#!/bin/bash

=============================================================================

ELK Stack Health Check Script

Description: Checks Elasticsearch, Logstash, Kibana, Filebeat, and system

resources relevant to the ELK stack.

Intended for: master3 (or wherever ELK services are running)

=============================================================================

set -euo pipefail

Colors

RED=’\033[0;31m’
GREEN=’\033[0;32m’
YELLOW=’\033[1;33m’
BLUE=’\033[0;34m’
BOLD=’\033[1m’
NC=’\033[0m’ # No Color

Configurable endpoints

ES_HOST=”${ES_HOST:-http://localhost:9200}“
KB_HOST=”${KB_HOST:-http://localhost:5601}“
LS_HOST=”${LS_HOST:-http://localhost:9600}“

PASS=0
WARN=0
FAIL=0

=============================================================================

Helper Functions

=============================================================================

print_header() {
echo -e “${BOLD}${BLUE}>>> $1${NC}”
}

pass() {
echo -e “${GREEN}[PASS]${NC} $1”
((PASS++))
}

warn() {
echo -e “${YELLOW}[WARN]${NC} $1”
((WARN++))
}

fail() {
echo -e “${RED}[FAIL]${NC} $1”
((FAIL++))
}

info() {
echo -e “${BLUE}[INFO]${NC} $1”
}

check_service() {
local svc=$1
if systemctl is-active –quiet “$svc” 2>/dev/null; then
pass “Service active: $svc”
return 0
elif systemctl is-failed –quiet “$svc” 2>/dev/null; then
fail “Service failed: $svc”
return 1
elif systemctl list-unit-files “$svc” &>/dev/null; then
fail “Service inactive: $svc”
return 1
else
warn “Service not installed: $svc”
return 1
fi
}

=============================================================================

Main

=============================================================================

echo -e “${BOLD}========================================${NC}”
echo -e “${BOLD} ELK Stack Health Check${NC}”
echo -e “${BOLD}========================================${NC}”
echo “”

——————————————————————————

1. Systemd Services

——————————————————————————

print_header “Checking Systemd Services”
check_service “elasticsearch.service”
check_service “logstash.service”
check_service “kibana.service”
check_service “filebeat.service”
echo “”

——————————————————————————

2. Elasticsearch Cluster Health

——————————————————————————

print_header “Checking Elasticsearch”
ES_HTTP=$(curl -s -o /dev/null -w “%{http_code}” “${ES_HOST}/_cluster/health” || true)
if [ “$ES_HTTP” == “200” ]; then
ES_JSON=$(curl -s “${ES_HOST}/_cluster/health” || true)
CLUSTER_STATUS=$(echo “$ES_JSON” | awk -F'”status”:”‘ ‘{print $2}’ | awk -F'”‘ ‘{print $1}’)
NODE_COUNT=$(echo “$ES_JSON” | awk -F'”number_of_nodes”:’ ‘{print $2}’ | awk -F’,’ ‘{print $1}’)
ACTIVE_SHARDS=$(echo “$ES_JSON” | awk -F'”active_shards”:’ ‘{print $2}’ | awk -F’,’ ‘{print $1}’)
RELocating_SHARDS=$(echo “$ES_JSON” | awk -F'”relocating_shards”:’ ‘{print $2}’ | awk -F’,’ ‘{print $1}’)
UNASSIGNED_SHARDS=$(echo “$ES_JSON” | awk -F'”unassigned_shards”:’ ‘{print $2}’ | awk -F’,’ ‘{print $1}’)

if [ "$CLUSTER_STATUS" == "green" ]; then
    pass "Cluster status: $CLUSTER_STATUS (nodes: $NODE_COUNT, active shards: $ACTIVE_SHARDS)"
elif [ "$CLUSTER_STATUS" == "yellow" ]; then
    warn "Cluster status: $CLUSTER_STATUS (nodes: $NODE_COUNT, active shards: $ACTIVE_SHARDS, unassigned: $UNASSIGNED_SHARDS)"
else
    fail "Cluster status: $CLUSTER_STATUS (nodes: $NODE_COUNT, active shards: $ACTIVE_SHARDS, unassigned: $UNASSIGNED_SHARDS)"
fi

else
fail “Elasticsearch not responding on $ES_HOST (HTTP ${ES_HTTP:-no response})”
fi
echo “”

——————————————————————————

3. Elasticsearch Indices

——————————————————————————

print_header “Checking Elasticsearch Indices”
INDICES=$(curl -s “${ES_HOST}/_cat/indices?v&bytes=mb” 2>/dev/null || true)
if [ -n “$INDICES” ]; then
echo “$INDICES” | head -15
COUNT=$(echo “$INDICES” | wc -l)
if [ “$COUNT” -gt 15 ]; then
info “$((COUNT – 15)) more indices not shown”
fi
else
fail “Could not retrieve index list from Elasticsearch”
fi
echo “”

——————————————————————————

4. Logstash Health

——————————————————————————

print_header “Checking Logstash”
LS_HTTP=$(curl -s -o /dev/null -w “%{http_code}” “${LS_HOST}/_node/stats” 2>/dev/null || true)
if [ “$LS_HTTP” == “200” ]; then
pass “Logstash API responding on port 9600”
LS_JSON=$(curl -s “${LS_HOST}/_node/stats” 2>/dev/null || true)
EVENTS_IN=$(echo “$LS_JSON” | grep -o ‘”in”:[0-9]‘ | head -1 | cut -d’:’ -f2 || true)
EVENTS_OUT=$(echo “$LS_JSON” | grep -o ‘”out”:[0-9]‘ | head -1 | cut -d’:’ -f2 || true)
if [ -n “$EVENTS_IN” ]; then
info “Pipeline events in: $EVENTS_IN, out: $EVENTS_OUT”
fi
else
# Fallback: check process and common ingestion port
if pgrep -f “logstash” > /dev/null 2>&1; then
warn “Logstash process running but API not responding on :9600”
else
fail “Logstash not responding and process not found”
fi
fi

Check if common Beats input port is listening

if ss -tlnp 2>/dev/null | grep -q “:5044 “; then
pass “Logstash Beats input port 5044 is listening”
elif netstat -tlnp 2>/dev/null | grep -q “:5044 “; then
pass “Logstash Beats input port 5044 is listening”
else
warn “Logstash Beats input port 5044 not found listening”
fi
echo “”

——————————————————————————

5. Kibana Status

——————————————————————————

print_header “Checking Kibana”
KB_HTTP=$(curl -s -o /dev/null -w “%{http_code}” “${KB_HOST}/api/status” 2>/dev/null || true)
if [ “$KB_HTTP” == “200” ]; then
KB_JSON=$(curl -s “${KB_HOST}/api/status” 2>/dev/null || true)
# Try to extract overall level
KB_STATE=$(echo “$KB_JSON” | grep -o ‘”level”:”[^”]*”‘ | head -1 | cut -d'”‘ -f4 || true)
if [ “$KB_STATE” == “available” ] || [ -z “$KB_STATE” ]; then
pass “Kibana is available ($KB_HOST)”
else
warn “Kibana state: $KB_STATE”
fi
else
fail “Kibana not responding on $KB_HOST (HTTP ${KB_HTTP:-no response})”
fi
echo “”

——————————————————————————

6. Filebeat Status

——————————————————————————

print_header “Checking Filebeat”
if systemctl is-active –quiet filebeat.service 2>/dev/null; then
# Try to check Filebeat logs for errors in last 50 lines
if journalctl -u filebeat.service –no-pager -n 50 2>/dev/null | grep -qi “error|fail”; then
warn “Filebeat is active but recent errors found in logs”
else
pass “Filebeat running with no recent errors”
fi
else
check_service “filebeat.service”
fi
echo “”

——————————————————————————

7. System Resources

——————————————————————————

print_header “Checking System Resources”

Disk usage for root (Elasticsearch is sensitive to disk watermarks)

DISK_USAGE=$(df / | awk ‘NR==2 {gsub(/%/,””); print $5}’)
if [ “$DISK_USAGE” -lt 70 ]; then
pass “Root disk usage: ${DISK_USAGE}%”
elif [ “$DISK_USAGE” -lt 85 ]; then
warn “Root disk usage: ${DISK_USAGE}% (approaching ES high watermark)”
else
fail “Root disk usage: ${DISK_USAGE}% (critical for Elasticsearch)”
fi

Memory check

MEM_USAGE=$(free | awk ‘NR==2{printf “%.0f”, $3/$2 * 100.0}’)
if [ “$MEM_USAGE” -lt 80 ]; then
pass “Memory usage: ${MEM_USAGE}%”
elif [ “$MEM_USAGE” -lt 90 ]; then
warn “Memory usage: ${MEM_USAGE}%”
else
fail “Memory usage: ${MEM_USAGE}%”
fi

Load average

LOAD1=$(awk ‘{print $1}’ /proc/loadavg)
LOAD5=$(awk ‘{print $2}’ /proc/loadavg)
info “Load average: $LOAD1 (1m), $LOAD5 (5m)”
echo “”

——————————————————————————

8. Listening Ports

——————————————————————————

print_header “Checking Listening Ports”
for port in 9200 5601 9600 5044; do
if command -v ss &>/dev/null; then
if ss -tlnp 2>/dev/null | grep -q “:$port “; then
pass “Port $port is listening”
else
fail “Port $port is not listening”
fi
else
if netstat -tlnp 2>/dev/null | grep -q “:$port “; then
pass “Port $port is listening”
else
fail “Port $port is not listening”
fi
fi
done
echo “”

=============================================================================

Summary

=============================================================================

echo -e “${BOLD}========================================${NC}”
echo -e “${BOLD} Summary${NC}”
echo -e “${BOLD}========================================${NC}”
echo -e “${GREEN}Passed: $PASS${NC}”
echo -e “${YELLOW}Warnings: $WARN${NC}”
echo -e “${RED}Failed: $FAIL${NC}”
echo “”

if [ $FAIL -eq 0 ] && [ $WARN -eq 0 ]; then
echo -e “${GREEN}All ELK components look healthy!${NC}”
elif [ $FAIL -eq 0 ]; then
echo -e “${YELLOW}ELK is functional but has warnings worth reviewing.${NC}”
else
echo -e “${RED}ELK stack has issues that need attention.${NC}”
fi