HH-442: isolate runtime processes and harden shutdown (#90)
* HH-442: isolate runtime processes and harden shutdown * HH-442: harden worker shutdown races * HH-442: gate dependency shutdown on active handlers --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
Regular → Executable
+78
-150
@@ -1,175 +1,103 @@
|
||||
#!/bin/bash
|
||||
# GoChat Health Check Script
|
||||
# Reference: Chatwoot health_check endpoint pattern
|
||||
# Checks: HTTP server, database, Redis, worker queues, WebSocket hub
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/health_check.sh [--full] [--json] [--timeout SECONDS]
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 = healthy
|
||||
# 1 = degraded (some checks failed but core is OK)
|
||||
# 2 = unhealthy (critical checks failed)
|
||||
# Exit codes: 0 healthy, 1 degraded, 2 unhealthy/invalid invocation.
|
||||
|
||||
set -euo pipefail
|
||||
set -uo pipefail
|
||||
|
||||
# Configuration
|
||||
GOCHAT_HOST="${GOCHAT_HOST:-localhost}"
|
||||
GOCHAT_PORT="${GOCHAT_PORT:-3000}"
|
||||
GOCHAT_METRICS_PORT="${GOCHAT_METRICS_PORT:-9090}"
|
||||
GOCHAT_METRICS_PORT="${GOCHAT_METRICS_PORT:-${GOCHAT_PORT}}"
|
||||
TIMEOUT="${TIMEOUT:-5}"
|
||||
FULL_CHECK=false
|
||||
JSON_OUTPUT=false
|
||||
RESULTS=()
|
||||
CRITICAL_FAIL=0
|
||||
DEGRADED_FAIL=0
|
||||
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--json" ]] && JSON_OUTPUT=true
|
||||
done
|
||||
|
||||
usage_error() {
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
printf '{"error":"%s","checks":[],"summary":{"critical_failures":1,"degraded_failures":0}}\n' "$1"
|
||||
else
|
||||
printf 'health_check: %s\n' "$1" >&2
|
||||
fi
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--full) FULL_CHECK=true; shift ;;
|
||||
--json) JSON_OUTPUT=true; shift ;;
|
||||
--timeout) TIMEOUT="$2"; shift 2 ;;
|
||||
--help) echo "Usage: $0 [--full] [--json] [--timeout SECONDS]"; exit 0 ;;
|
||||
*) echo "Unknown option: $1"; exit 1 ;;
|
||||
esac
|
||||
case "$1" in
|
||||
--full) FULL_CHECK=true; shift ;;
|
||||
--json) JSON_OUTPUT=true; shift ;;
|
||||
--timeout)
|
||||
[[ $# -ge 2 && "$2" =~ ^[1-9][0-9]*$ ]] || usage_error "--timeout requires a positive integer"
|
||||
TIMEOUT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
printf '{"usage":"health_check.sh [--full] [--json] [--timeout SECONDS]"}\n'
|
||||
else
|
||||
printf 'Usage: %s [--full] [--json] [--timeout SECONDS]\n' "$0"
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
*) usage_error "unknown option" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
BASE_URL="http://${GOCHAT_HOST}:${GOCHAT_PORT}"
|
||||
METRICS_URL="http://${GOCHAT_HOST}:${GOCHAT_METRICS_PORT}"
|
||||
|
||||
# Result tracking
|
||||
RESULTS=()
|
||||
CRITICAL_FAIL=0
|
||||
DEGRADED_FAIL=0
|
||||
|
||||
check_result() {
|
||||
local name="$1" status="$2" detail="$3"
|
||||
RESULTS+=("${name}|${status}|${detail}")
|
||||
if [[ "$status" == "CRITICAL_FAIL" ]]; then
|
||||
CRITICAL_FAIL=$((CRITICAL_FAIL + 1))
|
||||
elif [[ "$status" == "DEGRADED" ]]; then
|
||||
DEGRADED_FAIL=$((DEGRADED_FAIL + 1))
|
||||
fi
|
||||
local name="$1" status="$2" detail="$3"
|
||||
RESULTS+=("${name}|${status}|${detail}")
|
||||
[[ "${status}" == "CRITICAL_FAIL" ]] && CRITICAL_FAIL=$((CRITICAL_FAIL + 1))
|
||||
[[ "${status}" == "DEGRADED" ]] && DEGRADED_FAIL=$((DEGRADED_FAIL + 1))
|
||||
}
|
||||
|
||||
# ---- Check 1: HTTP liveness endpoint ----
|
||||
check_liveness() {
|
||||
local response
|
||||
response=$(curl -sf --max-time "${TIMEOUT}" "${BASE_URL}/live" 2>&1) && {
|
||||
check_result "liveness" "OK" "${response}"
|
||||
} || {
|
||||
check_result "liveness" "CRITICAL_FAIL" "HTTP /live endpoint unreachable"
|
||||
}
|
||||
check_url() {
|
||||
local name="$1" severity="$2" url="$3" ok_detail="$4" fail_detail="$5"
|
||||
if curl -fsS --max-time "${TIMEOUT}" -o /dev/null "${url}"; then
|
||||
check_result "${name}" "OK" "${ok_detail}"
|
||||
else
|
||||
check_result "${name}" "${severity}" "${fail_detail}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check 2: HTTP readiness endpoint ----
|
||||
check_readiness() {
|
||||
local response
|
||||
response=$(curl -sf --max-time "${TIMEOUT}" "${BASE_URL}/ready" 2>&1) && {
|
||||
check_result "readiness" "OK" "${response}"
|
||||
} || {
|
||||
check_result "readiness" "CRITICAL_FAIL" "HTTP /ready endpoint unreachable"
|
||||
}
|
||||
}
|
||||
check_url "liveness" "CRITICAL_FAIL" "${BASE_URL}/live" "process is alive" "liveness endpoint unreachable"
|
||||
check_url "readiness" "CRITICAL_FAIL" "${BASE_URL}/ready" "dependencies are ready" "readiness endpoint failed"
|
||||
|
||||
# ---- Check 3: Database connectivity ----
|
||||
check_database() {
|
||||
if [[ "${FULL_CHECK}" == "true" ]]; then
|
||||
local response
|
||||
response=$(curl -sf --max-time "${TIMEOUT}" "${BASE_URL}/health?check=database" 2>&1) && {
|
||||
check_result "database" "OK" "${response}"
|
||||
} || {
|
||||
check_result "database" "CRITICAL_FAIL" "Database connectivity failed"
|
||||
}
|
||||
else
|
||||
check_result "database" "SKIPPED" "Not in full mode"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check 4: Redis connectivity ----
|
||||
check_redis() {
|
||||
if [[ "${FULL_CHECK}" == "true" ]]; then
|
||||
local response
|
||||
response=$(curl -sf --max-time "${TIMEOUT}" "${BASE_URL}/health?check=redis" 2>&1) && {
|
||||
check_result "redis" "OK" "${response}"
|
||||
} || {
|
||||
check_result "redis" "DEGRADED" "Redis connectivity failed (non-critical)"
|
||||
}
|
||||
else
|
||||
check_result "redis" "SKIPPED" "Not in full mode"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check 5: Metrics endpoint ----
|
||||
check_metrics() {
|
||||
local response
|
||||
response=$(curl -sf --max-time "${TIMEOUT}" "${METRICS_URL}/metrics" 2>&1) && {
|
||||
check_result "metrics" "OK" "Prometheus metrics endpoint responding"
|
||||
} || {
|
||||
check_result "metrics" "DEGRADED" "Metrics endpoint unreachable (non-critical)"
|
||||
}
|
||||
}
|
||||
|
||||
# ---- Check 6: WebSocket hub (full mode) ----
|
||||
check_websocket() {
|
||||
if [[ "${FULL_CHECK}" == "true" ]]; then
|
||||
# WebSocket health check via HTTP status endpoint
|
||||
local response
|
||||
response=$(curl -sf --max-time "${TIMEOUT}" "${BASE_URL}/health?check=websocket" 2>&1) && {
|
||||
check_result "websocket" "OK" "${response}"
|
||||
} || {
|
||||
check_result "websocket" "DEGRADED" "WebSocket hub unreachable"
|
||||
}
|
||||
else
|
||||
check_result "websocket" "SKIPPED" "Not in full mode"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Run all checks ----
|
||||
check_liveness
|
||||
check_readiness
|
||||
check_database
|
||||
check_redis
|
||||
check_metrics
|
||||
check_websocket
|
||||
|
||||
# ---- Output results ----
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
# JSON output for programmatic consumption
|
||||
echo '{'
|
||||
echo ' "checks": ['
|
||||
for i in "${!RESULTS[@]}"; do
|
||||
IFS='|' read -r name status detail <<< "${RESULTS[$i]}"
|
||||
comma=""
|
||||
[[ $i -gt 0 ]] && comma=","
|
||||
echo " ${comma}{\"name\": \"${name}\", \"status\": \"${status}\", \"detail\": \"${detail}\"}"
|
||||
done
|
||||
echo ' ],'
|
||||
echo ' "summary": {'
|
||||
echo ' "critical_failures": ${CRITICAL_FAIL},'
|
||||
echo ' "degraded_failures": ${DEGRADED_FAIL}'
|
||||
echo ' }'
|
||||
echo '}'
|
||||
if [[ "${FULL_CHECK}" == "true" ]]; then
|
||||
check_url "database" "CRITICAL_FAIL" "${BASE_URL}/health?check=database" "database is healthy" "database check failed"
|
||||
check_url "redis" "CRITICAL_FAIL" "${BASE_URL}/health?check=redis" "redis is healthy" "redis check failed"
|
||||
else
|
||||
# Human-readable output
|
||||
echo "=== GoChat Health Check Report ==="
|
||||
for result in "${RESULTS[@]}"; do
|
||||
IFS='|' read -r name status detail <<< "${result}"
|
||||
case "$status" in
|
||||
OK) icon="✅" ;;
|
||||
CRITICAL_FAIL) icon="❌" ;;
|
||||
DEGRADED) icon="⚠️" ;;
|
||||
SKIPPED) icon="⏭️" ;;
|
||||
esac
|
||||
echo " ${icon} ${name}: ${status} — ${detail}"
|
||||
done
|
||||
echo ""
|
||||
echo "Critical failures: ${CRITICAL_FAIL} | Degraded: ${DEGRADED_FAIL}"
|
||||
check_result "database" "SKIPPED" "not in full mode"
|
||||
check_result "redis" "SKIPPED" "not in full mode"
|
||||
fi
|
||||
|
||||
# ---- Determine exit code ----
|
||||
if [[ ${CRITICAL_FAIL} -gt 0 ]]; then
|
||||
exit 2
|
||||
elif [[ ${DEGRADED_FAIL} -gt 0 ]]; then
|
||||
exit 1
|
||||
check_url "metrics" "DEGRADED" "${METRICS_URL}/metrics" "metrics endpoint is responding" "metrics endpoint unreachable"
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
printf '{"checks":['
|
||||
for i in "${!RESULTS[@]}"; do
|
||||
IFS='|' read -r name status detail <<< "${RESULTS[$i]}"
|
||||
[[ $i -gt 0 ]] && printf ','
|
||||
printf '{"name":"%s","status":"%s","detail":"%s"}' "${name}" "${status}" "${detail}"
|
||||
done
|
||||
printf '],"summary":{"critical_failures":%d,"degraded_failures":%d}}\n' "${CRITICAL_FAIL}" "${DEGRADED_FAIL}"
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
printf '=== GoChat Health Check Report ===\n'
|
||||
for result in "${RESULTS[@]}"; do
|
||||
IFS='|' read -r name status detail <<< "${result}"
|
||||
printf ' %s: %s — %s\n' "${name}" "${status}" "${detail}"
|
||||
done
|
||||
printf 'Critical failures: %d | Degraded: %d\n' "${CRITICAL_FAIL}" "${DEGRADED_FAIL}"
|
||||
fi
|
||||
|
||||
if [[ ${CRITICAL_FAIL} -gt 0 ]]; then
|
||||
exit 2
|
||||
elif [[ ${DEGRADED_FAIL} -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package scripts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHealthCheckJSONExitCodes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status func(string) int
|
||||
exitCode int
|
||||
}{
|
||||
{name: "healthy", status: func(string) int { return http.StatusOK }, exitCode: 0},
|
||||
{name: "degraded", status: func(path string) int {
|
||||
if path == "/metrics" {
|
||||
return http.StatusServiceUnavailable
|
||||
}
|
||||
return http.StatusOK
|
||||
}, exitCode: 1},
|
||||
{name: "unhealthy", status: func(path string) int {
|
||||
if path == "/live" || path == "/ready" {
|
||||
return http.StatusServiceUnavailable
|
||||
}
|
||||
return http.StatusOK
|
||||
}, exitCode: 2},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(test.status(r.URL.Path))
|
||||
}))
|
||||
defer server.Close()
|
||||
output, exitCode := runHealthScript(t, server.URL, "--json", "--full")
|
||||
require.Equal(t, test.exitCode, exitCode)
|
||||
var payload map[string]any
|
||||
require.NoError(t, json.Unmarshal(output, &payload), string(output))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCheckJSONInvalidInvocation(t *testing.T) {
|
||||
command := exec.Command("./health_check.sh", "--json", "--timeout")
|
||||
output, err := command.Output()
|
||||
var exitErr *exec.ExitError
|
||||
require.True(t, errors.As(err, &exitErr))
|
||||
require.Equal(t, 2, exitErr.ExitCode())
|
||||
require.JSONEq(t, `{"error":"--timeout requires a positive integer","checks":[],"summary":{"critical_failures":1,"degraded_failures":0}}`, string(output))
|
||||
}
|
||||
|
||||
func runHealthScript(t *testing.T, rawURL string, args ...string) ([]byte, int) {
|
||||
t.Helper()
|
||||
parsed, err := url.Parse(rawURL)
|
||||
require.NoError(t, err)
|
||||
host, port, err := net.SplitHostPort(parsed.Host)
|
||||
require.NoError(t, err)
|
||||
command := exec.Command("./health_check.sh", args...)
|
||||
command.Env = append(os.Environ(), "GOCHAT_HOST="+host, "GOCHAT_PORT="+port, "GOCHAT_METRICS_PORT="+port)
|
||||
output, err := command.Output()
|
||||
if err == nil {
|
||||
return output, 0
|
||||
}
|
||||
var exitErr *exec.ExitError
|
||||
require.True(t, errors.As(err, &exitErr), "health script failed to execute: %v", err)
|
||||
return output, exitErr.ExitCode()
|
||||
}
|
||||
Reference in New Issue
Block a user