HH-442: isolate runtime processes and harden shutdown (#90)

* HH-442: isolate runtime processes and harden shutdown

* HH-442: harden worker shutdown races

* HH-442: gate dependency shutdown on active handlers

---------

Co-authored-by: Rogee <rogee@ipao.vip>

backend/internal/handler/handler_test.go

@@ -1,7 +1,9 @@
 package handler
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -93,6 +95,49 @@ func TestReadyHandler_Ready(t *testing.T) {
 	assert.True(t, resp["ready"].(bool))
 }
 
+func TestReadyHandler_RequiredDependencyFailure(t *testing.T) {
+	db := newTestDB(t)
+	router := gin.New()
+	router.GET("/ready", ReadyHandler(db, DependencyCheck{Name: "redis", Check: func(context.Context) error {
+		return errors.New("connection refused")
+	}}))
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ready", nil))
+	assert.Equal(t, http.StatusServiceUnavailable, w.Code)
+	assert.JSONEq(t, `{"ready":false,"checks":{"database":"healthy","redis":"unhealthy: connection refused"}}`, w.Body.String())
+}
+
+func TestDatabaseOutageFailsReadinessButNotLiveness(t *testing.T) {
+	db := newTestDB(t)
+	sqlDB, err := db.DB()
+	require.NoError(t, err)
+	require.NoError(t, sqlDB.Close())
+	router := gin.New()
+	router.GET("/ready", ReadyHandler(db))
+	router.GET("/live", LiveHandler())
+
+	ready := httptest.NewRecorder()
+	router.ServeHTTP(ready, httptest.NewRequest(http.MethodGet, "/ready", nil))
+	assert.Equal(t, http.StatusServiceUnavailable, ready.Code)
+	live := httptest.NewRecorder()
+	router.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/live", nil))
+	assert.Equal(t, http.StatusOK, live.Code)
+}
+
+func TestHealthHandler_SelectedDependency(t *testing.T) {
+	db := newTestDB(t)
+	router := gin.New()
+	router.GET("/health", HealthHandler(db, time.Now(), "test", DependencyCheck{Name: "redis", Check: func(context.Context) error { return nil }}))
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/health?check=redis", nil))
+	assert.Equal(t, http.StatusOK, w.Code)
+	var response HealthResponse
+	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+	assert.Equal(t, map[string]string{"redis": "healthy"}, response.Checks)
+}
+
 func TestReadyHandler_NilDB_NotReady(t *testing.T) {
 	// ReadyHandler does not handle nil DB (panics on db.DB()).
 	t.Skip("ReadyHandler panics on nil DB; nil DB is not a valid runtime state")
@@ -144,6 +189,23 @@ func TestPrometheusHandler(t *testing.T) {
 	assert.Contains(t, body, "# TYPE")
 }
 
+func TestHTTPMetricsRecordsRequestsErrorsAndLatency(t *testing.T) {
+	metrics := NewHTTPMetrics(time.Now())
+	router := gin.New()
+	router.Use(metrics.Middleware())
+	router.GET("/items/:id", func(c *gin.Context) { c.Status(http.StatusInternalServerError) })
+	router.GET("/metrics", metrics.Handler())
+
+	router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/items/42", nil))
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+
+	body := w.Body.String()
+	assert.Contains(t, body, `http_requests_total{method="GET",route="/items/:id",status="500"} 1`)
+	assert.Contains(t, body, `http_request_errors_total{method="GET",route="/items/:id",status="500"} 1`)
+	assert.Contains(t, body, `http_request_duration_seconds_count{method="GET",route="/items/:id",status="500"} 1`)
+}
+
 func TestFormatGauge(t *testing.T) {
 	result := formatGauge("test_metric", 42)
 	assert.Equal(t, "test_metric 42", result)

backend/internal/handler/health_handler.go

@@ -1,6 +1,8 @@
 package handler
 
 import (
+	"context"
+	"errors"
 	"net/http"
 	"runtime"
 	"strconv"
@@ -10,6 +12,11 @@ import (
 	"gorm.io/gorm"
 )
 
+type DependencyCheck struct {
+	Name  string
+	Check func(context.Context) error
+}
+
 // HealthResponse is the structured health check response.
 type HealthResponse struct {
 	Status    string            `json:"status"`
@@ -19,52 +26,26 @@ type HealthResponse struct {
 	Checks    map[string]string `json:"checks"`
 }
 
-// HealthHandler returns application health status.
-// Reference: Chatwoot uses /health for monitoring in docker-compose.production.yaml
-func HealthHandler(db *gorm.DB, startTime time.Time, version string) gin.HandlerFunc {
+// HealthHandler reports dependency state and optional runtime diagnostics.
+func HealthHandler(db *gorm.DB, startTime time.Time, version string, dependencies ...DependencyCheck) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		checks := make(map[string]string)
-		overall := "healthy"
-
-		// Database check
-		sqlDB, err := db.DB()
-		if err != nil {
-			checks["database"] = "unhealthy: " + err.Error()
-			overall = "unhealthy"
-		} else if err := sqlDB.Ping(); err != nil {
-			checks["database"] = "unhealthy: " + err.Error()
-			overall = "unhealthy"
-		} else {
-			checks["database"] = "healthy"
-		}
-
-		// Memory check
-		var m runtime.MemStats
-		runtime.ReadMemStats(&m)
-		memMB := m.Alloc / 1024 / 1024
-		checks["memory_alloc_mb"] = strconv.FormatUint(memMB, 10)
-		if memMB > 500 {
-			checks["memory_warning"] = "high memory usage"
-		}
-
-		// Goroutine check
-		goroutines := runtime.NumGoroutine()
-		checks["goroutines"] = strconv.Itoa(goroutines)
-		if goroutines > 1000 {
-			checks["goroutine_warning"] = "high goroutine count"
-		}
-
-		// Uptime
+		selected := c.Query("check")
+		checks, healthy := runDependencyChecks(c.Request.Context(), db, selected, dependencies)
 		uptime := time.Since(startTime)
-		checks["uptime_seconds"] = strconv.FormatUint(uint64(uptime.Seconds()), 10)
-
-		statusCode := http.StatusOK
-		if overall == "unhealthy" {
-			statusCode = http.StatusServiceUnavailable
+		if selected == "" {
+			var memory runtime.MemStats
+			runtime.ReadMemStats(&memory)
+			checks["memory_alloc_mb"] = strconv.FormatUint(memory.Alloc/1024/1024, 10)
+			checks["goroutines"] = strconv.Itoa(runtime.NumGoroutine())
+			checks["uptime_seconds"] = strconv.FormatUint(uint64(uptime.Seconds()), 10)
 		}
 
+		status, statusCode := "healthy", http.StatusOK
+		if !healthy {
+			status, statusCode = "unhealthy", http.StatusServiceUnavailable
+		}
 		c.JSON(statusCode, HealthResponse{
-			Status:    overall,
+			Status:    status,
 			Timestamp: time.Now().UTC().Format(time.RFC3339),
 			Version:   version,
 			Uptime:    uptime.String(),
@@ -73,27 +54,64 @@ func HealthHandler(db *gorm.DB, startTime time.Time, version string) gin.Handler
 	}
 }
 
-// ReadyHandler returns whether the app is ready to accept traffic.
-// Used by K8s readiness probes — returns 503 if not ready.
-func ReadyHandler(db *gorm.DB) gin.HandlerFunc {
+// ReadyHandler returns 503 while draining or when a required dependency fails.
+func ReadyHandler(db *gorm.DB, dependencies ...DependencyCheck) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		sqlDB, err := db.DB()
-		if err != nil {
-			c.JSON(http.StatusServiceUnavailable, gin.H{"ready": false, "reason": "db error"})
-			return
+		checks, ready := runDependencyChecks(c.Request.Context(), db, "", dependencies)
+		statusCode := http.StatusOK
+		if !ready {
+			statusCode = http.StatusServiceUnavailable
 		}
-		if err := sqlDB.Ping(); err != nil {
-			c.JSON(http.StatusServiceUnavailable, gin.H{"ready": false, "reason": "db unreachable"})
-			return
-		}
-		c.JSON(http.StatusOK, gin.H{"ready": true})
+		c.JSON(statusCode, gin.H{"ready": ready, "checks": checks})
 	}
 }
 
-// LiveHandler returns whether the app process is alive.
-// Used by K8s liveness probes — simplest possible check.
+// LiveHandler only proves that the process can serve HTTP; dependency failures
+// belong to readiness so an outage does not trigger a restart loop.
 func LiveHandler() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		c.JSON(http.StatusOK, gin.H{"alive": true})
 	}
 }
+
+func runDependencyChecks(ctx context.Context, db *gorm.DB, selected string, dependencies []DependencyCheck) (map[string]string, bool) {
+	checks := make(map[string]string, len(dependencies)+1)
+	healthy := true
+	all := append([]DependencyCheck{{Name: "database", Check: databasePing(db)}}, dependencies...)
+	found := selected == ""
+	for _, dependency := range all {
+		if selected != "" && dependency.Name != selected {
+			continue
+		}
+		found = true
+		if dependency.Check == nil {
+			checks[dependency.Name] = "unhealthy: check is not configured"
+			healthy = false
+			continue
+		}
+		if err := dependency.Check(ctx); err != nil {
+			checks[dependency.Name] = "unhealthy: " + err.Error()
+			healthy = false
+		} else {
+			checks[dependency.Name] = "healthy"
+		}
+	}
+	if !found {
+		checks[selected] = "unhealthy: unknown check"
+		healthy = false
+	}
+	return checks, healthy
+}
+
+func databasePing(db *gorm.DB) func(context.Context) error {
+	return func(ctx context.Context) error {
+		if db == nil {
+			return errors.New("database is not configured")
+		}
+		sqlDB, err := db.DB()
+		if err != nil {
+			return err
+		}
+		return sqlDB.PingContext(ctx)
+	}
+}

backend/internal/handler/metrics_handler.go

@@ -4,79 +4,147 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
 )
 
-// MetricsHandler exposes Prometheus-compatible metrics in text exposition format.
-// Reference: Chatwoot uses Prometheus exporter for Sidekiq, Rails metrics
-// This provides Go runtime + application metrics on a dedicated port.
+var durationBuckets = [...]float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}
 
-// PrometheusHandler returns metrics in Prometheus text format.
-func PrometheusHandler(startTime time.Time) gin.HandlerFunc {
+type requestMetricKey struct {
+	Method string
+	Route  string
+	Status int
+}
+
+type requestMetric struct {
+	Count   uint64
+	Errors  uint64
+	Sum     float64
+	Buckets [len(durationBuckets)]uint64
+}
+
+// HTTPMetrics records bounded route-template labels without a Prometheus client dependency.
+type HTTPMetrics struct {
+	startTime time.Time
+	mu        sync.RWMutex
+	requests  map[requestMetricKey]requestMetric
+}
+
+func NewHTTPMetrics(startTime time.Time) *HTTPMetrics {
+	return &HTTPMetrics{startTime: startTime, requests: make(map[requestMetricKey]requestMetric)}
+}
+
+func (m *HTTPMetrics) Middleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
-		var m runtime.MemStats
-		runtime.ReadMemStats(&m)
-
-		uptime := time.Since(startTime).Seconds()
-
-		metrics := []string{
-			// Go runtime metrics
-			formatGauge("gochat_go_goroutines", uint64(runtime.NumGoroutine())),
-			formatGauge("gochat_go_memory_alloc_bytes", m.Alloc),
-			formatGauge("gochat_go_memory_sys_bytes", m.Sys),
-			formatGauge("gochat_go_memory_total_alloc_bytes", m.TotalAlloc),
-			formatGauge("gochat_go_gc_pause_total_ns", m.PauseTotalNs),
-			formatCounter("gochat_go_gc_count", uint64(m.NumGC)),
-			// Application metrics
-			formatGauge("gochat_uptime_seconds", uint64(uptime)),
-			formatGauge("gochat_threads_count", uint64(runtime.NumCPU())),
+		started := time.Now()
+		c.Next()
+		route := c.FullPath()
+		if route == "" {
+			route = "unmatched"
 		}
-
-		// HELP and TYPE annotations
-		help := []string{
-			"# HELP gochat_go_goroutines Number of goroutines currently running",
-			"# TYPE gochat_go_goroutines gauge",
-			"# HELP gochat_go_memory_alloc_bytes Bytes of allocated heap objects",
-			"# TYPE gochat_go_memory_alloc_bytes gauge",
-			"# HELP gochat_go_memory_sys_bytes Bytes obtained from system",
-			"# TYPE gochat_go_memory_sys_bytes gauge",
-			"# HELP gochat_uptime_seconds Application uptime in seconds",
-			"# TYPE gochat_uptime_seconds gauge",
-		}
-
-		output := ""
-		for _, h := range help {
-			output += h + "\n"
-		}
-		for _, m := range metrics {
-			output += m + "\n"
-		}
-
-		c.Header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-		c.String(http.StatusOK, output)
+		m.observe(requestMetricKey{Method: c.Request.Method, Route: route, Status: c.Writer.Status()}, time.Since(started).Seconds())
 	}
 }
 
-func formatGauge(name string, value uint64) string {
-	return name + " " + formatValue(value)
+func (m *HTTPMetrics) observe(key requestMetricKey, seconds float64) {
+	m.mu.Lock()
+	metric := m.requests[key]
+	metric.Count++
+	metric.Sum += seconds
+	if key.Status >= http.StatusInternalServerError {
+		metric.Errors++
+	}
+	for i, boundary := range durationBuckets {
+		if seconds <= boundary {
+			metric.Buckets[i]++
+		}
+	}
+	m.requests[key] = metric
+	m.mu.Unlock()
 }
 
-func formatCounter(name string, value uint64) string {
-	return name + " " + formatValue(value)
+func (m *HTTPMetrics) Handler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		var memory runtime.MemStats
+		runtime.ReadMemStats(&memory)
+		keys, snapshot := m.snapshot()
+
+		var output strings.Builder
+		output.WriteString("# HELP http_requests_total Total HTTP requests\n# TYPE http_requests_total counter\n")
+		output.WriteString("# HELP http_request_errors_total Total HTTP 5xx responses\n# TYPE http_request_errors_total counter\n")
+		output.WriteString("# HELP http_request_duration_seconds HTTP request duration\n# TYPE http_request_duration_seconds histogram\n")
+		for _, key := range keys {
+			metric := snapshot[key]
+			labels := requestLabels(key)
+			fmt.Fprintf(&output, "http_requests_total{%s} %d\n", labels, metric.Count)
+			fmt.Fprintf(&output, "http_request_errors_total{%s} %d\n", labels, metric.Errors)
+			for i, boundary := range durationBuckets {
+				fmt.Fprintf(&output, "http_request_duration_seconds_bucket{%s,le=%q} %d\n", labels, strconv.FormatFloat(boundary, 'g', -1, 64), metric.Buckets[i])
+			}
+			fmt.Fprintf(&output, "http_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, metric.Count)
+			fmt.Fprintf(&output, "http_request_duration_seconds_sum{%s} %s\n", labels, strconv.FormatFloat(metric.Sum, 'g', -1, 64))
+			fmt.Fprintf(&output, "http_request_duration_seconds_count{%s} %d\n", labels, metric.Count)
+		}
+
+		output.WriteString("# HELP gochat_go_goroutines Number of goroutines currently running\n# TYPE gochat_go_goroutines gauge\n")
+		output.WriteString("# HELP gochat_go_memory_alloc_bytes Bytes of allocated heap objects\n# TYPE gochat_go_memory_alloc_bytes gauge\n")
+		output.WriteString("# HELP gochat_go_memory_sys_bytes Bytes obtained from system\n# TYPE gochat_go_memory_sys_bytes gauge\n")
+		output.WriteString("# HELP gochat_uptime_seconds Application uptime in seconds\n# TYPE gochat_uptime_seconds gauge\n")
+		fmt.Fprintf(&output, "gochat_go_goroutines %d\n", runtime.NumGoroutine())
+		fmt.Fprintf(&output, "gochat_go_memory_alloc_bytes %d\n", memory.Alloc)
+		fmt.Fprintf(&output, "gochat_go_memory_sys_bytes %d\n", memory.Sys)
+		fmt.Fprintf(&output, "gochat_go_memory_total_alloc_bytes %d\n", memory.TotalAlloc)
+		fmt.Fprintf(&output, "gochat_go_gc_pause_total_ns %d\n", memory.PauseTotalNs)
+		fmt.Fprintf(&output, "gochat_go_gc_count %d\n", memory.NumGC)
+		fmt.Fprintf(&output, "gochat_uptime_seconds %s\n", strconv.FormatFloat(time.Since(m.startTime).Seconds(), 'f', 3, 64))
+		fmt.Fprintf(&output, "gochat_threads_count %d\n", runtime.NumCPU())
+
+		c.Data(http.StatusOK, "text/plain; version=0.0.4; charset=utf-8", []byte(output.String()))
+	}
 }
 
-func formatValue(v uint64) string {
-	// Simple uint64 formatting without strconv dependency
-	if v == 0 {
-		return "0"
+func (m *HTTPMetrics) snapshot() ([]requestMetricKey, map[requestMetricKey]requestMetric) {
+	m.mu.RLock()
+	snapshot := make(map[requestMetricKey]requestMetric, len(m.requests))
+	keys := make([]requestMetricKey, 0, len(m.requests))
+	for key, metric := range m.requests {
+		keys = append(keys, key)
+		snapshot[key] = metric
 	}
-	result := ""
-	for v > 0 {
-		digit := v % 10
-		result = fmt.Sprintf("%d%s", digit, result)
-		v /= 10
-	}
-	return result
+	m.mu.RUnlock()
+	sort.Slice(keys, func(i, j int) bool {
+		left, right := keys[i], keys[j]
+		if left.Route != right.Route {
+			return left.Route < right.Route
+		}
+		if left.Method != right.Method {
+			return left.Method < right.Method
+		}
+		return left.Status < right.Status
+	})
+	return keys, snapshot
 }
+
+func requestLabels(key requestMetricKey) string {
+	return fmt.Sprintf(`method="%s",route="%s",status="%s"`, escapeLabel(key.Method), escapeLabel(key.Route), strconv.Itoa(key.Status))
+}
+
+func escapeLabel(value string) string {
+	value = strings.ReplaceAll(value, `\`, `\\`)
+	value = strings.ReplaceAll(value, "\n", `\n`)
+	return strings.ReplaceAll(value, `"`, `\"`)
+}
+
+// PrometheusHandler is retained for callers that only need runtime metrics.
+func PrometheusHandler(startTime time.Time) gin.HandlerFunc {
+	return NewHTTPMetrics(startTime).Handler()
+}
+
+func formatGauge(name string, value uint64) string   { return name + " " + formatValue(value) }
+func formatCounter(name string, value uint64) string { return name + " " + formatValue(value) }
+func formatValue(value uint64) string                { return strconv.FormatUint(value, 10) }