* HH-442: isolate runtime processes and harden shutdown * HH-442: harden worker shutdown races * HH-442: gate dependency shutdown on active handlers --------- Co-authored-by: Rogee <rogee@ipao.vip>
151 lines
5.4 KiB
Go
151 lines
5.4 KiB
Go
package handler
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
var durationBuckets = [...]float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}
|
|
|
|
type requestMetricKey struct {
|
|
Method string
|
|
Route string
|
|
Status int
|
|
}
|
|
|
|
type requestMetric struct {
|
|
Count uint64
|
|
Errors uint64
|
|
Sum float64
|
|
Buckets [len(durationBuckets)]uint64
|
|
}
|
|
|
|
// HTTPMetrics records bounded route-template labels without a Prometheus client dependency.
|
|
type HTTPMetrics struct {
|
|
startTime time.Time
|
|
mu sync.RWMutex
|
|
requests map[requestMetricKey]requestMetric
|
|
}
|
|
|
|
func NewHTTPMetrics(startTime time.Time) *HTTPMetrics {
|
|
return &HTTPMetrics{startTime: startTime, requests: make(map[requestMetricKey]requestMetric)}
|
|
}
|
|
|
|
func (m *HTTPMetrics) Middleware() gin.HandlerFunc {
|
|
return func(c *gin.Context) {
|
|
started := time.Now()
|
|
c.Next()
|
|
route := c.FullPath()
|
|
if route == "" {
|
|
route = "unmatched"
|
|
}
|
|
m.observe(requestMetricKey{Method: c.Request.Method, Route: route, Status: c.Writer.Status()}, time.Since(started).Seconds())
|
|
}
|
|
}
|
|
|
|
func (m *HTTPMetrics) observe(key requestMetricKey, seconds float64) {
|
|
m.mu.Lock()
|
|
metric := m.requests[key]
|
|
metric.Count++
|
|
metric.Sum += seconds
|
|
if key.Status >= http.StatusInternalServerError {
|
|
metric.Errors++
|
|
}
|
|
for i, boundary := range durationBuckets {
|
|
if seconds <= boundary {
|
|
metric.Buckets[i]++
|
|
}
|
|
}
|
|
m.requests[key] = metric
|
|
m.mu.Unlock()
|
|
}
|
|
|
|
func (m *HTTPMetrics) Handler() gin.HandlerFunc {
|
|
return func(c *gin.Context) {
|
|
var memory runtime.MemStats
|
|
runtime.ReadMemStats(&memory)
|
|
keys, snapshot := m.snapshot()
|
|
|
|
var output strings.Builder
|
|
output.WriteString("# HELP http_requests_total Total HTTP requests\n# TYPE http_requests_total counter\n")
|
|
output.WriteString("# HELP http_request_errors_total Total HTTP 5xx responses\n# TYPE http_request_errors_total counter\n")
|
|
output.WriteString("# HELP http_request_duration_seconds HTTP request duration\n# TYPE http_request_duration_seconds histogram\n")
|
|
for _, key := range keys {
|
|
metric := snapshot[key]
|
|
labels := requestLabels(key)
|
|
fmt.Fprintf(&output, "http_requests_total{%s} %d\n", labels, metric.Count)
|
|
fmt.Fprintf(&output, "http_request_errors_total{%s} %d\n", labels, metric.Errors)
|
|
for i, boundary := range durationBuckets {
|
|
fmt.Fprintf(&output, "http_request_duration_seconds_bucket{%s,le=%q} %d\n", labels, strconv.FormatFloat(boundary, 'g', -1, 64), metric.Buckets[i])
|
|
}
|
|
fmt.Fprintf(&output, "http_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, metric.Count)
|
|
fmt.Fprintf(&output, "http_request_duration_seconds_sum{%s} %s\n", labels, strconv.FormatFloat(metric.Sum, 'g', -1, 64))
|
|
fmt.Fprintf(&output, "http_request_duration_seconds_count{%s} %d\n", labels, metric.Count)
|
|
}
|
|
|
|
output.WriteString("# HELP gochat_go_goroutines Number of goroutines currently running\n# TYPE gochat_go_goroutines gauge\n")
|
|
output.WriteString("# HELP gochat_go_memory_alloc_bytes Bytes of allocated heap objects\n# TYPE gochat_go_memory_alloc_bytes gauge\n")
|
|
output.WriteString("# HELP gochat_go_memory_sys_bytes Bytes obtained from system\n# TYPE gochat_go_memory_sys_bytes gauge\n")
|
|
output.WriteString("# HELP gochat_uptime_seconds Application uptime in seconds\n# TYPE gochat_uptime_seconds gauge\n")
|
|
fmt.Fprintf(&output, "gochat_go_goroutines %d\n", runtime.NumGoroutine())
|
|
fmt.Fprintf(&output, "gochat_go_memory_alloc_bytes %d\n", memory.Alloc)
|
|
fmt.Fprintf(&output, "gochat_go_memory_sys_bytes %d\n", memory.Sys)
|
|
fmt.Fprintf(&output, "gochat_go_memory_total_alloc_bytes %d\n", memory.TotalAlloc)
|
|
fmt.Fprintf(&output, "gochat_go_gc_pause_total_ns %d\n", memory.PauseTotalNs)
|
|
fmt.Fprintf(&output, "gochat_go_gc_count %d\n", memory.NumGC)
|
|
fmt.Fprintf(&output, "gochat_uptime_seconds %s\n", strconv.FormatFloat(time.Since(m.startTime).Seconds(), 'f', 3, 64))
|
|
fmt.Fprintf(&output, "gochat_threads_count %d\n", runtime.NumCPU())
|
|
|
|
c.Data(http.StatusOK, "text/plain; version=0.0.4; charset=utf-8", []byte(output.String()))
|
|
}
|
|
}
|
|
|
|
func (m *HTTPMetrics) snapshot() ([]requestMetricKey, map[requestMetricKey]requestMetric) {
|
|
m.mu.RLock()
|
|
snapshot := make(map[requestMetricKey]requestMetric, len(m.requests))
|
|
keys := make([]requestMetricKey, 0, len(m.requests))
|
|
for key, metric := range m.requests {
|
|
keys = append(keys, key)
|
|
snapshot[key] = metric
|
|
}
|
|
m.mu.RUnlock()
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
left, right := keys[i], keys[j]
|
|
if left.Route != right.Route {
|
|
return left.Route < right.Route
|
|
}
|
|
if left.Method != right.Method {
|
|
return left.Method < right.Method
|
|
}
|
|
return left.Status < right.Status
|
|
})
|
|
return keys, snapshot
|
|
}
|
|
|
|
func requestLabels(key requestMetricKey) string {
|
|
return fmt.Sprintf(`method="%s",route="%s",status="%s"`, escapeLabel(key.Method), escapeLabel(key.Route), strconv.Itoa(key.Status))
|
|
}
|
|
|
|
func escapeLabel(value string) string {
|
|
value = strings.ReplaceAll(value, `\`, `\\`)
|
|
value = strings.ReplaceAll(value, "\n", `\n`)
|
|
return strings.ReplaceAll(value, `"`, `\"`)
|
|
}
|
|
|
|
// PrometheusHandler is retained for callers that only need runtime metrics.
|
|
func PrometheusHandler(startTime time.Time) gin.HandlerFunc {
|
|
return NewHTTPMetrics(startTime).Handler()
|
|
}
|
|
|
|
func formatGauge(name string, value uint64) string { return name + " " + formatValue(value) }
|
|
func formatCounter(name string, value uint64) string { return name + " " + formatValue(value) }
|
|
func formatValue(value uint64) string { return strconv.FormatUint(value, 10) }
|