274 lines
11 KiB
Go
274 lines
11 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
hub "git.ipao.vip/rogee/creator-hub/internal/environment"
|
|
"github.com/gofiber/fiber/v3"
|
|
)
|
|
|
|
// setGatewayHealthProbeTimeout 临时替换探活超时，返回旧值供 restore 恢复。
|
|
func setGatewayHealthProbeTimeout(d time.Duration) time.Duration {
|
|
old := gatewayHealthProbeTimeout
|
|
gatewayHealthProbeTimeout = d
|
|
return old
|
|
}
|
|
|
|
// restoreGatewayHealthProbeTimeout 恢复探活超时旧值。
|
|
func restoreGatewayHealthProbeTimeout(old time.Duration) func() {
|
|
return func() { gatewayHealthProbeTimeout = old }
|
|
}
|
|
|
|
type recordedGatewayCheck struct {
|
|
name string
|
|
status string
|
|
reason string
|
|
}
|
|
|
|
// healthMemoryStore 在 memoryStore 之上记录 RecordGatewayCheck 调用，供健康检查断言。
|
|
type healthMemoryStore struct {
|
|
*memoryStore
|
|
mu sync.Mutex
|
|
checks []recordedGatewayCheck
|
|
}
|
|
|
|
func (s *healthMemoryStore) RecordGatewayCheck(_ context.Context, name, status, reason string) (hub.Gateway, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.checks = append(s.checks, recordedGatewayCheck{name: name, status: status, reason: reason})
|
|
return hub.Gateway{Name: name, HealthStatus: status, LastCheckReason: reason}, nil
|
|
}
|
|
|
|
func TestClassifyGatewayHealth(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
status int
|
|
callErr error
|
|
wantStatus string
|
|
wantReasonHad bool // 是否应携带非空 reason
|
|
}{
|
|
{name: "204 healthy without reason", status: http.StatusNoContent, wantStatus: "healthy", wantReasonHad: false},
|
|
{name: "200 healthy without reason", status: http.StatusOK, wantStatus: "healthy", wantReasonHad: false},
|
|
{name: "401 reachable but unauthorized", status: http.StatusUnauthorized, wantStatus: "healthy", wantReasonHad: true},
|
|
{name: "404 reachable but misrouted", status: http.StatusNotFound, wantStatus: "healthy", wantReasonHad: true},
|
|
{name: "503 unhealthy", status: http.StatusServiceUnavailable, wantStatus: "unhealthy", wantReasonHad: true},
|
|
{name: "connection error unhealthy", callErr: errors.New(`Get "http://gw:8081/healthz": dial tcp: connection refused`), wantStatus: "unhealthy", wantReasonHad: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
status, reason := classifyGatewayHealth(test.status, test.callErr)
|
|
if status != test.wantStatus {
|
|
t.Fatalf("classify status = %q, want %q (reason=%q)", status, test.wantStatus, reason)
|
|
}
|
|
if test.wantReasonHad != (reason != "") {
|
|
t.Fatalf("classify reason = %q, wantReasonHad=%v", reason, test.wantReasonHad)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestClassifyGatewayHealthTruncatesReason(t *testing.T) {
|
|
long := errors.New(string(make([]byte, 1024)))
|
|
_, reason := classifyGatewayHealth(0, long)
|
|
if len(reason) != 300 {
|
|
t.Fatalf("reason length = %d, want 300", len(reason))
|
|
}
|
|
}
|
|
|
|
func TestRecordGatewayHealthChecksPersistsLatestResult(t *testing.T) {
|
|
reachable := &fakeGateway{token: "token-a"}
|
|
reachableServer := httptest.NewServer(reachable.handler(t))
|
|
t.Cleanup(reachableServer.Close)
|
|
|
|
closedServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
|
closedServer.Close() // 端口已释放，保证连接失败
|
|
|
|
store := &healthMemoryStore{memoryStore: newMemoryStore()}
|
|
store.gateways["gw-live"] = hub.Gateway{Name: "gw-live", Endpoint: reachableServer.URL, Token: reachable.token}
|
|
store.gateways["gw-down"] = hub.Gateway{Name: "gw-down", Endpoint: closedServer.URL, Token: "unused-token"}
|
|
|
|
if err := recordGatewayHealthChecks(context.Background(), store); err != nil {
|
|
t.Fatalf("record gateway health checks returned error: %v", err)
|
|
}
|
|
|
|
byName := map[string]recordedGatewayCheck{}
|
|
for _, check := range store.checks {
|
|
byName[check.name] = check
|
|
}
|
|
if len(store.checks) != 2 {
|
|
t.Fatalf("expected one check per gateway, got %#v", store.checks)
|
|
}
|
|
if byName["gw-live"].status != "healthy" {
|
|
t.Fatalf("reachable gateway status = %q, want healthy", byName["gw-live"].status)
|
|
}
|
|
if byName["gw-live"].reason != "" {
|
|
t.Fatalf("reachable gateway reason = %q, want empty", byName["gw-live"].reason)
|
|
}
|
|
if byName["gw-down"].status != "unhealthy" || byName["gw-down"].reason == "" {
|
|
t.Fatalf("unreachable gateway = %#v, want unhealthy with reason", byName["gw-down"])
|
|
}
|
|
if requests := reachable.recorded(); len(requests) != 1 || requests[0].path != "/healthz" {
|
|
t.Fatalf("reachable gateway probes = %#v, want exactly one /healthz", requests)
|
|
}
|
|
}
|
|
|
|
func TestRecordGatewayHealthChecksEmptyListIsNoop(t *testing.T) {
|
|
store := &healthMemoryStore{memoryStore: newMemoryStore()}
|
|
if err := recordGatewayHealthChecks(context.Background(), store); err != nil {
|
|
t.Fatalf("empty list returned error: %v", err)
|
|
}
|
|
if len(store.checks) != 0 {
|
|
t.Fatalf("empty list recorded checks: %#v", store.checks)
|
|
}
|
|
}
|
|
|
|
// 编译期约束：探活超时常量必须明显小于调度间隔，避免探活互相堆叠。
|
|
func TestGatewayHealthProbeTimeoutBounded(t *testing.T) {
|
|
if gatewayHealthProbeTimeout > 10*time.Second {
|
|
t.Fatalf("probe timeout %v must stay below scheduling cadence", gatewayHealthProbeTimeout)
|
|
}
|
|
}
|
|
|
|
// waitForGatewayCheck 轮询等待指定网关的立即探活结果落库；超时判失败。
|
|
func waitForGatewayCheck(t *testing.T, store *healthMemoryStore, name string) recordedGatewayCheck {
|
|
t.Helper()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
store.mu.Lock()
|
|
for _, check := range store.checks {
|
|
if check.name == name {
|
|
store.mu.Unlock()
|
|
return check
|
|
}
|
|
}
|
|
store.mu.Unlock()
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
t.Fatalf("gateway %q health check was not recorded within deadline", name)
|
|
return recordedGatewayCheck{}
|
|
}
|
|
|
|
// healthzProbeServer 返回校验 Bearer token 后立即响应 204 的 /healthz 测试服务。
|
|
func healthzProbeServer(t *testing.T, token string, probeStarted chan struct{}, respond func(http.ResponseWriter)) *httptest.Server {
|
|
t.Helper()
|
|
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.URL.Path != "/healthz" {
|
|
response.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
if request.Header.Get("Authorization") != "Bearer "+token {
|
|
response.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if probeStarted != nil {
|
|
close(probeStarted)
|
|
}
|
|
if respond != nil {
|
|
respond(response)
|
|
return
|
|
}
|
|
response.WriteHeader(http.StatusNoContent)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
return server
|
|
}
|
|
|
|
// TestCreateGatewayProbesHealthImmediately：注册网关成功后立即按注册的 endpoint 与 token
|
|
// 对 /healthz 探活并落库，结果为 healthy。
|
|
func TestCreateGatewayProbesHealthImmediately(t *testing.T) {
|
|
store := &healthMemoryStore{memoryStore: newMemoryStore()}
|
|
probeStarted := make(chan struct{})
|
|
server := healthzProbeServer(t, "unit-test-gateway-token", probeStarted, nil)
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/gateways",
|
|
`{"name":"gw-new","endpoint":"`+server.URL+`","token":"unit-test-gateway-token"}`)
|
|
if response.Code != http.StatusCreated {
|
|
t.Fatalf("create gateway returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
|
|
select {
|
|
case <-probeStarted:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("gateway health probe was not triggered after create")
|
|
}
|
|
check := waitForGatewayCheck(t, store, "gw-new")
|
|
if check.status != "healthy" || check.reason != "" {
|
|
t.Fatalf("created gateway check = %#v, want healthy without reason", check)
|
|
}
|
|
}
|
|
|
|
// TestCreateGatewayDoesNotBlockOnSlowGateway：网关探活被拖住时注册请求必须立即返回，
|
|
// 后台探活超时后把 unhealthy 结果落库。
|
|
func TestCreateGatewayDoesNotBlockOnSlowGateway(t *testing.T) {
|
|
t.Cleanup(restoreGatewayHealthProbeTimeout(setGatewayHealthProbeTimeout(250 * time.Millisecond)))
|
|
store := &healthMemoryStore{memoryStore: newMemoryStore()}
|
|
server := healthzProbeServer(t, "unit-test-gateway-token", nil, func(response http.ResponseWriter) {
|
|
time.Sleep(800 * time.Millisecond) // 超过探活超时，若探活同步执行则注册请求会被拖住
|
|
response.WriteHeader(http.StatusNoContent)
|
|
})
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
started := time.Now()
|
|
response := do(app, http.MethodPost, "/api/gateways",
|
|
`{"name":"gw-slow","endpoint":"`+server.URL+`","token":"unit-test-gateway-token"}`)
|
|
if response.Code != http.StatusCreated {
|
|
t.Fatalf("create gateway returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if elapsed := time.Since(started); elapsed > 500*time.Millisecond {
|
|
t.Fatalf("create gateway blocked %v on health probe", elapsed)
|
|
}
|
|
check := waitForGatewayCheck(t, store, "gw-slow")
|
|
if check.status != "unhealthy" || check.reason == "" {
|
|
t.Fatalf("slow gateway check = %#v, want unhealthy with reason", check)
|
|
}
|
|
}
|
|
|
|
// TestUpdateGatewayProbesHealthImmediately：编辑网关成功后立即按新 endpoint 探活并覆盖落库。
|
|
func TestUpdateGatewayProbesHealthImmediately(t *testing.T) {
|
|
store := &healthMemoryStore{memoryStore: newMemoryStore()}
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: "http://gw-old:8081", Token: "unit-test-gateway-token"}
|
|
server := healthzProbeServer(t, "unit-test-gateway-token", nil, nil)
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPut, "/api/gateways/gw-1",
|
|
`{"name":"gw-1","endpoint":"`+server.URL+`","token":""}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("update gateway returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
check := waitForGatewayCheck(t, store, "gw-1")
|
|
if check.status != "healthy" || check.reason != "" {
|
|
t.Fatalf("updated gateway check = %#v, want healthy without reason", check)
|
|
}
|
|
}
|
|
|
|
// TestUpdateGatewayUnreachableRecordsUnhealthy：编辑指向不可达地址时保存照常成功，
|
|
// 后台探活把 unhealthy 结果落库供页面展示。
|
|
func TestUpdateGatewayUnreachableRecordsUnhealthy(t *testing.T) {
|
|
t.Cleanup(restoreGatewayHealthProbeTimeout(setGatewayHealthProbeTimeout(300 * time.Millisecond)))
|
|
store := &healthMemoryStore{memoryStore: newMemoryStore()}
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: "http://gw-old:8081", Token: "unit-test-gateway-token"}
|
|
closedServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
|
closedServer.Close() // 端口已释放，保证连接失败
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPut, "/api/gateways/gw-1",
|
|
`{"name":"gw-1","endpoint":"`+closedServer.URL+`","token":""}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("update gateway returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
check := waitForGatewayCheck(t, store, "gw-1")
|
|
if check.status != "unhealthy" || check.reason == "" {
|
|
t.Fatalf("unreachable gateway check = %#v, want unhealthy with reason", check)
|
|
}
|
|
}
|