Verify current HTTP reads and fail-closed outages

This commit is contained in:
2026-09-30 09:05:40 +08:00
parent 0e0537ebb5
commit 2b95966ba6
5 changed files with 148 additions and 17 deletions
@@ -71,7 +71,18 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
t.Fatal(err)
}
defer func() { _ = admin.QueueUnbind(result.Queue, result.BindingKey, result.Exchange, nil) }()
for _, queue := range []string{control.Queue, result.Queue} {
taskRoute, err := tenant.CurrentTaskRoute(dispatcherID, "task-asr")
if err != nil {
t.Fatal(err)
}
if _, err := admin.QueueDeclare(taskRoute.Queue, true, false, false, false, nil); err != nil {
t.Fatal(err)
}
defer func() { _, _ = admin.QueueDelete(taskRoute.Queue, false, false, false) }()
if err := admin.QueueBind(taskRoute.Queue, taskRoute.BindingKey, taskRoute.Exchange, false, nil); err != nil {
t.Fatal(err)
}
for _, queue := range []string{control.Queue, result.Queue, taskRoute.Queue} {
if _, err := admin.QueuePurge(queue, false); err != nil {
t.Fatal(err)
}
@@ -106,15 +117,15 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
go func() { _ = agentGRPC.Serve(agentListener) }()
defer agentGRPC.Stop()
sipBody, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", "config-read-sip.json"))
if err != nil {
t.Fatal(err)
examples := map[string][]byte{}
for _, name := range []string{"config-read-sip.json", "task-discovery-page.json", "task-discovery-end.json", "config-read-task-asr.json", "config-read-providers.json", "config-read-quota.json"} {
body, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", name))
if err != nil {
t.Fatal(err)
}
examples[name] = body
}
taskBody, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", "task-discovery-end.json"))
if err != nil {
t.Fatal(err)
}
var sipReads, taskReads atomic.Int64
var sipReads, discoveryReads, taskReads, providerReads, quotaReads atomic.Int64
saas := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("X-DISPATCHER-id") != dispatcherID || r.Header.Get("X-DISPATCHER-SECRET-KEY") != "isolated-secret" {
http.Error(w, "unapproved Dispatcher", http.StatusForbidden)
@@ -124,10 +135,26 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
switch r.URL.Path {
case "/internal/v1/dispatcher/sip":
sipReads.Add(1)
_, _ = w.Write(sipBody)
_, _ = w.Write(examples["config-read-sip.json"])
case "/internal/v1/dispatcher/tasks":
discoveryReads.Add(1)
switch r.URL.Query().Get("after") {
case "":
_, _ = w.Write(examples["task-discovery-page.json"])
case "opaque-page-token-1", "opaque-end-token":
_, _ = w.Write(examples["task-discovery-end.json"])
default:
http.Error(w, "unexpected Mock cursor", http.StatusBadRequest)
}
case "/internal/v1/dispatcher/task/task-asr":
taskReads.Add(1)
_, _ = w.Write(taskBody)
_, _ = w.Write(examples["config-read-task-asr.json"])
case "/internal/v1/dispatcher/ai-providers":
providerReads.Add(1)
_, _ = w.Write(examples["config-read-providers.json"])
case "/internal/v1/dispatcher/tenant/1001/quota":
quotaReads.Add(1)
_, _ = w.Write(examples["config-read-quota.json"])
default:
http.Error(w, "unexpected Mock read", http.StatusNotFound)
}
@@ -176,14 +203,33 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
finished := make(chan error, 1)
go func() { finished <- command.Execute() }()
deadline := time.After(8 * time.Second)
for sipReads.Load() == 0 || taskReads.Load() == 0 {
for sipReads.Load() < 2 || discoveryReads.Load() < 2 || taskReads.Load() == 0 || providerReads.Load() == 0 || quotaReads.Load() == 0 {
select {
case err := <-finished:
cancel()
t.Fatalf("isolated Dispatcher exited before HTTP bootstrap: %v", err)
t.Fatalf("isolated Dispatcher exited before the five HTTP resources were read: %v", err)
case <-deadline:
cancel()
t.Fatal("isolated Dispatcher never read SIP and the empty task snapshot")
t.Fatal("isolated Dispatcher did not read SIP, discovery, task, providers, and tenant quota")
case <-time.After(20 * time.Millisecond):
}
}
for {
queue, err := admin.QueueInspect(taskRoute.Queue)
if err != nil {
cancel()
t.Fatalf("assigned task queue cannot be inspected: %v", err)
}
if queue.Consumers > 0 {
break
}
select {
case err := <-finished:
cancel()
t.Fatalf("isolated Dispatcher exited before consuming its task queue: %v", err)
case <-deadline:
cancel()
t.Fatal("isolated Dispatcher did not consume its SaaS-provisioned task queue")
case <-time.After(20 * time.Millisecond):
}
}