Verify current HTTP reads and fail-closed outages
This commit is contained in:
@@ -71,7 +71,18 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = admin.QueueUnbind(result.Queue, result.BindingKey, result.Exchange, nil) }()
|
||||
for _, queue := range []string{control.Queue, result.Queue} {
|
||||
taskRoute, err := tenant.CurrentTaskRoute(dispatcherID, "task-asr")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := admin.QueueDeclare(taskRoute.Queue, true, false, false, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _, _ = admin.QueueDelete(taskRoute.Queue, false, false, false) }()
|
||||
if err := admin.QueueBind(taskRoute.Queue, taskRoute.BindingKey, taskRoute.Exchange, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, queue := range []string{control.Queue, result.Queue, taskRoute.Queue} {
|
||||
if _, err := admin.QueuePurge(queue, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -106,15 +117,15 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
|
||||
go func() { _ = agentGRPC.Serve(agentListener) }()
|
||||
defer agentGRPC.Stop()
|
||||
|
||||
sipBody, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", "config-read-sip.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
examples := map[string][]byte{}
|
||||
for _, name := range []string{"config-read-sip.json", "task-discovery-page.json", "task-discovery-end.json", "config-read-task-asr.json", "config-read-providers.json", "config-read-quota.json"} {
|
||||
body, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
examples[name] = body
|
||||
}
|
||||
taskBody, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", "task-discovery-end.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sipReads, taskReads atomic.Int64
|
||||
var sipReads, discoveryReads, taskReads, providerReads, quotaReads atomic.Int64
|
||||
saas := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("X-DISPATCHER-id") != dispatcherID || r.Header.Get("X-DISPATCHER-SECRET-KEY") != "isolated-secret" {
|
||||
http.Error(w, "unapproved Dispatcher", http.StatusForbidden)
|
||||
@@ -124,10 +135,26 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
|
||||
switch r.URL.Path {
|
||||
case "/internal/v1/dispatcher/sip":
|
||||
sipReads.Add(1)
|
||||
_, _ = w.Write(sipBody)
|
||||
_, _ = w.Write(examples["config-read-sip.json"])
|
||||
case "/internal/v1/dispatcher/tasks":
|
||||
discoveryReads.Add(1)
|
||||
switch r.URL.Query().Get("after") {
|
||||
case "":
|
||||
_, _ = w.Write(examples["task-discovery-page.json"])
|
||||
case "opaque-page-token-1", "opaque-end-token":
|
||||
_, _ = w.Write(examples["task-discovery-end.json"])
|
||||
default:
|
||||
http.Error(w, "unexpected Mock cursor", http.StatusBadRequest)
|
||||
}
|
||||
case "/internal/v1/dispatcher/task/task-asr":
|
||||
taskReads.Add(1)
|
||||
_, _ = w.Write(taskBody)
|
||||
_, _ = w.Write(examples["config-read-task-asr.json"])
|
||||
case "/internal/v1/dispatcher/ai-providers":
|
||||
providerReads.Add(1)
|
||||
_, _ = w.Write(examples["config-read-providers.json"])
|
||||
case "/internal/v1/dispatcher/tenant/1001/quota":
|
||||
quotaReads.Add(1)
|
||||
_, _ = w.Write(examples["config-read-quota.json"])
|
||||
default:
|
||||
http.Error(w, "unexpected Mock read", http.StatusNotFound)
|
||||
}
|
||||
@@ -176,14 +203,33 @@ func TestCurrentDispatcherCommandStartsWithIsolatedMQHTTPAndAgent(t *testing.T)
|
||||
finished := make(chan error, 1)
|
||||
go func() { finished <- command.Execute() }()
|
||||
deadline := time.After(8 * time.Second)
|
||||
for sipReads.Load() == 0 || taskReads.Load() == 0 {
|
||||
for sipReads.Load() < 2 || discoveryReads.Load() < 2 || taskReads.Load() == 0 || providerReads.Load() == 0 || quotaReads.Load() == 0 {
|
||||
select {
|
||||
case err := <-finished:
|
||||
cancel()
|
||||
t.Fatalf("isolated Dispatcher exited before HTTP bootstrap: %v", err)
|
||||
t.Fatalf("isolated Dispatcher exited before the five HTTP resources were read: %v", err)
|
||||
case <-deadline:
|
||||
cancel()
|
||||
t.Fatal("isolated Dispatcher never read SIP and the empty task snapshot")
|
||||
t.Fatal("isolated Dispatcher did not read SIP, discovery, task, providers, and tenant quota")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
for {
|
||||
queue, err := admin.QueueInspect(taskRoute.Queue)
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("assigned task queue cannot be inspected: %v", err)
|
||||
}
|
||||
if queue.Consumers > 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case err := <-finished:
|
||||
cancel()
|
||||
t.Fatalf("isolated Dispatcher exited before consuming its task queue: %v", err)
|
||||
case <-deadline:
|
||||
cancel()
|
||||
t.Fatal("isolated Dispatcher did not consume its SaaS-provisioned task queue")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user