package dispatcher import ( "context" "encoding/json" "errors" "log/slog" "net/http" "net/http/httptest" "path/filepath" "strings" "sync" "testing" "time" "git.ipao.vip/rogee/go-sip/internal/configread" "git.ipao.vip/rogee/go-sip/internal/store" ) type fakeControlAgent struct { calls []ControlSpec err error before func(ControlSpec) } func (a *fakeControlAgent) SendControl(_ context.Context, spec ControlSpec) error { if a.before != nil { a.before(spec) } a.calls = append(a.calls, spec) return a.err } func newControlFixture(t *testing.T) (*ControlController, *fakeControlAgent, *store.Store) { t.Helper() id := "c046b893-8628-4589-ae50-619d049248a6" snapshot := policySnapshot(t) approvedSIP, err := json.Marshal(snapshot.SIP) if err != nil { t.Fatal(err) } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") var body []byte switch r.URL.Path { case "/internal/v1/dispatcher/sip": body = approvedSIP case "/internal/v1/dispatcher/ai-providers": body = configExample(t, "config-read-providers") case "/internal/v1/dispatcher/task/task-asr": body = configExample(t, "config-read-task-asr") case "/internal/v1/dispatcher/tenant/1001/quota": body = configExample(t, "config-read-quota") default: t.Errorf("unexpected HTTP path %s", r.URL.Path) w.WriteHeader(http.StatusNotFound) return } _, _ = w.Write(body) })) t.Cleanup(server.Close) client, err := configread.NewClient(server.URL, id, "test-secret", server.Client()) if err != nil { t.Fatal(err) } providers, err := client.ReadProviders(context.Background()) if err != nil { t.Fatal(err) } s, err := store.Open(filepath.Join(t.TempDir(), "state.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = s.Close() }) if err := s.ApplyDiscoverySnapshot(id, []configread.DiscoveredTask{{TaskID: "task-asr", TenantID: 1001, TaskRevision: 1, Status: "running"}}); err != nil { t.Fatal(err) } if err := s.SaveSnapshot(snapshot); err != nil { t.Fatal(err) } if err := s.MarkReadyForSIP(id, snapshot.SIP.Revision); err != nil { t.Fatal(err) } agent := &fakeControlAgent{} controller := &ControlController{DispatcherID: id, Store: s, Client: client, Agent: agent, ApprovedSIP: &snapshot.SIP, ApprovedProviders: &providers, Now: func() time.Time { return monday(9, 30) }, PurgeTaskQueue: func(context.Context, string) (int, error) { return 0, nil }} return controller, agent, s } func controlBody(t *testing.T, eventID, action, policy string) []byte { t.Helper() body := string(configExample(t, "mq-control")) body = strings.Replace(body, `"event_id":"control-example"`, `"event_id":"`+eventID+`"`, 1) body = strings.Replace(body, `"action":"pause"`, `"action":"`+action+`"`, 1) body = strings.Replace(body, `"active_call_policy":"drain"`, `"active_call_policy":"`+policy+`"`, 1) if policy == "" { body = strings.Replace(body, `,"options":{"active_call_policy":""}`, ``, 1) } return []byte(body) } func TestControlPauseDefaultHangupAckAfterAgentDispatch(t *testing.T) { controller, agent, s := newControlFixture(t) agent.before = func(spec ControlSpec) { if spec.Action != "pause" || spec.ActiveCallPolicy != "hangup" { t.Errorf("wrong default control policy: %+v", spec) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { t.Errorf("task not blocked before Agent control: %v %v", admitted, err) } if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 { t.Errorf("ack written before Agent dispatch: %+v %v", outbox, err) } } body := controlBody(t, "pause-1", "pause", "") if err := controller.ProcessControl(context.Background(), body); err != nil { t.Fatal(err) } if len(agent.calls) != 1 { t.Fatalf("control was not dispatched: %+v", agent.calls) } outbox, err := s.ListPendingOutbox(controller.DispatcherID) if err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) { t.Fatalf("no durable applied ack: %+v %v", outbox, err) } if err := s.MarkOutboxConfirmed(controller.DispatcherID, "pause-1"); err != nil { t.Fatal(err) } agent.before = nil if err := controller.ProcessControl(context.Background(), body); err != nil || len(agent.calls) != 2 { t.Fatalf("control was deduplicated or failed: %d %v", len(agent.calls), err) } outbox, err = s.ListPendingOutbox(controller.DispatcherID) if err != nil || len(outbox) != 1 || outbox[0].EventID != "pause-1" { t.Fatalf("repeat control did not requeue original ack: %+v %v", outbox, err) } } func TestControlStopCannotResumeOrDispatchPending(t *testing.T) { controller, agent, s := newControlFixture(t) command := store.ExecuteCommand{DispatcherID: controller.DispatcherID, EventID: "queued-1", TenantID: 1001, TaskID: "task-asr", Callee: "15003164745", IssuedAt: "2026-09-21T01:00:00Z"} if _, _, err := s.RecordExecute(command); err != nil { t.Fatal(err) } if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-1", "stop", "")); err != nil { t.Fatal(err) } if len(agent.calls) != 1 || agent.calls[0].ActiveCallPolicy != "hangup" { t.Fatalf("stop did not dispatch hangup: %+v", agent.calls) } if pending, err := s.ListPendingExecute(controller.DispatcherID); err != nil || len(pending) != 0 { t.Fatalf("old pending call survived stop: %+v %v", pending, err) } if err := controller.ProcessControl(context.Background(), controlBody(t, "resume-1", "resume", "")); err != nil { t.Fatal(err) } if len(agent.calls) != 1 { t.Fatalf("stopped task sent resume: %+v", agent.calls) } outbox, err := s.ListPendingOutbox(controller.DispatcherID) if err != nil || len(outbox) != 2 || !strings.Contains(string(outbox[1].Body), `"status":"rejected"`) { t.Fatalf("stop/resume result wrong: %+v %v", outbox, err) } } func TestControlStopPurgesAfterBarrierBeforeAgentAndAck(t *testing.T) { controller, agent, s := newControlFixture(t) controller.PurgeTaskQueue = func(_ context.Context, taskID string) (int, error) { if taskID != "task-asr" { t.Fatalf("purged wrong task: %s", taskID) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, taskID); err != nil || admitted { t.Fatalf("stop barrier missing at purge: %v %v", admitted, err) } if len(agent.calls) != 0 { t.Fatal("Agent called before purge") } if pending, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(pending) != 0 { t.Fatalf("ack before purge: %v %v", pending, err) } return 42, nil } if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-purge", "stop", "")); err != nil { t.Fatal(err) } if len(agent.calls) != 1 { t.Fatalf("Agent not called after purge: %v", agent.calls) } } func TestControlStopPurgeFailureDoesNotAckOrCallAgent(t *testing.T) { controller, agent, s := newControlFixture(t) controller.PurgeTaskQueue = func(context.Context, string) (int, error) { return 0, errors.New("rabbitmq purge failed") } body := controlBody(t, "stop-fail", "stop", "") if err := controller.ProcessControl(context.Background(), body); err == nil || !strings.Contains(err.Error(), "rabbitmq purge failed") { t.Fatalf("purge failure hidden: %v", err) } if len(agent.calls) != 0 { t.Fatalf("Agent called after purge failure: %v", agent.calls) } if pending, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(pending) != 0 { t.Fatalf("stop falsely acknowledged: %v %v", pending, err) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { t.Fatalf("failed purge reopened admission: %v %v", admitted, err) } controller.PurgeTaskQueue = func(context.Context, string) (int, error) { return 0, nil } if err := controller.ProcessControl(context.Background(), body); err != nil { t.Fatalf("redelivered stop did not recover: %v", err) } } func TestStoppedTaskSilentlyAcknowledgesOldExecuteWithoutRecreatingInbox(t *testing.T) { control, _, s := newControlFixture(t) if err := control.ProcessControl(context.Background(), controlBody(t, "stop-before-execute", "stop", "")); err != nil { t.Fatal(err) } originator := &fakeOriginator{loaded: map[string]int64{"trunk-mock": 8}} execute := &ExecuteController{DispatcherID: control.DispatcherID, Store: s, Originator: originator, Publisher: &fakePublisher{}, Now: func() time.Time { return monday(9, 30) }} if err := execute.ProcessExecute(context.Background(), executeBody(t, "old-queued-1", "15003164745")); err != nil { t.Fatal(err) } if len(originator.calls) != 0 { t.Fatal("stopped task originated an old queued command") } if pending, err := s.ListPendingExecute(control.DispatcherID); err != nil || len(pending) != 0 { t.Fatalf("silently acknowledged old command was persisted as dispatchable: %+v %v", pending, err) } if outbox, err := s.ListPendingOutbox(control.DispatcherID); err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" { t.Fatalf("old unadmitted command emitted business result: %+v %v", outbox, err) } } func TestResumeUsesApprovedSIPButPendingRevisionStillClosesAdmission(t *testing.T) { controller, agent, s := newControlFixture(t) if err := controller.ProcessControl(context.Background(), controlBody(t, "pause-before-resume", "pause", "drain")); err != nil { t.Fatal(err) } if err := s.NoteSIPChange(controller.DispatcherID, 9); err != nil { t.Fatal(err) } if err := controller.ProcessControl(context.Background(), controlBody(t, "resume-after-load", "resume", "")); err != nil { t.Fatal(err) } if len(agent.calls) != 2 || agent.calls[1].Action != "resume" { t.Fatalf("resume did not reach Agent: %+v", agent.calls) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { t.Fatalf("pending SIP revision reopened admission: %v %v", admitted, err) } outbox, err := s.ListPendingOutbox(controller.DispatcherID) if err != nil || len(outbox) != 2 || outbox[1].EventID != "resume-after-load" { t.Fatalf("applied resume has no durable acknowledgment: %+v %v", outbox, err) } } func TestStartReadsNewTaskWithoutAgentControl(t *testing.T) { controller, agent, s := newControlFixture(t) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/internal/v1/dispatcher/sip": raw, _ := json.Marshal(policySnapshot(t).SIP) _, _ = w.Write(raw) case "/internal/v1/dispatcher/task/task-new": _, _ = w.Write([]byte(strings.ReplaceAll(string(configExample(t, "config-read-task-asr")), "task-asr", "task-new"))) case "/internal/v1/dispatcher/ai-providers": _, _ = w.Write(configExample(t, "config-read-providers")) case "/internal/v1/dispatcher/tenant/1001/quota": _, _ = w.Write(configExample(t, "config-read-quota")) default: t.Errorf("unexpected HTTP path %s", r.URL.Path) w.WriteHeader(http.StatusNotFound) } })) defer server.Close() var err error controller.Client, err = configread.NewClient(server.URL, controller.DispatcherID, "test-secret", server.Client()) if err != nil { t.Fatal(err) } body := strings.ReplaceAll(string(controlBody(t, "start-new", "start", "")), "task-asr", "task-new") if err := controller.ProcessControl(context.Background(), []byte(body)); err != nil { t.Fatal(err) } if len(agent.calls) != 0 { t.Fatalf("start unexpectedly sent Agent control: %+v", agent.calls) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-new"); err != nil || !admitted { t.Fatalf("new task not admitted after start: %v %v", admitted, err) } outbox, err := s.ListPendingOutbox(controller.DispatcherID) if err != nil || len(outbox) != 1 || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) { t.Fatalf("start acknowledgment missing: %+v %v", outbox, err) } } func TestStartDoesNotReopenPausedOrStoppedTasks(t *testing.T) { for _, action := range []string{"pause", "stop"} { t.Run(action, func(t *testing.T) { controller, agent, s := newControlFixture(t) if err := controller.ProcessControl(context.Background(), controlBody(t, "preceding-"+action, action, "")); err != nil { t.Fatal(err) } if err := controller.ProcessControl(context.Background(), controlBody(t, "late-start-"+action, "start", "")); err != nil { t.Fatal(err) } if len(agent.calls) != 1 { t.Fatalf("start unexpectedly reached Agent: %+v", agent.calls) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { t.Fatalf("start reopened %s task: %v %v", action, admitted, err) } outbox, err := s.ListPendingOutbox(controller.DispatcherID) if err != nil || len(outbox) != 2 || !strings.Contains(string(outbox[1].Body), `"status":"rejected"`) { t.Fatalf("start rejection acknowledgment missing: %+v %v", outbox, err) } }) } } func TestResumeRefreshesModifiedTaskRevision(t *testing.T) { controller, agent, s := newControlFixture(t) if err := controller.ProcessControl(context.Background(), controlBody(t, "pause-before-edit", "pause", "")); err != nil { t.Fatal(err) } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/internal/v1/dispatcher/sip": raw, _ := json.Marshal(policySnapshot(t).SIP) _, _ = w.Write(raw) case "/internal/v1/dispatcher/task/task-asr": _, _ = w.Write([]byte(strings.Replace(string(configExample(t, "config-read-task-asr")), `"task_revision":1`, `"task_revision":2`, 1))) case "/internal/v1/dispatcher/ai-providers": _, _ = w.Write(configExample(t, "config-read-providers")) case "/internal/v1/dispatcher/tenant/1001/quota": _, _ = w.Write(configExample(t, "config-read-quota")) default: t.Errorf("unexpected HTTP path %s", r.URL.Path) w.WriteHeader(http.StatusNotFound) } })) defer server.Close() var err error controller.Client, err = configread.NewClient(server.URL, controller.DispatcherID, "test-secret", server.Client()) if err != nil { t.Fatal(err) } if err := controller.ProcessControl(context.Background(), controlBody(t, "resume-after-edit", "resume", "")); err != nil { t.Fatal(err) } if len(agent.calls) != 2 || agent.calls[1].Action != "resume" { t.Fatalf("edited task was not resumed: %+v", agent.calls) } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || !admitted { t.Fatalf("edited task not admitted: %v %v", admitted, err) } } func TestRuntimeSurfacesControlFailureToCloseAdmission(t *testing.T) { controller, agent, s := newControlFixture(t) agent.err = errors.New("injected Agent control failure") runtime := &Runtime{Bootstrap: Bootstrap{DispatcherID: controller.DispatcherID, Store: s}, Control: *controller, Logger: slog.Default(), failures: make(chan error, 1), taskLocks: make(map[string]*sync.Mutex)} if err := runtime.handleControl(context.Background(), "", controlBody(t, "fatal-control", "pause", "")); err == nil { t.Fatal("Agent control failure was hidden") } select { case err := <-runtime.failures: if err == nil { t.Fatal("control failure signal was empty") } default: t.Fatal("consumer will requeue failed control indefinitely without closing admission") } } func TestControlAgentFailureKeepsBarrierWithoutSuccessAck(t *testing.T) { controller, agent, s := newControlFixture(t) agent.err = errors.New("injected Agent control RPC timeout") if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-timeout", "stop", "drain")); err == nil { t.Fatal("Agent control failure was hidden") } if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { t.Fatalf("control failure reopened task admission: %v %v", admitted, err) } if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 { t.Fatalf("false applied ack after Agent failure: %+v %v", outbox, err) } agent.err = nil if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-timeout", "stop", "drain")); err != nil || len(agent.calls) != 2 { t.Fatalf("redelivered control not dispatched: %+v %v", agent.calls, err) } }