394 lines
16 KiB
Go
394 lines
16 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
)
|
|
|
|
type fakeControlAgent struct {
|
|
calls []ControlSpec
|
|
err error
|
|
before func(ControlSpec)
|
|
}
|
|
|
|
func (a *fakeControlAgent) SendControl(_ context.Context, spec ControlSpec) error {
|
|
if a.before != nil {
|
|
a.before(spec)
|
|
}
|
|
a.calls = append(a.calls, spec)
|
|
return a.err
|
|
}
|
|
|
|
func newControlFixture(t *testing.T) (*ControlController, *fakeControlAgent, *store.Store) {
|
|
t.Helper()
|
|
id := "c046b893-8628-4589-ae50-619d049248a6"
|
|
snapshot := policySnapshot(t)
|
|
approvedSIP, err := json.Marshal(snapshot.SIP)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
var body []byte
|
|
switch r.URL.Path {
|
|
case "/internal/v1/dispatcher/sip":
|
|
body = approvedSIP
|
|
case "/internal/v1/dispatcher/ai-providers":
|
|
body = configExample(t, "config-read-providers")
|
|
case "/internal/v1/dispatcher/task/task-asr":
|
|
body = configExample(t, "config-read-task-asr")
|
|
case "/internal/v1/dispatcher/tenant/1001/quota":
|
|
body = configExample(t, "config-read-quota")
|
|
default:
|
|
t.Errorf("unexpected HTTP path %s", r.URL.Path)
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
_, _ = w.Write(body)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
client, err := configread.NewClient(server.URL, id, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
providers, err := client.ReadProviders(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := store.Open(filepath.Join(t.TempDir(), "state.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = s.Close() })
|
|
if err := s.ApplyDiscoverySnapshot(id, []configread.DiscoveredTask{{TaskID: "task-asr", TenantID: 1001, TaskRevision: 1, Status: "running"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.SaveSnapshot(snapshot); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.MarkReadyForSIP(id, snapshot.SIP.Revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
agent := &fakeControlAgent{}
|
|
controller := &ControlController{DispatcherID: id, Store: s, Client: client, Agent: agent, ApprovedSIP: &snapshot.SIP, ApprovedProviders: &providers, Now: func() time.Time { return monday(9, 30) }, PurgeTaskQueue: func(context.Context, string) (int, error) { return 0, nil }}
|
|
return controller, agent, s
|
|
}
|
|
|
|
func controlBody(t *testing.T, eventID, action, policy string) []byte {
|
|
t.Helper()
|
|
body := string(configExample(t, "mq-control"))
|
|
body = strings.Replace(body, `"event_id":"control-example"`, `"event_id":"`+eventID+`"`, 1)
|
|
body = strings.Replace(body, `"action":"pause"`, `"action":"`+action+`"`, 1)
|
|
body = strings.Replace(body, `"active_call_policy":"drain"`, `"active_call_policy":"`+policy+`"`, 1)
|
|
if policy == "" {
|
|
body = strings.Replace(body, `,"options":{"active_call_policy":""}`, ``, 1)
|
|
}
|
|
return []byte(body)
|
|
}
|
|
|
|
func TestControlPauseDefaultHangupAckAfterAgentDispatch(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
agent.before = func(spec ControlSpec) {
|
|
if spec.Action != "pause" || spec.ActiveCallPolicy != "hangup" {
|
|
t.Errorf("wrong default control policy: %+v", spec)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Errorf("task not blocked before Agent control: %v %v", admitted, err)
|
|
}
|
|
if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 {
|
|
t.Errorf("ack written before Agent dispatch: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
body := controlBody(t, "pause-1", "pause", "")
|
|
if err := controller.ProcessControl(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("control was not dispatched: %+v", agent.calls)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) {
|
|
t.Fatalf("no durable applied ack: %+v %v", outbox, err)
|
|
}
|
|
if err := s.MarkOutboxConfirmed(controller.DispatcherID, "pause-1"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
agent.before = nil
|
|
if err := controller.ProcessControl(context.Background(), body); err != nil || len(agent.calls) != 2 {
|
|
t.Fatalf("control was deduplicated or failed: %d %v", len(agent.calls), err)
|
|
}
|
|
outbox, err = s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 1 || outbox[0].EventID != "pause-1" {
|
|
t.Fatalf("repeat control did not requeue original ack: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestControlStopCannotResumeOrDispatchPending(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
command := store.ExecuteCommand{DispatcherID: controller.DispatcherID, EventID: "queued-1", TenantID: 1001, TaskID: "task-asr", Callee: "15003164745", IssuedAt: "2026-09-21T01:00:00Z"}
|
|
if _, _, err := s.RecordExecute(command); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-1", "stop", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 || agent.calls[0].ActiveCallPolicy != "hangup" {
|
|
t.Fatalf("stop did not dispatch hangup: %+v", agent.calls)
|
|
}
|
|
if pending, err := s.ListPendingExecute(controller.DispatcherID); err != nil || len(pending) != 0 {
|
|
t.Fatalf("old pending call survived stop: %+v %v", pending, err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "resume-1", "resume", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("stopped task sent resume: %+v", agent.calls)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 2 || !strings.Contains(string(outbox[1].Body), `"status":"rejected"`) {
|
|
t.Fatalf("stop/resume result wrong: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestControlStopPurgesAfterBarrierBeforeAgentAndAck(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
controller.PurgeTaskQueue = func(_ context.Context, taskID string) (int, error) {
|
|
if taskID != "task-asr" {
|
|
t.Fatalf("purged wrong task: %s", taskID)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, taskID); err != nil || admitted {
|
|
t.Fatalf("stop barrier missing at purge: %v %v", admitted, err)
|
|
}
|
|
if len(agent.calls) != 0 {
|
|
t.Fatal("Agent called before purge")
|
|
}
|
|
if pending, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(pending) != 0 {
|
|
t.Fatalf("ack before purge: %v %v", pending, err)
|
|
}
|
|
return 42, nil
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-purge", "stop", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("Agent not called after purge: %v", agent.calls)
|
|
}
|
|
}
|
|
|
|
func TestControlStopPurgeFailureDoesNotAckOrCallAgent(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
controller.PurgeTaskQueue = func(context.Context, string) (int, error) { return 0, errors.New("rabbitmq purge failed") }
|
|
body := controlBody(t, "stop-fail", "stop", "")
|
|
if err := controller.ProcessControl(context.Background(), body); err == nil || !strings.Contains(err.Error(), "rabbitmq purge failed") {
|
|
t.Fatalf("purge failure hidden: %v", err)
|
|
}
|
|
if len(agent.calls) != 0 {
|
|
t.Fatalf("Agent called after purge failure: %v", agent.calls)
|
|
}
|
|
if pending, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(pending) != 0 {
|
|
t.Fatalf("stop falsely acknowledged: %v %v", pending, err)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Fatalf("failed purge reopened admission: %v %v", admitted, err)
|
|
}
|
|
controller.PurgeTaskQueue = func(context.Context, string) (int, error) { return 0, nil }
|
|
if err := controller.ProcessControl(context.Background(), body); err != nil {
|
|
t.Fatalf("redelivered stop did not recover: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestStoppedTaskSilentlyAcknowledgesOldExecuteWithoutRecreatingInbox(t *testing.T) {
|
|
control, _, s := newControlFixture(t)
|
|
if err := control.ProcessControl(context.Background(), controlBody(t, "stop-before-execute", "stop", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originator := &fakeOriginator{loaded: map[string]int64{"trunk-mock": 8}}
|
|
execute := &ExecuteController{DispatcherID: control.DispatcherID, Store: s, Originator: originator, Publisher: &fakePublisher{}, Now: func() time.Time { return monday(9, 30) }}
|
|
if err := execute.ProcessExecute(context.Background(), executeBody(t, "old-queued-1", "15003164745")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(originator.calls) != 0 {
|
|
t.Fatal("stopped task originated an old queued command")
|
|
}
|
|
if pending, err := s.ListPendingExecute(control.DispatcherID); err != nil || len(pending) != 0 {
|
|
t.Fatalf("silently acknowledged old command was persisted as dispatchable: %+v %v", pending, err)
|
|
}
|
|
if outbox, err := s.ListPendingOutbox(control.DispatcherID); err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" {
|
|
t.Fatalf("old unadmitted command emitted business result: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestResumeUsesApprovedSIPButPendingRevisionStillClosesAdmission(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "pause-before-resume", "pause", "drain")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.NoteSIPChange(controller.DispatcherID, 9); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "resume-after-load", "resume", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 2 || agent.calls[1].Action != "resume" {
|
|
t.Fatalf("resume did not reach Agent: %+v", agent.calls)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Fatalf("pending SIP revision reopened admission: %v %v", admitted, err)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 2 || outbox[1].EventID != "resume-after-load" {
|
|
t.Fatalf("applied resume has no durable acknowledgment: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestStartReadsNewTaskWithoutAgentControl(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/internal/v1/dispatcher/sip":
|
|
raw, _ := json.Marshal(policySnapshot(t).SIP)
|
|
_, _ = w.Write(raw)
|
|
case "/internal/v1/dispatcher/task/task-new":
|
|
_, _ = w.Write([]byte(strings.ReplaceAll(string(configExample(t, "config-read-task-asr")), "task-asr", "task-new")))
|
|
case "/internal/v1/dispatcher/ai-providers":
|
|
_, _ = w.Write(configExample(t, "config-read-providers"))
|
|
case "/internal/v1/dispatcher/tenant/1001/quota":
|
|
_, _ = w.Write(configExample(t, "config-read-quota"))
|
|
default:
|
|
t.Errorf("unexpected HTTP path %s", r.URL.Path)
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
var err error
|
|
controller.Client, err = configread.NewClient(server.URL, controller.DispatcherID, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body := strings.ReplaceAll(string(controlBody(t, "start-new", "start", "")), "task-asr", "task-new")
|
|
if err := controller.ProcessControl(context.Background(), []byte(body)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 0 {
|
|
t.Fatalf("start unexpectedly sent Agent control: %+v", agent.calls)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-new"); err != nil || !admitted {
|
|
t.Fatalf("new task not admitted after start: %v %v", admitted, err)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 1 || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) {
|
|
t.Fatalf("start acknowledgment missing: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestStartDoesNotReopenPausedOrStoppedTasks(t *testing.T) {
|
|
for _, action := range []string{"pause", "stop"} {
|
|
t.Run(action, func(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "preceding-"+action, action, "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "late-start-"+action, "start", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("start unexpectedly reached Agent: %+v", agent.calls)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Fatalf("start reopened %s task: %v %v", action, admitted, err)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 2 || !strings.Contains(string(outbox[1].Body), `"status":"rejected"`) {
|
|
t.Fatalf("start rejection acknowledgment missing: %+v %v", outbox, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResumeRefreshesModifiedTaskRevision(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "pause-before-edit", "pause", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/internal/v1/dispatcher/sip":
|
|
raw, _ := json.Marshal(policySnapshot(t).SIP)
|
|
_, _ = w.Write(raw)
|
|
case "/internal/v1/dispatcher/task/task-asr":
|
|
_, _ = w.Write([]byte(strings.Replace(string(configExample(t, "config-read-task-asr")), `"task_revision":1`, `"task_revision":2`, 1)))
|
|
case "/internal/v1/dispatcher/ai-providers":
|
|
_, _ = w.Write(configExample(t, "config-read-providers"))
|
|
case "/internal/v1/dispatcher/tenant/1001/quota":
|
|
_, _ = w.Write(configExample(t, "config-read-quota"))
|
|
default:
|
|
t.Errorf("unexpected HTTP path %s", r.URL.Path)
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
var err error
|
|
controller.Client, err = configread.NewClient(server.URL, controller.DispatcherID, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "resume-after-edit", "resume", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 2 || agent.calls[1].Action != "resume" {
|
|
t.Fatalf("edited task was not resumed: %+v", agent.calls)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || !admitted {
|
|
t.Fatalf("edited task not admitted: %v %v", admitted, err)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeSurfacesControlFailureToCloseAdmission(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
agent.err = errors.New("injected Agent control failure")
|
|
runtime := &Runtime{Bootstrap: Bootstrap{DispatcherID: controller.DispatcherID, Store: s}, Control: *controller, Logger: slog.Default(), failures: make(chan error, 1), taskLocks: make(map[string]*sync.Mutex)}
|
|
if err := runtime.handleControl(context.Background(), "", controlBody(t, "fatal-control", "pause", "")); err == nil {
|
|
t.Fatal("Agent control failure was hidden")
|
|
}
|
|
select {
|
|
case err := <-runtime.failures:
|
|
if err == nil {
|
|
t.Fatal("control failure signal was empty")
|
|
}
|
|
default:
|
|
t.Fatal("consumer will requeue failed control indefinitely without closing admission")
|
|
}
|
|
}
|
|
|
|
func TestControlAgentFailureKeepsBarrierWithoutSuccessAck(t *testing.T) {
|
|
controller, agent, s := newControlFixture(t)
|
|
agent.err = errors.New("injected Agent control RPC timeout")
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-timeout", "stop", "drain")); err == nil {
|
|
t.Fatal("Agent control failure was hidden")
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Fatalf("control failure reopened task admission: %v %v", admitted, err)
|
|
}
|
|
if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 {
|
|
t.Fatalf("false applied ack after Agent failure: %+v %v", outbox, err)
|
|
}
|
|
agent.err = nil
|
|
if err := controller.ProcessControl(context.Background(), controlBody(t, "stop-timeout", "stop", "drain")); err != nil || len(agent.calls) != 2 {
|
|
t.Fatalf("redelivered control not dispatched: %+v %v", agent.calls, err)
|
|
}
|
|
}
|