250 lines
10 KiB
Go
250 lines
10 KiB
Go
package dispatcher
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/go-sip/internal/configread"
|
|
"git.ipao.vip/rogee/go-sip/internal/store"
|
|
)
|
|
|
|
type currentFakeControlAgent struct {
|
|
calls []CurrentControlSpec
|
|
err error
|
|
before func(CurrentControlSpec)
|
|
}
|
|
|
|
func (a *currentFakeControlAgent) SendControl(_ context.Context, spec CurrentControlSpec) error {
|
|
if a.before != nil {
|
|
a.before(spec)
|
|
}
|
|
a.calls = append(a.calls, spec)
|
|
return a.err
|
|
}
|
|
|
|
func newCurrentControlFixture(t *testing.T) (*CurrentControlController, *currentFakeControlAgent, *store.CurrentStore) {
|
|
t.Helper()
|
|
id := "c046b893-8628-4589-ae50-619d049248a6"
|
|
snapshot := currentPolicySnapshot(t)
|
|
approvedSIP, err := json.Marshal(snapshot.SIP)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
var body []byte
|
|
switch r.URL.Path {
|
|
case "/internal/v1/dispatcher/sip":
|
|
body = approvedSIP
|
|
case "/internal/v1/dispatcher/ai-providers":
|
|
body = currentConfigExample(t, "config-read-providers")
|
|
case "/internal/v1/dispatcher/task/task-asr":
|
|
body = currentConfigExample(t, "config-read-task-asr")
|
|
case "/internal/v1/dispatcher/tenant/1001/quota":
|
|
body = currentConfigExample(t, "config-read-quota")
|
|
default:
|
|
t.Errorf("unexpected HTTP path %s", r.URL.Path)
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
_, _ = w.Write(body)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
client, err := configread.NewClient(server.URL, id, "test-secret", server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := store.OpenCurrent(filepath.Join(t.TempDir(), "state.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = s.Close() })
|
|
if err := s.ApplyDiscoverySnapshot(id, []configread.CurrentDiscoveredTask{{TaskID: "task-asr", TenantID: 1001, TaskRevision: 1, Status: "running"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.SaveSnapshot(snapshot); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.MarkReadyForSIP(id, snapshot.SIP.Revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
agent := ¤tFakeControlAgent{}
|
|
controller := &CurrentControlController{DispatcherID: id, Store: s, Client: client, Agent: agent, VerifySIP: func(context.Context, configread.CurrentSIP) error { return nil }, Now: func() time.Time { return currentMonday(9, 30) }}
|
|
return controller, agent, s
|
|
}
|
|
|
|
func currentControlBody(t *testing.T, eventID, action, policy string) []byte {
|
|
t.Helper()
|
|
body := string(currentConfigExample(t, "mq-control"))
|
|
body = strings.Replace(body, `"event_id":"control-example"`, `"event_id":"`+eventID+`"`, 1)
|
|
body = strings.Replace(body, `"action":"pause"`, `"action":"`+action+`"`, 1)
|
|
body = strings.Replace(body, `"active_call_policy":"drain"`, `"active_call_policy":"`+policy+`"`, 1)
|
|
if policy == "" {
|
|
body = strings.Replace(body, `,"options":{"active_call_policy":""}`, ``, 1)
|
|
}
|
|
return []byte(body)
|
|
}
|
|
|
|
func TestCurrentControlPauseDefaultHangupAckAfterAgentDispatch(t *testing.T) {
|
|
controller, agent, s := newCurrentControlFixture(t)
|
|
agent.before = func(spec CurrentControlSpec) {
|
|
if spec.Action != "pause" || spec.ActiveCallPolicy != "hangup" {
|
|
t.Errorf("wrong default control policy: %+v", spec)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Errorf("task not blocked before Agent control: %v %v", admitted, err)
|
|
}
|
|
if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 {
|
|
t.Errorf("ack written before Agent dispatch: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
body := currentControlBody(t, "pause-1", "pause", "")
|
|
if err := controller.ProcessControl(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("control was not dispatched: %+v", agent.calls)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) {
|
|
t.Fatalf("no durable applied ack: %+v %v", outbox, err)
|
|
}
|
|
if err := s.MarkOutboxConfirmed(controller.DispatcherID, "pause-1"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
agent.before = nil
|
|
if err := controller.ProcessControl(context.Background(), body); err != nil || len(agent.calls) != 2 {
|
|
t.Fatalf("control was deduplicated or failed: %d %v", len(agent.calls), err)
|
|
}
|
|
outbox, err = s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 1 || outbox[0].EventID != "pause-1" {
|
|
t.Fatalf("repeat control did not requeue original ack: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestCurrentControlStopCannotResumeOrDispatchPending(t *testing.T) {
|
|
controller, agent, s := newCurrentControlFixture(t)
|
|
command := store.CurrentExecuteCommand{DispatcherID: controller.DispatcherID, EventID: "queued-1", TenantID: 1001, TaskID: "task-asr", Callee: "15003164745", IssuedAt: "2026-09-21T01:00:00Z"}
|
|
if _, _, err := s.RecordExecute(command); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "stop-1", "stop", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 || agent.calls[0].ActiveCallPolicy != "hangup" {
|
|
t.Fatalf("stop did not dispatch hangup: %+v", agent.calls)
|
|
}
|
|
if pending, err := s.ListPendingExecute(controller.DispatcherID); err != nil || len(pending) != 0 {
|
|
t.Fatalf("old pending call survived stop: %+v %v", pending, err)
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "resume-1", "resume", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("stopped task sent resume: %+v", agent.calls)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 2 || !strings.Contains(string(outbox[1].Body), `"status":"rejected"`) {
|
|
t.Fatalf("stop/resume result wrong: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestCurrentStoppedTaskSilentlyAcknowledgesOldExecuteWithoutRecreatingInbox(t *testing.T) {
|
|
control, _, s := newCurrentControlFixture(t)
|
|
if err := control.ProcessControl(context.Background(), currentControlBody(t, "stop-before-execute", "stop", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originator := ¤tFakeOriginator{loaded: map[string]int64{"trunk-mock": 8}}
|
|
execute := &CurrentExecuteController{DispatcherID: control.DispatcherID, Store: s, Originator: originator, Publisher: ¤tFakePublisher{}, Now: func() time.Time { return currentMonday(9, 30) }}
|
|
if err := execute.ProcessExecute(context.Background(), currentExecuteBody(t, "old-queued-1", "15003164745")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(originator.calls) != 0 {
|
|
t.Fatal("stopped task originated an old queued command")
|
|
}
|
|
if pending, err := s.ListPendingExecute(control.DispatcherID); err != nil || len(pending) != 0 {
|
|
t.Fatalf("silently acknowledged old command was persisted as dispatchable: %+v %v", pending, err)
|
|
}
|
|
if outbox, err := s.ListPendingOutbox(control.DispatcherID); err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" {
|
|
t.Fatalf("old unadmitted command emitted business result: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestCurrentResumeRequiresFreshTaskAndAppliedSIP(t *testing.T) {
|
|
controller, agent, s := newCurrentControlFixture(t)
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "pause-before-resume", "pause", "drain")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
approved := controller.VerifySIP
|
|
controller.VerifySIP = func(context.Context, configread.CurrentSIP) error {
|
|
return errors.New("SIP revision not yet applied by Asterisk")
|
|
}
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "resume-after-load", "resume", "")); err == nil {
|
|
t.Fatal("resumed task without applied SIP")
|
|
}
|
|
if len(agent.calls) != 1 {
|
|
t.Fatalf("unverified resume reached Agent: %+v", agent.calls)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Fatalf("failed SIP check reopened admission: %v %v", admitted, err)
|
|
}
|
|
controller.VerifySIP = approved
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "resume-after-load", "resume", "")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(agent.calls) != 2 || agent.calls[1].Action != "resume" || agent.calls[1].ActiveCallPolicy != "" {
|
|
t.Fatalf("valid fresh resume did not dispatch: %+v", agent.calls)
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || !admitted {
|
|
t.Fatalf("verified resume did not reopen admission: %v %v", admitted, err)
|
|
}
|
|
outbox, err := s.ListPendingOutbox(controller.DispatcherID)
|
|
if err != nil || len(outbox) != 2 || outbox[1].EventID != "resume-after-load" {
|
|
t.Fatalf("applied resume has no durable acknowledgment: %+v %v", outbox, err)
|
|
}
|
|
}
|
|
|
|
func TestCurrentRuntimeSurfacesControlFailureToCloseAdmission(t *testing.T) {
|
|
controller, agent, s := newCurrentControlFixture(t)
|
|
agent.err = errors.New("injected Agent control failure")
|
|
runtime := &CurrentRuntime{Bootstrap: CurrentBootstrap{DispatcherID: controller.DispatcherID, Store: s}, Control: *controller, Logger: slog.Default(), failures: make(chan error, 1), taskLocks: make(map[string]*sync.Mutex)}
|
|
if err := runtime.handleControl(context.Background(), "", currentControlBody(t, "fatal-control", "pause", "")); err == nil {
|
|
t.Fatal("Agent control failure was hidden")
|
|
}
|
|
select {
|
|
case err := <-runtime.failures:
|
|
if err == nil {
|
|
t.Fatal("control failure signal was empty")
|
|
}
|
|
default:
|
|
t.Fatal("consumer will requeue failed control indefinitely without closing admission")
|
|
}
|
|
}
|
|
|
|
func TestCurrentControlAgentFailureKeepsBarrierWithoutSuccessAck(t *testing.T) {
|
|
controller, agent, s := newCurrentControlFixture(t)
|
|
agent.err = errors.New("injected Agent control RPC timeout")
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "stop-timeout", "stop", "drain")); err == nil {
|
|
t.Fatal("Agent control failure was hidden")
|
|
}
|
|
if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted {
|
|
t.Fatalf("control failure reopened task admission: %v %v", admitted, err)
|
|
}
|
|
if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 {
|
|
t.Fatalf("false applied ack after Agent failure: %+v %v", outbox, err)
|
|
}
|
|
agent.err = nil
|
|
if err := controller.ProcessControl(context.Background(), currentControlBody(t, "stop-timeout", "stop", "drain")); err != nil || len(agent.calls) != 2 {
|
|
t.Fatalf("redelivered control not dispatched: %+v %v", agent.calls, err)
|
|
}
|
|
}
|