From c350c483a57cb2feb266806b10753844ce059ba8 Mon Sep 17 00:00:00 2001 From: Rogee Date: Tue, 29 Sep 2026 19:52:46 +0800 Subject: [PATCH] Implement durable MQ controls and isolated Dispatcher runtime --- .../saas-dispatcher-implementation.md | 51 ++ internal/dispatcher/current_control.go | 128 +++++ internal/dispatcher/current_control_test.go | 249 +++++++++ internal/dispatcher/current_discovery_test.go | 87 +++ internal/dispatcher/current_execute.go | 233 ++++++++ internal/dispatcher/current_execute_test.go | 176 ++++++ internal/dispatcher/current_gate_test.go | 64 +++ internal/dispatcher/current_runtime.go | 323 +++++++++++ .../current_runtime_integration_test.go | 380 +++++++++++++ internal/dispatcher/current_sip_reload.go | 127 +++++ .../dispatcher/current_sip_runtime_test.go | 91 +++ internal/mq/current.go | 516 ++++++++++++++++++ internal/mq/current_inbound_test.go | 58 ++ internal/mq/current_integration_test.go | 203 +++++++ internal/mq/current_pause_test.go | 45 ++ internal/mq/current_test.go | 59 ++ internal/store/current.go | 28 + internal/store/current_calls.go | 402 ++++++++++++++ internal/store/current_calls_test.go | 162 ++++++ internal/store/current_control_ack_test.go | 101 ++++ internal/store/current_control_flow.go | 177 ++++++ internal/store/current_control_test.go | 49 ++ internal/store/current_discovery_test.go | 128 +++++ internal/store/current_pending_test.go | 26 + internal/store/current_sip_change_test.go | 94 ++++ internal/store/current_test.go | 31 ++ scripts/check-current-mq-mock.sh | 55 ++ 27 files changed, 4043 insertions(+) create mode 100644 docs/evidence/saas-dispatcher-implementation.md create mode 100644 internal/dispatcher/current_control.go create mode 100644 internal/dispatcher/current_control_test.go create mode 100644 internal/dispatcher/current_discovery_test.go create mode 100644 internal/dispatcher/current_execute.go create mode 100644 internal/dispatcher/current_execute_test.go create mode 100644 internal/dispatcher/current_gate_test.go create mode 100644 internal/dispatcher/current_runtime.go create mode 100644 internal/dispatcher/current_runtime_integration_test.go create mode 100644 internal/dispatcher/current_sip_reload.go create mode 100644 internal/dispatcher/current_sip_runtime_test.go create mode 100644 internal/mq/current.go create mode 100644 internal/mq/current_inbound_test.go create mode 100644 internal/mq/current_integration_test.go create mode 100644 internal/mq/current_pause_test.go create mode 100644 internal/mq/current_test.go create mode 100644 internal/store/current_calls.go create mode 100644 internal/store/current_calls_test.go create mode 100644 internal/store/current_control_ack_test.go create mode 100644 internal/store/current_control_flow.go create mode 100644 internal/store/current_control_test.go create mode 100644 internal/store/current_discovery_test.go create mode 100644 internal/store/current_pending_test.go create mode 100644 internal/store/current_sip_change_test.go create mode 100644 scripts/check-current-mq-mock.sh diff --git a/docs/evidence/saas-dispatcher-implementation.md b/docs/evidence/saas-dispatcher-implementation.md new file mode 100644 index 0000000..c622506 --- /dev/null +++ b/docs/evidence/saas-dispatcher-implementation.md @@ -0,0 +1,51 @@ +# SaaS↔Dispatcher 项目内实施证据 + +## 改动前基线 + +- 基线提交：`f5c2d6a92036a579e0b070beb1d9381b0976f81e`；执行分支：`feat/saas-dispatcher-contract`；开始前已拉取并核对 `origin/main`，无待同步提交；Go 1.27.1。 +- 原工作树已有用户修改：`docs/thirds/v0.4.md`、`docs/thirds/第三方对接事件与请求消费顺序_v0.1.md`；未跟踪：`docs/plan-saas-dispatcher-v05-v0.1.md`、`docs/thirds/v0.5-proposal.md`、`docs/.plan-saas-dispatcher-v05-v0.1.md.swp`。这些内容不得作为已提交基线，不能静默删除、暂存或覆盖；历史来源在改名时逐一核验。 +- 全仓路径初筛发现 182 个代次式候选文件（包括历史证据/第三方合法版本，**不是** 182 个可删除目标）；源码、契约与脚本约 119 个候选。去后缀会发生实存碰撞：`internal/mq/amqp_v3.go`→`internal/mq/amqp.go`；历史证据文件也有碰撞，历史证据不能按文件名自动改写。 +- 尚未验收源码、契约、Proto 及数据兼容；此处只记录实施前状态，不构成 P01–P08 通过证明。 + +## 路径/符号映射与冲突处置（实施时逐项勾稽） + +| 旧路径/符号 | 目标职责与处置 | +| --- | --- | +| `internal/mq/amqp.go` 的旧声明/租户队列 Broker 与 `internal/mq/amqp_v3.go` 的预建队列 Broker | 保留预建队列单一路径，合并必要发布/错误处理后归入 `internal/mq/amqp.go` 的 `Broker`；删除旧运行入口，不能直接覆盖。 | +| `internal/configread/discovery_v04.go` | `internal/configread/discovery.go`，重做新 cursor 语义，删除旧 snapshot/watermark 路径。 | +| `internal/dispatcher/task_queue_v3.go` / `task_control_v3.go` / `task_runtime_v3.go` | `task_queue.go` / `task_control.go` / `task_runtime.go`，类型按职责命名，不保留 V3/LocalV01 别名。 | +| `internal/dispatcher/call_result_v01.go` / `local_v01.go` | `call_result.go` / `local.go`；只留已确认的对外事件。 | +| `proto/agent/v1/agent.proto` / `gen/agent/v1/` | `proto/agent/agent.proto` / `gen/agent/`，同步生成配置、Go 导入与服务名，重新生成而非手改生成物。 | +| `contracts/local/v0.x/`、`docs/contracts/*-v0.x*` 与按代次加载/校验器 | 各领域只有一份无代次当前 Schema、示例及来源/hash；历史来源只能归档，不参与运行校验。 | +| `internal/store/migrations/*_local_v0x_*.sql` | 保留有意义的迁移序号并更换实现代次名；先验证现存数据库的迁移跟踪/未交付状态，不能以改名重跑破坏数据。 | +| `docs/plan-saas-dispatcher-v05-v0.1.md` / `docs/thirds/v0.5-proposal.md` | 分别成为 `docs/plan-saas-dispatcher.md` / `docs/thirds/saas-dispatcher.md`；既有未跟踪内容须先核对，链接/哈希同步更新。 | + +## P01：项目内当前通信契约 + +- 新增 `contracts/local/config-read.schema.json`、`task-discovery.schema.json`、`mq.schema.json`、`mq-topology.json` 与 30 份正反例，manifest 覆盖 34 个当前 JSON 文件；`contracts.ReadCurrent` 只接受这五个当前文件，无旧目录回退。 +- `docs/thirds/saas-dispatcher.md` 记录五类 HTTP、入站/出站事件、队列归属/共享结果队列及 K01–K16 的业务边界；数字 tenant_id、原值 credential、ASR-only、revision、空录音、真实 outcome/reason_code、48h 失败留待人工等均有字段/行为约定。机器校验拒绝字符串租户、旧 schema_version/credential_ref/call_id、未知 recording 状态、非法控制 policy。 +- 测试：`bash scripts/check-current-contracts.sh`、`go test ./internal/contract ./contracts -count=1`、`git diff --check` 均通过；当前正例未出现旧配置或通话身份字段。来源/内容 SHA-256 在 manifest 中可重算。**这是当前合同文件的本地校验，不代表 P03–P08 运行行为、真实 SaaS 或旧合同清理通过。** + +## P02：Proto/生成物分批改造（未整体签收） + +- `proto/agent/v1/agent.proto`→`proto/agent/agent.proto`，包 `agent`；`gen/agent/v1/`→`gen/agent/`，源码 79 处导入/标识机械更新；旧生成路径已删除。已有 `RequestMeta.protocol_version=agent.v1` 是**有意义的线协议元数据**，没有为了去名改动字段或重编号。 +- 原 Buf STANDARD 的 `PACKAGE_VERSION_SUFFIX` 与已批准的无代次内部包名冲突；`buf.yaml` 仅对这一条规则作例外，其余 STANDARD 规则保持。隔离安装 Buf v1.50.0 / protoc-gen-go v1.36.12 / protoc-gen-go-grpc v1.5.1 于 `/tmp/sip-go-agent-tools/bin`，未改应用依赖。 +- `PATH=/tmp/sip-go-agent-tools/bin:$PATH sh scripts/check-proto.sh` 通过（lint/build/generate/新包测试/7 文件清单 hash）；`git diff --check`、`go test ./internal/rpc ./internal/agent ./internal/dispatcher ./cmd/sip-go-agent -count=1` 通过。其他旧实现/契约入口、文件名碰撞和迁移数据安全仍待 P02/P07，**不能据此称全仓已无代次或整体完成**。 + +## P03：HTTP 读取分批改造（未整体签收） + +- `contract.ValidateCurrent` 与 `configread` 按当前 Schema 读取 SIP、provider、task、quota 和 cursor 任务发现；严格检查数字 tenant_id、本 D 归属及不可变配置。provider 凭据原值只保留在内存快照，不写日志；Agent 参数中的显式 0/false 保真；无旧 Schema/旧配置回退。 +- `store.OpenCurrent` 新建数字租户 SQLite 状态；旧表、旧版当前布局、残缺布局均在写入前拒绝并保留原记录；不执行旧数据迁移或自动清理。启动时完整发现同一快照一次提交，分页增量逐页持久提交后才推进**内存** cursor；失败关闭准入，重启重新取完整快照。HTTP 的旧 running 不能解除 MQ 暂停/终止，同 revision 异内容及跨任务 SIP/租户额度冲突拒绝。 +- `CurrentBootstrap` 先关闭准入，核验 SIP 全量与 Agent/Asterisk 已加载 revision、读取任务和 provider/额度，再排空 MQ 控制积压，最后依据已验证 SIP revision 开准入；有更新的持久 SIP 通知时保持关闭但控制与结果处理仍可继续。`CurrentDiscoveryFollower` 逐页绑定任务快照；HTTP 错误、失效或授权不一致只失败，不回退旧读取。**目前只在隔离运行组件中调用，尚未接入 `cmd/sip-go-agent/main.go`；provider 向真实 Agent 交付及真实加载尚待 P05/P07。** +- TDD 与回归：`go test ./internal/configread ./internal/tenant ./internal/store ./internal/dispatcher -count=1`、`bash scripts/check-current-contracts.sh`、已提交 `a0118e3` 的干净归档测试通过；旧布局行/表原样保留由 `TestCurrentStoreRefusesPreviousCurrentLayoutBeforeModifyingDatabase` 覆盖。 + +## P04：隔离 MQ、控制与外呼接纳（仅项目内 Mock） + +- 固定 `v1` 精确路由与 SaaS 共享结果队列；各 D 控制/任务队列均由 SaaS 预建，D 仅被动检查和消费。隔离 RabbitMQ 实测无 `configure` 权限、D1/D2 不串收、shared queue 实际收讫、断绑后 mandatory 失败不算交付；畸形控制积压拒绝并阻止启动准入。 +- 执行消息先写持久 inbox 才 ACK；白名单号码格式错误单条拒绝，不暂停其它任务；任务/线路规则不满足时原执行身份留在 SQLite、该任务后续积压留在 SaaS 队列，窗口开放后重新核验才发出 Mock 指令。发指令前两次时窗/SIP/准入检查与持久额度占用；未知 Agent RPC 保持未知占用、不自动重拨。结果 outbox 只在 mandatory/return/confirm 成功后标记已入队，不宣称 SaaS 已处理。 +- pause/stop/resume 控制先持久挡住该任务新呼叫，Agent 确认收到指令后在同一事务提交应用状态和回执；省略策略默认 hangup，重复控制仍执行、重复回执复用事件身份，stopped 同 ID 不可恢复，未接纳旧外呼静默 ACK。SIP 通知先持久关闭准入，旧/未知通话未确认终结、SaaS 新版尚未分发或 Agent/Asterisk 未加载时不重开；等待期间控制和 outbox 仍可处理。 +- TDD 与隔离验证：`go test ./internal/store ./internal/dispatcher ./internal/mq -count=1`、`bash scripts/check-current-mq-mock.sh`、`git diff --check` 通过；Mock 覆盖任务积压、恢复、控制、重复投递、发布失败、SIP revision 栅栏与格式错误。当前 Agent side effect 为注入的**假外呼**，主 CLI 仍旧；真实 SaaS/RabbitMQ、供应商、线路、录音与 `call.result` 均**未验收**，分别留 P05–P08。 + +## 验收台账 + +A01–A12 的行为验证及 K01–K16 的运行时验证待 P03–P08 逐项填充；不得用本地 Mock 冒充外部签收。 diff --git a/internal/dispatcher/current_control.go b/internal/dispatcher/current_control.go new file mode 100644 index 0000000..e2adf66 --- /dev/null +++ b/internal/dispatcher/current_control.go @@ -0,0 +1,128 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/contract" + "git.ipao.vip/rogee/go-sip/internal/store" +) + +// CurrentControlSpec is an Agent control instruction, not evidence that a +// drain/hangup has completed. Never log a user-supplied reason or credentials. +type CurrentControlSpec struct { + DispatcherID string + TenantID int64 + TaskID string + Action string + ActiveCallPolicy string + Reason string +} + +type CurrentControlAgent interface { + SendControl(context.Context, CurrentControlSpec) error +} + +type CurrentControlController struct { + DispatcherID string + Store *store.CurrentStore + Client *configread.Client + Agent CurrentControlAgent + VerifySIP func(context.Context, configread.CurrentSIP) error + Now func() time.Time +} + +// ProcessControl applies each delivered control independently: it has no +// command_id, expected revision, or control-message deduplication. The state +// barrier is durable before Agent dispatch; applied state and MQ outbox are +// committed atomically after the Agent accepts the instruction. +func (c *CurrentControlController) ProcessControl(ctx context.Context, body []byte) error { + if c == nil || c.DispatcherID == "" || c.Store == nil || c.Client == nil || c.Agent == nil || c.VerifySIP == nil || c.Now == nil { + return errors.New("control processing requires Dispatcher, durable store, HTTP client, Agent, SIP verifier, and clock") + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return fmt.Errorf("invalid incoming task.control: %w", err) + } + var event struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + DispatcherID string `json:"dispatcher_id"` + TenantID int64 `json:"tenant_id"` + IssuedAt string `json:"issued_at"` + Payload struct { + TaskID string `json:"task_id"` + Action string `json:"action"` + Reason string `json:"reason"` + Options struct { + ActiveCallPolicy string `json:"active_call_policy"` + } `json:"options"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &event); err != nil { + return fmt.Errorf("decode task.control: %w", err) + } + if event.EventType != "task.control" || event.DispatcherID != c.DispatcherID || event.Payload.TaskID == "" || event.Payload.Action == "" { + return errors.New("control event type, task, or Dispatcher owner mismatch") + } + issuedAt, err := time.Parse(time.RFC3339Nano, event.IssuedAt) + if err != nil { + return fmt.Errorf("invalid task.control issued_at: %w", err) + } + if issuedAt.After(c.Now()) { + return fmt.Errorf("control event %q issued_at is in the future; do not apply early", event.EventID) + } + policy := event.Payload.Options.ActiveCallPolicy + if event.Payload.Action == "resume" { + if policy != "" { + return errors.New("resume control must not carry an active-call policy") + } + // A stale HTTP running status cannot override persisted pause/stop. + // Only a fresh approved task plus applied SIP snapshot may authorize + // the transition from a durable pause back to admission. + snapshot, err := c.Client.ReadCurrentTask(ctx, event.Payload.TaskID, event.TenantID) + if err != nil { + return fmt.Errorf("fresh resume task configuration: %w", err) + } + if snapshot.Task.Status != "running" { + return errors.New("fresh resume task is not running") + } + if err := c.VerifySIP(ctx, snapshot.SIP); err != nil { + return fmt.Errorf("resume SIP revision not applied: %w", err) + } + if err := c.Store.SaveSnapshot(snapshot); err != nil { + return fmt.Errorf("bind fresh resume task: %w", err) + } + } else { + if policy == "" { + policy = "hangup" + } + if policy != "hangup" && policy != "drain" { + return errors.New("unapproved active-call policy") + } + } + if err := c.Store.PrepareControl(event.DispatcherID, event.TenantID, event.Payload.TaskID, event.Payload.Action); err != nil { + if errors.Is(err, store.ErrCurrentControlRejected) { + if rejectErr := c.Store.RejectControl(event.DispatcherID, event.TenantID, event.EventID); rejectErr != nil { + return fmt.Errorf("persist rejected control %q: %w", event.EventID, rejectErr) + } + return nil + } + return fmt.Errorf("prepare task control %q: %w", event.EventID, err) + } + spec := CurrentControlSpec{ + DispatcherID: event.DispatcherID, TenantID: event.TenantID, + TaskID: event.Payload.TaskID, Action: event.Payload.Action, + ActiveCallPolicy: policy, Reason: event.Payload.Reason, + } + if err := c.Agent.SendControl(ctx, spec); err != nil { + return fmt.Errorf("Agent control %q dispatch failed: %w", event.EventID, err) + } + if err := c.Store.CompleteControl(event.DispatcherID, event.TenantID, event.Payload.TaskID, event.Payload.Action, event.EventID); err != nil { + return fmt.Errorf("Agent control %q dispatched but acknowledgment not durable: %w", event.EventID, err) + } + return nil +} diff --git a/internal/dispatcher/current_control_test.go b/internal/dispatcher/current_control_test.go new file mode 100644 index 0000000..2f1063e --- /dev/null +++ b/internal/dispatcher/current_control_test.go @@ -0,0 +1,249 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/store" +) + +type currentFakeControlAgent struct { + calls []CurrentControlSpec + err error + before func(CurrentControlSpec) +} + +func (a *currentFakeControlAgent) SendControl(_ context.Context, spec CurrentControlSpec) error { + if a.before != nil { + a.before(spec) + } + a.calls = append(a.calls, spec) + return a.err +} + +func newCurrentControlFixture(t *testing.T) (*CurrentControlController, *currentFakeControlAgent, *store.CurrentStore) { + t.Helper() + id := "c046b893-8628-4589-ae50-619d049248a6" + snapshot := currentPolicySnapshot(t) + approvedSIP, err := json.Marshal(snapshot.SIP) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var body []byte + switch r.URL.Path { + case "/internal/v1/dispatcher/sip": + body = approvedSIP + case "/internal/v1/dispatcher/ai-providers": + body = currentConfigExample(t, "config-read-providers") + case "/internal/v1/dispatcher/task/task-asr": + body = currentConfigExample(t, "config-read-task-asr") + case "/internal/v1/dispatcher/tenant/1001/quota": + body = currentConfigExample(t, "config-read-quota") + default: + t.Errorf("unexpected HTTP path %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + client, err := configread.NewClient(server.URL, id, "test-secret", server.Client()) + if err != nil { + t.Fatal(err) + } + s, err := store.OpenCurrent(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + if err := s.ApplyDiscoverySnapshot(id, []configread.CurrentDiscoveredTask{{TaskID: "task-asr", TenantID: 1001, TaskRevision: 1, Status: "running"}}); err != nil { + t.Fatal(err) + } + if err := s.SaveSnapshot(snapshot); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(id, snapshot.SIP.Revision); err != nil { + t.Fatal(err) + } + agent := ¤tFakeControlAgent{} + controller := &CurrentControlController{DispatcherID: id, Store: s, Client: client, Agent: agent, VerifySIP: func(context.Context, configread.CurrentSIP) error { return nil }, Now: func() time.Time { return currentMonday(9, 30) }} + return controller, agent, s +} + +func currentControlBody(t *testing.T, eventID, action, policy string) []byte { + t.Helper() + body := string(currentConfigExample(t, "mq-control")) + body = strings.Replace(body, `"event_id":"control-example"`, `"event_id":"`+eventID+`"`, 1) + body = strings.Replace(body, `"action":"pause"`, `"action":"`+action+`"`, 1) + body = strings.Replace(body, `"active_call_policy":"drain"`, `"active_call_policy":"`+policy+`"`, 1) + if policy == "" { + body = strings.Replace(body, `,"options":{"active_call_policy":""}`, ``, 1) + } + return []byte(body) +} + +func TestCurrentControlPauseDefaultHangupAckAfterAgentDispatch(t *testing.T) { + controller, agent, s := newCurrentControlFixture(t) + agent.before = func(spec CurrentControlSpec) { + if spec.Action != "pause" || spec.ActiveCallPolicy != "hangup" { + t.Errorf("wrong default control policy: %+v", spec) + } + if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Errorf("task not blocked before Agent control: %v %v", admitted, err) + } + if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 { + t.Errorf("ack written before Agent dispatch: %+v %v", outbox, err) + } + } + body := currentControlBody(t, "pause-1", "pause", "") + if err := controller.ProcessControl(context.Background(), body); err != nil { + t.Fatal(err) + } + if len(agent.calls) != 1 { + t.Fatalf("control was not dispatched: %+v", agent.calls) + } + outbox, err := s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) { + t.Fatalf("no durable applied ack: %+v %v", outbox, err) + } + if err := s.MarkOutboxConfirmed(controller.DispatcherID, "pause-1"); err != nil { + t.Fatal(err) + } + agent.before = nil + if err := controller.ProcessControl(context.Background(), body); err != nil || len(agent.calls) != 2 { + t.Fatalf("control was deduplicated or failed: %d %v", len(agent.calls), err) + } + outbox, err = s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(outbox) != 1 || outbox[0].EventID != "pause-1" { + t.Fatalf("repeat control did not requeue original ack: %+v %v", outbox, err) + } +} + +func TestCurrentControlStopCannotResumeOrDispatchPending(t *testing.T) { + controller, agent, s := newCurrentControlFixture(t) + command := store.CurrentExecuteCommand{DispatcherID: controller.DispatcherID, EventID: "queued-1", TenantID: 1001, TaskID: "task-asr", Callee: "15003164745", IssuedAt: "2026-09-21T01:00:00Z"} + if _, _, err := s.RecordExecute(command); err != nil { + t.Fatal(err) + } + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "stop-1", "stop", "")); err != nil { + t.Fatal(err) + } + if len(agent.calls) != 1 || agent.calls[0].ActiveCallPolicy != "hangup" { + t.Fatalf("stop did not dispatch hangup: %+v", agent.calls) + } + if pending, err := s.ListPendingExecute(controller.DispatcherID); err != nil || len(pending) != 0 { + t.Fatalf("old pending call survived stop: %+v %v", pending, err) + } + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "resume-1", "resume", "")); err != nil { + t.Fatal(err) + } + if len(agent.calls) != 1 { + t.Fatalf("stopped task sent resume: %+v", agent.calls) + } + outbox, err := s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(outbox) != 2 || !strings.Contains(string(outbox[1].Body), `"status":"rejected"`) { + t.Fatalf("stop/resume result wrong: %+v %v", outbox, err) + } +} + +func TestCurrentStoppedTaskSilentlyAcknowledgesOldExecuteWithoutRecreatingInbox(t *testing.T) { + control, _, s := newCurrentControlFixture(t) + if err := control.ProcessControl(context.Background(), currentControlBody(t, "stop-before-execute", "stop", "")); err != nil { + t.Fatal(err) + } + originator := ¤tFakeOriginator{loaded: map[string]int64{"trunk-mock": 8}} + execute := &CurrentExecuteController{DispatcherID: control.DispatcherID, Store: s, Originator: originator, Publisher: ¤tFakePublisher{}, Now: func() time.Time { return currentMonday(9, 30) }} + if err := execute.ProcessExecute(context.Background(), currentExecuteBody(t, "old-queued-1", "15003164745")); err != nil { + t.Fatal(err) + } + if len(originator.calls) != 0 { + t.Fatal("stopped task originated an old queued command") + } + if pending, err := s.ListPendingExecute(control.DispatcherID); err != nil || len(pending) != 0 { + t.Fatalf("silently acknowledged old command was persisted as dispatchable: %+v %v", pending, err) + } + if outbox, err := s.ListPendingOutbox(control.DispatcherID); err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" { + t.Fatalf("old unadmitted command emitted business result: %+v %v", outbox, err) + } +} + +func TestCurrentResumeRequiresFreshTaskAndAppliedSIP(t *testing.T) { + controller, agent, s := newCurrentControlFixture(t) + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "pause-before-resume", "pause", "drain")); err != nil { + t.Fatal(err) + } + approved := controller.VerifySIP + controller.VerifySIP = func(context.Context, configread.CurrentSIP) error { + return errors.New("SIP revision not yet applied by Asterisk") + } + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "resume-after-load", "resume", "")); err == nil { + t.Fatal("resumed task without applied SIP") + } + if len(agent.calls) != 1 { + t.Fatalf("unverified resume reached Agent: %+v", agent.calls) + } + if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("failed SIP check reopened admission: %v %v", admitted, err) + } + controller.VerifySIP = approved + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "resume-after-load", "resume", "")); err != nil { + t.Fatal(err) + } + if len(agent.calls) != 2 || agent.calls[1].Action != "resume" || agent.calls[1].ActiveCallPolicy != "" { + t.Fatalf("valid fresh resume did not dispatch: %+v", agent.calls) + } + if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || !admitted { + t.Fatalf("verified resume did not reopen admission: %v %v", admitted, err) + } + outbox, err := s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(outbox) != 2 || outbox[1].EventID != "resume-after-load" { + t.Fatalf("applied resume has no durable acknowledgment: %+v %v", outbox, err) + } +} + +func TestCurrentRuntimeSurfacesControlFailureToCloseAdmission(t *testing.T) { + controller, agent, s := newCurrentControlFixture(t) + agent.err = errors.New("injected Agent control failure") + runtime := &CurrentRuntime{Bootstrap: CurrentBootstrap{DispatcherID: controller.DispatcherID, Store: s}, Control: *controller, Logger: slog.Default(), failures: make(chan error, 1), taskLocks: make(map[string]*sync.Mutex)} + if err := runtime.handleControl(context.Background(), "", currentControlBody(t, "fatal-control", "pause", "")); err == nil { + t.Fatal("Agent control failure was hidden") + } + select { + case err := <-runtime.failures: + if err == nil { + t.Fatal("control failure signal was empty") + } + default: + t.Fatal("consumer will requeue failed control indefinitely without closing admission") + } +} + +func TestCurrentControlAgentFailureKeepsBarrierWithoutSuccessAck(t *testing.T) { + controller, agent, s := newCurrentControlFixture(t) + agent.err = errors.New("injected Agent control RPC timeout") + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "stop-timeout", "stop", "drain")); err == nil { + t.Fatal("Agent control failure was hidden") + } + if admitted, err := s.CanAdmit(controller.DispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("control failure reopened task admission: %v %v", admitted, err) + } + if outbox, err := s.ListPendingOutbox(controller.DispatcherID); err != nil || len(outbox) != 0 { + t.Fatalf("false applied ack after Agent failure: %+v %v", outbox, err) + } + agent.err = nil + if err := controller.ProcessControl(context.Background(), currentControlBody(t, "stop-timeout", "stop", "drain")); err != nil || len(agent.calls) != 2 { + t.Fatalf("redelivered control not dispatched: %+v %v", agent.calls, err) + } +} diff --git a/internal/dispatcher/current_discovery_test.go b/internal/dispatcher/current_discovery_test.go new file mode 100644 index 0000000..666a387 --- /dev/null +++ b/internal/dispatcher/current_discovery_test.go @@ -0,0 +1,87 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "git.ipao.vip/rogee/go-sip/internal/configread" +) + +func newCurrentFollowerFixture(t *testing.T, revision int64) (*CurrentDiscoveryFollower, func() bool) { + t.Helper() + executor, _, _, s := newCurrentExecuteFixture(t) + approved := currentPolicySnapshot(t).SIP + sipBody, err := json.Marshal(approved) + if err != nil { + t.Fatal(err) + } + terminalSeen := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var body []byte + switch r.URL.Path { + case "/internal/v1/dispatcher/tasks": + switch r.URL.Query().Get("after") { + case "initial": + body = []byte(fmt.Sprintf(`{"dispatcher_id":%q,"cursor":"middle","tasks":[{"task_id":"task-asr","tenant_id":1001,"status":"running","task_revision":%d}]}`, executor.DispatcherID, revision)) + case "middle": + terminalSeen = true + body = []byte(fmt.Sprintf(`{"dispatcher_id":%q,"cursor":"opaque-tail","tasks":[]}`, executor.DispatcherID)) + default: + t.Errorf("unexpected discovery cursor: %s", r.URL.RawQuery) + w.WriteHeader(http.StatusBadRequest) + return + } + case "/internal/v1/dispatcher/sip": + body = sipBody + case "/internal/v1/dispatcher/ai-providers": + body = currentConfigExample(t, "config-read-providers") + case "/internal/v1/dispatcher/task/task-asr": + body = currentConfigExample(t, "config-read-task-asr") + case "/internal/v1/dispatcher/tenant/1001/quota": + body = currentConfigExample(t, "config-read-quota") + default: + t.Errorf("unexpected config path: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + client, err := configread.NewClient(server.URL, executor.DispatcherID, "test-secret", server.Client()) + if err != nil { + t.Fatal(err) + } + follower := &CurrentDiscoveryFollower{DispatcherID: executor.DispatcherID, Client: client, Store: s, ApprovedSIP: approved, VerifySIP: func(context.Context, configread.CurrentSIP) error { return nil }, Cursor: "initial"} + return follower, func() bool { return terminalSeen } +} + +func TestCurrentDiscoveryFollowerPersistsPageBeforeAdvancingToTerminalEmpty(t *testing.T) { + follower, terminal := newCurrentFollowerFixture(t, 1) + if err := follower.Poll(context.Background()); err != nil { + t.Fatal(err) + } + if follower.Cursor != "opaque-tail" || !terminal() { + t.Fatalf("cursor did not advance through terminal empty page: %q %v", follower.Cursor, terminal()) + } + if admitted, err := follower.Store.CanAdmit(follower.DispatcherID, 1001, "task-asr"); err != nil || !admitted { + t.Fatalf("valid discovery closed admission: %v %v", admitted, err) + } +} + +func TestCurrentDiscoveryFollowerFailedPageKeepsCursorAndClosesAdmission(t *testing.T) { + follower, terminal := newCurrentFollowerFixture(t, 2) + if err := follower.Poll(context.Background()); err == nil { + t.Fatal("task revision mismatch was accepted") + } + if follower.Cursor != "initial" || terminal() { + t.Fatalf("failed page advanced cursor: %q %v", follower.Cursor, terminal()) + } + if admitted, err := follower.Store.CanAdmit(follower.DispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("failed page left task admission open: %v %v", admitted, err) + } +} diff --git a/internal/dispatcher/current_execute.go b/internal/dispatcher/current_execute.go new file mode 100644 index 0000000..4ba4d25 --- /dev/null +++ b/internal/dispatcher/current_execute.go @@ -0,0 +1,233 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/contract" + "git.ipao.vip/rogee/go-sip/internal/store" +) + +// CurrentCallSpec is a frozen instruction handed to one Agent. Its Snapshot +// includes immutable AI parameters and provider credentials; never log it. +type CurrentCallSpec struct { + DispatcherID string + EventID string + TenantID int64 + TaskID string + CallerProfileID string + CallerID string + Callee string + DialedCallee string + TrunkID string + RingTimeoutMS int64 + Deadline time.Time + Snapshot configread.CurrentSnapshot +} + +type CurrentOriginator interface { + LoadedTrunks(context.Context) (map[string]int64, error) + Originate(context.Context, CurrentCallSpec) error +} + +type CurrentPublisher interface { + Publish(context.Context, string, string, []byte) error +} + +type CurrentExecuteController struct { + DispatcherID string + Store *store.CurrentStore + Originator CurrentOriginator + Publisher CurrentPublisher + Now func() time.Time +} + +func (c *CurrentExecuteController) validate() error { + if c == nil || c.DispatcherID == "" || c.Store == nil || c.Originator == nil || c.Publisher == nil || c.Now == nil { + return errors.New("current execution requires Dispatcher identity, durable store, originator, publisher, and clock") + } + return nil +} + +// ProcessExecute persists the inbound identity before the transport ACK. +// Its business acknowledgment is a separate durable outbox event, written +// only after originate was actually dispatched or an individual number was +// definitively rejected. +func (c *CurrentExecuteController) ProcessExecute(ctx context.Context, body []byte) error { + if err := c.validate(); err != nil { + return err + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return fmt.Errorf("invalid incoming call.execute: %w", err) + } + var event struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + DispatcherID string `json:"dispatcher_id"` + TenantID int64 `json:"tenant_id"` + IssuedAt string `json:"issued_at"` + Payload struct { + TaskID string `json:"task_id"` + Callee string `json:"callee"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &event); err != nil { + return fmt.Errorf("decode call.execute: %w", err) + } + if event.EventType != "call.execute" || event.Payload.TaskID == "" || event.Payload.Callee == "" || event.DispatcherID != c.DispatcherID { + return errors.New("call instruction event type, task, or Dispatcher owner mismatch") + } + cmd := store.CurrentExecuteCommand{ + DispatcherID: event.DispatcherID, EventID: event.EventID, + TenantID: event.TenantID, TaskID: event.Payload.TaskID, + Callee: event.Payload.Callee, IssuedAt: event.IssuedAt, + } + stored, _, err := c.Store.RecordExecute(cmd) + if errors.Is(err, store.ErrCurrentStopped) { + // The stopped task never accepted this old queued command. RabbitMQ + // may ACK it without creating an execution or a per-call result. + return nil + } + if err != nil { + return fmt.Errorf("durable inbound call %q: %w", cmd.EventID, err) + } + if stored.Status != "pending" { + return nil + } + if _, whitelisted := currentCalleeWhitelist[cmd.Callee]; !whitelisted { + if err := c.Store.RejectExecute(cmd.DispatcherID, cmd.EventID, ErrCurrentCalleeRejected.Error()); err != nil { + return fmt.Errorf("persist rejected call %q: %w", cmd.EventID, err) + } + return nil + } + return c.dispatchPending(ctx, cmd) +} + +// ProcessPending reevaluates only durable, never-originated commands. This is +// used after rule/config changes; dispatching and unknown calls are excluded. +func (c *CurrentExecuteController) ProcessPending(ctx context.Context) error { + if err := c.validate(); err != nil { + return err + } + commands, err := c.Store.ListPendingExecute(c.DispatcherID) + if err != nil { + return err + } + var failures []error + for _, cmd := range commands { + if err := c.dispatchPending(ctx, cmd); err != nil { + failures = append(failures, err) + } + } + return errors.Join(failures...) +} + +func (c *CurrentExecuteController) dispatchPending(ctx context.Context, cmd store.CurrentExecuteCommand) error { + at := c.Now() + issued, err := time.Parse(time.RFC3339Nano, cmd.IssuedAt) + if err != nil { + return fmt.Errorf("persisted command %q has invalid issued_at: %w", cmd.EventID, err) + } + if issued.After(at) { + return nil + } + snapshot, err := c.Store.ReadSnapshot(cmd.DispatcherID, cmd.TenantID, cmd.TaskID) + if err != nil { + return fmt.Errorf("load authorized task %q: %w", cmd.TaskID, err) + } + loaded, err := c.Originator.LoadedTrunks(ctx) + if err != nil { + return fmt.Errorf("read applied Agent/Asterisk SIP revisions: %w", err) + } + occupied, err := c.Store.TrunkOccupancy(c.DispatcherID) + if err != nil { + return err + } + choice, err := SelectCurrentTrunk(snapshot, cmd.Callee, at, occupied, loaded) + if errors.Is(err, ErrCurrentRuleWait) { + return nil + } + if err != nil { + return fmt.Errorf("task %q rule validation: %w", cmd.TaskID, err) + } + reservation := store.CurrentCallReservation{ + TrunkID: choice.TrunkID, SIPRevision: snapshot.SIP.Revision, + CallerID: choice.CallerID, DialedCallee: choice.DialedCallee, + Deadline: choice.Deadline, + } + if err := c.Store.ReserveExecute(cmd.DispatcherID, cmd.EventID, reservation, at); err != nil { + if errors.Is(err, store.ErrCurrentCapacity) || errors.Is(err, store.ErrCurrentNotReady) || errors.Is(err, store.ErrCurrentAlreadyStarted) { + return nil + } + return fmt.Errorf("reserve call %q: %w", cmd.EventID, err) + } + // A reservation is not permission to dial after a clock/window/loaded SIP + // change. Recheck the fixed selection immediately before the only RPC. + actualAt := c.Now() + freshLoaded, err := c.Originator.LoadedTrunks(ctx) + if err != nil { + cancelErr := c.Store.CancelReservationBeforeOrigin(cmd.DispatcherID, cmd.EventID) + return errors.Join(fmt.Errorf("recheck applied SIP before originate: %w", err), cancelErr) + } + fresh, ruleErr := SelectCurrentTrunk(snapshot, cmd.Callee, actualAt, occupied, freshLoaded) + if !actualAt.Before(choice.Deadline) || ruleErr != nil || fresh.TrunkID != choice.TrunkID || fresh.CallerID != choice.CallerID { + cancelErr := c.Store.CancelReservationBeforeOrigin(cmd.DispatcherID, cmd.EventID) + if cancelErr != nil { + return fmt.Errorf("cancel expired/changed reservation: %w", cancelErr) + } + if ruleErr != nil && !errors.Is(ruleErr, ErrCurrentRuleWait) { + return fmt.Errorf("pre-dial task %q rule: %w", cmd.TaskID, ruleErr) + } + return nil + } + admitted, err := c.Store.CanAdmit(cmd.DispatcherID, cmd.TenantID, cmd.TaskID) + if err != nil || !admitted { + cancelErr := c.Store.CancelReservationBeforeOrigin(cmd.DispatcherID, cmd.EventID) + if err != nil { + return errors.Join(fmt.Errorf("recheck task admission before originate: %w", err), cancelErr) + } + return cancelErr + } + spec := CurrentCallSpec{ + DispatcherID: cmd.DispatcherID, EventID: cmd.EventID, TenantID: cmd.TenantID, + TaskID: cmd.TaskID, CallerProfileID: snapshot.Task.CallerProfileID, + CallerID: choice.CallerID, Callee: cmd.Callee, DialedCallee: choice.DialedCallee, + TrunkID: choice.TrunkID, RingTimeoutMS: snapshot.Task.RingTimeoutMS, + Deadline: choice.Deadline, Snapshot: snapshot, + } + if err := c.Originator.Originate(ctx, spec); err != nil { + unknownErr := c.Store.MarkExecuteUnknown(cmd.DispatcherID, cmd.EventID) + return errors.Join(fmt.Errorf("originator outcome unknown for call %q: %w", cmd.EventID, err), unknownErr) + } + if err := c.Store.MarkExecuteDispatched(cmd.DispatcherID, cmd.EventID); err != nil { + return fmt.Errorf("originated call %q has no durable acknowledgment: %w", cmd.EventID, err) + } + return nil +} + +// FlushOutbox uses the same event identity/body on every retry. Confirming a +// bound queue is not a SaaS application receipt; confirmed rows remain stored. +func (c *CurrentExecuteController) FlushOutbox(ctx context.Context) error { + if err := c.validate(); err != nil { + return err + } + pending, err := c.Store.ListPendingOutbox(c.DispatcherID) + if err != nil { + return err + } + var failures []error + for _, event := range pending { + if err := c.Publisher.Publish(ctx, "agent-call.saas.v1", event.RoutingKey, event.Body); err != nil { + failures = append(failures, fmt.Errorf("publish event %q to SaaS queue: %w", event.EventID, err)) + continue + } + if err := c.Store.MarkOutboxConfirmed(c.DispatcherID, event.EventID); err != nil { + failures = append(failures, fmt.Errorf("persist queue confirmation for %q: %w", event.EventID, err)) + } + } + return errors.Join(failures...) +} diff --git a/internal/dispatcher/current_execute_test.go b/internal/dispatcher/current_execute_test.go new file mode 100644 index 0000000..b15122f --- /dev/null +++ b/internal/dispatcher/current_execute_test.go @@ -0,0 +1,176 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/contract" + "git.ipao.vip/rogee/go-sip/internal/store" +) + +type currentFakeOriginator struct { + loaded map[string]int64 + calls []CurrentCallSpec + err error +} + +func (f *currentFakeOriginator) LoadedTrunks(context.Context) (map[string]int64, error) { + return f.loaded, nil +} +func (f *currentFakeOriginator) Originate(_ context.Context, spec CurrentCallSpec) error { + f.calls = append(f.calls, spec) + return f.err +} + +type currentFakePublisher struct { + bodies [][]byte + err error +} + +func (f *currentFakePublisher) Publish(_ context.Context, exchange, key string, body []byte) error { + if exchange != "agent-call.saas.v1" || !strings.HasSuffix(key, ".out") { + return errors.New("unexpected outbound MQ route") + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return err + } + f.bodies = append(f.bodies, append([]byte(nil), body...)) + return f.err +} + +func newCurrentExecuteFixture(t *testing.T) (*CurrentExecuteController, *currentFakeOriginator, *currentFakePublisher, *store.CurrentStore) { + t.Helper() + s, err := store.OpenCurrent(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + snapshot := currentPolicySnapshot(t) + if err := s.ApplyDiscoverySnapshot(snapshot.Task.DispatcherID, []configread.CurrentDiscoveredTask{{TaskID: snapshot.Task.TaskID, TenantID: snapshot.Task.TenantID, TaskRevision: snapshot.Task.TaskRevision, Status: "running"}}); err != nil { + t.Fatal(err) + } + if err := s.SaveSnapshot(snapshot); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(snapshot.Task.DispatcherID, snapshot.SIP.Revision); err != nil { + t.Fatal(err) + } + originator := ¤tFakeOriginator{loaded: map[string]int64{"trunk-mock": 8}} + publisher := ¤tFakePublisher{} + controller := &CurrentExecuteController{DispatcherID: snapshot.Task.DispatcherID, Store: s, Originator: originator, Publisher: publisher, Now: func() time.Time { return currentMonday(9, 30) }} + return controller, originator, publisher, s +} + +func currentExecuteBody(t *testing.T, eventID, callee string) []byte { + t.Helper() + body := string(currentConfigExample(t, "mq-execute")) + body = strings.Replace(body, `"event_id":"call-example"`, `"event_id":"`+eventID+`"`, 1) + body = strings.Replace(body, `"callee":"15003164745"`, `"callee":"`+callee+`"`, 1) + return []byte(body) +} + +func TestCurrentExecuteRejectsInvalidCalleeWithoutStoppingTask(t *testing.T) { + controller, originator, publisher, s := newCurrentExecuteFixture(t) + if err := controller.ProcessExecute(context.Background(), currentExecuteBody(t, "bad-1", "not-a-phone")); err != nil { + t.Fatal(err) + } + if len(originator.calls) != 0 { + t.Fatal("invalid number reached originator") + } + outbox, err := s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(outbox) != 1 || !strings.Contains(string(outbox[0].Body), `"status":"rejected"`) { + t.Fatalf("missing individual rejection: %+v %v", outbox, err) + } + if err := controller.ProcessExecute(context.Background(), currentExecuteBody(t, "good-1", "15003164745")); err != nil { + t.Fatal(err) + } + if len(originator.calls) != 1 { + t.Fatalf("invalid number paused entire task; originate calls=%d", len(originator.calls)) + } + if err := controller.ProcessExecute(context.Background(), currentExecuteBody(t, "good-1", "15003164745")); err != nil || len(originator.calls) != 1 { + t.Fatalf("duplicate command reoriginated: %d %v", len(originator.calls), err) + } + if err := controller.FlushOutbox(context.Background()); err != nil { + t.Fatal(err) + } + if len(publisher.bodies) != 2 { + t.Fatalf("expected rejected and dispatched acknowledgments, got %d", len(publisher.bodies)) + } + for _, body := range publisher.bodies { + var message struct { + EventType string `json:"event_type"` + Payload struct { + Status string `json:"status"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &message); err != nil || message.EventType != "call.execute" || (message.Payload.Status != "rejected" && message.Payload.Status != "dispatched") { + t.Fatalf("unexpected outbound message: %s %v", body, err) + } + } +} + +func TestCurrentExecuteWaitsForRulesThenDispatchesOriginalIdentity(t *testing.T) { + controller, originator, _, s := newCurrentExecuteFixture(t) + controller.Now = func() time.Time { return currentMonday(8, 59) } + body := currentExecuteBody(t, "waiting-1", "15003164745") + body = []byte(strings.Replace(string(body), `"issued_at":"2026-09-21T01:00:00Z"`, `"issued_at":"2026-09-20T00:00:00Z"`, 1)) + if err := controller.ProcessExecute(context.Background(), body); err != nil { + t.Fatal(err) + } + if len(originator.calls) != 0 { + t.Fatal("dialed before task/trunk window") + } + pending, err := s.ListPendingExecute(controller.DispatcherID) + if err != nil || len(pending) != 1 || pending[0].EventID != "waiting-1" { + t.Fatalf("pending command was lost: %+v %v", pending, err) + } + controller.Now = func() time.Time { return currentMonday(9, 30) } + if err := controller.ProcessPending(context.Background()); err != nil || len(originator.calls) != 1 || originator.calls[0].EventID != "waiting-1" { + t.Fatalf("original command did not resume once: calls=%+v err=%v", originator.calls, err) + } +} + +func TestCurrentExecuteTimeoutAndMQFailureDoNotRedial(t *testing.T) { + controller, originator, publisher, s := newCurrentExecuteFixture(t) + originator.err = context.DeadlineExceeded + body := currentExecuteBody(t, "unknown-1", "15003164745") + if err := controller.ProcessExecute(context.Background(), body); err == nil { + t.Fatal("originator timeout hidden") + } + if err := controller.ProcessExecute(context.Background(), body); err != nil || len(originator.calls) != 1 { + t.Fatalf("unknown call reoriginated: %d %v", len(originator.calls), err) + } + if err := controller.ProcessPending(context.Background()); err != nil || len(originator.calls) != 1 { + t.Fatalf("unknown call appeared pending: %d %v", len(originator.calls), err) + } + originator.err = nil + second := currentExecuteBody(t, "mq-loss-1", "15003164745") + if err := controller.ProcessExecute(context.Background(), second); err != nil { + t.Fatal(err) + } + publisher.err = errors.New("injected publisher confirm loss") + if err := controller.FlushOutbox(context.Background()); err == nil { + t.Fatal("MQ failure was hidden") + } + pending, err := s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(pending) != 1 { + t.Fatalf("outbox lost original ack: %+v %v", pending, err) + } + if err := controller.ProcessExecute(context.Background(), second); err != nil || len(originator.calls) != 2 { + t.Fatalf("MQ redelivery reoriginated: %d %v", len(originator.calls), err) + } + publisher.err = nil + if err := controller.FlushOutbox(context.Background()); err != nil { + t.Fatal(err) + } + pending, err = s.ListPendingOutbox(controller.DispatcherID) + if err != nil || len(pending) != 0 { + t.Fatalf("confirmed event not retained correctly: %+v %v", pending, err) + } +} diff --git a/internal/dispatcher/current_gate_test.go b/internal/dispatcher/current_gate_test.go new file mode 100644 index 0000000..16cd9c7 --- /dev/null +++ b/internal/dispatcher/current_gate_test.go @@ -0,0 +1,64 @@ +package dispatcher + +import ( + "context" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/store" +) + +func TestCurrentPreDialWindowChangeReleasesUnusedReservation(t *testing.T) { + controller, originator, _, s := newCurrentExecuteFixture(t) + first := time.Date(2026, 9, 21, 19, 59, 59, 900_000_000, time.FixedZone("Asia/Shanghai", 8*3600)) + second := time.Date(2026, 9, 21, 20, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)) + calls := 0 + controller.Now = func() time.Time { + calls++ + if calls == 1 { + return first + } + return second + } + if err := controller.ProcessExecute(context.Background(), currentExecuteBody(t, "boundary-1", "15003164745")); err != nil { + t.Fatal(err) + } + if len(originator.calls) != 0 { + t.Fatal("dialed after window closed") + } + pending, err := s.ListPendingExecute(controller.DispatcherID) + if err != nil || len(pending) != 1 || pending[0].EventID != "boundary-1" { + t.Fatalf("unused reservation not returned to pending: %+v %v", pending, err) + } + occupied, err := s.TrunkOccupancy(controller.DispatcherID) + if err != nil || occupied["trunk-mock"] != 0 { + t.Fatalf("unused reservation held quota: %+v %v", occupied, err) + } + controller.Now = func() time.Time { return time.Date(2026, 9, 28, 9, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)) } + if err := controller.ProcessPending(context.Background()); err != nil || len(originator.calls) != 1 { + t.Fatalf("eligible pending call did not resume once: calls=%d err=%v", len(originator.calls), err) + } +} + +func TestCurrentFutureIssuedAtNeverDialsEarly(t *testing.T) { + controller, originator, _, s := newCurrentExecuteFixture(t) + controller.Now = func() time.Time { return time.Date(2026, 9, 21, 8, 59, 59, 0, time.FixedZone("Asia/Shanghai", 8*3600)) } + body := currentExecuteBody(t, "future-1", "15003164745") + if err := controller.ProcessExecute(context.Background(), body); err != nil { + t.Fatal(err) + } + if len(originator.calls) != 0 { + t.Fatal("future issued_at dialed early") + } + pending, err := s.ListPendingExecute(controller.DispatcherID) + if err != nil || len(pending) != 1 { + t.Fatalf("future command not durable: %+v %v", pending, err) + } + controller.Now = func() time.Time { return time.Date(2026, 9, 21, 9, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)) } + if err := controller.ProcessPending(context.Background()); err != nil || len(originator.calls) != 1 { + t.Fatalf("future command did not dispatch after issued_at: %d %v", len(originator.calls), err) + } + if _, _, err := s.RecordExecute(store.CurrentExecuteCommand{DispatcherID: controller.DispatcherID, EventID: "future-1", TenantID: 1001, TaskID: "task-asr", Callee: "15830461047", IssuedAt: "2026-09-21T01:00:00Z"}); err == nil { + t.Fatal("conflicting identity silently accepted") + } +} diff --git a/internal/dispatcher/current_runtime.go b/internal/dispatcher/current_runtime.go new file mode 100644 index 0000000..8d5dee1 --- /dev/null +++ b/internal/dispatcher/current_runtime.go @@ -0,0 +1,323 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/mq" + "git.ipao.vip/rogee/go-sip/internal/store" + "git.ipao.vip/rogee/go-sip/internal/tenant" +) + +// CurrentRuntime owns task/control consumers for one Dispatcher. SaaS creates +// every queue/binding; this process only checks and consumes predeclared ones. +type CurrentRuntime struct { + Broker *mq.CurrentBroker + Bootstrap CurrentBootstrap + Execute CurrentExecuteController + Control CurrentControlController + PollInterval time.Duration + DiscoveryInterval time.Duration + Logger *slog.Logger + + gate sync.RWMutex // SIP admission barrier versus each pre-dial instruction + locksMu sync.Mutex + taskLocks map[string]*sync.Mutex + failures chan error +} + +// Serve closes admission on every shutdown/failure; it never clears durable +// calls, results, task queues, or the SQLite file. +func (r *CurrentRuntime) Serve(ctx context.Context) (result error) { + if r == nil || r.Broker == nil || r.Bootstrap.Client == nil || r.Bootstrap.Store == nil || r.Bootstrap.VerifySIP == nil || r.Bootstrap.DispatcherID == "" || r.Execute.Store != r.Bootstrap.Store || r.Control.Store != r.Bootstrap.Store || r.Execute.DispatcherID != r.Bootstrap.DispatcherID || r.Control.DispatcherID != r.Bootstrap.DispatcherID || r.PollInterval <= 0 || r.DiscoveryInterval <= 0 || r.Logger == nil || r.Bootstrap.DrainControls != nil { + return errors.New("current runtime requires one Dispatcher, durable state, verified SIP, independent clocks, and configured polling; external control drain is forbidden") + } + if err := r.Execute.validate(); err != nil { + return err + } + if r.Control.Client != r.Bootstrap.Client { + return errors.New("current runtime control and bootstrap must share the approved HTTP client") + } + r.failures = make(chan error, 1) + r.taskLocks = make(map[string]*sync.Mutex) + consumers := make(map[string]*mq.CurrentConsumer) + var controlConsumer *mq.CurrentConsumer + defer func() { + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for queue, consumer := range consumers { + if err := consumer.Stop(stopCtx); err != nil { + result = errors.Join(result, fmt.Errorf("stop task consumer %q: %w", queue, err)) + } + } + if controlConsumer != nil { + if err := controlConsumer.Stop(stopCtx); err != nil { + result = errors.Join(result, fmt.Errorf("stop control consumer: %w", err)) + } + } + if err := r.Bootstrap.Store.CloseAdmission(r.Bootstrap.DispatcherID); err != nil { + result = errors.Join(result, fmt.Errorf("close Dispatcher admission: %w", err)) + } + }() + + var cursor string + var sip configread.CurrentSIP + r.Bootstrap.Cursor = &cursor + r.Bootstrap.SIP = &sip + r.Bootstrap.DrainControls = func(ctx context.Context) error { + _, err := r.Broker.DrainControlPredeclared(ctx, r.Broker.ControlQueue(), r.handleControl) + if err != nil { + return err + } + // Start control consumption before bootstrap opens task admission. + consumer, err := r.Broker.StartPredeclaredConsumer(ctx, r.Broker.ControlQueue(), r.handleControl) + if err != nil { + return err + } + controlConsumer = consumer + go r.watchConsumer(ctx, "control", consumer) + if err := r.Execute.FlushOutbox(ctx); err != nil { + return fmt.Errorf("recover durable results before admission: %w", err) + } + return nil + } + if err := r.Bootstrap.Run(ctx); err != nil { + if !errors.Is(err, store.ErrCurrentSIPPending) { + return fmt.Errorf("bootstrap current Dispatcher: %w", err) + } + r.Logger.Warn("control and outbox stay active while newer SIP revision waits; task admission remains closed", "dispatcher_id", r.Bootstrap.DispatcherID, "error", err) + } + follower := &CurrentDiscoveryFollower{DispatcherID: r.Bootstrap.DispatcherID, Client: r.Bootstrap.Client, Store: r.Bootstrap.Store, ApprovedSIP: sip, VerifySIP: r.Bootstrap.VerifySIP, Cursor: cursor} + if err := r.syncTaskConsumers(ctx, consumers); err != nil { + return err + } + if err := r.Execute.FlushOutbox(ctx); err != nil { + return err + } + poll := time.NewTicker(r.PollInterval) + defer poll.Stop() + discovery := time.NewTicker(r.DiscoveryInterval) + defer discovery.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-r.failures: + return fmt.Errorf("current MQ processing failed: %w", err) + case <-poll.C: + if err := r.processPending(ctx); err != nil { + return err + } + if err := r.Execute.FlushOutbox(ctx); err != nil { + return fmt.Errorf("deliver persisted results: %w", err) + } + if err := r.syncTaskConsumers(ctx, consumers); err != nil { + return err + } + case <-discovery.C: + if err := r.refreshSIP(ctx, follower); err != nil { + return fmt.Errorf("refresh approved SIP: %w", err) + } + _, pending, err := r.Bootstrap.Store.SIPState(r.Bootstrap.DispatcherID) + if err != nil { + return err + } + if pending == 0 { + if err := follower.Poll(ctx); err != nil { + return fmt.Errorf("poll assigned tasks: %w", err) + } + } + if err := r.syncTaskConsumers(ctx, consumers); err != nil { + return err + } + } + } +} + +func (r *CurrentRuntime) watchConsumer(ctx context.Context, queue string, consumer *mq.CurrentConsumer) { + err := consumer.Wait(ctx) + if err != nil && ctx.Err() == nil { + r.Logger.Error("current MQ consumer failed", "dispatcher_id", r.Bootstrap.DispatcherID, "queue", queue, "error", err) + select { + case r.failures <- fmt.Errorf("queue %q: %w", queue, err): + default: + } + } +} + +// signalFailure forces the runtime to close admission after a consumer +// handler fails; RabbitMQ requeue alone would otherwise spin indefinitely. +func (r *CurrentRuntime) signalFailure(err error) { + if r.failures != nil { + select { + case r.failures <- err: + default: // the first failure is already being handled + } + } +} + +func (r *CurrentRuntime) withTask(taskID string, process func() error) error { + r.gate.RLock() + defer r.gate.RUnlock() + r.locksMu.Lock() + lock := r.taskLocks[taskID] + if lock == nil { + lock = new(sync.Mutex) + r.taskLocks[taskID] = lock + } + r.locksMu.Unlock() + lock.Lock() + defer lock.Unlock() + return process() +} + +func (r *CurrentRuntime) handleControl(ctx context.Context, _ string, body []byte) error { + var event struct { + EventType string `json:"event_type"` + Payload struct { + TaskID string `json:"task_id"` + Revision int64 `json:"revision"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &event); err != nil { + failure := fmt.Errorf("decode MQ control routing: %w", err) + r.signalFailure(failure) + return failure + } + switch event.EventType { + case "task.control": + return r.withTask(event.Payload.TaskID, func() error { + if err := r.Control.ProcessControl(ctx, body); err != nil { + r.Logger.Error("task control failed", "dispatcher_id", r.Bootstrap.DispatcherID, "task_id", event.Payload.TaskID, "error", err) + r.signalFailure(err) + return err + } + return nil + }) + case "sip.config": + r.gate.Lock() + err := r.Bootstrap.Store.NoteSIPChange(r.Bootstrap.DispatcherID, event.Payload.Revision) + r.gate.Unlock() + if err != nil { + failure := fmt.Errorf("persist SIP change before MQ ACK: %w", err) + r.signalFailure(failure) + return failure + } + r.Logger.Info("SIP notification persisted; task admission fenced until drain and loaded revision check", "dispatcher_id", r.Bootstrap.DispatcherID, "revision", event.Payload.Revision) + return nil + default: + failure := fmt.Errorf("unexpected MQ control event %q", event.EventType) + r.signalFailure(failure) + return failure + } +} + +func (r *CurrentRuntime) handleTask(ctx context.Context, taskID string, body []byte) error { + return r.withTask(taskID, func() error { + if err := r.Execute.ProcessExecute(ctx, body); err != nil { + r.Logger.Error("call instruction failed", "dispatcher_id", r.Bootstrap.DispatcherID, "task_id", taskID, "error", err) + r.signalFailure(err) + return err + } + return nil + }) +} + +func (r *CurrentRuntime) processPending(ctx context.Context) error { + pending, err := r.Bootstrap.Store.ListPendingExecute(r.Bootstrap.DispatcherID) + if err != nil { + return fmt.Errorf("read durable rule-wait instructions: %w", err) + } + for _, command := range pending { + cmd := command + if err := r.withTask(cmd.TaskID, func() error { return r.Execute.dispatchPending(ctx, cmd) }); err != nil { + return fmt.Errorf("retry eligible pending instruction %q: %w", cmd.EventID, err) + } + } + return nil +} + +func (r *CurrentRuntime) syncTaskConsumers(ctx context.Context, consumers map[string]*mq.CurrentConsumer) error { + assigned, err := r.Bootstrap.Store.ListAssignedTasks(r.Bootstrap.DispatcherID) + if err != nil { + return err + } + wanted := make(map[string]store.CurrentAssignedTask, len(assigned)) + for _, task := range assigned { + route, err := tenant.CurrentTaskRoute(r.Bootstrap.DispatcherID, task.TaskID) + if err != nil { + return err + } + if task.ControlState == "paused" || task.ControlState == "pausing" || task.ControlState == "resuming" || task.Status == "paused" { + continue + } + if task.ControlState != "stopped" && task.ControlState != "stopping" && task.Status != "stopped" { + admitted, err := r.Bootstrap.Store.CanAdmit(r.Bootstrap.DispatcherID, task.TenantID, task.TaskID) + if err != nil { + return err + } + if !admitted { + continue + } + waiting, err := r.Bootstrap.Store.PendingExecuteCount(r.Bootstrap.DispatcherID, task.TenantID, task.TaskID) + if err != nil { + return err + } + if waiting != 0 { + continue + } + } + wanted[route.Queue] = task + } + for queue, consumer := range consumers { + if _, ok := wanted[queue]; ok { + continue + } + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := consumer.Stop(stopCtx) + cancel() + if err != nil { + return fmt.Errorf("stop nonadmitted task queue %q: %w", queue, err) + } + delete(consumers, queue) + } + for queue, task := range wanted { + if _, ok := consumers[queue]; ok { + continue + } + id, tenantID := task.TaskID, task.TenantID + ready := make(chan struct{}) + var consumer *mq.CurrentConsumer + consumer, err = r.Broker.StartPredeclaredConsumer(ctx, queue, func(ctx context.Context, _ string, body []byte) error { + <-ready // the subscription is assigned before its first delivery can pause itself + if err := r.handleTask(ctx, id, body); err != nil { + return err + } + waiting, err := r.Bootstrap.Store.PendingExecuteCount(r.Bootstrap.DispatcherID, tenantID, id) + if err != nil { + return fmt.Errorf("check task-local rule wait: %w", err) + } + if waiting != 0 { + consumer.RequestStop() + } + return nil + }) + if err != nil { + return fmt.Errorf("start SaaS-owned task queue %q: %w", queue, err) + } + consumers[queue] = consumer + close(ready) + go r.watchConsumer(ctx, queue, consumer) + } + return nil +} + +// Compile-time interface checks: the same broker confirms bound persistent +// results and consumes SaaS-owned queues without configure permissions. +var _ CurrentPublisher = (*mq.CurrentBroker)(nil) diff --git a/internal/dispatcher/current_runtime_integration_test.go b/internal/dispatcher/current_runtime_integration_test.go new file mode 100644 index 0000000..06c514f --- /dev/null +++ b/internal/dispatcher/current_runtime_integration_test.go @@ -0,0 +1,380 @@ +//go:build integration + +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/mq" + "git.ipao.vip/rogee/go-sip/internal/store" + "git.ipao.vip/rogee/go-sip/internal/tenant" + amqp "github.com/rabbitmq/amqp091-go" +) + +type currentRuntimeMockAgent struct { + calls chan CurrentCallSpec + controls chan CurrentControlSpec +} + +func (a *currentRuntimeMockAgent) LoadedTrunks(context.Context) (map[string]int64, error) { + return map[string]int64{"trunk-mock": 8}, nil +} +func (a *currentRuntimeMockAgent) Originate(_ context.Context, spec CurrentCallSpec) error { + a.calls <- spec + return nil +} +func (a *currentRuntimeMockAgent) SendControl(_ context.Context, spec CurrentControlSpec) error { + a.controls <- spec + return nil +} + +func TestCurrentRuntimeIsolatedControlBacklogExecuteAndSharedResult(t *testing.T) { + brokerURL, adminURL := os.Getenv("RABBITMQ_URL"), os.Getenv("RABBITMQ_PROVISIONER_URL") + if brokerURL == "" || adminURL == "" { + t.Skip("requires isolated RabbitMQ mock with provisioner account") + } + id := "c046b893-8628-4589-ae50-619d049248a6" + adminConn, err := amqp.Dial(adminURL) + if err != nil { + t.Fatal(err) + } + defer adminConn.Close() + admin, err := adminConn.Channel() + if err != nil { + t.Fatal(err) + } + defer admin.Close() + for _, exchange := range []string{mq.CommandsExchangeCurrent, mq.ResultsExchangeCurrent, mq.DeadLetterExchangeCurrent} { + if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil { + t.Fatal(err) + } + } + controlRoute, _ := tenant.CurrentControlRoute(id) + taskRoute, _ := tenant.CurrentTaskRoute(id, "task-asr") + resultRoute, _ := tenant.CurrentResultRoute(id) + shared := "agent-call.saas.events.v1" + for _, queue := range []string{controlRoute.Queue, taskRoute.Queue, shared} { + if _, err := admin.QueueDeclare(queue, true, false, false, false, nil); err != nil { + t.Fatal(err) + } + defer func(name string) { _, _ = admin.QueueDelete(name, false, false, false) }(queue) + if _, err := admin.QueuePurge(queue, false); err != nil { + t.Fatal(err) + } + } + for _, route := range []tenant.CurrentRoute{controlRoute, taskRoute, resultRoute} { + queue := route.Queue + if route.BindingKey == resultRoute.BindingKey { + queue = shared + } + if err := admin.QueueBind(queue, route.BindingKey, route.Exchange, false, nil); err != nil { + t.Fatal(err) + } + defer func(q, key, exchange string) { _ = admin.QueueUnbind(q, key, exchange, nil) }(queue, route.BindingKey, route.Exchange) + } + publish := func(route tenant.CurrentRoute, body []byte) { + t.Helper() + if err := admin.PublishWithContext(context.Background(), route.Exchange, route.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil { + t.Fatal(err) + } + } + publish(controlRoute, currentControlBody(t, "control-example", "pause", "drain")) + publish(taskRoute, currentExecuteBody(t, "call-example", "15003164745")) + + snapshot := currentPolicySnapshot(t) + sipJSON, err := json.Marshal(snapshot.SIP) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var body []byte + switch r.URL.Path { + case "/internal/v1/dispatcher/sip": + body = sipJSON + case "/internal/v1/dispatcher/tasks": + if r.URL.Query().Get("after") == "" { + body = currentConfigExample(t, "task-discovery-page") + } else if r.URL.Query().Get("after") != "" { + body = currentConfigExample(t, "task-discovery-end") + } else { + w.WriteHeader(http.StatusBadRequest) + return + } + case "/internal/v1/dispatcher/task/task-asr": + body = currentConfigExample(t, "config-read-task-asr") + case "/internal/v1/dispatcher/ai-providers": + body = currentConfigExample(t, "config-read-providers") + case "/internal/v1/dispatcher/tenant/1001/quota": + body = currentConfigExample(t, "config-read-quota") + default: + t.Errorf("unexpected HTTP configuration path %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write(body) + })) + defer server.Close() + client, err := configread.NewClient(server.URL, id, "test-secret", server.Client()) + if err != nil { + t.Fatal(err) + } + db, err := store.OpenCurrent(filepath.Join(t.TempDir(), "dispatcher.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + broker, err := mq.OpenCurrent(brokerURL, id, 1) + if err != nil { + t.Fatal(err) + } + defer broker.Close() + agent := ¤tRuntimeMockAgent{calls: make(chan CurrentCallSpec, 3), controls: make(chan CurrentControlSpec, 3)} + var windowAllowed atomic.Bool + windowAllowed.Store(true) + verify := func(_ context.Context, sip configread.CurrentSIP) error { + if sip.Revision != 8 || sip.DispatcherID != id { + return fmt.Errorf("unloaded SIP revision") + } + return nil + } + runtime := &CurrentRuntime{ + Broker: broker, + Bootstrap: CurrentBootstrap{DispatcherID: id, Client: client, Store: db, VerifySIP: verify}, + Execute: CurrentExecuteController{DispatcherID: id, Store: db, Originator: agent, Publisher: broker, Now: func() time.Time { + if windowAllowed.Load() { + return currentMonday(9, 30) + } + return currentMonday(8, 59) + }}, + Control: CurrentControlController{DispatcherID: id, Store: db, Client: client, Agent: agent, VerifySIP: verify, Now: func() time.Time { return currentMonday(9, 30) }}, + PollInterval: 30 * time.Millisecond, DiscoveryInterval: 120 * time.Millisecond, Logger: slog.Default(), + } + ctx, cancel := context.WithCancel(context.Background()) + finished := make(chan struct{}) + var runtimeErr error + go func() { runtimeErr = runtime.Serve(ctx); close(finished) }() + defer func() { + cancel() + select { + case <-finished: + if runtimeErr != nil && !errors.Is(runtimeErr, context.Canceled) { + t.Errorf("runtime stopped: %v", runtimeErr) + } + case <-time.After(5 * time.Second): + t.Error("runtime did not stop") + } + }() + select { + case spec := <-agent.controls: + if spec.Action != "pause" || spec.ActiveCallPolicy != "drain" { + t.Fatalf("wrong drained control: %+v", spec) + } + case <-finished: + t.Fatalf("runtime failed during bootstrap: %v", runtimeErr) + case <-time.After(5 * time.Second): + t.Fatal("control backlog was not processed") + } + time.Sleep(120 * time.Millisecond) + select { + case call := <-agent.calls: + t.Fatalf("paused task originated call: %s", call.EventID) + default: + } + state, err := admin.QueueInspect(taskRoute.Queue) + if err != nil || state.Messages != 1 { + t.Fatalf("task queue was consumed before resume: %+v %v", state, err) + } + publish(controlRoute, currentControlBody(t, "resume-integration", "resume", "")) + select { + case spec := <-agent.controls: + if spec.Action != "resume" { + t.Fatalf("wrong control action: %+v", spec) + } + case <-finished: + t.Fatalf("runtime failed before resume: %v", runtimeErr) + case <-time.After(5 * time.Second): + t.Fatal("resume control not delivered") + } + select { + case spec := <-agent.calls: + if spec.EventID != "call-example" || spec.TrunkID != "trunk-mock" { + t.Fatalf("wrong dispatch: %+v", spec) + } + case <-finished: + t.Fatalf("runtime failed before call dispatch: %v", runtimeErr) + case <-time.After(5 * time.Second): + t.Fatal("queued call not dispatched after resume") + } + seen := map[string]string{} + until := time.After(5 * time.Second) + for len(seen) < 3 { + select { + case <-until: + t.Fatalf("missing SaaS shared-queue results: %+v", seen) + default: + } + msg, ok, err := admin.Get(shared, false) + if err != nil { + t.Fatal(err) + } + if !ok { + time.Sleep(20 * time.Millisecond) + continue + } + var event struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + Payload struct { + Status string `json:"status"` + } `json:"payload"` + } + if err := json.Unmarshal(msg.Body, &event); err != nil { + t.Fatal(err) + } + seen[event.EventID] = event.Payload.Status + if err := msg.Ack(false); err != nil { + t.Fatal(err) + } + } + if seen["control-example"] != "applied" || seen["resume-integration"] != "applied" || seen["call-example"] != "dispatched" { + t.Fatalf("wrong shared SaaS results: %+v", seen) + } + publish(taskRoute, currentExecuteBody(t, "call-example", "15003164745")) + select { + case spec := <-agent.calls: + t.Fatalf("redelivery reoriginated call %s", spec.EventID) + case <-time.After(200 * time.Millisecond): + } + // A temporary task rule wait is retained durably, then its consumer + // stops so further instructions remain in SaaS's task queue. Admission + // resumes from the original identity when the configured window opens. + windowAllowed.Store(false) + waiting := []byte(strings.Replace(string(currentExecuteBody(t, "window-wait-1", "15003164745")), `"issued_at":"2026-09-21T01:00:00Z"`, `"issued_at":"2026-09-20T00:00:00Z"`, 1)) + publish(taskRoute, waiting) + waitDeadline := time.After(5 * time.Second) + for { + count, err := db.PendingExecuteCount(id, 1001, "task-asr") + if err != nil { + t.Fatal(err) + } + queueState, err := admin.QueueInspect(taskRoute.Queue) + if err != nil { + t.Fatal(err) + } + if count == 1 && queueState.Consumers == 0 { + break + } + select { + case <-waitDeadline: + t.Fatalf("task-local waiting consumer stayed active: pending=%d consumers=%d", count, queueState.Consumers) + case <-time.After(20 * time.Millisecond): + } + } + publish(taskRoute, currentExecuteBody(t, "window-wait-2", "15003164745")) + time.Sleep(100 * time.Millisecond) + queueState, err := admin.QueueInspect(taskRoute.Queue) + if err != nil || queueState.Messages != 1 { + t.Fatalf("rule-wait backlog was not retained in SaaS queue: %+v %v", queueState, err) + } + select { + case spec := <-agent.calls: + t.Fatalf("called outside allowed window: %s", spec.EventID) + default: + } + windowAllowed.Store(true) + select { + case spec := <-agent.calls: + if spec.EventID != "window-wait-1" { + t.Fatalf("wrong waiting instruction resumed: %s", spec.EventID) + } + case <-finished: + t.Fatalf("runtime failed before window reopened: %v", runtimeErr) + case <-time.After(5 * time.Second): + t.Fatal("eligible retained instruction did not resume") + } + // A SIP notification is ACKed only after its revision is durable. The + // already-dispatched calls keep the revision change fenced, while the + // independent control queue and shared result publisher continue. + sipChange := []byte(strings.Replace(string(currentConfigExample(t, "mq-sip-change")), `"revision":8`, `"revision":9`, 1)) + publish(controlRoute, sipChange) + deadline := time.After(5 * time.Second) + for { + applied, pending, err := db.SIPState(id) + if err != nil { + t.Fatal(err) + } + if applied == 8 && pending == 9 { + break + } + select { + case <-deadline: + t.Fatalf("SIP notification was not persisted: applied=%d pending=%d", applied, pending) + case <-time.After(20 * time.Millisecond): + } + } + publish(taskRoute, currentExecuteBody(t, "after-sip-change", "15003164745")) + select { + case spec := <-agent.calls: + t.Fatalf("pending SIP change originated call %s", spec.EventID) + case <-time.After(120 * time.Millisecond): + } + publish(controlRoute, currentControlBody(t, "stop-after-sip", "stop", "")) + select { + case spec := <-agent.controls: + if spec.Action != "stop" || spec.ActiveCallPolicy != "hangup" { + t.Fatalf("control blocked or changed by SIP reload: %+v", spec) + } + case <-finished: + t.Fatalf("runtime failed while SIP was pending: %v", runtimeErr) + case <-time.After(5 * time.Second): + t.Fatal("stop control was blocked by pending SIP revision") + } + deadline = time.After(5 * time.Second) + for { + select { + case <-deadline: + t.Fatal("control acknowledgment did not reach shared SaaS result queue during SIP reload") + default: + } + msg, ok, err := admin.Get(shared, false) + if err != nil { + t.Fatal(err) + } + if !ok { + time.Sleep(20 * time.Millisecond) + continue + } + var ack struct { + EventID string `json:"event_id"` + Payload struct { + Status string `json:"status"` + } `json:"payload"` + } + if err := json.Unmarshal(msg.Body, &ack); err != nil { + t.Fatal(err) + } + if err := msg.Ack(false); err != nil { + t.Fatal(err) + } + if ack.EventID == "stop-after-sip" { + if ack.Payload.Status != "applied" { + t.Fatalf("stop control falsely acknowledged during SIP reload: %+v", ack) + } + break + } + } +} diff --git a/internal/dispatcher/current_sip_reload.go b/internal/dispatcher/current_sip_reload.go new file mode 100644 index 0000000..78ba832 --- /dev/null +++ b/internal/dispatcher/current_sip_reload.go @@ -0,0 +1,127 @@ +package dispatcher + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" +) + +// refreshSIP keeps control/result delivery running while a new approved SIP +// revision waits for old calls to drain and for Agent/Asterisk to load it. +// No HTTP snapshot or notification by itself authorizes a real call. +func (r *CurrentRuntime) refreshSIP(ctx context.Context, follower *CurrentDiscoveryFollower) error { + if r == nil || follower == nil || r.Bootstrap.Client == nil || r.Bootstrap.Store == nil || r.Bootstrap.VerifySIP == nil || r.Logger == nil { + return errors.New("SIP refresh requires durable state and applied-revision verifier") + } + current, err := r.Bootstrap.Client.ReadCurrentSIP(ctx) + if err != nil { + return r.closeSIPAdmission(fmt.Errorf("read approved SIP full snapshot: %w", err)) + } + old := follower.ApprovedSIP + if current.Revision < old.Revision { + return r.closeSIPAdmission(fmt.Errorf("approved SIP revision regressed from %d to %d", old.Revision, current.Revision)) + } + oldJSON, err := json.Marshal(old) + if err != nil { + return r.closeSIPAdmission(fmt.Errorf("encode current loaded SIP snapshot: %w", err)) + } + currentJSON, err := json.Marshal(current) + if err != nil { + return r.closeSIPAdmission(fmt.Errorf("encode latest approved SIP snapshot: %w", err)) + } + if current.Revision == old.Revision && !bytes.Equal(oldJSON, currentJSON) { + return r.closeSIPAdmission(fmt.Errorf("approved SIP revision %d changed content without a new revision", current.Revision)) + } + if current.Revision > old.Revision { + r.gate.Lock() + err := r.Bootstrap.Store.NoteSIPChange(r.Bootstrap.DispatcherID, current.Revision) + r.gate.Unlock() + if err != nil { + return fmt.Errorf("fence newly discovered SIP revision: %w", err) + } + } + _, pending, err := r.Bootstrap.Store.SIPState(r.Bootstrap.DispatcherID) + if err != nil { + return err + } + if pending == 0 { + return nil + } + if current.Revision < pending { + r.Logger.Warn("SaaS SIP snapshot is behind durable SIP notification; admission stays closed", "dispatcher_id", r.Bootstrap.DispatcherID, "pending_revision", pending, "available_revision", current.Revision) + return nil + } + occupied, err := r.Bootstrap.Store.OccupiedCalls(r.Bootstrap.DispatcherID) + if err != nil { + return err + } + if occupied != 0 { + r.Logger.Info("SIP reload waits for confirmed old-call drain", "dispatcher_id", r.Bootstrap.DispatcherID, "pending_revision", pending, "occupied_calls", occupied) + return nil + } + if err := r.Bootstrap.VerifySIP(ctx, current); err != nil { + r.Logger.Warn("SIP reload waits for actual Agent/Asterisk revision", "dispatcher_id", r.Bootstrap.DispatcherID, "pending_revision", pending, "error", err) + return nil + } + // A full discovery snapshot is required after an approved SIP revision + // change; a delta page cannot prove that all assigned tasks use the same + // loaded revision. MQ control consumption remains active during this read. + tasks, cursor, err := r.Bootstrap.Client.ReadAllCurrentTasks(ctx) + if err != nil { + return r.closeSIPAdmission(fmt.Errorf("reload complete assigned task list for SIP: %w", err)) + } + if err := r.Bootstrap.Store.ApplyDiscoverySnapshot(r.Bootstrap.DispatcherID, tasks); err != nil { + return r.closeSIPAdmission(fmt.Errorf("persist complete assigned task list for SIP: %w", err)) + } + for _, task := range tasks { + if task.Status == "stopped" { + continue + } + snapshot, err := r.Bootstrap.Client.ReadCurrentTask(ctx, task.TaskID, task.TenantID) + if err != nil { + return r.closeSIPAdmission(fmt.Errorf("reload task %q after SIP revision: %w", task.TaskID, err)) + } + if snapshot.Task.TaskRevision != task.TaskRevision || snapshot.Task.Status != task.Status { + return r.closeSIPAdmission(fmt.Errorf("task %q discovery conflicts with SIP reload configuration", task.TaskID)) + } + actualJSON, err := json.Marshal(snapshot.SIP) + if err != nil { + return r.closeSIPAdmission(fmt.Errorf("encode task %q SIP binding: %w", task.TaskID, err)) + } + if !bytes.Equal(currentJSON, actualJSON) { + return r.closeSIPAdmission(fmt.Errorf("task %q is not bound to approved SIP revision %d", task.TaskID, current.Revision)) + } + if err := r.Bootstrap.Store.SaveSnapshot(snapshot); err != nil { + return r.closeSIPAdmission(fmt.Errorf("persist task %q SIP binding: %w", task.TaskID, err)) + } + } + _, latestPending, err := r.Bootstrap.Store.SIPState(r.Bootstrap.DispatcherID) + if err != nil { + return err + } + if latestPending > current.Revision { + r.Logger.Info("newer SIP notification arrived during reload; admission remains closed", "dispatcher_id", r.Bootstrap.DispatcherID, "pending_revision", latestPending) + return nil + } + // MarkReadyForSIP rechecks the drain and every durable task binding in + // one SQLite transaction. No old version can reopen admission here. + if err := r.Bootstrap.Store.MarkReadyForSIP(r.Bootstrap.DispatcherID, current.Revision); err != nil { + return r.closeSIPAdmission(fmt.Errorf("verify and commit reloaded SIP revision: %w", err)) + } + follower.ApprovedSIP = current + follower.Cursor = cursor + if r.Bootstrap.SIP != nil { + *r.Bootstrap.SIP = current + } + r.Logger.Info("approved SIP revision is applied after confirmed drain", "dispatcher_id", r.Bootstrap.DispatcherID, "revision", current.Revision) + return nil +} + +func (r *CurrentRuntime) closeSIPAdmission(cause error) error { + if err := r.Bootstrap.Store.CloseAdmission(r.Bootstrap.DispatcherID); err != nil { + return errors.Join(cause, fmt.Errorf("close admission after SIP refresh failure: %w", err)) + } + return cause +} diff --git a/internal/dispatcher/current_sip_runtime_test.go b/internal/dispatcher/current_sip_runtime_test.go new file mode 100644 index 0000000..403345e --- /dev/null +++ b/internal/dispatcher/current_sip_runtime_test.go @@ -0,0 +1,91 @@ +package dispatcher + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "git.ipao.vip/rogee/go-sip/internal/configread" +) + +func TestCurrentSIPNotificationPersistsBarrierAndWaitsForLoadedFullSnapshot(t *testing.T) { + executor, _, _, s := newCurrentExecuteFixture(t) + approved := currentPolicySnapshot(t).SIP + changed := approved + changed.Revision = 9 + newSIP, err := json.Marshal(changed) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var body []byte + switch r.URL.Path { + case "/internal/v1/dispatcher/sip": + body = newSIP + case "/internal/v1/dispatcher/tasks": + if r.URL.Query().Get("after") == "" { + body = currentConfigExample(t, "task-discovery-page") + } else { + body = currentConfigExample(t, "task-discovery-end") + } + case "/internal/v1/dispatcher/ai-providers": + body = currentConfigExample(t, "config-read-providers") + case "/internal/v1/dispatcher/task/task-asr": + body = currentConfigExample(t, "config-read-task-asr") + case "/internal/v1/dispatcher/tenant/1001/quota": + body = currentConfigExample(t, "config-read-quota") + default: + t.Errorf("unexpected HTTP path %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write(body) + })) + defer server.Close() + client, err := configread.NewClient(server.URL, executor.DispatcherID, "test-secret", server.Client()) + if err != nil { + t.Fatal(err) + } + var loaded atomic.Bool + verify := func(_ context.Context, sip configread.CurrentSIP) error { + if sip.Revision != 9 || !loaded.Load() { + return errors.New("Agent and Asterisk have not applied SIP revision 9") + } + return nil + } + runtime := &CurrentRuntime{Bootstrap: CurrentBootstrap{DispatcherID: executor.DispatcherID, Client: client, Store: s, VerifySIP: verify}, Logger: slog.Default()} + follower := &CurrentDiscoveryFollower{DispatcherID: executor.DispatcherID, Client: client, Store: s, ApprovedSIP: approved, VerifySIP: verify, Cursor: "opaque-end-token"} + body := []byte(strings.Replace(string(currentConfigExample(t, "mq-sip-change")), `"revision":8`, `"revision":9`, 1)) + if err := runtime.handleControl(context.Background(), "", body); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(executor.DispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("SIP notification did not close admission: %v %v", admitted, err) + } + if applied, pending, err := s.SIPState(executor.DispatcherID); err != nil || applied != 8 || pending != 9 { + t.Fatalf("notification not durable before ACK: %d %d %v", applied, pending, err) + } + if err := runtime.refreshSIP(context.Background(), follower); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(executor.DispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("unloaded SIP reopened admission: %v %v", admitted, err) + } + loaded.Store(true) + if err := runtime.refreshSIP(context.Background(), follower); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(executor.DispatcherID, 1001, "task-asr"); err != nil || !admitted { + t.Fatalf("verified SIP did not reopen: %v %v", admitted, err) + } + if follower.ApprovedSIP.Revision != 9 || follower.Cursor != "opaque-end-token" { + t.Fatalf("reloaded SIP/cursor not bound: rev=%d cursor=%q", follower.ApprovedSIP.Revision, follower.Cursor) + } +} diff --git a/internal/mq/current.go b/internal/mq/current.go new file mode 100644 index 0000000..c48e84c --- /dev/null +++ b/internal/mq/current.go @@ -0,0 +1,516 @@ +package mq + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "git.ipao.vip/rogee/go-sip/internal/contract" + "git.ipao.vip/rogee/go-sip/internal/tenant" + amqp "github.com/rabbitmq/amqp091-go" +) + +const ( + CommandsExchangeCurrent = "agent-call.dispatchers.v1" + ResultsExchangeCurrent = "agent-call.saas.v1" + DeadLetterExchangeCurrent = "agent-call.dead-letter.v1" + MaxCurrentMessageBytes = 8 << 20 +) + +type CurrentBroker struct { + conn *amqp.Connection + dispatcherID string + prefetch int + controlQueue string + resultRoute string + closed <-chan *amqp.Error + mu sync.Mutex +} + +// OpenCurrent verifies SaaS-provisioned topology passively. It never declares, +// binds, or deletes exchanges or queues. +func CurrentControlQueueName(dispatcherID string) string { + return "agent-call.d." + dispatcherID + ".control.v1" +} + +func OpenCurrent(url, dispatcherID string, prefetch int) (*CurrentBroker, error) { + if strings.TrimSpace(url) == "" { + return nil, errors.New("rabbitmq URL is required") + } + if prefetch <= 0 { + return nil, errors.New("prefetch must be positive") + } + if err := tenant.ValidateDispatcherID(dispatcherID); err != nil { + return nil, err + } + conn, err := amqp.Dial(url) + if err != nil { + return nil, fmt.Errorf("dial rabbitmq: %w", err) + } + channel, err := conn.Channel() + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("open rabbitmq current channel: %w", err) + } + for _, exchange := range []string{CommandsExchangeCurrent, ResultsExchangeCurrent, DeadLetterExchangeCurrent} { + if err := channel.ExchangeDeclarePassive(exchange, "topic", true, false, false, false, nil); err != nil { + _ = channel.Close() + _ = conn.Close() + return nil, fmt.Errorf("required SaaS exchange %s unavailable: %w", exchange, err) + } + } + controlQueue := CurrentControlQueueName(dispatcherID) + if _, err := channel.QueueDeclarePassive(controlQueue, true, false, false, false, nil); err != nil { + _ = channel.Close() + _ = conn.Close() + return nil, fmt.Errorf("required SaaS control queue unavailable: %w", err) + } + // The shared SaaS result queue is not readable by Dispatcher credentials. + // mandatory + return + confirm and a SaaS-owned Mock consumer prove routing. + _ = channel.Close() + return &CurrentBroker{ + conn: conn, + dispatcherID: dispatcherID, + prefetch: prefetch, + controlQueue: controlQueue, + resultRoute: "d." + dispatcherID + ".out", + closed: conn.NotifyClose(make(chan *amqp.Error, 1)), + }, nil +} + +func (b *CurrentBroker) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.conn == nil { + return nil + } + err := b.conn.Close() + b.conn = nil + return err +} + +func (b *CurrentBroker) Done() <-chan *amqp.Error { return b.closed } + +func (b *CurrentBroker) Publish(ctx context.Context, exchange, routingKey string, body []byte) error { + if exchange != ResultsExchangeCurrent || routingKey != b.resultRoute || len(body) == 0 || len(body) > MaxCurrentMessageBytes { + return fmt.Errorf("current publish requires result exchange, Dispatcher route and 1..%d body bytes", MaxCurrentMessageBytes) + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return fmt.Errorf("outbound MQ contract: %w", err) + } + var identity struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + DispatcherID string `json:"dispatcher_id"` + Payload struct { + Status string `json:"status"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &identity); err != nil { + return fmt.Errorf("decode outbound MQ identity: %w", err) + } + if identity.DispatcherID != b.dispatcherID { + return errors.New("outbound MQ dispatcher owner mismatch") + } + if identity.EventType != "call.execute.result" && + !((identity.EventType == "task.control" || identity.EventType == "call.execute") && identity.Payload.Status != "") { + return fmt.Errorf("unapproved outbound MQ event %q", identity.EventType) + } + if len(identity.EventID) > 255 { + return errors.New("outbound event identity exceeds AMQP limit") + } + messageID := identity.EventID + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + b.mu.Lock() + defer b.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } + if b.conn == nil || b.conn.IsClosed() { + return errors.New("rabbitmq current connection is closed") + } + channel, err := b.conn.Channel() + if err != nil { + return fmt.Errorf("open current publication channel: %w", err) + } + defer channel.Close() + if err := channel.Confirm(false); err != nil { + return fmt.Errorf("enable current publisher confirms: %w", err) + } + returned := channel.NotifyReturn(make(chan amqp.Return, 1)) + confirmation, err := channel.PublishWithDeferredConfirmWithContext(ctx, exchange, routingKey, true, false, amqp.Publishing{ + ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body, MessageId: messageID, + }) + if err != nil { + return fmt.Errorf("publish current result: %w", err) + } + if confirmation == nil { + return errors.New("rabbitmq current publisher confirmation unavailable") + } + acked, err := confirmation.WaitContext(ctx) + if err != nil { + return fmt.Errorf("wait for current publisher confirmation: %w", err) + } + select { + case result, ok := <-returned: + if !ok { + return errors.New("current publication channel closed before routing was established") + } + return fmt.Errorf("current publication returned: code=%d", result.ReplyCode) + default: + } + if !acked { + return errors.New("rabbitmq current publisher was negatively acknowledged") + } + return nil +} + +type CurrentConsumer struct { + cancel context.CancelFunc + done chan struct{} + mu sync.Mutex + err error +} + +func (c *CurrentConsumer) Wait(ctx context.Context) error { + select { + case <-c.done: + c.mu.Lock() + defer c.mu.Unlock() + return c.err + case <-ctx.Done(): + return ctx.Err() + } +} + +// RequestStop cancels consumption without waiting for the current handler. +// Calling Stop from inside that handler would deadlock on its own completion. +func (c *CurrentConsumer) RequestStop() { + c.cancel() +} + +func (c *CurrentConsumer) Stop(ctx context.Context) error { + c.cancel() + err := c.Wait(ctx) + if errors.Is(err, context.Canceled) { + return nil + } + return err +} + +func (b *CurrentBroker) ConsumePredeclared(ctx context.Context, queue string, handler MessageHandler) error { + consumer, err := b.StartPredeclaredConsumer(ctx, queue, handler) + if err != nil { + return err + } + err = consumer.Wait(context.Background()) + if err != nil { + return err + } + return ctx.Err() +} + +// StartPredeclaredConsumer subscribes only to an existing SaaS-owned queue. +// Stopping it closes the channel so every unacknowledged delivery is requeued. +func (b *CurrentBroker) StartPredeclaredConsumer(ctx context.Context, queue string, handler MessageHandler) (*CurrentConsumer, error) { + if handler == nil { + return nil, errors.New("predeclared handler is required") + } + if err := b.validateQueue(queue, true); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + b.mu.Lock() + conn, prefetch, dispatcherID := b.conn, b.prefetch, b.dispatcherID + closed := conn == nil || conn.IsClosed() + b.mu.Unlock() + if closed { + return nil, errors.New("rabbitmq current connection is closed") + } + channel, err := conn.Channel() + if err != nil { + return nil, fmt.Errorf("open current consumer channel: %w", err) + } + if _, err := channel.QueueDeclarePassive(queue, true, false, false, false, nil); err != nil { + _ = channel.Close() + return nil, fmt.Errorf("required SaaS-owned queue %s unavailable: %w", queue, err) + } + if err := channel.Qos(prefetch, 0, false); err != nil { + _ = channel.Close() + return nil, fmt.Errorf("set current consumer prefetch: %w", err) + } + consumerTag := fmt.Sprintf("sip-go-agent-current-%d", consumerSequence.Add(1)) + deliveries, err := channel.Consume(queue, consumerTag, false, false, false, false, nil) + if err != nil { + _ = channel.Close() + return nil, fmt.Errorf("consume SaaS-owned current queue: %w", err) + } + consumerCtx, cancel := context.WithCancel(ctx) + consumer := &CurrentConsumer{cancel: cancel, done: make(chan struct{})} + go func() { + consumeErr := b.consumeCurrentDeliveries(consumerCtx, dispatcherID, queue, deliveries, handler) + if errors.Is(consumeErr, context.Canceled) && consumerCtx.Err() != nil { + consumeErr = nil + } + if err := channel.Cancel(consumerTag, false); err != nil { + consumeErr = errors.Join(consumeErr, fmt.Errorf("cancel current consumer: %w", err)) + } + if err := channel.Close(); err != nil { + consumeErr = errors.Join(consumeErr, fmt.Errorf("close current consumer channel: %w", err)) + } + consumer.mu.Lock() + consumer.err = consumeErr + consumer.mu.Unlock() + close(consumer.done) + }() + return consumer, nil +} + +func (b *CurrentBroker) DrainPredeclared(ctx context.Context, queue string) (int, error) { + if err := b.validateQueue(queue, false); err != nil { + return 0, err + } + if err := ctx.Err(); err != nil { + return 0, err + } + b.mu.Lock() + conn := b.conn + closed := conn == nil || conn.IsClosed() + b.mu.Unlock() + if closed { + return 0, errors.New("rabbitmq current connection is closed") + } + channel, err := conn.Channel() + if err != nil { + return 0, fmt.Errorf("open current drain channel: %w", err) + } + defer channel.Close() + if _, err := channel.QueueDeclarePassive(queue, true, false, false, false, nil); err != nil { + return 0, fmt.Errorf("required SaaS-owned queue %s unavailable: %w", queue, err) + } + drained := 0 + for { + if err := ctx.Err(); err != nil { + return drained, err + } + delivery, ok, err := channel.Get(queue, false) + if err != nil { + return drained, fmt.Errorf("read SaaS-owned queue %s for stop drain: %w", queue, err) + } + if !ok { + return drained, nil + } + if err := delivery.Ack(false); err != nil { + return drained, fmt.Errorf("ack stopped task backlog: %w", err) + } + drained++ + } +} + +// DrainControlPredeclared processes the already queued controls before task admission. +// Unlike stopped task backlogs, control deliveries must pass through the handler +// before ACK; a transient failure is requeued and closes startup admission. +func (b *CurrentBroker) DrainControlPredeclared(ctx context.Context, queue string, handler MessageHandler) (int, error) { + if queue == "" || queue != b.controlQueue || handler == nil { + return 0, errors.New("configured SaaS-owned control queue and handler are required") + } + if err := ctx.Err(); err != nil { + return 0, err + } + b.mu.Lock() + conn := b.conn + closed := conn == nil || conn.IsClosed() + b.mu.Unlock() + if closed { + return 0, errors.New("rabbitmq current connection is closed") + } + channel, err := conn.Channel() + if err != nil { + return 0, fmt.Errorf("open control backlog channel: %w", err) + } + defer channel.Close() + if _, err := channel.QueueDeclarePassive(queue, true, false, false, false, nil); err != nil { + return 0, fmt.Errorf("required SaaS-owned control queue %s unavailable: %w", queue, err) + } + processed := 0 + for { + if err := ctx.Err(); err != nil { + return processed, err + } + delivery, ok, err := channel.Get(queue, false) + if err != nil { + return processed, fmt.Errorf("read SaaS-owned control backlog %s: %w", queue, err) + } + if !ok { + return processed, nil + } + if len(delivery.Body) == 0 || len(delivery.Body) > MaxCurrentMessageBytes || !json.Valid(delivery.Body) { + if err := delivery.Reject(false); err != nil { + return processed, fmt.Errorf("reject invalid control backlog message: %w", err) + } + slog.Error("MQ control backlog invalid; admission stays closed", "dispatcher_id", b.dispatcherID, "delivery_tag", delivery.DeliveryTag, "reason", "invalid_size_or_json", "bytes", len(delivery.Body)) + processed++ + return processed, errors.New("invalid control backlog message; admission stays closed") + } + if err := b.validateInbound(queue, delivery.RoutingKey, delivery.Body); err != nil { + if rejectErr := delivery.Reject(false); rejectErr != nil { + return processed, fmt.Errorf("invalid control message %v; reject: %w", err, rejectErr) + } + slog.Error("MQ control backlog invalid; admission stays closed", "dispatcher_id", b.dispatcherID, "delivery_tag", delivery.DeliveryTag, "reason", err) + processed++ + return processed, fmt.Errorf("invalid control backlog message; admission stays closed: %w", err) + } + if err := handler(ctx, delivery.RoutingKey, delivery.Body); err != nil { + if IsPermanent(err) { + if rejectErr := delivery.Reject(false); rejectErr != nil { + return processed, fmt.Errorf("control handler error %v; reject: %w", err, rejectErr) + } + processed++ + return processed, fmt.Errorf("control backlog rejected permanently; admission stays closed: %w", err) + } + if nackErr := delivery.Nack(false, true); nackErr != nil { + return processed, fmt.Errorf("control handler error %v; nack: %w", err, nackErr) + } + return processed, fmt.Errorf("control backlog handler failed, admission stays closed: %w", err) + } + if err := delivery.Ack(false); err != nil { + return processed, fmt.Errorf("ack processed control backlog: %w", err) + } + processed++ + } +} + +func (b *CurrentBroker) ControlQueue() string { return b.controlQueue } + +// validateInbound prevents a wrongly bound queue, foreign D message, old +// envelope, or outbound acknowledgment from entering the command handler. +func (b *CurrentBroker) validateInbound(queue, routingKey string, body []byte) error { + if len(body) == 0 || len(body) > MaxCurrentMessageBytes { + return errors.New("invalid MQ message size") + } + if err := b.validateQueue(queue, true); err != nil { + return err + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return fmt.Errorf("inbound MQ contract: %w", err) + } + var identity struct { + DispatcherID string `json:"dispatcher_id"` + EventType string `json:"event_type"` + Payload struct { + TaskID string `json:"task_id"` + Action string `json:"action"` + Callee string `json:"callee"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &identity); err != nil { + return fmt.Errorf("decode MQ message identity: %w", err) + } + if identity.DispatcherID != b.dispatcherID { + return errors.New("MQ message belongs to another Dispatcher") + } + if queue == b.controlQueue { + if routingKey != "d."+b.dispatcherID+".control.in" { + return errors.New("control message used wrong routing key") + } + if identity.EventType == "sip.config" || (identity.EventType == "task.control" && identity.Payload.TaskID != "" && identity.Payload.Action != "") { + return nil + } + return errors.New("unapproved control-queue event") + } + prefix := "agent-call.d." + b.dispatcherID + ".task." + taskID := strings.TrimSuffix(strings.TrimPrefix(queue, prefix), ".v1") + route, err := tenant.CurrentTaskRoute(b.dispatcherID, taskID) + if err != nil || routingKey != route.BindingKey || identity.EventType != "call.execute" || identity.Payload.TaskID != taskID || identity.Payload.Callee == "" { + return errors.New("unapproved task-queue event or routing key") + } + return nil +} + +func (b *CurrentBroker) validateQueue(queue string, allowControl bool) error { + if allowControl && queue == b.controlQueue { + return nil + } + prefix := "agent-call.d." + b.dispatcherID + ".task." + if !strings.HasPrefix(queue, prefix) || !strings.HasSuffix(queue, ".v1") { + return errors.New("queue is not owned by this Dispatcher task") + } + taskID := strings.TrimSuffix(strings.TrimPrefix(queue, prefix), ".v1") + route, err := tenant.CurrentTaskRoute(b.dispatcherID, taskID) + if err != nil || route.Queue != queue { + return errors.New("invalid assigned task queue") + } + return nil +} + +func (b *CurrentBroker) consumeCurrentDeliveries(ctx context.Context, dispatcherID, queue string, deliveries <-chan amqp.Delivery, handler MessageHandler) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case delivery, ok := <-deliveries: + if !ok { + return errors.New("rabbitmq current delivery channel closed") + } + if len(delivery.Body) == 0 || len(delivery.Body) > MaxCurrentMessageBytes { + if err := delivery.Reject(false); err != nil { + return fmt.Errorf("reject invalid current message size: %w", err) + } + slog.Warn("MQ current message rejected", "dispatcher_id", dispatcherID, "delivery_tag", delivery.DeliveryTag, "reason", "invalid_message_size", "bytes", len(delivery.Body)) + if queue == b.controlQueue { + return errors.New("invalid control message size; admission stays closed") + } + continue + } + if !json.Valid(delivery.Body) { + if err := delivery.Reject(false); err != nil { + return fmt.Errorf("reject invalid current JSON: %w", err) + } + slog.Warn("MQ current message rejected", "dispatcher_id", dispatcherID, "delivery_tag", delivery.DeliveryTag, "reason", "invalid_json") + if queue == b.controlQueue { + return errors.New("invalid control JSON; admission stays closed") + } + continue + } + if err := b.validateInbound(queue, delivery.RoutingKey, delivery.Body); err != nil { + if rejectErr := delivery.Reject(false); rejectErr != nil { + return fmt.Errorf("invalid inbound MQ message %v; reject: %w", err, rejectErr) + } + slog.Warn("MQ current message rejected", "dispatcher_id", dispatcherID, "delivery_tag", delivery.DeliveryTag, "reason", err) + if queue == b.controlQueue { + return fmt.Errorf("invalid control message; admission stays closed: %w", err) + } + continue + } + if err := handler(ctx, delivery.RoutingKey, delivery.Body); err != nil { + if IsPermanent(err) { + if rejectErr := delivery.Reject(false); rejectErr != nil { + return fmt.Errorf("permanent current handler error %v; reject: %w", err, rejectErr) + } + if queue == b.controlQueue { + return fmt.Errorf("control handler rejected permanently; admission stays closed: %w", err) + } + continue + } + if nackErr := delivery.Nack(false, true); nackErr != nil { + return fmt.Errorf("current handler error %v; nack: %w", err, nackErr) + } + if queue == b.controlQueue { + return fmt.Errorf("control handler failed; admission stays closed: %w", err) + } + continue + } + if err := delivery.Ack(false); err != nil { + return fmt.Errorf("ack current delivery: %w", err) + } + } + } +} diff --git a/internal/mq/current_inbound_test.go b/internal/mq/current_inbound_test.go new file mode 100644 index 0000000..aaee1d0 --- /dev/null +++ b/internal/mq/current_inbound_test.go @@ -0,0 +1,58 @@ +package mq + +import ( + "strings" + "testing" + + "git.ipao.vip/rogee/go-sip/internal/tenant" +) + +func TestCurrentBrokerInboundRequiresExactQueueRouteOwnerAndSchema(t *testing.T) { + control, err := tenant.CurrentControlRoute(currentTestDispatcherID) + if err != nil { + t.Fatal(err) + } + task, err := tenant.CurrentTaskRoute(currentTestDispatcherID, "task-asr") + if err != nil { + t.Fatal(err) + } + broker := &CurrentBroker{dispatcherID: currentTestDispatcherID, controlQueue: control.Queue} + for _, tc := range []struct { + name, queue, route, fixture string + wantError bool + }{ + {"control", control.Queue, control.BindingKey, "mq-control", false}, + {"sip change", control.Queue, control.BindingKey, "mq-sip-change", false}, + {"execute", task.Queue, task.BindingKey, "mq-execute", false}, + {"wrong queue", "agent-call.d.c046b893-8628-4589-ae50-619d049248a7.task.task-asr.v1", task.BindingKey, "mq-execute", true}, + {"wrong route", task.Queue, control.BindingKey, "mq-execute", true}, + {"execute on control", control.Queue, control.BindingKey, "mq-execute", true}, + {"control on task", task.Queue, task.BindingKey, "mq-control", true}, + {"old schema", task.Queue, task.BindingKey, "invalid/mq-legacy-schema-version", true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := broker.validateInbound(tc.queue, tc.route, currentMessage(t, tc.fixture)) + if (err != nil) != tc.wantError { + t.Fatalf("wanted error=%v, got %v", tc.wantError, err) + } + }) + } + foreign := []byte(strings.Replace(string(currentMessage(t, "mq-execute")), currentTestDispatcherID, "c046b893-8628-4589-ae50-619d049248a7", 1)) + if err := broker.validateInbound(task.Queue, task.BindingKey, foreign); err == nil { + t.Fatal("accepted wrong Dispatcher owner") + } +} + +func TestCurrentBrokerRejectsControlQueueFromSilentTaskDrain(t *testing.T) { + control, err := tenant.CurrentControlRoute(currentTestDispatcherID) + if err != nil { + t.Fatal(err) + } + broker := &CurrentBroker{dispatcherID: currentTestDispatcherID, controlQueue: control.Queue} + if err := broker.validateQueue(control.Queue, false); err == nil { + t.Fatal("allowed task-drain to silently ACK controls") + } + if err := broker.validateQueue(control.Queue, true); err != nil { + t.Fatalf("control consumer cannot read own queue: %v", err) + } +} diff --git a/internal/mq/current_integration_test.go b/internal/mq/current_integration_test.go new file mode 100644 index 0000000..0c7be66 --- /dev/null +++ b/internal/mq/current_integration_test.go @@ -0,0 +1,203 @@ +//go:build integration + +package mq + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/tenant" + amqp "github.com/rabbitmq/amqp091-go" +) + +func TestCurrentBrokerSharedResultQueueAndNoConfigure(t *testing.T) { + url := os.Getenv("RABBITMQ_URL") + provisionerURL := os.Getenv("RABBITMQ_PROVISIONER_URL") + if url == "" || provisionerURL == "" { + t.Skip("isolated RabbitMQ Mock URLs not configured") + } + ids := []string{currentTestDispatcherID, "550e8400-e29b-41d4-a716-446655440000"} + conn, err := amqp.Dial(provisionerURL) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + admin, err := conn.Channel() + if err != nil { + t.Fatal(err) + } + defer admin.Close() + for _, exchange := range []string{CommandsExchangeCurrent, ResultsExchangeCurrent, DeadLetterExchangeCurrent} { + if err := admin.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil { + t.Fatal(err) + } + } + shared := "agent-call.saas.events.v1" + if _, err := admin.QueueDeclare(shared, true, false, false, false, nil); err != nil { + t.Fatal(err) + } + defer func() { _, _ = admin.QueueDelete(shared, false, false, false) }() + queues := []string{} + for _, id := range ids { + control, _ := tenant.CurrentControlRoute(id) + task, _ := tenant.CurrentTaskRoute(id, "task-asr") + result, _ := tenant.CurrentResultRoute(id) + for _, route := range []tenant.CurrentRoute{control, task} { + if _, err := admin.QueueDeclare(route.Queue, true, false, false, false, nil); err != nil { + t.Fatal(err) + } + queues = append(queues, route.Queue) + if err := admin.QueueBind(route.Queue, route.BindingKey, route.Exchange, false, nil); err != nil { + t.Fatal(err) + } + } + if err := admin.QueueBind(shared, result.BindingKey, result.Exchange, false, nil); err != nil { + t.Fatal(err) + } + defer func(key string) { _ = admin.QueueUnbind(shared, key, ResultsExchangeCurrent, nil) }(result.BindingKey) + } + defer func() { + for _, name := range queues { + _, _ = admin.QueueDelete(name, false, false, false) + } + }() + for _, name := range append(append([]string(nil), queues...), shared) { + if _, err := admin.QueuePurge(name, false); err != nil { + t.Fatal(err) + } + } + + // The Dispatcher account must not configure the topology, including the + // SaaS result queue. A failed declare closes its channel, not the connection. + probeConn, err := amqp.Dial(url) + if err != nil { + t.Fatal(err) + } + probe, err := probeConn.Channel() + if err != nil { + t.Fatal(err) + } + if _, err := probe.QueueDeclare("should-never-be-created", true, false, false, false, nil); err == nil { + _, _ = admin.QueueDelete("should-never-be-created", false, false, false) + t.Fatal("Dispatcher unexpectedly has configure permission") + } + _ = probe.Close() + _ = probeConn.Close() + + brokers := make([]*CurrentBroker, 0, 2) + for _, id := range ids { + broker, err := OpenCurrent(url, id, 1) + if err != nil { + t.Fatalf("Dispatcher without configure could not open %s: %v", id, err) + } + brokers = append(brokers, broker) + defer broker.Close() + } + control, _ := tenant.CurrentControlRoute(ids[0]) + task, _ := tenant.CurrentTaskRoute(ids[0], "task-asr") + controlBody := currentMessage(t, "mq-control") + if err := admin.PublishWithContext(context.Background(), CommandsExchangeCurrent, control.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: controlBody}); err != nil { + t.Fatal(err) + } + processed, err := brokers[0].DrainControlPredeclared(context.Background(), control.Queue, func(_ context.Context, key string, body []byte) error { + if key != control.BindingKey || string(body) != string(controlBody) { + return fmt.Errorf("wrong control route or body") + } + return nil + }) + if err != nil || processed != 1 { + t.Fatalf("control backlog: processed=%d err=%v", processed, err) + } + body := currentMessage(t, "mq-execute") + if err := admin.PublishWithContext(context.Background(), CommandsExchangeCurrent, task.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + got := make(chan []byte, 1) + done := make(chan error, 1) + go func() { + done <- brokers[0].ConsumePredeclared(ctx, task.Queue, func(_ context.Context, key string, message []byte) error { + if key != task.BindingKey { + return fmt.Errorf("wrong task routing key %q", key) + } + got <- append([]byte(nil), message...) + return nil + }) + }() + select { + case message := <-got: + if string(message) != string(body) { + t.Errorf("received wrong task body") + } + case err := <-done: + t.Fatalf("task consumer ended: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("task message not received") + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("consumer did not stop") + } + otherTask, _ := tenant.CurrentTaskRoute(ids[1], "task-asr") + if state, err := admin.QueueInspect(otherTask.Queue); err != nil || state.Messages != 0 { + t.Fatalf("D2 stole D1 command: %+v %v", state, err) + } + + resultBody := currentMessage(t, "mq-result-no-recording") + for i, broker := range brokers { + message := resultBody + if i == 1 { + message = []byte(strings.Replace(string(resultBody), ids[0], ids[1], 1)) + } + route, _ := tenant.CurrentResultRoute(ids[i]) + if err := broker.Publish(context.Background(), route.Exchange, route.BindingKey, message); err != nil { + t.Fatalf("result publish for D%d: %v", i+1, err) + } + } + seen := map[string]bool{} + for range ids { + delivery, ok, err := admin.Get(shared, false) + if err != nil || !ok { + t.Fatalf("shared SaaS queue missing confirmed result: %v", err) + } + seen[delivery.RoutingKey] = true + if err := delivery.Ack(false); err != nil { + t.Fatal(err) + } + } + for _, id := range ids { + if !seen["d."+id+".out"] { + t.Fatalf("missing D-specific output binding for %s", id) + } + } + for _, id := range ids { + route, _ := tenant.CurrentResultRoute(id) + if err := admin.QueueUnbind(shared, route.BindingKey, route.Exchange, nil); err != nil { + t.Fatal(err) + } + } + resultRoute, _ := tenant.CurrentResultRoute(ids[0]) + if err := brokers[0].Publish(context.Background(), resultRoute.Exchange, resultRoute.BindingKey, resultBody); err == nil { + t.Fatal("unroutable mandatory result falsely confirmed") + } + controlRoute, err := tenant.CurrentControlRoute(ids[0]) + if err != nil { + t.Fatal(err) + } + if err := admin.PublishWithContext(context.Background(), controlRoute.Exchange, controlRoute.BindingKey, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: []byte("{")}); err != nil { + t.Fatal(err) + } + processed, err = brokers[0].DrainControlPredeclared(context.Background(), controlRoute.Queue, func(context.Context, string, []byte) error { + t.Fatal("invalid backlog control reached handler") + return nil + }) + if err == nil || processed != 1 || !strings.Contains(err.Error(), "invalid control backlog") { + t.Fatalf("malformed control backlog was silently cleared: processed=%d err=%v", processed, err) + } +} diff --git a/internal/mq/current_pause_test.go b/internal/mq/current_pause_test.go new file mode 100644 index 0000000..007784b --- /dev/null +++ b/internal/mq/current_pause_test.go @@ -0,0 +1,45 @@ +package mq + +import ( + "context" + "strings" + "testing" + + "git.ipao.vip/rogee/go-sip/internal/tenant" + amqp "github.com/rabbitmq/amqp091-go" +) + +func TestCurrentConsumerRequestStopDoesNotWaitForItself(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + consumer := &CurrentConsumer{cancel: cancel, done: make(chan struct{})} + consumer.RequestStop() + select { + case <-ctx.Done(): + default: + t.Fatal("consumer cancellation was not requested") + } + consumer.RequestStop() +} + +type currentRejectRecorder struct{ rejected int } + +func (*currentRejectRecorder) Ack(uint64, bool) error { return nil } +func (*currentRejectRecorder) Nack(uint64, bool, bool) error { return nil } +func (a *currentRejectRecorder) Reject(uint64, bool) error { a.rejected++; return nil } + +func TestCurrentMalformedControlStopsConsumerAfterReject(t *testing.T) { + id := "c046b893-8628-4589-ae50-619d049248a6" + route, err := tenant.CurrentControlRoute(id) + if err != nil { + t.Fatal(err) + } + broker := &CurrentBroker{dispatcherID: id, controlQueue: route.Queue} + ack := ¤tRejectRecorder{} + deliveries := make(chan amqp.Delivery, 1) + deliveries <- amqp.Delivery{Body: []byte("{"), RoutingKey: route.BindingKey, Acknowledger: ack} + close(deliveries) + err = broker.consumeCurrentDeliveries(context.Background(), id, route.Queue, deliveries, func(context.Context, string, []byte) error { t.Fatal("invalid control reached handler"); return nil }) + if err == nil || !strings.Contains(err.Error(), "invalid control") || ack.rejected != 1 { + t.Fatalf("malformed control was silently discarded before admission: rejected=%d err=%v", ack.rejected, err) + } +} diff --git a/internal/mq/current_test.go b/internal/mq/current_test.go new file mode 100644 index 0000000..fe750f0 --- /dev/null +++ b/internal/mq/current_test.go @@ -0,0 +1,59 @@ +package mq + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const currentTestDispatcherID = "c046b893-8628-4589-ae50-619d049248a6" + +func currentMessage(t *testing.T, name string) []byte { + t.Helper() + body, err := os.ReadFile(filepath.Join("..", "..", "contracts", "local", "examples", name+".json")) + if err != nil { + t.Fatal(err) + } + return body +} + +func TestCurrentBrokerRejectsInvalidConfigBeforeConnecting(t *testing.T) { + for _, tc := range []struct { + url, id string + prefetch int + }{ + {"", currentTestDispatcherID, 1}, + {"amqp://127.0.0.1:1", "not-a-uuid", 1}, + {"amqp://127.0.0.1:1", currentTestDispatcherID, 0}, + } { + if _, err := OpenCurrent(tc.url, tc.id, tc.prefetch); err == nil { + t.Fatalf("accepted invalid broker settings: %+v", tc) + } + } +} + +func TestCurrentPublishRejectsLegacyAndWrongOwnerBeforeNetwork(t *testing.T) { + b := &CurrentBroker{dispatcherID: currentTestDispatcherID, resultRoute: "d." + currentTestDispatcherID + ".out"} + for _, tc := range []struct { + name string + body []byte + }{ + {"old version field", currentMessage(t, "invalid/mq-legacy-schema-version")}, + {"wrong owner", []byte(strings.Replace(string(currentMessage(t, "mq-result-no-recording")), currentTestDispatcherID, "c046b893-8628-4589-ae50-619d049248a7", 1))}, + {"unapproved event", currentMessage(t, "mq-sip-change")}, + {"empty event identity", []byte(strings.Replace(string(currentMessage(t, "mq-execute-ack")), `"event_id":"call-example"`, `"event_id":""`, 1))}, + } { + t.Run(tc.name, func(t *testing.T) { + err := b.Publish(context.Background(), "agent-call.saas.v1", b.resultRoute, tc.body) + if err == nil || strings.Contains(err.Error(), "connection is closed") { + t.Fatalf("message reached network path instead of being rejected: %v", err) + } + }) + } + valid := currentMessage(t, "mq-result-no-recording") + if err := b.Publish(context.Background(), "agent-call.saas.v1", b.resultRoute, valid); err == nil || !strings.Contains(err.Error(), "connection is closed") { + t.Fatalf("valid result did not reach network boundary: %v", err) + } +} diff --git a/internal/store/current.go b/internal/store/current.go index ded443c..91c5419 100644 --- a/internal/store/current.go +++ b/internal/store/current.go @@ -92,6 +92,7 @@ func OpenCurrent(path string) (_ *CurrentStore, err error) { return nil, fmt.Errorf("inspect SQLite schema before writing: %w", err) } allowed := map[string]bool{"dispatcher_state": true, "dispatcher_tasks": true, "dispatcher_configs": true, "dispatcher_inbox": true, "dispatcher_outbox": true} + seen := make(map[string]bool, len(allowed)) var unexpected []string for rows.Next() { var name string @@ -101,6 +102,8 @@ func OpenCurrent(path string) (_ *CurrentStore, err error) { } if !allowed[name] { unexpected = append(unexpected, name) + } else { + seen[name] = true } } err = rows.Err() @@ -111,6 +114,26 @@ func OpenCurrent(path string) (_ *CurrentStore, err error) { if len(unexpected) > 0 { return nil, fmt.Errorf("old or unknown SQLite tables %s: preserve database and reconcile manually before admission", strings.Join(unexpected, ",")) } + var version int + if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil { + return nil, fmt.Errorf("inspect SQLite layout version before writing: %w", err) + } + if len(seen) == 0 { + if version != 0 { + return nil, fmt.Errorf("unknown SQLite layout version %d without tables: preserve database before admission", version) + } + } else { + if len(seen) != len(allowed) || version != 1 { + return nil, fmt.Errorf("existing SQLite layout is incomplete or obsolete (tables=%d, version=%d): preserve data before admission", len(seen), version) + } + var requiredColumns int + if err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('dispatcher_state') WHERE name IN ('discovery_ready','applied_sip_revision','pending_sip_revision')`).Scan(&requiredColumns); err != nil { + return nil, fmt.Errorf("inspect durable SIP state layout: %w", err) + } + if requiredColumns != 3 { + return nil, errors.New("existing SQLite lacks durable SIP admission columns: preserve data before admission") + } + } for _, pragma := range []string{"PRAGMA busy_timeout=5000", "PRAGMA journal_mode=WAL", "PRAGMA synchronous=FULL", "PRAGMA foreign_keys=ON"} { if _, err := db.Exec(pragma); err != nil { return nil, fmt.Errorf("initialize SQLite %s: %w", pragma, err) @@ -119,6 +142,11 @@ func OpenCurrent(path string) (_ *CurrentStore, err error) { if _, err := db.Exec(currentSchema); err != nil { return nil, fmt.Errorf("create current SQLite schema: %w", err) } + if version == 0 { + if _, err := db.Exec(`PRAGMA user_version=1`); err != nil { + return nil, fmt.Errorf("commit current SQLite layout version: %w", err) + } + } return &CurrentStore{db: db}, nil } diff --git a/internal/store/current_calls.go b/internal/store/current_calls.go new file mode 100644 index 0000000..7588565 --- /dev/null +++ b/internal/store/current_calls.go @@ -0,0 +1,402 @@ +package store + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/contract" + "git.ipao.vip/rogee/go-sip/internal/tenant" +) + +var ( + ErrCurrentCapacity = errors.New("current task, tenant, or trunk capacity exhausted") + ErrCurrentNotReady = errors.New("current task admission is not ready") + ErrCurrentAlreadyStarted = errors.New("call instruction already began dispatch") + ErrCurrentStopped = errors.New("stopped task does not accept old call instructions") +) + +type CurrentExecuteCommand struct { + DispatcherID string + EventID string + TenantID int64 + TaskID string + Callee string + IssuedAt string +} + +type CurrentExecution struct { + CurrentExecuteCommand + Status string + SelectedTrunkID string +} + +type CurrentCallReservation struct { + TrunkID string + SIPRevision int64 + CallerID string + DialedCallee string + Deadline time.Time +} + +type CurrentOutboxEvent struct { + EventID string + EventType string + RoutingKey string + Body []byte +} + +// RecordExecute durably accepts the transport message before MQ ACK. A +// redelivery of the same message identity never creates another execution. +func (s *CurrentStore) RecordExecute(cmd CurrentExecuteCommand) (CurrentExecution, bool, error) { + if cmd.DispatcherID == "" || cmd.EventID == "" || len(cmd.EventID) > 255 || cmd.TenantID <= 0 || cmd.TaskID == "" || cmd.Callee == "" { + return CurrentExecution{}, false, errors.New("invalid call command identity or callee") + } + if _, err := time.Parse(time.RFC3339Nano, cmd.IssuedAt); err != nil { + return CurrentExecution{}, false, fmt.Errorf("invalid call issued_at: %w", err) + } + tx, err := s.db.Begin() + if err != nil { + return CurrentExecution{}, false, err + } + defer tx.Rollback() + var controlState string + var present int + err = tx.QueryRow(`SELECT control_state,present FROM dispatcher_tasks WHERE dispatcher_id=? AND tenant_id=? AND task_id=?`, cmd.DispatcherID, cmd.TenantID, cmd.TaskID).Scan(&controlState, &present) + if errors.Is(err, sql.ErrNoRows) || present != 1 { + return CurrentExecution{}, false, ErrCurrentNotReady + } + if err != nil { + return CurrentExecution{}, false, fmt.Errorf("read call task ownership: %w", err) + } + if controlState == "stopped" || controlState == "stopping" { + return CurrentExecution{}, false, ErrCurrentStopped + } + result, err := tx.Exec(`INSERT INTO dispatcher_inbox(dispatcher_id,event_id,tenant_id,task_id,callee,issued_at,status) + VALUES(?,?,?,?,?,?,'pending') ON CONFLICT(dispatcher_id,event_id) DO NOTHING`, cmd.DispatcherID, cmd.EventID, cmd.TenantID, cmd.TaskID, cmd.Callee, cmd.IssuedAt) + if err != nil { + return CurrentExecution{}, false, fmt.Errorf("persist call inbox: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return CurrentExecution{}, false, err + } + var found CurrentExecution + err = tx.QueryRow(`SELECT dispatcher_id,event_id,tenant_id,task_id,callee,issued_at,status,COALESCE(selected_trunk_id,'') + FROM dispatcher_inbox WHERE dispatcher_id=? AND event_id=?`, cmd.DispatcherID, cmd.EventID).Scan(&found.DispatcherID, &found.EventID, &found.TenantID, &found.TaskID, &found.Callee, &found.IssuedAt, &found.Status, &found.SelectedTrunkID) + if err != nil { + return CurrentExecution{}, false, fmt.Errorf("read call inbox: %w", err) + } + if found.CurrentExecuteCommand != cmd { + return CurrentExecution{}, false, fmt.Errorf("conflicting call message identity %q", cmd.EventID) + } + if err := tx.Commit(); err != nil { + return CurrentExecution{}, false, fmt.Errorf("commit call inbox: %w", err) + } + return found, affected == 1, nil +} + +// ReserveExecute is the final transactional quota/fencing boundary before +// sending a single originate. A crash or timeout after this point leaves an +// occupied dispatching/unknown execution, never an automatic redial. +func (s *CurrentStore) ReserveExecute(dispatcherID, eventID string, selected CurrentCallReservation, at time.Time) error { + if dispatcherID == "" || eventID == "" || selected.TrunkID == "" || selected.SIPRevision <= 0 || selected.CallerID == "" || selected.DialedCallee == "" || !selected.Deadline.After(at) { + return errors.New("invalid originate reservation or expired deadline") + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var cmd CurrentExecuteCommand + var status string + err = tx.QueryRow(`SELECT tenant_id,task_id,callee,issued_at,status FROM dispatcher_inbox WHERE dispatcher_id=? AND event_id=?`, dispatcherID, eventID).Scan(&cmd.TenantID, &cmd.TaskID, &cmd.Callee, &cmd.IssuedAt, &status) + if err != nil { + return fmt.Errorf("load durable call instruction: %w", err) + } + if status != "pending" { + return ErrCurrentAlreadyStarted + } + cmd.DispatcherID, cmd.EventID = dispatcherID, eventID + issued, err := time.Parse(time.RFC3339Nano, cmd.IssuedAt) + if err != nil { + return fmt.Errorf("decode persisted call issued_at: %w", err) + } + if issued.After(at) { + return fmt.Errorf("%w: call issued_at is in the future", ErrCurrentNotReady) + } + var body []byte + var revision int64 + err = tx.QueryRow(`SELECT c.snapshot_json,c.sip_revision FROM dispatcher_tasks t + JOIN dispatcher_state ds ON ds.dispatcher_id=t.dispatcher_id + JOIN dispatcher_configs c ON c.dispatcher_id=t.dispatcher_id AND c.tenant_id=t.tenant_id AND c.task_id=t.task_id AND c.task_revision=t.task_revision + WHERE t.dispatcher_id=? AND t.tenant_id=? AND t.task_id=? AND t.present=1 AND t.status='running' AND t.control_state='' AND ds.discovery_ready=1`, dispatcherID, cmd.TenantID, cmd.TaskID).Scan(&body, &revision) + if errors.Is(err, sql.ErrNoRows) { + return ErrCurrentNotReady + } + if err != nil { + return fmt.Errorf("load current task admission: %w", err) + } + if revision != selected.SIPRevision { + return fmt.Errorf("%w: SIP revision changed", ErrCurrentNotReady) + } + var snapshot struct { + Task configread.CurrentTask `json:"task"` + SIP configread.CurrentSIP `json:"sip"` + Quota configread.CurrentQuota `json:"quota"` + } + if err := json.Unmarshal(body, &snapshot); err != nil { + return fmt.Errorf("decode admission snapshot: %w", err) + } + if snapshot.Task.DispatcherID != dispatcherID || snapshot.Task.TenantID != cmd.TenantID || snapshot.Task.TaskID != cmd.TaskID || snapshot.SIP.Revision != revision || snapshot.Quota.TenantID != cmd.TenantID { + return errors.New("admission snapshot identity mismatch") + } + if snapshot.Task.MaxConcurrentCalls <= 0 || snapshot.Quota.MaxConcurrentCalls <= 0 { + return fmt.Errorf("%w: missing quota", ErrCurrentNotReady) + } + var trunks []struct { + TrunkID string `json:"trunk_id"` + Enabled bool `json:"enabled"` + MaxConcurrentCalls *int64 `json:"max_concurrent_calls"` + } + if err := json.Unmarshal(snapshot.SIP.Trunks, &trunks); err != nil { + return fmt.Errorf("decode selected SIP trunk: %w", err) + } + var trunkLimit int64 + for _, trunk := range trunks { + if trunk.TrunkID == selected.TrunkID && trunk.Enabled && trunk.MaxConcurrentCalls != nil { + trunkLimit = *trunk.MaxConcurrentCalls + break + } + } + if trunkLimit <= 0 { + return fmt.Errorf("%w: selected trunk is disabled or has unknown quota", ErrCurrentNotReady) + } + count := func(query string, args ...any) (int64, error) { + var n int64 + err := tx.QueryRow(query, args...).Scan(&n) + return n, err + } + const occupied = `status IN ('dispatching','dispatched','unknown')` + tenantUsed, err := count(`SELECT COUNT(*) FROM dispatcher_inbox WHERE dispatcher_id=? AND tenant_id=? AND `+occupied, dispatcherID, cmd.TenantID) + if err != nil { + return fmt.Errorf("count tenant occupancy: %w", err) + } + taskUsed, err := count(`SELECT COUNT(*) FROM dispatcher_inbox WHERE dispatcher_id=? AND tenant_id=? AND task_id=? AND `+occupied, dispatcherID, cmd.TenantID, cmd.TaskID) + if err != nil { + return fmt.Errorf("count task occupancy: %w", err) + } + trunkUsed, err := count(`SELECT COUNT(*) FROM dispatcher_inbox WHERE dispatcher_id=? AND selected_trunk_id=? AND `+occupied, dispatcherID, selected.TrunkID) + if err != nil { + return fmt.Errorf("count trunk occupancy: %w", err) + } + if tenantUsed >= snapshot.Quota.MaxConcurrentCalls || taskUsed >= snapshot.Task.MaxConcurrentCalls || trunkUsed >= trunkLimit { + return ErrCurrentCapacity + } + result, err := tx.Exec(`UPDATE dispatcher_inbox SET status='dispatching',selected_trunk_id=?,caller_id=?,dialed_callee=?,deadline=?,snapshot_json=? + WHERE dispatcher_id=? AND event_id=? AND status='pending'`, selected.TrunkID, selected.CallerID, selected.DialedCallee, selected.Deadline.UTC().Format(time.RFC3339Nano), body, dispatcherID, eventID) + if err != nil { + return fmt.Errorf("persist pre-originate fence: %w", err) + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n != 1 { + return ErrCurrentAlreadyStarted + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit pre-originate fence: %w", err) + } + return nil +} + +// CancelReservationBeforeOrigin is valid only before invoking the Agent RPC. +// It releases a reservation if the clock, SIP load, or schedule changed during +// the final pre-dial check; it must never be used after an unknown RPC outcome. +func (s *CurrentStore) CancelReservationBeforeOrigin(dispatcherID, eventID string) error { + result, err := s.db.Exec(`UPDATE dispatcher_inbox SET status=CASE WHEN EXISTS( + SELECT 1 FROM dispatcher_tasks t WHERE t.dispatcher_id=dispatcher_inbox.dispatcher_id + AND t.tenant_id=dispatcher_inbox.tenant_id AND t.task_id=dispatcher_inbox.task_id + AND t.control_state IN ('stopping','stopped')) THEN 'suppressed' ELSE 'pending' END, + selected_trunk_id=NULL,caller_id=NULL,dialed_callee=NULL,deadline=NULL,snapshot_json=NULL + WHERE dispatcher_id=? AND event_id=? AND status='dispatching'`, dispatcherID, eventID) + if err != nil { + return fmt.Errorf("cancel unused originate reservation: %w", err) + } + return requireOneRow(result, "cancel unused originate reservation") +} + +func (s *CurrentStore) MarkExecuteUnknown(dispatcherID, eventID string) error { + result, err := s.db.Exec(`UPDATE dispatcher_inbox SET status='unknown' WHERE dispatcher_id=? AND event_id=? AND status='dispatching'`, dispatcherID, eventID) + if err != nil { + return fmt.Errorf("persist unknown execution: %w", err) + } + return requireOneRow(result, "unknown execution") +} + +func (s *CurrentStore) FinishExecute(dispatcherID, eventID string) error { + result, err := s.db.Exec(`UPDATE dispatcher_inbox SET status='finished' WHERE dispatcher_id=? AND event_id=? AND status='dispatched'`, dispatcherID, eventID) + if err != nil { + return fmt.Errorf("persist confirmed call end: %w", err) + } + return requireOneRow(result, "confirmed call end") +} + +func (s *CurrentStore) RejectExecute(dispatcherID, eventID, reason string) error { + if reason == "" { + return errors.New("rejection reason is required") + } + return s.closeExecuteWithAck(dispatcherID, eventID, "pending", "rejected", map[string]any{"status": "rejected", "reason_code": nil, "reason_message": reason}) +} + +func (s *CurrentStore) MarkExecuteDispatched(dispatcherID, eventID string) error { + return s.closeExecuteWithAck(dispatcherID, eventID, "dispatching", "dispatched", map[string]any{"status": "dispatched"}) +} + +func (s *CurrentStore) closeExecuteWithAck(dispatcherID, eventID, expectedStatus, nextStatus string, payload any) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var tenantID int64 + var issuedAt, status string + err = tx.QueryRow(`SELECT tenant_id,issued_at,status FROM dispatcher_inbox WHERE dispatcher_id=? AND event_id=?`, dispatcherID, eventID).Scan(&tenantID, &issuedAt, &status) + if err != nil { + return fmt.Errorf("read call acknowledgment identity: %w", err) + } + if status != expectedStatus { + return fmt.Errorf("call %q cannot transition from %s to %s", eventID, status, nextStatus) + } + body, err := json.Marshal(struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + DispatcherID string `json:"dispatcher_id"` + TenantID int64 `json:"tenant_id"` + IssuedAt string `json:"issued_at"` + Payload any `json:"payload"` + }{eventID, "call.execute", dispatcherID, tenantID, issuedAt, payload}) + if err != nil { + return fmt.Errorf("encode call acknowledgment: %w", err) + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return fmt.Errorf("call acknowledgment violates current MQ contract: %w", err) + } + route, err := tenant.CurrentResultRoute(dispatcherID) + if err != nil { + return err + } + if _, err := tx.Exec(`UPDATE dispatcher_inbox SET status=? WHERE dispatcher_id=? AND event_id=? AND status=?`, nextStatus, dispatcherID, eventID, expectedStatus); err != nil { + return fmt.Errorf("persist call acknowledgment status: %w", err) + } + if _, err := tx.Exec(`INSERT INTO dispatcher_outbox(dispatcher_id,event_id,event_type,routing_key,body) VALUES(?,?,?,?,?)`, dispatcherID, eventID, "call.execute", route.BindingKey, body); err != nil { + return fmt.Errorf("persist call acknowledgment outbox: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit call status and acknowledgment: %w", err) + } + return nil +} + +// PendingExecuteCount is scoped to one task so a rule wait pauses only its +// SaaS-owned task queue; other tasks and the control queue keep consuming. +func (s *CurrentStore) PendingExecuteCount(dispatcherID string, tenantID int64, taskID string) (int64, error) { + if dispatcherID == "" || tenantID <= 0 || taskID == "" { + return 0, errors.New("pending task count requires durable task identity") + } + var count int64 + if err := s.db.QueryRow(`SELECT COUNT(*) FROM dispatcher_inbox WHERE dispatcher_id=? AND tenant_id=? AND task_id=? AND status='pending'`, dispatcherID, tenantID, taskID).Scan(&count); err != nil { + return 0, fmt.Errorf("count task-local rule waits: %w", err) + } + return count, nil +} + +// TrunkOccupancy includes dispatching and unknown executions. No timeout or +// lease expiry releases a call whose real end has not been confirmed. +func (s *CurrentStore) TrunkOccupancy(dispatcherID string) (map[string]int64, error) { + rows, err := s.db.Query(`SELECT selected_trunk_id, COUNT(*) FROM dispatcher_inbox + WHERE dispatcher_id=? AND status IN ('dispatching','dispatched','unknown') + AND selected_trunk_id IS NOT NULL GROUP BY selected_trunk_id`, dispatcherID) + if err != nil { + return nil, fmt.Errorf("count occupied SIP trunks: %w", err) + } + defer rows.Close() + occupied := make(map[string]int64) + for rows.Next() { + var trunkID string + var count int64 + if err := rows.Scan(&trunkID, &count); err != nil { + return nil, err + } + occupied[trunkID] = count + } + if err := rows.Err(); err != nil { + return nil, err + } + return occupied, nil +} + +func (s *CurrentStore) ListPendingExecute(dispatcherID string) ([]CurrentExecuteCommand, error) { + rows, err := s.db.Query(`SELECT dispatcher_id,event_id,tenant_id,task_id,callee,issued_at FROM dispatcher_inbox WHERE dispatcher_id=? AND status='pending' ORDER BY rowid`, dispatcherID) + if err != nil { + return nil, fmt.Errorf("list pending call commands: %w", err) + } + defer rows.Close() + var commands []CurrentExecuteCommand + for rows.Next() { + var cmd CurrentExecuteCommand + if err := rows.Scan(&cmd.DispatcherID, &cmd.EventID, &cmd.TenantID, &cmd.TaskID, &cmd.Callee, &cmd.IssuedAt); err != nil { + return nil, err + } + commands = append(commands, cmd) + } + if err := rows.Err(); err != nil { + return nil, err + } + return commands, nil +} + +func (s *CurrentStore) ListPendingOutbox(dispatcherID string) ([]CurrentOutboxEvent, error) { + rows, err := s.db.Query(`SELECT event_id,event_type,routing_key,body FROM dispatcher_outbox WHERE dispatcher_id=? AND confirmed=0 ORDER BY rowid`, dispatcherID) + if err != nil { + return nil, fmt.Errorf("list pending SaaS events: %w", err) + } + defer rows.Close() + var events []CurrentOutboxEvent + for rows.Next() { + var event CurrentOutboxEvent + if err := rows.Scan(&event.EventID, &event.EventType, &event.RoutingKey, &event.Body); err != nil { + return nil, err + } + events = append(events, event) + } + if err := rows.Err(); err != nil { + return nil, err + } + return events, nil +} + +// Confirm means delivery to a bound queue, not application receipt. The row +// remains durable and is never deleted without separate explicit evidence. +func (s *CurrentStore) MarkOutboxConfirmed(dispatcherID, eventID string) error { + result, err := s.db.Exec(`UPDATE dispatcher_outbox SET confirmed=1,confirmed_at=? WHERE dispatcher_id=? AND event_id=? AND confirmed=0`, time.Now().UTC().Format(time.RFC3339Nano), dispatcherID, eventID) + if err != nil { + return fmt.Errorf("persist MQ queue confirm: %w", err) + } + return requireOneRow(result, "MQ queue confirm") +} + +func requireOneRow(result sql.Result, operation string) error { + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return fmt.Errorf("%s expected exactly one state transition, got %d", operation, count) + } + return nil +} diff --git a/internal/store/current_calls_test.go b/internal/store/current_calls_test.go new file mode 100644 index 0000000..fc8f23e --- /dev/null +++ b/internal/store/current_calls_test.go @@ -0,0 +1,162 @@ +package store + +import ( + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/configread" +) + +func preparedCurrentCallStore(t *testing.T) *CurrentStore { + t.Helper() + s, err := OpenCurrent(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + task := currentStoreSnapshot(t) + task.SIP.Trunks = []byte(strings.Replace(string(task.SIP.Trunks), `"max_concurrent_calls":null`, `"max_concurrent_calls":2`, 1)) + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{{TaskID: task.Task.TaskID, TenantID: task.Task.TenantID, TaskRevision: task.Task.TaskRevision, Status: "running"}}); err != nil { + t.Fatal(err) + } + if err := s.SaveSnapshot(task); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 8); err != nil { + t.Fatal(err) + } + return s +} + +func currentCall(id string) CurrentExecuteCommand { + return CurrentExecuteCommand{DispatcherID: currentDispatcherID, EventID: id, TenantID: 1001, TaskID: "task-asr", Callee: "15003164745", IssuedAt: "2026-09-21T01:30:00Z"} +} +func currentReservation() CurrentCallReservation { + return CurrentCallReservation{TrunkID: "trunk-mock", SIPRevision: 8, CallerID: "BD00000000", DialedCallee: "15003164745", Deadline: time.Date(2026, 9, 21, 1, 32, 0, 0, time.UTC)} +} + +func TestCurrentExecuteInboxPreventsDuplicateOriginationAfterRestart(t *testing.T) { + s := preparedCurrentCallStore(t) + cmd := currentCall("call-1") + first, created, err := s.RecordExecute(cmd) + if err != nil || !created || first.Status != "pending" { + t.Fatalf("first durable receipt: %+v created=%v err=%v", first, created, err) + } + second, created, err := s.RecordExecute(cmd) + if err != nil || created || second.Status != "pending" { + t.Fatalf("redelivery not deduplicated: %+v created=%v err=%v", second, created, err) + } + if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + if _, created, err := s.RecordExecute(cmd); err != nil || created { + t.Fatalf("reserved call was reaccepted: created=%v err=%v", created, err) + } + if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC)); err == nil { + t.Fatal("duplicate receipt reserved a second originate") + } + pending, err := s.ListPendingExecute(currentDispatcherID) + if err != nil || len(pending) != 0 { + t.Fatalf("unknown/reserved call retried: %+v %v", pending, err) + } + if err := s.MarkExecuteUnknown(cmd.DispatcherID, cmd.EventID); err != nil { + t.Fatal(err) + } + if _, created, err := s.RecordExecute(cmd); err != nil || created { + t.Fatalf("unknown call reaccepted: created=%v err=%v", created, err) + } +} + +func TestCurrentExecuteRejectsOnlyOnceWithoutFinalResult(t *testing.T) { + s := preparedCurrentCallStore(t) + cmd := currentCall("call-invalid") + if _, _, err := s.RecordExecute(cmd); err != nil { + t.Fatal(err) + } + if err := s.RejectExecute(cmd.DispatcherID, cmd.EventID, "callee not permitted"); err != nil { + t.Fatal(err) + } + outbox, err := s.ListPendingOutbox(currentDispatcherID) + if err != nil || len(outbox) != 1 || outbox[0].EventType != "call.execute" || !strings.Contains(string(outbox[0].Body), `"status":"rejected"`) { + t.Fatalf("rejection outbox: %+v %v", outbox, err) + } + if _, created, err := s.RecordExecute(cmd); err != nil || created { + t.Fatalf("rejected command was accepted twice: %v %v", created, err) + } + if err := s.RejectExecute(cmd.DispatcherID, cmd.EventID, "callee not permitted"); err == nil { + t.Fatal("duplicate rejection created another outbox event") + } + if err := s.MarkOutboxConfirmed(currentDispatcherID, outbox[0].EventID); err != nil { + t.Fatal(err) + } + if pending, err := s.ListPendingOutbox(currentDispatcherID); err != nil || len(pending) != 0 { + t.Fatalf("confirmed outbox was republished: %+v %v", pending, err) + } + var total int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM dispatcher_outbox WHERE dispatcher_id=?`, currentDispatcherID).Scan(&total); err != nil || total != 1 { + t.Fatalf("confirmed outbox vanished before SaaS receipt: %d %v", total, err) + } +} + +func TestCurrentExecuteQuotaCountsUnknownAndReleasesOnlyConfirmedEnd(t *testing.T) { + s := preparedCurrentCallStore(t) + at := time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC) + for _, id := range []string{"call-1", "call-2"} { + cmd := currentCall(id) + if _, _, err := s.RecordExecute(cmd); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), at); err != nil { + t.Fatal(err) + } + } + third := currentCall("call-3") + if _, _, err := s.RecordExecute(third); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(third.DispatcherID, third.EventID, currentReservation(), at); !errors.Is(err, ErrCurrentCapacity) { + t.Fatalf("third call exceeded task/trunk capacity: %v", err) + } + if err := s.MarkExecuteUnknown(currentDispatcherID, "call-1"); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(third.DispatcherID, third.EventID, currentReservation(), at); !errors.Is(err, ErrCurrentCapacity) { + t.Fatalf("unknown call released occupancy: %v", err) + } + if err := s.MarkExecuteDispatched(currentDispatcherID, "call-2"); err != nil { + t.Fatal(err) + } + if err := s.FinishExecute(currentDispatcherID, "call-2"); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(third.DispatcherID, third.EventID, currentReservation(), at); err != nil { + t.Fatalf("finished call failed to release capacity: %v", err) + } +} + +func TestCurrentExecuteOutboxFailureDoesNotMarkOriginatedCallDelivered(t *testing.T) { + s := preparedCurrentCallStore(t) + cmd := currentCall("call-fault") + if _, _, err := s.RecordExecute(cmd); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_outbox BEFORE INSERT ON dispatcher_outbox BEGIN SELECT RAISE(ABORT,'injected outbox failure'); END`); err != nil { + t.Fatal(err) + } + if err := s.MarkExecuteDispatched(cmd.DispatcherID, cmd.EventID); err == nil { + t.Fatal("marked dispatch despite outbox failure") + } + var status string + if err := s.db.QueryRow(`SELECT status FROM dispatcher_inbox WHERE dispatcher_id=? AND event_id=?`, cmd.DispatcherID, cmd.EventID).Scan(&status); err != nil || status != "dispatching" { + t.Fatalf("lost pre-originate fence after outbox failure: %q %v", status, err) + } + if _, created, err := s.RecordExecute(cmd); err != nil || created { + t.Fatalf("originated command redialed after outbox failure: %v %v", created, err) + } +} diff --git a/internal/store/current_control_ack_test.go b/internal/store/current_control_ack_test.go new file mode 100644 index 0000000..6d2bdb0 --- /dev/null +++ b/internal/store/current_control_ack_test.go @@ -0,0 +1,101 @@ +package store + +import ( + "strings" + "testing" +) + +func TestCurrentControlAckOnlyAfterDispatchAndInSameTransaction(t *testing.T) { + s := preparedCurrentCallStore(t) + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "pause"); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("pause preparation failed to close task admission: %v %v", admitted, err) + } + if events, err := s.ListPendingOutbox(currentDispatcherID); err != nil || len(events) != 0 { + t.Fatalf("ack existed before Agent dispatch: %+v %v", events, err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "pause", "pause-event-1"); err != nil { + t.Fatal(err) + } + outbox, err := s.ListPendingOutbox(currentDispatcherID) + if err != nil || len(outbox) != 1 || outbox[0].EventType != "task.control" || !strings.Contains(string(outbox[0].Body), `"status":"applied"`) { + t.Fatalf("missing applied control ack: %+v %v", outbox, err) + } + if err := s.MarkOutboxConfirmed(currentDispatcherID, "pause-event-1"); err != nil { + t.Fatal(err) + } + // A repeated control must be dispatched again, not suppressed by event ID. + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "pause"); err != nil { + t.Fatal(err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "pause", "pause-event-1"); err != nil { + t.Fatal(err) + } + outbox, err = s.ListPendingOutbox(currentDispatcherID) + if err != nil || len(outbox) != 1 || outbox[0].EventID != "pause-event-1" { + t.Fatalf("repeated control failed to resend same acknowledgment: %+v %v", outbox, err) + } +} + +func TestCurrentResumeDoesNotOpenAdmissionWhenAckOutboxFails(t *testing.T) { + s := preparedCurrentCallStore(t) + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "pause"); err != nil { + t.Fatal(err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "pause", "pause-1"); err != nil { + t.Fatal(err) + } + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "resume"); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_resume_ack BEFORE INSERT ON dispatcher_outbox WHEN NEW.event_id='resume-1' BEGIN SELECT RAISE(ABORT,'injected MQ outbox failure'); END`); err != nil { + t.Fatal(err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "resume", "resume-1"); err == nil { + t.Fatal("opened admission without durable resume acknowledgment") + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("resume outbox failure opened admission: %v %v", admitted, err) + } + if _, err := s.db.Exec(`DROP TRIGGER fail_resume_ack`); err != nil { + t.Fatal(err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "resume", "resume-1"); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || !admitted { + t.Fatalf("successful resume did not open admission: %v %v", admitted, err) + } +} + +func TestCurrentStopRejectsResumeAndSuppressesOldCommands(t *testing.T) { + s := preparedCurrentCallStore(t) + cmd := currentCall("queued-stop") + if _, _, err := s.RecordExecute(cmd); err != nil { + t.Fatal(err) + } + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "stop"); err != nil { + t.Fatal(err) + } + if commands, err := s.ListPendingExecute(currentDispatcherID); err != nil || len(commands) != 0 { + t.Fatalf("stopped task retained dispatchable old command: %+v %v", commands, err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "stop", "stop-1"); err != nil { + t.Fatal(err) + } + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "resume"); err == nil { + t.Fatal("stopped task accepted resume") + } + if err := s.RejectControl(currentDispatcherID, 1001, "resume-1"); err != nil { + t.Fatal(err) + } + outbox, err := s.ListPendingOutbox(currentDispatcherID) + if err != nil || len(outbox) != 2 { + t.Fatalf("stop/rejected resume acknowledgments missing: %+v %v", outbox, err) + } + if strings.Contains(string(outbox[1].Body), `"status":"applied"`) { + t.Fatalf("stopped task resume falsely applied: %s", outbox[1].Body) + } +} diff --git a/internal/store/current_control_flow.go b/internal/store/current_control_flow.go new file mode 100644 index 0000000..e32ebfc --- /dev/null +++ b/internal/store/current_control_flow.go @@ -0,0 +1,177 @@ +package store + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + + "git.ipao.vip/rogee/go-sip/internal/contract" + "git.ipao.vip/rogee/go-sip/internal/tenant" +) + +var ErrCurrentControlRejected = errors.New("task control rejected by durable task state") + +// PrepareControl closes only this task's admission before requesting the +// Agent action. No successful control acknowledgment exists at this point. +// Repeated commands are prepared and dispatched again; this is not control +// deduplication or an expected-revision/CAS API. +func (s *CurrentStore) PrepareControl(dispatcherID string, tenantID int64, taskID, action string) error { + if dispatcherID == "" || tenantID <= 0 || taskID == "" { + return errors.New("invalid task control identity") + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var state, status string + var revision int64 + var present int + err = tx.QueryRow(`SELECT control_state,status,task_revision,present FROM dispatcher_tasks + WHERE dispatcher_id=? AND tenant_id=? AND task_id=?`, dispatcherID, tenantID, taskID).Scan(&state, &status, &revision, &present) + if err != nil { + return fmt.Errorf("load task control state: %w", err) + } + if present != 1 { + return fmt.Errorf("%w: task is not in the assigned discovery list", ErrCurrentControlRejected) + } + var prepared string + switch action { + case "pause": + if state == "stopped" || state == "stopping" { + return fmt.Errorf("%w: stopped task cannot be paused", ErrCurrentControlRejected) + } + prepared = "pausing" + case "stop": + prepared = "stopping" + case "resume": + if state != "paused" && state != "resuming" { + return fmt.Errorf("%w: resume requires a persistently paused, non-stopped task", ErrCurrentControlRejected) + } + if status != "running" { + return fmt.Errorf("%w: fresh task is not running", ErrCurrentControlRejected) + } + var configuredRevision int64 + err = tx.QueryRow(`SELECT task_revision FROM dispatcher_configs WHERE dispatcher_id=? AND tenant_id=? AND task_id=?`, dispatcherID, tenantID, taskID).Scan(&configuredRevision) + if err != nil || configuredRevision != revision { + return errors.New("resume requires the latest verified task configuration") + } + prepared = "resuming" + default: + return fmt.Errorf("invalid task control action %q", action) + } + if _, err := tx.Exec(`UPDATE dispatcher_tasks SET control_state=? WHERE dispatcher_id=? AND tenant_id=? AND task_id=?`, prepared, dispatcherID, tenantID, taskID); err != nil { + return fmt.Errorf("persist task control barrier: %w", err) + } + if action == "stop" { + if _, err := tx.Exec(`UPDATE dispatcher_inbox SET status='suppressed' WHERE dispatcher_id=? AND tenant_id=? AND task_id=? AND status='pending'`, dispatcherID, tenantID, taskID); err != nil { + return fmt.Errorf("suppress unstarted stopped-task commands: %w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit task control barrier: %w", err) + } + return nil +} + +// CompleteControl atomically records the applied state and durable outbox +// only after an Agent control RPC accepted the action. It does not claim that +// active-call drain or hangup has already finished. +func (s *CurrentStore) CompleteControl(dispatcherID string, tenantID int64, taskID, action, eventID string) error { + var expected, final string + switch action { + case "pause": + expected, final = "pausing", "paused" + case "stop": + expected, final = "stopping", "stopped" + case "resume": + expected, final = "resuming", "" + default: + return fmt.Errorf("invalid task control action %q", action) + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var state, status string + err = tx.QueryRow(`SELECT control_state,status FROM dispatcher_tasks WHERE dispatcher_id=? AND tenant_id=? AND task_id=? AND present=1`, dispatcherID, tenantID, taskID).Scan(&state, &status) + if err != nil { + return fmt.Errorf("read prepared task control: %w", err) + } + if state != expected || (action == "resume" && status != "running") { + return fmt.Errorf("task %q control %q is no longer prepared or authorized", taskID, action) + } + if _, err := tx.Exec(`UPDATE dispatcher_tasks SET control_state=? WHERE dispatcher_id=? AND tenant_id=? AND task_id=?`, final, dispatcherID, tenantID, taskID); err != nil { + return fmt.Errorf("persist applied task control: %w", err) + } + if err := enqueueControlAck(tx, dispatcherID, tenantID, eventID, "applied"); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit task control and outbox: %w", err) + } + return nil +} + +func (s *CurrentStore) RejectControl(dispatcherID string, tenantID int64, eventID string) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := enqueueControlAck(tx, dispatcherID, tenantID, eventID, "rejected"); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit rejected task control: %w", err) + } + return nil +} + +func enqueueControlAck(tx *sql.Tx, dispatcherID string, tenantID int64, eventID, status string) error { + if eventID == "" || len(eventID) > 255 || tenantID <= 0 || (status != "applied" && status != "rejected") { + return errors.New("invalid task control acknowledgment identity or status") + } + route, err := tenant.CurrentResultRoute(dispatcherID) + if err != nil { + return err + } + body, err := json.Marshal(struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + DispatcherID string `json:"dispatcher_id"` + TenantID int64 `json:"tenant_id"` + Payload struct { + Status string `json:"status"` + } `json:"payload"` + }{EventID: eventID, EventType: "task.control", DispatcherID: dispatcherID, TenantID: tenantID, Payload: struct { + Status string `json:"status"` + }{status}}) + if err != nil { + return fmt.Errorf("encode task control acknowledgment: %w", err) + } + if err := contract.ValidateCurrent("mq", body); err != nil { + return fmt.Errorf("task control acknowledgment violates MQ contract: %w", err) + } + var oldType string + var oldBody []byte + err = tx.QueryRow(`SELECT event_type,body FROM dispatcher_outbox WHERE dispatcher_id=? AND event_id=?`, dispatcherID, eventID).Scan(&oldType, &oldBody) + if errors.Is(err, sql.ErrNoRows) { + if _, err := tx.Exec(`INSERT INTO dispatcher_outbox(dispatcher_id,event_id,event_type,routing_key,body) VALUES(?,?,?,?,?)`, dispatcherID, eventID, "task.control", route.BindingKey, body); err != nil { + return fmt.Errorf("persist task control outbox: %w", err) + } + return nil + } + if err != nil { + return fmt.Errorf("inspect existing task control outbox: %w", err) + } + if oldType != "task.control" || string(oldBody) != string(body) { + return fmt.Errorf("control event identity %q conflicts with another outbound result", eventID) + } + if _, err := tx.Exec(`UPDATE dispatcher_outbox SET confirmed=0,confirmed_at=NULL WHERE dispatcher_id=? AND event_id=?`, dispatcherID, eventID); err != nil { + return fmt.Errorf("requeue repeated control acknowledgment: %w", err) + } + return nil +} diff --git a/internal/store/current_control_test.go b/internal/store/current_control_test.go new file mode 100644 index 0000000..9803886 --- /dev/null +++ b/internal/store/current_control_test.go @@ -0,0 +1,49 @@ +package store + +import ( + "testing" + "time" +) + +func TestCurrentStopSuppressesUnstartedCallsButPreservesUnknown(t *testing.T) { + s := preparedCurrentCallStore(t) + pending := currentCall("queued-before-stop") + if _, _, err := s.RecordExecute(pending); err != nil { + t.Fatal(err) + } + unknown := currentCall("unknown-before-stop") + if _, _, err := s.RecordExecute(unknown); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(unknown.DispatcherID, unknown.EventID, currentReservation(), currentMondayUTC()); err != nil { + t.Fatal(err) + } + if err := s.MarkExecuteUnknown(unknown.DispatcherID, unknown.EventID); err != nil { + t.Fatal(err) + } + if err := s.ApplyControl(currentDispatcherID, 1001, "task-asr", "stop"); err != nil { + t.Fatal(err) + } + calls, err := s.ListPendingExecute(currentDispatcherID) + if err != nil || len(calls) != 0 { + t.Fatalf("stopped task still has dispatchable commands: %+v %v", calls, err) + } + var pendingStatus, unknownStatus string + if err := s.db.QueryRow(`SELECT status FROM dispatcher_inbox WHERE event_id=?`, pending.EventID).Scan(&pendingStatus); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT status FROM dispatcher_inbox WHERE event_id=?`, unknown.EventID).Scan(&unknownStatus); err != nil { + t.Fatal(err) + } + if pendingStatus != "suppressed" || unknownStatus != "unknown" { + t.Fatalf("stop did not preserve unknown and silently suppress unstarted: pending=%s unknown=%s", pendingStatus, unknownStatus) + } + if err := s.ApplyControl(currentDispatcherID, 1001, "task-asr", "resume"); err == nil { + t.Fatal("stopped task resumed") + } + if outbox, err := s.ListPendingOutbox(currentDispatcherID); err != nil || len(outbox) != 0 { + t.Fatalf("stop emitted per-call results: %+v %v", outbox, err) + } +} + +func currentMondayUTC() time.Time { return time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC) } diff --git a/internal/store/current_discovery_test.go b/internal/store/current_discovery_test.go new file mode 100644 index 0000000..3f1273e --- /dev/null +++ b/internal/store/current_discovery_test.go @@ -0,0 +1,128 @@ +package store + +import ( + "path/filepath" + "testing" + + "git.ipao.vip/rogee/go-sip/internal/configread" +) + +func currentDiscoveredTask(status string, revision int64) configread.CurrentDiscoveredTask { + return configread.CurrentDiscoveredTask{TaskID: "task-asr", TenantID: 1001, TaskRevision: revision, Status: status} +} + +func TestCurrentHTTPStopCannotBeUndoneByStaleRunningAndSuppressesInbox(t *testing.T) { + s, err := OpenCurrent(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + base := currentStoreSnapshot(t) + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("running", 1)}); err != nil { + t.Fatal(err) + } + if err := s.SaveSnapshot(base); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 8); err != nil { + t.Fatal(err) + } + if _, _, err := s.RecordExecute(currentCall("not-admitted-before-http-stop")); err != nil { + t.Fatal(err) + } + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("stopped", 1)}); err != nil { + t.Fatal(err) + } + if pending, err := s.ListPendingExecute(currentDispatcherID); err != nil || len(pending) != 0 { + t.Fatalf("HTTP stop retained pending command: %+v %v", pending, err) + } + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("running", 1)}); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 8); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("stale HTTP running undid irrevocable stop: %v %v", admitted, err) + } + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "resume"); err == nil { + t.Fatal("stopped task accepted resume") + } +} + +func TestCurrentHTTPPausedRequiresExplicitMQResume(t *testing.T) { + s, err := OpenCurrent(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + base := currentStoreSnapshot(t) + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("running", 1)}); err != nil { + t.Fatal(err) + } + if err := s.SaveSnapshot(base); err != nil { + t.Fatal(err) + } + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("paused", 1)}); err != nil { + t.Fatal(err) + } + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("running", 1)}); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 8); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("HTTP running overrode persisted pause: %v %v", admitted, err) + } + if err := s.PrepareControl(currentDispatcherID, 1001, "task-asr", "resume"); err != nil { + t.Fatal(err) + } + if err := s.CompleteControl(currentDispatcherID, 1001, "task-asr", "resume", "resume-http-pause"); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || !admitted { + t.Fatalf("explicit resume failed: %v %v", admitted, err) + } +} + +func TestCurrentDiscoveryPageCommitsAtomicallyAndPreservesControl(t *testing.T) { + s, err := OpenCurrent(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.ApplyDiscoverySnapshot(currentDispatcherID, []configread.CurrentDiscoveredTask{currentDiscoveredTask("running", 1)}); err != nil { + t.Fatal(err) + } + if err := s.ApplyControl(currentDispatcherID, 1001, "task-asr", "pause"); err != nil { + t.Fatal(err) + } + tasks := []configread.CurrentDiscoveredTask{currentDiscoveredTask("running", 2), {TaskID: "task-second", TenantID: 0, TaskRevision: 1, Status: "running"}} + if err := s.ApplyDiscoveryPage(currentDispatcherID, tasks); err == nil { + t.Fatal("invalid page was partly committed") + } + var revision int64 + if err := s.db.QueryRow(`SELECT task_revision FROM dispatcher_tasks WHERE task_id='task-asr'`).Scan(&revision); err != nil || revision != 1 { + t.Fatalf("page committed despite later error: revision=%d err=%v", revision, err) + } + tasks[1].TenantID = 1001 + if err := s.ApplyDiscoveryPage(currentDispatcherID, tasks); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT task_revision FROM dispatcher_tasks WHERE task_id='task-asr'`).Scan(&revision); err != nil || revision != 2 { + t.Fatalf("valid page not committed: revision=%d err=%v", revision, err) + } + var state string + if err := s.db.QueryRow(`SELECT control_state FROM dispatcher_tasks WHERE task_id='task-asr'`).Scan(&state); err != nil || state != "paused" { + t.Fatalf("HTTP page overrode MQ pause: %s %v", state, err) + } + var count int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM dispatcher_tasks WHERE task_id='task-second' AND present=1`).Scan(&count); err != nil || count != 1 { + t.Fatalf("second task of valid page absent: %d %v", count, err) + } + assigned, err := s.ListAssignedTasks(currentDispatcherID) + if err != nil || len(assigned) != 2 || assigned[0].TaskID != "task-asr" || assigned[0].ControlState != "paused" || assigned[1].TaskID != "task-second" { + t.Fatalf("assigned task queues not discoverable: %+v %v", assigned, err) + } +} diff --git a/internal/store/current_pending_test.go b/internal/store/current_pending_test.go new file mode 100644 index 0000000..f67c7ae --- /dev/null +++ b/internal/store/current_pending_test.go @@ -0,0 +1,26 @@ +package store + +import ( + "testing" + "time" +) + +func TestCurrentPendingCountPausesOnlyAffectedTaskQueue(t *testing.T) { + s := preparedCurrentCallStore(t) + cmd := currentCall("waiting-for-task-window") + if _, _, err := s.RecordExecute(cmd); err != nil { + t.Fatal(err) + } + if count, err := s.PendingExecuteCount(currentDispatcherID, 1001, "task-asr"); err != nil || count != 1 { + t.Fatalf("durable wait not counted: %d %v", count, err) + } + if count, err := s.PendingExecuteCount(currentDispatcherID, 1001, "another-task"); err != nil || count != 0 { + t.Fatalf("unrelated task queue was paused: %d %v", count, err) + } + if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + if count, err := s.PendingExecuteCount(currentDispatcherID, 1001, "task-asr"); err != nil || count != 0 { + t.Fatalf("reserved command still blocked its queue: %d %v", count, err) + } +} diff --git a/internal/store/current_sip_change_test.go b/internal/store/current_sip_change_test.go new file mode 100644 index 0000000..293521a --- /dev/null +++ b/internal/store/current_sip_change_test.go @@ -0,0 +1,94 @@ +package store + +import ( + "path/filepath" + "testing" +) + +func TestCurrentSIPNotificationRequiresFullDrainAndExactLoadedRevision(t *testing.T) { + s := preparedCurrentCallStore(t) + if err := s.MarkReadyForSIP(currentDispatcherID, 8); err != nil { + t.Fatal(err) + } + cmd := currentCall("ongoing-before-sip-change") + if _, _, err := s.RecordExecute(cmd); err != nil { + t.Fatal(err) + } + if err := s.ReserveExecute(cmd.DispatcherID, cmd.EventID, currentReservation(), currentMondayUTC()); err != nil { + t.Fatal(err) + } + if err := s.MarkExecuteUnknown(cmd.DispatcherID, cmd.EventID); err != nil { + t.Fatal(err) + } + if count, err := s.OccupiedCalls(currentDispatcherID); err != nil || count != 1 { + t.Fatalf("unknown call was not counted: %d %v", count, err) + } + if err := s.NoteSIPChange(currentDispatcherID, 9); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || admitted { + t.Fatalf("SIP change failed to close admission: %v %v", admitted, err) + } + if applied, pending, err := s.SIPState(currentDispatcherID); err != nil || applied != 8 || pending != 9 { + t.Fatalf("SIP notification not durable: applied=%d pending=%d %v", applied, pending, err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 9); err == nil { + t.Fatal("changed SIP opened without matching task snapshot or drain") + } + snapshot, err := s.ReadSnapshot(currentDispatcherID, 1001, "task-asr") + if err != nil { + t.Fatal(err) + } + snapshot.SIP.Revision = 9 + if err := s.SaveSnapshot(snapshot); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 9); err == nil { + t.Fatal("unknown ongoing call was released by SIP change") + } + // An external definitive call end must be recorded before old SIP loads can + // drain. Unknown outcomes never expire or auto-release on their own. + if _, err := s.db.Exec(`UPDATE dispatcher_inbox SET status='finished' WHERE dispatcher_id=? AND event_id=? AND status='unknown'`, cmd.DispatcherID, cmd.EventID); err != nil { + t.Fatal(err) + } + if err := s.MarkReadyForSIP(currentDispatcherID, 9); err != nil { + t.Fatal(err) + } + if admitted, err := s.CanAdmit(currentDispatcherID, 1001, "task-asr"); err != nil || !admitted { + t.Fatalf("drained verified SIP revision did not reopen: %v %v", admitted, err) + } + if applied, pending, err := s.SIPState(currentDispatcherID); err != nil || applied != 9 || pending != 0 { + t.Fatalf("SIP state not committed after reload: %d %d %v", applied, pending, err) + } + if err := s.NoteSIPChange(currentDispatcherID, 8); err != nil { + t.Fatal(err) + } + if applied, pending, err := s.SIPState(currentDispatcherID); err != nil || applied != 9 || pending != 0 { + t.Fatalf("stale notification downgraded applied SIP: %d %d %v", applied, pending, err) + } +} + +func TestCurrentPendingSIPChangeSurvivesRestartWithoutOldDataDeletion(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := OpenCurrent(path) + if err != nil { + t.Fatal(err) + } + if err := s.NoteSIPChange(currentDispatcherID, 12); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenCurrent(path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if applied, pending, err := reopened.SIPState(currentDispatcherID); err != nil || applied != 0 || pending != 12 { + t.Fatalf("lost pending SIP notification on restart: %d %d %v", applied, pending, err) + } + if err := reopened.MarkReadyForSIP(currentDispatcherID, 11); err == nil { + t.Fatal("older loaded SIP overrode durable pending notification") + } +} diff --git a/internal/store/current_test.go b/internal/store/current_test.go index 2adb27f..fdac7ea 100644 --- a/internal/store/current_test.go +++ b/internal/store/current_test.go @@ -135,3 +135,34 @@ func TestCurrentStoreRefusesOldSchemaWithoutDeletingRows(t *testing.T) { t.Fatalf("old data changed or disappeared: %q, %v", tenantKey, err) } } + +func TestCurrentStoreRefusesPreviousCurrentLayoutBeforeModifyingDatabase(t *testing.T) { + path := filepath.Join(t.TempDir(), "previous-current.db") + old, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := old.Exec(`CREATE TABLE dispatcher_state(dispatcher_id TEXT PRIMARY KEY,discovery_ready INTEGER NOT NULL); INSERT INTO dispatcher_state VALUES('original-dispatcher',1)`); err != nil { + t.Fatal(err) + } + if err := old.Close(); err != nil { + t.Fatal(err) + } + if opened, err := OpenCurrent(path); err == nil { + opened.Close() + t.Fatal("accepted prior current SQLite layout without explicit data decision") + } + old, err = sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer old.Close() + var id string + var ready, count int + if err := old.QueryRow(`SELECT dispatcher_id,discovery_ready FROM dispatcher_state`).Scan(&id, &ready); err != nil || id != "original-dispatcher" || ready != 1 { + t.Fatalf("old admission state changed: %q %d %v", id, ready, err) + } + if err := old.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("old database was modified before rejection: tables=%d err=%v", count, err) + } +} diff --git a/scripts/check-current-mq-mock.sh b/scripts/check-current-mq-mock.sh new file mode 100644 index 0000000..1d1e937 --- /dev/null +++ b/scripts/check-current-mq-mock.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +image=rabbitmq:4.1-management-alpine +if ! command -v docker >/dev/null || ! docker info >/dev/null 2>&1; then + echo 'isolated RabbitMQ Mock blocked: Docker is unavailable' >&2 + exit 1 +fi +if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "isolated RabbitMQ Mock blocked: local image $image is unavailable (no automatic pull)" >&2 + exit 1 +fi +uid=$(docker run --rm --entrypoint id "$image" -u rabbitmq) +gid=$(docker run --rm --entrypoint id "$image" -g rabbitmq) +name="sip-go-mq-mock-$$" +cleanup() { docker rm -f "$name" >/dev/null 2>&1 || true; } +trap cleanup EXIT +# Only a local isolated container, with ephemeral data/cookie and an +# automatically assigned localhost-only AMQP port. No real SaaS connection. +docker run -d --name "$name" --tmpfs "/var/lib/rabbitmq:rw,uid=$uid,gid=$gid,mode=0700" \ + -p 127.0.0.1::5672 "$image" >/dev/null +ready=0 +for _ in $(seq 1 60); do + state=$(docker inspect -f '{{.State.Status}}' "$name") + if [ "$state" != running ]; then + echo "isolated RabbitMQ Mock stopped: $state" >&2 + docker logs --tail 24 "$name" >&2 + exit 1 + fi + if docker exec -u rabbitmq "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 +done +if [ "$ready" -ne 1 ]; then + echo 'isolated RabbitMQ Mock did not become ready' >&2 + docker logs --tail 24 "$name" >&2 + exit 1 +fi +port=$(docker port "$name" 5672/tcp | awk -F: 'NR==1{print $NF}') +saas_pw=$(openssl rand -hex 12) +d_pw=$(openssl rand -hex 12) +if ! docker exec -u rabbitmq "$name" rabbitmqctl add_user saas_mock "$saas_pw" >/dev/null 2>&1 || + ! docker exec -u rabbitmq "$name" rabbitmqctl add_user dispatcher_mock "$d_pw" >/dev/null 2>&1 || + ! docker exec -u rabbitmq "$name" rabbitmqctl set_permissions -p / saas_mock '.*' '.*' '.*' >/dev/null 2>&1 || + ! docker exec -u rabbitmq "$name" rabbitmqctl set_permissions -p / dispatcher_mock '^$' '^agent-call\.saas\.v1$' '^agent-call\.d\..*\.(control|task\..*)\.v1$' >/dev/null 2>&1; then + echo 'isolated RabbitMQ Mock user/permission setup failed' >&2 + exit 1 +fi +export RABBITMQ_URL="amqp://dispatcher_mock:${d_pw}@127.0.0.1:${port}/" +export RABBITMQ_PROVISIONER_URL="amqp://saas_mock:${saas_pw}@127.0.0.1:${port}/" +go test -tags=integration ./internal/mq -run '^TestCurrentBrokerSharedResultQueueAndNoConfigure$' -count=1 -v +go test -tags=integration ./internal/dispatcher -run '^TestCurrentRuntimeIsolated' -count=1 -v