From 3ea5f3458386781e1e38db60c4fff96bd8266bb3 Mon Sep 17 00:00:00 2001 From: Rogee Date: Sun, 27 Sep 2026 22:45:51 +0800 Subject: [PATCH] Implement Dispatcher v0.4 local state and controls --- AGENTS.md | 10 +- contracts/contracts.go | 4 +- .../call-execute-v0.4-proposal.schema.json | 27 +++ .../task-control-v0.4-proposal.schema.json | 27 +++ .../task-discovery-v0.4-proposal.schema.json | 57 ++++++ contracts/local_v04_test.go | 53 +++++ ...spatcher有界接纳与控制通道改造计划_v0.1.md | 2 +- .../call-execute-v0.4-proposal.schema.json | 27 +++ .../examples/call-execute-altcallee-v0.4.json | 11 + .../call-execute-invalid-expiry-v0.4.json | 12 ++ .../examples/call-execute-old-v0.4.json | 11 + .../examples/call-execute-recent-v0.4.json | 11 + .../task-control-invalid-drain-v0.4.json | 10 + .../task-control-invalid-expiry-v0.4.json | 11 + .../examples/task-control-pause-v0.4.json | 10 + .../examples/task-control-resume-v0.4.json | 10 + .../examples/task-control-stop-v0.4.json | 10 + .../examples/task-discovery-changes-v0.4.json | 9 + .../examples/task-discovery-error-v0.4.json | 8 + .../examples/task-discovery-invalid-v0.4.json | 10 + .../task-discovery-snapshot-page1-v0.4.json | 11 + .../task-discovery-snapshot-page2-v0.4.json | 11 + .../local-contract-manifest-v0.4.json | 87 ++++++++ .../task-control-v0.4-proposal.schema.json | 27 +++ .../task-discovery-v0.4-proposal.schema.json | 57 ++++++ .../dispatcher-v04-local-acceptance.md | 42 ++++ docs/plan-0926.md | 2 +- docs/plan-config-read-v0.1.md | 13 +- docs/plan-dispatcher-state-v0.1.md | 44 ++++ docs/thirds/v0.4.md | 35 ++++ internal/configread/discovery_v04.go | 133 ++++++++++++ internal/configread/discovery_v04_test.go | 86 ++++++++ internal/contract/contract.go | 12 ++ internal/contract/schema.go | 2 + internal/contract/schema_test.go | 34 ++++ internal/dispatcher/config_read.go | 6 - internal/dispatcher/local_config_read_test.go | 2 +- internal/dispatcher/local_origination.go | 10 - internal/dispatcher/local_v01.go | 18 +- .../dispatcher/local_v01_integration_test.go | 94 ++++++++- internal/dispatcher/local_v01_test.go | 108 +++++----- internal/dispatcher/task_control_v3.go | 59 +++--- internal/dispatcher/task_control_v3_test.go | 107 +++++++++- .../dispatcher/task_discovery_resume_test.go | 95 +++------ internal/dispatcher/task_queue_v3.go | 8 +- .../task_queue_v3_stop_active_test.go | 27 ++- internal/dispatcher/task_queue_v3_test.go | 10 + .../task_runtime_v03_recovery_test.go | 129 +++--------- internal/dispatcher/task_runtime_v03_test.go | 192 +++++++++++++----- .../task_runtime_v04_control_test.go | 55 +++++ internal/dispatcher/task_runtime_v3.go | 136 ++++++++++--- internal/dispatcher/task_runtime_v3_test.go | 46 +++-- internal/mq/amqp_v3.go | 65 ++++++ internal/mq/amqp_v3_integration_test.go | 51 ++++- .../store/local_discovery_helpers_test.go | 41 +--- internal/store/local_origination.go | 4 +- internal/store/local_v01_discovery.go | 2 +- internal/store/local_v03_admission_test.go | 13 +- internal/store/local_v03_config_gate_test.go | 15 +- internal/store/local_v03_discovery.go | 2 +- internal/store/local_v04_cleanup.go | 90 ++++++++ internal/store/local_v04_cleanup_test.go | 128 ++++++++++++ internal/store/local_v04_discovery.go | 177 ++++++++++++++++ internal/store/local_v04_discovery_test.go | 96 +++++++++ internal/store/local_v04_lookup_test.go | 56 +++++ .../migrations/019_local_v04_discovery.sql | 6 + scripts/generate-local-contract-bundle.sh | 3 +- scripts/validate-local-contracts.py | 58 ++++++ 68 files changed, 2375 insertions(+), 460 deletions(-) create mode 100644 contracts/local/v0.4/call-execute-v0.4-proposal.schema.json create mode 100644 contracts/local/v0.4/task-control-v0.4-proposal.schema.json create mode 100644 contracts/local/v0.4/task-discovery-v0.4-proposal.schema.json create mode 100644 contracts/local_v04_test.go create mode 100644 docs/contracts/call-execute-v0.4-proposal.schema.json create mode 100644 docs/contracts/examples/call-execute-altcallee-v0.4.json create mode 100644 docs/contracts/examples/call-execute-invalid-expiry-v0.4.json create mode 100644 docs/contracts/examples/call-execute-old-v0.4.json create mode 100644 docs/contracts/examples/call-execute-recent-v0.4.json create mode 100644 docs/contracts/examples/task-control-invalid-drain-v0.4.json create mode 100644 docs/contracts/examples/task-control-invalid-expiry-v0.4.json create mode 100644 docs/contracts/examples/task-control-pause-v0.4.json create mode 100644 docs/contracts/examples/task-control-resume-v0.4.json create mode 100644 docs/contracts/examples/task-control-stop-v0.4.json create mode 100644 docs/contracts/examples/task-discovery-changes-v0.4.json create mode 100644 docs/contracts/examples/task-discovery-error-v0.4.json create mode 100644 docs/contracts/examples/task-discovery-invalid-v0.4.json create mode 100644 docs/contracts/examples/task-discovery-snapshot-page1-v0.4.json create mode 100644 docs/contracts/examples/task-discovery-snapshot-page2-v0.4.json create mode 100644 docs/contracts/local-contract-manifest-v0.4.json create mode 100644 docs/contracts/task-control-v0.4-proposal.schema.json create mode 100644 docs/contracts/task-discovery-v0.4-proposal.schema.json create mode 100644 docs/evidence/dispatcher-v04-local-acceptance.md create mode 100644 docs/plan-dispatcher-state-v0.1.md create mode 100644 docs/thirds/v0.4.md create mode 100644 internal/configread/discovery_v04.go create mode 100644 internal/configread/discovery_v04_test.go create mode 100644 internal/dispatcher/task_runtime_v04_control_test.go create mode 100644 internal/store/local_v04_cleanup.go create mode 100644 internal/store/local_v04_cleanup_test.go create mode 100644 internal/store/local_v04_discovery.go create mode 100644 internal/store/local_v04_discovery_test.go create mode 100644 internal/store/local_v04_lookup_test.go create mode 100644 internal/store/migrations/019_local_v04_discovery.sql diff --git a/AGENTS.md b/AGENTS.md index b407d8c..404e444 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ ## 当前范围 -- 后续Agent必须先读 **`docs/plan-config-read-v0.1.md`**：按 F 工作包/文档索引读取需求与契约，确认 C/L/M 和 I/M/G 前置、授权及写入边界后实施；并行时按该计划§9认领，子Agent只写独占模块/证据，§8本轮总台账、公共文件及合并状态由集成负责人单写。已恢复原内容的 `docs/archive/plan-0918.md` 仅保留旧 W00–W16 历史，不是现行计划入口。任务发现现行项目内合同另读 `docs/plan-0926.md` 和 `docs/thirds/v0.3.md`；计划中 v0.2 状态仅作历史。计划不替代权威Schema或验收，不重复审批已确认方向，不自动授权真实云/付费/拨号。 +- 后续Agent必须先读 **`docs/plan-dispatcher-state-v0.1.md`** 和 **`docs/thirds/v0.4.md`**；历史 F 工作包索引见 `docs/plan-config-read-v0.1.md`：按 F 工作包/文档索引读取需求与契约，确认 C/L/M 和 I/M/G 前置、授权及写入边界后实施；并行时按该计划§9认领，子Agent只写独占模块/证据，§8本轮总台账、公共文件及合并状态由集成负责人单写。已恢复原内容的 `docs/archive/plan-0918.md` 仅保留旧 W00–W16 历史，不是现行计划入口。本地 Dispatcher v0.4 任务发现、MQ 控制及外呼入站规则另读 `docs/plan-dispatcher-state-v0.1.md`、`docs/thirds/v0.4.md`；`docs/plan-0926.md`、`docs/thirds/v0.3.md` 及 v0.2 状态仅作历史。**v0.4 的 Schema、隔离 Mock 与代码已在项目内实施，真实 SaaS/management 尚未签收或联调；仅显式条件清理已终止任务的任务配置副本，不自动清除现有 SQLite 数据。没有 SaaS 应用收讫的执行、幂等及 outbox 记录继续保留。日志不得记录密钥、完整音频或对话。** 计划不替代权威Schema或验收，不重复审批已确认方向，不自动授权真实云/付费/拨号。 - 当前MQ-only/OSS调整目标已获用户明确批准由当前Agent独立执行，**不启动子Agent**，不得再以子Agent模型/fast环境阻塞本目标；此前按任务类型自动分工规则在本目标不适用。若未来另获启动授权，仍遵守下条模型要求。 - 用户指定：若启动开发及配套审查子Agent，固定 **gpt-5.6-luna、max思考、fast模式**。启动前查询精确provider/model和runner支持并显式配置（当前工具用模型`:max`后缀及`fast: true`，不继承默认）；不可用/不支持/无法核验则报告阻塞，不静默换模型、降思考档、关fast或换CLI。此为后续执行约束，本轮仅修复计划，未启动开发子Agent。 - 并行开发须先有获授权的可追溯Git/契约基线、一lane一工作区/测试资源、无交叠写集合及每批合并后回归；当前子项目尚未跟踪的文件不能假定存在于HEAD/worktree。不得自动提交、暂存或清理父项目无关改动；详情见 `docs/plan-config-read-v0.1.md` §9。 @@ -155,7 +155,7 @@ - **现行合同（尚未切换）**：Dispatcher按MQ任务agent_version_id，经RabbitMQ专用Topic向SaaS取得不可变AI配置/授权，响应回原Dispatcher；校验租户/源Schema/不可变摘要/能力并持久绑定后向Agent交付执行快照。过去已废弃的独立AI版本GET不复活；新获批准的方向是**任务配置只读接口内含智能体**，须另冻合同而非旧GET兼容层。Agent不直连SaaS，不从CLI/env/源码常量或SDK默认覆盖AI业务值，不新增任务MQ模式字段或调参后台。 - 已有model/prompt/voice/speed/ASR输入与识别/temperature/max_tokens/timeout及对话控制必须实际传入SDK或控制器；热词/VAD/top_p/音量/阶段时限等所需扩展先在上游补GAP-09，再生成校验。严格additionalProperties不放宽，不借metadata/raw_request透传。 - **现行 AI 合同**：SaaS新版本供新任务引用，无需改代码/重启D/A；在途/原排队任务固定快照，同版本异内容拒绝。缓存按租户+版本隔离，断SaaS无有效授权缓存拒新准入；显式0/false与未提供保真，并发通话不得共享可变SDK参数。 -- **本轮用户授权的项目内配置目标（现行外部 v1 不变）**：除任务发现已按 `docs/thirds/v0.3.md` 项目内版本切换外，以 `docs/thirds/第三方对接事件与请求消费顺序_v0.1.md` 为其余本地契约；F01/F07 Schema、正反例、来源/hash 与 Mock C 通过后可继续本地实现，不等待外部签收/连通。实际 SaaS 字段兼容性仍未验证。拟定 SaaS 只读路径 `/internal/v1/dispatcher/sip`、`/internal/v1/dispatcher/task/:task_id`，以及下一轮归属任务发现 `/internal/v1/dispatcher/tasks`；新增第四条拟定 `/internal/v1/dispatcher/tenant/:tenant_id/quota`（项目路径草案）在取得任务后按 tenant_id 读取分给该D的租户总份额；前两条返回获 management 批准的 SIP 全量和含智能体获授权不可变版本的任务配置；D 用 `X-DISPATCHER-id`/`X-DISPATCHER-SECRET-KEY` 按需获取，不将凭据写入源码、样例或日志。SIP 在启动/重启先取全量、核验 Agent/Asterisk 已加载再读执行队列；任务按租户+任务缓存约60秒，活跃且有待接纳执行时到期主动刷新；已接纳/运行中执行固定原快照，未接纳允许最多约一分钟配置生效延迟。不采用 ETag/304，缓存到期重新读取完整 200 响应；过期或 HTTP 不可用时拒绝新执行，**不使用过期缓存或MQ配置回退**；停/暂停控制不随配置缓存延迟；**现行** opt-out 仍以 MQ 即时事件通知，下一版仅在最终 `call.result` 中通知 SaaS，业务签收前不可冒称两者等价。任务终结后仅清配置缓存，不删执行恢复/幂等/outbox。SIP 变更须关执行准入、排空/对账、确认加载后恢复；不能保证一分钟内完成发布。旧 `call.execute` 固定引用/排队 AI 固定规则须经新合同定义重绑定与修改-准入竞态；未发布新版前不可变现行语义。 +- **项目内配置与执行（已发布外部 v1 不变）**：项目内 v0.4 任务发现、控制及外呼入站依据 `docs/thirds/v0.4.md`；其它必要边界见 `docs/thirds/第三方对接事件与请求消费顺序_v0.1.md`。拟定 SaaS 只读路径为 `/internal/v1/dispatcher/sip`、`/internal/v1/dispatcher/task/:task_id`、`/internal/v1/dispatcher/tasks` 和 `/internal/v1/dispatcher/tenant/:tenant_id/quota`；均为项目内字段/路径，实际 SaaS 兼容性未验证。D 用 `X-DISPATCHER-id`/`X-DISPATCHER-SECRET-KEY` 按需读取，不在源码、配置样例或日志暴露凭据。SIP 启动先取获批全量并核验 Agent/Asterisk 加载后才准入；任务按租户+任务缓存约 60 秒，到期重新读取完整 200，无 ETag、过期缓存或 MQ 配置回退；已接纳执行固定原快照，暂停/停止控制不等待缓存。项目内 v0.4 `call.execute` 入站只有 `task_id/callee`，调用路由、主叫、AI 和时限绑定已授权任务快照；历史 MQ 命令不因年龄过期，但拨号前门禁不放宽。任务终止仅允许**显式、条件式**清理任务配置副本，不删执行恢复、幂等或 outbox。SIP 改动须关准入、排空及核验实际加载；外部 opt-out 现行 MQ 即时语义不可擅自改称已切换为最终 `call.result`。F01/F07 本地 Schema、正反例、来源/hash 与 Mock C 已通过，外部签收/连通仍未验证。 - 凭据/供应商端点来自受控引用且有授权/出口校验，不能因可调参数绕过安全硬限额或启用不安全重试。OpenAI默认自动重试显式关闭；日志只留脱敏版本/摘要/有效参数，不打印prompt/变量/密钥。 - 静态发布只约束SIP/节点制品，不将AI配置硬编码；GAP-08/09及SDK参数PoC为P1门禁，验证入口见验收§5.1（现有E/L项子场景，不新增虚假通过数）。 @@ -167,11 +167,11 @@ - **本轮项目内目标（尚未替换真实外部运行）**：任务（含智能体）、SIP、任务发现和按 tenant_id 的租户额度走四条只读 HTTP；呼叫、控制及其必要回执/单份最终结果走 MQ，取消对外业务查询/补传和分散通话事件。不提供配置HTTP→MQ回退，也不恢复其它业务HTTP通道。SaaS须分发 management 已批准的唯一 SIP 版本，management 仍为唯一编辑/审批面。该本地语义按第三方契约及版本化 Schema/示例/hash 冻结，Mock C 是本地门禁，不等待外部签收；真实切换仍需另行授权。 - 新HTTP配置字段阶段草案见 `docs/contracts/config-read-fields-v0.1-proposal.md` 及同目录 `config-read-v0.1.schema.json`/mock示例；截图只证实UI含义，英文响应键为项目自定义，绝非SaaS现网接口已确认字段。用户新增任务排除日期、线路时段等未见截图项按项目需求设计；SIP传输/鉴权/注册及额度未知不能猜默认值。Schema/Mock 校验可满足项目内 C，但不代表 SaaS/management 已发布或真实兼容；真实响应、审批来源、摘要和 Agent/Asterisk 实际加载仍须单独验证。当前唯一权威运行契约不因草案变化。 - **每个Dispatcher必须有独立、全局唯一且不重复的ID及独立接收Topic/队列**；指定D的任务/现行MQ配置结果/上传结果不能由其它D抢收，也不能广播后仅靠正文过滤；新目标只读HTTP配置由该D UUID+SECRETKEY获取且须核对任务归属。身份与tenant/Agent/Cell ID、dispatcher_epoch分开；现行合同保留租户独立队列及原值tenant_key，完整新路由长度预算须重验。具体ID生成/持久化、Topic/绑定、消息字段/关联/错误/期限须随W01新版本冻结，不凭本文给旧严格Schema添加字段。 -- **本轮项目内任务队列所有权硬边界（见第三方契约 §1.2 与 `mq-topology-v0.1-proposal.json`）：任务队列和 RabbitMQ 绑定只能由 SaaS 创建、维护、退役；Dispatcher 只消费，不能自行建队、绑定或删除；F03 v0.3 已本地验证，外部兼容性未验证。** SaaS 须先确认持久任务/控制队列与绑定就绪，再向归属 D+任务 ID 的队列发布 persistent 命令；D 离线时已存在队列可积压，缺队列而未入队的原消息由 SaaS 保留并按原身份重发。现行项目内任务发现从 `GET /internal/v1/dispatcher/tasks?after=0` 开始，以持久十进制事件游标逐页读取同一 `tasks[]`（每页最多 256、每 D 活跃任务最多 256），后续约每 30 秒从已提交游标续读；同任务更新可再次出现，`status=removed` 为墓碑，空页保持游标并表示追平。**无 410/游标过期重建、`changes` 兼容或配置回退；不得用旧 v0.2 checkpoint 解释事件 ID。** SaaS 必须保证分页完整有序、序号及撤销墓碑长期可读；累计墓碑不受活跃任务上限约束，容量和灾备连续性尚待外部签收。每页任务/归属/响应/游标同一 SQLite 事务提交；首次未追平、请求/响应/持久化错误均持久关新准入、停任务消费，但 MQ 即时控制和结果恢复照常处理，不自动拨号或重置游标。轮询周期不是端到端发现 SLA，也不能取代 MQ 控制。发现页的较新 `running` 只更新任务状态和游标，不解除已持久的 paused；pause 保留积压，只有 MQ resume 重新读取单任务并确认 running 后才恢复原积压且不延长期限；stop 持久生效后未接纳旧消息**静默消费/ACK，不拨号、不回逐条结果**，不删队列；控制本身及在途通话仍有处理/最终结果，本地错误/计数不静默。任务带 tenant_id/原值 tenant_key，同 D 各任务共用按 tenant_id 取得的租户额度；额度 0/缺失/过期拒新，降额不强挂或清未知占用，跨 D 仍需权威份额。已移除 `--tenant-key` 及自建租户队列，按 D+task ID 推导 SaaS 预建队列；本地 RabbitMQ 无 `configure` 权限测试通过。v0.1/v0.2 发现证据仅作历史，v0.3 本地证据见 `docs/evidence/f09-local-acceptance-v0.3.md`；不证明真实 SaaS/management、多 D 业务或生产可用。 +- **v0.4 任务发现与队列所有权硬边界**：SaaS 独占创建、维护、退役每 D 的独立控制队列与每任务任务队列及绑定；Dispatcher 仅消费，不能自行建队、绑定或删除。本地 RabbitMQ 无 `configure` 权限 Mock 已通过，真实兼容性未验证。新加入或重启 D 先 `GET /internal/v1/dispatcher/tasks?mode=snapshot` 逐页取得同一 `snapshot_id`/`watermark` 的完整清单，全部校验后一次 SQLite 事务提交；独立控制队列积压处理完成前不开任务准入。运行期 `GET .../tasks?after=<内存游标>` 仅发现归属变更，每页持久提交后推进内存游标；重启不恢复旧 v0.3 持久事件游标，分页或持久化失败关新准入、停任务消费，MQ 控制及结果恢复照常处理。HTTP 的偶发状态与 MQ 已应用状态冲突则关准入，不让发现页覆盖已持久的 pause/stop；只有 MQ resume 经单任务新鲜状态确认才恢复原积压。stop 挂断、排空后回自身控制回执，未接纳旧外呼静默 ACK、不拨号、不回逐条结果，也不删 SaaS 任务队列。历史 `task.control` 与 `call.execute` 不因消息年龄过期，外呼仍在接纳和拨号前独立检查任务/白名单/时段/授权/额度及通话时限，未来 `issued_at` 不提前接纳。保留原值 tenant_key、租户额度和未知占用；跨 D 份额仍待外部冻结。v0.1–v0.3 发现证据仅作历史，v0.4 本地证据见 `docs/evidence/dispatcher-v04-local-acceptance.md`；不能证明真实 SaaS/management、多 D 或生产可用。 - **OSS相关配置存于Dispatcher配置文件，Agent向Dispatcher领取临时上传TOKEN后直传OSS，不保存长期凭据；SaaS不再下发OSS配置/TOKEN。** D复用官方SDK提供受限TOKEN/目标信息，配置缺失/无效明确失败；不在样例、源码、日志或证据中保存实际密钥/完整TOKEN。过期只允许A显式向D重新申请，不自动续期或向SaaS申请TOKEN；精确配置格式/TOKEN形态/UploadGrant映射另行核验，不猜字段。 - **本轮项目内目标：上传仅负责 Agent 直传及 D 可靠通知 MQ，SaaS 后续处理不属本项目职责；外部 v2 的现行上传事实不因本地目标自动改变。** D保留签发能力、不转发文件；R13持久保存事实与recording.uploaded outbox，消息为persistent、进入指定durable队列/绑定、mandatory无return且publisher confirm成功后才记交付完成。只写本地outbox不算入队；不申请SaaS会话、不等待verified/OSS ID、不新增VERIFYING、不伪造SaaS结果。 - 新版recording.uploaded取代本项目recording.ready，字段为call_id/recording_id/upload_id/bucket/object_key/format/channels/sample_rate_hz/duration_ms/size_bytes/checksum_sha256；不含TOKEN/密钥/签名URL。MQ失败/确认丢失/重启只恢复原消息身份的交付，不重新PUT或新建资产。现行AI/控制等必要请求响应不受此收缩影响；新目标AI改用任务只读HTTP获取，控制仍走MQ。 -- 前一轮MQ-only文档纠正已结束；以下是**现行 MQ-only v1 合同的历史基线**，不得误当作新只读配置HTTP方案的限制或验收通过：现行机器依据见 `contracts/upstream/v1/mq-topology.json`、`mq.schema.json` 和 `event-payloads.schema.json`，第三方顺序说明见 `docs/thirds/第三方对接事件与请求消费顺序_v0.1.md`：纯Topic精确绑定、拒绝独立通配词段、tenant_key预算196 UTF-8字节、稳定UUID v4、旧版严格JSON配置及15分钟SDK预签名PUT；新目标保留业务MQ及OSS规则，仅替换配置获取，须另发版本化合同，不原地复用旧Schema。当前仅保留现有代码需要的契约包；未发生契约迁移前不新增契约，旧快照不作为当前工作树输入。新版须完成Schema/正反例/哈希验证后发布，不把方案确认当实现完成。历史 W01/W02/W04/W05/W07/W08/W11/W12/W13/W14 范围见归档 `docs/archive/plan-0918.md` §8.2；本轮新配置/控制路径须按 `docs/plan-config-read-v0.1.md` §4–§8 重新验证。全局唯一D身份/专用Topic及本地D1/D2隔离fixture为当前合同要求，不授权双D业务运行、HA或共享额度。 +- 前一轮MQ-only文档纠正已结束；以下是**现行 MQ-only v1 合同的历史基线**，不得误当作新只读配置HTTP方案的限制或验收通过：现行机器依据见 `contracts/upstream/v1/mq-topology.json`、`mq.schema.json` 和 `event-payloads.schema.json`，第三方顺序说明见 `docs/thirds/第三方对接事件与请求消费顺序_v0.1.md`：纯Topic精确绑定、拒绝独立通配词段、tenant_key预算196 UTF-8字节、稳定UUID v4、旧版严格JSON配置及15分钟SDK预签名PUT；新目标保留业务MQ及OSS规则，仅替换配置获取，须另发版本化合同，不原地复用旧Schema。当前仅保留现有代码需要的契约包；未发生契约迁移前不新增契约，旧快照不作为当前工作树输入。新版须完成Schema/正反例/哈希验证后发布，不把方案确认当实现完成。历史 W01/W02/W04/W05/W07/W08/W11/W12/W13/W14 范围见归档 `docs/archive/plan-0918.md` §8.2；项目内 v0.4 配置/控制路径以 `docs/plan-dispatcher-state-v0.1.md` 和本地验收证据为当前入口；外部合同另行签收。全局唯一D身份/专用Topic及本地D1/D2隔离fixture为当前合同要求，不授权双D业务运行、HA或共享额度。 - **现行已发布合同**中的实时文字事件名为transcript.updated，不新增call.transcript别名；OSS文本归档不能冒充当前实时文字/opt-out，缺少专用资产授权接口时明确未启用，不能伪装recording.ready。 - **用户已批准的项目内精简契约目标**见 `docs/plan-config-read-v0.1.md` §3.5/§4 和第三方对接契约：SaaS↔D 移除对外查询/补传命令及分散通话/转写/拒联/录音事件，保留呼叫/控制命令和必要回执；录音仍上传 OSS，D 每通话只回传一份含最终转写、拒联事实与 OSS 路径的 `call.result`。本地实现以 F01/F07 Schema、正反例、来源/hash 和 Mock C 为门禁，无需额外外部/业务签收；本地验收更新后不要求通话前实时文字或拒联。无录音按 `not_created` 收口；预期录音明确失败立即 `unavailable`，未知最迟于 `call.ended_at + 15m` 收口。通话确认终结释放执行资源后释放占用，不等待 OSS 或 MQ confirm；未知仍占用。现行外部 Schema/代码保持原样，真实兼容性未验证。 - 现有MQ信封command_type/command_id、event_type/aggregate_*与正文已对齐；事件payload专属约束尚需补齐，不把通用object校验当完整验收。字段索引只读生成，不手改成第二套Schema。 @@ -180,7 +180,7 @@ - 配额覆盖所有 Cell/实例及未知占用；租约过期不自动释放不明通话。**现行 v2** 控制 CAS 为 expected_task_revision，pause 与 stop 的 drain/hangup 区分；paused 可按新授权恢复，stopped 不可恢复。现行整体补传仅 call_id/source_command_id；下一版对外查询/补传目标取消但严格合同/代码未改。最后发起许可、权限和屏障须故障注入。 - **用户已批准的项目内精简契约目标（不能混写现行 v2；外部签收不阻塞本地 C）**：`call.execute.payload` 仅 `task_id/callee`，外呼信封身份仍用于防止重拨；路由/主叫/智能体版本与 `ring_timeout_ms/max_call_duration_ms` 均从已批准的任务配置取得并持久绑定。`task.control` 的 pause/resume/stop 均不带 `command_id` 或 `expected_task_revision`，不设计控制去重，但 D 必须回 task/action/status 处理结果；乱序、控制重投/回执丢失与 stop 后 resume 的本地判定写入第三方契约并由 Mock 测试；外部签收不是本地阻塞项，不偷偷重引入 CAS/去重掩盖。任务配置新增明确 `caller_profile_id`，按allowed_trunk_ids顺序选首个时段/额度/加载/主叫均匹配的线路，选后固定、不自动换线重拨；有效通话上限取任务与已授权AI两者较小值。旧running配置/发现不得解除已持久的paused/stopped，resume须强制最新任务核验，stopped同ID不可逆。拟定配置响应不再包含 `agent.content_sha256`；SIP 快照/录音 checksum 为另有用途的字段，不误删。参见 `docs/plan-config-read-v0.1.md` 和 `docs/thirds/第三方对接事件与请求消费顺序_v0.1.md`。 - management是SIP配置唯一编辑/审批面。**现行 P1**通过批准的版本化静态制品和受控部署入口交付，D核验目标/准入屏障，Agent加载并报告；不要求在线发布控制面。静态交接合同须批准，旧直接写Agent面不能同时启用；成功必须证明精确快照已被Asterisk加载。 -- **本轮项目内 SIP 配置读取目标（现行外部运行未切换）**：management 仍是 SIP 唯一编辑/审批方，SaaS 仅经只读 HTTP 提供该 D 资源分区的获批全量快照；新加入/重启 D 先查询、核验所属 Agent/Asterisk 实际加载的版本/摘要后才消费执行队列。后续约每60秒核对版本，有变化就关执行准入、排空旧活动通话并确认新版已加载；控制MQ照常处理。SaaS/management 分属不同配置系统时须证明SaaS分发的是同一份获批制品；不引入共享业务DB或MQ配置回退。本地 SIP 读取路径按本轮第三方契约、计划和 Mock C 实施；当前运行代码尚未切换。真实 SaaS/management 兼容性及多 D 资源配额份额仍未验证、不属本轮验收，详见 `docs/architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md` §3.2。 +- **本轮项目内 SIP 配置读取目标（现行外部运行未切换）**：management 仍是 SIP 唯一编辑/审批方，SaaS 仅经只读 HTTP 提供该 D 资源分区的获批全量快照；新加入/重启 D 先查询、核验所属 Agent/Asterisk 实际加载的版本/摘要后才消费执行队列。后续约每60秒核对版本，有变化就关执行准入、排空旧活动通话并确认新版已加载；控制MQ照常处理。SaaS/management 分属不同配置系统时须证明SaaS分发的是同一份获批制品；不引入共享业务DB或MQ配置回退。项目内 SIP 读取及准入链路已用隔离 Mock 验证；真实 Agent/Asterisk 加载与 SaaS/management 切换尚未验证。真实 SaaS/management 兼容性及多 D 资源配额份额仍未验证、不属本轮验收，详见 `docs/architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md` §3.2。 ## 安全和真实验证 diff --git a/contracts/contracts.go b/contracts/contracts.go index 5e5ca7c..d8031a0 100644 --- a/contracts/contracts.go +++ b/contracts/contracts.go @@ -11,13 +11,13 @@ import ( // Files contains pinned upstream schemas and the active local contract versions. // Runtime code never reads a checkout or resolves schema refs online. // -//go:embed upstream local/v0.1 local/v0.3 +//go:embed upstream local/v0.1 local/v0.3 local/v0.4 var Files embed.FS const SourceCommit = "v1" func ReadLocal(version, name string) ([]byte, error) { - if (version != "v0.1" && version != "v0.3") || name == "" || path.Base(name) != name || !strings.HasSuffix(name, ".schema.json") { + if (version != "v0.1" && version != "v0.3" && version != "v0.4") || name == "" || path.Base(name) != name || !strings.HasSuffix(name, ".schema.json") { return nil, fmt.Errorf("invalid project-local schema %q/%q", version, name) } return Files.ReadFile(path.Join("local", version, name)) diff --git a/contracts/local/v0.4/call-execute-v0.4-proposal.schema.json b/contracts/local/v0.4/call-execute-v0.4-proposal.schema.json new file mode 100644 index 0000000..bf88383 --- /dev/null +++ b/contracts/local/v0.4/call-execute-v0.4-proposal.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://go-sip.local/contracts/proposals/call-execute-v0.4-proposal.schema.json", + "title": "Project-local call.execute ingress; external SaaS compatibility unverified", + "$comment": "Only replaces call.execute ingress in command-next.v0.1-proposal. This MQ command has no expiry; dispatcher must obtain and check fresh independent authorization before admission and originate. The command.result egress remains v0.1.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "dispatcher_id", "tenant_id", "tenant_key", "trace_id", "issued_at", "command_id", "command_type", "payload"], + "properties": { + "schema_version": {"const": "call-execute.v0.4-proposal"}, + "dispatcher_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/dispatcherId"}, + "tenant_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "tenant_key": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/tenantKey"}, + "trace_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "issued_at": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/time"}, + "command_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "command_type": {"const": "call.execute"}, + "payload": { + "type": "object", "additionalProperties": false, + "required": ["task_id", "callee"], + "properties": { + "task_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{1,128}$"}, + "callee": {"type": "string", "minLength": 1} + } + } + } +} diff --git a/contracts/local/v0.4/task-control-v0.4-proposal.schema.json b/contracts/local/v0.4/task-control-v0.4-proposal.schema.json new file mode 100644 index 0000000..c519298 --- /dev/null +++ b/contracts/local/v0.4/task-control-v0.4-proposal.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://go-sip.local/contracts/proposals/task-control-v0.4-proposal.schema.json", + "title": "Project-local inbound pause/resume/stop commands; external compatibility unverified", + "$comment": "Replaces only task.control ingress in command-next.v0.1-proposal. Existing command.result egress remains under the v0.1 contract. Pause and stop always require hangup; no active_call_policy or MQ command expiry field.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "dispatcher_id", "tenant_id", "tenant_key", "trace_id", "issued_at", "command_type", "payload"], + "properties": { + "schema_version": {"const": "task-control.v0.4-proposal"}, + "dispatcher_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/dispatcherId"}, + "tenant_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "tenant_key": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/tenantKey"}, + "trace_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "issued_at": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/time"}, + "command_type": {"const": "task.control"}, + "payload": { + "type": "object", "additionalProperties": false, + "required": ["task_id", "action", "reason"], + "properties": { + "task_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{1,128}$"}, + "action": {"enum": ["pause", "resume", "stop"]}, + "reason": {"type": "string", "minLength": 1, "maxLength": 512} + } + } + } +} diff --git a/contracts/local/v0.4/task-discovery-v0.4-proposal.schema.json b/contracts/local/v0.4/task-discovery-v0.4-proposal.schema.json new file mode 100644 index 0000000..4f13a0a --- /dev/null +++ b/contracts/local/v0.4/task-discovery-v0.4-proposal.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://go-sip.local/contracts/proposals/task-discovery-v0.4-proposal.schema.json", + "title": "Project-local restart snapshot and live assignment changes; SaaS compatibility unverified", + "oneOf": [ + {"$ref": "#/$defs/snapshot_page"}, + {"$ref": "#/$defs/change_page"}, + {"$ref": "#/$defs/error_response"} + ], + "$defs": { + "dispatcher_id": {"$ref": "https://go-sip.local/contracts/proposals/task-discovery-v0.3-proposal.schema.json#/$defs/dispatcher_id"}, + "cursor": {"$ref": "https://go-sip.local/contracts/proposals/task-discovery-v0.3-proposal.schema.json#/$defs/cursor"}, + "task": {"$ref": "https://go-sip.local/contracts/proposals/task-discovery-v0.3-proposal.schema.json#/$defs/task"}, + "snapshot_page": { + "type": "object", "additionalProperties": false, + "required": ["schema_version", "mode", "dispatcher_id", "snapshot_id", "watermark", "next_page_token", "tasks"], + "properties": { + "schema_version": {"const": "task-discovery.v0.4-proposal"}, + "mode": {"const": "snapshot"}, + "dispatcher_id": {"$ref": "#/$defs/dispatcher_id"}, + "snapshot_id": {"type": "string", "format": "uuid", "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"}, + "watermark": {"$ref": "#/$defs/cursor"}, + "next_page_token": {"type": ["string", "null"], "minLength": 1, "maxLength": 256}, + "tasks": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/task"}} + }, + "$comment": "Every page has identical snapshot_id and watermark; null next_page_token ends this consistent complete inventory. No task appears twice, and removed tasks are omitted only from the complete inventory. Server retains stopped tasks with not-yet-retired queues. Semantic client checks enforce consistency." + }, + "change_page": { + "type": "object", "additionalProperties": false, + "required": ["schema_version", "mode", "dispatcher_id", "next_cursor", "tasks"], + "properties": { + "schema_version": {"const": "task-discovery.v0.4-proposal"}, + "mode": {"const": "changes"}, + "dispatcher_id": {"$ref": "#/$defs/dispatcher_id"}, + "next_cursor": {"$ref": "#/$defs/cursor"}, + "tasks": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/task"}} + }, + "$comment": "Live-only event cursor; ordered complete event positions per dispatcher, no server expiry while the process is running. Removed tombstones are explicit; status values are not online control commands. A nonempty page advances the cursor, and an empty page repeats it." + }, + "error_response": { + "type": "object", "additionalProperties": false, + "required": ["schema_version", "resource", "error"], + "properties": { + "schema_version": {"const": "task-discovery.v0.4-proposal"}, + "resource": {"const": "error"}, + "error": { + "type": "object", "additionalProperties": false, + "required": ["code", "message"], + "properties": { + "code": {"enum": ["invalid_request", "invalid_page_token", "snapshot_unavailable", "unauthorized", "dispatcher_not_authorized", "service_unavailable"]}, + "message": {"type": "string", "minLength": 1, "maxLength": 256} + } + } + } + } + } +} diff --git a/contracts/local_v04_test.go b/contracts/local_v04_test.go new file mode 100644 index 0000000..8353b3f --- /dev/null +++ b/contracts/local_v04_test.go @@ -0,0 +1,53 @@ +package contracts + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestReadLocalV04SchemasAndRejectInvalidPaths(t *testing.T) { + for _, name := range []string{ + "task-discovery-v0.4-proposal.schema.json", + "task-control-v0.4-proposal.schema.json", + "call-execute-v0.4-proposal.schema.json", + } { + data, err := ReadLocal("v0.4", name) + if err != nil || !json.Valid(data) { + t.Fatalf("embedded v0.4 schema %s: valid=%v err=%v", name, json.Valid(data), err) + } + original, err := Files.ReadFile("local/v0.4/" + name) + if err != nil || !bytes.Equal(data, original) { + t.Fatalf("runtime schema differs from embedded bundle: %s err=%v", name, err) + } + } + for _, input := range []struct{ version, name string }{ + {"v0.2", "task-control-v0.4-proposal.schema.json"}, + {"v0.4", ""}, + {"v0.4", "../task-control-v0.4-proposal.schema.json"}, + {"v0.4", "task-control-v0.4-proposal.json"}, + {"v0.4", "does-not-exist.schema.json"}, + } { + if _, err := ReadLocal(input.version, input.name); err == nil { + t.Fatalf("invalid or missing embedded schema accepted: %+v", input) + } + } +} + +func TestReadUpstreamContractAndDecodeErrors(t *testing.T) { + data, err := Read("mq.schema.json") + if err != nil || !json.Valid(data) { + t.Fatalf("upstream MQ schema: valid=%v err=%v", json.Valid(data), err) + } + var schema map[string]json.RawMessage + if err := ReadJSON("mq.schema.json", &schema); err != nil || len(schema) == 0 { + t.Fatalf("decode upstream MQ schema: fields=%d err=%v", len(schema), err) + } + if err := ReadJSON("missing.schema.json", &schema); err == nil { + t.Fatal("missing upstream contract did not return an error") + } + var scalar int + if err := ReadJSON("mq.schema.json", &scalar); err == nil { + t.Fatal("invalid upstream schema destination did not return a decode error") + } +} diff --git a/docs/architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md b/docs/architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md index b9e9a4c..9da15eb 100644 --- a/docs/architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md +++ b/docs/architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md @@ -1,6 +1,6 @@ # Dispatcher 有界接纳与控制通道改造计划 v0.1 -**项目内 P1 设计计划，尚未实施；本地 F01/F07 Schema、正反例、来源/hash 与 Mock C 是实施门禁，不要求 SaaS 外部签收。** 唯一计划/台账是[plan-config-read-v0.1](../plan-config-read-v0.1.md)；项目内对接结构见[第三方对接契约](../thirds/第三方对接事件与请求消费顺序_v0.1.md)。本文同步替代此前的 ETag/304、控制 CAS、暂停丢弃旧积压和逐条停止回执方案，不修改现行 `contracts/upstream/v1/`；外部兼容性未验证。 +**历史设计计划；不能单独作为当前运行/验收状态的依据。** 当前状态以[总计划](../plan-config-read-v0.1.md)和[任务发现 v0.3](../thirds/v0.3.md)为准；下一版“重启全量、在线增量、pause/stop 全部挂断及条件清理”仅见[新计划](../plan-dispatcher-state-v0.1.md)/[v0.4 草案](../thirds/v0.4.md)。以下启动全量/水位、drain/hangup 与旧分页验收项保留历史，不作为已实现 v0.3 或 v0.4 的冻结合同。原设计 F01/F07 Schema、正反例、来源/hash 与 Mock C 是项目内门禁，不要求 SaaS 外部签收；外部兼容性仍未验证。 路径来源：`/internal/v1/dispatcher/sip`、`/internal/v1/dispatcher/task/:task_id`、`/internal/v1/dispatcher/tasks`（含 `?after=`）为用户给定路径；`/internal/v1/dispatcher/tenant/:tenant_id/quota` 与任务路由字段 `route_policy_id`、`caller_profile_id`、`allowed_trunk_ids` 为项目内目标，按第三方对接契约和 F01/F07 Schema 实施；不代表 SaaS 现网字段，真实兼容性另记。 diff --git a/docs/contracts/call-execute-v0.4-proposal.schema.json b/docs/contracts/call-execute-v0.4-proposal.schema.json new file mode 100644 index 0000000..bf88383 --- /dev/null +++ b/docs/contracts/call-execute-v0.4-proposal.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://go-sip.local/contracts/proposals/call-execute-v0.4-proposal.schema.json", + "title": "Project-local call.execute ingress; external SaaS compatibility unverified", + "$comment": "Only replaces call.execute ingress in command-next.v0.1-proposal. This MQ command has no expiry; dispatcher must obtain and check fresh independent authorization before admission and originate. The command.result egress remains v0.1.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "dispatcher_id", "tenant_id", "tenant_key", "trace_id", "issued_at", "command_id", "command_type", "payload"], + "properties": { + "schema_version": {"const": "call-execute.v0.4-proposal"}, + "dispatcher_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/dispatcherId"}, + "tenant_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "tenant_key": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/tenantKey"}, + "trace_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "issued_at": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/time"}, + "command_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "command_type": {"const": "call.execute"}, + "payload": { + "type": "object", "additionalProperties": false, + "required": ["task_id", "callee"], + "properties": { + "task_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{1,128}$"}, + "callee": {"type": "string", "minLength": 1} + } + } + } +} diff --git a/docs/contracts/examples/call-execute-altcallee-v0.4.json b/docs/contracts/examples/call-execute-altcallee-v0.4.json new file mode 100644 index 0000000..da4cfaa --- /dev/null +++ b/docs/contracts/examples/call-execute-altcallee-v0.4.json @@ -0,0 +1,11 @@ +{ + "schema_version": "call-execute.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "command_id": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + "issued_at": "2026-09-27T02:00:00Z", + "command_type": "call.execute", + "payload": {"task_id": "a02", "callee": "15830461047"} +} diff --git a/docs/contracts/examples/call-execute-invalid-expiry-v0.4.json b/docs/contracts/examples/call-execute-invalid-expiry-v0.4.json new file mode 100644 index 0000000..45cedee --- /dev/null +++ b/docs/contracts/examples/call-execute-invalid-expiry-v0.4.json @@ -0,0 +1,12 @@ +{ + "schema_version": "call-execute.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "44444444-4444-4444-8444-444444444444", + "command_id": "77777777-7777-4777-8777-777777777777", + "issued_at": "2026-09-20T01:00:00Z", + "not_after": "2026-09-20T01:05:00Z", + "command_type": "call.execute", + "payload": {"task_id": "a01", "callee": "15003164745"} +} diff --git a/docs/contracts/examples/call-execute-old-v0.4.json b/docs/contracts/examples/call-execute-old-v0.4.json new file mode 100644 index 0000000..20a5575 --- /dev/null +++ b/docs/contracts/examples/call-execute-old-v0.4.json @@ -0,0 +1,11 @@ +{ + "schema_version": "call-execute.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "44444444-4444-4444-8444-444444444444", + "command_id": "77777777-7777-4777-8777-777777777777", + "issued_at": "2026-09-20T01:00:00Z", + "command_type": "call.execute", + "payload": {"task_id": "a01", "callee": "15003164745"} +} diff --git a/docs/contracts/examples/call-execute-recent-v0.4.json b/docs/contracts/examples/call-execute-recent-v0.4.json new file mode 100644 index 0000000..b4c1048 --- /dev/null +++ b/docs/contracts/examples/call-execute-recent-v0.4.json @@ -0,0 +1,11 @@ +{ + "schema_version": "call-execute.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "88888888-8888-4888-8888-888888888888", + "command_id": "99999999-9999-4999-8999-999999999999", + "issued_at": "2026-09-27T01:00:00Z", + "command_type": "call.execute", + "payload": {"task_id": "a01", "callee": "15003164745"} +} diff --git a/docs/contracts/examples/task-control-invalid-drain-v0.4.json b/docs/contracts/examples/task-control-invalid-drain-v0.4.json new file mode 100644 index 0000000..6a4d38c --- /dev/null +++ b/docs/contracts/examples/task-control-invalid-drain-v0.4.json @@ -0,0 +1,10 @@ +{ + "schema_version": "task-control.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "44444444-4444-4444-8444-444444444444", + "issued_at": "2026-09-27T01:00:00Z", + "command_type": "task.control", + "payload": {"task_id": "a01", "action": "pause", "active_call_policy": "drain", "reason": "paused by SaaS"} +} diff --git a/docs/contracts/examples/task-control-invalid-expiry-v0.4.json b/docs/contracts/examples/task-control-invalid-expiry-v0.4.json new file mode 100644 index 0000000..43a6122 --- /dev/null +++ b/docs/contracts/examples/task-control-invalid-expiry-v0.4.json @@ -0,0 +1,11 @@ +{ + "schema_version": "task-control.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "44444444-4444-4444-8444-444444444444", + "issued_at": "2026-09-20T01:00:00Z", + "not_after": "2026-09-20T01:05:00Z", + "command_type": "task.control", + "payload": {"task_id": "a01", "action": "stop", "reason": "stopped by SaaS"} +} diff --git a/docs/contracts/examples/task-control-pause-v0.4.json b/docs/contracts/examples/task-control-pause-v0.4.json new file mode 100644 index 0000000..e5ebddc --- /dev/null +++ b/docs/contracts/examples/task-control-pause-v0.4.json @@ -0,0 +1,10 @@ +{ + "schema_version": "task-control.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "44444444-4444-4444-8444-444444444444", + "issued_at": "2026-09-27T01:00:00Z", + "command_type": "task.control", + "payload": {"task_id": "a01", "action": "pause", "reason": "paused by SaaS"} +} diff --git a/docs/contracts/examples/task-control-resume-v0.4.json b/docs/contracts/examples/task-control-resume-v0.4.json new file mode 100644 index 0000000..ab3d8f0 --- /dev/null +++ b/docs/contracts/examples/task-control-resume-v0.4.json @@ -0,0 +1,10 @@ +{ + "schema_version": "task-control.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "66666666-6666-4666-8666-666666666666", + "issued_at": "2026-09-27T01:02:00Z", + "command_type": "task.control", + "payload": {"task_id": "a01", "action": "resume", "reason": "resumed by SaaS"} +} diff --git a/docs/contracts/examples/task-control-stop-v0.4.json b/docs/contracts/examples/task-control-stop-v0.4.json new file mode 100644 index 0000000..6ab6ff6 --- /dev/null +++ b/docs/contracts/examples/task-control-stop-v0.4.json @@ -0,0 +1,10 @@ +{ + "schema_version": "task-control.v0.4-proposal", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tenant_id": "33333333-3333-4333-8333-333333333333", + "tenant_key": "tenant-A", + "trace_id": "55555555-5555-4555-8555-555555555555", + "issued_at": "2026-09-27T01:01:00Z", + "command_type": "task.control", + "payload": {"task_id": "a01", "action": "stop", "reason": "stopped by SaaS"} +} diff --git a/docs/contracts/examples/task-discovery-changes-v0.4.json b/docs/contracts/examples/task-discovery-changes-v0.4.json new file mode 100644 index 0000000..828072c --- /dev/null +++ b/docs/contracts/examples/task-discovery-changes-v0.4.json @@ -0,0 +1,9 @@ +{ + "schema_version": "task-discovery.v0.4-proposal", + "mode": "changes", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "tasks": [ + {"task_id": "a01", "tenant_id": "tenant-1", "tenant_key": "tenant-A", "status": "removed", "task_revision": 6} + ], + "next_cursor": "43" +} diff --git a/docs/contracts/examples/task-discovery-error-v0.4.json b/docs/contracts/examples/task-discovery-error-v0.4.json new file mode 100644 index 0000000..612622e --- /dev/null +++ b/docs/contracts/examples/task-discovery-error-v0.4.json @@ -0,0 +1,8 @@ +{ + "schema_version": "task-discovery.v0.4-proposal", + "resource": "error", + "error": { + "code": "snapshot_unavailable", + "message": "Snapshot is no longer available; restart full discovery with admission closed" + } +} diff --git a/docs/contracts/examples/task-discovery-invalid-v0.4.json b/docs/contracts/examples/task-discovery-invalid-v0.4.json new file mode 100644 index 0000000..7ba7168 --- /dev/null +++ b/docs/contracts/examples/task-discovery-invalid-v0.4.json @@ -0,0 +1,10 @@ +{ + "schema_version": "task-discovery.v0.4-proposal", + "mode": "snapshot", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "snapshot_id": "22222222-2222-4222-8222-222222222222", + "watermark": "42", + "tasks": [], + "next_page_token": null, + "unexpected_unapproved_field": true +} diff --git a/docs/contracts/examples/task-discovery-snapshot-page1-v0.4.json b/docs/contracts/examples/task-discovery-snapshot-page1-v0.4.json new file mode 100644 index 0000000..9238cdb --- /dev/null +++ b/docs/contracts/examples/task-discovery-snapshot-page1-v0.4.json @@ -0,0 +1,11 @@ +{ + "schema_version": "task-discovery.v0.4-proposal", + "mode": "snapshot", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "snapshot_id": "22222222-2222-4222-8222-222222222222", + "watermark": "42", + "tasks": [ + {"task_id": "a01", "tenant_id": "tenant-1", "tenant_key": "tenant-A", "status": "running", "task_revision": 5} + ], + "next_page_token": "page-2" +} diff --git a/docs/contracts/examples/task-discovery-snapshot-page2-v0.4.json b/docs/contracts/examples/task-discovery-snapshot-page2-v0.4.json new file mode 100644 index 0000000..4cda94f --- /dev/null +++ b/docs/contracts/examples/task-discovery-snapshot-page2-v0.4.json @@ -0,0 +1,11 @@ +{ + "schema_version": "task-discovery.v0.4-proposal", + "mode": "snapshot", + "dispatcher_id": "11111111-1111-4111-8111-111111111111", + "snapshot_id": "22222222-2222-4222-8222-222222222222", + "watermark": "42", + "tasks": [ + {"task_id": "a02", "tenant_id": "tenant-1", "tenant_key": "tenant-A", "status": "paused", "task_revision": 3} + ], + "next_page_token": null +} diff --git a/docs/contracts/local-contract-manifest-v0.4.json b/docs/contracts/local-contract-manifest-v0.4.json new file mode 100644 index 0000000..1900424 --- /dev/null +++ b/docs/contracts/local-contract-manifest-v0.4.json @@ -0,0 +1,87 @@ +{ + "manifest_version": "local-contract-manifest.v0.4", + "hash_algorithm": "SHA-256", + "status": "project-local proposal; external SaaS compatibility unverified", + "source": { + "path": "docs/thirds/v0.4.md", + "sha256": "85a72ab93526a49aed78b9f762b005c46eaccfecaaaff50013f3af6be18a3470" + }, + "artifacts": [ + { + "path": "docs/contracts/task-discovery-v0.4-proposal.schema.json", + "sha256": "95c02504d432c01ee57d8074c63ea60dc1118dd92057ae138b2d1e4340b7d73c" + }, + { + "path": "docs/contracts/task-control-v0.4-proposal.schema.json", + "sha256": "5aa94f799da190a3e97c2fa4e911980ceac52662322e85bc2f57eecb5b21de3c" + }, + { + "path": "docs/contracts/task-discovery-v0.3-proposal.schema.json", + "sha256": "da8eda2e8f5416b2fb35f68e09a98f37d1417a878e9271b8e6d0d6824b94814c" + }, + { + "path": "contracts/upstream/v1/mq.schema.json", + "sha256": "d566899abbc8f0f8404cfe79ea58b0d1e7bf7e9fc663b09b282c15d5c74f505d" + }, + { + "path": "docs/contracts/examples/task-discovery-snapshot-page1-v0.4.json", + "sha256": "5085883ed0fd591644de091b944acd9d725714b6d5c48247b0428221ebc33867" + }, + { + "path": "docs/contracts/examples/task-discovery-snapshot-page2-v0.4.json", + "sha256": "064243f15bc33530d459f71c08f6b9d00fc776151ecaf4be4dc9a8582f5f993a" + }, + { + "path": "docs/contracts/examples/task-discovery-changes-v0.4.json", + "sha256": "f3afc04d6cae564d31c9165e889e99c109f92c7e1bdbb3853421321963c30b6e" + }, + { + "path": "docs/contracts/examples/task-discovery-error-v0.4.json", + "sha256": "71dd9986b49f56db792348443cf8595d7743447783129daf302d4550e73d5e96" + }, + { + "path": "docs/contracts/examples/task-discovery-invalid-v0.4.json", + "sha256": "ff840c254f7abb1f6a2d9d472f10bda9c646f2ee6cfd485da1d4b4c04018314c" + }, + { + "path": "docs/contracts/examples/task-control-pause-v0.4.json", + "sha256": "0adbeaed2517c33bd5b5b2e4e72c0c663263c4f32858b807f5f70190ad98a543" + }, + { + "path": "docs/contracts/examples/task-control-resume-v0.4.json", + "sha256": "95c1cb22147c79751f09310fcade441835d02893bb6dc337fc49a1ab5e870358" + }, + { + "path": "docs/contracts/examples/task-control-stop-v0.4.json", + "sha256": "d84b2be049bf9348fd67446f3e9a075df2f248777b81ed76153bbef4f4894d4d" + }, + { + "path": "docs/contracts/examples/task-control-invalid-drain-v0.4.json", + "sha256": "36e6d1bee9f2255158696fb61d9754e968dafe1927145df56412ea8d23e46b39" + }, + { + "path": "docs/contracts/call-execute-v0.4-proposal.schema.json", + "sha256": "0e019e1775810bfc0616683f5dd6bd49d9d97de92b9c063c56966a894d2461ed" + }, + { + "path": "docs/contracts/examples/call-execute-altcallee-v0.4.json", + "sha256": "1c58585f9e30745b4ca21c029f22f416931ad40a00e24e6c8389f113e964435c" + }, + { + "path": "docs/contracts/examples/call-execute-invalid-expiry-v0.4.json", + "sha256": "b332703b0113f64e1f8bb77bac2ee312d9a22d6d4a69214784f557381137edc7" + }, + { + "path": "docs/contracts/examples/call-execute-old-v0.4.json", + "sha256": "8e2285c5dada3ee3e5511d5a687577142e6bdcf034da789a4a8b4001e3127327" + }, + { + "path": "docs/contracts/examples/call-execute-recent-v0.4.json", + "sha256": "d03f5f4b04fde957e21ac300246098286ff6260c45b68a192bacb658af4e8cde" + }, + { + "path": "docs/contracts/examples/task-control-invalid-expiry-v0.4.json", + "sha256": "614fda38465be236224858d3706e304a76dd651b905f8b84347492b68e96755e" + } + ] +} diff --git a/docs/contracts/task-control-v0.4-proposal.schema.json b/docs/contracts/task-control-v0.4-proposal.schema.json new file mode 100644 index 0000000..c519298 --- /dev/null +++ b/docs/contracts/task-control-v0.4-proposal.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://go-sip.local/contracts/proposals/task-control-v0.4-proposal.schema.json", + "title": "Project-local inbound pause/resume/stop commands; external compatibility unverified", + "$comment": "Replaces only task.control ingress in command-next.v0.1-proposal. Existing command.result egress remains under the v0.1 contract. Pause and stop always require hangup; no active_call_policy or MQ command expiry field.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "dispatcher_id", "tenant_id", "tenant_key", "trace_id", "issued_at", "command_type", "payload"], + "properties": { + "schema_version": {"const": "task-control.v0.4-proposal"}, + "dispatcher_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/dispatcherId"}, + "tenant_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "tenant_key": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/tenantKey"}, + "trace_id": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/id"}, + "issued_at": {"$ref": "https://go-sip.local/contracts/v1/mq.schema.json#/$defs/time"}, + "command_type": {"const": "task.control"}, + "payload": { + "type": "object", "additionalProperties": false, + "required": ["task_id", "action", "reason"], + "properties": { + "task_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{1,128}$"}, + "action": {"enum": ["pause", "resume", "stop"]}, + "reason": {"type": "string", "minLength": 1, "maxLength": 512} + } + } + } +} diff --git a/docs/contracts/task-discovery-v0.4-proposal.schema.json b/docs/contracts/task-discovery-v0.4-proposal.schema.json new file mode 100644 index 0000000..4f13a0a --- /dev/null +++ b/docs/contracts/task-discovery-v0.4-proposal.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://go-sip.local/contracts/proposals/task-discovery-v0.4-proposal.schema.json", + "title": "Project-local restart snapshot and live assignment changes; SaaS compatibility unverified", + "oneOf": [ + {"$ref": "#/$defs/snapshot_page"}, + {"$ref": "#/$defs/change_page"}, + {"$ref": "#/$defs/error_response"} + ], + "$defs": { + "dispatcher_id": {"$ref": "https://go-sip.local/contracts/proposals/task-discovery-v0.3-proposal.schema.json#/$defs/dispatcher_id"}, + "cursor": {"$ref": "https://go-sip.local/contracts/proposals/task-discovery-v0.3-proposal.schema.json#/$defs/cursor"}, + "task": {"$ref": "https://go-sip.local/contracts/proposals/task-discovery-v0.3-proposal.schema.json#/$defs/task"}, + "snapshot_page": { + "type": "object", "additionalProperties": false, + "required": ["schema_version", "mode", "dispatcher_id", "snapshot_id", "watermark", "next_page_token", "tasks"], + "properties": { + "schema_version": {"const": "task-discovery.v0.4-proposal"}, + "mode": {"const": "snapshot"}, + "dispatcher_id": {"$ref": "#/$defs/dispatcher_id"}, + "snapshot_id": {"type": "string", "format": "uuid", "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"}, + "watermark": {"$ref": "#/$defs/cursor"}, + "next_page_token": {"type": ["string", "null"], "minLength": 1, "maxLength": 256}, + "tasks": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/task"}} + }, + "$comment": "Every page has identical snapshot_id and watermark; null next_page_token ends this consistent complete inventory. No task appears twice, and removed tasks are omitted only from the complete inventory. Server retains stopped tasks with not-yet-retired queues. Semantic client checks enforce consistency." + }, + "change_page": { + "type": "object", "additionalProperties": false, + "required": ["schema_version", "mode", "dispatcher_id", "next_cursor", "tasks"], + "properties": { + "schema_version": {"const": "task-discovery.v0.4-proposal"}, + "mode": {"const": "changes"}, + "dispatcher_id": {"$ref": "#/$defs/dispatcher_id"}, + "next_cursor": {"$ref": "#/$defs/cursor"}, + "tasks": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/task"}} + }, + "$comment": "Live-only event cursor; ordered complete event positions per dispatcher, no server expiry while the process is running. Removed tombstones are explicit; status values are not online control commands. A nonempty page advances the cursor, and an empty page repeats it." + }, + "error_response": { + "type": "object", "additionalProperties": false, + "required": ["schema_version", "resource", "error"], + "properties": { + "schema_version": {"const": "task-discovery.v0.4-proposal"}, + "resource": {"const": "error"}, + "error": { + "type": "object", "additionalProperties": false, + "required": ["code", "message"], + "properties": { + "code": {"enum": ["invalid_request", "invalid_page_token", "snapshot_unavailable", "unauthorized", "dispatcher_not_authorized", "service_unavailable"]}, + "message": {"type": "string", "minLength": 1, "maxLength": 256} + } + } + } + } + } +} diff --git a/docs/evidence/dispatcher-v04-local-acceptance.md b/docs/evidence/dispatcher-v04-local-acceptance.md new file mode 100644 index 0000000..fb6d12e --- /dev/null +++ b/docs/evidence/dispatcher-v04-local-acceptance.md @@ -0,0 +1,42 @@ +# Dispatcher v0.4 项目内验收证据 + +**记录时间：**2026-09-27 UTC。仅限单 Dispatcher、单 Agent、单 Cell、单租户的隔离 Mock；没有真实 SaaS、OSS、ECS、供应商、生产部署或真实拨号。 + +## 合同和环境 + +- Go `go1.27.1 linux/amd64`；分支 `feat/dispatcher-state-v04`。未修改 `contracts/upstream/v1/`，未自动暂存或提交。 +- `python3 scripts/validate-local-contracts.py`：v0.4 发现、控制、外呼三类 Schema，**10 个正例、4 个反例、20 个带 SHA-256 的文件**全部通过；内嵌 Schema 与项目内源文件逐字节一致。历史 v0.1/v0.3 校验仍通过。v0.4 是项目内提案，**不是 SaaS 已签收的外部合同**。 +- `scripts/mq-integration-local.sh`：使用本地缓存的 `rabbitmq:4.1-management-alpine`（镜像 ID `sha256:fcc273cebb0880ec25845c9bfd97687122ac9cc391538f053cb5873f4181f35f`）。Mock SaaS 预建每 D 控制队列、每任务任务队列及结果队列；Dispatcher 的 RabbitMQ `configure` 权限为 `^$`，不能自建任务队列。仅在隔离本机 Docker 中测试，运行后临时容器已由脚本清理。 + +## 故障与恢复矩阵 + +| 场景 | 本地证据与结果 | +| --- | --- | +| 重启全量/多页完整性 | `TestRuntimeCommitsCompleteSnapshotBeforeControlAndTaskAdmission`：完整页先提交，控制积压处理前准入保持关闭；`TestRuntimeInterruptedSnapshotRestartsFromFirstPage`：第二页故障不提交部分清单，从第一页重读。 | +| 重启不继承旧游标、在线归属变化 | `TestRuntimeRestartDiscardsLiveCursorAndReloadsCompleteSnapshot`、`TestRuntimeCatchesChangesArrivingAfterSnapshotWatermark`、`TestRuntimeCatchesChangeCommittedDuringSnapshotPagination`：每次重启全量，运行内存水位接续，撤销不丢；Mock HTTP 实际请求 `mode=snapshot` 后 `after=42`。 | +| 状态冲突与控制积压 | `TestV04LiveChangesOnlyManageMembershipNotExistingStatus`、`TestRuntimeHTTPRunningAfterPauseNeedsMQResume`、`task_runtime_v04_control_test.go`：HTTP 增量不代替 MQ resume，状态冲突关准入；控制先于任务消费。 | +| 历史 pause/resume/stop | `TestOldQueuedPauseResumeStopPreservesOrderAndEveryReceipt`：一个月前发出的三条命令逐条按序应用并各有回执；`TestOldQueuedStopStillHangsUpAndDrains`：旧 stop 不因年龄拒绝。RabbitMQ Mock 在真实预建控制队列验证旧 stop、原消息身份重排队、stop 排空未接纳任务队列并发布回执。 | +| 挂断与确认 | `task_queue_v3_stop_active_test.go`、`task_control_v3_test.go`：pause/stop 发挂断、Agent 终态未知或失败不伪报完成；resume 不挂断；stop 回执在排空后。 | +| 历史外呼、新鲜门禁、未来命令 | `TestAcceptLocalV01CommandPersistsReceiptAndAcceptsOldBacklog`：旧外呼不因年龄拒绝；`TestOldQueuedCallRejectsCurrentlyUnlistedCallee`：积压一个月仍检查当前号码白名单；`TestOldQueuedCallOriginationRejectsPauseAndStop`、`TestOldQueuedCallOriginationRechecksBoundWindowAndClaimsOnlyOnce`：旧请求在拨号前仍检查最新任务控制和绑定的时间/授权截止，不二次发起；`TestFutureIssuedCallExecuteNeverReservesOrDials`：未来 `issued_at` 不提前接纳。`local_origination_quota_test.go` 和 `dial_policy_test.go` 另覆盖额度、任务/线路时间窗及较小通话时限。 | +| 结果发送、确认丢失与复投 | `TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery`：先存事实再 ACK，结果队列不存在时 outbox 保留，恢复后原消息发出；确认只说明 RabbitMQ 接收，不等于 SaaS 已处理。 | +| 安全的 SQLite 局部清理 | `TestTerminalTaskCacheEvictionPreservesRecoveryAndDedupRecords`：**只显式删除终止任务的任务配置副本**，命令重复投递继续使用原身份和回执；inbox、outbox、归属与租户绑定均保留。`TestTerminalTaskCacheEvictionFailClosedWhileWorkOrDeliveryIsUnknown`：未知/在途执行、额度占用、上传中和待发送结果均阻断；`TestTerminalTaskCacheEvictionDatabaseFailureRetainsSnapshot`：SQLite 写失败回滚。无现有数据自动扫描/删除。 | +| 历史任务增长 | `TestSingleTaskControlLookupUsesIndexedIdentityWithGrowingHistory`：600 条归属历史，按 Dispatcher+task ID 直接查询，SQLite 查询计划使用索引而非全表扫描。 | + +## 工具检查与覆盖率 + +`bash scripts/acceptance-local.sh` 已通过合同校验、格式检查、`go vet ./...`、`go test -race ./...`、`go build ./...` 和 RabbitMQ 隔离集成测试；额外执行 `go test -cover` 统计相关业务模块。覆盖率以实际执行测试集为准： + +| 模块 | 覆盖率 | 测试集 | +| --- | ---: | --- | +| `internal/dispatcher` | 70.3% | 隔离 RabbitMQ 集成测试 | +| `internal/store` | 67.6% | 单元测试 | +| `internal/configread` | 75.9% | 单元测试 | +| `internal/contract` | 77.4% | 单元测试 | +| `contracts` | 100.0% | 内嵌文件读写及错误路径单元测试 | +| `internal/mq` | 67.5% | 隔离 RabbitMQ 集成测试；纯单元测试单独执行不足 65%，不能混称 | + +## 不可代签的外部阻断 + +1. SaaS/management 尚未提供或签收真实的 v0.4 分页快照、水位、增量保留和控制交接合同；本地 Schema 与 Mock 不证明接口真实兼容。 +2. SaaS 应用收讫、可查询恢复依据和最长命令重投期限尚未确认。**本轮不删除**执行身份、幂等、未知占用、上传、终结事实或 outbox；RabbitMQ publisher confirm 不能替代应用收讫。任务配置副本的局部清理仅能显式调用且全部条件通过，不能宣称数据库已整体清理。 +3. 真实供应商/OSS/ECS/生产部署与拨号、容量及切换均不在本次本地验收范围；需要独立授权和实测。 diff --git a/docs/plan-0926.md b/docs/plan-0926.md index 4147a2c..8218597 100644 --- a/docs/plan-0926.md +++ b/docs/plan-0926.md @@ -1,6 +1,6 @@ # 任务发现统一事件游标分页计划（设计稿 v0.1） -> 状态：本文件保留用户确认的目标与门禁；**项目内 v0.3 合同、运行代码和本地 Mock/F06 检查已完成，外部 SaaS/management 未签收，也未部署或拨号**。当前本地任务发现依据为 [`docs/thirds/v0.3.md`](thirds/v0.3.md) 及其独立 Schema；[v0.2](thirds/v0.2.md) 仅保留历史。实施顺序及总体状态仍以 [`docs/plan-config-read-v0.1.md`](plan-config-read-v0.1.md) 为入口。 +> 状态：**历史 v0.3 设计依据**；下文“当前”均指当时的 v0.3 实现，不是现行项目内运行版本。现行 Dispatcher 项目内 v0.4 已完成本地实现与隔离验收，见 [`新计划`](plan-dispatcher-state-v0.1.md)、[`v0.4 合同`](thirds/v0.4.md)及[验收证据](evidence/dispatcher-v04-local-acceptance.md)；真实 SaaS/management 仍未签收、未联调，也未部署或拨号。[v0.2](thirds/v0.2.md) 与本文件 v0.3 仅保留历史；未改写已发布外部合同。 ## 1. 目标和不变边界 diff --git a/docs/plan-config-read-v0.1.md b/docs/plan-config-read-v0.1.md index d107486..17b4aa9 100644 --- a/docs/plan-config-read-v0.1.md +++ b/docs/plan-config-read-v0.1.md @@ -1,6 +1,6 @@ # Go SIP Agent：配置读取与有界外呼迭代计划 v0.1 -**当前迭代唯一执行入口。状态：用户已授权 P1 本地迭代；任务发现 v0.3 项目内合同、运行与 F09/F06 本地门禁已完成，外部签收与真实切换未完成。** 现行 `contracts/upstream/v1/` 仍是旧外部基线；本文不授权真实 SaaS、生产部署、拨号或切换。分项设计：[有界消费与控制通道](architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md)；其他本地 SaaS 业务契约仍按[第三方对接 v0.1](thirds/第三方对接事件与请求消费顺序_v0.1.md)；任务发现现用[计划 0926](plan-0926.md)、[v0.3 项目内合同](thirds/v0.3.md)及[独立 Schema](contracts/task-discovery-v0.3-proposal.schema.json)，[v0.2](thirds/v0.2.md)仅保留历史。机器校验件：[Schema/正反例](contracts/)。分项材料须与本文一起更新，不能继续使用旧 ETag/304、控制 CAS 或即时反馈目标。 +**历史 v0.3 的执行入口。任务发现 v0.3 项目内合同、运行与 F09/F06 本地门禁仅保留历史证据。现行项目内 v0.4 已完成代码及隔离验收，见 [`plan-dispatcher-state-v0.1.md`](plan-dispatcher-state-v0.1.md)、[`thirds/v0.4.md`](thirds/v0.4.md)和[验收证据](evidence/dispatcher-v04-local-acceptance.md)；外部 SaaS 签收与真实切换仍未完成。本文下述“当前”均指编写时的 v0.3，不能当作 v0.4 合同。** 现行 `contracts/upstream/v1/` 仍是旧外部基线；本文不授权真实 SaaS、生产部署、拨号或切换。分项设计：[有界消费与控制通道](architecture/Dispatcher有界接纳与控制通道改造计划_v0.1.md)；其他本地 SaaS 业务契约仍按[第三方对接 v0.1](thirds/第三方对接事件与请求消费顺序_v0.1.md)；历史 v0.3 发现依据为[计划 0926](plan-0926.md)、[v0.3 项目内合同](thirds/v0.3.md)及[独立 Schema](contracts/task-discovery-v0.3-proposal.schema.json)，[v0.2](thirds/v0.2.md)也仅保留历史；现行项目内发现见上述 v0.4 合同。机器校验件：[Schema/正反例](contracts/)。分项材料须与本文一起更新，不能继续使用旧 ETag/304、控制 CAS 或即时反馈目标。 ## 1. 范围和完成标准 @@ -12,7 +12,7 @@ | --- | --- | | `/internal/v1/dispatcher/sip` | 用户指定路径；本 D 获管理面批准的 SIP 全量。 | | `/internal/v1/dispatcher/task/:task_id` | 用户指定路径；任务归属、路由/主叫、智能体、时段及两项任务级超时。 | -| `/internal/v1/dispatcher/tasks?after=` | 用户指定路径；首次 `after=0`、重启从已提交事件游标续读，首次与后续均按统一 `tasks[]` 分页，约每 30 秒发现新变化。 | +| `/internal/v1/dispatcher/tasks?after=` | **历史 v0.3 运行方式**：首次 `after=0`、重启从已提交事件游标续读，首次与后续均按统一 `tasks[]` 分页。现行项目内 v0.4 改为启动 `mode=snapshot` 全量、运行时 `after=<内存游标>` 在线增量，已通过隔离 Mock；真实 SaaS 未联调。 | | `/internal/v1/dispatcher/tenant/:tenant_id/quota` | 本地项目定义路径：取得任务后按其 `tenant_id` 读取该租户分配给本 D 的并发额度；真实 SaaS 路径兼容性未验证。 | **任务队列及精确绑定只由 SaaS 创建、维护和退役；D 只消费，不建队、不绑定、不删除。** SaaS 先确认持久队列 ready 再发布 persistent 消息，mandatory 无 return 且 confirm 成功才记入队。D 离线时消息可积压；无队列时未成功入队的原消息由 SaaS 保留，就绪后按原身份重新发布，不能靠 D 重启倒灌。任务队列按 D+任务 ID 定位；租户身份保留原值用于核验和额度汇总。 @@ -42,7 +42,9 @@ 额度成功核验后最多缓存约 60 秒且不超过响应有效截止时间；缺失、身份错配、过期或刷新失败，关闭该租户所有任务的新准入，不使用无限额/默认额，不阻塞 stop 静默排空。额度调低时不强挂已接通通话、不抹掉已占用/未知执行，直到占用降至上限以下才再接新呼叫。D 以 SQLite 单事务检查并预留租户+任务+Cell/线路/AI 等额度，不能先分别判断后并发超卖。确认通话终结并释放执行资源后释放通话额度，**不等待录音上传或 MQ 最终结果确认**；未知通话继续占额。多 D 时各份额之和≤租户总额，未经 F05 授权不开放多 D 运行。 -### 3.3 暂停、恢复、停止与状态来源 +### 3.3 暂停、恢复、停止与状态来源（当前已实现合同） + +> 下一版目标改为 pause/stop 均挂断在途通话、resume 不挂断，且在线任务状态切换以独立控制队列为准；见 [`下一版计划`](plan-dispatcher-state-v0.1.md)。以下 drain/hangup 和发现状态规则在代码/Schema 切换前仍属**当前合同**，不是新目标已实现。 SaaS 先持久修改权威任务状态，再向独立 D 控制队列发布 `task.control`。请求不含 `command_id/expected_task_revision`，本阶段不设计控制去重；D 按任务串行处理控制与接纳，并持久记录生效状态后才 ACK/回执。 @@ -52,7 +54,9 @@ SaaS 先持久修改权威任务状态，再向独立 D 控制队列发布 `task 4. **优先级：**本地 stopped 永久高于任何 running；本地 paused 只能由有效 resume 解除。`tasks[]` 事件页及缓存更新可使状态更严格；即使较新的发现页把 `paused` 改为 `running`，也只更新 SaaS 状态和游标，不清除已持久的暂停/停止屏障。暂停只由上述 MQ `resume` 配合新鲜任务读取解除，且低于已知 `task_revision` 的状态不得覆盖新状态。冷启动恢复本地屏障并从已提交游标追平 SaaS 事件页后取更严格者；冲突/缺失关闭新准入，不丢已有执行事实。 5. **无编号乱序：**控制重投可能重复返回回执，不保证按消息身份“只处理一次”。pause/stop 先关闭准入；与 SaaS 最新状态不符、不可核验或接收乱序时保持关闭并返回明确失败，不根据到达先后自动恢复。恢复必须重新发送有效 resume 并核对最新权威状态；不引入替代 command_id 或控制 CAS。SaaS 不可把发布成功视为控制已应用，丢失回执下状态不明不得主动扩量。 -### 3.4 任务发现、发布与期限 +### 3.4 任务发现、发布与期限（当前 v0.3） + +> 重启全量、运行中内存游标增量为 [`v0.4 未实施草案`](thirds/v0.4.md)；本节只描述当前运行，不得拿旧持久游标解释新快照。 D 启动/重启从已提交的 v0.3 事件游标续读（无新游标时请求 `/internal/v1/dispatcher/tasks?after=0`），运行中约每 30 秒继续读取。首次与后续均返回统一 `tasks[]` 事件页，每页最多 256 项、每 D 活跃归属最多 256 项；`after` 是规范十进制事件序号，不是最大 task_id。非空页的 `next_cursor` 必须前进，空页的 `next_cursor` 保持请求游标并表示本轮追平；同一任务更新后可再次出现，`status=removed` 为撤销墓碑，不能根据某页未出现任务推断撤销。响应不携带 queue/key，D 按 D+task ID 推导并仅消费 SaaS 预建队列。每页任务、响应和游标同事务提交，首次未追平或任何错误页都拒绝新准入，控制 MQ 保持可用。**不设游标过期/410 自动重建，v0.2 checkpoint 不可解释成 v0.3 事件 ID；错误不可静默重置或回退旧 `changes` 路径。** SaaS 必须保证页内完整有序、旧游标及撤销墓碑长期可读；容量/灾备连续性仍待外部签收。发现项和任务配置须提供 tenant_id/tenant_key 以便核验额度；无变化不撤销消费关系。 @@ -132,6 +136,7 @@ SaaS 清单须保留已 stopped 但未排空任务；撤销握手按第三方契 | F09 | v0.3 首次多页、空页、同任务重现及墓碑、翻页中更新、断页重启、坏游标/410 失败、SQLite 写失败同事务回滚、MQ 控制竞态（含发现 `paused→running` 仅更新状态/游标、须 MQ `resume`＋新鲜任务核验才恢复队列）、重复执行不重拨完成单节点/单租户本地 Mock 验证；旧 F09 v0.2 证据仅作历史。证据：[`f09-local-acceptance-v0.3.md`](evidence/f09-local-acceptance-v0.3.md)。 | `scripts/acceptance-local.sh`（格式/合同/Proto/hash、race、vet、构建、受限 RabbitMQ）通过；P1 业务包 66.4%–85.4%（MQ 含 integration）。CLI 30.0% 和历史 callruntime 61.7% 单列技术债，不声称全仓达标；SQLite 写失败用触发器注入，未物理填盘。外部有序返回/墓碑容量/真实验收未完成。 | | F06 | Go 1.27.1 一次性本地制品/校验和将 v0.1 其他业务＋v0.3 任务发现＋v3 MQ 拓扑及 Proto 版本/hash 写入清单；已有 release/package/归档拒绝覆盖，制品 mixed/real 在开 DB 前拒绝；旧 CLI 和分散事件不在新运行路径。证据：[`f06-local-release-gates-v0.3.md`](evidence/f06-local-release-gates-v0.3.md)。 | `make release-check-local` 与本地 acceptance 通过；来源 dirty、`production_approval=false`。外部 SaaS 有序返回/长期墓碑容量、v0.2 旧状态受控切换、真实部署/拨号与生产签收均**未完成**；本地阻断不代签真实切换。 | | F05 | 多D/第二租户不在本轮运行范围。 | 另获阶段与额度/资源授权。 | +| 下一版 Dispatcher 状态/临时数据（仅文档） | [`plan-dispatcher-state-v0.1.md`](plan-dispatcher-state-v0.1.md) 与 [`thirds/v0.4.md`](thirds/v0.4.md) 记录用户本轮提出的重启全量、状态来源收敛、pause/stop 强制挂断与任务结束条件清理；当前代码、Schema、F09/F06 证据**未切换**。日志全部原样的要求与现行禁令冲突，不列为已批准实施。 | 冻结全量分页与控制交接、SaaS 应用收讫/恢复依据、清理期限及新严格 Schema/Mock；TDD 实施后再重跑本地门禁。外部兼容、真实拨号和切换另签收。 | ## 9. 协作与版本记录 diff --git a/docs/plan-dispatcher-state-v0.1.md b/docs/plan-dispatcher-state-v0.1.md new file mode 100644 index 0000000..d3883eb --- /dev/null +++ b/docs/plan-dispatcher-state-v0.1.md @@ -0,0 +1,44 @@ +# Dispatcher 任务状态、控制与临时数据迭代计划 v0.1 + +**状态：项目内 v0.4 Schema、隔离 Mock、代码及本地验收已完成，证据见 [`dispatcher-v04-local-acceptance.md`](evidence/dispatcher-v04-local-acceptance.md)；外部 SaaS/management 未签收、未联调。** 历史项目内 v0.3 见 [`thirds/v0.3.md`](thirds/v0.3.md)，上一轮计划见 [`plan-config-read-v0.1.md`](plan-config-read-v0.1.md)；已发布外部合同不因本地切换改变。下一版目标合同见 [`thirds/v0.4.md`](thirds/v0.4.md)。本计划不授权真实拨号、ECS、生产切换或清理任何现存数据。 + +## 1. 问题与完成定义 + +- 历史 v0.3 的重启从 SQLite 持久事件游标续读；项目内 v0.4 已改为**每次重启取得完整任务归属清单**，游标仅在该次进程内用于在线增量发现。发现页不再代替独立的 MQ 控制队列切换任务状态。 +- 历史 `pause/stop` 可选择 `drain`；项目内 v0.4 已改为两者均挂断该任务的在途通话并核验终态，`resume` 不挂断。两类 MQ 命令均不因消息年龄失效；历史积压控制与外呼、新鲜拨号授权及未来 `issued_at` 外呼的边界需通过失败测试与回归证明。 +- Dispatcher SQLite 持有接纳/幂等、未知通话占用、执行快照、录音/最终结果与 outbox；v0.4 仅增加已终止任务配置副本的显式条件清理，不自动删除现存数据。SaaS 尚无可验证的全部执行事实和应用收讫，**不能把这些 SQLite 记录当作可无条件删除的缓存**；不以日志代替事务和恢复事实。单节点 SQLite 不要求高可用，不因此允许已接纳的未知呼叫重拨。 +- 历史控制处理为定位一个任务读取全部任务归属；v0.4 已按 D+task ID 直接索引查询，600 条历史任务的查询计划测试未扫描全表。 + +**完成标准：**项目内下一版严格合同/Schema/正反例/来源与哈希、Mock SaaS 故障矩阵先通过；再基于 TDD 替换运行路径且不保留已废弃模式。验证重启全量、在线增量、控制顺序/挂断、停止排空、历史积压外呼的新鲜授权、未来 issued_at、任务结束清理及数据库故障；至少执行本模块格式检查、`go vet ./...`、`go test -race ./...`、构建，业务模块覆盖率 ≥65%。不能将计划或 Mock 当作已交付功能/真实 SaaS 验收。 + +## 2. 必须先冻结的合同边界 + +| 项目 | 下一版目标 | 冻结前不得声称已解决的问题 | +| --- | --- | --- | +| SaaS→D 入站队列 | **两类仍保留**：每 D 一条独立控制队列 `task.control`，每归属任务一条任务队列 `call.execute`；均由 SaaS 创建/退役，D 只消费。结果另走 D→SaaS 队列。 | 不将任务列表轮询替代控制队列，不将控制队列误当任务队列。 | +| 重启全量 | 每次进程重启从 SaaS 读取本 D 的**完整、一致、有界分页**任务清单（含队列未退役的 stopped 任务），在全部页核验、应用前关闭新执行；不以本地旧游标跳过全量。 | 冻结全量分页标识/边界、水位、完整性、遗漏任务与 removed 的含义；未拿到完整清单不得从已有队列拨号。与现行 v0.3 `after=0` 事件回放不同，不能重解释旧游标。 | +| 在线增量 | 内存游标仅在运行期使用，发现**新增/撤销/归属**变化；状态切换由独立 MQ 控制队列实时处理。重启重新全量，无需为发现长期保留旧事件/无限墓碑。 | 冻结增量页丢失、无序/中断、连续性及全量重取规则；变更页 incidental `status` 不可自动解除暂停/停止，也不能成为第二个状态命令。身份或状态冲突须关闭准入并报错。 | +| 全量与控制交接 | 在重启清单核验和控制积压处理期间关闭新执行；按任务串行处理有效控制及接纳。SaaS 先持久状态再发控制；不得先开任务消费后才处理已积压控制。 | 冻结快照水位与已发布/积压控制顺序、历史积压、重复/迟到控制逐条处理的证明；MQ 命令不因时间积压过期。队列暂时为空不是 SaaS 不再发布的保证；不凭消息到达先后把旧 pause 当作最新 resume。 | +| 控制副作用 | `pause/stop` 关闭准入并对该任务所有在途通话发挂断，失败保持关闭并可追踪；`resume` 不挂断。stop 仍静默 ACK 未接纳积压，停止回执只在排空条件达成后应用。 | 删除 `drain` 作为 pause/stop 有效选项前，冻结 Agent 实际终结确认、超时/重试边界、错误回执以及已接纳通话的唯一最终结果。不能把 Agent 收到指令等同于通话已结束。 | +| 控制积压 | 不合并为“每个任务只执行最后一条”：pause 挂断后 resume、stop 不可逆和逐条控制回执有中间副作用。 | 只有另行变更为纯状态设置、移除中间副作用及原回执合同，才能重新讨论合并；本轮不做。 | +| SaaS 证据与清理 | SaaS 是业务配置与最终结果的目标来源；D 仅保存执行中所需最少的本地事务/恢复事实。 | 当前 RabbitMQ publisher confirm **不是** SaaS 应用收讫；必须先决定 SaaS 应用回执/可查询恢复事实及最长重投期限，才能冻结何时安全删除本地执行身份、已发布 outbox 和防重记录。未确认时保留并报告阻塞。 | +| 日志 | 文件系统诊断日志记录原值任务/通话/消息 ID、号码、时间、状态、错误、挂断与清理阶段；日志只用于排查。 | 用户要求“所有内容原样记录”与现行禁止密钥、私钥、完整音频/对话入日志的约束冲突，**不纳入实施目标**；该冲突未获解除前不得写入这些内容，也不得用日志代替可靠状态。 | + +不为全量/增量自行猜测新 JSON 字段、SaaS 应用回执或保留期限；这些须作为 F01/F07 合同、严格 Schema、正反例与 Mock 的同一版本冻结。新合同不原地改写 `contracts/upstream/v1/` 或已运行的 `docs/contracts/task-discovery-v0.3-proposal.schema.json`，也不复活 v0.2 的旧 `changes`/410 字段来冒充新版本。 + +## 3. 实施顺序（项目内完成；外部签收仍未完成） + +1. **合同与迁移门禁。** 与 SaaS 明确全量分页/增量连续性、归属撤销、单任务只读配置继续存在、控制队列积压及任务队列退役、SaaS 最终结果应用收讫或可恢复查询范围；发布项目内新版本 Schema、拓扑/正反例、来源/hash 与 Mock C。历史项目内 v0.3 与已发布外部 MQ v0.1 不能冒充 v0.4；已发布 upstream/v1 未修改。 +2. **先写失败用例再切发现。** 重启不使用旧发现游标直接作为续读起点，先读一致全量，全部页面核验后才能接任务队列；在线增量只增删归属，不从状态字段擅自恢复暂停；失败/分页断裂关准入，控制队列照常消费。迁移时排空/确认旧任务、旧未确认占用和 outbox，不能直接删旧 SQLite 或并跑新旧发现。 +3. **控制路径。** 保留独立 D 控制队列和逐条回执；暂停/停止首先关闭新执行、退回未接纳交付，及时对该任务所有在途通话挂断，终结事实另外确认；停止仍须排空未接纳队列后回 stopped/applied。历史 stop 即使积压很久仍须按序挂断、排空并回逐条结果，不能只停 consumer 留队列。两类 SaaS→D MQ 命令无 `not_after` 到期语义；历史外呼在接纳与拨号前仍按当前任务、白名单、时段、独立授权、额度及通话时限校验，失败拒绝且不自动重拨。未来 `issued_at` 的 `call.execute` 拒绝提前接纳。 +4. **临时 SQLite 收敛。** 盘点每张当前表的写入/引用和所有外键，区分在途状态、未知占用、未交付结果、SaaS 已核收事实及有界防重记录；仅对满足全部安全条件的**已完成任务**批量清理。先保证幂等身份与可恢复结果不会因删除而重拨/丢失，再清配置副本/历史终结事实。清理阶段留日志、计数、耗时及失败原因；失败显式阻断或重试清理，不影响仍需恢复的记录。具体保留期和 SQLite/WAL 空间回收频率以容量测试和 SaaS 回执合同为依据，不猜定值。 +5. **性能与文档。** 控制按 D+task ID 直查；在大量历史任务下验证单次控制查询不随全部历史线性增长。同步更新项目入口、任务发现/控制合同、代码发布版本与部署/测试证据；旧文档加历史标识但保留可追溯证据。 + +## 4. 本地验收场景 + +- 重启多页完整清单、分页期间新增/移除、崩溃后再次从全量恢复；运行期增量断页和旧游标故障不跳项、不误撤销；旧状态资料不能覆盖新消费状态。SaaS 停止发布前任务队列不能由 D 删除；独立控制队列即使发现不可用也能处理控制。 +- 在途两通分别收到 pause、stop 后均挂断；resume 不挂断；Agent 挂断失败、响应成功但终态未知、控制回执丢失均不误报完成、不重拨。pause→resume、pause→stop、stop→resume、重复/历史积压控制按实际副作用与终态验收，不只检查最后状态。 +- stop 未接纳积压静默 ACK、已接纳通话各发唯一最终结果；SaaS 停止发布/队列排空/回执/退役的顺序可追溯。历史 stop/断线重启不遗留永不排空的队列；未来 issued_at 的外呼绝不提前发出。 +- 任务完成但 outbox 未确认、SaaS 尚未核收、上传中、未知呼叫、重复命令晚到和 SQLite 写失败时不误清；仅显式清理已证实安全的任务配置副本，历史任务查询使用索引；数据库整体大小/长期保留期仍依赖未签收的 SaaS 收讫与重投期限，不能宣称有界；原值诊断字段可追查，日志无实际凭据、完整音频/对话。 + +**外部阻断项：**真实 SaaS 尚未签收全量/水位和控制交接合同、没有可验证的 SaaS 应用收讫/恢复事实与重投期限，因此本轮仅完成项目内 v0.4 实现和隔离 Mock，不得宣称 SQLite 可随时清空或生产可用。现行单节点/单 Cell/单租户、禁止真实拨号和不自动提交/清理工作树的边界不变。 diff --git a/docs/thirds/v0.4.md b/docs/thirds/v0.4.md new file mode 100644 index 0000000..b1868e9 --- /dev/null +++ b/docs/thirds/v0.4.md @@ -0,0 +1,35 @@ +# SaaS ↔ Dispatcher：任务清单、控制与临时状态 v0.4（项目内目标草案） + +**项目内 v0.4 Schema、隔离 Mock、运行代码及本地验收已完成（见 [`dispatcher-v04-local-acceptance.md`](../evidence/dispatcher-v04-local-acceptance.md)）；真实 SaaS 尚未签收或联调。** 已发布外部合同仍以 `contracts/upstream/v1/` 为基线；v0.3 历史发现见 [`v0.3`](v0.3.md)，其余历史规则见[第三方对接顺序 v0.1](第三方对接事件与请求消费顺序_v0.1.md)。本文件不授权真实呼叫或部署。实施步骤与待冻字段见 [`plan-dispatcher-state-v0.1.md`](../plan-dispatcher-state-v0.1.md)。 + +## 1. 不变的两类 SaaS → D 队列 + +- 每个 Dispatcher 一条 SaaS 预建、独占的**控制队列**，接收 `task.control` 的 pause/resume/stop；每个归属任务另有一条 SaaS 预建**任务队列**，接收 `call.execute`。D 只消费，不能创建、绑定或删除。D→SaaS 的控制/命令回执和 `call.result` 仍走既有 per-D 结果路由，不能把结果队列误认为入站任务队列。 +- 控制须先于任务新接纳生效，SaaS 先持久任务状态再按每任务顺序发布控制；控制回执是应用事实，RabbitMQ publisher confirm 不是 SaaS 已处理。不可将控制积压按 task 只保留最后一条：中间挂断、stop 排空及逐条回执仍须执行。 +- SaaS 先停止向被 stop 的任务发布，D 持久屏障、处理在途执行并静默 ACK 所有未接纳积压；仅在原任务队列排空后回 stopped/applied，SaaS 确认回执后退役任务队列。无回执不得提前删队列。`resume` 不得重新启用本地已 stopped 的同一 task ID。 + +## 2. 重启全量、运行中增量（项目内已替换 v0.3 §2.5） + +- **每次 D 进程启动/重启**，在关闭新执行的状态下读取 SaaS 对此 D 的完整归属清单；如需多页，各页必须属同一个有界、一致的快照。清单包含仍需排空的 stopped 任务，不能仅列 running；明确撤销后的任务不得被当成仍归属。全量页不完整、身份冲突、归属缺失或列表不可用时保持关闭，不能拿旧 SQLite 游标跳过全量。所有页核验完成后才应用任务归属，继续处理重启期间积压的控制；控制队列在发现失败时也不得静默停摆。 +- 项目内拟定同一路径 `GET /internal/v1/dispatcher/tasks?mode=snapshot` 返回 `schema_version=task-discovery.v0.4-proposal`、`mode=snapshot`、`dispatcher_id`、UUID v4 `snapshot_id`、十进制事件 `watermark`、`tasks[]`、`next_page_token`（最后一页 `null`）。后续用 `?snapshot_id=&page_token=` 取同一快照下一页；每页 ≤256、全 D 当前归属/待退役任务合计 ≤256（不能重复/跳项），各页 `snapshot_id/watermark` 不得变化；空页有非空 token 或有新 token 却零任务都按不完整失败。快照不可继续时返回 HTTP 409 + `snapshot_unavailable`，丢弃整个未提交快照并从头重取，**不**回退旧游标；page token 不受客户端解析。起始快照的水位覆盖快照生成前所有任务归属事件，不用任何旧 v0.3 SQLite 游标。 +- 在线 `GET /internal/v1/dispatcher/tasks?after=` 返回 `schema_version=task-discovery.v0.4-proposal`、`mode=changes`、`dispatcher_id`、`tasks[]` 和 `next_cursor`。非空页严格前进，空页等于本次请求游标；SaaS 保证每 D 事件连续、分页完整、有序、不得提前丢失未消费增量，无法满足即显式报错并关闭新执行、重新全量。HTTP 400 不合法请求、403 D 无归属、503 服务不可用；错误响应严格见 Schema，**不复用** v0.2 `changes[]`/410。具体身份验证仍按现有只读 HTTP 配置合同。 +- **仅在本次进程运行期间**按发现水位定时查询增量，用于新增任务归属、撤销/退役和必要身份校验。水位不是最大 task_id；异常页/缺页不得推进游标或视为空变更。重启重新全量，不要求从上一次进程的永久事件游标续读；旧 v0.3 的 `after=0` 事件回放和 SQLite 持久游标不得被当作新全量响应。 +- 任务列表的任务状态可以作为启动快照的初始状态及增量身份/一致性校验，**运行中 pause/resume/stop 由独立 MQ 控制队列生效**；增量页 `running` 不能自动解除暂停或不可逆停止。若列表状态与已应用的 MQ 控制矛盾，关闭该任务新接纳并报错、等待受控恢复，不按 HTTP 到达顺序偷偷切换状态。`GET /internal/v1/dispatcher/task/:task_id` 的授权配置和 resume 的新鲜状态核验仍保留，不等同于恢复“发现页控制状态”机制。 +- 项目内发现字段已在 [`task-discovery-v0.4-proposal.schema.json`](../contracts/task-discovery-v0.4-proposal.schema.json) 独立严格声明，任务条目引用现有 v0.3 `$defs/task`，不重造业务字段；控制入站严格格式见 [`task-control-v0.4-proposal.schema.json`](../contracts/task-control-v0.4-proposal.schema.json)。机器 Schema 只约束消息结构；多页相同水位、完整性和快照与 MQ 积压的交接顺序须另由 Mock/代码验证。真实 SaaS 的这些字段、应用收讫、增量保留期限**仍未签收**，不能以项目内 Schema/Mock 自证兼容；不直接引用 v0.2 的 `changes`/410 或原地复用 v0.3 的严格 Schema。 + +## 3. 控制、通话与积压命令边界 + +- 新控制入站使用 `schema_version=task-control.v0.4-proposal`、原始信封 `dispatcher_id/tenant_id/tenant_key/trace_id/issued_at/command_type` 与严格 `payload={task_id,action,reason}`；**不再携带 `active_call_policy`**，逐条回执格式沿用现行 `command.result` v0.1，不能把旧 v0.1 控制请求误认为新请求。**有效 pause**：立即持久关闭该任务新接纳，停止消费任务队列并退回已交付但未接纳消息；向该任务所有已接纳且仍在途的执行发挂断。**有效 stop**：不可逆关闭新接纳，挂断全部在途执行，同时按 §1 排空未接纳积压；已接纳呼叫按原身份形成最终结果。两者不再提供 `drain` 行为；`resume` 仅在新鲜任务状态为 running 且授权有效、本地未 stopped 时恢复原任务队列，不挂断。 +- 在途挂断的“已发命令”“Agent 已应用”和“实际通话终结”是三个不同事实；不因 RPC 成功伪称已终结。失败或终态未知时维持关闭、保留未知占用与可追查错误，何时回控制的 applied/failed、终结最长等待及 retry 边界须由新控制合同与 Agent 能力验证后冻结。不得因自动重投二次 originate。 +- 两类 SaaS→D 入站命令 `task.control`、`call.execute` 均**不带 `not_after`，不因积压时间拒绝**。历史 pause/resume/stop 逐条按控制队列顺序处理，不合并、延迟或悄悄丢弃；stop 依 §1 排空未接纳积压并回逐条结果。新版外呼请求严格见 [`call-execute-v0.4-proposal.schema.json`](../contracts/call-execute-v0.4-proposal.schema.json)，旧 v0.1 入站请求不能当作新版请求；已有出站 `command.result` 仍按其原合同校验。 +- 未来 `issued_at` 的 `call.execute` **不得提前接纳**，这是时间先后校验而不是命令到期。历史外呼仍须在持久接纳及实际拨号前核对当前任务运行状态、原值号码白名单、任务及选中线路允许时段、SaaS 授权与额度、租户及供应商份额、任务与 AI 中较小通话时限；任何条件缺失、过期或不确定均拒绝新执行，不等下一窗口、不换线、不自动重拨。删除 MQ 消息期限不放宽这些独立授权期限。 + +## 4. Dispatcher SQLite 与日志 + +- SQLite 是单 D **临时可靠事务账本**，不要求多节点高可用，也不是 SaaS 的永久业务库。执行中仍需短期保存命令幂等、未知占用、执行固定快照、上传/最终结果以及未交付 outbox；文件日志只能排查，不能取代这些用于恢复与防重复拨号的事实。 +- 本地只提供**显式且有条件**的已终止任务配置副本删除：仍有未知执行/额度、待上传或未发布 outbox 时拒绝；保留任务归属、执行快照、幂等命令、终结事实、上传、占用及 outbox。无自动扫描或迁移删除现有 SQLite 数据。SaaS 应用层回执/可查询恢复事实与最长重投期限尚未冻结；MQ publisher confirm **不等于** SaaS 收讫，不能据此删除依赖应用收讫的历史和防重证据。阻断原因、实际删除数量及 SQLite 错误可追查。 +- 文件系统日志记录原值任务 ID、执行 ID、号码、状态、时间、错误及清理阶段；**不写入密钥、密码、私钥、完整用户音频或完整对话**。用户另提“全部内容无需脱敏”，与项目现行日志禁令冲突，不作为本版本合同已批准项。 + +## 5. 版本与验收边界 + +项目内 F07 的发现、控制与外呼严格 Schema、正反例、来源与 SHA-256 已校验；隔离 Mock C 与 TDD 回归已覆盖重启多页、积压控制/外呼、挂断失败、stop 排空、未来 `issued_at`、独立拨号门禁、条件清理及复投。已发布外部合同、真实单任务配置字段、SaaS 可恢复/应用回执与重投期限仍待 F01/F07 外部核对；`v0.3` 和 v0.1 历史证据不自动升级成 v0.4。真实 SaaS、生产、云和拨号仍须单独授权与验证。 diff --git a/internal/configread/discovery_v04.go b/internal/configread/discovery_v04.go new file mode 100644 index 0000000..10c9f55 --- /dev/null +++ b/internal/configread/discovery_v04.go @@ -0,0 +1,133 @@ +package configread + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "git.ipao.vip/rogee/go-sip/internal/contract" +) + +// TaskSnapshotPage belongs to one immutable, complete snapshot of this Dispatcher's tasks. +type TaskSnapshotPage struct { + DispatcherID string + SnapshotID string + Watermark string + NextPageToken string + Tasks []DiscoveredTask + Body json.RawMessage +} + +// ReadTaskSnapshot reads either the first snapshot page or the next page of the same snapshot. +// Cross-page identity, completeness, and duplicate checks belong to the caller, before admission. +func (c *Client) ReadTaskSnapshot(ctx context.Context, snapshotID, pageToken string) (TaskSnapshotPage, error) { + var query url.Values + switch { + case snapshotID == "" && pageToken == "": + query = url.Values{"mode": {"snapshot"}} + case snapshotID != "" && pageToken != "": + query = url.Values{"snapshot_id": {snapshotID}, "page_token": {pageToken}} + default: + return TaskSnapshotPage{}, errors.New("snapshot ID and page token must both be present or absent") + } + body, err := c.getTaskDiscoveryV04(ctx, configReadPath+"/tasks?"+query.Encode()) + if err != nil { + return TaskSnapshotPage{}, err + } + var response struct { + Mode string `json:"mode"` + DispatcherID string `json:"dispatcher_id"` + SnapshotID string `json:"snapshot_id"` + Watermark string `json:"watermark"` + NextPageToken *string `json:"next_page_token"` + Tasks []DiscoveredTask `json:"tasks"` + } + if err := json.Unmarshal(body, &response); err != nil { + return TaskSnapshotPage{}, fmt.Errorf("decode task snapshot: %w", err) + } + if response.Mode != "snapshot" || response.DispatcherID != c.dispatcherID { + return TaskSnapshotPage{}, errors.New("task snapshot mode or dispatcher identity mismatch") + } + if _, err := parseEventCursor(response.Watermark); err != nil { + return TaskSnapshotPage{}, fmt.Errorf("invalid task snapshot watermark: %w", err) + } + if snapshotID != "" && response.SnapshotID != snapshotID { + return TaskSnapshotPage{}, errors.New("task snapshot identity changed between pages") + } + if err := validateV04Tasks(response.Tasks, false); err != nil { + return TaskSnapshotPage{}, err + } + var next string + if response.NextPageToken != nil { + next = *response.NextPageToken + if len(response.Tasks) == 0 || next == pageToken { + return TaskSnapshotPage{}, errors.New("task snapshot has an empty or repeated next page") + } + } + return TaskSnapshotPage{DispatcherID: response.DispatcherID, SnapshotID: response.SnapshotID, + Watermark: response.Watermark, NextPageToken: next, Tasks: response.Tasks, Body: append(json.RawMessage(nil), body...)}, nil +} + +// ReadTaskChanges uses the in-memory cursor acquired from a completed startup snapshot. +func (c *Client) ReadTaskChanges(ctx context.Context, after string) (TaskDiscovery, error) { + from, err := parseEventCursor(after) + if err != nil { + return TaskDiscovery{}, fmt.Errorf("invalid live discovery cursor: %w", err) + } + body, err := c.getTaskDiscoveryV04(ctx, configReadPath+"/tasks?"+url.Values{"after": {after}}.Encode()) + if err != nil { + return TaskDiscovery{}, err + } + var response struct { + Mode string `json:"mode"` + DispatcherID string `json:"dispatcher_id"` + NextCursor string `json:"next_cursor"` + Tasks []DiscoveredTask `json:"tasks"` + } + if err := json.Unmarshal(body, &response); err != nil { + return TaskDiscovery{}, fmt.Errorf("decode live task changes: %w", err) + } + if response.Mode != "changes" || response.DispatcherID != c.dispatcherID { + return TaskDiscovery{}, errors.New("live task changes mode or dispatcher identity mismatch") + } + to, err := parseEventCursor(response.NextCursor) + if err != nil || (len(response.Tasks) == 0 && to != from) || (len(response.Tasks) != 0 && to <= from) { + return TaskDiscovery{}, errors.New("live task changes cursor does not match returned tasks") + } + if err := validateV04Tasks(response.Tasks, true); err != nil { + return TaskDiscovery{}, err + } + return TaskDiscovery{DispatcherID: response.DispatcherID, FromCursor: after, + NextCursor: response.NextCursor, Tasks: response.Tasks, Body: append(json.RawMessage(nil), body...)}, nil +} + +func validateV04Tasks(tasks []DiscoveredTask, allowRemoved bool) error { + seen := make(map[string]struct{}, len(tasks)) + tenants, keys := make(map[string]string), make(map[string]string) + for _, task := range tasks { + if !allowRemoved && task.Status == "removed" { + return errors.New("complete task snapshot cannot include removed tombstones") + } + if _, duplicate := seen[task.TaskID]; duplicate { + return fmt.Errorf("task discovery contains duplicate task ID %q", task.TaskID) + } + seen[task.TaskID] = struct{}{} + if err := validateTenantBinding(tenants, keys, task.TenantID, task.TenantKey); err != nil { + return err + } + } + return nil +} + +func (c *Client) getTaskDiscoveryV04(ctx context.Context, path string) (json.RawMessage, error) { + body, err := c.get(ctx, path) + if err != nil { + return nil, err + } + if err := contract.ValidateLocalTaskDiscoveryV04(body); err != nil { + return nil, fmt.Errorf("validate v0.4 task discovery response: %w", err) + } + return body, nil +} diff --git a/internal/configread/discovery_v04_test.go b/internal/configread/discovery_v04_test.go new file mode 100644 index 0000000..6c45912 --- /dev/null +++ b/internal/configread/discovery_v04_test.go @@ -0,0 +1,86 @@ +package configread + +import ( + "context" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" +) + +func TestV04SnapshotAndLiveChangesUseSeparateRequests(t *testing.T) { + var paths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.RequestURI()) + if r.Header.Get(dispatcherIDHeader) != eventTestDispatcherID || r.Header.Get(dispatcherSecretHeader) != mockTestSecret { + t.Error("missing Dispatcher credentials") + } + var fixture string + switch r.URL.RawQuery { + case "mode=snapshot": + fixture = "task-discovery-snapshot-page1-v0.4.json" + case "page_token=page-2&snapshot_id=22222222-2222-4222-8222-222222222222": + fixture = "task-discovery-snapshot-page2-v0.4.json" + case "after=42": + fixture = "task-discovery-changes-v0.4.json" + default: + t.Errorf("unexpected request %s", r.URL.RequestURI()) + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(readConfigFixture(t, fixture)) + })) + defer server.Close() + client := eventTestClient(t, server) + first, err := client.ReadTaskSnapshot(context.Background(), "", "") + if err != nil { + t.Fatal(err) + } + if first.SnapshotID != "22222222-2222-4222-8222-222222222222" || first.Watermark != "42" || first.NextPageToken != "page-2" || len(first.Tasks) != 1 { + t.Fatalf("first page=%+v", first) + } + second, err := client.ReadTaskSnapshot(context.Background(), first.SnapshotID, first.NextPageToken) + if err != nil { + t.Fatal(err) + } + if second.SnapshotID != first.SnapshotID || second.Watermark != first.Watermark || second.NextPageToken != "" || len(second.Tasks) != 1 { + t.Fatalf("second page=%+v", second) + } + changes, err := client.ReadTaskChanges(context.Background(), first.Watermark) + if err != nil { + t.Fatal(err) + } + if changes.NextCursor != "43" || len(changes.Tasks) != 1 || changes.Tasks[0].Status != "removed" { + t.Fatalf("changes=%+v", changes) + } + want := []string{"/internal/v1/dispatcher/tasks?mode=snapshot", "/internal/v1/dispatcher/tasks?page_token=page-2&snapshot_id=22222222-2222-4222-8222-222222222222", "/internal/v1/dispatcher/tasks?after=42"} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("queries=%v want=%v", paths, want) + } +} + +func TestV04SnapshotRejectsInvalidPayloadOrRequest(t *testing.T) { + for name, fixture := range map[string]string{ + "extra field": "task-discovery-invalid-v0.4.json", + "changes in snapshot": "task-discovery-changes-v0.4.json", + "old cursor page": "task-discovery-page-v0.3.json", + } { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(readConfigFixture(t, fixture)) })) + defer server.Close() + if _, err := eventTestClient(t, server).ReadTaskSnapshot(context.Background(), "", ""); err == nil { + t.Fatal("invalid snapshot accepted") + } + }) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write(readConfigFixture(t, "task-discovery-error-v0.4.json")) + })) + defer server.Close() + if _, err := eventTestClient(t, server).ReadTaskSnapshot(context.Background(), "", ""); err == nil || !strings.Contains(err.Error(), "409") { + t.Fatalf("unavailable snapshot error=%v", err) + } +} diff --git a/internal/contract/contract.go b/internal/contract/contract.go index 89485d6..0aa57a2 100644 --- a/internal/contract/contract.go +++ b/internal/contract/contract.go @@ -76,6 +76,18 @@ func ValidateLocalTaskDiscovery(raw []byte) error { return validateLocalSchema("task-discovery-v0.3-proposal.schema.json", raw) } +func ValidateLocalTaskDiscoveryV04(raw []byte) error { + return validateLocalSchema("task-discovery-v0.4-proposal.schema.json", raw) +} + +func ValidateLocalTaskControlV04(raw []byte) error { + return validateLocalSchema("task-control-v0.4-proposal.schema.json", raw) +} + +func ValidateLocalCallExecuteV04(raw []byte) error { + return validateLocalSchema("call-execute-v0.4-proposal.schema.json", raw) +} + func ValidateLocalCommandNext(raw []byte) error { return validateLocalSchema("command-next-v0.1-proposal.schema.json", raw) } diff --git a/internal/contract/schema.go b/internal/contract/schema.go index 51fb0f6..84cd689 100644 --- a/internal/contract/schema.go +++ b/internal/contract/schema.go @@ -64,6 +64,8 @@ func localSchemaVersion(name string) string { return "v0.1" case "task-discovery-v0.3-proposal.schema.json": return "v0.3" + case "task-discovery-v0.4-proposal.schema.json", "task-control-v0.4-proposal.schema.json", "call-execute-v0.4-proposal.schema.json": + return "v0.4" default: return "" } diff --git a/internal/contract/schema_test.go b/internal/contract/schema_test.go index fad9299..a262055 100644 --- a/internal/contract/schema_test.go +++ b/internal/contract/schema_test.go @@ -78,3 +78,37 @@ func TestProjectLocalConfigurationSchemasValidatePositivesAndRejectNegatives(t * } } } + +func TestV04LocalSchemasValidateExamples(t *testing.T) { + for _, tt := range []struct { + name string + validate func([]byte) error + valid bool + }{ + {"task-discovery-snapshot-page1-v0.4.json", ValidateLocalTaskDiscoveryV04, true}, + {"task-discovery-snapshot-page2-v0.4.json", ValidateLocalTaskDiscoveryV04, true}, + {"task-discovery-changes-v0.4.json", ValidateLocalTaskDiscoveryV04, true}, + {"task-discovery-error-v0.4.json", ValidateLocalTaskDiscoveryV04, true}, + {"task-discovery-invalid-v0.4.json", ValidateLocalTaskDiscoveryV04, false}, + {"task-control-pause-v0.4.json", ValidateLocalTaskControlV04, true}, + {"task-control-resume-v0.4.json", ValidateLocalTaskControlV04, true}, + {"task-control-stop-v0.4.json", ValidateLocalTaskControlV04, true}, + {"task-control-invalid-drain-v0.4.json", ValidateLocalTaskControlV04, false}, + {"task-control-invalid-expiry-v0.4.json", ValidateLocalTaskControlV04, false}, + {"call-execute-old-v0.4.json", ValidateLocalCallExecuteV04, true}, + {"call-execute-recent-v0.4.json", ValidateLocalCallExecuteV04, true}, + {"call-execute-altcallee-v0.4.json", ValidateLocalCallExecuteV04, true}, + {"call-execute-invalid-expiry-v0.4.json", ValidateLocalCallExecuteV04, false}, + } { + t.Run(tt.name, func(t *testing.T) { + body, err := os.ReadFile(filepath.Join("..", "..", "docs", "contracts", "examples", tt.name)) + if err != nil { + t.Fatal(err) + } + err = tt.validate(body) + if (err == nil) != tt.valid { + t.Fatalf("valid=%v, err=%v", tt.valid, err) + } + }) + } +} diff --git a/internal/dispatcher/config_read.go b/internal/dispatcher/config_read.go index 67ea50b..c8a5bda 100644 --- a/internal/dispatcher/config_read.go +++ b/internal/dispatcher/config_read.go @@ -69,16 +69,11 @@ func (d *Dispatcher) LoadProjectConfig(ctx context.Context, client *configread.C if snapshot.TenantKey != assigned.TenantKey || snapshot.TaskRevision != assigned.TaskRevision || snapshot.TaskStatus != assigned.Status { return fail(errors.New("task configuration and durable discovery disagree")) } - cursor, exists, err := d.store.LocalTaskDiscoveryCursor(d.dispatcherID) - if err != nil || !exists { - return fail(errors.New("durable task discovery cursor is unavailable")) - } now := d.now().UTC() quotaExpiry := minTime(now.Add(projectConfigTTL), snapshot.QuotaValidUntil) taskExpiry := minTime(now.Add(projectConfigTTL), snapshot.AgentAuthorizationExpiresAt) sipExpiry := now.Add(projectConfigTTL) discoveryExpiry := assigned.DiscoveryObservedAt.Add(taskDiscoveryTTL) - snapshot.DiscoveryCursor = cursor snapshot.FetchedAt = now snapshot.ExpiresAt = minTime(quotaExpiry, taskExpiry, sipExpiry, discoveryExpiry) if !snapshot.ExpiresAt.After(now) { @@ -111,7 +106,6 @@ func (d *Dispatcher) LoadProjectConfig(ctx context.Context, client *configread.C current.SIPCellID = snapshot.SIPCellID current.SIPArtifactRevision = snapshot.SIPArtifactRevision current.SIPArtifactConfigSHA256 = snapshot.SIPArtifactConfigSHA256 - current.DiscoveryCursor = snapshot.DiscoveryCursor current.ExpiresAt = minTime(current.ExpiresAt, sharedExpiry) if key.tenantID == snapshot.TenantID { current.TenantQuota = append(current.TenantQuota[:0], snapshot.TenantQuota...) diff --git a/internal/dispatcher/local_config_read_test.go b/internal/dispatcher/local_config_read_test.go index d75cf28..db46b39 100644 --- a/internal/dispatcher/local_config_read_test.go +++ b/internal/dispatcher/local_config_read_test.go @@ -139,7 +139,7 @@ func TestDispatcherSharesUpdatedTenantQuotaAcrossTasks(t *testing.T) { if err != nil { t.Fatal(err) } - seedLocalEventDiscovery(t, st, now, "task-b") + seedLocalSnapshot(t, st, now, localTestTaskID, "task-b") if err := d.LoadProjectConfig(context.Background(), client, localTestSIPConfigVerifier(), "task-b", localTestTenantID); err != nil { t.Fatal(err) } diff --git a/internal/dispatcher/local_origination.go b/internal/dispatcher/local_origination.go index c3bd22e..413b5a9 100644 --- a/internal/dispatcher/local_origination.go +++ b/internal/dispatcher/local_origination.go @@ -86,14 +86,7 @@ func (d *Dispatcher) AuthorizeLocalOrigination(executionID, agentID string) (aut command.Payload.TaskID != candidate.TaskID || command.Payload.Callee != candidate.Callee { return LocalDialAuthorization{}, fmt.Errorf("bound call.execute identity does not match accepted execution") } - notAfter, err := time.Parse(time.RFC3339, command.NotAfter) - if err != nil { - return LocalDialAuthorization{}, fmt.Errorf("invalid bound command expiry: %w", err) - } at := d.now() - if !at.Before(notAfter) { - return LocalDialAuthorization{}, fmt.Errorf("%w: bound call.execute expired before origination", callwindow.ErrWindowClosed) - } // The accepted snapshot is immutable. Cache expiry prevents NEW admissions; // it does not silently replace an already accepted call's routing policy. bound := configread.Snapshot{SIP: candidate.Snapshot.SIP, Task: candidate.Snapshot.Task} @@ -122,9 +115,6 @@ func (d *Dispatcher) AuthorizeLocalOrigination(executionID, agentID string) (aut if quotaUntil.Before(decision.AllowedUntil) { decision.AllowedUntil = quotaUntil } - if notAfter.Before(decision.AllowedUntil) { - decision.AllowedUntil = notAfter - } if current := d.now(); !current.Before(decision.AllowedUntil) { return LocalDialAuthorization{}, fmt.Errorf("%w: bound dial deadline passed before claim", callwindow.ErrWindowClosed) } diff --git a/internal/dispatcher/local_v01.go b/internal/dispatcher/local_v01.go index 4a6c17b..2eb5c9f 100644 --- a/internal/dispatcher/local_v01.go +++ b/internal/dispatcher/local_v01.go @@ -15,13 +15,15 @@ import ( "git.ipao.vip/rogee/go-sip/internal/callwindow" "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/contract" "git.ipao.vip/rogee/go-sip/internal/mq" "git.ipao.vip/rogee/go-sip/internal/store" "git.ipao.vip/rogee/go-sip/internal/tenant" "github.com/google/uuid" ) -const localCommandSchemaVersion = "command-next.v0.1-proposal" +const localCommandSchemaVersion = "command-next.v0.1-proposal" // command.result egress +const localExecuteSchemaVersion = "call-execute.v0.4-proposal" var localTaskIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) @@ -32,7 +34,6 @@ type localExecuteCommand struct { TenantKey string `json:"tenant_key"` TraceID string `json:"trace_id"` IssuedAt string `json:"issued_at"` - NotAfter string `json:"not_after"` CommandID string `json:"command_id"` CommandType string `json:"command_type"` Payload struct { @@ -111,10 +112,6 @@ func (d *Dispatcher) AcceptLocalV01Command(ctx context.Context, routingKey strin if err != nil { return mq.Permanent(fmt.Errorf("invalid issued_at: %w", err)) } - notAfter, err := time.Parse(time.RFC3339Nano, command.NotAfter) - if err != nil || !notAfter.After(issuedAt) { - return mq.Permanent(errors.New("not_after must be later than issued_at")) - } d.configRefreshMu.RLock() defer d.configRefreshMu.RUnlock() @@ -123,8 +120,8 @@ func (d *Dispatcher) AcceptLocalV01Command(ctx context.Context, routingKey strin reason := "" var projectConfig configread.Snapshot var dial dialDecision - if !now.Before(notAfter) { - reason = "command_expired" + if now.Before(issuedAt) { + reason = "command_not_yet_valid" } else if command.Payload.Callee != "15003164745" && command.Payload.Callee != "15830461047" { reason = "callee_not_allowed" } else { @@ -245,6 +242,9 @@ func (d *Dispatcher) localConfigSnapshot(command localExecuteCommand) (configrea } func decodeLocalExecuteCommand(body []byte) (localExecuteCommand, error) { + if err := contract.ValidateLocalCallExecuteV04(body); err != nil { + return localExecuteCommand{}, fmt.Errorf("invalid v0.4 call.execute: %w", err) + } var command localExecuteCommand decoder := json.NewDecoder(bytes.NewReader(body)) decoder.DisallowUnknownFields() @@ -257,7 +257,7 @@ func decodeLocalExecuteCommand(body []byte) (localExecuteCommand, error) { } return localExecuteCommand{}, err } - if command.SchemaVersion != localCommandSchemaVersion || command.CommandType != "call.execute" { + if command.SchemaVersion != localExecuteSchemaVersion || command.CommandType != "call.execute" { return localExecuteCommand{}, errors.New("unsupported command schema version or type") } return command, nil diff --git a/internal/dispatcher/local_v01_integration_test.go b/internal/dispatcher/local_v01_integration_test.go index 6a29eda..37e7480 100644 --- a/internal/dispatcher/local_v01_integration_test.go +++ b/internal/dispatcher/local_v01_integration_test.go @@ -13,6 +13,7 @@ import ( "reflect" "strings" "sync" + "sync/atomic" "testing" "time" @@ -40,6 +41,7 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { } var requestsMu sync.Mutex var requests []string + var stopped atomic.Bool mockHTTP := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestsMu.Lock() requests = append(requests, r.Method+" "+r.URL.RequestURI()) @@ -51,11 +53,11 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { } if r.URL.Path == "/internal/v1/dispatcher/tasks" { w.Header().Set("Content-Type", "application/json") - switch r.URL.Query().Get("after") { - case "0": - fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"running","task_revision":2}],"next_cursor":"1"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) - case "1": - fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[],"next_cursor":"1"}`, localTestDispatcherID) + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 2, "")) + case "after=42": + fmt.Fprint(w, emptyChangesPage("42")) default: w.WriteHeader(http.StatusBadRequest) } @@ -68,6 +70,23 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { return } w.Header().Set("Content-Type", "application/json") + if stopped.Load() && r.URL.Path == "/internal/v1/dispatcher/task/"+localTestTaskID { + var task map[string]json.RawMessage + if err := json.Unmarshal(body, &task); err != nil { + t.Errorf("Mock task status fixture: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + task["status"] = json.RawMessage(`"stopped"`) + task["task_revision"] = json.RawMessage(`3`) + var err error + body, err = json.Marshal(task) + if err != nil { + t.Errorf("Mock task status encoding: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + } _, _ = w.Write(body) })) defer mockHTTP.Close() @@ -152,9 +171,14 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { t.Fatal(err) } runtime := &LocalV01Runtime{dispatcher: d, client: configClient} + runtime.queues = newV3TaskQueueController(d, broker, nil) + runtime.controls = newLocalTaskControlProcessor(d, configClient, runtime.queues) if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) + } if err := d.LoadProjectConfig(context.Background(), configClient, localTestSIPConfigVerifier(), localTestTaskID, localTestTenantID); err != nil { t.Fatal(err) } @@ -162,8 +186,7 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { gotRequests := append([]string(nil), requests...) requestsMu.Unlock() wantRequests := []string{ - "GET /internal/v1/dispatcher/tasks?after=0", - "GET /internal/v1/dispatcher/tasks?after=1", + "GET /internal/v1/dispatcher/tasks?mode=snapshot", "GET /internal/v1/dispatcher/sip", "GET /internal/v1/dispatcher/task/" + localTestTaskID, "GET /internal/v1/dispatcher/tenant/" + localTestTenantID + "/quota", @@ -171,6 +194,15 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { if !reflect.DeepEqual(gotRequests, wantRequests) { t.Fatalf("Mock SaaS GET flow = %#v, want %#v", gotRequests, wantRequests) } + if err := runtime.refreshDiscoveryState(context.Background()); err != nil || runtime.cursor != "42" { + t.Fatalf("Mock SaaS live discovery from snapshot watermark: cursor=%q err=%v", runtime.cursor, err) + } + requestsMu.Lock() + lastDiscoveryRequest := requests[len(requests)-1] + requestsMu.Unlock() + if lastDiscoveryRequest != "GET /internal/v1/dispatcher/tasks?after=42" { + t.Fatalf("live discovery missed the Mock SaaS change endpoint: %s", lastDiscoveryRequest) + } ctx, cancel := context.WithCancel(context.Background()) consumeErr := make(chan error, 1) @@ -294,6 +326,54 @@ func TestLocalDispatcherMockSaaSEndToEndWithOutboxRecovery(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("Dispatcher task consumer did not stop") } + + // The stopped task still has a SaaS-owned queue. A month-old control must + // drain its unaccepted backlog and emit its own receipt, not expire. + stopped.Store(true) + for _, command := range []struct { + route string + body []byte + }{ + {taskRoute, localExecuteCommandBody(t, "pending-after-stop", now.Add(-30*24*time.Hour))}, + {controlRoute, localTaskControlBody(t, now.Add(-30*24*time.Hour), "stop")}, + } { + if err := channel.PublishWithContext(context.Background(), mq.CommandsExchangeV3, command.route, true, false, amqp.Publishing{ + ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: command.body, + }); err != nil { + t.Fatal(err) + } + } + processed, err := broker.DrainControlPredeclared(context.Background(), controlQueue, runtime.controls.Handle) + if err != nil || processed != 1 { + t.Fatalf("old stop control was not processed: count=%d err=%v", processed, err) + } + remaining, err := channel.QueueInspect(taskQueue) + if err != nil || remaining.Messages != 0 { + t.Fatalf("old stop left pending task messages: messages=%d err=%v", remaining.Messages, err) + } + if n, err := d.FlushOutbox(context.Background(), 2); err != nil || n != 1 { + t.Fatalf("old stop receipt publication: count=%d err=%v", n, err) + } + var stopReceipt struct { + EventType string `json:"event_type"` + Payload struct { + Action string `json:"action"` + Status string `json:"status"` + TaskState string `json:"task_state"` + } `json:"payload"` + } + if err := json.Unmarshal(receiveLocalV3Event(t, resultDeliveries), &stopReceipt); err != nil { + t.Fatal(err) + } + if stopReceipt.EventType != "command.result" || stopReceipt.Payload.Action != "stop" || stopReceipt.Payload.Status != "applied" || stopReceipt.Payload.TaskState != "stopped" { + t.Fatalf("old stop receipt=%+v", stopReceipt) + } + requestsMu.Lock() + lastRequest := requests[len(requests)-1] + requestsMu.Unlock() + if lastRequest != "GET /internal/v1/dispatcher/task/"+localTestTaskID { + t.Fatalf("stop did not verify fresh task status: last HTTP request=%s", lastRequest) + } } func readLocalFixture(t *testing.T, name string) []byte { diff --git a/internal/dispatcher/local_v01_test.go b/internal/dispatcher/local_v01_test.go index 386ca8c..acf93e1 100644 --- a/internal/dispatcher/local_v01_test.go +++ b/internal/dispatcher/local_v01_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "strconv" "strings" "testing" "time" @@ -52,7 +51,7 @@ func TestValidLocalIDMatchesContractLengthAndWhitespace(t *testing.T) { } } -func TestAcceptLocalV01CommandPersistsReceiptAndRejectsExpiredCommand(t *testing.T) { +func TestAcceptLocalV01CommandPersistsReceiptAndAcceptsOldBacklog(t *testing.T) { now := time.Date(2026, 9, 21, 2, 0, 0, 0, time.UTC) d, st, server := newLocalV01TestDispatcher(t, now) defer server.Close() @@ -100,20 +99,47 @@ func TestAcceptLocalV01CommandPersistsReceiptAndRejectsExpiredCommand(t *testing t.Fatal(err) } - expiredBody := localExecuteCommandBody(t, "command-expired", now.Add(-time.Second)) - if err := d.AcceptLocalV01Command(context.Background(), route, expiredBody); err != nil { + oldBody := localExecuteCommandBody(t, "command-old", now.Add(-time.Second)) + if err := d.AcceptLocalV01Command(context.Background(), route, oldBody); err != nil { t.Fatal(err) } - expired, err := st.ClaimOutbox(10) - if err != nil || len(expired) != 1 { - t.Fatalf("claim expired receipt: count=%d err=%v", len(expired), err) + old, err := st.ClaimOutbox(10) + if err != nil || len(old) != 1 { + t.Fatalf("claim old-command receipt: count=%d err=%v", len(old), err) } - var expiredReceipt localTestCommandResult - if err := json.Unmarshal(expired[0].Body, &expiredReceipt); err != nil { + var oldReceipt localTestCommandResult + if err := json.Unmarshal(old[0].Body, &oldReceipt); err != nil { t.Fatal(err) } - if expiredReceipt.Payload.Status != "rejected" || expiredReceipt.Payload.ReasonCode != "command_expired" || expiredReceipt.Payload.ExecutionID != "" { - t.Fatalf("unexpected expired receipt: %+v", expiredReceipt) + if oldReceipt.Payload.Status != "accepted" || oldReceipt.Payload.ExecutionID == "" { + t.Fatalf("old MQ command rejected solely by age: %+v", oldReceipt) + } +} + +func TestFutureIssuedCallExecuteNeverReservesOrDials(t *testing.T) { + now := time.Date(2026, 9, 21, 2, 0, 0, 0, time.UTC) + d, st, server := newLocalV01TestDispatcher(t, now) + defer server.Close() + defer st.Close() + body := localExecuteCommandBody(t, "future-command", now.Add(3*time.Hour)) // issued_at is one hour in the future. + route := "d." + localTestDispatcherID + ".task." + localTestTaskID + ".in" + if err := d.AcceptLocalV01Command(context.Background(), route, body); err != nil { + t.Fatal(err) + } + records, err := st.ClaimOutbox(1) + if err != nil || len(records) != 1 { + t.Fatalf("future command receipt: count=%d err=%v", len(records), err) + } + var receipt localTestCommandResult + if err := json.Unmarshal(records[0].Body, &receipt); err != nil { + t.Fatal(err) + } + if receipt.Payload.Status != "rejected" || receipt.Payload.ReasonCode != "command_not_yet_valid" || receipt.Payload.ExecutionID != "" { + t.Fatalf("future-issued execution was accepted: %+v", receipt) + } + var tasks int + if err := st.DB().QueryRow(`SELECT COUNT(*) FROM tasks`).Scan(&tasks); err != nil || tasks != 0 { + t.Fatalf("future-issued execution reserved a call: tasks=%d err=%v", tasks, err) } } @@ -134,13 +160,13 @@ type localTestCommandResult struct { } `json:"payload"` } -func TestAuthorizeLocalOriginationRechecksBoundWindowAndClaimsOnlyOnce(t *testing.T) { +func TestOldQueuedCallOriginationRechecksBoundWindowAndClaimsOnlyOnce(t *testing.T) { at := time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC) // Shanghai 09:30. d, st, server := newLocalV01TestDispatcher(t, at) defer server.Close() defer st.Close() route := "d." + localTestDispatcherID + ".task." + localTestTaskID + ".in" - if err := d.AcceptLocalV01Command(context.Background(), route, localExecuteCommandBody(t, "command-final-gate", at.Add(2*time.Hour))); err != nil { + if err := d.AcceptLocalV01Command(context.Background(), route, localExecuteCommandBody(t, "command-final-gate", at.Add(-30*24*time.Hour))); err != nil { t.Fatal(err) } outbox, err := st.ClaimOutbox(1) @@ -213,7 +239,7 @@ func TestAuthorizeLocalOriginationRechecksBoundWindowAndClaimsOnlyOnce(t *testin } } -func TestAuthorizeLocalOriginationRejectsPauseAndExpiredCommand(t *testing.T) { +func TestOldQueuedCallOriginationRejectsPauseAndStop(t *testing.T) { for _, tc := range []struct { name string barrier string @@ -221,7 +247,6 @@ func TestAuthorizeLocalOriginationRejectsPauseAndExpiredCommand(t *testing.T) { }{ {"pause before origination", "paused", time.Minute}, {"stop before origination", "stopped", time.Minute}, - {"command expires before origination", "", 2 * time.Minute}, } { t.Run(tc.name, func(t *testing.T) { at := time.Date(2026, 9, 21, 1, 30, 0, 0, time.UTC) @@ -229,7 +254,7 @@ func TestAuthorizeLocalOriginationRejectsPauseAndExpiredCommand(t *testing.T) { defer server.Close() defer st.Close() route := "d." + localTestDispatcherID + ".task." + localTestTaskID + ".in" - if err := d.AcceptLocalV01Command(context.Background(), route, localExecuteCommandBody(t, "command-control-gate", at.Add(90*time.Second))); err != nil { + if err := d.AcceptLocalV01Command(context.Background(), route, localExecuteCommandBody(t, "command-control-gate", at.Add(-30*24*time.Hour))); err != nil { t.Fatal(err) } outbox, err := st.ClaimOutbox(1) @@ -247,7 +272,7 @@ func TestAuthorizeLocalOriginationRejectsPauseAndExpiredCommand(t *testing.T) { } d.now = func() time.Time { return at.Add(tc.advance) } if _, err := d.AuthorizeLocalOrigination(receipt.Payload.ExecutionID, "agent-1"); err == nil { - t.Fatal("a paused task or expired command was authorized to dial") + t.Fatal("a paused or stopped task was authorized to dial") } var status string if err := st.DB().QueryRow(`SELECT status FROM tasks WHERE execution_id=?`, receipt.Payload.ExecutionID).Scan(&status); err != nil || status != "finished" { @@ -445,14 +470,14 @@ func TestAcceptLocalV01CommandRejectsOutsideCurrentCallWindow(t *testing.T) { } } -func TestAcceptLocalV01CommandRejectsUnlistedCallee(t *testing.T) { +func TestOldQueuedCallRejectsCurrentlyUnlistedCallee(t *testing.T) { now := time.Date(2026, 9, 21, 2, 0, 0, 0, time.UTC) d, st, server := newLocalV01TestDispatcher(t, now) defer server.Close() defer st.Close() var command localExecuteCommand - if err := json.Unmarshal(localExecuteCommandBody(t, "command-unlisted", now.Add(time.Minute)), &command); err != nil { + if err := json.Unmarshal(localExecuteCommandBody(t, "command-unlisted", now.Add(-30*24*time.Hour)), &command); err != nil { t.Fatal(err) } command.Payload.Callee = "15000000000" @@ -477,11 +502,11 @@ func TestAcceptLocalV01CommandRejectsUnlistedCallee(t *testing.T) { } } -func localExecuteCommandBody(t *testing.T, commandID string, expiresAt time.Time) []byte { - return localExecuteTaskCommandBody(t, commandID, localTestTaskID, expiresAt) +func localExecuteCommandBody(t *testing.T, commandID string, referenceAt time.Time) []byte { + return localExecuteTaskCommandBody(t, commandID, localTestTaskID, referenceAt) } -func localExecuteTaskCommandBody(t *testing.T, commandID, taskID string, expiresAt time.Time) []byte { +func localExecuteTaskCommandBody(t *testing.T, commandID, taskID string, referenceAt time.Time) []byte { t.Helper() body, err := json.Marshal(struct { SchemaVersion string `json:"schema_version"` @@ -490,7 +515,6 @@ func localExecuteTaskCommandBody(t *testing.T, commandID, taskID string, expires TenantKey string `json:"tenant_key"` TraceID string `json:"trace_id"` IssuedAt string `json:"issued_at"` - NotAfter string `json:"not_after"` CommandID string `json:"command_id"` CommandType string `json:"command_type"` Payload struct { @@ -498,10 +522,10 @@ func localExecuteTaskCommandBody(t *testing.T, commandID, taskID string, expires Callee string `json:"callee"` } `json:"payload"` }{ - SchemaVersion: "command-next.v0.1-proposal", DispatcherID: localTestDispatcherID, + SchemaVersion: "call-execute.v0.4-proposal", DispatcherID: localTestDispatcherID, TenantID: localTestTenantID, TenantKey: localTestTenantKey, - TraceID: "trace-v2", - IssuedAt: expiresAt.Add(-time.Hour).UTC().Format(time.RFC3339Nano), NotAfter: expiresAt.UTC().Format(time.RFC3339Nano), + TraceID: "trace-v2", + IssuedAt: referenceAt.Add(-2 * time.Hour).UTC().Format(time.RFC3339Nano), CommandID: commandID, CommandType: "call.execute", Payload: struct { TaskID string `json:"task_id"` @@ -548,7 +572,7 @@ func newLocalV01TestDispatcher(t *testing.T, now time.Time) (*Dispatcher, *store server.Close() t.Fatal(err) } - seedLocalEventDiscovery(t, st, now, localTestTaskID) + seedLocalSnapshot(t, st, now, localTestTaskID) if err := d.LoadProjectConfig(context.Background(), client, localTestSIPConfigVerifier(), localTestTaskID, localTestTenantID); err != nil { st.Close() server.Close() @@ -557,38 +581,16 @@ func newLocalV01TestDispatcher(t *testing.T, now time.Time) (*Dispatcher, *store return d, st, server } -func seedLocalEventDiscovery(t *testing.T, st *store.Store, now time.Time, taskIDs ...string) { +func seedLocalSnapshot(t *testing.T, st *store.Store, now time.Time, taskIDs ...string) { t.Helper() - var tasks []store.LocalDiscoveredTask - var wire []map[string]any + tasks := make([]store.LocalDiscoveredTask, 0, len(taskIDs)) for _, taskID := range taskIDs { tasks = append(tasks, store.LocalDiscoveredTask{TaskID: taskID, TenantID: localTestTenantID, TenantKey: localTestTenantKey, Status: "running", TaskRevision: 2}) - wire = append(wire, map[string]any{"task_id": taskID, "tenant_id": localTestTenantID, "tenant_key": localTestTenantKey, "status": "running", "task_revision": 2}) } - after, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID) - if err != nil { + if err := st.ApplyLocalTaskSnapshot(localTestDispatcherID, tasks, now); err != nil { t.Fatal(err) } - if !exists { - after = "0" - } - position, err := strconv.ParseUint(after, 10, 64) - if err != nil || position == ^uint64(0) { - t.Fatalf("invalid test discovery cursor %q: %v", after, err) - } - next := strconv.FormatUint(position+1, 10) - body, err := json.Marshal(map[string]any{"schema_version": "task-discovery.v0.3-proposal", "dispatcher_id": localTestDispatcherID, "tasks": wire, "next_cursor": next}) - if err != nil { - t.Fatal(err) - } - if err := st.ApplyLocalTaskDiscoveryPage(store.LocalTaskDiscoveryPage{DispatcherID: localTestDispatcherID, FromCursor: after, NextCursor: next, Tasks: tasks, Body: body, ObservedAt: now}); err != nil { - t.Fatal(err) - } - empty, err := json.Marshal(map[string]any{"schema_version": "task-discovery.v0.3-proposal", "dispatcher_id": localTestDispatcherID, "tasks": []any{}, "next_cursor": next}) - if err != nil { - t.Fatal(err) - } - if err := st.ApplyLocalTaskDiscoveryPage(store.LocalTaskDiscoveryPage{DispatcherID: localTestDispatcherID, FromCursor: next, NextCursor: next, Body: empty, ObservedAt: now}); err != nil { + if err := st.MarkLocalTaskDiscoveryReadyV04(localTestDispatcherID, now); err != nil { t.Fatal(err) } } diff --git a/internal/dispatcher/task_control_v3.go b/internal/dispatcher/task_control_v3.go index a71fc6b..7b77d7c 100644 --- a/internal/dispatcher/task_control_v3.go +++ b/internal/dispatcher/task_control_v3.go @@ -2,6 +2,7 @@ package dispatcher import ( "context" + "database/sql" "encoding/json" "errors" "fmt" @@ -41,13 +42,11 @@ type localTaskControlCommand struct { TenantKey string `json:"tenant_key"` TraceID string `json:"trace_id"` IssuedAt string `json:"issued_at"` - NotAfter string `json:"not_after"` CommandType string `json:"command_type"` Payload struct { - TaskID string `json:"task_id"` - Action string `json:"action"` - ActiveCallPolicy string `json:"active_call_policy"` - Reason string `json:"reason"` + TaskID string `json:"task_id"` + Action string `json:"action"` + Reason string `json:"reason"` } `json:"payload"` } @@ -85,14 +84,14 @@ func (p *localTaskControlProcessor) Handle(ctx context.Context, routingKey strin if p == nil || p.dispatcher == nil || p.statusReader == nil || p.queueControl == nil { return errors.New("task control processor dependencies are required") } - if err := contract.ValidateLocalCommandNext(body); err != nil { - return mq.Permanent(fmt.Errorf("invalid local task-control message: %w", err)) + if err := contract.ValidateLocalTaskControlV04(body); err != nil { + return mq.Permanent(fmt.Errorf("invalid v0.4 task-control message: %w", err)) } var command localTaskControlCommand if err := json.Unmarshal(body, &command); err != nil { return mq.Permanent(fmt.Errorf("decode local task-control message: %w", err)) } - if command.SchemaVersion != localCommandSchemaVersion || command.CommandType != "task.control" { + if command.SchemaVersion != "task-control.v0.4-proposal" || command.CommandType != "task.control" { return mq.Permanent(errors.New("message is not a local task-control command")) } if command.DispatcherID != p.dispatcher.dispatcherID || tenant.ValidateDispatcherID(command.DispatcherID) != nil { @@ -108,39 +107,31 @@ func (p *localTaskControlProcessor) Handle(ctx context.Context, routingKey strin if err != nil { return mq.Permanent(fmt.Errorf("invalid task-control issued_at: %w", err)) } - notAfter, err := time.Parse(time.RFC3339Nano, command.NotAfter) - if err != nil || !notAfter.After(issuedAt) { - return mq.Permanent(errors.New("task-control not_after must be later than issued_at")) - } // The single control consumer already serializes deliveries; this mutex also // protects direct/test callers and preserves task-transition ordering. p.controlMu.Lock() defer p.controlMu.Unlock() - assignments, err := p.dispatcher.store.LocalTaskAssignments(p.dispatcher.dispatcherID) + assignment, err := p.dispatcher.store.LocalTaskAssignment(p.dispatcher.dispatcherID, command.Payload.TaskID) + if errors.Is(err, sql.ErrNoRows) { + ready, readinessErr := p.dispatcher.store.LocalTaskDiscoveryReadyV04(p.dispatcher.dispatcherID) + if readinessErr != nil { + return fmt.Errorf("read task ownership readiness: %w", readinessErr) + } + if !ready { + return fmt.Errorf("task ownership awaits complete snapshot: %w", store.ErrLocalDiscoveryUnavailable) + } + return p.persistUnknownTaskResult(command, "task_unavailable", "stopped") + } if err != nil { return err } - var assignment store.LocalTaskAssignment - found := false - for _, candidate := range assignments { - if candidate.TaskID == command.Payload.TaskID { - assignment, found = candidate, true - break - } - } - if !found { - return p.persistUnknownTaskResult(command, "task_unavailable", "stopped") - } if assignment.TenantID != command.TenantID || assignment.TenantKey != command.TenantKey { return mq.Permanent(errors.New("task-control tenant binding mismatch")) } now := p.dispatcher.now().UTC() - if !now.Before(notAfter) { - return p.persistControlResult(ctx, command, assignment, nil, "rejected", "command_expired", publicTaskAdmissionState(assignment.AdmissionState)) - } if now.Before(issuedAt) { return p.persistControlResult(ctx, command, assignment, nil, "rejected", "command_not_yet_valid", publicTaskAdmissionState(assignment.AdmissionState)) } @@ -165,6 +156,11 @@ func (p *localTaskControlProcessor) pause(ctx context.Context, command localTask if err := p.queueControl.StopTask(ctx, assignment); err != nil { return p.persistControlResult(ctx, command, assignment, nil, "rejected", "queue_unavailable", publicTaskAdmissionState(assignment.AdmissionState)) } + // An authenticated pause interrupts in-flight calls even if the SaaS + // status lookup subsequently fails; the durable barrier stays closed. + if err := p.queueControl.ApplyActiveCallPolicy(ctx, assignment, "pause", "hangup"); err != nil { + return p.persistControlResult(ctx, command, assignment, nil, "rejected", "active_call_policy_failed", publicTaskAdmissionState(assignment.AdmissionState)) + } status, statusErr := p.readAuthoritativeStatus(ctx, command) if statusErr != nil { return p.persistControlResult(ctx, command, assignment, nil, "rejected", "task_unavailable", publicTaskAdmissionState(assignment.AdmissionState)) @@ -175,9 +171,6 @@ func (p *localTaskControlProcessor) pause(ctx context.Context, command localTask if status.Status != "paused" { return p.persistControlResult(ctx, command, assignment, &status, "rejected", "state_mismatch", publicTaskAdmissionState(assignment.AdmissionState)) } - if err := p.queueControl.ApplyActiveCallPolicy(ctx, assignment, "pause", command.Payload.ActiveCallPolicy); err != nil { - return p.persistControlResult(ctx, command, assignment, &status, "rejected", "active_call_policy_failed", publicTaskAdmissionState(assignment.AdmissionState)) - } return p.persistControlResult(ctx, command, assignment, &status, "applied", "applied", "paused") } @@ -220,6 +213,9 @@ func (p *localTaskControlProcessor) stop(ctx context.Context, command localTaskC if err := p.queueControl.StopTask(ctx, assignment); err != nil { return p.persistControlResult(ctx, command, assignment, nil, "rejected", "queue_unavailable", publicTaskAdmissionState(assignment.AdmissionState)) } + if err := p.queueControl.ApplyActiveCallPolicy(ctx, assignment, "stop", "hangup"); err != nil { + return p.persistControlResult(ctx, command, assignment, nil, "rejected", "active_call_policy_failed", publicTaskAdmissionState(assignment.AdmissionState)) + } status, statusErr := p.readAuthoritativeStatus(ctx, command) if statusErr != nil { return p.persistControlResult(ctx, command, assignment, nil, "rejected", "task_unavailable", publicTaskAdmissionState(assignment.AdmissionState)) @@ -230,9 +226,6 @@ func (p *localTaskControlProcessor) stop(ctx context.Context, command localTaskC if assignment.AdmissionState == "finished" || assignment.AdmissionState == "removed" { return p.persistControlResult(ctx, command, assignment, &status, "rejected", "task_stopped", publicTaskAdmissionState(assignment.AdmissionState)) } - if err := p.queueControl.ApplyActiveCallPolicy(ctx, assignment, "stop", command.Payload.ActiveCallPolicy); err != nil { - return p.persistControlResult(ctx, command, assignment, &status, "rejected", "active_call_policy_failed", publicTaskAdmissionState(assignment.AdmissionState)) - } if err := p.dispatcher.store.RecordLocalTaskStatusObservation( assignment.DispatcherID, assignment.TaskID, assignment.TenantID, assignment.TenantKey, status.Status, status.TaskRevision, ); err != nil { diff --git a/internal/dispatcher/task_control_v3_test.go b/internal/dispatcher/task_control_v3_test.go index e69775e..23b1d66 100644 --- a/internal/dispatcher/task_control_v3_test.go +++ b/internal/dispatcher/task_control_v3_test.go @@ -4,10 +4,13 @@ import ( "context" "encoding/json" "errors" + "path/filepath" + "reflect" "testing" "time" "git.ipao.vip/rogee/go-sip/internal/configread" + "git.ipao.vip/rogee/go-sip/internal/mq" "git.ipao.vip/rogee/go-sip/internal/store" ) @@ -85,7 +88,7 @@ func TestTaskControlPauseClosesAdmissionBeforeStoppingConsumer(t *testing.T) { if err != nil || len(assignments) != 1 || assignments[0].AdmissionState != "paused" { t.Fatalf("assignments=%+v err=%v", assignments, err) } - if len(queues.events) != 1 || queues.events[0] != "stop" || reader.calls != 1 { + if len(queues.events) != 2 || queues.events[0] != "stop" || queues.events[1] != "hangup" || reader.calls != 1 { t.Fatalf("queue events=%v status calls=%d", queues.events, reader.calls) } assertTaskControlReceipt(t, st, "pause", "applied", "applied", "paused") @@ -102,8 +105,8 @@ func TestTaskControlStopDrainsOnlyAfterAuthoritativeStop(t *testing.T) { if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "stop")); err != nil { t.Fatal(err) } - if len(queues.events) != 2 || queues.events[0] != "stop" || queues.events[1] != "drain" { - t.Fatalf("stop/drain ordering = %v", queues.events) + if len(queues.events) != 3 || queues.events[0] != "stop" || queues.events[1] != "hangup" || queues.events[2] != "drain" { + t.Fatalf("stop/hangup/drain ordering = %v", queues.events) } assertTaskControlReceipt(t, st, "stop", "applied", "applied", "stopped") } @@ -268,6 +271,98 @@ func localControlRoutingKey() string { return "d." + localTestDispatcherID + ".control.in" } +func TestTaskControlUnknownTaskBeforeCompleteSnapshotRequeues(t *testing.T) { + now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC) + st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + d, err := NewV3(localTestDispatcherID, st, nil, func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + processor := newLocalTaskControlProcessor(d, &fakeTaskControlStatusReader{status: localControlStatus("paused", 3)}, &fakeTaskQueueController{}) + err = processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now, "pause")) + if err == nil || mq.IsPermanent(err) { + t.Fatalf("unresolved ownership was ACKed/rejected instead of requeued: %v", err) + } + var outboxCount int + if err := st.DB().QueryRow(`SELECT COUNT(*) FROM outbox`).Scan(&outboxCount); err != nil || outboxCount != 0 { + t.Fatalf("unknown task produced receipt before snapshot: count=%d err=%v", outboxCount, err) + } +} + +func TestOldQueuedStopStillHangsUpAndDrains(t *testing.T) { + now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC) + d, st, server := newLocalV01TestDispatcher(t, now) + defer st.Close() + defer server.Close() + queues := &fakeTaskQueueController{store: st} + processor := newLocalTaskControlProcessor(d, &fakeTaskControlStatusReader{status: localControlStatus("stopped", 3)}, queues) + if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now.Add(-30*24*time.Hour), "stop")); err != nil { + t.Fatal(err) + } + if len(queues.events) != 3 || queues.events[0] != "stop" || queues.events[1] != "hangup" || queues.events[2] != "drain" { + t.Fatalf("old stop did not hang up and drain: %v", queues.events) + } + assertTaskControlReceipt(t, st, "stop", "applied", "applied", "stopped") +} + +func TestOldQueuedPauseResumeStopPreservesOrderAndEveryReceipt(t *testing.T) { + now := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC) + d, st, server := newLocalV01TestDispatcher(t, now) + defer st.Close() + defer server.Close() + reader := &fakeTaskControlStatusReader{} + queues := &fakeTaskQueueController{} + processor := newLocalTaskControlProcessor(d, reader, queues) + for index, step := range []struct { + action string + status string + }{{"pause", "paused"}, {"resume", "running"}, {"stop", "stopped"}} { + reader.status = localControlStatus(step.status, int64(index+3)) + if err := processor.Handle(context.Background(), localControlRoutingKey(), localTaskControlBody(t, now.Add(-30*24*time.Hour), step.action)); err != nil { + t.Fatalf("old %s: %v", step.action, err) + } + } + wantEvents := []string{"stop", "hangup", "start", "stop", "hangup", "drain"} + if !reflect.DeepEqual(queues.events, wantEvents) || reader.calls != 3 { + t.Fatalf("old control effects skipped/reordered: events=%v reads=%d", queues.events, reader.calls) + } + rows, err := st.DB().Query(`SELECT body FROM outbox ORDER BY rowid`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var actions []string + for rows.Next() { + var body []byte + if err := rows.Scan(&body); err != nil { + t.Fatal(err) + } + var result struct { + Payload struct { + Action string `json:"action"` + Status string `json:"status"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &result); err != nil { + t.Fatal(err) + } + if result.Payload.Status != "applied" { + t.Fatalf("control receipt not applied: %+v", result.Payload) + } + actions = append(actions, result.Payload.Action) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actions, []string{"pause", "resume", "stop"}) { + t.Fatalf("lost or reordered old control receipts: %v", actions) + } +} + func localTaskControlBody(t *testing.T, now time.Time, action string) []byte { t.Helper() payload := map[string]any{ @@ -275,17 +370,13 @@ func localTaskControlBody(t *testing.T, now time.Time, action string) []byte { "action": action, "reason": "local test control", } - if action == "pause" || action == "stop" { - payload["active_call_policy"] = "drain" - } body, err := json.Marshal(map[string]any{ - "schema_version": "command-next.v0.1-proposal", + "schema_version": "task-control.v0.4-proposal", "dispatcher_id": localTestDispatcherID, "tenant_id": localTestTenantID, "tenant_key": localTestTenantKey, "trace_id": "11111111-1111-4111-8111-111111111111", "issued_at": now.Add(-time.Minute).Format(time.RFC3339Nano), - "not_after": now.Add(time.Minute).Format(time.RFC3339Nano), "command_type": "task.control", "payload": payload, }) diff --git a/internal/dispatcher/task_discovery_resume_test.go b/internal/dispatcher/task_discovery_resume_test.go index d999bae..3104044 100644 --- a/internal/dispatcher/task_discovery_resume_test.go +++ b/internal/dispatcher/task_discovery_resume_test.go @@ -2,7 +2,6 @@ package dispatcher import ( "context" - "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -18,54 +17,19 @@ func TestTaskDiscoveryRunningAfterPauseRequiresMQResumeToRestartQueue(t *testing defer server.Close() defer st.Close() - for _, update := range []struct { - after, next, status string - revision int64 - }{ - {after: "1", next: "2", status: "paused", revision: 3}, - {after: "2", next: "3", status: "running", revision: 4}, - } { - body, err := json.Marshal(map[string]any{ - "schema_version": "task-discovery.v0.3-proposal", - "dispatcher_id": localTestDispatcherID, - "next_cursor": update.next, - "tasks": []map[string]any{{ - "task_id": localTestTaskID, "tenant_id": localTestTenantID, "tenant_key": localTestTenantKey, - "status": update.status, "task_revision": update.revision, - }}, - }) - if err != nil { - t.Fatal(err) - } - if err := st.ApplyLocalTaskDiscoveryPage(store.LocalTaskDiscoveryPage{ - DispatcherID: localTestDispatcherID, FromCursor: update.after, NextCursor: update.next, - Tasks: []store.LocalDiscoveredTask{{TaskID: localTestTaskID, TenantID: localTestTenantID, TenantKey: localTestTenantKey, - Status: update.status, TaskRevision: update.revision}}, Body: body, ObservedAt: now, - }); err != nil { - t.Fatal(err) - } - } - empty, err := json.Marshal(map[string]any{ - "schema_version": "task-discovery.v0.3-proposal", "dispatcher_id": localTestDispatcherID, - "next_cursor": "3", "tasks": []any{}, - }) - if err != nil { + if _, err := st.SetLocalTaskAdmissionBarrier(localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, "paused"); err != nil { t.Fatal(err) } - if err := st.ApplyLocalTaskDiscoveryPage(store.LocalTaskDiscoveryPage{ - DispatcherID: localTestDispatcherID, FromCursor: "3", NextCursor: "3", Body: empty, ObservedAt: now, - }); err != nil { + if err := st.ApplyLocalTaskChanges(localTestDispatcherID, []store.LocalDiscoveredTask{{ + TaskID: localTestTaskID, TenantID: localTestTenantID, TenantKey: localTestTenantKey, + Status: "running", TaskRevision: 4, + }}, now); err != nil { t.Fatal(err) } - assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) if err != nil || assignment.Status != "running" || assignment.TaskRevision != 4 || assignment.AdmissionState != "paused" || taskAssignmentCanConsume(assignment) { t.Fatalf("new discovery status reopened paused queue without MQ resume: %+v err=%v", assignment, err) } - cursor, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID) - if err != nil || !exists || cursor != "3" { - t.Fatalf("discovery update was not durably applied: cursor=%q exists=%v err=%v", cursor, exists, err) - } reader := &fakeTaskControlStatusReader{status: localControlStatus("running", 4)} queues := &fakeTaskQueueController{} processor := newLocalTaskControlProcessor(d, reader, queues) @@ -82,24 +46,26 @@ func TestTaskDiscoveryRunningAfterPauseRequiresMQResumeToRestartQueue(t *testing assertTaskControlReceipt(t, st, "resume", "applied", "applied", "running") } -func TestEventRuntimeHTTPPausedToRunningNeedsMQResume(t *testing.T) { - page := func(status string, revision int, cursor string) []byte { - return []byte(fmt.Sprintf(`{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":%q,"task_revision":%d}],"next_cursor":%q}`, - localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, status, revision, cursor)) - } - pages := map[string][]byte{ - "0": page("paused", 1, "1"), - "1": page("running", 2, "2"), - "2": []byte(fmt.Sprintf(`{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[],"next_cursor":"2"}`, localTestDispatcherID)), - } +func TestRuntimeHTTPRunningAfterPauseNeedsMQResume(t *testing.T) { + var queries []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, ok := pages[r.URL.Query().Get("after")] - if r.URL.Path != "/internal/v1/dispatcher/tasks" || !ok { + queries = append(queries, r.URL.RawQuery) + if r.URL.Path != "/internal/v1/dispatcher/tasks" { w.WriteHeader(http.StatusBadRequest) return } w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(body) + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "")) + case "after=42": + fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"running","task_revision":2}],"next_cursor":"43"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) + case "after=43": + fmt.Fprint(w, emptyChangesPage("43")) + default: + t.Errorf("unexpected query %q", r.URL.RawQuery) + w.WriteHeader(http.StatusBadRequest) + } })) defer server.Close() runtime, st := newEventRuntime(t, server) @@ -109,16 +75,21 @@ func TestEventRuntimeHTTPPausedToRunningNeedsMQResume(t *testing.T) { if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := st.SetLocalTaskAdmissionBarrier(localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, "paused"); err != nil { + t.Fatal(err) + } + if err := runtime.refreshDiscoveryState(context.Background()); err != nil { + t.Fatal(err) + } assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) - if err != nil || assignment.Status != "running" || assignment.TaskRevision != 2 || assignment.AdmissionState != "paused" { - t.Fatalf("HTTP discovery did not retain paused admission: %+v err=%v", assignment, err) + if err != nil || assignment.Status != "running" || assignment.TaskRevision != 2 || assignment.AdmissionState != "paused" || taskAssignmentCanConsume(assignment) { + t.Fatalf("HTTP discovery reopened paused admission: %+v err=%v", assignment, err) } - if len(broker.started) != 0 { - t.Fatalf("HTTP discovery started a paused task queue: %v", broker.started) - } - cursor, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID) - if err != nil || !exists || cursor != "2" { - t.Fatalf("HTTP discovery failed to commit status and cursor: cursor=%q exists=%v err=%v", cursor, exists, err) + if len(broker.started) != 0 || runtime.cursor != "43" || len(queries) != 3 { + t.Fatalf("discovery started queue or lost in-memory cursor: started=%v cursor=%q queries=%v", broker.started, runtime.cursor, queries) } reader := &fakeTaskControlStatusReader{status: localControlStatus("running", 2)} processor := newLocalTaskControlProcessor(runtime.dispatcher, reader, runtime.queues) diff --git a/internal/dispatcher/task_queue_v3.go b/internal/dispatcher/task_queue_v3.go index f68e064..70d09eb 100644 --- a/internal/dispatcher/task_queue_v3.go +++ b/internal/dispatcher/task_queue_v3.go @@ -21,6 +21,7 @@ type taskQueueConsumer interface { type taskQueueBroker interface { StartPredeclaredConsumer(context.Context, string, mq.MessageHandler) (taskQueueConsumer, error) DrainPredeclared(context.Context, string) (int, error) + DrainControlPredeclared(context.Context, string, mq.MessageHandler) (int, error) } type v3TaskBrokerAdapter struct{ broker *mq.V3Broker } @@ -33,6 +34,10 @@ func (a v3TaskBrokerAdapter) DrainPredeclared(ctx context.Context, queue string) return a.broker.DrainPredeclared(ctx, queue) } +func (a v3TaskBrokerAdapter) DrainControlPredeclared(ctx context.Context, queue string, handler mq.MessageHandler) (int, error) { + return a.broker.DrainControlPredeclared(ctx, queue, handler) +} + type v3TaskQueueController struct { dispatcher *Dispatcher broker taskQueueBroker @@ -252,7 +257,8 @@ func (q *v3TaskQueueController) ApplyActiveCallPolicy(ctx context.Context, assig return fmt.Errorf("hang up active execution %s: %w", target.ExecutionID, err) } if response == nil || response.GetReceipt().GetResult() != agentv1.ResultCode_RESULT_CODE_APPLIED || - response.GetAppliedTaskRevision() != target.Binding.TaskRevision+1 { + response.GetAppliedTaskRevision() != target.Binding.TaskRevision+1 || + response.GetState() != agentv1.ExecutionState_EXECUTION_STATE_TERMINAL { return fmt.Errorf("Agent did not confirm hangup for execution %s", target.ExecutionID) } } diff --git a/internal/dispatcher/task_queue_v3_stop_active_test.go b/internal/dispatcher/task_queue_v3_stop_active_test.go index 9cb6478..b8cffe6 100644 --- a/internal/dispatcher/task_queue_v3_stop_active_test.go +++ b/internal/dispatcher/task_queue_v3_stop_active_test.go @@ -2,7 +2,6 @@ package dispatcher import ( "context" - "encoding/json" "testing" "time" @@ -10,6 +9,7 @@ import ( ) type mockActiveStopRecorder struct { + state agentv1.ExecutionState calls []struct { agentID string executionID string @@ -25,9 +25,14 @@ func (r *mockActiveStopRecorder) Control(_ context.Context, agentID string, bind action agentv1.ControlAction policy agentv1.ActiveCallPolicy }{agentID, binding.ExecutionId, action, policy}) + state := r.state + if state == agentv1.ExecutionState_EXECUTION_STATE_UNSPECIFIED { + state = agentv1.ExecutionState_EXECUTION_STATE_TERMINAL + } return &agentv1.ApplyTaskControlResponse{ Receipt: &agentv1.OperationReceipt{Result: agentv1.ResultCode_RESULT_CODE_APPLIED}, AppliedTaskRevision: binding.TaskRevision + 1, + State: state, }, nil } @@ -73,21 +78,21 @@ func TestAuthorizedMockInFlightExecutionsRemainAddressableForStop(t *testing.T) t.Fatalf("stop selected an unrelated or unbound call: %+v", target) } } - control := &mockActiveStopRecorder{} + control := &mockActiveStopRecorder{state: agentv1.ExecutionState_EXECUTION_STATE_OBSERVED} broker := &fakeQueueBroker{} queues := newTaskQueueController(d, broker, control) - reader := &fakeTaskControlStatusReader{status: localControlStatus("stopped", 3)} - processor := newLocalTaskControlProcessor(d, reader, queues) - var stop map[string]any - if err := json.Unmarshal(localTaskControlBody(t, at, "stop"), &stop); err != nil { - t.Fatal(err) - } - stop["payload"].(map[string]any)["active_call_policy"] = "hangup" - body, err := json.Marshal(stop) + assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) if err != nil { t.Fatal(err) } - if err := processor.Handle(ctx, localControlRoutingKey(), body); err != nil { + if err := queues.ApplyActiveCallPolicy(ctx, assignment, "pause", "hangup"); err == nil { + t.Fatal("Agent applied receipt without terminal call state was accepted as hangup") + } + control.calls = nil + control.state = agentv1.ExecutionState_EXECUTION_STATE_TERMINAL + reader := &fakeTaskControlStatusReader{status: localControlStatus("stopped", 3)} + processor := newLocalTaskControlProcessor(d, reader, queues) + if err := processor.Handle(ctx, localControlRoutingKey(), localTaskControlBody(t, at, "stop")); err != nil { t.Fatalf("persist stop, hang up both calls and drain the owned queue: %v", err) } if len(control.calls) != 2 || len(broker.drained) != 1 { diff --git a/internal/dispatcher/task_queue_v3_test.go b/internal/dispatcher/task_queue_v3_test.go index 1850e32..30ea167 100644 --- a/internal/dispatcher/task_queue_v3_test.go +++ b/internal/dispatcher/task_queue_v3_test.go @@ -16,6 +16,8 @@ type fakeQueueBroker struct { consumers []*fakeQueueConsumer consumerWaitErr error handler mq.MessageHandler + controlDrain func(context.Context) (int, error) + controlDrained bool } func (f *fakeQueueBroker) StartPredeclaredConsumer(_ context.Context, queue string, handler mq.MessageHandler) (taskQueueConsumer, error) { @@ -31,6 +33,14 @@ func (f *fakeQueueBroker) DrainPredeclared(_ context.Context, queue string) (int return 3, nil } +func (f *fakeQueueBroker) DrainControlPredeclared(ctx context.Context, _ string, _ mq.MessageHandler) (int, error) { + f.controlDrained = true + if f.controlDrain != nil { + return f.controlDrain(ctx) + } + return 0, nil +} + type fakeQueueConsumer struct { done chan struct{} once sync.Once diff --git a/internal/dispatcher/task_runtime_v03_recovery_test.go b/internal/dispatcher/task_runtime_v03_recovery_test.go index ddb45ac..c8d46f8 100644 --- a/internal/dispatcher/task_runtime_v03_recovery_test.go +++ b/internal/dispatcher/task_runtime_v03_recovery_test.go @@ -5,76 +5,12 @@ import ( "fmt" "net/http" "net/http/httptest" - "path/filepath" "reflect" - "sync" "sync/atomic" "testing" ) -func TestEventRuntimeResumesAtCommittedCursorAfterRestart(t *testing.T) { - var fail atomic.Bool - fail.Store(true) - var mu sync.Mutex - var requests []string - pages := map[string]string{ - "0": "task-discovery-page-v0.3.json", - "2": "task-discovery-updated-v0.3.json", - "3": "task-discovery-removed-v0.3.json", - "4": "task-discovery-empty-v0.3.json", - } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - after := r.URL.Query().Get("after") - mu.Lock() - requests = append(requests, after) - mu.Unlock() - w.Header().Set("Content-Type", "application/json") - if r.URL.Path != "/internal/v1/dispatcher/tasks" || pages[after] == "" { - w.WriteHeader(http.StatusBadRequest) - return - } - if after == "2" && fail.Load() { - w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.3-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`)) - return - } - _, _ = w.Write(eventRuntimeFixture(t, pages[after])) - })) - defer server.Close() - path := filepath.Join(t.TempDir(), "dispatcher.db") - runtime, st := newEventRuntimeAt(t, server, path) - if err := runtime.refreshDiscoveryState(context.Background()); err == nil { - t.Fatal("partial discovery accepted before restart") - } - cursor, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID) - if err != nil || !exists || cursor != "2" { - t.Fatalf("checkpoint before restart=%q exists=%v err=%v", cursor, exists, err) - } - if err := st.Close(); err != nil { - t.Fatal(err) - } - fail.Store(false) - restarted, reopened := newEventRuntimeAt(t, server, path) - if err := restarted.refreshDiscoveryState(context.Background()); err != nil { - t.Fatal(err) - } - mu.Lock() - got := append([]string(nil), requests...) - mu.Unlock() - if !reflect.DeepEqual(got, []string{"0", "2", "2", "3", "4"}) { - t.Fatalf("restart reset or skipped event cursor: requests=%v", got) - } - var ready int - if err := reopened.DB().QueryRow(`SELECT cursor,ready FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&cursor, &ready); err != nil || cursor != "4" || ready != 1 { - t.Fatalf("checkpoint after restart=%q ready=%d err=%v", cursor, ready, err) - } - assignments, err := reopened.LocalTaskAssignments(localTestDispatcherID) - if err != nil || len(assignments) != 2 || assignments[0].Status != "paused" || !assignments[1].Removed { - t.Fatalf("replayed assignments=%+v err=%v", assignments, err) - } -} - -func TestEventRuntimeCatchesUpdateArrivingDuringPagination(t *testing.T) { +func TestRuntimeCatchesChangeCommittedDuringSnapshotPagination(t *testing.T) { firstPageRequested := make(chan struct{}) updateCommitted := make(chan struct{}) var updated atomic.Bool @@ -83,41 +19,27 @@ func TestEventRuntimeCatchesUpdateArrivingDuringPagination(t *testing.T) { updated.Store(true) close(updateCommitted) }() - var mu sync.Mutex var requests []string - task := func(id, status string) string { - return fmt.Sprintf(`{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":%q,"task_revision":2}`, id, localTestTenantID, localTestTenantKey, status) - } - page := func(tasks, next string) []byte { - return []byte(fmt.Sprintf(`{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[%s],"next_cursor":%q}`, localTestDispatcherID, tasks, next)) - } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - after := r.URL.Query().Get("after") - mu.Lock() - requests = append(requests, after) - mu.Unlock() + requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") - if r.URL.Path != "/internal/v1/dispatcher/tasks" { - w.WriteHeader(http.StatusBadRequest) - return - } - switch after { - case "0": - body := page(task(localTestTaskID, "running"), "1") + switch r.URL.RawQuery { + case "mode=snapshot": close(firstPageRequested) <-updateCommitted - _, _ = w.Write(body) - case "1": - _, _ = w.Write(page(task("task-b", "running"), "2")) - case "2": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2")) + case "page_token=page-2&snapshot_id=" + testSnapshotID: + fmt.Fprint(w, emptySnapshotPage()) + case "after=42": if !updated.Load() { w.WriteHeader(http.StatusServiceUnavailable) return } - _, _ = w.Write(page(task(localTestTaskID, "paused"), "3")) - case "3": - _, _ = w.Write(page("", "3")) + fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"removed","task_revision":2}],"next_cursor":"43"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) + case "after=43": + fmt.Fprint(w, emptyChangesPage("43")) default: + t.Errorf("unexpected query %q", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) } })) @@ -126,18 +48,23 @@ func TestEventRuntimeCatchesUpdateArrivingDuringPagination(t *testing.T) { if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } - mu.Lock() - got := append([]string(nil), requests...) - mu.Unlock() - if !reflect.DeepEqual(got, []string{"0", "1", "2", "3"}) { - t.Fatalf("concurrent update lost or pages skipped: %v", got) + runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil) + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) } - assignments, err := st.LocalTaskAssignments(localTestDispatcherID) - if err != nil || len(assignments) != 2 { - t.Fatalf("incomplete assignments: %+v err=%v", assignments, err) + if err := runtime.refreshDiscoveryState(context.Background()); err != nil { + t.Fatal(err) } - status := map[string]string{assignments[0].TaskID: assignments[0].Status, assignments[1].TaskID: assignments[1].Status} - if status[localTestTaskID] != "paused" || status["task-b"] != "running" { - t.Fatalf("update did not supersede earlier page: %+v", assignments) + assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) + if err != nil || !assignment.Removed || assignment.TaskRevision != 2 || runtime.cursor != "43" { + t.Fatalf("change during snapshot was lost: %+v err=%v cursor=%q", assignment, err, runtime.cursor) + } + want := []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID, "after=42", "after=43"} + if !reflect.DeepEqual(requests, want) { + t.Fatalf("discovery requests=%v want=%v", requests, want) + } + var ready int + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { + t.Fatalf("live changes did not reopen admission: ready=%d err=%v", ready, err) } } diff --git a/internal/dispatcher/task_runtime_v03_test.go b/internal/dispatcher/task_runtime_v03_test.go index 2f4237c..987b4be 100644 --- a/internal/dispatcher/task_runtime_v03_test.go +++ b/internal/dispatcher/task_runtime_v03_test.go @@ -1,8 +1,8 @@ package dispatcher import ( - "bytes" "context" + "fmt" "net/http" "net/http/httptest" "path/filepath" @@ -15,10 +15,7 @@ import ( "git.ipao.vip/rogee/go-sip/internal/store" ) -func eventRuntimeFixture(t *testing.T, name string) []byte { - t.Helper() - return bytes.ReplaceAll(localConfigFixture(t, name), []byte("11111111-1111-4111-8111-111111111111"), []byte(localTestDispatcherID)) -} +const testSnapshotID = "22222222-2222-4222-8222-222222222222" func newEventRuntime(t *testing.T, server *httptest.Server) (*LocalV01Runtime, *store.Store) { t.Helper() @@ -43,83 +40,182 @@ func newEventRuntimeAt(t *testing.T, server *httptest.Server, path string) (*Loc return &LocalV01Runtime{dispatcher: d, client: client}, st } -func TestEventRuntimeBootsOnlyAfterAllPagesAndEmptyPage(t *testing.T) { - pages := map[string]string{"0": "task-discovery-page-v0.3.json", "2": "task-discovery-updated-v0.3.json", "3": "task-discovery-removed-v0.3.json", "4": "task-discovery-empty-v0.3.json"} - var afters []string +func snapshotTaskPage(taskID, status string, revision int64, next string) string { + nextJSON := "null" + if next != "" { + nextJSON = fmt.Sprintf("%q", next) + } + return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":%q,"watermark":"42","tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":%q,"task_revision":%d}],"next_page_token":%s}`, + localTestDispatcherID, testSnapshotID, taskID, localTestTenantID, localTestTenantKey, status, revision, nextJSON) +} + +func emptySnapshotPage() string { + return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":%q,"watermark":"42","tasks":[],"next_page_token":null}`, localTestDispatcherID, testSnapshotID) +} + +func emptyChangesPage(cursor string) string { + return fmt.Sprintf(`{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[],"next_cursor":%q}`, localTestDispatcherID, cursor) +} + +func TestRuntimeCommitsCompleteSnapshotBeforeControlAndTaskAdmission(t *testing.T) { + var requests []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - after := r.URL.Query().Get("after") - afters = append(afters, after) - if r.URL.Path != "/internal/v1/dispatcher/tasks" || r.URL.Query().Has("page_token") || pages[after] == "" { - t.Errorf("bad discovery request: %s", r.URL.RequestURI()) - w.WriteHeader(http.StatusBadRequest) - return - } + requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(eventRuntimeFixture(t, pages[after])) + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2")) + case "page_token=page-2&snapshot_id=" + testSnapshotID: + fmt.Fprint(w, snapshotTaskPage("task-b", "paused", 1, "")) + case "after=42": + fmt.Fprint(w, emptyChangesPage("42")) + default: + t.Errorf("unexpected query: %s", r.URL.RawQuery) + w.WriteHeader(http.StatusBadRequest) + } })) defer server.Close() runtime, st := newEventRuntime(t, server) if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(afters, []string{"0", "2", "3", "4"}) { - t.Fatalf("pages read=%v", afters) + if !reflect.DeepEqual(requests, []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID}) || !runtime.complete || runtime.cursor != "42" { + t.Fatalf("snapshot incomplete or lost in-memory watermark: queries=%v complete=%v cursor=%q", requests, runtime.complete, runtime.cursor) } var ready int - if err := st.DB().QueryRow(`SELECT ready FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { - t.Fatalf("complete discovery ready=%d err=%v", ready, err) + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 { + t.Fatalf("task admitted before control backlog: ready=%d err=%v", ready, err) } assignments, err := st.LocalTaskAssignments(localTestDispatcherID) - if err != nil || len(assignments) != 2 || assignments[0].Status != "paused" || assignments[1].Status != "removed" { - t.Fatalf("assignments=%+v err=%v", assignments, err) + if err != nil || len(assignments) != 2 || assignments[0].TaskID != "task-b" || assignments[1].TaskID != localTestTaskID { + t.Fatalf("complete snapshot assignments=%+v err=%v", assignments, err) + } + runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil) + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) } if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } - if afters[len(afters)-1] != "4" { - t.Fatalf("poll did not resume durable cursor: %v", afters) + if requests[len(requests)-1] != "after=42" || runtime.cursor != "42" { + t.Fatalf("live discovery did not start from snapshot watermark: %v cursor=%q", requests, runtime.cursor) + } + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { + t.Fatalf("control backlog did not open admission: %d %v", ready, err) } } -func TestEventRuntimeInterruptedPageResumesWithoutReset(t *testing.T) { +func TestRuntimeInterruptedSnapshotRestartsFromFirstPage(t *testing.T) { var fail atomic.Bool fail.Store(true) - var afters []string + var requests []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - after := r.URL.Query().Get("after") - afters = append(afters, after) + requests = append(requests, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") - if after == "2" && fail.Load() { - w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.3-proposal","resource":"error","error":{"code":"service_unavailable","message":"test outage"}}`)) - return - } - name := map[string]string{"0": "task-discovery-page-v0.3.json", "2": "task-discovery-updated-v0.3.json", "3": "task-discovery-removed-v0.3.json", "4": "task-discovery-empty-v0.3.json"}[after] - if name == "" { - t.Errorf("unexpected cursor %q", after) + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "page-2")) + case "page_token=page-2&snapshot_id=" + testSnapshotID: + if fail.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, `{"schema_version":"task-discovery.v0.4-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`) + return + } + fmt.Fprint(w, emptySnapshotPage()) + default: + t.Errorf("unexpected query %q", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) - return } - _, _ = w.Write(eventRuntimeFixture(t, name)) })) defer server.Close() runtime, st := newEventRuntime(t, server) if err := runtime.refreshDiscoveryState(context.Background()); err == nil { - t.Fatal("partial page accepted") + t.Fatal("partial snapshot accepted") } - cursor, exists, err := st.LocalTaskDiscoveryCursor(localTestDispatcherID) - if err != nil || !exists || cursor != "2" { - t.Fatalf("partial cursor=%q exists=%v err=%v", cursor, exists, err) - } - var ready int - if err := st.DB().QueryRow(`SELECT ready FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 { - t.Fatalf("partial response opened admission: ready=%d err=%v", ready, err) + assignments, err := st.LocalTaskAssignments(localTestDispatcherID) + if err != nil || len(assignments) != 0 || runtime.complete || runtime.cursor != "" { + t.Fatalf("failed snapshot persisted partial assignments: %+v err=%v complete=%v cursor=%q", assignments, err, runtime.complete, runtime.cursor) } fail.Store(false) if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(afters, []string{"0", "2", "2", "3", "4"}) { - t.Fatalf("retry must use persisted cursor, got %v", afters) + want := []string{"mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID, "mode=snapshot", "page_token=page-2&snapshot_id=" + testSnapshotID} + if !reflect.DeepEqual(requests, want) || !runtime.complete || runtime.cursor != "42" { + t.Fatalf("partial snapshot resumed instead of restarting: requests=%v complete=%v cursor=%q", requests, runtime.complete, runtime.cursor) + } +} + +func TestRuntimeRestartDiscardsLiveCursorAndReloadsCompleteSnapshot(t *testing.T) { + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.URL.RawQuery) + w.Header().Set("Content-Type", "application/json") + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "")) + case "after=42": + fmt.Fprint(w, emptyChangesPage("42")) + default: + t.Errorf("unexpected query %q", r.URL.RawQuery) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + path := filepath.Join(t.TempDir(), "dispatcher.db") + first, st := newEventRuntimeAt(t, server, path) + if err := first.refreshDiscoveryState(context.Background()); err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + second, _ := newEventRuntimeAt(t, server, path) + if second.complete || second.cursor != "" { + t.Fatalf("new runtime reused old cursor: %q complete=%v", second.cursor, second.complete) + } + if err := second.refreshDiscoveryState(context.Background()); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(requests, []string{"mode=snapshot", "mode=snapshot"}) { + t.Fatalf("restart skipped full snapshot: requests=%v", requests) + } +} + +func TestRuntimeCatchesChangesArrivingAfterSnapshotWatermark(t *testing.T) { + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.URL.RawQuery) + w.Header().Set("Content-Type", "application/json") + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprint(w, snapshotTaskPage(localTestTaskID, "running", 1, "")) + case "after=42": + fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"removed","task_revision":2}],"next_cursor":"43"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) + case "after=43": + fmt.Fprint(w, emptyChangesPage("43")) + default: + t.Errorf("unexpected query %q", r.URL.RawQuery) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + runtime, st := newEventRuntime(t, server) + if err := runtime.refreshDiscoveryState(context.Background()); err != nil { + t.Fatal(err) + } + runtime.queues = newTaskQueueController(runtime.dispatcher, &fakeQueueBroker{}, nil) + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) + } + if err := runtime.refreshDiscoveryState(context.Background()); err != nil { + t.Fatal(err) + } + assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) + if err != nil || !assignment.Removed || assignment.TaskRevision != 2 || runtime.cursor != "43" { + t.Fatalf("event after snapshot was lost: %+v err=%v cursor=%q", assignment, err, runtime.cursor) + } + if !reflect.DeepEqual(requests, []string{"mode=snapshot", "after=42", "after=43"}) { + t.Fatalf("changes sequence=%v", requests) } } diff --git a/internal/dispatcher/task_runtime_v04_control_test.go b/internal/dispatcher/task_runtime_v04_control_test.go new file mode 100644 index 0000000..77fa537 --- /dev/null +++ b/internal/dispatcher/task_runtime_v04_control_test.go @@ -0,0 +1,55 @@ +package dispatcher + +import ( + "context" + "path/filepath" + "testing" + "time" + + "git.ipao.vip/rogee/go-sip/internal/store" +) + +func TestV04ControlBacklogClosesAdmissionBeforeTaskConsumption(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + now := time.Now().UTC() + assignment := store.LocalDiscoveredTask{TaskID: localTestTaskID, TenantID: localTestTenantID, TenantKey: localTestTenantKey, Status: "running", TaskRevision: 1} + if err := st.ApplyLocalTaskSnapshot(localTestDispatcherID, []store.LocalDiscoveredTask{assignment}, now); err != nil { + t.Fatal(err) + } + d, err := NewV3(localTestDispatcherID, st, nil, time.Now) + if err != nil { + t.Fatal(err) + } + broker := &fakeQueueBroker{controlDrain: func(_ context.Context) (int, error) { + var ready int + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 { + t.Fatalf("control backlog processed with task admission open: ready=%d err=%v", ready, err) + } + if _, err := st.SetLocalTaskAdmissionBarrier(localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey, "paused"); err != nil { + return 0, err + } + return 1, nil + }} + queues := newTaskQueueController(d, broker, nil) + runtime := &LocalV01Runtime{dispatcher: d, queues: queues, controls: &localTaskControlProcessor{}} + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) + } + if !broker.controlDrained { + t.Fatal("control backlog was not processed") + } + var ready int + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { + t.Fatalf("admission did not open after backlog: %d %v", ready, err) + } + if err := runtime.reconcileTaskQueues(context.Background()); err != nil { + t.Fatal(err) + } + if len(broker.started) != 0 { + t.Fatalf("task queue started despite queued pause: %v", broker.started) + } +} diff --git a/internal/dispatcher/task_runtime_v3.go b/internal/dispatcher/task_runtime_v3.go index 5136f0d..4c3ea90 100644 --- a/internal/dispatcher/task_runtime_v3.go +++ b/internal/dispatcher/task_runtime_v3.go @@ -18,8 +18,8 @@ const ( outboxFlushBatchSize = 64 ) -// LocalV01Runtime owns the V3 control and task consumers plus durable discovery -// polling. Broker queues remain SaaS-owned; this runtime only passively consumes +// LocalV01Runtime owns the control and task consumers plus full startup discovery +// and in-memory live changes. Broker queues remain SaaS-owned; it only consumes // them and publishes Dispatcher outbox records. type LocalV01Runtime struct { dispatcher *Dispatcher @@ -28,6 +28,8 @@ type LocalV01Runtime struct { verifier SIPConfigVerifier queues *v3TaskQueueController controls *localTaskControlProcessor + cursor string // live event ID; never persisted across a process restart + complete bool // a full snapshot of this process's assignments has been committed } func NewLocalV01Runtime(d *Dispatcher, broker *mq.V3Broker, client *configread.Client, verifier SIPConfigVerifier, taskController TaskController) (*LocalV01Runtime, error) { @@ -60,12 +62,15 @@ func (r *LocalV01Runtime) Run(ctx context.Context) error { if r == nil || r.dispatcher == nil || r.broker == nil || r.client == nil || r.verifier == nil || r.queues == nil || r.controls == nil { return errors.New("local v0.1 runtime is not configured") } - if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmission(r.dispatcher.dispatcherID); err != nil { + if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmissionV04(r.dispatcher.dispatcherID); err != nil { return fmt.Errorf("close admission before initial discovery: %w", err) } initialDiscoveryErr := r.refreshDiscoveryState(ctx) + if initialDiscoveryErr == nil { + initialDiscoveryErr = r.catchUpControlBeforeTasks(ctx) + } if initialDiscoveryErr != nil && !errors.Is(initialDiscoveryErr, context.Canceled) { - slog.Error("initial task discovery failed; task admission remains closed", "dispatcher_id", r.dispatcher.dispatcherID, "error", initialDiscoveryErr) + slog.Error("initial task discovery/control backlog failed; task admission remains closed", "dispatcher_id", r.dispatcher.dispatcherID, "error", initialDiscoveryErr) } controlConsumer, err := r.broker.StartPredeclaredConsumer(ctx, r.broker.ControlQueue(), r.controls.Handle) if err != nil { @@ -118,8 +123,27 @@ func (r *LocalV01Runtime) Run(ctx context.Context) error { return fmt.Errorf("task queue consumer stopped: %w", err) case <-discoveryTicker.C: r.recoverMockResults(ctx) - if err := r.refreshDiscoveryState(ctx); err != nil { - slog.Error("task discovery refresh failed; task admission closed", "dispatcher_id", r.dispatcher.dispatcherID, "error", err) + catchingUp := !r.complete + if catchingUp { + // Never process the same control backlog with two consumers. + if err := controlConsumer.Stop(ctx); err != nil { + return fmt.Errorf("stop control consumer for snapshot catch-up: %w", err) + } + } + discoveryErr := r.refreshDiscoveryState(ctx) + if catchingUp && discoveryErr == nil { + discoveryErr = r.catchUpControlBeforeTasks(ctx) + } + if catchingUp { + controlConsumer, err = r.broker.StartPredeclaredConsumer(ctx, r.broker.ControlQueue(), r.controls.Handle) + if err != nil { + return fmt.Errorf("restart SaaS-predeclared control consumer: %w", err) + } + controlDone = make(chan error, 1) + go func(done chan error, consumer *mq.V3Consumer) { done <- consumer.Wait(context.Background()) }(controlDone, controlConsumer) + } + if discoveryErr != nil { + slog.Error("task discovery/control backlog refresh failed; task admission closed", "dispatcher_id", r.dispatcher.dispatcherID, "error", discoveryErr) continue } if err := r.reconcileTaskQueues(ctx); err != nil { @@ -149,43 +173,99 @@ func (r *LocalV01Runtime) recoverMockResults(ctx context.Context) { } func (r *LocalV01Runtime) refreshDiscoveryState(ctx context.Context) error { - cursor, exists, err := r.dispatcher.store.LocalTaskDiscoveryCursor(r.dispatcher.dispatcherID) - if err != nil { - return r.failDiscovery(fmt.Errorf("read durable task-discovery cursor: %w", err)) - } - if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmission(r.dispatcher.dispatcherID); err != nil { + if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmissionV04(r.dispatcher.dispatcherID); err != nil { return r.failDiscovery(fmt.Errorf("close task admission before discovery: %w", err)) } - if !exists { - cursor = "0" - } pageCtx, cancel := context.WithTimeout(ctx, taskDiscoveryPollInterval) defer cancel() + if !r.complete { + var snapshotID, watermark, pageToken string + seen, tokens := make(map[string]bool), make(map[string]bool) + var tasks []store.LocalDiscoveredTask + for { + page, err := r.client.ReadTaskSnapshot(pageCtx, snapshotID, pageToken) + if err != nil { + return r.failDiscovery(fmt.Errorf("read complete task snapshot: %w", err)) + } + if snapshotID == "" { + snapshotID, watermark = page.SnapshotID, page.Watermark + } else if page.SnapshotID != snapshotID || page.Watermark != watermark { + return r.failDiscovery(errors.New("task snapshot identity or watermark changed between pages")) + } + for _, task := range page.Tasks { + if seen[task.TaskID] { + return r.failDiscovery(fmt.Errorf("task %s appears twice in complete snapshot", task.TaskID)) + } + seen[task.TaskID] = true + tasks = append(tasks, discoveredLocalTask(task)) + } + if len(tasks) > 256 { + return r.failDiscovery(errors.New("complete task snapshot exceeds per-Dispatcher task limit")) + } + if page.NextPageToken == "" { + break + } + if tokens[page.NextPageToken] { + return r.failDiscovery(errors.New("task snapshot repeats a page token")) + } + tokens[page.NextPageToken] = true + pageToken = page.NextPageToken + } + if err := r.dispatcher.store.ApplyLocalTaskSnapshot(r.dispatcher.dispatcherID, tasks, r.dispatcher.now().UTC()); err != nil { + return r.failDiscovery(fmt.Errorf("persist complete task snapshot: %w", err)) + } + slog.Info("complete task snapshot applied", "dispatcher_id", r.dispatcher.dispatcherID, "snapshot_id", snapshotID, "watermark", watermark, "task_count", len(tasks)) + r.cursor, r.complete = watermark, true + return nil + } for { - page, err := r.client.ReadTaskDiscovery(pageCtx, cursor) + page, err := r.client.ReadTaskChanges(pageCtx, r.cursor) if err != nil { - return r.failDiscovery(fmt.Errorf("read task discovery after %s: %w", cursor, err)) + return r.failDiscovery(fmt.Errorf("read live task changes after %s: %w", r.cursor, err)) + } + if len(page.Tasks) == 0 { + if err := r.dispatcher.store.MarkLocalTaskDiscoveryReadyV04(r.dispatcher.dispatcherID, r.dispatcher.now().UTC()); err != nil { + return r.failDiscovery(fmt.Errorf("mark live task changes caught up: %w", err)) + } + return nil } tasks := make([]store.LocalDiscoveredTask, 0, len(page.Tasks)) for _, task := range page.Tasks { - tasks = append(tasks, store.LocalDiscoveredTask{TaskID: task.TaskID, TenantID: task.TenantID, - TenantKey: task.TenantKey, TaskRevision: task.TaskRevision, Status: task.Status}) + tasks = append(tasks, discoveredLocalTask(task)) } - if err := r.dispatcher.store.ApplyLocalTaskDiscoveryPage(store.LocalTaskDiscoveryPage{ - DispatcherID: page.DispatcherID, FromCursor: cursor, NextCursor: page.NextCursor, - Tasks: tasks, Body: page.Body, ObservedAt: r.dispatcher.now().UTC(), - }); err != nil { - return r.failDiscovery(fmt.Errorf("persist task-discovery page after %s: %w", cursor, err)) + if err := r.dispatcher.store.ApplyLocalTaskChanges(r.dispatcher.dispatcherID, tasks, r.dispatcher.now().UTC()); err != nil { + return r.failDiscovery(fmt.Errorf("persist live task changes after %s: %w", r.cursor, err)) } - if len(tasks) == 0 { - return nil - } - cursor = page.NextCursor + r.cursor = page.NextCursor } } +func discoveredLocalTask(task configread.DiscoveredTask) store.LocalDiscoveredTask { + return store.LocalDiscoveredTask{TaskID: task.TaskID, TenantID: task.TenantID, + TenantKey: task.TenantKey, TaskRevision: task.TaskRevision, Status: task.Status} +} + +// catchUpControlBeforeTasks processes queued controls while no task may be +// admitted. It must run without a concurrent control consumer. +func (r *LocalV01Runtime) catchUpControlBeforeTasks(ctx context.Context) error { + if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmissionV04(r.dispatcher.dispatcherID); err != nil { + return r.failDiscovery(fmt.Errorf("close admission before control backlog: %w", err)) + } + count, err := r.queues.broker.DrainControlPredeclared(ctx, mq.V3ControlQueueName(r.dispatcher.dispatcherID), r.controls.Handle) + if err != nil { + return r.failDiscovery(fmt.Errorf("process SaaS-owned control backlog: %w", err)) + } + if err := r.dispatcher.store.MarkLocalTaskDiscoveryReadyV04(r.dispatcher.dispatcherID, r.dispatcher.now().UTC()); err != nil { + return r.failDiscovery(fmt.Errorf("open admission after control backlog: %w", err)) + } + slog.Info("SaaS-owned control backlog processed", "dispatcher_id", r.dispatcher.dispatcherID, "messages", count) + return nil +} + func (r *LocalV01Runtime) failDiscovery(cause error) error { - if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmission(r.dispatcher.dispatcherID); err != nil { + r.complete = false // failed changes require a new complete snapshot, never a persisted cursor + r.cursor = "" + if err := r.dispatcher.store.CloseLocalTaskDiscoveryAdmissionV04(r.dispatcher.dispatcherID); err != nil { cause = errors.Join(cause, fmt.Errorf("close admission after task discovery failure: %w", err)) } if r.queues != nil { diff --git a/internal/dispatcher/task_runtime_v3_test.go b/internal/dispatcher/task_runtime_v3_test.go index 885842c..0fd0dfb 100644 --- a/internal/dispatcher/task_runtime_v3_test.go +++ b/internal/dispatcher/task_runtime_v3_test.go @@ -16,31 +16,31 @@ import ( func TestLocalV01RuntimeDiscoveryFailureClosesAdmissionAndStopsTaskConsumers(t *testing.T) { var phase atomic.Int32 + var requested []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - after := r.URL.Query().Get("after") + requested = append(requested, r.URL.RawQuery) if r.URL.Path != "/internal/v1/dispatcher/tasks" { t.Errorf("unexpected path: %s", r.URL.Path) w.WriteHeader(http.StatusNotFound) return } - switch phase.Load() { - case 0: - if after == "0" { - fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"running","task_revision":2}],"next_cursor":"1"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) - return - } - case 1: + if phase.Load() == 1 { w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.3-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`)) + _, _ = w.Write([]byte(`{"schema_version":"task-discovery.v0.4-proposal","resource":"error","error":{"code":"service_unavailable","message":"mock outage"}}`)) return } - if after != "1" { - t.Errorf("unexpected event cursor: %q", after) + switch r.URL.RawQuery { + case "mode=snapshot": + fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":"22222222-2222-4222-8222-222222222222","watermark":"42","tasks":[{"task_id":%q,"tenant_id":%q,"tenant_key":%q,"status":"running","task_revision":2}],"next_page_token":"page-2"}`, localTestDispatcherID, localTestTaskID, localTestTenantID, localTestTenantKey) + case "page_token=page-2&snapshot_id=22222222-2222-4222-8222-222222222222": + fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"snapshot","dispatcher_id":%q,"snapshot_id":"22222222-2222-4222-8222-222222222222","watermark":"42","tasks":[],"next_page_token":null}`, localTestDispatcherID) + case "after=42": + fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.4-proposal","mode":"changes","dispatcher_id":%q,"tasks":[],"next_cursor":"42"}`, localTestDispatcherID) + default: + t.Errorf("unexpected discovery query: %q", r.URL.RawQuery) w.WriteHeader(http.StatusBadRequest) - return } - fmt.Fprintf(w, `{"schema_version":"task-discovery.v0.3-proposal","dispatcher_id":%q,"tasks":[],"next_cursor":"1"}`, localTestDispatcherID) })) defer server.Close() st, err := store.Open(filepath.Join(t.TempDir(), "dispatcher.db")) @@ -58,10 +58,16 @@ func TestLocalV01RuntimeDiscoveryFailureClosesAdmissionAndStopsTaskConsumers(t * } broker := &fakeQueueBroker{} queues := newTaskQueueController(d, broker, nil) - runtime := &LocalV01Runtime{dispatcher: d, client: client, queues: queues} + runtime := &LocalV01Runtime{dispatcher: d, client: client, queues: queues, controls: &localTaskControlProcessor{}} if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } + if len(requested) != 2 || requested[0] != "mode=snapshot" || requested[1] != "page_token=page-2&snapshot_id=22222222-2222-4222-8222-222222222222" { + t.Fatalf("snapshot was not fully paginated before admission: %v", requested) + } + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) + } assignment, err := st.LocalTaskAssignment(localTestDispatcherID, localTestTaskID) if err != nil { t.Fatal(err) @@ -74,7 +80,7 @@ func TestLocalV01RuntimeDiscoveryFailureClosesAdmissionAndStopsTaskConsumers(t * t.Fatal("outage accepted as fresh discovery") } var ready int - if err := st.DB().QueryRow(`SELECT ready FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 { + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 0 { t.Fatalf("discovery gate after outage=%d err=%v", ready, err) } select { @@ -86,8 +92,14 @@ func TestLocalV01RuntimeDiscoveryFailureClosesAdmissionAndStopsTaskConsumers(t * if err := runtime.refreshDiscoveryState(context.Background()); err != nil { t.Fatal(err) } - if err := st.DB().QueryRow(`SELECT ready FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { - t.Fatalf("discovery gate after fresh empty page=%d err=%v", ready, err) + if len(requested) < 5 || requested[2] != "after=42" || requested[3] != "mode=snapshot" { + t.Fatalf("outage did not restart a full snapshot: %v", requested) + } + if err := runtime.catchUpControlBeforeTasks(context.Background()); err != nil { + t.Fatal(err) + } + if err := st.DB().QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, localTestDispatcherID).Scan(&ready); err != nil || ready != 1 { + t.Fatalf("discovery gate after new snapshot=%d err=%v", ready, err) } } diff --git a/internal/mq/amqp_v3.go b/internal/mq/amqp_v3.go index 0b61949..bc92bd1 100644 --- a/internal/mq/amqp_v3.go +++ b/internal/mq/amqp_v3.go @@ -306,6 +306,71 @@ func (b *V3Broker) DrainPredeclared(ctx context.Context, queue string) (int, err } } +// DrainControlPredeclared processes the already queued controls before task admission. +// Unlike stopped task backlogs, control deliveries must pass through the handler +// before ACK; a transient failure is requeued and closes startup admission. +func (b *V3Broker) DrainControlPredeclared(ctx context.Context, queue string, handler MessageHandler) (int, error) { + if queue == "" || queue != b.controlQueue || handler == nil { + return 0, errors.New("configured SaaS-owned control queue and handler are required") + } + if err := ctx.Err(); err != nil { + return 0, err + } + b.mu.Lock() + conn := b.conn + closed := conn == nil || conn.IsClosed() + b.mu.Unlock() + if closed { + return 0, errors.New("rabbitmq v3 connection is closed") + } + channel, err := conn.Channel() + if err != nil { + return 0, fmt.Errorf("open control backlog channel: %w", err) + } + defer channel.Close() + if _, err := channel.QueueDeclarePassive(queue, true, false, false, false, nil); err != nil { + return 0, fmt.Errorf("required SaaS-owned control queue %s unavailable: %w", queue, err) + } + processed := 0 + for { + if err := ctx.Err(); err != nil { + return processed, err + } + delivery, ok, err := channel.Get(queue, false) + if err != nil { + return processed, fmt.Errorf("read SaaS-owned control backlog %s: %w", queue, err) + } + if !ok { + return processed, nil + } + if len(delivery.Body) == 0 || len(delivery.Body) > MaxV3MessageBytes || !json.Valid(delivery.Body) { + if err := delivery.Reject(false); err != nil { + return processed, fmt.Errorf("reject invalid control backlog message: %w", err) + } + slog.Warn("MQ control backlog message rejected", "dispatcher_id", b.dispatcherID, "delivery_tag", delivery.DeliveryTag, "reason", "invalid_size_or_json", "bytes", len(delivery.Body)) + processed++ + continue + } + if err := handler(ctx, delivery.RoutingKey, delivery.Body); err != nil { + if IsPermanent(err) { + if rejectErr := delivery.Reject(false); rejectErr != nil { + return processed, fmt.Errorf("control handler error %v; reject: %w", err, rejectErr) + } + processed++ + continue + } + if nackErr := delivery.Nack(false, true); nackErr != nil { + return processed, fmt.Errorf("control handler error %v; nack: %w", err, nackErr) + } + return processed, fmt.Errorf("control backlog handler failed, admission stays closed: %w", err) + } + if err := delivery.Ack(false); err != nil { + return processed, fmt.Errorf("ack processed control backlog: %w", err) + } + processed++ + } +} + func (b *V3Broker) ControlQueue() string { return b.controlQueue } func (b *V3Broker) consumeV3Deliveries(ctx context.Context, dispatcherID string, deliveries <-chan amqp.Delivery, handler MessageHandler) error { diff --git a/internal/mq/amqp_v3_integration_test.go b/internal/mq/amqp_v3_integration_test.go index d8e5af2..13b1951 100644 --- a/internal/mq/amqp_v3_integration_test.go +++ b/internal/mq/amqp_v3_integration_test.go @@ -4,6 +4,7 @@ package mq import ( "context" + "errors" "fmt" "os" "testing" @@ -117,6 +118,54 @@ func TestV3BrokerConsumesSaaSOwnedQueueAndPublishesConfirmedResult(t *testing.T) t.Fatal(err) } defer broker.Close() + if broker.ControlQueue() != controlQueue { + t.Fatalf("wrong SaaS-owned control queue: %q", broker.ControlQueue()) + } + oldStop := []byte(`{"action":"stop","issued_at":"2025-01-01T00:00:00Z"}`) + publishControl := func(body []byte) { + t.Helper() + if err := mockChannel.PublishWithContext(context.Background(), CommandsExchangeV3, controlRoute, true, false, amqp.Publishing{ + ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body, + }); err != nil { + t.Fatal(err) + } + } + publishControl(oldStop) + attempts := 0 + processed, err := broker.DrainControlPredeclared(context.Background(), controlQueue, func(_ context.Context, key string, body []byte) error { + attempts++ + if key != controlRoute || string(body) != string(oldStop) { + return fmt.Errorf("wrong historical control: key=%s body=%s", key, body) + } + return errors.New("mock SQLite write failure") + }) + if err == nil || processed != 0 || attempts != 1 { + t.Fatalf("transient control failure was ACKed or not surfaced: processed=%d attempts=%d err=%v", processed, attempts, err) + } + processed, err = broker.DrainControlPredeclared(context.Background(), controlQueue, func(_ context.Context, key string, body []byte) error { + attempts++ + if key != controlRoute || string(body) != string(oldStop) { + return fmt.Errorf("old stop not redelivered: key=%s body=%s", key, body) + } + return nil + }) + if err != nil || processed != 1 || attempts != 2 { + t.Fatalf("old stop did not recover under its original identity: processed=%d attempts=%d err=%v", processed, attempts, err) + } + publishControl([]byte(`{`)) + publishControl([]byte(`{"action":"invalid-control"}`)) + processed, err = broker.DrainControlPredeclared(context.Background(), controlQueue, func(_ context.Context, key string, body []byte) error { + if key != controlRoute || string(body) != `{"action":"invalid-control"}` { + return fmt.Errorf("invalid JSON delivered to control handler: key=%s body=%s", key, body) + } + return Permanent(errors.New("invalid control contract")) + }) + if err != nil || processed != 2 { + t.Fatalf("malformed and permanently invalid controls were not rejected: processed=%d err=%v", processed, err) + } + if state, err := mockChannel.QueueInspect(controlQueue); err != nil || state.Messages != 0 { + t.Fatalf("control backlog did not drain: messages=%d err=%v", state.Messages, err) + } missingQueue := "agent-call.d." + dispatcherID + ".task.missing.v3" missingCtx, missingCancel := context.WithTimeout(context.Background(), time.Second) if err := broker.ConsumePredeclared(missingCtx, missingQueue, func(context.Context, string, []byte) error { return nil }); err == nil { @@ -196,7 +245,7 @@ func TestV3BrokerConsumesSaaSOwnedQueueAndPublishesConfirmedResult(t *testing.T) releaseRetry := make(chan struct{}) barrierSeen := make(chan struct{}, 1) retryBarrierBody := []byte(`{"event_id":"sqlite-recovery-barrier"}`) - attempts := 0 + attempts = 0 retryConsumer, err := broker.StartPredeclaredConsumer(retryCtx, taskQueue, func(_ context.Context, _ string, body []byte) error { if string(body) == string(retryBarrierBody) { barrierSeen <- struct{}{} diff --git a/internal/store/local_discovery_helpers_test.go b/internal/store/local_discovery_helpers_test.go index cd2e084..5b39828 100644 --- a/internal/store/local_discovery_helpers_test.go +++ b/internal/store/local_discovery_helpers_test.go @@ -1,9 +1,7 @@ package store import ( - "encoding/json" "path/filepath" - "strconv" "testing" "time" ) @@ -40,53 +38,24 @@ func seedLocalCommandAssignments(t *testing.T, st *Store, records ...LocalComman } tasks = append(tasks, localDiscoveredTask(dispatcherID, record.TaskID, record.TenantID, record.TenantKey, revision, "running")) } - seedLocalEventTasks(t, st, dispatcherID, tasks...) + seedLocalSnapshotTasks(t, st, dispatcherID, tasks...) } func seedLocalDiscovery(t *testing.T, st *Store) { t.Helper() - seedLocalEventTasks(t, st, "d-1", + seedLocalSnapshotTasks(t, st, "d-1", localDiscoveredTask("d-1", "task-a", "tenant-a", "tenant-key-a", 1, "running"), localDiscoveredTask("d-1", "task-b", "tenant-a", "tenant-key-a", 1, "running"), ) } -func seedLocalEventTasks(t *testing.T, st *Store, dispatcherID string, tasks ...LocalDiscoveredTask) { +func seedLocalSnapshotTasks(t *testing.T, st *Store, dispatcherID string, tasks ...LocalDiscoveredTask) { t.Helper() - cursor, exists, err := st.LocalTaskDiscoveryCursor(dispatcherID) - if err != nil { - t.Fatal(err) - } - if !exists { - cursor = "0" - } - eventID, err := strconv.ParseUint(cursor, 10, 64) - if err != nil || eventID == ^uint64(0) { - t.Fatalf("invalid test event cursor %q: %v", cursor, err) - } - next := strconv.FormatUint(eventID+1, 10) - wire := make([]map[string]any, 0, len(tasks)) - for _, task := range tasks { - wire = append(wire, map[string]any{"task_id": task.TaskID, "tenant_id": task.TenantID, - "tenant_key": task.TenantKey, "status": task.Status, "task_revision": task.TaskRevision}) - } - body, err := json.Marshal(map[string]any{"schema_version": "task-discovery.v0.3-proposal", "dispatcher_id": dispatcherID, - "tasks": wire, "next_cursor": next}) - if err != nil { - t.Fatal(err) - } now := time.Now().UTC() - if err := st.ApplyLocalTaskDiscoveryPage(LocalTaskDiscoveryPage{DispatcherID: dispatcherID, FromCursor: cursor, - NextCursor: next, Tasks: tasks, Body: body, ObservedAt: now}); err != nil { + if err := st.ApplyLocalTaskSnapshot(dispatcherID, tasks, now); err != nil { t.Fatal(err) } - empty, err := json.Marshal(map[string]any{"schema_version": "task-discovery.v0.3-proposal", "dispatcher_id": dispatcherID, - "tasks": []any{}, "next_cursor": next}) - if err != nil { - t.Fatal(err) - } - if err := st.ApplyLocalTaskDiscoveryPage(LocalTaskDiscoveryPage{DispatcherID: dispatcherID, FromCursor: next, - NextCursor: next, Body: empty, ObservedAt: now}); err != nil { + if err := st.MarkLocalTaskDiscoveryReadyV04(dispatcherID, now); err != nil { t.Fatal(err) } } diff --git a/internal/store/local_origination.go b/internal/store/local_origination.go index 81bb002..afb0200 100644 --- a/internal/store/local_origination.go +++ b/internal/store/local_origination.go @@ -47,7 +47,7 @@ func (s *Store) LoadLocalOrigination(dispatcherID, executionID string) (LocalOri JOIN local_v01_execution_configs c ON c.execution_id=t.execution_id JOIN reservations r ON r.execution_id=t.execution_id JOIN local_v01_task_assignments a ON a.dispatcher_id=? AND a.task_id=t.task_id - JOIN local_v03_task_discovery_state d ON d.dispatcher_id=a.dispatcher_id + JOIN local_v04_task_discovery_state d ON d.dispatcher_id=a.dispatcher_id WHERE t.execution_id=?`, dispatcherID, executionID).Scan(&candidate.TenantID, &candidate.TenantKey, &candidate.TaskID, &candidate.CommandID, &candidate.Callee, &taskStatus, &candidate.CommandBody, &inboxHash, &inboxStatus, &snapshotBody, &candidate.SnapshotSHA256, &reservationState, &admissionState, &removed, &ready) if errors.Is(err, sql.ErrNoRows) { @@ -104,7 +104,7 @@ func (s *Store) ClaimLocalOrigination(dispatcherID, executionID, agentID string, SELECT t.execution_id,?,'issued','',? FROM tasks t WHERE t.execution_id=? AND t.status='reserved' AND EXISTS (SELECT 1 FROM reservations r WHERE r.execution_id=t.execution_id AND r.state='held') AND EXISTS (SELECT 1 FROM local_v01_task_assignments a WHERE a.dispatcher_id=? AND a.task_id=t.task_id AND a.tenant_id=t.tenant_id AND a.tenant_key=t.tenant_key AND a.removed=0 AND a.admission_state='running') - AND EXISTS (SELECT 1 FROM local_v03_task_discovery_state d WHERE d.dispatcher_id=? AND d.ready=1) + AND EXISTS (SELECT 1 FROM local_v04_task_discovery_state d WHERE d.dispatcher_id=? AND d.ready=1) AND NOT EXISTS (SELECT 1 FROM controls c WHERE c.execution_id=t.execution_id AND c.action IN ('pause','drain','stop','hangup')) AND NOT EXISTS (SELECT 1 FROM local_v02_origination_decisions old WHERE old.execution_id=t.execution_id)`, dispatcherID, decidedAt.UTC().Format(time.RFC3339Nano), executionID, dispatcherID, dispatcherID) diff --git a/internal/store/local_v01_discovery.go b/internal/store/local_v01_discovery.go index 0c8432e..9028c23 100644 --- a/internal/store/local_v01_discovery.go +++ b/internal/store/local_v01_discovery.go @@ -229,7 +229,7 @@ func scanLocalTaskAssignment(row interface{ Scan(...any) error }) (LocalTaskAssi func requireLocalTaskRunningTx(tx *sql.Tx, dispatcherID, tenantID, tenantKey, taskID string) error { var ready int - if err := tx.QueryRow(`SELECT ready FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, dispatcherID).Scan(&ready); err != nil { + if err := tx.QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, dispatcherID).Scan(&ready); err != nil { if errors.Is(err, sql.ErrNoRows) { return ErrLocalDiscoveryUnavailable } diff --git a/internal/store/local_v03_admission_test.go b/internal/store/local_v03_admission_test.go index c667826..77edd25 100644 --- a/internal/store/local_v03_admission_test.go +++ b/internal/store/local_v03_admission_test.go @@ -6,7 +6,7 @@ import ( "time" ) -func TestEventDiscoveryAdmissionRequiresRunningTask(t *testing.T) { +func TestSnapshotAdmissionRequiresRunningTask(t *testing.T) { for _, tc := range []struct { name string discovered bool @@ -21,7 +21,16 @@ func TestEventDiscoveryAdmissionRequiresRunningTask(t *testing.T) { t.Run(tc.name, func(t *testing.T) { st := openLocalDiscoveryTestStore(t) if tc.discovered { - seedLocalEventTasks(t, st, "d-1", eventTask("task-a", tc.status, 1)) + initialStatus := tc.status + if initialStatus == "removed" { + initialStatus = "running" + } + seedLocalSnapshotTasks(t, st, "d-1", eventTask("task-a", initialStatus, 1)) + if tc.status == "removed" { + if err := st.ApplyLocalTaskChanges("d-1", []LocalDiscoveredTask{eventTask("task-a", "removed", 2)}, time.Now().UTC()); err != nil { + t.Fatal(err) + } + } } now := time.Date(2026, 9, 21, 2, 0, 0, 0, time.UTC) bundle := localConfigBundle("d-1", "tenant-a", "tenant-key-a", "task-a", 1, 1, 1, 1, 1, now) diff --git a/internal/store/local_v03_config_gate_test.go b/internal/store/local_v03_config_gate_test.go index d3d81d0..0ba2cca 100644 --- a/internal/store/local_v03_config_gate_test.go +++ b/internal/store/local_v03_config_gate_test.go @@ -3,17 +3,19 @@ package store import ( "errors" "testing" + "time" ) -func TestEventConfigReadRequiresCompleteDiscoveryAndTenantBinding(t *testing.T) { +func TestSnapshotConfigReadRequiresControlCatchupAndTenantBinding(t *testing.T) { st := openLocalDiscoveryTestStore(t) - if err := st.ApplyLocalTaskDiscoveryPage(eventPage("0", "1", eventTask("task-a", "running", 1))); err != nil { + now := time.Now().UTC() + if err := st.ApplyLocalTaskSnapshot("d-1", []LocalDiscoveredTask{eventTask("task-a", "running", 1)}, now); err != nil { t.Fatal(err) } if _, err := st.LocalDiscoveredTaskForConfig("d-1", "task-a", "tenant-a"); !errors.Is(err, ErrLocalDiscoveryUnavailable) { - t.Fatalf("partial discovery admitted config read: %v", err) + t.Fatalf("snapshot without control catch-up admitted config read: %v", err) } - if err := st.ApplyLocalTaskDiscoveryPage(eventPage("1", "1")); err != nil { + if err := st.MarkLocalTaskDiscoveryReadyV04("d-1", now); err != nil { t.Fatal(err) } assignment, err := st.LocalDiscoveredTaskForConfig("d-1", "task-a", "tenant-a") @@ -23,10 +25,7 @@ func TestEventConfigReadRequiresCompleteDiscoveryAndTenantBinding(t *testing.T) if _, err := st.LocalDiscoveredTaskForConfig("d-1", "task-a", "another-tenant"); !errors.Is(err, ErrTenantBindingConflict) { t.Fatalf("cross-tenant config read: %v", err) } - if err := st.ApplyLocalTaskDiscoveryPage(eventPage("1", "2", eventTask("task-a", "removed", 1))); err != nil { - t.Fatal(err) - } - if err := st.ApplyLocalTaskDiscoveryPage(eventPage("2", "2")); err != nil { + if err := st.ApplyLocalTaskChanges("d-1", []LocalDiscoveredTask{eventTask("task-a", "removed", 2)}, now.Add(time.Second)); err != nil { t.Fatal(err) } if _, err := st.LocalDiscoveredTaskForConfig("d-1", "task-a", "tenant-a"); !errors.Is(err, ErrLocalTaskRemoved) { diff --git a/internal/store/local_v03_discovery.go b/internal/store/local_v03_discovery.go index 7651189..44e5928 100644 --- a/internal/store/local_v03_discovery.go +++ b/internal/store/local_v03_discovery.go @@ -130,7 +130,7 @@ func (s *Store) LocalDiscoveredTaskForConfig(dispatcherID, taskID, tenantID stri defer s.mu.Unlock() var ready int var observedAt string - err := s.db.QueryRow(`SELECT ready,updated_at FROM local_v03_task_discovery_state WHERE dispatcher_id=?`, dispatcherID).Scan(&ready, &observedAt) + err := s.db.QueryRow(`SELECT ready,updated_at FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, dispatcherID).Scan(&ready, &observedAt) if errors.Is(err, sql.ErrNoRows) || (err == nil && ready != 1) { return LocalTaskAssignment{}, ErrLocalDiscoveryUnavailable } diff --git a/internal/store/local_v04_cleanup.go b/internal/store/local_v04_cleanup.go new file mode 100644 index 0000000..82172c6 --- /dev/null +++ b/internal/store/local_v04_cleanup.go @@ -0,0 +1,90 @@ +package store + +import ( + "database/sql" + "errors" + "fmt" + "log" +) + +// LocalTaskConfigEviction reports a narrowly scoped cache eviction. BlockedBy +// is a durable reason to retry later, not permission to discard recovery data. +type LocalTaskConfigEviction struct { + Deleted int64 + BlockedBy string +} + +// EvictTerminalTaskConfig explicitly removes only a terminal task's derived +// HTTP task-config copy. It never removes command identities, assignments, +// execution snapshots, reservations, uploads, terminal facts or outbox rows: +// publisher confirm is not SaaS application receipt. There is no automatic +// migration or sweep of existing data. +func (s *Store) EvictTerminalTaskConfig(dispatcherID, taskID string) (LocalTaskConfigEviction, error) { + if dispatcherID == "" || taskID == "" { + return LocalTaskConfigEviction{}, ErrLocalTaskUnassigned + } + s.mu.Lock() + defer s.mu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return LocalTaskConfigEviction{}, err + } + defer tx.Rollback() + + var tenantID, status, admission string + var removed int + err = tx.QueryRow(`SELECT tenant_id,saas_status,admission_state,removed FROM local_v01_task_assignments + WHERE dispatcher_id=? AND task_id=?`, dispatcherID, taskID).Scan(&tenantID, &status, &admission, &removed) + if errors.Is(err, sql.ErrNoRows) { + return LocalTaskConfigEviction{}, ErrLocalTaskUnassigned + } + if err != nil { + return LocalTaskConfigEviction{}, fmt.Errorf("load task before cache eviction: %w", err) + } + terminal := func(value string) bool { return value == "stopped" || value == "finished" || value == "removed" } + if !terminal(status) || !terminal(admission) || (removed != 0 && status != "removed") { + log.Printf("task config eviction blocked dispatcher_id=%s task_id=%s reason=task_active", dispatcherID, taskID) + return LocalTaskConfigEviction{BlockedBy: "task_active"}, nil + } + checks := []struct { + reason string + query string + args []any + }{ + {"execution_unconfirmed", `SELECT EXISTS(SELECT 1 FROM tasks t + LEFT JOIN reservations r ON r.execution_id=t.execution_id + LEFT JOIN local_v01_call_terminals c ON c.execution_id=t.execution_id + LEFT JOIN outbox o ON o.event_id=c.result_event_id + WHERE t.tenant_id=? AND t.task_id=? AND + (t.status!='finished' OR r.reservation_id IS NULL OR r.state!='released' OR + c.execution_id IS NULL OR c.result_event_id IS NULL OR o.status IS NULL OR o.status!='published'))`, []any{tenantID, taskID}}, + {"quota_occupied", `SELECT EXISTS(SELECT 1 FROM quotas WHERE scope IN (?,?) AND (reserved_value!=0 OR unknown_value!=0))`, + []any{LocalTaskQuotaScope(dispatcherID, tenantID, taskID), LocalTenantQuotaScope(dispatcherID, tenantID)}}, + {"upload_incomplete", `SELECT EXISTS(SELECT 1 FROM uploads WHERE state='granted')`, nil}, + {"outbox_pending", `SELECT EXISTS(SELECT 1 FROM outbox WHERE status!='published')`, nil}, + } + for _, check := range checks { + var blocked int + if err := tx.QueryRow(check.query, check.args...).Scan(&blocked); err != nil { + return LocalTaskConfigEviction{}, fmt.Errorf("check %s before task cache eviction: %w", check.reason, err) + } + if blocked != 0 { + log.Printf("task config eviction blocked dispatcher_id=%s task_id=%s reason=%s", dispatcherID, taskID, check.reason) + return LocalTaskConfigEviction{BlockedBy: check.reason}, nil + } + } + result, err := tx.Exec(`DELETE FROM local_v01_config_snapshots WHERE dispatcher_id=? AND resource='task' AND tenant_id=? AND task_id=?`, + dispatcherID, tenantID, taskID) + if err != nil { + return LocalTaskConfigEviction{}, fmt.Errorf("evict terminal task config: %w", err) + } + deleted, err := result.RowsAffected() + if err != nil || deleted > 1 { + return LocalTaskConfigEviction{}, fmt.Errorf("terminal task config rows=%d: %w", deleted, errors.Join(err, ErrCommandConflict)) + } + if err := tx.Commit(); err != nil { + return LocalTaskConfigEviction{}, fmt.Errorf("commit terminal task config eviction: %w", err) + } + log.Printf("task config eviction completed dispatcher_id=%s task_id=%s deleted=%d", dispatcherID, taskID, deleted) + return LocalTaskConfigEviction{Deleted: deleted}, nil +} diff --git a/internal/store/local_v04_cleanup_test.go b/internal/store/local_v04_cleanup_test.go new file mode 100644 index 0000000..8a1fa8b --- /dev/null +++ b/internal/store/local_v04_cleanup_test.go @@ -0,0 +1,128 @@ +package store + +import ( + "database/sql" + "errors" + "testing" + "time" +) + +func cleanupTestStore(t *testing.T) *Store { + t.Helper() + st := openLocalDiscoveryTestStore(t) + seedLocalSnapshotTasks(t, st, "d-1", eventTask("task-a", "running", 1)) + bundle := localConfigBundle("d-1", "tenant-a", "tenant-key-a", "task-a", 1, 1, 1, 3, 3, time.Now().UTC()) + if err := st.SaveLocalConfigBundle(bundle); err != nil { + t.Fatal(err) + } + return st +} + +func stopCleanupTestTask(t *testing.T, st *Store) { + t.Helper() + if _, err := st.db.Exec(`UPDATE local_v01_task_assignments SET saas_status='stopped',admission_state='stopped' WHERE dispatcher_id='d-1' AND task_id='task-a'`); err != nil { + t.Fatal(err) + } +} + +func expectCleanupTaskCache(t *testing.T, st *Store, present bool) { + t.Helper() + _, err := st.LocalConfigSnapshot("d-1", "task", "tenant-a", "task-a") + if present && err != nil || !present && !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("task config present=%v error=%v", present, err) + } +} + +func TestTerminalTaskCacheEvictionPreservesRecoveryAndDedupRecords(t *testing.T) { + st := cleanupTestStore(t) + command := LocalCommandRecord{ + DispatcherID: "d-1", TaskID: "task-a", TenantID: "tenant-a", TenantKey: "tenant-key-a", + CommandID: "original-command", CommandType: "call.execute", Body: []byte(`{"task_id":"task-a"}`), + ReceiptID: "result-event", Exchange: "events", RoutingKey: "d-1", ReceiptBody: []byte(`{}`), Status: "rejected", + } + if duplicate, err := st.PersistLocalCommand(command); err != nil || duplicate { + t.Fatalf("seed rejected command: duplicate=%v err=%v", duplicate, err) + } + if _, err := st.db.Exec(`UPDATE outbox SET status='published',published_at='2026-09-22T10:01:00Z' WHERE event_id='result-event'`); err != nil { + t.Fatal(err) + } + result, err := st.EvictTerminalTaskConfig("d-1", "task-a") + if err != nil || result.Deleted != 0 || result.BlockedBy != "task_active" { + t.Fatalf("active task cleanup=%+v err=%v", result, err) + } + stopCleanupTestTask(t, st) + result, err = st.EvictTerminalTaskConfig("d-1", "task-a") + if err != nil || result.Deleted != 1 || result.BlockedBy != "" { + t.Fatalf("safe terminal config cleanup=%+v err=%v", result, err) + } + expectCleanupTaskCache(t, st, false) + for _, scope := range []struct{ resource, tenantID string }{{"sip", ""}, {"tenant_quota", "tenant-a"}} { + if _, err := st.LocalConfigSnapshot("d-1", scope.resource, scope.tenantID, ""); err != nil { + t.Fatalf("cleaned shared %s: %v", scope.resource, err) + } + } + for table := range map[string]bool{"inbox": true, "outbox": true, "local_v01_task_assignments": true, "local_v01_tenant_bindings": true} { + var count int + if err := st.db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&count); err != nil || count != 1 { + t.Fatalf("lost durable %s: count=%d err=%v", table, count, err) + } + } + result, err = st.EvictTerminalTaskConfig("d-1", "task-a") + if err != nil || result.Deleted != 0 || result.BlockedBy != "" { + t.Fatalf("repeat cleanup should be idempotent: %+v err=%v", result, err) + } + if duplicate, err := st.PersistLocalCommand(command); err != nil || !duplicate { + t.Fatalf("old command after config eviction lost original identity: duplicate=%v err=%v", duplicate, err) + } + var status string + if err := st.db.QueryRow(`SELECT status FROM outbox WHERE event_id='result-event'`).Scan(&status); err != nil || status != "pending" { + t.Fatalf("old command receipt not redelivered with original event id: %q err=%v", status, err) + } + var tasks int + if err := st.db.QueryRow(`SELECT count(*) FROM tasks`).Scan(&tasks); err != nil || tasks != 0 { + t.Fatalf("old command re-reserved after cleanup: tasks=%d err=%v", tasks, err) + } + expectCleanupTaskCache(t, st, false) +} + +func TestTerminalTaskCacheEvictionFailClosedWhileWorkOrDeliveryIsUnknown(t *testing.T) { + cases := []struct { + name string + seed string + params []any + want string + }{ + {"pending outbox", `INSERT INTO outbox(event_id,tenant_key,exchange,routing_key,body,status,created_at) VALUES('event','tenant-key-a','events','call.result',x'7b7d','pending','2026-09-22T10:00:00Z')`, nil, "outbox_pending"}, + {"inflight upload", `INSERT INTO uploads(upload_id,binding,asset,grant,object_key,state,created_at) VALUES('upload-1',x'7b7d',x'7b7d',x'7b7d','path','granted','2026-09-22T10:00:00Z')`, nil, "upload_incomplete"}, + {"unknown quota", `UPDATE quotas SET unknown_value=1 WHERE scope=?`, []any{LocalTaskQuotaScope("d-1", "tenant-a", "task-a")}, "quota_occupied"}, + {"unknown call", `INSERT INTO tasks(execution_id,tenant_key,tenant_id,task_id,task_item_id,task_revision,trace_id,callee,route_policy_id,caller_profile_id,agent_version_id,variables,ring_timeout_ms,max_call_duration_ms,status,created_at,updated_at) VALUES('exe-1','tenant-key-a','tenant-a','task-a','item-1',1,'trace','15003164745','route','caller','agent',x'7b7d',1000,2000,'unknown','2026-09-22T10:00:00Z','2026-09-22T10:00:00Z')`, nil, "execution_unconfirmed"}, + {"finished call without final result", `INSERT INTO tasks(execution_id,tenant_key,tenant_id,task_id,task_item_id,task_revision,trace_id,callee,route_policy_id,caller_profile_id,agent_version_id,variables,ring_timeout_ms,max_call_duration_ms,status,created_at,updated_at) VALUES('exe-2','tenant-key-a','tenant-a','task-a','item-2',1,'trace','15003164745','route','caller','agent',x'7b7d',1000,2000,'finished','2026-09-22T10:00:00Z','2026-09-22T10:00:00Z')`, nil, "execution_unconfirmed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + st := cleanupTestStore(t) + stopCleanupTestTask(t, st) + if _, err := st.db.Exec(tc.seed, tc.params...); err != nil { + t.Fatal(err) + } + result, err := st.EvictTerminalTaskConfig("d-1", "task-a") + if err != nil || result.Deleted != 0 || result.BlockedBy != tc.want { + t.Fatalf("unsafe cleanup=%+v err=%v want blocker=%s", result, err, tc.want) + } + expectCleanupTaskCache(t, st, true) + }) + } +} + +func TestTerminalTaskCacheEvictionDatabaseFailureRetainsSnapshot(t *testing.T) { + st := cleanupTestStore(t) + stopCleanupTestTask(t, st) + if _, err := st.db.Exec(`CREATE TRIGGER cleanup_failure BEFORE DELETE ON local_v01_config_snapshots + WHEN OLD.resource='task' BEGIN SELECT RAISE(ABORT, 'test database write failure'); END`); err != nil { + t.Fatal(err) + } + if _, err := st.EvictTerminalTaskConfig("d-1", "task-a"); err == nil { + t.Fatal("SQLite delete failure swallowed") + } + expectCleanupTaskCache(t, st, true) +} diff --git a/internal/store/local_v04_discovery.go b/internal/store/local_v04_discovery.go new file mode 100644 index 0000000..051efb6 --- /dev/null +++ b/internal/store/local_v04_discovery.go @@ -0,0 +1,177 @@ +package store + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// CloseLocalTaskDiscoveryAdmissionV04 fails closed without persisting a live cursor. +func (s *Store) CloseLocalTaskDiscoveryAdmissionV04(dispatcherID string) error { + s.mu.Lock() + defer s.mu.Unlock() + _, err := s.db.Exec(`INSERT INTO local_v04_task_discovery_state(dispatcher_id,ready,updated_at) VALUES(?,0,?) + ON CONFLICT(dispatcher_id) DO UPDATE SET ready=0,updated_at=excluded.updated_at`, dispatcherID, time.Now().UTC().Format(time.RFC3339Nano)) + return err +} + +// LocalTaskDiscoveryReadyV04 reports whether a complete snapshot and control +// backlog have been applied. An absent row is not ready on a fresh host. +func (s *Store) LocalTaskDiscoveryReadyV04(dispatcherID string) (bool, error) { + if dispatcherID == "" { + return false, ErrLocalDiscoveryUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + var ready int + if err := s.db.QueryRow(`SELECT ready FROM local_v04_task_discovery_state WHERE dispatcher_id=?`, dispatcherID).Scan(&ready); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return false, err + } + return ready == 1, nil +} + +// MarkLocalTaskDiscoveryReadyV04 opens admission after the live change stream is caught up. +func (s *Store) MarkLocalTaskDiscoveryReadyV04(dispatcherID string, observedAt time.Time) error { + if dispatcherID == "" || observedAt.IsZero() { + return ErrLocalDiscoveryUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + result, err := s.db.Exec(`UPDATE local_v04_task_discovery_state SET ready=1,updated_at=? WHERE dispatcher_id=?`, observedAt.UTC().Format(time.RFC3339Nano), dispatcherID) + if err != nil { + return err + } + if n, err := result.RowsAffected(); err != nil || n != 1 { + return ErrLocalDiscoveryUnavailable + } + return nil +} + +// ApplyLocalTaskSnapshot replaces membership only after every page of one snapshot +// has been verified by the caller. Control barriers and execution records survive. +func (s *Store) ApplyLocalTaskSnapshot(dispatcherID string, tasks []LocalDiscoveredTask, observedAt time.Time) error { + if dispatcherID == "" || observedAt.IsZero() || len(tasks) > 256 { + return ErrLocalDiscoveryUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + if err := validateV04AssignmentSet(dispatcherID, tasks); err != nil { + return err + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + now := observedAt.UTC().Format(time.RFC3339Nano) + seen := make(map[string]bool, len(tasks)) + for _, task := range tasks { + if task.Status == "removed" { + return ErrLocalDiscoveryUnavailable + } + seen[task.TaskID] = true + if err := applyLocalDiscoveredTask(tx, dispatcherID, task, now); err != nil { + return err + } + } + rows, err := tx.Query(`SELECT task_id FROM local_v01_task_assignments WHERE dispatcher_id=? AND removed=0`, dispatcherID) + if err != nil { + return err + } + var absent []string + for rows.Next() { + var taskID string + if err := rows.Scan(&taskID); err != nil { + rows.Close() + return err + } + if !seen[taskID] { + absent = append(absent, taskID) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + for _, taskID := range absent { + if _, err := tx.Exec(`UPDATE local_v01_task_assignments SET saas_status='removed',admission_state='removed',removed=1,updated_at=? WHERE dispatcher_id=? AND task_id=?`, now, dispatcherID, taskID); err != nil { + return err + } + } + if _, err := tx.Exec(`INSERT INTO local_v04_task_discovery_state(dispatcher_id,ready,updated_at) VALUES(?,0,?) + ON CONFLICT(dispatcher_id) DO UPDATE SET ready=0,updated_at=excluded.updated_at`, dispatcherID, now); err != nil { + return err + } + return tx.Commit() +} + +// ApplyLocalTaskChanges updates task membership. An existing task's status is +// controlled by MQ; contradictory HTTP status fails admission instead of racing it. +func (s *Store) ApplyLocalTaskChanges(dispatcherID string, tasks []LocalDiscoveredTask, observedAt time.Time) error { + if dispatcherID == "" || observedAt.IsZero() || len(tasks) > 256 { + return ErrLocalDiscoveryUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + if err := validateV04AssignmentSet(dispatcherID, tasks); err != nil { + return err + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + for _, task := range tasks { + var existingStatus string + err := tx.QueryRow(`SELECT saas_status FROM local_v01_task_assignments WHERE dispatcher_id=? AND task_id=?`, dispatcherID, task.TaskID).Scan(&existingStatus) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if err == nil && task.Status != "removed" && task.Status != existingStatus { + if err := tx.Rollback(); err != nil { + return err + } + if _, err := s.db.Exec(`UPDATE local_v04_task_discovery_state SET ready=0 WHERE dispatcher_id=?`, dispatcherID); err != nil { + return err + } + return fmt.Errorf("task %s: HTTP status %s conflicts with MQ-applied %s: %w", task.TaskID, task.Status, existingStatus, ErrLocalDiscoveryUnavailable) + } + } + now := observedAt.UTC().Format(time.RFC3339Nano) + for _, task := range tasks { + if err := applyLocalDiscoveredTask(tx, dispatcherID, task, now); err != nil { + return err + } + } + if _, err := tx.Exec(`UPDATE local_v04_task_discovery_state SET updated_at=? WHERE dispatcher_id=?`, now, dispatcherID); err != nil { + return err + } + return tx.Commit() +} + +func validateV04AssignmentSet(dispatcherID string, tasks []LocalDiscoveredTask) error { + seen := make(map[string]bool, len(tasks)) + ids, keys := make(map[string]string), make(map[string]string) + for _, task := range tasks { + if err := validateLocalDiscoveredTask(dispatcherID, task); err != nil { + return err + } + if seen[task.TaskID] { + return fmt.Errorf("duplicate discovered task %s: %w", task.TaskID, ErrLocalDiscoveryUnavailable) + } + seen[task.TaskID] = true + if key, ok := ids[task.TenantID]; ok && key != task.TenantKey { + return ErrTenantBindingConflict + } + if id, ok := keys[task.TenantKey]; ok && id != task.TenantID { + return ErrTenantBindingConflict + } + ids[task.TenantID], keys[task.TenantKey] = task.TenantKey, task.TenantID + } + return nil +} diff --git a/internal/store/local_v04_discovery_test.go b/internal/store/local_v04_discovery_test.go new file mode 100644 index 0000000..783d744 --- /dev/null +++ b/internal/store/local_v04_discovery_test.go @@ -0,0 +1,96 @@ +package store + +import ( + "errors" + "testing" + "time" +) + +func TestV04CompleteSnapshotReplacesMembershipWithoutReopeningPause(t *testing.T) { + st := openLocalDiscoveryTestStore(t) + now := time.Now().UTC() + if err := st.ApplyLocalTaskSnapshot("d-1", []LocalDiscoveredTask{eventTask("a", "running", 1), eventTask("b", "running", 1)}, now); err != nil { + t.Fatal(err) + } + if _, err := st.LocalDiscoveredTaskForConfig("d-1", "a", "tenant-a"); !errors.Is(err, ErrLocalDiscoveryUnavailable) { + t.Fatalf("snapshot allowed admission before control backlog: %v", err) + } + if err := st.MarkLocalTaskDiscoveryReadyV04("d-1", now); err != nil { + t.Fatal(err) + } + if _, err := st.SetLocalTaskAdmissionBarrier("d-1", "a", "tenant-a", "tenant-key-a", "paused"); err != nil { + t.Fatal(err) + } + if err := st.CloseLocalTaskDiscoveryAdmissionV04("d-1"); err != nil { + t.Fatal(err) + } + if err := st.ApplyLocalTaskSnapshot("d-1", []LocalDiscoveredTask{eventTask("a", "paused", 2)}, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + a, err := st.LocalTaskAssignment("d-1", "a") + if err != nil || a.AdmissionState != "paused" || a.TaskRevision != 2 { + t.Fatalf("snapshot reopened paused task: %+v %v", a, err) + } + b, err := st.LocalTaskAssignment("d-1", "b") + if err != nil || !b.Removed || b.AdmissionState != "removed" { + t.Fatalf("snapshot failed to revoke missing task: %+v %v", b, err) + } + if err := st.MarkLocalTaskDiscoveryReadyV04("d-1", now.Add(time.Second)); err != nil { + t.Fatal(err) + } + if _, err := st.LocalDiscoveredTaskForConfig("d-1", "a", "tenant-a"); err != nil { + t.Fatalf("completed snapshot+control catch-up not ready: %v", err) + } +} + +func TestV04InvalidSnapshotDoesNotPartiallyCommitOrOpenAdmission(t *testing.T) { + st := openLocalDiscoveryTestStore(t) + now := time.Now().UTC() + if err := st.ApplyLocalTaskSnapshot("d-1", []LocalDiscoveredTask{eventTask("a", "running", 1)}, now); err != nil { + t.Fatal(err) + } + if err := st.CloseLocalTaskDiscoveryAdmissionV04("d-1"); err != nil { + t.Fatal(err) + } + bad := eventTask("b", "running", 1) + bad.TenantKey = "changed-binding" + if err := st.ApplyLocalTaskSnapshot("d-1", []LocalDiscoveredTask{eventTask("a", "running", 2), bad}, now.Add(time.Second)); err == nil { + t.Fatal("conflicting tenant binding accepted") + } + if _, err := st.LocalDiscoveredTaskForConfig("d-1", "a", "tenant-a"); !errors.Is(err, ErrLocalDiscoveryUnavailable) { + t.Fatalf("failed snapshot opened admission: %v", err) + } + if _, err := st.LocalTaskAssignment("d-1", "b"); err == nil { + t.Fatal("failed snapshot partially inserted task b") + } + a, err := st.LocalTaskAssignment("d-1", "a") + if err != nil || a.TaskRevision != 1 { + t.Fatalf("failed snapshot changed a: %+v %v", a, err) + } +} + +func TestV04LiveChangesOnlyManageMembershipNotExistingStatus(t *testing.T) { + st := openLocalDiscoveryTestStore(t) + now := time.Now().UTC() + if err := st.ApplyLocalTaskSnapshot("d-1", []LocalDiscoveredTask{eventTask("a", "running", 1), eventTask("b", "running", 1)}, now); err != nil { + t.Fatal(err) + } + if _, err := st.SetLocalTaskAdmissionBarrier("d-1", "a", "tenant-a", "tenant-key-a", "paused"); err != nil { + t.Fatal(err) + } + if err := st.ApplyLocalTaskChanges("d-1", []LocalDiscoveredTask{eventTask("a", "running", 2), eventTask("b", "removed", 2), eventTask("c", "running", 1)}, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + for taskID, want := range map[string]string{"a": "paused", "b": "removed", "c": "running"} { + row, err := st.LocalTaskAssignment("d-1", taskID) + if err != nil || row.AdmissionState != want { + t.Fatalf("task %s state=%+v err=%v", taskID, row, err) + } + } + if err := st.ApplyLocalTaskChanges("d-1", []LocalDiscoveredTask{eventTask("a", "stopped", 3)}, now.Add(2*time.Second)); !errors.Is(err, ErrLocalDiscoveryUnavailable) { + t.Fatalf("incremental status switched without MQ: %v", err) + } + if _, err := st.LocalDiscoveredTaskForConfig("d-1", "a", "tenant-a"); !errors.Is(err, ErrLocalDiscoveryUnavailable) { + t.Fatalf("status conflict should close admission: %v", err) + } +} diff --git a/internal/store/local_v04_lookup_test.go b/internal/store/local_v04_lookup_test.go new file mode 100644 index 0000000..86c0e67 --- /dev/null +++ b/internal/store/local_v04_lookup_test.go @@ -0,0 +1,56 @@ +package store + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestSingleTaskControlLookupUsesIndexedIdentityWithGrowingHistory(t *testing.T) { + st := openLocalDiscoveryTestStore(t) + for batch := 0; batch < 3; batch++ { + tasks := make([]LocalDiscoveredTask, 0, 200) + for i := batch * 200; i < (batch+1)*200; i++ { + tasks = append(tasks, eventTask(fmt.Sprintf("hist-%d", i), "running", 1)) + } + if err := st.ApplyLocalTaskSnapshot("d-1", tasks, time.Now().UTC()); err != nil { + t.Fatal(err) + } + } + if err := st.MarkLocalTaskDiscoveryReadyV04("d-1", time.Now().UTC()); err != nil { + t.Fatal(err) + } + row, err := st.LocalTaskAssignment("d-1", "hist-599") + if err != nil || row.Removed || row.TaskID != "hist-599" { + t.Fatalf("active assignment lookup: %+v err=%v", row, err) + } + row, err = st.LocalTaskAssignment("d-1", "hist-0") + if err != nil || !row.Removed { + t.Fatalf("historical assignment lost: %+v err=%v", row, err) + } + plan, err := st.db.Query(`EXPLAIN QUERY PLAN SELECT tenant_id,tenant_key,saas_status,admission_state,task_revision,removed + FROM local_v01_task_assignments WHERE dispatcher_id=? AND task_id=?`, "d-1", "hist-599") + if err != nil { + t.Fatal(err) + } + defer plan.Close() + var indexed bool + for plan.Next() { + var id, parent, unused int + var detail string + if err := plan.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "SCAN local_v01_task_assignments") { + t.Fatalf("single task control query scans historical assignments: %s", detail) + } + indexed = indexed || strings.Contains(detail, "SEARCH local_v01_task_assignments USING INDEX") + } + if err := plan.Err(); err != nil { + t.Fatal(err) + } + if !indexed { + t.Fatal("single task query did not use dispatcher/task identity index") + } +} diff --git a/internal/store/migrations/019_local_v04_discovery.sql b/internal/store/migrations/019_local_v04_discovery.sql new file mode 100644 index 0000000..0d59ba4 --- /dev/null +++ b/internal/store/migrations/019_local_v04_discovery.sql @@ -0,0 +1,6 @@ +-- v0.4 stores only the current process's discovery readiness; the live event cursor remains in memory. +CREATE TABLE IF NOT EXISTS local_v04_task_discovery_state ( + dispatcher_id TEXT PRIMARY KEY, + ready INTEGER NOT NULL CHECK(ready IN (0, 1)), + updated_at TEXT NOT NULL +); diff --git a/scripts/generate-local-contract-bundle.sh b/scripts/generate-local-contract-bundle.sh index 4eeeab6..2d45986 100644 --- a/scripts/generate-local-contract-bundle.sh +++ b/scripts/generate-local-contract-bundle.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "$0")/.." -mkdir -p contracts/local/v0.1 contracts/local/v0.3 +mkdir -p contracts/local/v0.1 contracts/local/v0.3 contracts/local/v0.4 rm -f contracts/local/v0.1/task-discovery-v0.1-proposal.schema.json cp docs/contracts/config-read-v0.1.schema.json docs/contracts/command-next-v0.1-proposal.schema.json docs/contracts/call-result-v0.1-proposal.schema.json docs/contracts/local-mock-recording-failure-v0.1.schema.json contracts/local/v0.1/ cp docs/contracts/task-discovery-v0.3-proposal.schema.json contracts/local/v0.3/ +cp docs/contracts/task-discovery-v0.4-proposal.schema.json docs/contracts/task-control-v0.4-proposal.schema.json docs/contracts/call-execute-v0.4-proposal.schema.json contracts/local/v0.4/ diff --git a/scripts/validate-local-contracts.py b/scripts/validate-local-contracts.py index 369aa61..81af9ba 100644 --- a/scripts/validate-local-contracts.py +++ b/scripts/validate-local-contracts.py @@ -384,6 +384,64 @@ for entry in [event_source, *event_artifacts]: raise SystemExit(f"v0.3 manifest SHA-256 mismatch: {relative}") print(f"Task discovery v0.3 proposal: positive={event_positive + len(seen_event_errors)}, negative={event_negative}, manifest files={1 + len(event_artifacts)}") +# v0.4 is a new project-local contract, not a mutation of v0.3 or upstream v1. +v4_doc = ROOT / "docs/thirds/v0.4.md" +v4_schema_paths = { + name: ROOT / f"docs/contracts/{name}-v0.4-proposal.schema.json" + for name in ("task-discovery", "task-control", "call-execute") +} +v4_registry = registry.with_resource(event_schema["$id"], Resource.from_contents(event_schema)) +v4_examples = {name: sorted(EXAMPLES.glob(f"{name}-*-v0.4.json")) for name in v4_schema_paths} +v4_positive = v4_negative = 0 +for name, schema_path in v4_schema_paths.items(): + schema = load_json(schema_path) + Draft202012Validator.check_schema(schema) + if schema_versions(schema) != {f"{name}.v0.4-proposal"}: + raise SystemExit(f"unexpected v0.4 schema version: {schema_path}") + embedded = ROOT / "contracts/local/v0.4" / schema_path.name + if embedded.read_bytes() != schema_path.read_bytes(): + raise SystemExit(f"embedded v0.4 schema differs from project source: {schema_path.name}") + validator = Draft202012Validator(schema, registry=v4_registry, format_checker=FormatChecker()) + positive = negative = 0 + for path in v4_examples[name]: + sample = load_json(path) + if "invalid" in path.name: + try: + validator.validate(sample) + except ValidationError: + negative += 1 + else: + raise SystemExit(f"invalid v0.4 fixture passed: {path.relative_to(ROOT)}") + else: + validator.validate(sample) + positive += 1 + if positive < 3 or negative < 1: + raise SystemExit(f"missing v0.4 positive/negative fixtures for {schema_path.name}") + v4_positive += positive + v4_negative += negative +v4_manifest = load_json(ROOT / "docs/contracts/local-contract-manifest-v0.4.json") +if v4_manifest.get("manifest_version") != "local-contract-manifest.v0.4" or v4_manifest.get("hash_algorithm") != "SHA-256": + raise SystemExit("invalid v0.4 manifest version or hash algorithm") +v4_source = v4_manifest.get("source", {}) +v4_artifacts = v4_manifest.get("artifacts", []) +required_v4 = { + *(p.relative_to(ROOT).as_posix() for p in v4_schema_paths.values()), + *(p.relative_to(ROOT).as_posix() for examples in v4_examples.values() for p in examples), + event_schema_path.relative_to(ROOT).as_posix(), + "contracts/upstream/v1/mq.schema.json", +} +v4_paths = [entry.get("path") for entry in v4_artifacts if isinstance(entry, dict)] +if v4_source.get("path") != v4_doc.relative_to(ROOT).as_posix() or len(v4_paths) != len(v4_artifacts) or len(v4_paths) != len(set(v4_paths)) or set(v4_paths) != required_v4: + raise SystemExit("v0.4 manifest source/artifacts mismatch") +for entry in [v4_source, *v4_artifacts]: + relative = Path(entry.get("path", "")) + digest = entry.get("sha256", "") + if relative.is_absolute() or ".." in relative.parts or not re.fullmatch(r"[0-9a-f]{64}", str(digest)): + raise SystemExit(f"invalid v0.4 manifest entry: {entry}") + if not (ROOT / relative).is_file() or hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() != digest: + raise SystemExit(f"v0.4 manifest SHA-256 mismatch: {relative}") +print(f"Task discovery/control/execute v0.4 proposal: positive={v4_positive}, negative={v4_negative}, manifest files={1 + len(v4_artifacts)}") + # The approved Agent→Dispatcher failure fact is Mock-only. Its independent # manifest must not alter the historical SaaS v0.1 or discovery v0.2 baselines. mock_doc = ROOT / "docs/contracts/local-mock-recording-failure-v0.1.md"