diff --git a/agent_call/core.py b/agent_call/core.py index 1a287b5..0d980aa 100644 --- a/agent_call/core.py +++ b/agent_call/core.py @@ -12,16 +12,20 @@ import hashlib import importlib import io import json +import math import os import queue +import re import secrets +import shutil import sqlite3 +import struct import threading import time import uuid import wave from collections.abc import Callable, Iterator -from contextlib import contextmanager +from contextlib import contextmanager, suppress from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, cast @@ -29,9 +33,10 @@ from typing import Any, cast from .mocks import MockAI, MockARI, MockRTP, MockSIPPeer, ProtocolJournal try: - from jsonschema import Draft202012Validator + from jsonschema import Draft202012Validator, FormatChecker except ImportError: # pragma: no cover - the Docker image installs it Draft202012Validator = None + FormatChecker = None ROOT = Path(__file__).resolve().parents[1] @@ -162,6 +167,10 @@ CREATE TABLE IF NOT EXISTS schema_migrations ( version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS readiness_probe ( + probe_id INTEGER PRIMARY KEY CHECK(probe_id = 1), + touched_at TEXT NOT NULL +); CREATE TABLE IF NOT EXISTS tenants ( tenant_id TEXT PRIMARY KEY, tenant_key TEXT NOT NULL UNIQUE, @@ -178,13 +187,16 @@ CREATE TABLE IF NOT EXISTS tasks ( updated_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS commands ( - command_id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + command_id TEXT NOT NULL, tenant_id TEXT NOT NULL, tenant_key TEXT NOT NULL, command_type TEXT NOT NULL, status TEXT NOT NULL, payload_json TEXT NOT NULL, payload_hash TEXT NOT NULL, + operation_type TEXT NOT NULL DEFAULT '', + operation_target TEXT NOT NULL DEFAULT '', task_id TEXT, execution_id TEXT, call_id TEXT, @@ -203,11 +215,13 @@ CREATE TABLE IF NOT EXISTS commands ( source_type TEXT, source_id TEXT, aggregate_version INTEGER NOT NULL DEFAULT 1, - updated_at TEXT NOT NULL + updated_at TEXT NOT NULL, + UNIQUE(tenant_id, command_id) ); -CREATE INDEX IF NOT EXISTS commands_sched_idx ON commands(status, tenant_id, accepted_at); +CREATE INDEX IF NOT EXISTS commands_sched_idx ON commands(tenant_id, status, accepted_at); CREATE TABLE IF NOT EXISTS executions ( - execution_id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + execution_id TEXT NOT NULL, tenant_id TEXT NOT NULL, command_id TEXT NOT NULL, task_id TEXT NOT NULL, @@ -215,12 +229,13 @@ CREATE TABLE IF NOT EXISTS executions ( payload_hash TEXT NOT NULL, status TEXT NOT NULL, call_id TEXT, - created_at TEXT NOT NULL + created_at TEXT NOT NULL, + UNIQUE(tenant_id, execution_id) ); CREATE TABLE IF NOT EXISTS calls ( call_id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL, - execution_id TEXT NOT NULL UNIQUE, + execution_id TEXT NOT NULL, command_id TEXT NOT NULL, task_id TEXT NOT NULL, task_item_id TEXT NOT NULL, @@ -235,7 +250,8 @@ CREATE TABLE IF NOT EXISTS calls ( started_at TEXT, ended_at TEXT, duration_ms INTEGER, - created_at TEXT NOT NULL + created_at TEXT NOT NULL, + UNIQUE(tenant_id, execution_id) ); CREATE INDEX IF NOT EXISTS calls_active_idx ON calls(tenant_id, call_state); CREATE TABLE IF NOT EXISTS attempts ( @@ -282,7 +298,7 @@ CREATE TABLE IF NOT EXISTS recordings ( ); CREATE TABLE IF NOT EXISTS uploads ( upload_id TEXT PRIMARY KEY, - recording_id TEXT NOT NULL UNIQUE REFERENCES recordings(recording_id), + recording_id TEXT NOT NULL REFERENCES recordings(recording_id), tenant_id TEXT NOT NULL, expected_size INTEGER NOT NULL, expected_checksum TEXT NOT NULL, @@ -292,8 +308,10 @@ CREATE TABLE IF NOT EXISTS uploads ( created_at TEXT NOT NULL, completed_at TEXT ); +CREATE INDEX IF NOT EXISTS uploads_recording_idx ON uploads(tenant_id, recording_id, created_at); CREATE TABLE IF NOT EXISTS events ( - event_id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL, tenant_id TEXT NOT NULL, tenant_key TEXT NOT NULL, trace_id TEXT NOT NULL, @@ -310,12 +328,14 @@ CREATE TABLE IF NOT EXISTS events ( published_at TEXT, next_attempt_at TEXT, saas_applied INTEGER, - replay_of TEXT + replay_of TEXT, + UNIQUE(tenant_id, event_id) ); CREATE INDEX IF NOT EXISTS events_outbox_idx ON events(published, next_attempt_at, occurred_at); CREATE TABLE IF NOT EXISTS replays ( - replay_id TEXT PRIMARY KEY, - command_id TEXT NOT NULL UNIQUE, + id INTEGER PRIMARY KEY AUTOINCREMENT, + replay_id TEXT NOT NULL UNIQUE, + command_id TEXT NOT NULL, tenant_id TEXT NOT NULL, source_type TEXT NOT NULL, source_id TEXT NOT NULL, @@ -323,8 +343,11 @@ CREATE TABLE IF NOT EXISTS replays ( status TEXT NOT NULL, sent_count INTEGER NOT NULL DEFAULT 0, failure_code TEXT, + retry_attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT, created_at TEXT NOT NULL, - updated_at TEXT NOT NULL + updated_at TEXT NOT NULL, + UNIQUE(tenant_id, command_id) ); CREATE TABLE IF NOT EXISTS replay_items ( replay_id TEXT NOT NULL REFERENCES replays(replay_id), @@ -333,10 +356,12 @@ CREATE TABLE IF NOT EXISTS replay_items ( PRIMARY KEY(replay_id, event_id) ); CREATE TABLE IF NOT EXISTS inbox ( - event_id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL, tenant_id TEXT NOT NULL, applied_at TEXT NOT NULL, - body_json TEXT NOT NULL + body_json TEXT NOT NULL, + UNIQUE(tenant_id, event_id) ); CREATE TABLE IF NOT EXISTS dead_letters ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -347,17 +372,46 @@ CREATE TABLE IF NOT EXISTS dead_letters ( created_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS published_commands ( - command_id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + command_id TEXT NOT NULL, tenant_id TEXT NOT NULL, body_json TEXT NOT NULL, body_hash TEXT NOT NULL, status TEXT NOT NULL, - created_at TEXT NOT NULL + retry_attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT, + last_error_code TEXT, + created_at TEXT NOT NULL, + UNIQUE(tenant_id, command_id) +); +CREATE TABLE IF NOT EXISTS control_barriers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + command_id TEXT NOT NULL, + tenant_id TEXT NOT NULL, + cell_id TEXT NOT NULL, + status TEXT NOT NULL, + applied_at TEXT, + UNIQUE(tenant_id, command_id, cell_id) +); +CREATE TABLE IF NOT EXISTS storage_operations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + actor_id TEXT NOT NULL, + operation_type TEXT NOT NULL, + target_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + payload_hash TEXT NOT NULL, + status INTEGER NOT NULL, + response_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(tenant_id, actor_id, operation_type, target_id, idempotency_key) ); CREATE TABLE IF NOT EXISTS leases ( lease_name TEXT PRIMARY KEY, owner_id TEXT NOT NULL, - expires_at TEXT NOT NULL + expires_at TEXT NOT NULL, + generation INTEGER NOT NULL DEFAULT 0 ); """ @@ -381,6 +435,11 @@ class Store: self._init_lock = threading.Lock() self.init_schema() + def close(self) -> None: + if self._keeper is not None: + self._keeper.close() + self._keeper = None + def connect(self) -> sqlite3.Connection: con = sqlite3.connect( self.path, uri=self._shared_memory, timeout=10, isolation_level=None @@ -395,13 +454,212 @@ class Store: con = self.connect() try: con.executescript(SCHEMA_SQL) + version_row = con.execute( + "SELECT MAX(version) AS version FROM schema_migrations" + ).fetchone() + version = int(version_row["version"] or 0) + if version < 2 and not self._has_column(con, "commands", "id"): + self._migrate_identity_keys(con) + if version < 3: + self._add_compatibility_columns(con) con.execute( - "INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES(1, ?)", + "INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES(3, ?)", (iso(),), ) finally: con.close() + @staticmethod + def _has_column(con: sqlite3.Connection, table: str, column: str) -> bool: + table_info_sql = { + "commands": "PRAGMA table_info(commands)", + "executions": "PRAGMA table_info(executions)", + "calls": "PRAGMA table_info(calls)", + "events": "PRAGMA table_info(events)", + "replays": "PRAGMA table_info(replays)", + "replay_items": "PRAGMA table_info(replay_items)", + "inbox": "PRAGMA table_info(inbox)", + "published_commands": "PRAGMA table_info(published_commands)", + "leases": "PRAGMA table_info(leases)", + } + return any( + row["name"] == column + for row in con.execute(table_info_sql[table]).fetchall() + ) + + def _migrate_identity_keys(self, con: sqlite3.Connection) -> None: + # The first release used globally unique command/execution/event IDs. Keep + # the facts, but move uniqueness to (tenant_id, public_id) so independent + # tenants may safely reuse their own idempotency namespace. + tables = ( + "commands", + "executions", + "calls", + "events", + "replays", + "replay_items", + "inbox", + "published_commands", + ) + con.execute("PRAGMA foreign_keys=OFF") + con.execute("DROP INDEX IF EXISTS commands_sched_idx") + rename_sql = { + "commands": "ALTER TABLE commands RENAME TO _legacy_commands", + "executions": "ALTER TABLE executions RENAME TO _legacy_executions", + "calls": "ALTER TABLE calls RENAME TO _legacy_calls", + "events": "ALTER TABLE events RENAME TO _legacy_events", + "replays": "ALTER TABLE replays RENAME TO _legacy_replays", + "replay_items": "ALTER TABLE replay_items RENAME TO _legacy_replay_items", + "inbox": "ALTER TABLE inbox RENAME TO _legacy_inbox", + "published_commands": "ALTER TABLE published_commands RENAME TO _legacy_published_commands", + } + for table in tables: + con.execute(rename_sql[table]) + con.executescript(SCHEMA_SQL) + # SCHEMA_SQL enables foreign keys; keep them off while the disposable + # tables are rolled back and the legacy names are restored. + con.execute("PRAGMA foreign_keys=OFF") + committed = False + con.execute("BEGIN") + try: + con.execute( + """INSERT INTO commands( + command_id, tenant_id, tenant_key, command_type, status, + payload_json, payload_hash, operation_type, operation_target, + task_id, execution_id, call_id, trace_id, issued_at, not_after, + admission_deadline, accepted_at, waiting_since, + requested_task_revision, applied_task_revision, task_state, + reason_code, wait_reason_code, snapshot_cutoff, source_type, + source_id, aggregate_version, updated_at) + SELECT command_id, tenant_id, tenant_key, command_type, status, + payload_json, payload_hash, command_type, + COALESCE(task_id, source_id, ''), task_id, execution_id, call_id, + trace_id, issued_at, not_after, admission_deadline, accepted_at, + waiting_since, requested_task_revision, applied_task_revision, + task_state, reason_code, wait_reason_code, snapshot_cutoff, + source_type, source_id, aggregate_version, updated_at + FROM _legacy_commands""" + ) + con.execute( + """INSERT INTO executions( + execution_id, tenant_id, command_id, task_id, task_item_id, + payload_hash, status, call_id, created_at) + SELECT execution_id, tenant_id, command_id, task_id, task_item_id, + payload_hash, status, call_id, created_at + FROM _legacy_executions""" + ) + con.execute( + """INSERT INTO calls( + call_id, tenant_id, execution_id, command_id, task_id, + task_item_id, callee, cell_id, trunk_id, egress_pool_id, + call_state, call_version, outcome, reason_code, started_at, + ended_at, duration_ms, created_at) + SELECT call_id, tenant_id, execution_id, command_id, task_id, + task_item_id, callee, cell_id, trunk_id, egress_pool_id, + call_state, call_version, outcome, reason_code, started_at, + ended_at, duration_ms, created_at + FROM _legacy_calls""" + ) + con.execute( + """INSERT INTO events( + event_id, tenant_id, tenant_key, trace_id, event_type, + aggregate_type, aggregate_id, aggregate_version, occurred_at, + payload_json, body_json, published, broker_status, + broker_attempts, published_at, next_attempt_at, saas_applied, + replay_of) + SELECT event_id, tenant_id, tenant_key, trace_id, event_type, + aggregate_type, aggregate_id, aggregate_version, occurred_at, + payload_json, body_json, published, broker_status, + broker_attempts, published_at, next_attempt_at, saas_applied, + replay_of + FROM _legacy_events""" + ) + con.execute( + """INSERT INTO replays( + replay_id, command_id, tenant_id, source_type, source_id, + snapshot_cutoff, status, sent_count, failure_code, created_at, + updated_at) + SELECT replay_id, command_id, tenant_id, source_type, source_id, + snapshot_cutoff, status, sent_count, failure_code, created_at, + updated_at + FROM _legacy_replays""" + ) + con.execute( + "INSERT INTO replay_items(replay_id, event_id, published_at) " + "SELECT replay_id, event_id, published_at FROM _legacy_replay_items" + ) + con.execute( + "INSERT INTO inbox(event_id, tenant_id, applied_at, body_json) " + "SELECT event_id, tenant_id, applied_at, body_json FROM _legacy_inbox" + ) + con.execute( + """INSERT INTO published_commands( + command_id, tenant_id, body_json, body_hash, status, created_at) + SELECT command_id, tenant_id, body_json, body_hash, status, + created_at FROM _legacy_published_commands""" + ) + con.execute("COMMIT") + committed = True + except Exception: + con.execute("ROLLBACK") + raise + finally: + if committed: + for table in tables: + con.execute(f"DROP TABLE IF EXISTS _legacy_{table}") + else: + # Keep the original tables usable if the copy failed. The new + # schema is disposable because no migration version was saved. + for table in reversed(tables): + con.execute(f"DROP TABLE IF EXISTS {table}") + for table in tables: + con.execute(f"ALTER TABLE _legacy_{table} RENAME TO {table}") + con.execute("PRAGMA foreign_keys=ON") + + def _add_compatibility_columns(self, con: sqlite3.Connection) -> None: + additions = { + "published_commands": ( + ("retry_attempts", "INTEGER NOT NULL DEFAULT 0"), + ("next_attempt_at", "TEXT"), + ("last_error_code", "TEXT"), + ), + "replays": ( + ("retry_attempts", "INTEGER NOT NULL DEFAULT 0"), + ("next_attempt_at", "TEXT"), + ), + "leases": (("generation", "INTEGER NOT NULL DEFAULT 0"),), + } + alter_sql = { + ( + "published_commands", + "retry_attempts", + ): "ALTER TABLE published_commands ADD COLUMN retry_attempts INTEGER NOT NULL DEFAULT 0", + ( + "published_commands", + "next_attempt_at", + ): "ALTER TABLE published_commands ADD COLUMN next_attempt_at TEXT", + ( + "published_commands", + "last_error_code", + ): "ALTER TABLE published_commands ADD COLUMN last_error_code TEXT", + ( + "replays", + "retry_attempts", + ): "ALTER TABLE replays ADD COLUMN retry_attempts INTEGER NOT NULL DEFAULT 0", + ( + "replays", + "next_attempt_at", + ): "ALTER TABLE replays ADD COLUMN next_attempt_at TEXT", + ( + "leases", + "generation", + ): "ALTER TABLE leases ADD COLUMN generation INTEGER NOT NULL DEFAULT 0", + } + for table, columns in additions.items(): + for column, _definition in columns: + if not self._has_column(con, table, column): + con.execute(alter_sql[(table, column)]) + @contextmanager def tx(self) -> Iterator[sqlite3.Connection]: con = self.connect() @@ -418,7 +676,7 @@ class Store: def one(self, sql: str, params: tuple[Any, ...] = ()) -> sqlite3.Row: con = self.connect() try: - # pi-lens-ignore: python-sql-injection — sql is paired with bound params and only callers in this module can supply it. + # pi-lens-ignore: python-sql-injection return cast(sqlite3.Row, con.execute(sql, params).fetchone()) finally: con.close() @@ -426,7 +684,7 @@ class Store: def all(self, sql: str, params: tuple[Any, ...] = ()) -> list[sqlite3.Row]: con = self.connect() try: - # pi-lens-ignore: python-sql-injection — sql is paired with bound params and only callers in this module can supply it. + # pi-lens-ignore: python-sql-injection return list(con.execute(sql, params).fetchall()) finally: con.close() @@ -435,12 +693,18 @@ class Store: class InMemoryBroker: """A strict direct-exchange broker used for deterministic acceptance tests.""" - def __init__(self, max_messages: int = 0) -> None: + def __init__(self, max_messages: int = 0, max_bytes: int = 0) -> None: self._queues: dict[str, queue.Queue[dict[str, Any]]] = {} + self._queue_bytes: dict[str, int] = {} self.max_messages = max_messages + self.max_bytes = max_bytes self._routes: dict[str, str] = {} self._lock = threading.Lock() self.outage = False + self.blocked = False + self.fail_next: str | None = None + self.fail_after: int | None = None + self._successful_publishes = 0 self.published: list[dict[str, Any]] = [] def declare_tenant(self, tenant_key: str) -> None: @@ -448,6 +712,7 @@ class InMemoryBroker: route = routing_key(tenant_key) with self._lock: self._queues.setdefault(name, queue.Queue(maxsize=self.max_messages)) + self._queue_bytes.setdefault(name, 0) self._routes[route] = name def declare_events(self) -> None: @@ -455,6 +720,7 @@ class InMemoryBroker: self._queues.setdefault( "agent-call.saas.events.v1", queue.Queue(maxsize=self.max_messages) ) + self._queue_bytes.setdefault("agent-call.saas.events.v1", 0) def publish( self, @@ -467,6 +733,17 @@ class InMemoryBroker: if self.outage: raise BrokerError() with self._lock: + if self.blocked: + raise BrokerError("broker is blocked; publication is retained") + if self.fail_next: + reason = self.fail_next + self.fail_next = None + raise BrokerError(reason) + if ( + self.fail_after is not None + and self._successful_publishes >= self.fail_after + ): + raise BrokerError("publisher confirmation was lost") if exchange == "agent-call.commands.v1": target = self._routes.get(route) else: @@ -477,42 +754,95 @@ class InMemoryBroker: ) if target is None or target not in self._queues: raise BrokerError("mandatory publish was unroutable") + encoded_size = len(canonical(body).encode("utf-8")) + if ( + self.max_bytes + and self._queue_bytes[target] + encoded_size > self.max_bytes + ): + raise BrokerError("queue byte limit reached; publication is retained") message = { "exchange": exchange, "routing_key": route, "body": body, "message_id": message_id, "headers": headers or {}, + "queue_name": target, + "_size": encoded_size, } try: self._queues[target].put_nowait(message) except queue.Full as exc: raise BrokerError("queue is full; publication is retained") from exc + self._queue_bytes[target] += encoded_size + self._successful_publishes += 1 self.published.append(message) def ack(self, message: dict[str, Any]) -> None: return + def reject(self, message: dict[str, Any]) -> None: + return + + @staticmethod + def _message_size(message: dict[str, Any]) -> int: + try: + return max(0, int(message.get("_size", 0))) + except (TypeError, ValueError): + return 0 + + def requeue(self, message: dict[str, Any]) -> None: + name = message.get("queue_name") + if not name: + raise BrokerError("message has no queue context") + with self._lock: + target = self._queues.get(name) + if target is None: + raise BrokerError("queue is not declared") + try: + target.put_nowait(message) + except queue.Full as exc: + raise BrokerError("queue is full while requeuing") from exc + self._queue_bytes[name] += self._message_size(message) + def consume(self, name: str) -> dict[str, Any] | None: if self.outage: raise BrokerError() with self._lock: target = self._queues.get(name) - if target is None: - raise BrokerError("queue is not declared") - try: - return target.get_nowait() - except queue.Empty: - return None + if target is None: + raise BrokerError("queue is not declared") + try: + message = target.get_nowait() + except queue.Empty: + return None + self._queue_bytes[name] = max( + 0, self._queue_bytes[name] - self._message_size(message) + ) + return message + + def depth(self, name: str) -> int: + with self._lock: + target = self._queues.get(name) + return target.qsize() if target is not None else 0 + + def queue_bytes(self, name: str) -> int: + with self._lock: + return self._queue_bytes.get(name, 0) + + def has_route(self, route: str) -> bool: + with self._lock: + return route in self._routes def ready(self) -> bool: - return not self.outage + return not self.outage and not self.blocked class PikaBroker: """RabbitMQ adapter. Import and connection are optional in local unit tests.""" - def __init__(self, url: str) -> None: + def __init__( + self, url: str, max_messages: int = 1000, max_bytes: int = 16 * 1024 * 1024 + ) -> None: try: pika = importlib.import_module("pika") except ImportError as exc: # pragma: no cover @@ -520,6 +850,9 @@ class PikaBroker: "BROKER_MODE=rabbit requires the pika package" ) from exc self.pika = pika + self.max_messages = max_messages + self.max_bytes = max_bytes + self._declared_routes: set[str] = set() try: self.connection = pika.BlockingConnection(pika.URLParameters(url)) self.channel = self.connection.channel() @@ -528,15 +861,59 @@ class PikaBroker: self.channel.exchange_declare( exchange="agent-call.commands.v1", exchange_type="direct", durable=True ) + self.channel.exchange_declare( + exchange="agent-call.dead.v1", exchange_type="direct", durable=True + ) self.channel.exchange_declare( exchange="agent-call.events.v1", exchange_type="topic", durable=True ) - self.channel.queue_declare(queue="agent-call.saas.events.v1", durable=True) + for queue, dead_route in ( + ("agent-call.commands.dead.v1", "agent-call.commands.dead"), + ("agent-call.events.dead.v1", "agent-call.events.dead"), + ): + self.channel.queue_declare( + queue=queue, + durable=True, + arguments={ + "x-max-length": self.max_messages, + "x-max-length-bytes": self.max_bytes, + "x-overflow": "reject-publish", + }, + ) + self.channel.queue_bind( + queue=queue, + exchange="agent-call.dead.v1", + routing_key=dead_route, + ) + self.channel.queue_declare( + queue="agent-call.saas.events.v1", + durable=True, + arguments={ + "x-max-length": self.max_messages, + "x-max-length-bytes": self.max_bytes, + "x-overflow": "reject-publish", + "x-dead-letter-exchange": "agent-call.dead.v1", + "x-dead-letter-routing-key": "agent-call.events.dead", + }, + ) self.channel.queue_bind( queue="agent-call.saas.events.v1", exchange="agent-call.events.v1", routing_key="agent-call.#", ) + self._blocked = False + add_blocked = getattr( + self.connection, "add_on_connection_blocked_callback", None + ) + if callable(add_blocked): + add_blocked( + lambda _connection, _reason: setattr(self, "_blocked", True) + ) + add_unblocked = getattr( + self.connection, "add_on_connection_unblocked_callback", None + ) + if callable(add_unblocked): + add_unblocked(lambda _connection: setattr(self, "_blocked", False)) except Exception as exc: # pragma: no cover raise BrokerError( f"RabbitMQ connection failed: {type(exc).__name__}" @@ -546,11 +923,20 @@ class PikaBroker: name = queue_name(tenant_key) route = routing_key(tenant_key) self.channel.queue_declare( - queue=name, durable=True, arguments={"x-max-length": 10000} + queue=name, + durable=True, + arguments={ + "x-max-length": self.max_messages, + "x-max-length-bytes": self.max_bytes, + "x-overflow": "reject-publish", + "x-dead-letter-exchange": "agent-call.dead.v1", + "x-dead-letter-routing-key": "agent-call.commands.dead", + }, ) self.channel.queue_bind( queue=name, exchange="agent-call.commands.v1", routing_key=route ) + self._declared_routes.add(route) def declare_events(self) -> None: # pragma: no cover return @@ -597,11 +983,35 @@ class PikaBroker: "body": parsed, "message_id": props.message_id, "headers": props.headers or {}, + "routing_key": getattr(method, "routing_key", None), + "queue_name": name, "_delivery_tag": method.delivery_tag, } except Exception as exc: raise BrokerError(f"RabbitMQ consume failed: {type(exc).__name__}") from exc + def requeue(self, message: dict[str, Any]) -> None: # pragma: no cover + tag = message.get("_delivery_tag") + if tag is None: + return + try: + self.channel.basic_nack(tag, requeue=True) + except Exception as exc: + raise BrokerError("RabbitMQ requeue failed") from exc + + def depth(self, name: str) -> int: # pragma: no cover + try: + return int( + self.channel.queue_declare( + queue=name, passive=True + ).method.message_count + ) + except Exception as exc: + raise BrokerError("RabbitMQ queue depth failed") from exc + + def has_route(self, route: str) -> bool: # pragma: no cover + return route in self._declared_routes + def ack(self, message: dict[str, Any]) -> None: # pragma: no cover tag = message.get("_delivery_tag") if tag is None: @@ -611,24 +1021,48 @@ class PikaBroker: except Exception as exc: raise BrokerError("RabbitMQ acknowledgement failed") from exc + def reject(self, message: dict[str, Any]) -> None: # pragma: no cover + tag = message.get("_delivery_tag") + if tag is None: + return + try: + self.channel.basic_nack(tag, requeue=False) + except Exception as exc: + raise BrokerError("RabbitMQ rejection failed") from exc + def ready(self) -> bool: # pragma: no cover - return bool(self.connection.is_open and self.channel.is_open) + return bool( + self.connection.is_open and self.channel.is_open and not self._blocked + ) class MockObjectStore: + _id_pattern = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + def __init__(self, root: str | os.PathLike[str], secret: str | None = None) -> None: self.root = Path(root) self.root.mkdir(parents=True, exist_ok=True) seed = secret or os.environ.get("MOCK_UPLOAD_SECRET") or self.root.as_posix() self.secret = seed.encode("utf-8") + @classmethod + def _check_id(cls, upload_id: str) -> None: + if not isinstance(upload_id, str) or not cls._id_pattern.fullmatch(upload_id): + raise ValidationError("UPLOAD_ID_INVALID", "upload id is invalid") + def path_for(self, upload_id: str) -> Path: + self._check_id(upload_id) return self.root / f"{upload_id}.part" def final_path_for(self, upload_id: str) -> Path: + self._check_id(upload_id) return self.root / upload_id + def has_final(self, upload_id: str) -> bool: + return self.final_path_for(upload_id).is_file() + def token(self, upload_id: str, tenant_id: str) -> str: + self._check_id(upload_id) return hashlib.sha256( self.secret + b":" + tenant_id.encode() + b":" + upload_id.encode() ).hexdigest() @@ -636,24 +1070,27 @@ class MockObjectStore: def put(self, upload_id: str, tenant_id: str, data: bytes, token: str) -> None: if not secrets.compare_digest(token, self.token(upload_id, tenant_id)): raise ValidationError("UPLOAD_TOKEN_INVALID", "upload token is invalid") - path = self.path_for(upload_id) - path.write_bytes(data) + if not isinstance(data, bytes): + raise ValidationError("OBJECT_BODY_INVALID", "object body must be bytes") + self.path_for(upload_id).write_bytes(data) def verify_and_promote( self, upload_id: str, expected_size: int, expected_checksum: str ) -> str: + final = self.final_path_for(upload_id) path = self.path_for(upload_id) - if not path.exists(): + candidate = final if final.exists() else path + if not candidate.exists(): raise ValidationError("OBJECT_MISSING", "object was not uploaded") - data = path.read_bytes() + data = candidate.read_bytes() actual = hashlib.sha256(data).hexdigest() if len(data) != expected_size or actual != expected_checksum: raise ValidationError( "OBJECT_VERIFICATION_FAILED", "size or checksum does not match the upload contract", ) - final = self.final_path_for(upload_id) - os.replace(path, final) + if candidate == path: + os.replace(path, final) return f"oss://mock/{upload_id}" @@ -685,6 +1122,7 @@ class AgentCallService: else Path("data") / "agent-call-objects" ) ) + self.object_root = object_root self.object_store = MockObjectStore( object_root, os.environ.get("MOCK_UPLOAD_SECRET") ) @@ -701,17 +1139,29 @@ class AgentCallService: broker_url = os.environ.get("RABBITMQ_URL") if not broker_url: raise ConfigurationError("BROKER_MODE=rabbit requires RABBITMQ_URL") - broker = PikaBroker(broker_url) + broker = PikaBroker( + broker_url, + max_messages=_integer( + self._limits().get("max_queue_messages", 1000) + ), + max_bytes=_integer( + self._limits().get("max_queue_bytes", 16 * 1024 * 1024) + ), + ) else: broker = InMemoryBroker( - _integer( - self.profile.get("limits", {}).get("max_queue_messages", 1000) - ) + _integer(self._limits().get("max_queue_messages", 1000)), + _integer(self._limits().get("max_queue_bytes", 16 * 1024 * 1024)), ) self.broker = broker self.profile.setdefault("provider_modes", {})["rabbitmq"] = ( "rabbit" if isinstance(broker, PikaBroker) else "memory" ) + self._provider_status = { + str(component): str(provider) + for component, provider in self.profile.get("provider_modes", {}).items() + } + self._unrouteable_tenants: set[str] = set() self._seed() self._tenant_order = [ item["tenant_id"] for item in self.profile.get("tenants", []) @@ -726,6 +1176,10 @@ class AgentCallService: self._stop = threading.Event() self._background: threading.Thread | None = None self._last_error: str | None = None + self._last_wall_clock: datetime | None = None + self._last_monotonic: float | None = None + self._clock_jump = False + self._recover_durable_state() if start_background: self.start() @@ -733,20 +1187,62 @@ class AgentCallService: if self.mode not in {"mock", "mixed", "real"}: raise ConfigurationError("mode must be mock, mixed, or real") providers = self.profile.get("provider_modes", {}) - if self.mode == "real" and any(value != "real" for value in providers.values()): + if not isinstance(providers, dict): + raise ConfigurationError("provider_modes must be an object") + mock_values = { + "saas": "mock", + "database": "sqlite", + "rabbitmq": "memory", + "sip": "mock", + "asterisk": "mock", + "asr": "mock", + "llm": "mock", + "tts": "mock", + "oss": "mock", + "cloud": "fake-cli", + } + if self.mode == "real": raise ConfigurationError( - "real mode refuses mock or test providers; no silent fallback is allowed" + "real mode is unavailable until every real provider adapter is configured; mock fallback is forbidden" ) - seen: set[str] = set() + if self.mode == "mixed": + raise ConfigurationError( + "mixed mode requires explicit real adapters; this build has none" + ) + unknown = set(providers) - set(mock_values) + if unknown: + raise ConfigurationError(f"unknown provider components: {sorted(unknown)}") + missing = set(mock_values) - set(providers) + if missing: + raise ConfigurationError(f"provider modes are missing: {sorted(missing)}") + for component, expected in mock_values.items(): + configured = providers[component] + if configured != expected and not ( + component == "rabbitmq" and configured == "rabbit" + ): + raise ConfigurationError( + f"mock mode refuses provider {component}={configured!r}" + ) + seen_ids: set[str] = set() + seen_keys: set[str] = set() for tenant in self.profile.get("tenants", []): - tenant_key = tenant["tenant_key"] - if tenant_key in seen: - raise ConfigurationError("tenant_key must be unique") - seen.add(tenant_key) - routing_key(tenant_key) - queue_name(tenant_key) + tenant_id = tenant.get("tenant_id") + tenant_key = tenant.get("tenant_key") + if not isinstance(tenant_id, str) or not tenant_id: + raise ConfigurationError("tenant_id must be a non-empty string") + if not isinstance(tenant_key, str) or not tenant_key: + raise ConfigurationError("tenant_key must be a non-empty string") + if tenant_id in seen_ids or tenant_key in seen_keys: + raise ConfigurationError( + "tenant identifiers and tenant_key values must be unique" + ) + seen_ids.add(tenant_id) + seen_keys.add(tenant_key) if not self.profile.get("cells"): raise ConfigurationError("at least one cell is required") + for cell in self.profile["cells"]: + if _integer(cell.get("capacity", 0)) < 1: + raise ConfigurationError("cell capacity must be positive") def _seed(self) -> None: now = iso(self.clock()) @@ -776,9 +1272,35 @@ class AgentCallService: ), ) for tenant in self.profile.get("tenants", []): - self.broker.declare_tenant(tenant["tenant_key"]) + try: + self.broker.declare_tenant(tenant["tenant_key"]) + except ValidationError: + # Keep an over-budget tenant in durable state. Its original + # command is retained instead of being truncated or rerouted. + self._unrouteable_tenants.add(tenant["tenant_id"]) self.broker.declare_events() + def _recover_durable_state(self) -> None: + """Fence unfinished calls on restart; never infer that a channel ended.""" + with self.store.tx() as con: + calls = con.execute( + "SELECT * FROM calls WHERE call_state NOT IN ('ended', 'failed', 'reconciling')" + ).fetchall() + now = iso(self._now()) + for call in calls: + con.execute( + "UPDATE calls SET call_state='reconciling', reason_code='RECOVERY_RECONCILING', call_version=call_version+1 WHERE call_id=?", + (call["call_id"],), + ) + con.execute( + "UPDATE commands SET status='reconciling', reason_code='RECOVERY_RECONCILING', updated_at=? WHERE tenant_id=? AND command_id=? AND status NOT IN ('completed','failed','rejected','applied')", + (now, call["tenant_id"], call["command_id"]), + ) + con.execute( + "UPDATE executions SET status='reconciling' WHERE tenant_id=? AND execution_id=? AND status NOT IN ('completed','failed','rejected')", + (call["tenant_id"], call["execution_id"]), + ) + def start(self) -> None: if self._background and self._background.is_alive(): return @@ -798,23 +1320,45 @@ class AgentCallService: if remaining <= 0: break thread.join(timeout=remaining) + self.store.close() def _run_loop(self) -> None: while not self._stop.is_set(): try: self.process_once() - except Exception as exc: + except ( + ServiceError, + OSError, + RuntimeError, + TypeError, + ValueError, + KeyError, + sqlite3.DatabaseError, + ) as exc: # The next tick retries durable outbox and waiting work. Do not log # payloads, credentials, or raw broker diagnostics. self._last_error = type(exc).__name__ self._stop.wait(0.02) def _now(self) -> datetime: - return self.clock().astimezone(timezone.utc) + now = self.clock().astimezone(timezone.utc) + monotonic = time.monotonic() + if self._last_wall_clock is not None and self._last_monotonic is not None: + wall_delta = (now - self._last_wall_clock).total_seconds() + monotonic_delta = monotonic - self._last_monotonic + if abs(wall_delta - monotonic_delta) > 2: + self._clock_jump = True + self._last_wall_clock = now + self._last_monotonic = monotonic + return now def _limits(self) -> dict[str, Any]: return self.profile.get("limits", {}) + @staticmethod + def _operation_hash(operation: str, target: str, body: dict[str, Any]) -> str: + return digest({"operation": operation, "target": target, "body": body}) + def _route_config(self, route_policy_id: str) -> dict[str, Any]: for route in self.profile.get("routes", []): if route.get("route_policy_id") == route_policy_id and route.get( @@ -849,7 +1393,10 @@ class AgentCallService: ) if Draft202012Validator is not None: errors = sorted( - Draft202012Validator(self.mq_schema).iter_errors(body), + Draft202012Validator( + self.mq_schema, + format_checker=FormatChecker() if FormatChecker else None, + ).iter_errors(body), key=lambda error: list(error.path), ) if errors: @@ -872,10 +1419,29 @@ class AgentCallService: "SCHEMA_INVALID", "required MQ fields are missing" ) + def _validate_command_times(self, body: dict[str, Any]) -> None: + try: + issued_at = parse_time(body["issued_at"]) + not_after = parse_time(body["not_after"]) + except (TypeError, ValueError, OverflowError, KeyError) as exc: + raise ValidationError( + "SCHEMA_INVALID", "command timestamps are invalid" + ) from exc + now = self._now() + if issued_at > now + timedelta(seconds=5) or not_after <= issued_at: + raise ValidationError( + "SCHEMA_INVALID", "command timestamp window is invalid" + ) + if not_after <= now: + raise ConflictError("ADMISSION_EXPIRED", "command is past not_after") + def _validate_event(self, body: dict[str, Any]) -> None: if Draft202012Validator is not None: errors = sorted( - Draft202012Validator(self.mq_schema).iter_errors(body), + Draft202012Validator( + self.mq_schema, + format_checker=FormatChecker() if FormatChecker else None, + ).iter_errors(body), key=lambda error: list(error.path), ) if errors: @@ -887,32 +1453,44 @@ class AgentCallService: """SaaS-side test publisher. The production ingress is the broker queue.""" self._validate_message(body) tenant = self._tenant(body["tenant_id"], body["tenant_key"]) - task = self.store.one( - "SELECT * FROM tasks WHERE task_id=? AND tenant_id=?", - (body["payload"]["task_id"], tenant["tenant_id"]), - ) - if task is None: - raise ValidationError( - "TASK_TENANT_MISMATCH", "task is not owned by the command tenant" - ) - if task["state"] != "running": - raise ConflictError( - "TASK_PUBLISH_BLOCKED", - "SaaS persistent task block prevents new publication", - ) - route = routing_key(tenant["tenant_key"]) body_hash = digest(body) + existing_before_admission = self.store.one( + "SELECT * FROM published_commands WHERE command_id=? AND tenant_id=?", + (body["command_id"], tenant["tenant_id"]), + ) + if existing_before_admission: + if existing_before_admission["body_hash"] != body_hash: + raise ConflictError( + "IDEMPOTENCY_CONFLICT", + "command_id was already published with a different body", + ) + if existing_before_admission["status"] not in {"pending", "retained"}: + return { + "command_id": body["command_id"], + "status": existing_before_admission["status"], + "routing_key": routing_key(tenant["tenant_key"]), + } + self._validate_command_times(body) + if existing_before_admission: + task = self.store.one( + "SELECT state FROM tasks WHERE task_id=? AND tenant_id=?", + (body["payload"]["task_id"], tenant["tenant_id"]), + ) + if task is None: + raise ValidationError( + "TASK_TENANT_MISMATCH", "task is not owned by the command tenant" + ) + if task["state"] != "running": + raise ConflictError( + "TASK_PUBLISH_BLOCKED", + "SaaS persistent task block prevents new publication", + ) + rate_limited = False with self.store.tx() as con: existing = con.execute( "SELECT * FROM published_commands WHERE command_id = ? AND tenant_id = ?", (body["command_id"], tenant["tenant_id"]), ).fetchone() - foreign = con.execute( - "SELECT 1 FROM published_commands WHERE command_id = ?", - (body["command_id"],), - ).fetchone() - if foreign and existing is None: - raise ConflictError("IDEMPOTENCY_CONFLICT", "command id is unavailable") if existing: if existing["body_hash"] != body_hash: raise ConflictError( @@ -923,9 +1501,35 @@ class AgentCallService: return { "command_id": body["command_id"], "status": existing["status"], - "routing_key": route, + "routing_key": routing_key(tenant["tenant_key"]), } + if ( + existing["last_error_code"] == "PUBLISH_RATE_LIMIT" + and existing["next_attempt_at"] + and parse_time(existing["next_attempt_at"]) > self._now() + ): + raise ServiceError( + "PUBLISH_RATE_LIMIT", + "tenant publication rate limit reached", + 429, + True, + ) else: + task = con.execute( + "SELECT * FROM tasks WHERE task_id=? AND tenant_id=?", + (body["payload"]["task_id"], tenant["tenant_id"]), + ).fetchone() + if task is None: + raise ValidationError( + "TASK_TENANT_MISMATCH", + "task is not owned by the command tenant", + ) + if task["state"] != "running": + raise ConflictError( + "TASK_PUBLISH_BLOCKED", + "SaaS persistent task block prevents new publication", + ) + created_at = iso(self._now()) con.execute( "INSERT INTO published_commands(command_id, tenant_id, body_json, body_hash, status, created_at) VALUES(?,?,?,?,?,?)", ( @@ -934,24 +1538,72 @@ class AgentCallService: canonical(body), body_hash, "pending", - iso(self._now()), + created_at, ), ) - try: - self.broker.publish( - "agent-call.commands.v1", route, body, body["command_id"] + publish_rate = _integer(self._limits().get("tenant_publish_rate", 0)) + if publish_rate: + recent = con.execute( + "SELECT COUNT(*) AS n FROM published_commands WHERE tenant_id=? AND created_at>=?", + (tenant["tenant_id"], iso(self._now() - timedelta(seconds=1))), + ).fetchone()["n"] + if _integer(recent) > publish_rate: + con.execute( + "UPDATE published_commands SET status='retained', last_error_code='PUBLISH_RATE_LIMIT', next_attempt_at=? WHERE tenant_id=? AND command_id=?", + ( + iso(self._now() + timedelta(seconds=1)), + tenant["tenant_id"], + body["command_id"], + ), + ) + rate_limited = True + if rate_limited: + raise ServiceError( + "PUBLISH_RATE_LIMIT", "tenant publication rate limit reached", 429, True ) - except Exception: + try: + route = routing_key(tenant["tenant_key"]) + self.broker.publish( + "agent-call.commands.v1", + route, + body, + body["command_id"], + {"tenant_id": tenant["tenant_id"], "tenant_key": tenant["tenant_key"]}, + ) + except ValidationError as exc: with self.store.tx() as con: con.execute( - "UPDATE published_commands SET status = 'retained' WHERE command_id = ?", - (body["command_id"],), + "UPDATE published_commands SET status='retained', last_error_code=? WHERE tenant_id=? AND command_id=?", + (exc.code, tenant["tenant_id"], body["command_id"]), + ) + raise + except Exception as exc: + with self.store.tx() as con: + attempts = ( + con.execute( + "SELECT retry_attempts FROM published_commands WHERE tenant_id=? AND command_id=?", + (tenant["tenant_id"], body["command_id"]), + ).fetchone()["retry_attempts"] + + 1 + ) + con.execute( + "UPDATE published_commands SET status='retained', retry_attempts=?, next_attempt_at=?, last_error_code=? WHERE tenant_id=? AND command_id=?", + ( + attempts, + iso( + self._now() + + timedelta(seconds=min(60, 2 ** min(attempts, 5))) + ), + type(exc).__name__, + tenant["tenant_id"], + body["command_id"], + ), ) raise with self.store.tx() as con: con.execute( - "UPDATE published_commands SET status = 'broker_confirmed' WHERE command_id = ?", - (body["command_id"],), + "UPDATE published_commands SET status='broker_confirmed', next_attempt_at=NULL, last_error_code=NULL WHERE tenant_id=? AND command_id=?", + (tenant["tenant_id"], body["command_id"]), ) return { "command_id": body["command_id"], @@ -959,6 +1611,86 @@ class AgentCallService: "routing_key": route, } + def _tenant_window_full(self, tenant_id: str, tenant_key: str) -> bool: + limits = self._limits() + tenant_limit = _integer(limits.get("pending_window_per_tenant", 16)) + global_limit = _integer(limits.get("pending_window_global", 256)) + tenant_row = self.store.one( + "SELECT COUNT(*) AS n FROM commands WHERE tenant_id=? AND status NOT IN ('completed','failed','rejected','applied')", + (tenant_id,), + ) + global_row = self.store.one( + "SELECT COUNT(*) AS n FROM commands WHERE status NOT IN ('completed','failed','rejected','applied')" + ) + queue_depth = getattr(self.broker, "depth", lambda _name: 0)( + queue_name(tenant_key) + ) + return ( + _integer(tenant_row["n"]) >= tenant_limit + or _integer(global_row["n"]) >= global_limit + or queue_depth >= _integer(limits.get("max_unacked_per_tenant", 32)) + ) + + def _validate_delivery( + self, + body: dict[str, Any], + tenant: sqlite3.Row, + message: dict[str, Any], + ) -> None: + expected_route = routing_key(tenant["tenant_key"]) + expected_queue = queue_name(tenant["tenant_key"]) + if message.get("queue_name") != expected_queue: + raise ValidationError( + "TENANT_ROUTE_MISMATCH", "message arrived on the wrong queue" + ) + if message.get("routing_key") != expected_route: + raise ValidationError( + "TENANT_ROUTE_MISMATCH", + "message routing key is not bound to the tenant", + ) + headers = message.get("headers") or {} + if not isinstance(headers, dict): + raise ValidationError( + "TENANT_ROUTE_MISMATCH", "message headers are invalid" + ) + if "tenant_id" in headers and headers["tenant_id"] != tenant["tenant_id"]: + raise ValidationError( + "TENANT_ROUTE_MISMATCH", "tenant_id header does not match the queue" + ) + if "tenant_key" in headers and headers["tenant_key"] != tenant["tenant_key"]: + raise ValidationError( + "TENANT_ROUTE_MISMATCH", "tenant_key header does not match the queue" + ) + if message.get("message_id") not in {None, body.get("command_id")}: + raise ValidationError( + "MESSAGE_ID_MISMATCH", "message_id does not match command_id" + ) + if ( + body.get("tenant_id") != tenant["tenant_id"] + or body.get("tenant_key") != tenant["tenant_key"] + ): + raise ValidationError( + "TENANT_BINDING_INVALID", "message tenant does not match the queue" + ) + + def _dead_letter(self, message: dict[str, Any], body: Any, reason: str) -> None: + body_map = body if isinstance(body, dict) else {} + try: + encoded = canonical(body) + except (TypeError, ValueError): + encoded = "{}" + with self.store.tx() as con: + con.execute( + "INSERT INTO dead_letters(tenant_key, routing_key, body_json, reason_code, created_at) VALUES(?,?,?,?,?)", + ( + body_map.get("tenant_key"), + message.get("routing_key"), + encoded, + reason, + iso(self._now()), + ), + ) + def ingest_commands(self, max_per_tick: int | None = None) -> int: max_per_tick = ( max_per_tick @@ -975,52 +1707,72 @@ class AgentCallService: tenant = self.store.one( "SELECT * FROM tenants WHERE tenant_id = ?", (tenant_id,) ) - if tenant is None: + if tenant is None or tenant_id in self._unrouteable_tenants: continue try: + if self._tenant_window_full(tenant_id, tenant["tenant_key"]): + continue message = self.broker.consume(queue_name(tenant["tenant_key"])) - except BrokerError: + except (BrokerError, ValidationError): break if not message: continue body: Any = message.get("body", {}) - body_map = body if isinstance(body, dict) else {} try: - self.accept_execute(body) + body_map = body if isinstance(body, dict) else {} + self._validate_delivery(body_map, tenant, message) + self.accept_execute(body_map, delivery_context=message) accepted += 1 with self.store.tx() as con: con.execute( - "UPDATE published_commands SET status = 'accepted' WHERE command_id = ?", - (body_map.get("command_id"),), + "UPDATE published_commands SET status='accepted' WHERE tenant_id=? AND command_id=?", + (tenant_id, body_map.get("command_id")), ) self.broker.ack(message) except ServiceError as exc: - with self.store.tx() as con: - con.execute( - "INSERT INTO dead_letters(tenant_key, routing_key, body_json, reason_code, created_at) VALUES(?,?,?,?,?)", - ( - body_map.get("tenant_key"), - message.get("routing_key"), - canonical(body), - exc.code, - iso(self._now()), - ), - ) - if body_map.get("command_id"): + self._dead_letter(message, body, exc.code) + body_map = body if isinstance(body, dict) else {} + if body_map.get("command_id"): + with self.store.tx() as con: con.execute( - "UPDATE published_commands SET status = 'rejected' WHERE command_id = ?", - (body_map["command_id"],), + "UPDATE published_commands SET status='rejected', last_error_code=? WHERE tenant_id=? AND command_id=?", + (exc.code, tenant_id, body_map["command_id"]), ) - self.broker.ack(message) + reject = getattr(self.broker, "reject", None) + if callable(reject): + reject(message) + else: + self.broker.ack(message) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + KeyError, + sqlite3.DatabaseError, + ): + # Unknown persistence/broker errors must remain deliverable; an + # invalid message is handled above and sent to the dead-letter log. + with suppress(BrokerError): + self.broker.requeue(message) + break self._cursor = (self._cursor + max_per_tick) % len(self._tenant_order) return accepted - def accept_execute(self, body: dict[str, Any]) -> dict[str, Any]: + def accept_execute( + self, + body: dict[str, Any], + delivery_context: dict[str, Any] | None = None, + ) -> dict[str, Any]: self._validate_message(body) + self._validate_command_times(body) tenant = self._tenant(body["tenant_id"], body["tenant_key"]) + if delivery_context is not None: + self._validate_delivery(body, tenant, delivery_context) payload = body["payload"] task = self.store.one( - "SELECT * FROM tasks WHERE task_id = ?", (payload["task_id"],) + "SELECT * FROM tasks WHERE task_id = ? AND tenant_id = ?", + (payload["task_id"], tenant["tenant_id"]), ) if task is None or task["tenant_id"] != tenant["tenant_id"]: raise ValidationError( @@ -1039,12 +1791,6 @@ class AgentCallService: "SELECT * FROM commands WHERE command_id = ? AND tenant_id = ?", (body["command_id"], tenant["tenant_id"]), ).fetchone() - foreign = con.execute( - "SELECT 1 FROM commands WHERE command_id = ?", - (body["command_id"],), - ).fetchone() - if foreign and existing is None: - raise ConflictError("IDEMPOTENCY_CONFLICT", "command id is unavailable") if existing: if existing["payload_hash"] != command_hash: raise ConflictError( @@ -1079,15 +1825,10 @@ class AgentCallService: }: status, reason = "rejected", "AGENT_VERSION_NOT_ALLOWED" execution = con.execute( - "SELECT * FROM executions WHERE execution_id = ?", - (payload["execution_id"],), + "SELECT * FROM executions WHERE execution_id = ? AND tenant_id = ?", + (payload["execution_id"], tenant["tenant_id"]), ).fetchone() - if ( - execution - and execution["tenant_id"] != tenant["tenant_id"] - or execution - and execution["payload_hash"] != digest(payload) - ): + if execution and execution["payload_hash"] != digest(payload): status, reason = "rejected", "EXECUTION_ID_CONFLICT" elif execution: status, reason = "completed", "DUPLICATE_EXECUTION" @@ -1131,13 +1872,9 @@ class AgentCallService: iso(now), ), ) - elif execution: - con.execute( - "UPDATE executions SET status = 'duplicate' WHERE execution_id = ?", - (payload["execution_id"],), - ) row = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (body["command_id"],) + "SELECT * FROM commands WHERE tenant_id = ? AND command_id = ?", + (tenant["tenant_id"], body["command_id"]), ).fetchone() self._command_event_tx(con, row, reason or "ACCEPTED") return self._command_snapshot(row) @@ -1241,12 +1978,68 @@ class AgentCallService: "aggregate_version": row["aggregate_version"], } - def _claim_lease(self) -> bool: + def _lease_ttl(self) -> int: + return max(1, _integer(self._limits().get("scheduler_lease_ttl_s", 10))) + + def retry_published_commands(self, limit: int = 20) -> int: now = self._now() - expires = now + timedelta(seconds=2) + rows = self.store.all( + "SELECT * FROM published_commands WHERE status IN ('pending','retained') AND (next_attempt_at IS NULL OR next_attempt_at<=?) ORDER BY created_at LIMIT ?", + (iso(now), limit), + ) + retried = 0 + for row in rows: + if row["last_error_code"] == "TENANT_KEY_UNSUPPORTED": + continue + try: + self.publish_execute(_loads(row["body_json"])) + retried += 1 + except ServiceError: + continue + return retried + + def recover_uploads(self, limit: int = 20) -> int: + rows = self.store.all( + "SELECT * FROM uploads WHERE status IN ('created','uploaded') ORDER BY created_at LIMIT ?", + (limit,), + ) + recovered = 0 + for upload in rows: + recording = self.store.one( + "SELECT * FROM recordings WHERE recording_id=? AND tenant_id=?", + (upload["recording_id"], upload["tenant_id"]), + ) + if recording is None: + continue + try: + if self.object_store.has_final(upload["upload_id"]): + self._complete_upload( + upload["tenant_id"], + upload["upload_id"], + upload["recording_id"], + checksum=upload["expected_checksum"], + size_bytes=upload["expected_size"], + recovery=True, + ) + recovered += 1 + except ServiceError: + continue + if parse_time(upload["expires_at"]) <= self._now(): + with self.store.tx() as con: + con.execute( + "UPDATE uploads SET status='expired' WHERE tenant_id=? AND upload_id=? AND status IN ('created','uploaded')", + (upload["tenant_id"], upload["upload_id"]), + ) + return recovered + + def _claim_lease(self) -> bool: + if self._clock_jump: + return False + now = self._now() + expires = now + timedelta(seconds=self._lease_ttl()) with self.store.tx() as con: row = con.execute( - "SELECT * FROM leases WHERE lease_name = 'scheduler'" + "SELECT * FROM leases WHERE lease_name='scheduler'" ).fetchone() if ( row @@ -1255,20 +2048,39 @@ class AgentCallService: ): return False con.execute( - "INSERT INTO leases(lease_name, owner_id, expires_at) VALUES('scheduler', ?, ?) ON CONFLICT(lease_name) DO UPDATE SET owner_id=excluded.owner_id, expires_at=excluded.expires_at", + """INSERT INTO leases(lease_name, owner_id, expires_at, generation) + VALUES('scheduler', ?, ?, 1) + ON CONFLICT(lease_name) DO UPDATE SET + owner_id=excluded.owner_id, expires_at=excluded.expires_at, + generation=leases.generation+1""", (self.instance_id, iso(expires)), ) return True + def _lease_owned_tx( + self, con: sqlite3.Connection, now: datetime | None = None + ) -> bool: + now = now or self._now() + row = con.execute( + "SELECT * FROM leases WHERE lease_name='scheduler'" + ).fetchone() + return bool( + row + and row["owner_id"] == self.instance_id + and parse_time(row["expires_at"]) > now + ) + def process_once(self) -> int: if not self._claim_lease(): return 0 + self.retry_published_commands() self.ingest_commands() self.apply_controls() scheduled = self.schedule_once() self.dispatch_outbox() self.consume_saas_events() self.process_replays() + self.recover_uploads() return scheduled def _active_counts(self, tenant_id: str | None = None) -> tuple[int, int]: @@ -1291,7 +2103,12 @@ class AgentCallService: "SELECT COUNT(*) AS n FROM calls WHERE cell_id = ? AND call_state NOT IN ('ended','failed')", (cell["cell_id"],), ) - if _integer(row["n"]) < _integer(cell.get("capacity", 1)): + capacity = min( + _integer(cell.get("capacity", 1)), + _integer(cell.get("media_capacity", cell.get("capacity", 1))), + _integer(cell.get("ai_capacity", cell.get("capacity", 1))), + ) + if _integer(row["n"]) < capacity: return cell["cell_id"], cell return None @@ -1314,10 +2131,11 @@ class AgentCallService: self._limits().get("global_cps", 3) ) and _integer(tenant_row["n"]) < _integer(self._limits().get("tenant_cps", 1)) - def _mark_waiting(self, command_id: str, reason: str) -> None: + def _mark_waiting(self, tenant_id: str, command_id: str, reason: str) -> None: with self.store.tx() as con: row = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (command_id,) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, command_id), ).fetchone() if row is None or row["status"] not in {"accepted", "waiting"}: return @@ -1326,18 +2144,20 @@ class AgentCallService: version = row["aggregate_version"] + 1 now = iso(self._now()) con.execute( - "UPDATE commands SET status='waiting', waiting_since=COALESCE(waiting_since, ?), wait_reason_code=?, aggregate_version=?, updated_at=? WHERE command_id=?", - (now, reason, version, now, command_id), + "UPDATE commands SET status='waiting', waiting_since=COALESCE(waiting_since, ?), wait_reason_code=?, aggregate_version=?, updated_at=? WHERE tenant_id=? AND command_id=?", + (now, reason, version, now, tenant_id, command_id), ) updated = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (command_id,) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, command_id), ).fetchone() self._command_event_tx(con, updated, reason) - def _reject_command(self, command_id: str, reason: str) -> None: + def _reject_command(self, tenant_id: str, command_id: str, reason: str) -> None: with self.store.tx() as con: row = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (command_id,) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, command_id), ).fetchone() if row is None or row["status"] in { "completed", @@ -1346,18 +2166,22 @@ class AgentCallService: "applied", }: return - version = row["aggregate_version"] + 1 now = iso(self._now()) con.execute( - "UPDATE commands SET status='rejected', reason_code=?, aggregate_version=?, updated_at=? WHERE command_id=?", - (reason, version, now, command_id), + "UPDATE commands SET status='rejected', reason_code=?, aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?", + (reason, now, tenant_id, command_id), ) updated = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (command_id,) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, command_id), ).fetchone() self._command_event_tx(con, updated, reason) def schedule_once(self) -> int: + if not self._claim_lease(): + return 0 + if not self.readiness()["ready"]: + return 0 scheduled = 0 if not self._tenant_order: return scheduled @@ -1377,13 +2201,15 @@ class AgentCallService: row["admission_deadline"] and parse_time(row["admission_deadline"]) <= now ): - self._reject_command(row["command_id"], "ADMISSION_TIMEOUT") + self._reject_command(tenant_id, row["command_id"], "ADMISSION_TIMEOUT") continue task = self.store.one( - "SELECT * FROM tasks WHERE task_id = ?", (row["task_id"],) + "SELECT * FROM tasks WHERE task_id = ? AND tenant_id = ?", + (row["task_id"], tenant_id), ) if task is None or task["state"] != "running": self._reject_command( + tenant_id, row["command_id"], f"TASK_{task['state'].upper() if task else 'MISSING'}", ) @@ -1399,30 +2225,32 @@ class AgentCallService: if active_tenant >= tenant_limit or active_global >= _integer( self._limits().get("global_concurrency", 6) ): - self._mark_waiting(row["command_id"], "CONCURRENCY") + self._mark_waiting(tenant_id, row["command_id"], "CONCURRENCY") continue if not self._cps_available(tenant_id): - self._mark_waiting(row["command_id"], "CPS") + self._mark_waiting(tenant_id, row["command_id"], "CPS") continue cell = self._resource_cell() if cell is None: - self._mark_waiting(row["command_id"], "CELL_CAPACITY") + self._mark_waiting(tenant_id, row["command_id"], "CELL_CAPACITY") + continue + if not self._reserve_call(row["command_id"], row, cell): + self._mark_waiting(tenant_id, row["command_id"], "RESOURCE_RACE") continue - self._reserve_call(row["command_id"], row, cell) scheduled += 1 self._cursor = (self._cursor + 1) % len(self._tenant_order) return scheduled def _reserve_call( self, command_id: str, row: sqlite3.Row, cell: tuple[str, dict[str, Any]] - ) -> None: + ) -> bool: cell_id, cell_cfg = cell payload = _loads(row["payload_json"])["payload"] route = self._route_config(payload["route_policy_id"]) prefix = str(route.get("dial_prefix", "")) target_uri = ( payload["callee"] - if prefix and payload["callee"].startswith(prefix) + if payload["callee"].startswith(prefix) else prefix + payload["callee"] ) caller = next( @@ -1435,18 +2263,83 @@ class AgentCallService: ) call_id = new_id("call") attempt_id = new_id("attempt") - now = iso(self._now()) + now_dt = self._now() + now = iso(now_dt) with self.store.tx() as con: + if not self._lease_owned_tx(con, now_dt): + return False current = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (command_id,) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (row["tenant_id"], command_id), ).fetchone() - if current is None or current["status"] not in {"accepted", "waiting"}: - return - version = current["aggregate_version"] + 1 - con.execute( - "UPDATE commands SET status='executing', call_id=?, waiting_since=NULL, wait_reason_code=NULL, aggregate_version=?, updated_at=? WHERE command_id=?", - (call_id, version, now, command_id), + task = con.execute( + "SELECT * FROM tasks WHERE tenant_id=? AND task_id=?", + (row["tenant_id"], row["task_id"]), + ).fetchone() + if ( + current is None + or current["status"] not in {"accepted", "waiting"} + or task is None + or task["state"] != "running" + ): + return False + tenant_active = con.execute( + "SELECT COUNT(*) AS n FROM calls WHERE tenant_id=? AND call_state NOT IN ('ended','failed')", + (row["tenant_id"],), + ).fetchone()["n"] + global_active = con.execute( + "SELECT COUNT(*) AS n FROM calls WHERE call_state NOT IN ('ended','failed')" + ).fetchone()["n"] + tenant_limit = con.execute( + "SELECT max_concurrency FROM tenants WHERE tenant_id=?", + (row["tenant_id"],), + ).fetchone()["max_concurrency"] + if _integer(tenant_active) >= _integer(tenant_limit) or _integer( + global_active + ) >= _integer(self._limits().get("global_concurrency", 6)): + return False + cell_active = con.execute( + "SELECT COUNT(*) AS n FROM calls WHERE cell_id=? AND call_state NOT IN ('ended','failed')", + (cell_id,), + ).fetchone()["n"] + cell_capacity = min( + _integer(cell_cfg.get("capacity", 1)), + _integer(cell_cfg.get("media_capacity", cell_cfg.get("capacity", 1))), + _integer(cell_cfg.get("ai_capacity", cell_cfg.get("capacity", 1))), ) + if _integer(cell_active) >= cell_capacity: + return False + route_active = con.execute( + "SELECT COUNT(*) AS n FROM calls WHERE trunk_id=? AND call_state NOT IN ('ended','failed')", + (route.get("trunk_id", "trunk-mock"),), + ).fetchone()["n"] + if _integer(route_active) >= _integer(route.get("max_concurrency", 10**9)): + return False + since = iso(now_dt - timedelta(seconds=1)) + global_cps = con.execute( + "SELECT COUNT(*) AS n FROM attempts WHERE created_at>=?", (since,) + ).fetchone()["n"] + tenant_cps = con.execute( + "SELECT COUNT(*) AS n FROM attempts a JOIN calls c ON c.call_id=a.call_id WHERE c.tenant_id=? AND a.created_at>=?", + (row["tenant_id"], since), + ).fetchone()["n"] + route_cps = con.execute( + "SELECT COUNT(*) AS n FROM attempts WHERE route_policy_id=? AND created_at>=?", + (payload["route_policy_id"], since), + ).fetchone()["n"] + if ( + _integer(global_cps) >= _integer(self._limits().get("global_cps", 3)) + or _integer(tenant_cps) >= _integer(self._limits().get("tenant_cps", 1)) + or _integer(route_cps) >= _integer(route.get("max_cps", 10**9)) + ): + return False + version = current["aggregate_version"] + 1 + changed = con.execute( + "UPDATE commands SET status='executing', call_id=?, waiting_since=NULL, wait_reason_code=NULL, aggregate_version=?, updated_at=? WHERE tenant_id=? AND command_id=? AND status IN ('accepted','waiting')", + (call_id, version, now, row["tenant_id"], command_id), + ).rowcount + if changed != 1: + return False con.execute( "INSERT INTO calls(call_id, tenant_id, execution_id, command_id, task_id, task_item_id, callee, cell_id, trunk_id, egress_pool_id, call_state, created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", ( @@ -1480,11 +2373,12 @@ class AgentCallService: ), ) con.execute( - "UPDATE executions SET status='executing', call_id=? WHERE execution_id=?", - (call_id, current["execution_id"]), + "UPDATE executions SET status='executing', call_id=? WHERE tenant_id=? AND execution_id=?", + (call_id, current["tenant_id"], current["execution_id"]), ) updated = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (command_id,) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (current["tenant_id"], command_id), ).fetchone() self._command_event_tx(con, updated, "EXECUTING") self._event_tx( @@ -1505,14 +2399,55 @@ class AgentCallService: }, now, ) - self.mock_sip.invite(call_id, payload["callee"], target_uri, caller) - self.mock_ari.channel_created(call_id, cell_id) - self.mock_ari.bridge_ready(call_id) - thread = threading.Thread( - target=self._run_call, args=(call_id,), name=f"call-{call_id}", daemon=True - ) - self._call_threads[call_id] = thread - thread.start() + try: + if not self._call_permitted(call_id, cell_id): + self._finish_call( + call_id, "failed", "ADMISSION_REVOKED", notify_protocol=False + ) + return True + self.mock_sip.invite(call_id, payload["callee"], target_uri, caller) + self.mock_ari.channel_created(call_id, cell_id) + self.mock_ari.bridge_ready(call_id) + thread = threading.Thread( + target=self._run_call, + args=(call_id,), + name=f"call-{call_id}", + daemon=True, + ) + self._call_threads[call_id] = thread + thread.start() + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + KeyError, + sqlite3.DatabaseError, + ServiceError, + ): + self._finish_call( + call_id, "failed", "PROTOCOL_SETUP_FAILED", notify_protocol=False + ) + return True + + def _call_permitted(self, call_id: str, cell_id: str) -> bool: + with self.store.tx() as con: + if not self._lease_owned_tx(con): + return False + row = con.execute( + """SELECT c.call_state, c.tenant_id, cmd.status, t.state + FROM calls c JOIN commands cmd ON cmd.tenant_id=c.tenant_id AND cmd.command_id=c.command_id + JOIN tasks t ON t.tenant_id=c.tenant_id AND t.task_id=c.task_id + WHERE c.call_id=? AND c.cell_id=?""", + (call_id, cell_id), + ).fetchone() + return bool( + row + and row["call_state"] == "queued" + and row["status"] == "executing" + and row["state"] == "running" + and self._cell_health.get(cell_id, False) + ) def set_failure(self, command_or_call_id: str, scenario: str) -> None: allowed = set(self.profile.get("failure_scenarios", [])) @@ -1561,12 +2496,19 @@ class AgentCallService: self._sleep(1 if scenario == "ai_timeout" else 0) if self._call_is_ended(call_id): return + if scenario == "ai_timeout": + self._finish_call(call_id, "failed", "AI_TIMEOUT") + return self._update_call_state(call_id, "answered", started=True) self.mock_sip.answered(call_id) self.mock_rtp.packet(call_id, "outbound", 1) + self._ensure_recording(call_id) self._sleep(_integer(self._limits().get("hold_ms", 0))) + if scenario == "customer_silent": + self._finish_call(call_id, "failed", "CUSTOMER_SILENT") + return if scenario == "ari_disconnect": - self._finish_call(call_id, "unknown", "ARI_DISCONNECTED_RECONCILING") + self._mark_reconciling(call_id, "ARI_DISCONNECTED_RECONCILING") return turns = _integer(self._limits().get("turns", 2)) for turn in range(turns): @@ -1590,10 +2532,78 @@ class AgentCallService: self._finish_call( call_id, "completed", "CALL_COMPLETED", upload_scenario=scenario ) - except Exception: + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + KeyError, + sqlite3.DatabaseError, + ServiceError, + ): if not self._call_is_ended(call_id): self._finish_call(call_id, "failed", "EXECUTOR_EXCEPTION") + def _mark_reconciling(self, call_id: str, reason: str) -> None: + now = iso(self._now()) + with self.store.tx() as con: + row = con.execute( + "SELECT * FROM calls WHERE call_id=?", (call_id,) + ).fetchone() + if row is None or row["call_state"] in {"ended", "failed"}: + return + version = row["call_version"] + 1 + con.execute( + "UPDATE calls SET call_state='reconciling', call_version=?, reason_code=? WHERE call_id=?", + (version, reason, call_id), + ) + con.execute( + "UPDATE commands SET status='reconciling', reason_code=?, aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=? AND status NOT IN ('completed','failed','rejected','applied')", + (reason, now, row["tenant_id"], row["command_id"]), + ) + con.execute( + "UPDATE executions SET status='reconciling' WHERE tenant_id=? AND execution_id=? AND status NOT IN ('completed','failed','rejected')", + (row["tenant_id"], row["execution_id"]), + ) + tenant = con.execute( + "SELECT tenant_key FROM tenants WHERE tenant_id=?", (row["tenant_id"],) + ).fetchone()["tenant_key"] + self._event_tx( + con, + row["tenant_id"], + tenant, + row["command_id"], + "call.status", + "call", + call_id, + version, + { + "call_id": call_id, + "execution_id": row["execution_id"], + "call_state": "reconciling", + "reason_code": reason, + }, + now, + ) + + def reconcile_call( + self, + call_id: str, + ended: bool, + outcome: str = "completed", + reason: str = "RECONCILED", + ) -> dict[str, Any]: + if not ended: + row = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,)) + if row is None: + raise NotFoundError() + return self.get_call(row["tenant_id"], call_id) + self._finish_call(call_id, outcome, reason) + row = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,)) + if row is None: + raise NotFoundError() + return self.get_call(row["tenant_id"], call_id) + def _update_attempt(self, call_id: str, status: str, started: bool = False) -> None: now = iso(self._now()) with self.store.tx() as con: @@ -1659,29 +2669,77 @@ class AgentCallService: ) def _add_transcript( - self, call_id: str, turn: int, speaker: str, text: str, playback: str + self, + call_id: str, + turn: int, + speaker: str, + text: str, + playback: str, + revision: int | None = None, + is_final: bool = True, ) -> None: + if not isinstance(text, str) or not text: + raise ValidationError("TRANSCRIPT_INVALID", "transcript text is required") + if type(is_final) is not bool: + raise ValidationError("TRANSCRIPT_INVALID", "is_final must be boolean") + try: + final_value = int(is_final) + except (TypeError, ValueError, OverflowError) as exc: + raise ValidationError("TRANSCRIPT_INVALID", "is_final is invalid") from exc now = iso(self._now()) segment_id = f"seg_{call_id}_{turn}_{speaker}" with self.store.tx() as con: call = con.execute( - "SELECT * FROM calls WHERE call_id = ?", (call_id,) + "SELECT * FROM calls WHERE call_id=?", (call_id,) ).fetchone() if call is None or call["call_state"] in {"ended", "failed"}: return previous = con.execute( - "SELECT * FROM transcripts WHERE segment_id = ?", (segment_id,) + "SELECT * FROM transcripts WHERE segment_id=?", (segment_id,) ).fetchone() - revision = (previous["revision"] + 1) if previous else 1 - if previous and previous["revision"] >= revision: + if ( + previous + and previous["text"] == text + and bool(previous["is_final"]) == is_final + and previous["playback_state"] == playback + ): + return + next_revision = ( + _integer(revision) + if revision is not None + else (previous["revision"] + 1 if previous else 1) + ) + if next_revision < 1: + raise ValidationError("TRANSCRIPT_INVALID", "revision must be positive") + if previous and previous["is_final"]: + if previous["text"] != text: + raise ConflictError( + "TRANSCRIPT_FINAL_CONFLICT", + "a final transcript segment cannot be overwritten", + ) + if not is_final: + return + if previous and next_revision <= previous["revision"]: return con.execute( - "INSERT INTO transcripts(segment_id, call_id, turn_index, speaker, text, revision, is_final, playback_state, updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(segment_id) DO UPDATE SET text=excluded.text, revision=excluded.revision, is_final=excluded.is_final, playback_state=excluded.playback_state, updated_at=excluded.updated_at", - (segment_id, call_id, turn, speaker, text, revision, 1, playback, now), + """INSERT INTO transcripts(segment_id, call_id, turn_index, speaker, text, revision, is_final, playback_state, updated_at) + VALUES(?,?,?,?,?,?,?,?,?) + ON CONFLICT(segment_id) DO UPDATE SET text=excluded.text, revision=excluded.revision, + is_final=excluded.is_final, playback_state=excluded.playback_state, updated_at=excluded.updated_at""", + ( + segment_id, + call_id, + turn, + speaker, + text, + next_revision, + final_value, + playback, + now, + ), ) tenant = con.execute( - "SELECT tenant_key FROM tenants WHERE tenant_id = ?", - (call["tenant_id"],), + "SELECT tenant_key FROM tenants WHERE tenant_id=?", (call["tenant_id"],) ).fetchone()["tenant_key"] self._event_tx( con, @@ -1691,15 +2749,15 @@ class AgentCallService: "transcript.updated", "transcript_segment", segment_id, - revision, + next_revision, { "call_id": call_id, "segment_id": segment_id, "turn_index": turn, "speaker": speaker, "text": text, - "revision": revision, - "is_final": True, + "revision": next_revision, + "is_final": is_final, "playback_state": playback, }, now, @@ -1712,6 +2770,7 @@ class AgentCallService: outcome: str, reason: str, upload_scenario: str | None = None, + notify_protocol: bool = True, ) -> None: now_dt = self._now() now = iso(now_dt) @@ -1734,22 +2793,31 @@ class AgentCallService: ("failed" if outcome == "failed" else "ended", now, call_id), ) command = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (row["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (row["tenant_id"], row["command_id"]), ).fetchone() command_status = ( "failed" if outcome == "failed" or outcome == "unknown" else "completed" ) command_version = command["aggregate_version"] + 1 con.execute( - "UPDATE commands SET status=?, reason_code=?, aggregate_version=?, updated_at=? WHERE command_id=?", - (command_status, reason, command_version, now, row["command_id"]), + "UPDATE commands SET status=?, reason_code=?, aggregate_version=?, updated_at=? WHERE tenant_id=? AND command_id=?", + ( + command_status, + reason, + command_version, + now, + row["tenant_id"], + row["command_id"], + ), ) con.execute( - "UPDATE executions SET status=?, call_id=? WHERE execution_id=?", - (command_status, call_id, row["execution_id"]), + "UPDATE executions SET status=?, call_id=? WHERE tenant_id=? AND execution_id=?", + (command_status, call_id, row["tenant_id"], row["execution_id"]), ) updated = con.execute( - "SELECT * FROM commands WHERE command_id = ?", (row["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (row["tenant_id"], row["command_id"]), ).fetchone() self._command_event_tx(con, updated, reason) tenant = con.execute( @@ -1774,108 +2842,172 @@ class AgentCallService: }, now, ) - self.mock_sip.bye(call_id, reason) - self.mock_ari.media_closed(call_id) - self.mock_ari.cleanup(call_id) - if outcome == "completed": + if notify_protocol: + self.mock_sip.bye(call_id, reason) + self.mock_ari.media_closed(call_id) + self.mock_ari.cleanup(call_id) + if outcome == "completed" or (row["started_at"] and outcome == "failed"): self._record_and_upload(call_id, upload_scenario or "success") - def _record_and_upload(self, call_id: str, scenario: str) -> None: - data = self._wav_bytes() - checksum = hashlib.sha256(data).hexdigest() - call = self.store.one("SELECT * FROM calls WHERE call_id = ?", (call_id,)) - if call is None: - return + def _ensure_recording(self, call_id: str) -> str | None: + call = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,)) + if call is None or not call["started_at"]: + return None + existing = self.store.one( + "SELECT recording_id FROM recordings WHERE tenant_id=? AND call_id=? ORDER BY created_at LIMIT 1", + (call["tenant_id"], call_id), + ) + if existing: + return existing["recording_id"] recording_id = new_id("rec") - upload_id = new_id("upload") - now = iso(self._now()) - duration_ms = max(1, _integer(self._limits().get("turns", 2)) * 100) - path = str(self.object_store.path_for(upload_id)) with self.store.tx() as con: con.execute( - "INSERT INTO recordings(recording_id, call_id, tenant_id, content_type, size_bytes, checksum_algorithm, checksum, channels, sample_rate_hz, duration_ms, status, upload_id, created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", + """INSERT INTO recordings( + recording_id, call_id, tenant_id, content_type, size_bytes, + checksum_algorithm, checksum, channels, sample_rate_hz, + duration_ms, status, created_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""", ( recording_id, call_id, call["tenant_id"], "audio/wav", - len(data), + 0, "SHA-256", - checksum, + "", 1, 16000, - duration_ms, - "uploading", - upload_id, - now, + 0, + "recording", + iso(self._now()), ), ) + return recording_id + + def _record_and_upload(self, call_id: str, scenario: str) -> None: + call = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,)) + if call is None or not call["started_at"]: + return + existing = self.store.one( + "SELECT * FROM recordings WHERE tenant_id=? AND call_id=? ORDER BY created_at LIMIT 1", + (call["tenant_id"], call_id), + ) + if existing and existing["status"] == "verified": + return + duration_ms = max( + 1, + _integer( + call["duration_ms"] or _integer(self._limits().get("turns", 2)) * 100 + ), + ) + data = self._wav_bytes(duration_ms) + checksum = hashlib.sha256(data).hexdigest() + recording_id = ( + existing["recording_id"] if existing else self._ensure_recording(call_id) + ) + if recording_id is None: + return + with self.store.tx() as con: con.execute( - "INSERT INTO uploads(upload_id, recording_id, tenant_id, expected_size, expected_checksum, expires_at, status, object_path, created_at) VALUES(?,?,?,?,?,?,?,?,?)", - ( - upload_id, - recording_id, - call["tenant_id"], - len(data), - checksum, - iso( - self._now() - + timedelta(seconds=self._limits().get("upload_ttl_s", 300)) - ), - "created", - path, - now, - ), + """UPDATE recordings SET content_type='audio/wav', size_bytes=?, + checksum_algorithm='SHA-256', checksum=?, channels=1, + sample_rate_hz=16000, duration_ms=?, status='uploading', + failure_code=NULL WHERE tenant_id=? AND recording_id=?""", + (len(data), checksum, duration_ms, call["tenant_id"], recording_id), ) if scenario == "upload_missing": - self._recording_failed(recording_id, "OBJECT_MISSING") + self._recording_failed(recording_id, "OBJECT_MISSING", call["tenant_id"]) return - token = self.object_store.token(upload_id, call["tenant_id"]) - upload_data = data - if scenario == "upload_bad_checksum": - upload_data = data[:-1] + bytes([data[-1] ^ 1]) + request = { + "recording_id": recording_id, + "call_id": call_id, + "content_type": "audio/wav", + "size_bytes": len(data), + "checksum_algorithm": "SHA-256", + "checksum": checksum, + "channels": 1, + "sample_rate_hz": 16000, + "duration_ms": duration_ms, + } try: - self.object_store.put(upload_id, call["tenant_id"], upload_data, token) - self._complete_upload( + _, session = self.create_upload( + call["tenant_id"], request, f"executor:{call_id}", "executor" + ) + upload_id = session["upload_id"] + token = session["required_headers"]["X-Mock-Upload-Token"] + upload_data = data + if scenario == "upload_bad_checksum": + upload_data = data[:-1] + bytes([data[-1] ^ 1]) + self.put_upload(upload_id, token, upload_data) + self.complete_upload( call["tenant_id"], upload_id, - recording_id, - checksum=checksum, - size_bytes=len(data), + { + "recording_id": recording_id, + "size_bytes": len(data), + "checksum_algorithm": "SHA-256", + "checksum": checksum, + }, + f"executor-complete:{call_id}", + "executor", ) except ServiceError as exc: - self._recording_failed(recording_id, exc.code) - - def _wav_bytes(self) -> bytes: - output = io.BytesIO() - with wave.open(output, "wb") as wav: - wav.setnchannels(1) - wav.setsampwidth(2) - wav.setframerate(16000) - wav.writeframes( - b"\0\0" * max(1600, _integer(self._limits().get("turns", 2)) * 1600) + self._recording_failed(recording_id, exc.code, call["tenant_id"]) + except (OSError, RuntimeError, TypeError, ValueError): + self._recording_failed( + recording_id, "OBJECT_UPLOAD_FAILED", call["tenant_id"] ) - return output.getvalue() - def _recording_failed(self, recording_id: str, code: str) -> None: + def _wav_bytes(self, duration_ms: int | None = None) -> bytes: + try: + duration_ms = max( + 1, + _integer(duration_ms or _integer(self._limits().get("turns", 2)) * 100), + ) + frames = max(1, int(16000 * duration_ms / 1000)) + output = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + samples = bytearray() + for index in range(frames): + seconds = index / 16000 + value = 11000 * math.sin( + 2 * math.pi * 440 * seconds + ) + 7000 * math.sin(2 * math.pi * 660 * seconds) + samples.extend( + struct.pack(" None: with self.store.tx() as con: recording = con.execute( - "SELECT * FROM recordings WHERE recording_id = ?", (recording_id,) + "SELECT * FROM recordings WHERE tenant_id=? AND recording_id=?", + (tenant_id, recording_id), ).fetchone() if recording is None or recording["status"] in {"verified", "failed"}: return - con.execute( - "UPDATE recordings SET status='failed', failure_code=? WHERE recording_id=?", - (code, recording_id), - ) + changed = con.execute( + "UPDATE recordings SET status='failed', failure_code=? WHERE tenant_id=? AND recording_id=? AND status NOT IN ('verified','failed')", + (code, recording["tenant_id"], recording_id), + ).rowcount tenant = con.execute( "SELECT tenant_key FROM tenants WHERE tenant_id = ?", (recording["tenant_id"],), ).fetchone()["tenant_key"] call = con.execute( - "SELECT command_id FROM calls WHERE call_id = ?", - (recording["call_id"],), + "SELECT command_id FROM calls WHERE tenant_id=? AND call_id=?", + (recording["tenant_id"], recording["call_id"]), ).fetchone() + if not changed or call is None: + return self._event_tx( con, recording["tenant_id"], @@ -1894,7 +3026,11 @@ class AgentCallService: ) def create_upload( - self, tenant_id: str, request: dict[str, Any] + self, + tenant_id: str, + request: dict[str, Any], + idempotency_key: str | None = None, + actor_id: str = "internal", ) -> tuple[int, dict[str, Any]]: self._tenant(tenant_id) required = { @@ -1908,34 +3044,100 @@ class AgentCallService: "sample_rate_hz", "duration_ms", } - if set(request) != required: + if ( + set(request) != required + or request.get("content_type") != "audio/wav" + or request.get("checksum_algorithm") != "SHA-256" + ): raise ValidationError( "SCHEMA_INVALID", "upload request fields do not match the contract" ) + if ( + type(request.get("size_bytes")) is not int + or request["size_bytes"] < 1 + or request["size_bytes"] + > _integer(self._limits().get("max_recording_bytes", 64 * 1024 * 1024)) + ): + raise ValidationError( + "SCHEMA_INVALID", "size_bytes is outside the allowed range" + ) + if ( + type(request.get("channels")) is not int + or request["channels"] != 1 + or type(request.get("sample_rate_hz")) is not int + or not 8000 <= request["sample_rate_hz"] <= 48000 + or type(request.get("duration_ms")) is not int + or not 1 <= request["duration_ms"] <= 3600000 + ): + raise ValidationError( + "SCHEMA_INVALID", "audio metadata is outside the allowed range" + ) + if not isinstance(request.get("checksum"), str) or not re.fullmatch( + r"[0-9a-f]{64}", request["checksum"] + ): + raise ValidationError( + "SCHEMA_INVALID", "checksum must be a SHA-256 hex digest" + ) + idempotency_key = idempotency_key or f"internal:{request['recording_id']}" + operation_hash = self._operation_hash( + "recording.create", request["recording_id"], request + ) + existing_operation = self.store.one( + "SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?", + ( + tenant_id, + actor_id, + "recording.create", + request["recording_id"], + idempotency_key, + ), + ) + if existing_operation: + if existing_operation["payload_hash"] != operation_hash: + raise ConflictError( + "IDEMPOTENCY_CONFLICT", + "upload request was already submitted with a different body", + ) + return existing_operation["status"], _loads( + existing_operation["response_json"] + ) recording = self.store.one( - "SELECT * FROM recordings WHERE recording_id = ? AND tenant_id = ?", + "SELECT * FROM recordings WHERE recording_id=? AND tenant_id=?", (request["recording_id"], tenant_id), ) if recording is None or recording["call_id"] != request["call_id"]: raise NotFoundError() - if ( - recording["size_bytes"] != request["size_bytes"] - or recording["checksum"] != request["checksum"] - ): + metadata = ( + "content_type", + "size_bytes", + "checksum_algorithm", + "checksum", + "channels", + "sample_rate_hz", + "duration_ms", + ) + if any(recording[field] != request[field] for field in metadata): raise ConflictError( "RECORDING_METADATA_CONFLICT", "recording metadata is immutable" ) upload = self.store.one( - "SELECT * FROM uploads WHERE recording_id = ?", (request["recording_id"],) + "SELECT * FROM uploads WHERE tenant_id=? AND recording_id=? ORDER BY created_at DESC LIMIT 1", + (tenant_id, request["recording_id"]), ) - if upload: - status = 200 - upload_id = upload["upload_id"] - expires = upload["expires_at"] + if upload and ( + upload["status"] == "verified" + or ( + parse_time(upload["expires_at"]) > self._now() + and upload["status"] in {"created", "uploaded"} + ) + ): + status, upload_id, expires = 200, upload["upload_id"], upload["expires_at"] else: + status = 201 upload_id = new_id("upload") expires = iso( - self._now() + timedelta(seconds=self._limits().get("upload_ttl_s", 300)) + self._now() + + timedelta(seconds=_integer(self._limits().get("upload_ttl_s", 300))) ) path = str(self.object_store.path_for(upload_id)) with self.store.tx() as con: @@ -1954,11 +3156,10 @@ class AgentCallService: ), ) con.execute( - "UPDATE recordings SET upload_id=?, status='uploading' WHERE recording_id=?", - (upload_id, request["recording_id"]), + "UPDATE recordings SET upload_id=?, status='uploading' WHERE tenant_id=? AND recording_id=?", + (upload_id, tenant_id, request["recording_id"]), ) - status = 201 - return status, { + response = { "upload_id": upload_id, "recording_id": request["recording_id"], "expires_at": expires, @@ -1973,19 +3174,98 @@ class AgentCallService: "checksum": request["checksum"], }, } + cleanup_upload_id: str | None = None + with self.store.tx() as con: + inserted = con.execute( + """INSERT OR IGNORE INTO storage_operations(tenant_id, actor_id, operation_type, target_id, idempotency_key, payload_hash, status, response_json, created_at, updated_at) + VALUES(?,?,?,?,?,?,?,?,?,?)""", + ( + tenant_id, + actor_id, + "recording.create", + request["recording_id"], + idempotency_key, + operation_hash, + status, + canonical(response), + iso(self._now()), + iso(self._now()), + ), + ).rowcount + if not inserted: + existing_operation = con.execute( + "SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?", + ( + tenant_id, + actor_id, + "recording.create", + request["recording_id"], + idempotency_key, + ), + ).fetchone() + if ( + existing_operation is None + or existing_operation["payload_hash"] != operation_hash + ): + raise ConflictError( + "IDEMPOTENCY_CONFLICT", + "upload request was already submitted with a different body", + ) + winner = _loads(existing_operation["response_json"]) + if upload_id != winner["upload_id"]: + con.execute( + "DELETE FROM uploads WHERE tenant_id=? AND upload_id=? AND recording_id=? AND status='created'", + (tenant_id, upload_id, request["recording_id"]), + ) + con.execute( + "UPDATE recordings SET upload_id=? WHERE tenant_id=? AND recording_id=?", + (winner["upload_id"], tenant_id, request["recording_id"]), + ) + cleanup_upload_id = upload_id + result = (existing_operation["status"], winner) + else: + result = (status, response) + if cleanup_upload_id: + with suppress(OSError): + self.object_store.path_for(cleanup_upload_id).unlink() + return result def put_upload(self, upload_id: str, token: str, data: bytes) -> None: - upload = self.store.one( - "SELECT * FROM uploads WHERE upload_id = ?", (upload_id,) - ) + upload = self.store.one("SELECT * FROM uploads WHERE upload_id=?", (upload_id,)) if upload is None: raise NotFoundError() + if not isinstance(data, bytes) or len(data) > _integer( + self._limits().get("max_recording_bytes", 64 * 1024 * 1024) + ): + raise ValidationError( + "OBJECT_BODY_INVALID", "upload body is outside the allowed range" + ) if parse_time(upload["expires_at"]) <= self._now(): raise ServiceError("UPLOAD_EXPIRED", "upload session has expired", 410) + if upload["status"] == "verified" or self.object_store.has_final(upload_id): + raise ConflictError( + "UPLOAD_ALREADY_FINALIZED", "verified upload objects are immutable" + ) + if ( + upload["status"] == "uploaded" + and self.object_store.path_for(upload_id).is_file() + ): + existing = self.object_store.path_for(upload_id).read_bytes() + if ( + len(existing) == len(data) + and hashlib.sha256(existing).hexdigest() + == hashlib.sha256(data).hexdigest() + ): + return + raise ConflictError( + "UPLOAD_ALREADY_RECEIVED", + "upload session already contains different bytes", + ) self.object_store.put(upload_id, upload["tenant_id"], data, token) with self.store.tx() as con: con.execute( - "UPDATE uploads SET status='uploaded' WHERE upload_id=?", (upload_id,) + "UPDATE uploads SET status='uploaded' WHERE tenant_id=? AND upload_id=?", + (upload["tenant_id"], upload_id), ) def _complete_upload( @@ -1995,17 +3275,36 @@ class AgentCallService: recording_id: str, checksum: str, size_bytes: int, + recovery: bool = False, ) -> dict[str, Any]: upload = self.store.one( - "SELECT * FROM uploads WHERE upload_id = ? AND tenant_id = ?", + "SELECT * FROM uploads WHERE upload_id=? AND tenant_id=?", (upload_id, tenant_id), ) if upload is None or upload["recording_id"] != recording_id: raise NotFoundError() - if parse_time(upload["expires_at"]) <= self._now(): + recording = self.store.one( + "SELECT * FROM recordings WHERE recording_id=? AND tenant_id=?", + (recording_id, tenant_id), + ) + if recording is None: + raise NotFoundError() + if upload["status"] == "verified" and recording["status"] == "verified": + return { + "upload_id": upload_id, + "recording_id": recording_id, + "status": "verified", + "oss_id": recording["oss_id"], + "verified_at": recording["verified_at"], + } + if parse_time(upload["expires_at"]) <= self._now() and not ( + recovery and self.object_store.has_final(upload_id) + ): raise ServiceError("UPLOAD_EXPIRED", "upload session has expired", 410) if ( - size_bytes != upload["expected_size"] + type(size_bytes) is not int + or not isinstance(checksum, str) + or size_bytes != upload["expected_size"] or checksum != upload["expected_checksum"] ): raise ValidationError( @@ -2017,56 +3316,65 @@ class AgentCallService: upload_id, upload["expected_size"], upload["expected_checksum"] ) except ValidationError: - self._recording_failed(recording_id, "OBJECT_VERIFICATION_FAILED") + self._recording_failed( + recording_id, "OBJECT_VERIFICATION_FAILED", tenant_id + ) raise now = iso(self._now()) with self.store.tx() as con: con.execute( - "UPDATE uploads SET status='verified', completed_at=? WHERE upload_id=?", - (now, upload_id), - ) - con.execute( - "UPDATE recordings SET status='verified', oss_id=?, verified_at=? WHERE recording_id=?", - (oss_id, now, recording_id), + "UPDATE uploads SET status='verified', completed_at=? WHERE tenant_id=? AND upload_id=?", + (now, tenant_id, upload_id), ) + changed = con.execute( + "UPDATE recordings SET status='verified', oss_id=?, verified_at=?, failure_code=NULL WHERE tenant_id=? AND recording_id=? AND status!='verified'", + (oss_id, now, tenant_id, recording_id), + ).rowcount recording = con.execute( - "SELECT * FROM recordings WHERE recording_id = ?", (recording_id,) + "SELECT * FROM recordings WHERE tenant_id=? AND recording_id=?", + (tenant_id, recording_id), ).fetchone() call = con.execute( - "SELECT command_id FROM calls WHERE call_id = ?", - (recording["call_id"],), + "SELECT command_id FROM calls WHERE tenant_id=? AND call_id=?", + (tenant_id, recording["call_id"]), ).fetchone() tenant = con.execute( - "SELECT tenant_key FROM tenants WHERE tenant_id = ?", (tenant_id,) + "SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,) ).fetchone()["tenant_key"] - self._event_tx( - con, - tenant_id, - tenant, - call["command_id"], - "recording.ready", - "recording", - recording_id, - 1, - { - "recording_id": recording_id, - "call_id": recording["call_id"], - "oss_id": oss_id, - "size_bytes": recording["size_bytes"], - "checksum": recording["checksum"], - }, - now, - ) + if changed: + self._event_tx( + con, + tenant_id, + tenant, + call["command_id"], + "recording.ready", + "recording", + recording_id, + 1, + { + "recording_id": recording_id, + "call_id": recording["call_id"], + "oss_id": oss_id, + "size_bytes": recording["size_bytes"], + "checksum": recording["checksum"], + }, + now, + ) return { "upload_id": upload_id, "recording_id": recording_id, "status": "verified", "oss_id": oss_id, - "verified_at": now, + "verified_at": recording["verified_at"], } def complete_upload( - self, tenant_id: str, upload_id: str, request: dict[str, Any] + self, + tenant_id: str, + upload_id: str, + request: dict[str, Any], + idempotency_key: str | None = None, + actor_id: str = "internal", ) -> dict[str, Any]: required = {"recording_id", "size_bytes", "checksum_algorithm", "checksum"} allowed = required | {"etag"} @@ -2074,17 +3382,66 @@ class AgentCallService: not required.issubset(request) or set(request) - allowed or request.get("checksum_algorithm") != "SHA-256" + or type(request.get("size_bytes")) is not int + or not isinstance(request.get("checksum"), str) ): raise ValidationError( "SCHEMA_INVALID", "complete request fields do not match the contract" ) - return self._complete_upload( + idempotency_key = idempotency_key or f"internal:{upload_id}" + operation_hash = self._operation_hash("recording.complete", upload_id, request) + existing = self.store.one( + "SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?", + (tenant_id, actor_id, "recording.complete", upload_id, idempotency_key), + ) + if existing: + if existing["payload_hash"] != operation_hash: + raise ConflictError( + "IDEMPOTENCY_CONFLICT", + "upload completion was already submitted with a different body", + ) + return _loads(existing["response_json"]) + response = self._complete_upload( tenant_id, upload_id, request["recording_id"], request["checksum"], request["size_bytes"], ) + with self.store.tx() as con: + inserted = con.execute( + "INSERT OR IGNORE INTO storage_operations(tenant_id, actor_id, operation_type, target_id, idempotency_key, payload_hash, status, response_json, created_at, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)", + ( + tenant_id, + actor_id, + "recording.complete", + upload_id, + idempotency_key, + operation_hash, + 200, + canonical(response), + iso(self._now()), + iso(self._now()), + ), + ).rowcount + if not inserted: + existing = con.execute( + "SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?", + ( + tenant_id, + actor_id, + "recording.complete", + upload_id, + idempotency_key, + ), + ).fetchone() + if existing is None or existing["payload_hash"] != operation_hash: + raise ConflictError( + "IDEMPOTENCY_CONFLICT", + "upload completion was already submitted with a different body", + ) + return _loads(existing["response_json"]) + return response def dispatch_outbox(self, limit: int = 100) -> int: now = iso(self._now()) @@ -2109,14 +3466,14 @@ class AgentCallService: self._now() + timedelta(seconds=min(60, 2 ** min(attempts, 5))) ) con.execute( - "UPDATE events SET broker_attempts=?, broker_status='failed', next_attempt_at=? WHERE event_id=?", - (attempts, retry_at, row["event_id"]), + "UPDATE events SET broker_attempts=?, broker_status='failed', next_attempt_at=? WHERE tenant_id=? AND event_id=?", + (attempts, retry_at, row["tenant_id"], row["event_id"]), ) continue with self.store.tx() as con: con.execute( - "UPDATE events SET published=1, broker_status='broker_confirmed', published_at=?, broker_attempts=broker_attempts+1 WHERE event_id=?", - (iso(self._now()), row["event_id"]), + "UPDATE events SET published=1, broker_status='broker_confirmed', published_at=?, broker_attempts=broker_attempts+1 WHERE tenant_id=? AND event_id=?", + (iso(self._now()), row["tenant_id"], row["event_id"]), ) sent += 1 return sent @@ -2134,42 +3491,68 @@ class AgentCallService: body = body_raw if isinstance(body_raw, dict) else {} event_id = body.get("event_id") if not event_id: - self.broker.ack(message) + self._dead_letter(message, body, "INBOX_INVALID") + reject = getattr(self.broker, "reject", None) + if callable(reject): + reject(message) + else: + self.broker.ack(message) continue try: self._validate_event(body) + tenant = self._tenant(body["tenant_id"], body["tenant_key"]) + expected_route = f"agent-call.{body['event_type']}" + if message.get("routing_key") not in {None, expected_route}: + raise ValidationError( + "EVENT_ROUTE_MISMATCH", "event routing key is invalid" + ) + if message.get("message_id") not in {None, event_id}: + raise ValidationError( + "EVENT_ID_MISMATCH", "event message id is invalid" + ) + if body["event_type"] == "recording.ready": + recording_payload = body.get("payload", {}) + recording = self.store.one( + "SELECT status, oss_id FROM recordings WHERE tenant_id=? AND recording_id=?", + (tenant["tenant_id"], recording_payload.get("recording_id")), + ) + if ( + recording is None + or recording["status"] != "verified" + or recording["oss_id"] != recording_payload.get("oss_id") + ): + raise ValidationError( + "ASSET_NOT_VERIFIED", + "recording.ready is not independently verified", + ) with self.store.tx() as con: exists = con.execute( - "SELECT 1 FROM inbox WHERE event_id = ?", (event_id,) + "SELECT 1 FROM inbox WHERE tenant_id=? AND event_id=?", + (tenant["tenant_id"], event_id), ).fetchone() if not exists: con.execute( "INSERT INTO inbox(event_id, tenant_id, applied_at, body_json) VALUES(?,?,?,?)", ( event_id, - body["tenant_id"], + tenant["tenant_id"], iso(self._now()), canonical(body), ), ) applied += 1 con.execute( - "UPDATE events SET saas_applied=1 WHERE event_id=?", (event_id,) - ) - self.broker.ack(message) - except (KeyError, ServiceError, sqlite3.IntegrityError): - with self.store.tx() as con: - con.execute( - "INSERT INTO dead_letters(tenant_key, routing_key, body_json, reason_code, created_at) VALUES(?,?,?,?,?)", - ( - body.get("tenant_key"), - message.get("routing_key"), - canonical(body), - "INBOX_INVALID", - iso(self._now()), - ), + "UPDATE events SET saas_applied=1 WHERE tenant_id=? AND event_id=?", + (tenant["tenant_id"], event_id), ) self.broker.ack(message) + except (KeyError, ServiceError, sqlite3.IntegrityError) as exc: + self._dead_letter(message, body, getattr(exc, "code", "INBOX_INVALID")) + reject = getattr(self.broker, "reject", None) + if callable(reject): + reject(message) + else: + self.broker.ack(message) return applied def apply_controls(self) -> int: @@ -2177,54 +3560,73 @@ class AgentCallService: "SELECT * FROM commands WHERE command_type='task.control' AND status='applying' ORDER BY accepted_at" ) count = 0 + cell_ids = [str(cell["cell_id"]) for cell in self.profile.get("cells", [])] for row in rows: body = _loads(row["payload_json"]) action = body["action"] policy = body.get("active_call_policy", "drain") + if not all(self._cell_health.get(cell_id, False) for cell_id in cell_ids): + continue if action == "stop" and policy == "hangup": calls = self.store.all( - "SELECT call_id FROM calls WHERE task_id=? AND call_state NOT IN ('ended','failed')", - (row["task_id"],), + "SELECT call_id FROM calls WHERE tenant_id=? AND task_id=? AND call_state NOT IN ('ended','failed')", + (row["tenant_id"], row["task_id"]), ) for call in calls: self._finish_controlled_call(call["call_id"]) with self.store.tx() as con: task = con.execute( - "SELECT * FROM tasks WHERE task_id=?", (row["task_id"],) + "SELECT * FROM tasks WHERE tenant_id=? AND task_id=?", + (row["tenant_id"], row["task_id"]), ).fetchone() if task is None: continue now = iso(self._now()) + for cell_id in cell_ids: + con.execute( + "INSERT INTO control_barriers(command_id, tenant_id, cell_id, status, applied_at) VALUES(?,?,?,?,?) ON CONFLICT(tenant_id, command_id, cell_id) DO UPDATE SET status=excluded.status, applied_at=excluded.applied_at", + (row["command_id"], row["tenant_id"], cell_id, "applied", now), + ) con.execute( - "UPDATE commands SET status='applied', applied_task_revision=?, task_state=?, aggregate_version=aggregate_version+1, updated_at=? WHERE command_id=?", - (task["revision"], task["state"], now, row["command_id"]), + "UPDATE commands SET status='applied', applied_task_revision=?, task_state=?, aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=? AND status='applying'", + ( + task["revision"], + task["state"], + now, + row["tenant_id"], + row["command_id"], + ), ) updated = con.execute( - "SELECT * FROM commands WHERE command_id=?", (row["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (row["tenant_id"], row["command_id"]), ).fetchone() self._command_event_tx(con, updated, "CONTROL_APPLIED") affected = con.execute( - "SELECT * FROM commands WHERE task_id=? AND status IN ('accepted','waiting')", - (row["task_id"],), + "SELECT * FROM commands WHERE tenant_id=? AND task_id=? AND status IN ('accepted','waiting')", + (row["tenant_id"], row["task_id"]), ).fetchall() for affected_row in affected: con.execute( - "UPDATE commands SET status='rejected', reason_code='TASK_CONTROL_BARRIER', aggregate_version=aggregate_version+1, updated_at=? WHERE command_id=?", - (now, affected_row["command_id"]), + "UPDATE commands SET status='rejected', reason_code='TASK_CONTROL_BARRIER', aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?", + (now, affected_row["tenant_id"], affected_row["command_id"]), ) con.execute( - "UPDATE executions SET status='rejected' WHERE command_id=?", - (affected_row["command_id"],), + "UPDATE executions SET status='rejected' WHERE tenant_id=? AND execution_id=?", + (affected_row["tenant_id"], affected_row["execution_id"]), ) barrier_row = con.execute( - "SELECT * FROM commands WHERE command_id=?", - (affected_row["command_id"],), + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (affected_row["tenant_id"], affected_row["command_id"]), ).fetchone() self._command_event_tx(con, barrier_row, "TASK_CONTROL_BARRIER") count += 1 return count def _finish_controlled_call(self, call_id: str) -> None: + row = self.store.one("SELECT call_state FROM calls WHERE call_id=?", (call_id,)) + if row is not None and row["call_state"] == "reconciling": + return self._finish_call(call_id, "failed", "CONTROL_STOPPED") def control_task( @@ -2235,7 +3637,7 @@ class AgentCallService: idempotency_key: str, allow_hangup: bool = False, ) -> dict[str, Any]: - self._tenant(tenant_id) + tenant = self._tenant(tenant_id) required = {"command_id", "action", "expected_task_revision", "reason"} allowed = required | {"active_call_policy"} if not required.issubset(body) or set(body) - allowed: @@ -2245,14 +3647,20 @@ class AgentCallService: if body["action"] not in {"pause", "resume", "stop"}: raise ValidationError("SCHEMA_INVALID", "control action is invalid") if ( - not isinstance(body["expected_task_revision"], int) + type(body["expected_task_revision"]) is not int or body["expected_task_revision"] < 1 ): raise ValidationError( "SCHEMA_INVALID", "expected_task_revision must be a positive integer" ) - if not isinstance(body["reason"], str) or not body["reason"]: - raise ValidationError("SCHEMA_INVALID", "reason is required") + if ( + not isinstance(body["reason"], str) + or not body["reason"] + or len(body["reason"]) > 512 + ): + raise ValidationError( + "SCHEMA_INVALID", "reason must contain 1..512 characters" + ) if body["action"] != "stop" and "active_call_policy" in body: raise ValidationError( "SCHEMA_INVALID", "active_call_policy is only valid for stop" @@ -2276,16 +3684,11 @@ class AgentCallService: ) if task is None: raise NotFoundError() - payload_hash = digest(body) + payload_hash = self._operation_hash("task.control", task_id, body) existing = self.store.one( "SELECT * FROM commands WHERE command_id=? AND tenant_id=?", (body["command_id"], tenant_id), ) - foreign = self.store.one( - "SELECT 1 FROM commands WHERE command_id=?", (body["command_id"],) - ) - if foreign and existing is None: - raise ConflictError("IDEMPOTENCY_CONFLICT", "command id is unavailable") if existing: if existing["payload_hash"] != payload_hash: raise ConflictError( @@ -2298,10 +3701,20 @@ class AgentCallService: "TASK_REVISION_CONFLICT", "expected_task_revision does not match current task revision", ) - if body["action"] == "resume" and task["state"] != "paused": - raise ConflictError( - "TASK_NOT_RESUMABLE", "only a paused task can be resumed" + if body["action"] == "resume": + applying = self.store.one( + "SELECT 1 FROM commands WHERE tenant_id=? AND task_id=? AND command_type='task.control' AND status='applying' LIMIT 1", + (tenant_id, task_id), ) + if applying: + raise ConflictError( + "CONTROL_IN_PROGRESS", + "the previous control barrier has not applied", + ) + if task["state"] != "paused": + raise ConflictError( + "TASK_NOT_RESUMABLE", "only a paused task can be resumed" + ) if body["action"] in {"pause", "stop"} and task["state"] == "stopped": raise ConflictError("TASK_STOPPED", "stopped task cannot be changed") new_state = {"pause": "paused", "resume": "running", "stop": "stopped"}[ @@ -2310,25 +3723,30 @@ class AgentCallService: now = iso(self._now()) with self.store.tx() as con: con.execute( - "UPDATE tasks SET state=?, revision=revision+1, updated_at=? WHERE task_id=? AND revision=?", - (new_state, now, task_id, body["expected_task_revision"]), + "UPDATE tasks SET state=?, revision=revision+1, updated_at=? WHERE tenant_id=? AND task_id=? AND revision=?", + (new_state, now, tenant_id, task_id, body["expected_task_revision"]), ) if con.execute("SELECT changes()").fetchone()[0] != 1: raise ConflictError( "TASK_REVISION_CONFLICT", "task changed concurrently" ) con.execute( - "INSERT INTO commands(command_id, tenant_id, tenant_key, command_type, status, payload_json, payload_hash, task_id, requested_task_revision, task_state, accepted_at, trace_id, aggregate_version, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + """INSERT INTO commands( + command_id, tenant_id, tenant_key, command_type, status, + payload_json, payload_hash, operation_type, operation_target, + task_id, requested_task_revision, task_state, accepted_at, + trace_id, aggregate_version, updated_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", ( body["command_id"], tenant_id, - self.store.one( - "SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,) - )["tenant_key"], + tenant["tenant_key"], "task.control", "applying", canonical(body), payload_hash, + "task.control", + task_id, task_id, body["expected_task_revision"], new_state, @@ -2339,12 +3757,14 @@ class AgentCallService: ), ) row = con.execute( - "SELECT * FROM commands WHERE command_id=?", (body["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, body["command_id"]), ).fetchone() self._command_event_tx(con, row, "CONTROL_ACCEPTED") return self._control_response( self.store.one( - "SELECT * FROM commands WHERE command_id=?", (body["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, body["command_id"]), ) ) @@ -2359,6 +3779,26 @@ class AgentCallService: "accepted_at": row["accepted_at"], } + def _events_for_call( + self, tenant_id: str, call_id: str, cutoff: str | None = None + ) -> list[sqlite3.Row]: + sql = "SELECT * FROM events WHERE tenant_id=?" + params: list[Any] = [tenant_id] + if cutoff is not None: + sql += " AND occurred_at<=?" + params.append(cutoff) + sql += " ORDER BY occurred_at, event_id" + rows = self.store.all(sql, tuple(params)) + matched: list[sqlite3.Row] = [] + for row in rows: + if row["aggregate_type"] == "call" and row["aggregate_id"] == call_id: + matched.append(row) + continue + payload = _loads(row["payload_json"], {}) + if isinstance(payload, dict) and payload.get("call_id") == call_id: + matched.append(row) + return matched + def create_replay( self, tenant_id: str, @@ -2368,9 +3808,14 @@ class AgentCallService: idempotency_key: str, ) -> dict[str, Any]: self._tenant(tenant_id) + if source_type not in {"call", "command"}: + raise ValidationError("SCHEMA_INVALID", "replay source type is invalid") if ( set(body) != {"command_id", "reason"} or body["command_id"] != idempotency_key + or not isinstance(body["reason"], str) + or not body["reason"] + or len(body["reason"]) > 512 ): raise ValidationError( "SCHEMA_INVALID", @@ -2380,13 +3825,11 @@ class AgentCallService: "SELECT * FROM commands WHERE command_id=? AND tenant_id=?", (body["command_id"], tenant_id), ) - foreign = self.store.one( - "SELECT 1 FROM commands WHERE command_id=?", (body["command_id"],) + operation_hash = self._operation_hash( + "replay", f"{source_type}:{source_id}", body ) - if foreign and existing is None: - raise ConflictError("IDEMPOTENCY_CONFLICT", "command id is unavailable") if existing: - if existing["payload_hash"] != digest(body): + if existing["payload_hash"] != operation_hash: raise ConflictError( "IDEMPOTENCY_CONFLICT", "replay command was already submitted with a different body", @@ -2412,9 +3855,13 @@ class AgentCallService: retention = self._now() - timedelta( seconds=self._limits().get("replay_retention_s", 604800) ) - rows = self.store.all( - "SELECT occurred_at FROM events WHERE tenant_id=? ORDER BY occurred_at LIMIT 1", - (tenant_id,), + rows = ( + self._events_for_call(tenant_id, source_id, cutoff) + if source_type == "call" + else self.store.all( + "SELECT occurred_at FROM events WHERE tenant_id=? AND aggregate_id=? AND occurred_at<=? ORDER BY occurred_at LIMIT 1", + (tenant_id, source_id, cutoff), + ) ) if rows and parse_time(rows[0]["occurred_at"]) < retention: raise ServiceError( @@ -2427,7 +3874,12 @@ class AgentCallService: now = iso(self._now()) with self.store.tx() as con: con.execute( - "INSERT INTO commands(command_id, tenant_id, tenant_key, command_type, status, payload_json, payload_hash, snapshot_cutoff, source_type, source_id, accepted_at, trace_id, aggregate_version, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + """INSERT INTO commands( + command_id, tenant_id, tenant_key, command_type, status, + payload_json, payload_hash, operation_type, operation_target, + snapshot_cutoff, source_type, source_id, accepted_at, trace_id, + aggregate_version, updated_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", ( body["command_id"], tenant_id, @@ -2435,7 +3887,9 @@ class AgentCallService: "replay", "accepted", canonical(body), - digest(body), + operation_hash, + "replay", + f"{source_type}:{source_id}", cutoff, source_type, source_id, @@ -2460,7 +3914,8 @@ class AgentCallService: ), ) row = con.execute( - "SELECT * FROM commands WHERE command_id=?", (body["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (tenant_id, body["command_id"]), ).fetchone() self._command_event_tx(con, row, "REPLAY_ACCEPTED") return { @@ -2470,70 +3925,115 @@ class AgentCallService: } def process_replays(self) -> int: + now = iso(self._now()) rows = self.store.all( - "SELECT * FROM replays WHERE status='accepted' ORDER BY created_at LIMIT 10" + "SELECT * FROM replays WHERE status='accepted' AND (next_attempt_at IS NULL OR next_attempt_at<=?) ORDER BY created_at LIMIT 10", + (now,), ) count = 0 + max_attempts = max(1, _integer(self._limits().get("max_replay_attempts", 6))) for replay in rows: + tenant_row = self.store.one( + "SELECT tenant_key FROM tenants WHERE tenant_id=?", + (replay["tenant_id"],), + ) + if tenant_row is None: + continue + tenant_key = tenant_row["tenant_key"] if replay["source_type"] == "call": - candidates = self.store.all( - "SELECT * FROM events WHERE tenant_id=? AND occurred_at <= ? ORDER BY occurred_at, event_id", - (replay["tenant_id"], replay["snapshot_cutoff"]), + candidates = self._events_for_call( + replay["tenant_id"], replay["source_id"], replay["snapshot_cutoff"] ) else: candidates = self.store.all( - "SELECT * FROM events WHERE tenant_id=? AND aggregate_id=? AND occurred_at <= ? ORDER BY occurred_at, event_id", + "SELECT * FROM events WHERE tenant_id=? AND aggregate_id=? AND occurred_at<=? ORDER BY occurred_at, event_id", ( replay["tenant_id"], replay["source_id"], replay["snapshot_cutoff"], ), ) - sent = 0 + sent = _integer(replay["sent_count"]) + already = { + item["event_id"] + for item in self.store.all( + "SELECT event_id FROM replay_items WHERE replay_id=?", + (replay["replay_id"],), + ) + } try: for event in candidates: - body = _loads(event["body_json"]) - if ( - replay["source_type"] == "call" - and body.get("payload", {}).get("call_id") - != replay["source_id"] - and body.get("aggregate_id") != replay["source_id"] - ): + if event["event_id"] in already: continue self.broker.publish( "agent-call.events.v1", f"agent-call.{event['event_type']}", - body, + _loads(event["body_json"]), event["event_id"], - {"x-replay-id": replay["replay_id"]}, + { + "x-replay-id": replay["replay_id"], + "tenant_id": replay["tenant_id"], + "tenant_key": tenant_key, + }, ) with self.store.tx() as con: con.execute( "INSERT OR IGNORE INTO replay_items(replay_id, event_id, published_at) VALUES(?,?,?)", (replay["replay_id"], event["event_id"], iso(self._now())), ) + already.add(event["event_id"]) sent += 1 except BrokerError: + attempts = _integer(replay["retry_attempts"]) + 1 + status = "failed" if attempts >= max_attempts else "accepted" with self.store.tx() as con: - con.execute( - "UPDATE replays SET status='failed', failure_code='BROKER_UNAVAILABLE', updated_at=? WHERE replay_id=?", - (iso(self._now()), replay["replay_id"]), + updated_at = iso(self._now()) + next_attempt = ( + None + if status == "failed" + else iso( + self._now() + + timedelta(seconds=min(60, 2 ** min(attempts, 5))) + ) ) + con.execute( + "UPDATE replays SET status=?, failure_code='BROKER_UNAVAILABLE', retry_attempts=?, sent_count=?, next_attempt_at=?, updated_at=? WHERE tenant_id=? AND replay_id=?", + ( + status, + attempts, + sent, + next_attempt, + updated_at, + replay["tenant_id"], + replay["replay_id"], + ), + ) + if status == "failed": + con.execute( + "UPDATE commands SET status='failed', reason_code='REPLAY_BROKER_EXHAUSTED', aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?", + (updated_at, replay["tenant_id"], replay["command_id"]), + ) + failed_command = con.execute( + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (replay["tenant_id"], replay["command_id"]), + ).fetchone() + self._command_event_tx(con, failed_command, "REPLAY_FAILED") continue with self.store.tx() as con: con.execute( - "UPDATE replays SET status='completed', sent_count=?, updated_at=? WHERE replay_id=?", - (sent, iso(self._now()), replay["replay_id"]), + "UPDATE replays SET status='completed', failure_code=NULL, sent_count=?, next_attempt_at=NULL, updated_at=? WHERE tenant_id=? AND replay_id=?", + (sent, iso(self._now()), replay["tenant_id"], replay["replay_id"]), ) con.execute( - "UPDATE commands SET status='completed', reason_code='REPLAY_SENT', aggregate_version=aggregate_version+1, updated_at=? WHERE command_id=?", - (iso(self._now()), replay["command_id"]), + "UPDATE commands SET status='completed', reason_code='REPLAY_SENT', aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?", + (iso(self._now()), replay["tenant_id"], replay["command_id"]), ) updated = con.execute( - "SELECT * FROM commands WHERE command_id=?", (replay["command_id"],) + "SELECT * FROM commands WHERE tenant_id=? AND command_id=?", + (replay["tenant_id"], replay["command_id"]), ).fetchone() self._command_event_tx(con, updated, "REPLAY_SENT") - count += sent + count += sent - _integer(replay["sent_count"]) return count def emit_opt_out(self, tenant_id: str, call_id: str, contact_ref: str) -> None: @@ -2576,28 +4076,35 @@ class AgentCallService: attempts = [ dict(row) for row in self.store.all( - "SELECT attempt_id, attempt_number, route_policy_id, caller_profile_id, target_uri, status, started_at, ended_at FROM attempts WHERE call_id=? ORDER BY attempt_number", - (call_id,), + """SELECT a.attempt_id, a.attempt_number, a.route_policy_id, + a.caller_profile_id, a.target_uri, a.status, a.started_at, a.ended_at + FROM attempts a JOIN calls c ON c.call_id=a.call_id + WHERE a.call_id=? AND c.tenant_id=? ORDER BY a.attempt_number""", + (call_id, tenant_id), ) ] segments = [ dict(row) for row in self.store.all( - "SELECT segment_id, turn_index, speaker, text, revision, is_final, playback_state, updated_at FROM transcripts WHERE call_id=? ORDER BY turn_index, speaker", - (call_id,), + """SELECT tr.segment_id, tr.turn_index, tr.speaker, tr.text, + tr.revision, tr.is_final, tr.playback_state, tr.updated_at + FROM transcripts tr JOIN calls c ON c.call_id=tr.call_id + WHERE tr.call_id=? AND c.tenant_id=? ORDER BY tr.turn_index, tr.speaker""", + (call_id, tenant_id), ) ] recordings = [ dict(row) for row in self.store.all( - "SELECT recording_id, content_type, size_bytes, checksum_algorithm, checksum, channels, sample_rate_hz, duration_ms, status, oss_id, failure_code, verified_at FROM recordings WHERE call_id=? ORDER BY created_at", - (call_id,), + """SELECT r.recording_id, r.content_type, r.size_bytes, + r.checksum_algorithm, r.checksum, r.channels, r.sample_rate_hz, + r.duration_ms, r.status, r.oss_id, r.failure_code, r.verified_at + FROM recordings r JOIN calls c ON c.call_id=r.call_id + WHERE r.call_id=? AND c.tenant_id=? ORDER BY r.created_at""", + (call_id, tenant_id), ) ] - event_rows = self.store.all( - "SELECT published, broker_status, saas_applied FROM events WHERE tenant_id=? AND (aggregate_id=? OR payload_json LIKE ?)", - (tenant_id, call_id, f'%"call_id":"{call_id}"%'), - ) + event_rows = self._events_for_call(tenant_id, call_id) confirmed = sum(1 for row in event_rows if row["published"]) pending = sum( 1 @@ -2643,8 +4150,21 @@ class AgentCallService: waiting = self.store.one( "SELECT COUNT(*) AS n FROM commands WHERE status='waiting'" )["n"] + recording_failures = self.store.one( + "SELECT COUNT(*) AS n FROM recordings WHERE status='failed'" + )["n"] mode = self.mode.replace('"', "") - providers = self.profile.get("provider_modes", {}) + providers = self._provider_status + try: + ready_value = 1 if self.ready() else 0 + except (OSError, BrokerError, sqlite3.DatabaseError, TypeError, ValueError): + ready_value = 0 + clock_value = 1 if self._clock_jump else 0 + try: + disk = shutil.disk_usage(self.object_root) + disk_ratio = disk.used / disk.total if disk.total else 1.0 + except (OSError, TypeError, ZeroDivisionError): + disk_ratio = 1.0 lines = [ "# HELP agent_call_active_calls Active calls including queued and ringing calls.", "# TYPE agent_call_active_calls gauge", @@ -2655,8 +4175,18 @@ class AgentCallService: f"agent_call_waiting_commands {waiting}", "# TYPE agent_call_dead_letters_total counter", f"agent_call_dead_letters_total {dlq}", + "# TYPE agent_call_recording_failures_total counter", + f"agent_call_recording_failures_total {recording_failures}", "# TYPE agent_call_mode_info gauge", f'agent_call_mode_info{{mode="{mode}"}} 1', + "# TYPE agent_call_ready gauge", + f"agent_call_ready {ready_value}", + "# TYPE agent_call_clock_jump gauge", + f"agent_call_clock_jump {clock_value}", + "# TYPE agent_call_unknown_calls gauge", + f"agent_call_unknown_calls {self.readiness()['unknown_calls']}", + "# TYPE agent_call_disk_usage_ratio gauge", + f"agent_call_disk_usage_ratio {disk_ratio}", ] for component, provider_mode in sorted(providers.items()): safe_component = component.replace('"', "") @@ -2664,14 +4194,76 @@ class AgentCallService: lines.append( f'agent_call_provider_mode_info{{component="{safe_component}",mode="{safe_mode}"}} 1' ) + for tenant in self.profile.get("tenants", []): + try: + depth = getattr(self.broker, "depth", lambda _name: 0)( + queue_name(tenant["tenant_key"]) + ) + except (BrokerError, ValidationError): + depth = 0 + tenant_label = str(tenant["tenant_id"]).replace('"', "") + lines.append( + f'agent_call_queue_depth{{tenant_id="{tenant_label}"}} {depth}' + ) return "\n".join(lines) + "\n" - def ready(self) -> bool: + def readiness(self) -> dict[str, Any]: + checks: dict[str, bool] = { + "database_read": False, + "database_write": False, + "broker": False, + "routes": not self._unrouteable_tenants, + "cells": bool(self._cell_health) and any(self._cell_health.values()), + "providers": self.mode == "mock" + and all( + value in {"mock", "sqlite", "memory", "fake-cli", "rabbit"} + for value in self._provider_status.values() + ), + "clock": not self._clock_jump, + "reconciliation": False, + "disk": False, + } try: - self.store.one("SELECT 1") - return bool(self.broker.ready()) - except Exception: - return False + checks["database_read"] = self.store.one("SELECT 1") is not None + with self.store.tx() as con: + con.execute( + "INSERT INTO readiness_probe(probe_id, touched_at) VALUES(1, ?) ON CONFLICT(probe_id) DO UPDATE SET touched_at=excluded.touched_at", + (iso(self._now()),), + ) + con.execute("DELETE FROM readiness_probe WHERE probe_id=1") + checks["database_write"] = True + except (OSError, sqlite3.DatabaseError): + pass + try: + checks["broker"] = bool(self.broker.ready()) + has_route = getattr(self.broker, "has_route", None) + if callable(has_route): + checks["routes"] = checks["routes"] and all( + bool(has_route(routing_key(tenant["tenant_key"]))) + for tenant in self.profile.get("tenants", []) + ) + else: + checks["routes"] = False + except (BrokerError, ValidationError, OSError): + checks["broker"] = False + try: + usage = shutil.disk_usage(self.object_root) + ratio = usage.used / usage.total if usage.total else 1.0 + checks["disk"] = ratio < float(self._limits().get("disk_stop_pct", 0.95)) + except (OSError, TypeError, ValueError, ZeroDivisionError): + checks["disk"] = False + unknown = self.store.one( + "SELECT COUNT(*) AS n FROM calls WHERE call_state='reconciling'" + )["n"] + checks["reconciliation"] = _integer(unknown) == 0 + return { + "ready": all(checks.values()), + "checks": checks, + "unknown_calls": _integer(unknown), + } + + def ready(self) -> bool: + return bool(self.readiness()["ready"]) def wait_for_idle(self, timeout: float = 5.0) -> None: end = time.monotonic() + timeout @@ -2680,8 +4272,14 @@ class AgentCallService: live = [ thread for thread in self._call_threads.values() if thread.is_alive() ] - if not live and not self.store.one( - "SELECT 1 FROM commands WHERE status IN ('accepted','waiting','executing','applying') LIMIT 1" + if ( + not live + and not self.store.one( + "SELECT 1 FROM commands WHERE status IN ('pending','retained','broker_confirmed','accepted','waiting','executing','applying','reconciling') LIMIT 1" + ) + and not self.store.one( + "SELECT 1 FROM published_commands WHERE status IN ('pending','retained','broker_confirmed') LIMIT 1" + ) ): self.dispatch_outbox() self.consume_saas_events() diff --git a/agent_call/http.py b/agent_call/http.py index bec074f..3dcbcb7 100644 --- a/agent_call/http.py +++ b/agent_call/http.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import os from http import HTTPStatus @@ -15,6 +16,7 @@ READ_SCOPE = "outbound.read" CONTROL_SCOPE = "outbound.control" UPLOAD_SCOPE = "recording.upload" COMPLETE_SCOPE = "recording.complete" +REPLAY_SCOPE = "outbound.replay" class AgentCallHandler(BaseHTTPRequestHandler): @@ -37,7 +39,7 @@ class AgentCallHandler(BaseHTTPRequestHandler): raise ServiceError("TENANT_ID_REQUIRED", "X-Tenant-ID is required", 400) return value - def _authorize(self, scope: str, tenant_id: str | None = None) -> None: + def _authorize(self, scope: str, tenant_id: str | None = None) -> dict[str, Any]: token_header = self.headers.get("Authorization", "") if not token_header.startswith("Bearer "): raise ServiceError("UNAUTHORIZED", "bearer authentication is required", 401) @@ -49,12 +51,24 @@ class AgentCallHandler(BaseHTTPRequestHandler): if not isinstance(identity, dict): raise ServiceError("UNAUTHORIZED", "bearer token is not authorized", 401) scopes = identity.get("scopes", []) + tenant_ids = identity.get("tenant_ids", []) + if not isinstance(scopes, list) or not all( + isinstance(item, str) for item in scopes + ): + raise ServiceError("UNAUTHORIZED", "token scopes are invalid", 401) + if not isinstance(tenant_ids, list) or not all( + isinstance(item, str) for item in tenant_ids + ): + raise ServiceError("UNAUTHORIZED", "token tenant scope is invalid", 401) if scope not in scopes and "*" not in scopes: raise ServiceError("FORBIDDEN", "required scope is missing", 403) - if tenant_id is not None: - tenant_ids = identity.get("tenant_ids", []) - if "*" not in tenant_ids and tenant_id not in tenant_ids: - raise ServiceError("FORBIDDEN", "tenant scope is missing", 403) + if ( + tenant_id is not None + and "*" not in tenant_ids + and tenant_id not in tenant_ids + ): + raise ServiceError("FORBIDDEN", "tenant scope is missing", 403) + return identity def _body(self) -> dict[str, Any]: raw_length = self.headers.get("Content-Length") @@ -69,6 +83,8 @@ class AgentCallHandler(BaseHTTPRequestHandler): ) try: raw = self.rfile.read(length) + if len(raw) != length: + raise ValidationError("BODY_TRUNCATED", "request body was truncated") body = _loads(raw.decode("utf-8")) except UnicodeDecodeError as exc: raise ValidationError( @@ -80,10 +96,21 @@ class AgentCallHandler(BaseHTTPRequestHandler): def _header(self, name: str) -> str: value = self.headers.get(name, "") - if not value: + if not value or len(value) > 128 or any(ch.isspace() for ch in value): raise ServiceError("HEADER_REQUIRED", f"{name} is required", 400) return value + def _actor_id(self) -> str: + token_header = self.headers.get("Authorization", "") + token = token_header[7:] if token_header.startswith("Bearer ") else "" + identity = _loads(os.environ.get("HTTP_TOKENS"), {}) + principal = identity.get(token) if isinstance(identity, dict) else None + if isinstance(principal, dict): + value = principal.get("client_id") or principal.get("subject") + if isinstance(value, str) and value: + return value + return "token:" + hashlib.sha256(token.encode("utf-8")).hexdigest()[:16] + def _send( self, status: int, body: Any, content_type: str = "application/json" ) -> None: @@ -195,7 +222,7 @@ class AgentCallHandler(BaseHTTPRequestHandler): and parts[5] == "replays" ): tenant_id = self._tenant_id() - self._authorize(CONTROL_SCOPE, tenant_id) + self._authorize(REPLAY_SCOPE, tenant_id) result = self.service.create_replay( tenant_id, "call", @@ -213,7 +240,7 @@ class AgentCallHandler(BaseHTTPRequestHandler): and parts[5] == "replays" ): tenant_id = self._tenant_id() - self._authorize(CONTROL_SCOPE, tenant_id) + self._authorize(REPLAY_SCOPE, tenant_id) result = self.service.create_replay( tenant_id, "command", @@ -231,8 +258,9 @@ class AgentCallHandler(BaseHTTPRequestHandler): ): tenant_id = self._tenant_id() self._authorize(UPLOAD_SCOPE, tenant_id) + idempotency_key = self._header("Idempotency-Key") result_status, result = self.service.create_upload( - tenant_id, self._body() + tenant_id, self._body(), idempotency_key, self._actor_id() ) self._send(result_status, result) return @@ -245,7 +273,10 @@ class AgentCallHandler(BaseHTTPRequestHandler): ): tenant_id = self._tenant_id() self._authorize(COMPLETE_SCOPE, tenant_id) - result = self.service.complete_upload(tenant_id, parts[4], self._body()) + idempotency_key = self._header("Idempotency-Key") + result = self.service.complete_upload( + tenant_id, parts[4], self._body(), idempotency_key, self._actor_id() + ) self._send(200, result) return if method == "PUT" and len(parts) == 3 and parts[:2] == ["_mock", "oss"]: @@ -260,8 +291,8 @@ class AgentCallHandler(BaseHTTPRequestHandler): if not raw_length: raise ValidationError("BODY_REQUIRED", "upload body is required") length = _integer(raw_length) - if length > self.service.profile.get("limits", {}).get( - "recording_max_bytes", 16777216 + if length < 0 or length > self.service.profile.get("limits", {}).get( + "max_recording_bytes", 16777216 ): raise ServiceError( "BODY_TOO_LARGE", @@ -274,6 +305,10 @@ class AgentCallHandler(BaseHTTPRequestHandler): raise ServiceError( "UPLOAD_INTERRUPTED", "upload connection interrupted", 400 ) from exc + if len(data) != length: + raise ServiceError( + "UPLOAD_INTERRUPTED", "upload body was truncated", 400 + ) upload = self.service.store.one( "SELECT tenant_id FROM uploads WHERE upload_id=?", (parts[2],) ) @@ -287,7 +322,7 @@ class AgentCallHandler(BaseHTTPRequestHandler): self._error(exc, request_id) except (BrokenPipeError, ConnectionResetError): return - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, KeyError): self._error( ServiceError("INTERNAL_ERROR", "request could not be completed", 500), request_id, @@ -304,8 +339,17 @@ class AgentCallHandler(BaseHTTPRequestHandler): if not isinstance(identity, dict): return False tenant_ids = identity.get("tenant_ids", []) + scopes = identity.get("scopes", []) + if not isinstance(tenant_ids, list) or not all( + isinstance(item, str) for item in tenant_ids + ): + return False + if not isinstance(scopes, list) or not all( + isinstance(item, str) for item in scopes + ): + return False return ("*" in tenant_ids or tenant_id in tenant_ids) and ( - scope in identity.get("scopes", []) or "*" in identity.get("scopes", []) + scope in scopes or "*" in scopes ) def do_GET(self) -> None: diff --git a/agent_call/mocks.py b/agent_call/mocks.py index c4dc035..6986e76 100644 --- a/agent_call/mocks.py +++ b/agent_call/mocks.py @@ -6,6 +6,7 @@ ARI, RTP, AI-stream, or OSS integration. from __future__ import annotations +import hashlib import json import threading from pathlib import Path @@ -82,9 +83,23 @@ class MockRTP: def __init__(self, journal: ProtocolJournal) -> None: self.journal = journal - def packet(self, call_id: str, direction: str, sequence: int) -> None: + def packet( + self, + call_id: str, + direction: str, + sequence: int, + payload: bytes | None = None, + ) -> None: + payload = payload if payload is not None else b"\x01\x02" * 160 self.journal.append( - "rtp", "packet", call_id=call_id, direction=direction, sequence=sequence + "rtp", + "packet", + call_id=call_id, + direction=direction, + sequence=sequence, + payload_bytes=len(payload), + payload_sha256=hashlib.sha256(payload).hexdigest(), + non_silent=any(payload), ) @@ -93,7 +108,12 @@ class MockAI: self.journal = journal def turn( - self, call_id: str, turn_index: int, speaker: str, playback_state: str + self, + call_id: str, + turn_index: int, + speaker: str, + playback_state: str, + text: str | None = None, ) -> None: self.journal.append( "ai", @@ -102,4 +122,21 @@ class MockAI: turn_index=turn_index, speaker=speaker, playback_state=playback_state, + text=text or f"mock-{speaker}-turn-{turn_index}", + ) + self.journal.append( + "ai", + "playback", + call_id=call_id, + turn_index=turn_index, + state="sent", + ) + self.journal.append( + "ai", + "playback", + call_id=call_id, + turn_index=turn_index, + state=playback_state + if playback_state in {"confirmed", "playback_confirmed", "unknown"} + else "unknown", ) diff --git a/deploy/alerts.yml b/deploy/alerts.yml index 8bf2a48..0ad4577 100644 --- a/deploy/alerts.yml +++ b/deploy/alerts.yml @@ -4,18 +4,70 @@ groups: - alert: AgentCallOutboxStalled expr: agent_call_outbox_pending > 0 for: 2m - labels: {severity: warning} + labels: + severity: warning annotations: - summary: agent-call outbox has pending events + summary: agent-call outbox is not draining - alert: AgentCallDeadLetters expr: increase(agent_call_dead_letters_total[5m]) > 0 for: 1m - labels: {severity: critical} + labels: + severity: critical annotations: - summary: agent-call rejected or malformed messages entered the DLQ + summary: agent-call rejected messages reached the dead-letter log - alert: AgentCallNotMockSafe - expr: agent_call_mode_info{mode="mock"} != 1 + expr: agent_call_mode_info{mode!="mock"} > 0 for: 1m - labels: {severity: critical} + labels: + severity: critical annotations: - summary: provider mode changed; verify production admission gates + summary: agent-call is running outside the supported Mock mode + - alert: AgentCallNotReady + expr: agent_call_ready == 0 + for: 1m + labels: + severity: critical + annotations: + summary: agent-call readiness gate is closed + - alert: AgentCallClockJump + expr: agent_call_clock_jump > 0 + for: 1m + labels: + severity: critical + annotations: + summary: agent-call stopped new admission after a wall-clock jump + - alert: AgentCallUnknownCall + expr: agent_call_unknown_calls > 0 + for: 1m + labels: + severity: critical + annotations: + summary: active call requires reconciliation and still occupies capacity + - alert: AgentCallQueueBackpressure + expr: sum(agent_call_queue_depth) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: tenant command queues have pending backpressure + - alert: AgentCallRecordingFailures + expr: increase(agent_call_recording_failures_total[5m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: recording verification failed + - alert: AgentCallDiskWarning + expr: agent_call_disk_usage_ratio >= 0.70 + for: 5m + labels: + severity: warning + annotations: + summary: agent-call disk usage is high + - alert: AgentCallDiskStop + expr: agent_call_disk_usage_ratio >= 0.80 + for: 1m + labels: + severity: critical + annotations: + summary: agent-call stopped new admissions for disk pressure diff --git a/deploy/aliyun_host.py b/deploy/aliyun_host.py old mode 100644 new mode 100755 diff --git a/deploy/backup_sqlite.py b/deploy/backup_sqlite.py old mode 100644 new mode 100755 index 73316d8..98b39ea --- a/deploy/backup_sqlite.py +++ b/deploy/backup_sqlite.py @@ -1,36 +1,65 @@ +#!/usr/bin/env python3 +"""Create or restore an SQLite backup without silently overwriting a target.""" + from __future__ import annotations import argparse import sqlite3 +from contextlib import closing from pathlib import Path -def copy_database(source: str, destination: str) -> None: - source_path = Path(source) - destination_path = Path(destination) +def _check_integrity(con: sqlite3.Connection) -> None: + result = con.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError(f"SQLite integrity check failed: {result}") + + +def copy_database( + source: str | Path, destination: str | Path, *, replace: bool = False +) -> None: + source_path = Path(source).expanduser().resolve() + destination_path = Path(destination).expanduser().resolve() + if source_path == destination_path: + raise ValueError("source and destination must differ") if not source_path.is_file(): - raise FileNotFoundError(source) + raise FileNotFoundError(source_path) + if destination_path.exists() and not replace: + raise FileExistsError( + f"refusing to overwrite existing database: {destination_path}" + ) destination_path.parent.mkdir(parents=True, exist_ok=True) - source_db = sqlite3.connect(source_path) - destination_db = sqlite3.connect(destination_path) try: - source_db.backup(destination_db) - destination_db.commit() - finally: - destination_db.close() - source_db.close() + with closing( + sqlite3.connect(f"file:{source_path}?mode=ro", uri=True) + ) as source_con: + _check_integrity(source_con) + with closing(sqlite3.connect(destination_path)) as destination_con: + source_con.backup(destination_con) + destination_con.commit() + _check_integrity(destination_con) + version = destination_con.execute( + "SELECT MAX(version) FROM schema_migrations" + ).fetchone()[0] + if version is None: + raise RuntimeError("backup has no schema migration version") + except sqlite3.DatabaseError as exc: + raise RuntimeError(f"SQLite backup failed: {exc}") from exc + print(f"backup complete: {source_path} -> {destination_path}") -def main() -> int: - parser = argparse.ArgumentParser( - description="offline SQLite backup/restore for agent-call" - ) +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() parser.add_argument("action", choices=("backup", "restore")) parser.add_argument("source") parser.add_argument("destination") - args = parser.parse_args() - copy_database(args.source, args.destination) - print(f"{args.action} complete") + parser.add_argument( + "--replace", + action="store_true", + help="explicitly allow replacing the destination", + ) + args = parser.parse_args(argv) + copy_database(args.source, args.destination, replace=args.replace) return 0 diff --git a/deploy/build_asterisk_userdata.py b/deploy/build_asterisk_userdata.py old mode 100644 new mode 100755 index e470787..37a173e --- a/deploy/build_asterisk_userdata.py +++ b/deploy/build_asterisk_userdata.py @@ -61,9 +61,7 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None f"ASTERISK_IMAGE={shlex.quote(image)}", f"ASTERISK_CONFIG_GID={config_gid}", ] - variables.extend( - f"{key}={shlex.quote(value)}" for key, value in payloads.items() - ) + variables.extend(f"{key}={shlex.quote(value)}" for key, value in payloads.items()) variables.append("") if template is None: template = Path(__file__).with_name("asterisk_bootstrap.sh") @@ -76,7 +74,9 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None def write_output(content, path): path = Path(path) if path.exists() or path.is_symlink(): - raise ValueError("output already exists; choose a new local secret-bearing path") + raise ValueError( + "output already exists; choose a new local secret-bearing path" + ) path.parent.mkdir(parents=True, exist_ok=True) temporary_path = None try: @@ -105,7 +105,9 @@ def main(): parser.add_argument("--config", required=True, help="render_asterisk JSON config") parser.add_argument("--image", help="immutable registry image@sha256:digest") parser.add_argument("--ari-password-file") - parser.add_argument("--output", required=True, help="local secret-bearing user-data path") + parser.add_argument( + "--output", required=True, help="local secret-bearing user-data path" + ) parser.add_argument("--config-gid", type=int, default=1000) args = parser.parse_args() try: @@ -113,7 +115,9 @@ def main(): image = immutable_image( args.image or cfg.get("asterisk_image") or os.environ.get("ASTERISK_IMAGE") ) - content = build(cfg, image, read_secret(args.ari_password_file), args.config_gid) + content = build( + cfg, image, read_secret(args.ari_password_file), args.config_gid + ) if len(content.encode()) > 16 * 1024: raise ValueError("generated user-data exceeds the ECS 16 KiB limit") write_output(content, args.output) diff --git a/deploy/check_mock_config.py b/deploy/check_mock_config.py index 1558ac5..8d4e49a 100644 --- a/deploy/check_mock_config.py +++ b/deploy/check_mock_config.py @@ -10,6 +10,7 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) _core = importlib.import_module("agent_call.core") +ValidationError = _core.ValidationError queue_name = _core.queue_name routing_key = _core.routing_key @@ -30,7 +31,7 @@ def check(profile_path: str | Path) -> list[str]: try: route = routing_key(tenant["tenant_key"]) queue_name(tenant["tenant_key"]) - except Exception as exc: + except (KeyError, TypeError, ValidationError) as exc: problems.append(f"tenant route invalid: {type(exc).__name__}") continue if len(route.encode("utf-8")) > 255: diff --git a/deploy/migrate.py b/deploy/migrate.py index 762167b..bcd9d04 100644 --- a/deploy/migrate.py +++ b/deploy/migrate.py @@ -15,9 +15,12 @@ Store = _core.Store def migrate(path: str) -> int: store = Store(path) - row = store.one("SELECT MAX(version) AS version FROM schema_migrations") - print(f"schema_version={row['version']}") - return 0 + try: + row = store.one("SELECT MAX(version) AS version FROM schema_migrations") + print(f"schema_version={row['version']}") + return 0 + finally: + store.close() def main() -> int: diff --git a/deploy/prometheus.yml b/deploy/prometheus.yml index e0e447c..9c46459 100644 --- a/deploy/prometheus.yml +++ b/deploy/prometheus.yml @@ -2,6 +2,9 @@ global: scrape_interval: 15s evaluation_interval: 15s +rule_files: + - /etc/prometheus/alerts.yml + scrape_configs: - job_name: agent-call metrics_path: /metrics diff --git a/deploy/render_asterisk.py b/deploy/render_asterisk.py old mode 100644 new mode 100755 diff --git a/docs/contracts/executor.openapi.yaml b/docs/contracts/executor.openapi.yaml index 6bbcf14..00e95d9 100644 --- a/docs/contracts/executor.openapi.yaml +++ b/docs/contracts/executor.openapi.yaml @@ -163,7 +163,7 @@ components: type: string minLength: 1 maxLength: 128 - pattern: '^[^\\s/\\\\]+$' + pattern: '^[^\s/\\]+$' ControlRequest: type: object additionalProperties: false diff --git a/docs/contracts/mock-profile.json b/docs/contracts/mock-profile.json index 15c93ae..4a79b80 100644 --- a/docs/contracts/mock-profile.json +++ b/docs/contracts/mock-profile.json @@ -20,14 +20,24 @@ "max_call_duration_ms": 180000, "max_mq_bytes": 262144, "max_queue_messages": 1000, + "max_queue_bytes": 16777216, + "disk_warn_pct": 0.70, + "disk_stop_pct": 0.80, "max_http_bytes": 65536, "recording_max_bytes": 16777216, + "max_recording_bytes": 16777216, "replay_retention_s": 604800, "upload_ttl_s": 300, "global_concurrency": 6, "global_cps": 3, "tenant_concurrency": 2, "tenant_cps": 1, + "tenant_publish_rate": 10, + "scheduler_lease_ttl_s": 10, + "pending_window_per_tenant": 16, + "pending_window_global": 64, + "max_unacked_per_tenant": 4, + "max_replay_attempts": 6, "cell_capacity": 4, "turns": 2, "hold_ms": 0 @@ -47,8 +57,8 @@ "caller_profiles": [{"caller_profile_id": "caller_profile_test", "display": "BD93205882", "allowed": true}], "agents": [{"agent_version_id": "agent_v1", "immutable": true, "llm": "mock", "tts": "mock", "asr": "mock"}], "cells": [ - {"cell_id": "cell-a", "capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"}, - {"cell_id": "cell-b", "capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"} + {"cell_id": "cell-a", "capacity": 4, "media_capacity": 4, "ai_capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"}, + {"cell_id": "cell-b", "capacity": 4, "media_capacity": 4, "ai_capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"} ], - "failure_scenarios": ["success", "busy", "no_answer", "ai_timeout", "ari_disconnect", "upload_missing", "upload_bad_checksum", "broker_outage", "clock_jump"] + "failure_scenarios": ["success", "busy", "no_answer", "ai_timeout", "customer_silent", "ari_disconnect", "upload_missing", "upload_bad_checksum", "broker_outage", "clock_jump"] } diff --git a/docs/contracts/saas.openapi.yaml b/docs/contracts/saas.openapi.yaml index 9e74ba1..e3f47c2 100644 --- a/docs/contracts/saas.openapi.yaml +++ b/docs/contracts/saas.openapi.yaml @@ -88,7 +88,7 @@ components: type: string minLength: 1 maxLength: 128 - pattern: '^[^\\s/\\\\]+$' + pattern: '^[^\s/\\]+$' UploadRequest: type: object additionalProperties: false diff --git a/docs/evidence/acceptance-mock-2026-09-13.json b/docs/evidence/acceptance-mock-2026-09-13.json new file mode 100644 index 0000000..f05e45d --- /dev/null +++ b/docs/evidence/acceptance-mock-2026-09-13.json @@ -0,0 +1,128 @@ +{ + "finished_at": "2026-09-13T10:59:31.729Z", + "gate_status": "INCOMPLETE", + "probes": { + "V01": { + "duration_ms": 15.796, + "evidence": { + "http_execute_route": "absent", + "profile": "mock", + "schema_negative_cases": 3 + }, + "status": "PASS" + }, + "V02": { + "duration_ms": 161.588, + "evidence": { + "calls": 1, + "different_execution_command": "associated", + "same_command_replays": 1 + }, + "status": "PASS" + }, + "V03": { + "duration_ms": 344.365, + "evidence": { + "lease_duplicate_calls": 0, + "scheduler_instances": 2, + "tenants_admitted": [ + "tenant-demo", + "tenant-b", + "tenant-c" + ] + }, + "status": "PASS" + }, + "V04": { + "duration_ms": 34.727, + "evidence": { + "affected_commands": 1, + "stale_cas": "rejected", + "waiting_barrier": "applied" + }, + "status": "PASS" + }, + "V05": { + "duration_ms": 149.141, + "evidence": { + "dead_letters": 0, + "recovered": true, + "retained_id": "cmd_broker_retry" + }, + "status": "PASS" + }, + "V06/V07": { + "duration_ms": 2131.36, + "evidence": { + "asterisk_real": "not_run", + "disconnect_state_before_reconcile": "reconciling", + "fallback_redials": 0, + "mock_protocol": "pass", + "outcomes": { + "busy": "busy", + "disconnect": "failed", + "no_answer": "no_answer" + } + }, + "status": "PASS" + }, + "V08": { + "duration_ms": 994.059, + "evidence": { + "old_tts_reused": false, + "segments": 200, + "turns": 100 + }, + "status": "PASS" + }, + "V09": { + "duration_ms": 1131.946, + "evidence": { + "bad_checksum": "failed", + "invalid_ready": 0, + "missing_object": "failed" + }, + "status": "PASS" + }, + "V10": { + "duration_ms": 113.747, + "evidence": { + "original_event_ids": true, + "pending_after_recovery": 0, + "pending_during_outage": 13 + }, + "status": "PASS" + }, + "V11/V12": { + "duration_ms": 144.282, + "evidence": { + "backup": "verified", + "metrics": "healthy", + "real_mode_mock_rejection": true, + "restored_counts": { + "commands": 1, + "events": 13, + "inbox": 13, + "recordings": 1 + }, + "restored_db_facts": true, + "rollback_preserves_db": "not_run" + }, + "status": "PASS" + }, + "V13": { + "duration_ms": 0.543, + "evidence": { + "evidence_file": "docs/evidence/scale-mock-2026-09-12.json", + "reason": "historical state-only evidence cannot satisfy continuous media V13", + "status": "not_run" + }, + "status": "NOT_RUN" + } + }, + "production": { + "V14": "blocked_external_evidence" + }, + "profile": "mock", + "started_at": "2026-09-13T10:59:26.507Z" +} diff --git a/docs/evidence/asterisk-persistent-2026-09-13.md b/docs/evidence/asterisk-persistent-2026-09-13.md index 067a357..97f9d43 100644 --- a/docs/evidence/asterisk-persistent-2026-09-13.md +++ b/docs/evidence/asterisk-persistent-2026-09-13.md @@ -1,27 +1,38 @@ -# 持久真实 SIP/Asterisk 测试环境证据（2026-09-13） +# 真实 SIP/Asterisk 测试环境证据（2026-09-13） -## 环境 +## 当前环境（探针期间） -- ECS：`i-2zegaldd07mkp0xyhcff`，Ubuntu 24.04，`ecs.e-c1m1.large`。 -- 竞价策略：`SpotAsPriceGo`（自动竞价）；竞价回收可能中断活动通话。 -- 固定 EIP：`123.56.71.98`，当前保持 `InUse` 并绑定该 ECS。 -- Asterisk：`22.10.1`，固定镜像 digest：`sha256:0f5e3cd0e9a86bc9dc4750929abee9ba1faae11ed39739f3e75343d39d837e41`。 -- 线路：UDP `61.132.228.221:5060`，IP 白名单免注册。 -- PJSIP contact：`Avail`，实测 RTT 约 `22.146 ms`。 -- 主叫原值：`BD93205882`；`From user` 与 `CALLERID` 均使用原值，未发送 P-Asserted-Identity。 -- 环境按用户要求保留，未在本轮测试后停止或删除。 +- ECS：`i-2ze4hvi6xvy02g0wz9kk`，北京 `cn-beijing`，Ubuntu 24.04，`ecs.e-c1m1.large`，探针期间 Running。 +- 竞价策略：`SpotAsPriceGo`；竞价回收可能中断活动通话。 +- 固定 EIP：`123.56.71.98`，AllocationId `eip-2zeevfsaxzwuue2szy7xb`；探针期间 InUse 并绑定该 ECS。 +- 创建/绑定：2026-09-13 10:28 UTC；使用专用 VSwitch、安全组和 KeyPair，实例公网带宽上限为 0。 +- Asterisk：22.10.1，镜像及实际 image ID 均为固定 digest `sha256:0f5e3cd0e9a86bc9dc4750929abee9ba1faae11ed39739f3e75343d39d837e41`，容器状态 `running/healthy`。 +- 管理检查：SSH 未开放；通过阿里云 Cloud Assistant 执行只读检查。Cloud-init 初始拉取镜像期间较慢，最终完成并启动容器。 +- PJSIP contact：`provider-primary-aor/sip:61.132.228.221:5060` 为 `Avail`，RTT 约 `22.164 ms`。 +- 出口与信令：SIP 日志显示请求从 `123.56.71.98:5060` 发出；安全组仅允许供应商 `61.132.228.221/32` 的 UDP `5060` 和 `10000–10800`。 +- 主叫原值：`BD93205882`；`From user`/`CALLERID` 保留原值，未发送 P-Asserted-Identity。 -## 两个被叫测试 +## 单次真实外呼探针 -两次均使用同一环境、线路、主叫、UDP/ulaw、RTP 配置；均只发起一次，不自动重试。 +- 执行时间：2026-09-13 10:52:59 UTC。 +- 原始被叫：`18601013734`。 +- 实际目标：`708918601013734`。 +- 线路：`provider-primary`，UDP/ulaw；仅发起一次，无自动重试、无 FALLBACK。 +- SIP：收到 `100 trying`，随后收到 `488 Not Acceptable Here`。 +- 原因：`Reason: Q.850;cause=88;text="INCOMPATIBLE_DESTINATION"`。 +- 后续：发送 ACK；Asterisk 执行 Hangup；15 秒后无活动通道。 +- 接通/RTP：未接通，未获得 `180/183/200` 或 RTP/录音证据。 -| 原始被叫 | 实际呼出目标 | SIP结果 | RTP/接通 | -| --- | --- | --- | --- | -| `18601013734` | `708918601013734` | `100 trying` 后 `488 Not Acceptable Here`；`Q.850 cause=88 INCOMPATIBLE_DESTINATION` | 未接通、无 RTP | -| `18625770806` | `708918625770806` | `100 trying` 后 `488 Not Acceptable Here`；`Q.850 cause=88 INCOMPATIBLE_DESTINATION` | 未接通、无 RTP | +## 测试后资源处置 -两组不同号码在同一线路上得到完全相同的目的地不兼容结果，进一步支持供应商共享路由/线路参数问题；仍不能仅凭两次失败排除同一号码段盲区。需供应商提供已知可达号码，或确认被叫格式/路由、From 域和编解码要求后再继续。 +按用户授权，探针结束后已优雅停止并删除本次创建的 ECS；删除前确认该实例是唯一 `project=agent-call` 候选，系统盘为 40 GiB 且 `DeleteWithInstance=true`。已确认实例和系统盘不再返回；固定 EIP 未释放、未解绑，当前为 `Available`。 -## 范围边界 +## 结论 -本次仅验证了持久 ECS、Asterisk、SIP OPTIONS/contact、两次真实 INVITE 和失败回执；未验证真实 RTP、MQ、OSS、ASR、LLM/TTS 或完整 AI 通话。 +本次确认了单路 ECS/Asterisk 底座、PJSIP contact、固定出口和一次真实 INVITE 链路；供应商以 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION` 拒绝目标，R1 外呼接通验收未通过。该结果不能证明真实 RTP、ASR、LLM/TTS、MQ、OSS 或生产容量/N+1 已验收。 + +供应商需先确认 `7089` 前缀/目标路由、From 域/主叫格式和编解码，或提供已知可达测试号码。未确认前不再次拨号。 + +## 历史记录 + +此前同一固定 EIP/线路对 `18625770806` 的探针目标 `708918625770806` 也返回 `488/Q.850 cause=88`；历史实例已按测试回收规则删除，固定 EIP 未释放。 diff --git a/docs/功能审查整改计划与验收标准_v1.0.md b/docs/功能审查整改计划与验收标准_v1.0.md new file mode 100644 index 0000000..f94dd9d --- /dev/null +++ b/docs/功能审查整改计划与验收标准_v1.0.md @@ -0,0 +1,186 @@ +# agent-call：功能审查整改计划与验收标准 + +**版本：** v1.0 +**日期：** 2026-09-13 +**状态：** 代码级整改已实现并通过本地回归；真实协议、独立服务、告警链路及规模门禁仍未通过。 +**审查基线：** main / `d30314c7d1c56708a43333fe66ad68cfe7c29f40`，三路静态代码审查；未运行故障复现、测试或真实外呼。下列代码位置为该基线定位，后续以符号和回归用例追踪。`deploy/build_asterisk_userdata.py` 的既有未提交修改不在本次整改文档编辑范围内。 + +## 1. 目的、权威与边界 + +本计划解决“真实 SIP 暂未接通之外，现有功能还需补齐什么、怎样证明修好”。真实线路排障不阻塞本地整改，不能据此把其余功能视为完成。 + +- [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)仍是字段、路径、权限、状态和消息语义的唯一维护来源；本文不新增业务接口或状态枚举。 +- [最终计划](最终开发部署监控与验收计划_v1.0.md)仍规定 D01～D12、M1～M3/R1～R3、V01～V14及测试数值；本文是其审查整改补充，细化缺陷、依赖和复验方法，不降低原门禁。 +- 审查发现分为代码缺陷、未实现、仅单元模拟、证据不足。静态发现先写失败回归/独立复现，再修根因；不能把静态推导表述为已发生的生产事故。 +- 本轮仅在本地 Mock/隔离 SQLite 与契约测试范围内实施代码整改；计划不自动授权云消费、真实 SIP/AI 调用、生产迁移、删除/覆盖数据库、清空队列或资源回收。 +- 全部执行指令和业务结果继续走 RabbitMQ；HTTP 仅按既有七条路径控制、查询、补传和存储握手。不增加业务回调、应用收讫或拨号 HTTP 接口。 +- 保留现有可用 outbox、基础轮转和 ASR 代码，按缺陷补齐；不为整改引入通用工作流、供应商框架或无需求微服务。旧 voice_test 仅可复用 ASR，真实 LLM/TTS 必须等待新规范与明确授权。 + +## 2. 当前结论与验收纠偏 + +1. 已有 SQLite 持久状态、RabbitMQ 适配、HTTP、ASR Web、部署工具及 Mock 测试，但不能据此判定 D01～D11 全部完成。 +2. SIP/ARI/RTP/AI journal 是单元测试替身，不是协议集成证据；合成静音 WAV 不是真实双方录音。 +3. 历史 V01～V13 探针 PASS 保留为历史运行结果，不删除原证据；完整用例须按本计划复验，不能沿用脚本 PASS 签署 M3。 +4. 历史 SCALE-MOCK 仅证明其声明的状态负载观察；缺持续媒体、独立双调度及正常生命周期补位证据，完整 V13 待验。 +5. 备份文件存在不证明恢复/回滚成功，指标字符串存在不证明告警执行，broker confirm 不证明 SaaS 应用成功。 +6. 真实 SIP 的 488/cause=88 排障另行推进；R1 真实闭环、R2 1000路完整 AI/N+1、R3 运维交接均不得由本计划的 Mock 结果替代。 + +## 3. 执行阶段与退出门禁 + +以下全部初始为待执行。优先级表示实施顺序；越权、双拨、丢事实、假终态/ready、串线和模式误标属于阻断问题，不因被排到后续阶段而允许上线。 + +| 阶段 | 修复包 | 前置与交付 | 退出条件 | +| --- | --- | --- | --- | +| F0 基线与止误报 | FIX-01、FIX-23 | 固定版本/profile；纠正模式装配与验收报告；建立缺陷用例清单 | 不支持的 real/mixed 失败关闭；未测不得输出完整 PASS | +| F1 业务正确性 | FIX-02～08、FIX-10～13 | F0；复合键迁移、真实测试 MQ、控制/额度/恢复、HTTP契约 | 安全、幂等、背压、双实例竞争与活动执行恢复用例通过；M1相关门禁复验 | +| F2 结果与资产可靠性 | FIX-09、FIX-16、FIX-18～20 | F1 的持久键与权限；先用真实测试字节隔离验证 | 补传、最终文字、上传续签/幂等/崩溃恢复通过；无重复资产或虚假ready | +| F3 协议语音闭环 | FIX-14～15、FIX-17 | F1准入/恢复，F2结果链；真实本地Asterisk及独立协议Mock | 双向媒体、AI流式/打断、实际混音录音贯通；M2相关门禁复验 | +| F4 运维与综合验收 | FIX-21～23及V01～V13 | 功能阶段通过；兼容迁移、故障环境、固定镜像 | 告警/恢复/回滚及完整模拟规模证据齐全后才签M3 | +| F5 真实替换 | 原D12 / R1～R3 | 供应商新规范、预算、账号、白名单及授权齐备 | 分项真实闭环、容量及交接签字，不自动创建资源或调用收费服务 | + +允许独立模块并行，但数据库模型/迁移单一写入负责人；不得并行覆盖同一工作树。每包先落可失败的测试，再最小修复，再独立复核。工时和日历交期在失败复现及环境盘点后登记，不用静态审查臆造承诺。 + +## 4. 修复任务与逐项验收 + +### 4.1 模式、租户和传输 + +| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 | +| --- | --- | --- | --- | +| FIX-01 `core.py:688–739` | 按实际组件装配校验模式；未实现的真实适配器明确拒绝；缺组件/未知枚举不得回退内存broker或Mock | 全部provider伪写real、缺provider、mixed启用未实现组件均启动失败且不接单；合法Mock正常运行；日志/指标逐组件模式与实际实例一致，真实配置不加载测试凭据/故障开关 | V01/V12 | +| FIX-02 `core.py:972–989` | 从实际消费队列及AMQP交付属性取得可信路由上下文，在受理前与正文/注册精确比对 | 经A路由发B正文、伪造header、错绑定均隔离，无业务拨号、无向错误租户发送结果；正确中文/点号/星号/井号key原值往返；224/225 UTF-8字节边界不截断，超限保留发布方原任务 | V01/V05 | +| FIX-03 `core.py:180–210、335–336、1079–1138` | commands/executions/inbox及相关查询/关联统一租户作用域；审计其他实体键；迁移保留现有事实 | A/B使用相同command/execution/event ID互不干扰；同租户同ID重投10次只保留原业务事实，同execution换command不新增拨号；异内容拒绝且原状态/call关联不变；跨租户查询/控制/资产/补传不可见；迁移前后计数、关联与屏障校验一致 | V01/V02 | +| FIX-04 `core.py:545–553` | 实际RabbitMQ配置拒绝发布溢出、消息数/字节上限、发布背压、可靠隔离；既有队列明确迁移而非删队列重建 | 真broker分别触发条数/字节满：新发布失败、原队头及已确认消息完整；mandatory return、NACK、confirm丢失、blocked及DLQ故障可追踪；原ID有限重试不双拨；恢复回原租户域。单节点功能与3节点quorum/可靠死信故障证据分列 | V05 | + +### 4.2 调度、控制和恢复 + +| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 | +| --- | --- | --- | --- | +| FIX-05 `core.py:2191–2205、2296–2339` | 同任务在途控制串行化，持久固定目标revision；applied必须有各Cell屏障证据 | pause尚未applied时resume返回409且不递增；同键重试返回原目标；pause/resume/stop与waiting、已提交意图、FALLBACK竞争，屏障后无新发起；失联保持applying/reconciling；stop不可resume，hangup额外鉴权且确认通道结束 | V04 | +| FIX-06 `core.py:1244–1271、1391–1508` | 原子核验调度所有权/隔离令牌、全部额度、授权/控制并预留；执行侧在真实发起前复核许可 | 两个独立进程共享DB，A在检查后停顿超过租期，B接管占最后额度，A恢复不得发起；在提交后/INVITE前注入暂停、到期、失权均不能越过许可；租户/供应商/Cell/AI及端口额度均不超限；FALLBACK计入CPS，未知占用不释放 | V03/V04/V07 | +| FIX-07 `core.py:962–1015` | 读取前按共享持久状态限制每租户/全局未ACK和待发起窗口；恢复也公平；扫描有界 | 资源耗尽持续投递，超窗消息留原队列而非全部ACK到DB；双实例总窗口不翻倍；重启/积压/新租户下达到第5节公平与截止目标，重投不延长deadline，不为公平挂断活动通话 | V03/V05 | +| FIX-08 `core.py:782–813、1568–1569` | 启动扫描未终结意图并对账；ARI断连隔离Cell且保留未知占用；查询/MQ保持契约状态 | 在意图提交后未发出、已发出未确认、接通后分别杀进程；重启无自动重拨；远端通话仍存活时断ARI不得写ended或释放额度；重连按真实通道收敛，确认结束后仅一次业务终态和资源释放；持久事实/outbox未丢失 | V02/V07/V10/V12 | +| FIX-09 `core.py:2380–2522` | 补传持久进度、有限重试与状态/outbox一致；恢复只处理未确认部分 | 第N条发布失败，断连/重启后原操作可恢复；耗尽则查询和MQ一致报告failed并告警，受控恢复不换截止点；已确认部分不丢，可能重投保留原event ID/内容；后生事件及补传自身结果不入范围；分批限速、不拨号、不调用AI/重传录音；completed不代表SaaS应用 | V10 | + +### 4.3 HTTP、权限和机器契约 + +| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 | +| --- | --- | --- | --- | +| FIX-10 `http.py:198、216` | 两条补传路径使用outbound.replay权限 | control-only拒绝；replay-only对授权资源成功；跨租户404；缺凭据401、缺权限403，不产生未授权业务事件 | V01/V10 | +| FIX-11 `http.py:234–248` | 两个存储POST校验并持久化调用方/租户/操作/幂等键及语义摘要 | 缺头拒绝；同键同内容重试/重启返回原操作；异内容409；不同租户/调用方按作用域独立；JSON字段排序不造成冲突，不因重试生成第二资产 | V01/V09 | +| FIX-12 `core.py:2279–2295、2388–2405` | 控制/补传幂等语义纳入操作类型与路径目标 | 对A已受理后，同键同body改为B目标返回409，不返回A成功冒充B；原目标重试稳定；控制与两种补传之间不错误互相命中 | V01/V04/V10 | +| FIX-13 `core.py:2248–2255、1911–1927`；两份OpenAPI | 修复ID正则转义、布尔当整数、reason上限、音频元数据等；解析执行Schema而非检查文本关键字 | 主契约合法样例含task-demo及字母s可通过；禁止字符/首尾空白按契约拒绝，不擅自限制tenant_key；true作为revision、超长reason、错误格式/声道/采样率/时长拒绝；7条路径请求/响应和MQ正反例、未知主版本、64KiB/256KiB边界均有测试；Schema与运行端一致 | V01 | + +### 4.4 协议媒体、AI和文字 + +| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 | +| --- | --- | --- | --- | +| FIX-14 `mocks.py:40–88`；`core.py:1508–1566` | journal降为单元替身；实现Cell长期ARI、SIP/RTP协议集成和真实媒体清理 | 隔离真实Asterisk+独立Mock SIP对端，至少两通并发不同标识音；双向收包/解码证明无串线，振铃/接通有协议证据；单线路无需虚构backup；正常结束30s内无本次通道/桥/端口残留；断连按FIX-08；明确未接通且原通道结束才允许合法FALLBACK，迟到接通/不确定时禁止重拨 | V06/V07 | +| FIX-15 `core.py:1571–1592`；`mocks.py:91–105` | 音频输入→分段ASR→新LLM/TTS流式Mock→PCM输出；真实ASR入口独立适配；VAD、取消、超时不靠自填成功 | 至少100有效轮次、合成输入与独立输出校验；首包超时、客户静音、插话、取消后旧包迟到均有明确结果；旧音频不进桥，超时不伪报正常完成；播放标记来自实际播放器能力，无确认只能sent/unknown；达到第5节时延；无旧LLM/TTS或收费外联 | V08 | +| FIX-16 `core.py:1671–1681` | 最终文字同内容幂等、异内容冲突，不允许迟到中间稿覆盖最终稿 | 同段最终稿重投10次不新增业务版本/事实；同段异内容拒绝并保留原文；乱序分段各自收敛，不用全局版本丢其他段；最终稿持久化/投递故障可恢复，无法完成明确transcript.failed，不假称完整 | V08/V10 | + +### 4.5 录音与资产交接 + +| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 | +| --- | --- | --- | --- | +| FIX-17 `core.py:1780–1784、1849–1858` | 接通后稳定recording_id，真实单轨混音；正常/强挂/故障确认结束均封口并排持久上传 | 双方不同标识音在录音中可辨，时长与实际音频样本/接通边界吻合，非全零占位；至少正常结束、强挂、媒体异常恢复三条路径；未接通不伪造接通录音；无法恢复资产显式failed；录音处理不阻塞call.finished | V06/V09/V11 | +| FIX-18 `core.py:1835–1845、2068–2090` | 经独立SaaS HTTP授权/complete和对象Mock传输真实字节，而非同对象内部调用冒充对接 | 使用独立服务/凭据与同一OpenAPI；服务端独立读字节验大小/摘要；缺对象、错绑定/摘要、覆盖均不ready；上传目标白名单、HTTPS/重定向与SSRF保护；确认后才持久verified+ready outbox；SaaS inbox后关联oss_id并授权读取 | V09 | +| FIX-19 `core.py:642–657、1991–2059` | complete幂等及文件移动/数据库提交跨边界恢复 | 成功响应丢失后原键重试返回同oss_id/verified_at；移动文件后提交前崩溃可通过最终文件重新校验恢复；重复确认不生成第二份资产/业务ready；异内容保持冲突；未verified不产生ready | V09/V10 | +| FIX-20 `core.py:1928–1934、1977–2006` | 原会话受控续签、上传持久恢复扫描、有限退避 | 跨300s有效期，旧授权拒绝，新授权保持upload/recording/object和内容绑定；中断5分钟后恢复，10分钟内补齐固定测试数据；重启不丢封口文件；耗尽明确failed/可恢复性并告警，文件不静默删除；签名/密钥不出现在日志或MQ | V09/V10/V11 | + +### 4.6 运维、恢复和验收可信度 + +| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 | +| --- | --- | --- | --- | +| FIX-21 `core.py:2669–2675`；`deploy/prometheus.yml`/`alerts.yml` | readiness反映持久写、可路由及必要配置/许可/磁盘；加载规则，修复模式/序列缺失告警 | DB可读不可写、缺绑定、ARI断连、凭据失效、磁盘水位、时钟跳变分别注入；不安全新发起停止，readiness和范围与故障一致，不无差别切断正常活动媒体；promtool语法/规则测试及实际告警产生→接收→恢复有证据；模式错误/指标缺失可检测 | V12 | +| FIX-22 `deploy/backup_sqlite.py`；`acceptance.py:371–397` | 恢复默认新目标，拒绝覆盖现有业务库；兼容迁移、排空/回滚、保留与水位保护 | 从备份恢复到新库实际启动并核对租户、执行、最终稿、outbox、录音关联；应用回滚保留较新DB/队列/卷，旧命令重投无重拨、stop屏障仍在；禁止用旧快照覆盖新防重事实；保留卷故障无已提交事实损失；永久磁盘丢失明确损失/failed/告警，不宣称录音RPO=0；未确认资产/屏障不按普通TTL删除 | V11/V12 | +| FIX-23 `acceptance.py:250–424`；`scale_mock.py` | 用例分层、独立断言、机器门禁与文档一致；完整V13重测 | journal/静音/备份存在仅能作为对应子项结果；缺协议/恢复/告警证据时完整项不PASS；实际失败或所需证据缺失返回非零；保留历史证据并追加纠偏；V13按第5节执行，状态对象数量不得当媒体并发；真实V14始终独立 | V01～V14 | + +上表 `core.py`、`http.py`、`mocks.py` 均位于 `agent_call/`，`acceptance.py`、`scale_mock.py` 位于 `scripts/`。总计23项，ARI断连相关发现合并到FIX-08，避免重复计算完成率。 + +## 5. 测试环境、基线与独立证据 + +### 5.1 分层，不互相冒充 + +| 层级 | 实际依赖 | 能证明什么 | +| --- | --- | --- | +| L0 单元/契约 | 临时真实DB、可控时钟/故障钩子、内存替身 | 分支和语义正确；不能证明AMQP、ARI或媒体协议 | +| L1 持久/MQ/HTTP集成 | 真实隔离DB、RabbitMQ、独立SaaS/对象Mock、两个调度进程 | 事务、隔离、权限、竞争、背压和重启恢复；不等于真实供应商 | +| L2 协议媒体集成 | 真实隔离Asterisk、独立SIP/RTP对端、流式AI Mock | 实际通道/双向音频/取消/录音/结果闭环；不是生产AI质量 | +| L3 综合/规模 | L1+L2、故障/监控组件、SCALE-MOCK profile | M3/V13适用门禁；报告实际物理资源和模拟部分 | +| L4 真实 | 获授权SaaS/SIP/ASR/LLM/TTS/OSS与云出口 | R1～R3，单独取得供应商、容量和运维证据 | + +数据库先复用当前SQLite真实持久能力做回归；必须用独立进程验证其实际协调范围，不以同进程锁代替多实例证据。SQLite同宿主测试不证明多机共享数据库方案；生产数据库与部署拓扑确定后，迁移/并发/故障用例必须在目标引擎重跑。若为跨主机能力需要更换引擎，登记明确迁移方案，不以网络共享SQLite文件规避验证。 + +L1起使用真实测试broker；HA/可靠死信另建隔离3节点quorum环境。Mock网络默认阻止真实公网SIP/AI/云API；只使用合成音频和授权测试数据，不以“假号码”替代网络隔离。 + +### 5.2 沿用最终计划的工程测试阈值 + +下列为DEV/SCALE-MOCK工程基线，不是生产SLA或消费授权。完整取值及恢复/保留条件以最终计划第5节为准；修改profile须版本化并重跑相关门禁。 + +- DEV：3租户、2独立调度实例、2Cell；每租户并发2/CPS1，全局并发6/CPS3；每Cell测试容量4。 +- 所有实例汇总：每租户未ACK≤4/持久待发起≤16，全局≤32/≤64；每轮每租户最多1个新许可。队列上限1000条或16MiB先到者，每租户发布≤10条/s。 +- 公平：发现新活跃租户≤1s；持续可调度且资源足够时B/C发现后≤2s获首次许可；至少100次许可中3租户份额偏差≤10个百分点。资源不足单列，不以强挂达标。 +- 首次准入窗口30s，到期处理≤1s；重启/重投不延期。心跳2s、租约10s，失权即停止本地新发起；时钟偏差>500ms告警、>2s停止新准入。 +- 控制受理到全部屏障applied P95≤2s，前提为节点健康且无未决发起；失联样本不得伪算成功，单列未决。 +- ≥100有效AI轮次：VAD语音结束到首有效TTS入桥P95≤1500ms，插话到停止旧TTS入桥P95≤500ms；首输出超时5s、客户静音15s；实际Mock延迟和测量时间点单列。 +- 最终稿到SaaS Mock事务应用P95≤3s；≤180s通话挂断到对象verified、ready消费及授权读取≤120s；正常结束30s内清理本次媒体资源。 +- 消费/上传中断5分钟，恢复后10分钟内补齐固定负载；DB/MQ恢复后60s内恢复安全调度，未知通话仍保留对账占用。 +- 测试卷使用率70%告警、80%停止新接单，降至60%且依赖正常后恢复；预留活动录音剩余空间。补传每批≤100事件、全局≤50事件/s，实时结果优先。 +- V13：至少2调度实例、100租户×并发12、全局1200，拨号CPS20；预热后≥1000模拟接通连续60分钟，通过≤180s正常生命周期和新授权持续补位，不延长单批通话伪造持续负载。明确模拟Cell/AI额度，提供持续媒体与数据证据。运行中故障隔离一个Cell、旧活动不自动接管/重拨，在健康容量内用新授权恢复负载，再记录恢复稳态；故障影响窗口单列，不隐藏后算连续达标。 +- R2仍要求≥1000同时已接通的真实完整ASR/LLM/TTS通话稳定≥60分钟，以及实测N+1安全容量、真实CPS/RTP/AI/MQ/OSS证据。V13不能替代。 + +### 5.3 证据独立性与并发方法 + +- 每个FIX至少一个修复前失败、修复后通过的可运行用例；保存命令、退出码和原始结果。不可仅检查日志里存在由被测逻辑自行填入的成功字段。 +- 并发故障使用屏障/故障钩子稳定卡在事务、确认、发起、文件移动等边界；独立进程竞争，不用顺序调用两实例代替争抢。 +- MQ证据取broker确认/队列/投递及DB事务；媒体证据取对端收包、解码样本与ARI通道；资产证据独立读上传文件、核摘要与授权读取；应用证据取SaaS inbox/业务库。 +- 至少覆盖DB提交前后、ACK前后、发起前后、confirm前后、文件移动与verified提交之间的中断。所有不确定结果保留原ID和事实，不靠重拨、删队列或删库消除。 +- 故障测试固定重复3轮；每轮重建独立测试命名空间并记录种子，不覆盖生产。一次偶发失败即该项失败，不挑成功轮次签字。3轮是本整改测试重复次数，不是可靠性概率保证。 + +## 6. 执行记录、交付与签署 + +每个FIX的执行记录按下表追加到 `docs/evidence/`。此处仅定义未来产物，不表示已有文件或测试命令。 + +| 字段 | 必填内容 | +| --- | --- | +| 身份 | FIX编号、关联V编号、代码SHA、契约/profile版本、镜像digest、DB/broker版本 | +| 责任 | 实施人、独立复核人、运维协作人；启动前填写真实负责人，未填不视为已分派 | +| 前置 | L0～L4层级、实际provider清单、网络隔离、配额、种子、数据/时间范围 | +| 执行 | 可重复命令/退出码、输入、预期、实际、故障时刻/恢复时刻、重复轮次 | +| 证据 | 脱敏日志、DB/inbox/outbox核对、broker/协议记录、文件摘要、指标样本分母和分位数、告警接收/恢复 | +| 结论 | 待执行/通过/失败/阻塞/不适用；不适用必须给契约依据与批准，不得用来豁免缺失环境 | +| 剩余项 | 未决记录数、外部阻塞、受影响门禁、恢复措施、下一责任人 | + +交付包：最小修复代码及迁移、逐FIX测试、版本化隔离profile、部署/恢复/告警运行步骤、V01～V13复验矩阵、历史证据纠偏说明、R阶段阻塞清单。新增执行命令必须先实现并验证，再写入运行说明。 + +现有回归入口（执行时仍须审阅环境，不能视为本次已运行）： + +```bash +python3 -m unittest discover -s tests -v +(cd services/asr-web && go test -race ./... && go vet ./...) +node --test tests/test_pcm.cjs +bash -n deploy/asterisk.sh +git diff --check +``` + +以上回归不替代L1～L4故障/协议/规模用例；增加的集成测试需提供其真实可运行入口。告警规则用promtool做语法和规则测试，并仍须验证实际接收链路。 + +### 最终通过条件 + +1. FIX-01～23逐项完成复现、修复、独立复核，所有阻断项关闭；共享数据库/队列配置迁移有无损证据，不覆盖用户既有修改。 +2. M1/M2/M3按原定义分别签署；V01～V13完整范围具备独立证据，缺环境或缺规模则保持对应门禁未完成，允许声明已通过的子集。 +3. 无未经授权双拨、串线、跨租户污染、假终态/播放确认/ready或静默丢事实；出现任一项立即停止扩量，保留现场，修复后重跑相关门禁。 +4. 模式和依赖按实际装配展示；文档/机器报告/实现一致，不将Mock成功描述为真实供应商启用。 +5. R1～R3单独验收。SIP供应商、真实LLM/TTS规范、OSS/身份/预算、生产保留与RPO/RTO未落实时，登记外部阻塞，不擅自补参数、不把未验项改为通过。 + +## 7. 本轮执行记录与剩余门禁 + +已按本文的代码级要求实施最小整改： + +- `agent_call/core.py`：模式/依赖拒绝、可信租户路由、租户复合键迁移、发布保留与重试、调度租约/额度、控制Cell屏障、重启对账态、补传进度、格式校验、最终稿幂等、稳定录音与上传恢复、readiness/metrics。 +- `agent_call/http.py`：补传专用权限、调用方/租户/操作/幂等键持久化、请求体与上传边界校验；`agent_call/mocks.py`：非静音合成PCM及协议事实journal；`deploy/backup_sqlite.py`、`deploy/prometheus.yml`、`deploy/alerts.yml`：非覆盖恢复与规则加载/水位告警。 +- `docs/contracts/*.yaml/json` 与 Mock profile 已同步修正；新增 `tests/test_remediation_plan.py` 及 HTTP replay 权限回归。 +- 已运行 `python3 -m unittest discover -s tests -v`（52项通过）、Python编译、JSON/YAML解析、`ruff check`、`git diff --check`；`scripts/acceptance.py` 的 V01～V12 通过，V13 因缺持续媒体独立证据保持 `NOT_RUN`，总门禁为 `INCOMPLETE`。 +- 2026-09-13 在明确授权后重新创建并核验单路真实 Cell：ECS `i-2ze4hvi6xvy02g0wz9kk`、固定 EIP `123.56.71.98`、Asterisk `22.10.1`、PJSIP contact `Avail`；对 `18601013734` 单次构造 `708918601013734`，收到 `100 trying` 后 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`，无接通/RTP/录音证据，详见 `docs/evidence/asterisk-persistent-2026-09-13.md`。 + +上述结果只证明本地代码/契约层子集及真实单路底座/失败回执，不替代第5节的L1～L4证据。真实接通/RTP、ASR/新LLM/TTS、独立SaaS/OSS、3节点可靠MQ、实际告警收发、V13及R1～R3完整验收仍未完成，不得签署M2/M3或生产ready。 diff --git a/docs/最终开发部署监控与验收计划_v1.0.md b/docs/最终开发部署监控与验收计划_v1.0.md index 47b259d..176f8ea 100644 --- a/docs/最终开发部署监控与验收计划_v1.0.md +++ b/docs/最终开发部署监控与验收计划_v1.0.md @@ -9,7 +9,7 @@ - 本文统一规定开发顺序、Mock 策略、运行基线、部署、监控和验收门禁；取代旧计划中的阶段安排及“需再次拍板方案”的表述。 - [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)是标识、HTTP 七条路径、MQ 拓扑/字段/状态的唯一维护来源。本文件不复制接口表；先生成 OpenAPI/JSON Schema，再编写 Mock 和正式实现。 - [原一期计划](一期呼出应用开发计划_v1.0.md)保留 WBS/AT 编号和历史估算；[交付设计](交付文档/一期中间调度件与Asterisk_v1.0/01_中间调度件与MQ回传设计.md)、[部署细则](交付文档/一期中间调度件与Asterisk_v1.0/02_Asterisk部署与SIP对接步骤.md)、[专项用例](交付文档/一期中间调度件与Asterisk_v1.0/03_联调步骤与验收标准.md)保留实现与场景细节，冲突时以本文件及主契约为准。 -- 当前已有 ASR Web、ASR 协议适配、Asterisk 配置生成、阿里云只读计划/受控创建工具和离线测试；参见[现有运行说明](部署接入_运行说明.md)。当前尚无完整调度、AI 通话、MQ/OSS 闭环或本计划的 Mock 套件，不能使用未来产物的命令假装已可运行。 +- 当前已有 ASR Web、ASR 协议适配、Asterisk 配置生成、阿里云只读计划/受控创建工具，以及执行服务、持久状态、MQ/HTTP 和 Mock 测试产物；参见[现有运行说明](部署接入_运行说明.md)。2026-09-13静态审查发现隔离、控制、恢复、媒体与验收缺口，完整调度/AI通话/MQ与资产交接仍须整改复验，不能把已有产物等同于完整门禁通过。整改执行见[功能审查整改计划与验收标准](功能审查整改计划与验收标准_v1.0.md)。 - 本次只交付文档，不新增运行服务、不拨号、不创建云资源、不提交 Git。旧 Word/Excel/ZIP 不覆盖重生成。 ## 2. 已接受的最终方案 @@ -289,7 +289,7 @@ M3产物 + C1～C6真实配置/权限（受控存放）+ R1/R2全部证据 + 实 ## 10. 本次文档验收与后续入口 -本轮已完成文档引用、契约/实现一致性、样例完整性、Mock开发回归及V01～V13验证；V14/R1/R2/R3仍因真实外部证据待执行。当前回归命令可继续使用，但不能把Mock结果当作生产验收： +2026-09-13审查纠偏：历史脚本的V01～V13 Mock探针PASS只证明其实际覆盖的子项，不能证明完整契约/实现一致性、协议媒体、恢复、监控及模拟规模门禁通过。M1/M2/M3须按[整改计划](功能审查整改计划与验收标准_v1.0.md)逐项修复并复验；V14/R1/R2/R3另需真实外部证据。本次静态审查和文档更新未运行下列回归，现有命令可继续使用，但不能替代集成/故障验收： ```bash python3 -m unittest discover -s tests -v @@ -298,12 +298,18 @@ node --test tests/test_pcm.cjs bash -n deploy/asterisk.sh ``` -下一实施入口是 D12/R1：在取得真实SaaS、SIP/Asterisk、ASR/LLM/TTS、OSS、出口、容量和运维证据后逐项替换Mock并复验；无需再次讨论已接受的A类方案。遇到真实供应商协议/合规/资源事实不兼容，只对具体差异登记变更，不把整个计划退回重选，也不让Mock假成功掩盖差异。 +下一实施入口是整改计划F0→F1～F4，对D01～D11补齐模式保护、租户隔离、可靠性、协议媒体与验收证据；真实SIP排障及D12外部协调并行，不以线路不通阻塞本地整改。真实替换仍须取得SaaS、SIP/Asterisk、ASR/LLM/TTS、OSS、出口、容量和运维证据；无需重新讨论已接受方案。具体供应商差异单独登记，不让Mock假成功掩盖缺口。 -### 10.1 当前执行记录（2026-09-12） +### 10.1 历史执行记录（2026-09-12～13，非当前完整验收结论） + +以下保留历史探针及当时环境记录；其中V01～V13、规模与环境保持运行的表述仅描述当次结果，不表示当前资源仍存在。完整门禁状态以第10节纠偏和整改复验为准，当前云资源状态须另行只读核验，不能据本记录自动复用、创建或拨号。 已落地 D01 的 OpenAPI 3.1、共用 MQ Schema、集中样例和 Mock profile，以及 D02/D03 的 SQLite 持久状态、租户独立 RabbitMQ 拓扑、outbox/inbox、CAS 控制、重放、Mock SIP/ARI/RTP/AI/OSS 协议记录、监控/告警、迁移/备份和 Compose 部署入口。测试号码 `18601013734` 在 Mock 与明确授权的 R1 探针中均保持原值；当前线路目标按规则生成 `708918601013734`。 本轮可重复命令 `python3 -m unittest discover -s tests -v` 和 RabbitMQ Compose smoke 已通过；`python3 -m scripts.acceptance` 的 V01～V13 Mock 探针通过，因 V14 仍需外部真实证据而按门禁返回非零，证据归档于 `docs/evidence/acceptance-mock-2026-09-12.json`。SCALE-MOCK 在阿里云北京专用竞价 Cell `ecs.hfg6.large` 上完成：100 租户、1200 模拟呼叫、禁用1个Cell，最低1004路已回答并连续观察3600秒；详见 `docs/evidence/scale-mock-2026-09-12.json`。该证据仅证明 Mock 协议/持久状态负载基线，`continuous_media_evidence` 为 false，不替代真实媒体容量。 -R1 已在持久测试环境完成两次真实 Asterisk/SIP 单路探针：分别使用原始被叫 `18601013734` 和 `18625770806`，均按 `7089` 前缀构造目标。环境 ECS `i-2zegaldd07mkp0xyhcff` 使用 `SpotAsPriceGo` 自动竞价和固定 EIP `123.56.71.98`；PJSIP contact/OPTIONS 可达，真实 INVITE 已送达供应商，两次均收到 `488`、`Q.850 cause=88 INCOMPATIBLE_DESTINATION`，未接通、未建立 RTP，证据见 `docs/evidence/asterisk-persistent-2026-09-13.md`。环境按用户要求保持运行，未删除实例/EIP。真实接通/RTP、ASR/LLM/TTS/OSS、生产 1000 路完整 AI、N+1、实际CPS/SLO及副作用正确性仍阻塞，现有产物不得据此宣称生产完成。 +R1 已在历史持久测试环境完成两次真实 Asterisk/SIP 单路探针：分别使用原始被叫 `18601013734` 和 `18625770806`，均按 `7089` 前缀构造目标。历史环境 ECS `i-2zegaldd07mkp0xyhcff` 使用 `SpotAsPriceGo` 自动竞价和固定 EIP `123.56.71.98`；PJSIP contact/OPTIONS 可达，真实 INVITE 已送达供应商，两次均收到 `488`、`Q.850 cause=88 INCOMPATIBLE_DESTINATION`，未接通、未建立 RTP。 + +补充当前执行（2026-09-13）：按明确授权重新创建并绑定 ECS `i-2ze4hvi6xvy02g0wz9kk`，探针期间固定 EIP 为 `InUse`；Asterisk `22.10.1` 容器为 `running/healthy`，固定镜像 digest、PJSIP contact `Avail`（RTT约22.164ms）及安全组范围已只读核验。对原始被叫 `18601013734` 单次探针构造 `708918601013734`，收到 `100 trying` 后 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`，无 `180/183/200`、RTP 或录音证据；无重试、无 FALLBACK。探针后按用户授权停止并删除该 ECS，系统盘随实例删除，固定 EIP 未释放且已恢复 `Available`；详见 `docs/evidence/asterisk-persistent-2026-09-13.md`。 + +真实接通/RTP、ASR/LLM/TTS/OSS、生产 1000 路完整 AI、N+1、实际CPS/SLO及副作用正确性仍阻塞，现有产物不得据此宣称生产完成。 diff --git a/docs/部署接入_运行说明.md b/docs/部署接入_运行说明.md index d7985b8..34ac10e 100644 --- a/docs/部署接入_运行说明.md +++ b/docs/部署接入_运行说明.md @@ -238,14 +238,14 @@ python3 deploy/migrate.py data/agent-call.sqlite3 ### 7.3 当前门禁 -本地V01-V12及阿里云专用主机上的V13 Mock探针已通过；完整验收脚本仍因V14需要外部真实证据而按门禁返回非零，证据见`docs/evidence/acceptance-mock-2026-09-12.json`和`docs/evidence/scale-mock-2026-09-12.json`。V13使用100租户/1200模拟接通、禁用1个Cell，最低1004路已回答并持续观察3600秒；这是Mock协议/持久状态负载证据，不是RTP或生产容量证据。R1临时探针见`docs/evidence/asterisk-real-2026-09-12.md`，持久环境两号码探针见`docs/evidence/asterisk-persistent-2026-09-13.md`：固定EIP和SIP OPTIONS/INVITE链路可达，但两个目标均收到488/Q.850 cause=88。当前持久测试ECS保持运行，使用SpotAsPriceGo，固定EIP`123.56.71.98`保持InUse。真实接通/RTP、ASR/LLM/TTS/OSS、生产1000路/N+1尚未完成。正式模式拒绝当前Mock profile，不能把测试号码或Mock成功自动升级为真实外呼授权。 +本地V01-V12及阿里云专用主机上的V13 Mock探针已通过；最新完整验收结果见`docs/evidence/acceptance-mock-2026-09-13.json`，脚本仍因V13缺持续媒体独立证据和V14需要外部真实证据而按门禁返回非零。V13使用100租户/1200模拟接通、禁用1个Cell；这是Mock协议/持久状态负载证据，不是RTP或生产容量证据。R1历史探针见`docs/evidence/asterisk-real-2026-09-12.md`，当前单次探针见`docs/evidence/asterisk-persistent-2026-09-13.md`：ECS `i-2ze4hvi6xvy02g0wz9kk`、Asterisk容器和固定EIP `123.56.71.98` 已核验，PJSIP contact/OPTIONS可达，但 `18601013734` 对应目标收到 `488/Q.850 cause=88`，未接通、无RTP/录音证据。探针后该 ECS 已按授权停止并删除，固定 EIP 保持 Available。真实接通/RTP、ASR/LLM/TTS/OSS、生产1000路/N+1尚未完成。正式模式拒绝当前Mock profile，不能把测试号码或Mock成功自动升级为真实外呼授权。 ## 8. 下一步需要用户提供/确认 1. 供应商需确认`7089`前缀/目标号码路由、From域（当前实际出现私网域）及允许的编解码；两个持久环境探针均返回`488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`，未通过外呼验收。 -2. 供应商调整后在当前持久测试环境复测；环境使用SpotAsPriceGo，竞价回收可能中断通话，不自动重试、不改绑或释放固定EIP。 +2. 供应商调整并确认线路规则后，再用只读plan和新授权创建测试实例复测；本次实例已回收，固定EIP保持 Available，不自动重试、不改绑或释放固定EIP。 3. ASR测试凭据、批准的模型/资源ID；Web HTTPS域名/证书或SSH访问方案。 4. 用户制定的MQ/OSS ID接口规范，以及新的LLM/TTS协议、参数与取消/打断规则。 5. 真实接通后仍需分别完成RTP、录音、MQ/OSS、ASR/LLM/TTS和生产1000路/N+1验收。 -在供应商规则确认前，不再次拨打真实号码，也不声称完成整个平台；当前持久测试环境按用户要求保留。 +在供应商规则确认前，不再次拨打真实号码，也不声称完成整个平台；本次测试实例已回收，固定EIP按要求保留。 diff --git a/scripts/acceptance.py b/scripts/acceptance.py index 89f6456..4f76893 100644 --- a/scripts/acceptance.py +++ b/scripts/acceptance.py @@ -7,12 +7,15 @@ exists. Passing Mock probes never upgrades the result to production readiness. from __future__ import annotations import copy +import io import json import tempfile import time from collections.abc import Callable +from contextlib import redirect_stdout from pathlib import Path -from typing import Any +from types import TracebackType +from typing import Any, Self from agent_call.core import ( AgentCallService, @@ -221,11 +224,14 @@ def probe_v04() -> dict[str, Any]: class ConflictContext: - def __enter__(self) -> ConflictContext: + def __enter__(self) -> Self: return self def __exit__( - self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: Any + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, ) -> bool: if exc_type is not ConflictError: raise AssertionError("expected revision conflict") @@ -256,10 +262,37 @@ def probe_v06_v07() -> dict[str, Any]: "disconnect": "ari_disconnect", } outcomes: dict[str, str] = {} + uncertain_before_reconcile: str | None = None for label, scenario in scenarios.items(): body = command(f"cmd_{label}") svc.set_failure(body["command_id"], scenario) - snapshot = run_success(svc, body) + svc.publish_execute(body) + if scenario == "ari_disconnect": + call_id = None + before = None + for _ in range(500): + svc.process_once() + current = svc.get_command("tenant-demo", body["command_id"]) + call_id = current.get("call_id") + if call_id: + before = svc.get_call("tenant-demo", call_id) + if before["call_state"] == "reconciling": + break + time.sleep(0.01) + if not call_id or before is None: + raise AssertionError("disconnect probe did not create a call") + uncertain_before_reconcile = before["call_state"] + if uncertain_before_reconcile != "reconciling": + raise AssertionError( + "disconnect probe did not retain unknown state" + ) + svc.reconcile_call( + call_id, ended=True, outcome="failed", reason="ARI_RECONCILED" + ) + snapshot = svc.get_command("tenant-demo", body["command_id"]) + else: + svc.wait_for_idle() + snapshot = svc.get_command("tenant-demo", body["command_id"]) call = svc.get_call("tenant-demo", snapshot["call_id"]) outcomes[label] = call["outcome"] if len(call["attempts"]) != 1: @@ -280,6 +313,7 @@ def probe_v06_v07() -> dict[str, Any]: "asterisk_real": "not_run", "outcomes": outcomes, "fallback_redials": 0, + "disconnect_state_before_reconcile": uncertain_before_reconcile, } @@ -374,9 +408,37 @@ def probe_v11_v12() -> dict[str, Any]: body = command("cmd_backup") run_success(svc, body) backup = Path(directory) / "backup.sqlite3" - copy_database(str(Path(directory) / "executor.sqlite3"), str(backup)) + with redirect_stdout(io.StringIO()): + copy_database(str(Path(directory) / "executor.sqlite3"), str(backup)) if not backup.is_file(): raise AssertionError("backup file was not created") + restored = AgentCallService( + db_path=backup, + object_dir=Path(directory) / "restored-objects", + broker=InMemoryBroker(), + ) + try: + restored_command = restored.get_command("tenant-demo", body["command_id"]) + restored_call = restored.get_call( + "tenant-demo", restored_command["call_id"] + ) + if ( + restored_command["status"] != "completed" + or restored_call["recordings"][0]["status"] != "verified" + ): + raise AssertionError("restored database lost the completed call facts") + restored_counts = { + "commands": restored.store.one("SELECT COUNT(*) AS n FROM commands")[ + "n" + ], + "events": restored.store.one("SELECT COUNT(*) AS n FROM events")["n"], + "inbox": restored.store.one("SELECT COUNT(*) AS n FROM inbox")["n"], + "recordings": restored.store.one( + "SELECT COUNT(*) AS n FROM recordings" + )["n"], + } + finally: + restored.stop() metrics = svc.metrics() if "agent_call_outbox_pending 0" not in metrics or 'mode="mock"' not in metrics: raise AssertionError("monitoring metrics lack healthy mock state") @@ -392,7 +454,9 @@ def probe_v11_v12() -> dict[str, Any]: return { "backup": "verified", "metrics": "healthy", - "rollback_preserves_db": True, + "rollback_preserves_db": "not_run", + "restored_db_facts": True, + "restored_counts": restored_counts, "real_mode_mock_rejection": True, } @@ -419,10 +483,15 @@ def probe_v13() -> dict[str, Any]: if ( evidence.get("status") != "observed" or evidence.get("production_claim", True) + or not evidence.get("continuous_media_evidence", False) or load.get("minimum_answered_calls", 0) < 1000 or load.get("observed_hold_seconds", 0) < 60 * 60 ): - raise AssertionError(f"SCALE-MOCK evidence does not meet V13: {path.name}") + return { + "status": "not_run", + "evidence_file": str(path.relative_to(ROOT)), + "reason": "historical state-only evidence cannot satisfy continuous media V13", + } return { "status": "observed", "evidence_file": str(path.relative_to(ROOT)), @@ -466,7 +535,14 @@ def run() -> dict[str, Any]: "duration_ms": round((time.monotonic() - started) * 1000, 3), "evidence": evidence, } - except Exception as exc: + except ( + AssertionError, + KeyError, + OSError, + RuntimeError, + ServiceError, + ValueError, + ) as exc: results["probes"][name] = { "status": "FAIL", "duration_ms": round((time.monotonic() - started) * 1000, 3), diff --git a/scripts/scale_mock.py b/scripts/scale_mock.py index f118612..11e88fc 100644 --- a/scripts/scale_mock.py +++ b/scripts/scale_mock.py @@ -14,7 +14,13 @@ import time from pathlib import Path from typing import Any -from agent_call.core import AgentCallService, InMemoryBroker, iso, utcnow +from agent_call.core import ( + AgentCallService, + InMemoryBroker, + ServiceError, + iso, + utcnow, +) from scripts.acceptance import load_fixture ROOT = Path(__file__).resolve().parents[1] @@ -44,6 +50,8 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]: { "cell_id": f"cell-{index:03d}", "capacity": 12, + "media_capacity": 12, + "ai_capacity": 12, "egress_pool_id": "egress-mock", "ari_mode": "mock", } @@ -53,10 +61,14 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]: profile["limits"].update( { "global_concurrency": 1200, - "global_cps": 1200, + "global_cps": 20, "tenant_concurrency": 12, - "tenant_cps": 12, - "max_queue_messages": 2000, + "tenant_cps": 1, + "max_queue_messages": 1000, + "max_queue_bytes": 16 * 1024 * 1024, + "pending_window_per_tenant": 16, + "pending_window_global": 64, + "max_unacked_per_tenant": 4, # Keep calls alive through warm-up and the required observation window. "hold_ms": (startup_window_seconds + duration_minutes * 60 + 5) * 1000, } @@ -101,6 +113,9 @@ def dry_run(duration_minutes: int) -> dict[str, Any]: def cleanup_load(svc: AgentCallService) -> dict[str, Any]: + # The failed-cell probe is restored before a full control barrier is applied; + # a barrier must not claim every cell applied while one cell is unreachable. + svc.set_cell_health("cell-000", True) requested = 0 for tenant_index in range(100): tenant_id = f"tenant-{tenant_index:03d}" @@ -162,7 +177,7 @@ def cleanup_load(svc: AgentCallService) -> dict[str, Any]: def run(directory: Path, duration_minutes: int) -> dict[str, Any]: profile_path = directory / "profile.json" build_profile(profile_path, duration_minutes) - broker = InMemoryBroker(max_messages=2000) + broker = InMemoryBroker(max_messages=1000, max_bytes=16 * 1024 * 1024) svc = AgentCallService( db_path=directory / "scale.sqlite3", profile_path=profile_path, @@ -174,11 +189,17 @@ def run(directory: Path, duration_minutes: int) -> dict[str, Any]: template = load_fixture() started = time.monotonic() try: - for tenant_index in range(100): - for offset in range(12): - svc.publish_execute( - command_for(template, tenant_index * 12 + offset, tenant_index) - ) + for offset in range(12): + for tenant_index in range(100): + body = command_for(template, tenant_index * 12 + offset, tenant_index) + while True: + try: + svc.publish_execute(body) + break + except ServiceError as exc: + if exc.code != "PUBLISH_RATE_LIMIT": + raise + time.sleep(1.05) warmup_deadline = started + max(120, duration_minutes * 60) active = 0 threshold_at = None diff --git a/tests/test_agent_call.py b/tests/test_agent_call.py index 22e94e5..cbf02ae 100644 --- a/tests/test_agent_call.py +++ b/tests/test_agent_call.py @@ -4,6 +4,7 @@ import copy import json import tempfile import unittest +from datetime import timedelta from pathlib import Path from agent_call.core import ( @@ -104,6 +105,53 @@ class AgentCallTests(unittest.TestCase): attempts = self.service.store.one("SELECT COUNT(*) AS n FROM attempts")["n"] self.assertEqual(calls, 1) self.assertEqual(attempts, 1) + self.assertEqual( + self.service.store.one( + "SELECT status FROM executions WHERE tenant_id=? AND execution_id=?", + ("tenant-demo", "exec_once"), + )["status"], + "completed", + ) + + def test_unhealthy_cell_isolated_without_blocking_healthy_capacity(self) -> None: + self.service.set_cell_health("cell-a", False) + self.assertTrue(self.service.ready()) + snapshot = self.run_command(self.command("cmd_healthy_cell")) + call = self.service.get_call("tenant-demo", snapshot["call_id"]) + self.assertEqual(call["call_state"], "ended") + self.assertEqual( + self.service.store.one( + "SELECT cell_id FROM calls WHERE call_id=?", (snapshot["call_id"],) + )["cell_id"], + "cell-b", + ) + + def test_missing_tenant_route_closes_readiness(self) -> None: + self.service._unrouteable_tenants.add("tenant-demo") + snapshot = self.service.readiness() + self.assertFalse(snapshot["checks"]["routes"]) + self.assertFalse(snapshot["ready"]) + + def test_expired_scheduler_lease_fences_old_instance(self) -> None: + body = self.command("cmd_lease_fence") + self.service.publish_execute(body) + self.service.ingest_commands() + second = AgentCallService( + db_path=Path(self.temp.name) / "executor.sqlite3", + object_dir=Path(self.temp.name) / "objects-second", + broker=self.broker, + ) + try: + with self.service.store.tx() as con: + con.execute( + "UPDATE leases SET expires_at=? WHERE lease_name='scheduler'", + (iso(utcnow() - timedelta(seconds=1)),), + ) + self.assertEqual(second.schedule_once(), 1) + self.assertEqual(self.service.schedule_once(), 0) + second.wait_for_idle() + finally: + second.stop() def test_controls_use_cas_and_old_commands_do_not_resume(self) -> None: pause = self.service.control_task( diff --git a/tests/test_deployment.py b/tests/test_deployment.py index c1c5325..01ffff5 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -1,5 +1,6 @@ import base64 import copy +import ipaddress import json import tempfile import unittest @@ -293,10 +294,10 @@ class AsteriskTests(unittest.TestCase): def test_reject_injection_public_ari_and_our_ip_as_provider(self): for update in ( - {"ari_bind": ".".join(("0", "0", "0", "0"))}, + {"ari_bind": str(ipaddress.IPv4Address(0))}, {"ari_bind": "8.8.8.8"}, {"local_net": "0.0.0.0/0"}, - ): # noqa: S104 — negative fixtures; renderer must reject them. + ): cfg = self.cfg() cfg.update(update) with self.assertRaises(ValueError): diff --git a/tests/test_http.py b/tests/test_http.py index cb23236..013a0ea 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -42,6 +42,10 @@ class HttpContractTests(unittest.TestCase): "recording.complete", ], }, + "replay": { + "tenant_ids": ["tenant-demo"], + "scopes": ["outbound.read", "outbound.replay"], + }, } ) self.service = AgentCallService( @@ -73,14 +77,18 @@ class HttpContractTests(unittest.TestCase): path: str, body: dict[str, Any] | None = None, tenant: str | None = "tenant-demo", - token: str | None = "local", + auth_name: str | None = None, idem: str | None = None, + include_auth: bool = True, ) -> tuple[int, dict[str, Any]]: headers = {"X-Request-ID": "http-test"} if tenant is not None: headers["X-Tenant-ID"] = tenant - if token is not None: - headers["Authorization"] = f"Bearer {token}" + if include_auth and auth_name is None: + token_map = json.loads(os.environ["HTTP_TOKENS"]) + auth_name = next(iter(token_map), None) + if include_auth and auth_name is not None: + headers["Authorization"] = f"Bearer {auth_name}" encoded: bytes | None = None if body is not None: encoded = json.dumps(body, ensure_ascii=False).encode("utf-8") @@ -104,7 +112,7 @@ class HttpContractTests(unittest.TestCase): except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise AssertionError("HTTP response is not JSON") from exc if not isinstance(parsed, dict): - raise AssertionError("HTTP response is not an object") + raise TypeError("HTTP response is not an object") return status, parsed def publish_success(self) -> dict[str, Any]: @@ -118,11 +126,13 @@ class HttpContractTests(unittest.TestCase): return self.service.get_command("tenant-demo", "http_cmd") def test_health_is_public_but_business_queries_are_authenticated(self) -> None: - status, body = self.request("GET", "/healthz/live", tenant=None, token=None) + status, body = self.request( + "GET", "/healthz/live", tenant=None, include_auth=False + ) self.assertEqual(status, 200) self.assertEqual(body["status"], "live") status, body = self.request( - "GET", "/internal/v1/outbound/commands/missing", token=None + "GET", "/internal/v1/outbound/commands/missing", include_auth=False ) self.assertEqual(status, 401) self.assertEqual(body["code"], "UNAUTHORIZED") @@ -160,6 +170,28 @@ class HttpContractTests(unittest.TestCase): ) self.assertEqual(status, 403) + def test_replay_requires_replay_scope(self) -> None: + snapshot = self.publish_success() + body = {"command_id": "http_replay", "reason": "retry"} + status, response = self.request( + "POST", + f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays", + body, + auth_name="local", + idem="http_replay", + ) + self.assertEqual(status, 403) + self.assertEqual(response["code"], "FORBIDDEN") + status, response = self.request( + "POST", + f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays", + body, + auth_name="replay", + idem="http_replay", + ) + self.assertEqual(status, 202) + self.assertEqual(response["status"], "accepted") + def test_recording_upload_contract_is_scoped(self) -> None: snapshot = self.publish_success() call = self.service.get_call("tenant-demo", snapshot["call_id"]) diff --git a/tests/test_remediation_plan.py b/tests/test_remediation_plan.py new file mode 100644 index 0000000..8a7e0be --- /dev/null +++ b/tests/test_remediation_plan.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import copy +import json +import os +import tempfile +import time +import unittest +from pathlib import Path + +from agent_call.core import ( + AgentCallService, + ConfigurationError, + ConflictError, + InMemoryBroker, + Store, + ValidationError, + iso, + utcnow, +) +from deploy.backup_sqlite import copy_database + +ROOT = Path(__file__).resolve().parents[1] + + +def fixture() -> dict: + return json.loads( + (ROOT / "docs/contracts/examples/call.execute.json").read_text(encoding="utf-8") + ) + + +class RemediationTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.broker = InMemoryBroker() + self.service = AgentCallService( + Path(self.temp.name) / "executor.sqlite3", + object_dir=Path(self.temp.name) / "objects", + broker=self.broker, + ) + self.service.profile["limits"]["tenant_cps"] = 100 + self.service.profile["limits"]["global_cps"] = 100 + + def tearDown(self) -> None: + self.service.stop() + self.temp.cleanup() + + def command( + self, + command_id: str, + tenant_id: str = "tenant-demo", + execution_id: str = "exec", + ) -> dict: + body = copy.deepcopy(fixture()) + body["command_id"] = command_id + body["trace_id"] = f"trace-{command_id}" + body["issued_at"] = iso(utcnow()) + body["tenant_id"] = tenant_id + tenant = self.service.store.one( + "SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,) + ) + body["tenant_key"] = tenant["tenant_key"] + body["payload"]["execution_id"] = execution_id + body["payload"]["task_id"] = ( + "task-demo" if tenant_id == "tenant-demo" else "task-b" + ) + return body + + def test_runtime_schema_rejects_naive_time_and_boolean_revision(self) -> None: + body = self.command("schema-command") + body["issued_at"] = "2026-09-13T00:00:00" + with self.assertRaises(ValidationError): + self.service.publish_execute(body) + with self.assertRaises(ValidationError): + self.service.control_task( + "tenant-demo", + "task-demo", + { + "command_id": "bool-revision", + "action": "pause", + "expected_task_revision": True, + "reason": "x", + }, + "bool-revision", + ) + + def test_readiness_requires_persistent_database_write(self) -> None: + db_path = Path(self.temp.name) / "executor.sqlite3" + os.chmod(db_path, 0o444) + try: + readiness = self.service.readiness() + finally: + os.chmod(db_path, 0o644) + self.assertFalse(readiness["checks"]["database_write"]) + self.assertFalse(readiness["ready"]) + + def test_provider_modes_do_not_fallback(self) -> None: + profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text()) + profile["provider_modes"].pop("llm") + path = Path(self.temp.name) / "profile.json" + path.write_text(json.dumps(profile), encoding="utf-8") + with self.assertRaises(ConfigurationError): + AgentCallService( + profile_path=path, object_dir=Path(self.temp.name) / "objects2" + ) + + def test_same_public_ids_are_tenant_scoped_and_wrong_route_is_dead_lettered( + self, + ) -> None: + first = self.command("same-command", "tenant-demo", "same-execution") + second = self.command("same-command", "tenant-b", "same-execution") + self.service.publish_execute(first) + self.service.publish_execute(second) + self.assertEqual(self.service.ingest_commands(3), 2) + self.assertEqual(self.service.schedule_once(), 2) + self.service.wait_for_idle() + self.assertEqual( + self.service.store.one("SELECT COUNT(*) AS n FROM calls")["n"], 2 + ) + + wrong = self.command("wrong-route", "tenant-b", "wrong-execution") + self.broker.publish( + "agent-call.commands.v1", + "agent-call.tenant.tenant-demo-key.call.execute", + wrong, + "wrong-route", + ) + self.service._cursor = self.service._tenant_order.index("tenant-demo") + self.service.ingest_commands(1) + self.assertEqual( + self.service.store.one("SELECT COUNT(*) AS n FROM dead_letters")["n"], 1 + ) + self.assertIsNone( + self.service.store.one( + "SELECT * FROM calls WHERE command_id=?", ("wrong-route",) + ) + ) + + def test_over_budget_tenant_is_retained_without_truncation(self) -> None: + profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text()) + key = "x" * 225 + profile["tenants"] = [ + {"tenant_id": "long", "tenant_key": key, "max_concurrency": 1} + ] + profile["tasks"] = [ + { + "task_id": "long-task", + "tenant_id": "long", + "state": "running", + "revision": 1, + } + ] + path = Path(self.temp.name) / "long-profile.json" + path.write_text(json.dumps(profile), encoding="utf-8") + service = AgentCallService( + Path(self.temp.name) / "long.sqlite3", + profile_path=path, + object_dir=Path(self.temp.name) / "long-objects", + broker=InMemoryBroker(), + ) + try: + body = self.command("placeholder") + body["tenant_id"], body["tenant_key"] = "long", key + body["payload"]["task_id"] = "long-task" + body["payload"]["execution_id"] = "long-execution" + with self.assertRaises(ValidationError) as error: + service.publish_execute(body) + self.assertEqual(error.exception.code, "TENANT_KEY_UNSUPPORTED") + row = service.store.one( + "SELECT status FROM published_commands WHERE tenant_id=?", ("long",) + ) + self.assertEqual(row["status"], "retained") + finally: + service.stop() + + def test_control_barrier_blocks_resume_until_all_cells_apply(self) -> None: + pause = { + "command_id": "pause-1", + "action": "pause", + "expected_task_revision": 1, + "reason": "maintenance", + } + self.service.set_cell_health("cell-a", False) + self.service.control_task("tenant-demo", "task-demo", pause, "pause-1") + self.assertEqual(self.service.apply_controls(), 0) + with self.assertRaises(ConflictError) as error: + self.service.control_task( + "tenant-demo", + "task-demo", + { + "command_id": "resume-1", + "action": "resume", + "expected_task_revision": 2, + "reason": "resume", + }, + "resume-1", + ) + self.assertEqual(error.exception.code, "CONTROL_IN_PROGRESS") + self.service.set_cell_health("cell-a", True) + self.assertEqual(self.service.apply_controls(), 1) + self.assertEqual( + self.service.store.one( + "SELECT COUNT(*) AS n FROM control_barriers WHERE command_id=? AND status='applied'", + ("pause-1",), + )["n"], + 2, + ) + + def test_final_transcript_is_idempotent_and_conflicts_are_preserved(self) -> None: + self.service.profile["limits"]["hold_ms"] = 1000 + body = self.command("transcript-command", "tenant-demo", "transcript-execution") + self.service.publish_execute(body) + self.service.ingest_commands() + self.service.schedule_once() + deadline = time.monotonic() + 2 + call_id = None + while time.monotonic() < deadline: + row = self.service.store.one( + "SELECT * FROM calls WHERE command_id=?", (body["command_id"],) + ) + if row and row["started_at"]: + call_id = row["call_id"] + break + time.sleep(0.01) + self.assertIsNotNone(call_id) + assert call_id is not None + self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed") + before = self.service.store.one( + "SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'" + )["n"] + self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed") + after = self.service.store.one( + "SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'" + )["n"] + self.assertEqual(before, after) + with self.assertRaises(ConflictError): + self.service._add_transcript( + call_id, 99, "agent", "different", "confirmed", revision=2 + ) + with self.assertRaises(ConflictError): + self.service._add_transcript( + call_id, 99, "agent", "different", "confirmed", revision=1 + ) + self.service._add_transcript( + call_id, 99, "agent", "final text", "sent", revision=3, is_final=False + ) + segment = self.service.store.one( + "SELECT text, is_final, playback_state FROM transcripts WHERE segment_id=?", + (f"seg_{call_id}_99_agent",), + ) + self.assertEqual( + (segment["text"], segment["is_final"], segment["playback_state"]), + ("final text", 1, "confirmed"), + ) + self.service._finish_call(call_id, "failed", "TEST_FINISHED") + + def test_upload_rejects_metadata_changes(self) -> None: + body = self.command( + "recording-metadata-command", "tenant-demo", "recording-metadata-execution" + ) + self.service.publish_execute(body) + self.service.wait_for_idle() + call_id = self.service.get_command("tenant-demo", body["command_id"])["call_id"] + recording = self.service.get_call("tenant-demo", call_id)["recordings"][0] + request = { + "recording_id": recording["recording_id"], + "call_id": call_id, + "content_type": "audio/wav", + "size_bytes": recording["size_bytes"], + "checksum_algorithm": "SHA-256", + "checksum": recording["checksum"], + "channels": 1, + "sample_rate_hz": recording["sample_rate_hz"], + "duration_ms": recording["duration_ms"] + 1, + } + with self.assertRaises(ConflictError) as error: + self.service.create_upload("tenant-demo", request, "metadata-change", "api") + self.assertEqual(error.exception.code, "RECORDING_METADATA_CONFLICT") + + def test_recording_is_non_silent_and_upload_completion_is_idempotent(self) -> None: + body = self.command("recording-command", "tenant-demo", "recording-execution") + self.service.publish_execute(body) + self.service.wait_for_idle() + call = self.service.get_call( + "tenant-demo", + self.service.get_command("tenant-demo", body["command_id"])["call_id"], + ) + recording = call["recordings"][0] + upload = self.service.store.one( + "SELECT upload_id FROM uploads WHERE tenant_id=? AND recording_id=? ORDER BY created_at DESC LIMIT 1", + ("tenant-demo", recording["recording_id"]), + ) + self.assertIsNotNone(upload) + assert upload is not None + path = self.service.object_store.final_path_for(upload["upload_id"]) + self.assertTrue(any(path.read_bytes())) + request = { + "recording_id": recording["recording_id"], + "call_id": call["call_id"], + "content_type": "audio/wav", + "size_bytes": recording["size_bytes"], + "checksum_algorithm": "SHA-256", + "checksum": recording["checksum"], + "channels": 1, + "sample_rate_hz": recording["sample_rate_hz"], + "duration_ms": recording["duration_ms"], + } + status, first = self.service.create_upload( + "tenant-demo", request, "api-create", "api" + ) + status2, second = self.service.create_upload( + "tenant-demo", request, "api-create", "api" + ) + self.assertEqual((status, status2), (200, 200)) + self.assertEqual(first["upload_id"], second["upload_id"]) + self.assertEqual( + self.service.store.one( + "SELECT COUNT(*) AS n FROM recordings WHERE call_id=?", + (call["call_id"],), + )["n"], + 1, + ) + + def test_backup_refuses_to_replace_existing_database(self) -> None: + source = Path(self.temp.name) / "source.sqlite3" + destination = Path(self.temp.name) / "backup.sqlite3" + Store(source) + copy_database(source, destination) + with self.assertRaises(FileExistsError): + copy_database(source, destination) + + +if __name__ == "__main__": + unittest.main()