rogee/agent-call
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b95d2aee81792ced273eace4f47b4bca6b2f87ae
agent-call/agent_call/core.py
T

4289 lines
172 KiB
Python
Raw Blame History

"""Small, durable mock-first call executor.
The module intentionally keeps the state machine in one place. SQLite is used as
an actual relational store for local acceptance; RabbitMQ is an optional
publisher/consumer adapter selected by configuration. No HTTP endpoint accepts
an execution command.
"""
from __future__ import annotations
import hashlib
import importlib
import io
import json
import math
import os
import queue
import re
import secrets
import shutil
import sqlite3
import struct
import threading
import time
import uuid
import wave
from collections.abc import Callable, Iterator
from contextlib import contextmanager, suppress
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, cast
from .mocks import MockAI, MockARI, MockRTP, MockSIPPeer, ProtocolJournal
try:
from jsonschema import Draft202012Validator, FormatChecker
except ImportError: # pragma: no cover - the Docker image installs it
Draft202012Validator = None
FormatChecker = None
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_PROFILE = ROOT / "docs" / "contracts" / "mock-profile.json"
DEFAULT_MQ_SCHEMA = ROOT / "docs" / "contracts" / "mq.schema.json"
class ServiceError(Exception):
def __init__(
self, code: str, detail: str, status: int = 400, retryable: bool = False
) -> None:
super().__init__(detail)
self.code = code
self.detail = detail
self.status = status
self.retryable = retryable
class BrokerError(ServiceError):
def __init__(self, detail: str = "message broker unavailable") -> None:
super().__init__("BROKER_UNAVAILABLE", detail, 503, True)
class ConfigurationError(ServiceError):
def __init__(self, detail: str) -> None:
super().__init__("CONFIGURATION_INVALID", detail, 500, False)
class NotFoundError(ServiceError):
def __init__(self, detail: str = "resource not found") -> None:
super().__init__("NOT_FOUND", detail, 404, False)
class ConflictError(ServiceError):
def __init__(self, code: str, detail: str) -> None:
super().__init__(code, detail, 409, False)
class ValidationError(ServiceError):
def __init__(self, code: str, detail: str) -> None:
super().__init__(code, detail, 422, False)
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def iso(value: datetime | None = None) -> str:
value = value or utcnow()
return (
value.astimezone(timezone.utc)
.isoformat(timespec="milliseconds")
.replace("+00:00", "Z")
)
def parse_time(value: str) -> datetime:
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except (TypeError, ValueError) as exc:
raise ValidationError("INVALID_TIME", "timestamp must be RFC3339") from exc
if parsed.tzinfo is None:
raise ValidationError("INVALID_TIME", "timestamp must include a timezone")
return parsed.astimezone(timezone.utc)
def new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex}"
def canonical(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def digest(value: Any) -> str:
return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()
def routing_key(tenant_key: str) -> str:
value = f"agent-call.tenant.{tenant_key}.call.execute"
if len(value.encode("utf-8")) > 255:
raise ValidationError(
"TENANT_KEY_UNSUPPORTED",
"tenant routing key exceeds the 255-byte broker limit; command is retained and not sent",
)
return value
def queue_name(tenant_key: str) -> str:
value = f"agent-call.executor.{tenant_key}.v1"
if len(value.encode("utf-8")) > 255:
raise ValidationError(
"TENANT_KEY_UNSUPPORTED", "tenant queue exceeds the broker limit"
)
return value
def _json(value: Any) -> str:
return canonical(value)
def _loads(value: str | None, default: Any = None) -> Any:
if value is None:
return default
try:
return json.loads(value)
except (TypeError, json.JSONDecodeError) as exc:
raise ValidationError("INVALID_JSON", "stored JSON is invalid") from exc
def _load_json_file(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ConfigurationError(f"cannot load contract file: {path.name}") from exc
def _integer(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError) as exc:
raise ValidationError("INVALID_INTEGER", "numeric value is invalid") from exc
SCHEMA_SQL = """
PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
applied_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS readiness_probe (
probe_id INTEGER PRIMARY KEY CHECK(probe_id = 1),
touched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS tenants (
tenant_id TEXT PRIMARY KEY,
tenant_key TEXT NOT NULL UNIQUE,
enabled INTEGER NOT NULL DEFAULT 1,
max_concurrency INTEGER NOT NULL DEFAULT 2,
max_cps INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS tasks (
task_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES tenants(tenant_id),
state TEXT NOT NULL,
revision INTEGER NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS commands (
id INTEGER PRIMARY KEY AUTOINCREMENT,
command_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
tenant_key TEXT NOT NULL,
command_type TEXT NOT NULL,
status TEXT NOT NULL,
payload_json TEXT NOT NULL,
payload_hash TEXT NOT NULL,
operation_type TEXT NOT NULL DEFAULT '',
operation_target TEXT NOT NULL DEFAULT '',
task_id TEXT,
execution_id TEXT,
call_id TEXT,
trace_id TEXT,
issued_at TEXT,
not_after TEXT,
admission_deadline TEXT,
accepted_at TEXT,
waiting_since TEXT,
requested_task_revision INTEGER,
applied_task_revision INTEGER,
task_state TEXT,
reason_code TEXT,
wait_reason_code TEXT,
snapshot_cutoff TEXT,
source_type TEXT,
source_id TEXT,
aggregate_version INTEGER NOT NULL DEFAULT 1,
updated_at TEXT NOT NULL,
UNIQUE(tenant_id, command_id)
);
CREATE INDEX IF NOT EXISTS commands_sched_idx ON commands(tenant_id, status, accepted_at);
CREATE TABLE IF NOT EXISTS executions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
execution_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
command_id TEXT NOT NULL,
task_id TEXT NOT NULL,
task_item_id TEXT NOT NULL,
payload_hash TEXT NOT NULL,
status TEXT NOT NULL,
call_id TEXT,
created_at TEXT NOT NULL,
UNIQUE(tenant_id, execution_id)
);
CREATE TABLE IF NOT EXISTS calls (
call_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
execution_id TEXT NOT NULL,
command_id TEXT NOT NULL,
task_id TEXT NOT NULL,
task_item_id TEXT NOT NULL,
callee TEXT NOT NULL,
cell_id TEXT,
trunk_id TEXT,
egress_pool_id TEXT,
call_state TEXT NOT NULL,
call_version INTEGER NOT NULL DEFAULT 1,
outcome TEXT,
reason_code TEXT,
started_at TEXT,
ended_at TEXT,
duration_ms INTEGER,
created_at TEXT NOT NULL,
UNIQUE(tenant_id, execution_id)
);
CREATE INDEX IF NOT EXISTS calls_active_idx ON calls(tenant_id, call_state);
CREATE TABLE IF NOT EXISTS attempts (
attempt_id TEXT PRIMARY KEY,
call_id TEXT NOT NULL REFERENCES calls(call_id),
attempt_number INTEGER NOT NULL,
route_policy_id TEXT NOT NULL,
caller_profile_id TEXT NOT NULL,
target_uri TEXT NOT NULL,
status TEXT NOT NULL,
started_at TEXT,
ended_at TEXT,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS transcripts (
segment_id TEXT PRIMARY KEY,
call_id TEXT NOT NULL REFERENCES calls(call_id),
turn_index INTEGER NOT NULL,
speaker TEXT NOT NULL,
text TEXT NOT NULL,
revision INTEGER NOT NULL,
is_final INTEGER NOT NULL,
playback_state TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(call_id, turn_index, speaker)
);
CREATE TABLE IF NOT EXISTS recordings (
recording_id TEXT PRIMARY KEY,
call_id TEXT NOT NULL REFERENCES calls(call_id),
tenant_id TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
checksum_algorithm TEXT NOT NULL,
checksum TEXT NOT NULL,
channels INTEGER NOT NULL,
sample_rate_hz INTEGER NOT NULL,
duration_ms INTEGER NOT NULL,
status TEXT NOT NULL,
oss_id TEXT,
upload_id TEXT,
failure_code TEXT,
created_at TEXT NOT NULL,
verified_at TEXT
);
CREATE TABLE IF NOT EXISTS uploads (
upload_id TEXT PRIMARY KEY,
recording_id TEXT NOT NULL REFERENCES recordings(recording_id),
tenant_id TEXT NOT NULL,
expected_size INTEGER NOT NULL,
expected_checksum TEXT NOT NULL,
expires_at TEXT NOT NULL,
status TEXT NOT NULL,
object_path TEXT NOT NULL,
created_at TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS uploads_recording_idx ON uploads(tenant_id, recording_id, created_at);
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
tenant_key TEXT NOT NULL,
trace_id TEXT NOT NULL,
event_type TEXT NOT NULL,
aggregate_type TEXT NOT NULL,
aggregate_id TEXT NOT NULL,
aggregate_version INTEGER NOT NULL,
occurred_at TEXT NOT NULL,
payload_json TEXT NOT NULL,
body_json TEXT NOT NULL,
published INTEGER NOT NULL DEFAULT 0,
broker_status TEXT NOT NULL DEFAULT 'pending',
broker_attempts INTEGER NOT NULL DEFAULT 0,
published_at TEXT,
next_attempt_at TEXT,
saas_applied INTEGER,
replay_of TEXT,
UNIQUE(tenant_id, event_id)
);
CREATE INDEX IF NOT EXISTS events_outbox_idx ON events(published, next_attempt_at, occurred_at);
CREATE TABLE IF NOT EXISTS replays (
id INTEGER PRIMARY KEY AUTOINCREMENT,
replay_id TEXT NOT NULL UNIQUE,
command_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
source_type TEXT NOT NULL,
source_id TEXT NOT NULL,
snapshot_cutoff TEXT NOT NULL,
status TEXT NOT NULL,
sent_count INTEGER NOT NULL DEFAULT 0,
failure_code TEXT,
retry_attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(tenant_id, command_id)
);
CREATE TABLE IF NOT EXISTS replay_items (
replay_id TEXT NOT NULL REFERENCES replays(replay_id),
event_id TEXT NOT NULL,
published_at TEXT NOT NULL,
PRIMARY KEY(replay_id, event_id)
);
CREATE TABLE IF NOT EXISTS inbox (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
applied_at TEXT NOT NULL,
body_json TEXT NOT NULL,
UNIQUE(tenant_id, event_id)
);
CREATE TABLE IF NOT EXISTS dead_letters (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_key TEXT,
routing_key TEXT,
body_json TEXT NOT NULL,
reason_code TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS published_commands (
id INTEGER PRIMARY KEY AUTOINCREMENT,
command_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
body_json TEXT NOT NULL,
body_hash TEXT NOT NULL,
status TEXT NOT NULL,
retry_attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
last_error_code TEXT,
created_at TEXT NOT NULL,
UNIQUE(tenant_id, command_id)
);
CREATE TABLE IF NOT EXISTS control_barriers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
command_id TEXT NOT NULL,
tenant_id TEXT NOT NULL,
cell_id TEXT NOT NULL,
status TEXT NOT NULL,
applied_at TEXT,
UNIQUE(tenant_id, command_id, cell_id)
);
CREATE TABLE IF NOT EXISTS storage_operations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_id TEXT NOT NULL,
actor_id TEXT NOT NULL,
operation_type TEXT NOT NULL,
target_id TEXT NOT NULL,
idempotency_key TEXT NOT NULL,
payload_hash TEXT NOT NULL,
status INTEGER NOT NULL,
response_json TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(tenant_id, actor_id, operation_type, target_id, idempotency_key)
);
CREATE TABLE IF NOT EXISTS leases (
lease_name TEXT PRIMARY KEY,
owner_id TEXT NOT NULL,
expires_at TEXT NOT NULL,
generation INTEGER NOT NULL DEFAULT 0
);
"""
class Store:
def __init__(self, path: str | os.PathLike[str]) -> None:
requested = str(path)
self._shared_memory = requested == ":memory:"
self.path = (
f"file:agent_call_{uuid.uuid4().hex}?mode=memory&cache=shared"
if self._shared_memory
else requested
)
self._keeper: sqlite3.Connection | None = None
if self._shared_memory:
self._keeper = sqlite3.connect(
self.path, uri=True, timeout=10, isolation_level=None
)
else:
Path(self.path).parent.mkdir(parents=True, exist_ok=True)
self._init_lock = threading.Lock()
self.init_schema()
def close(self) -> None:
if self._keeper is not None:
self._keeper.close()
self._keeper = None
def connect(self) -> sqlite3.Connection:
con = sqlite3.connect(
self.path, uri=self._shared_memory, timeout=10, isolation_level=None
)
con.row_factory = sqlite3.Row
con.execute("PRAGMA foreign_keys = ON")
con.execute("PRAGMA busy_timeout = 10000")
return con
def init_schema(self) -> None:
with self._init_lock:
con = self.connect()
try:
con.executescript(SCHEMA_SQL)
version_row = con.execute(
"SELECT MAX(version) AS version FROM schema_migrations"
).fetchone()
version = int(version_row["version"] or 0)
if version < 2 and not self._has_column(con, "commands", "id"):
self._migrate_identity_keys(con)
if version < 3:
self._add_compatibility_columns(con)
con.execute(
"INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES(3, ?)",
(iso(),),
)
finally:
con.close()
@staticmethod
def _has_column(con: sqlite3.Connection, table: str, column: str) -> bool:
table_info_sql = {
"commands": "PRAGMA table_info(commands)",
"executions": "PRAGMA table_info(executions)",
"calls": "PRAGMA table_info(calls)",
"events": "PRAGMA table_info(events)",
"replays": "PRAGMA table_info(replays)",
"replay_items": "PRAGMA table_info(replay_items)",
"inbox": "PRAGMA table_info(inbox)",
"published_commands": "PRAGMA table_info(published_commands)",
"leases": "PRAGMA table_info(leases)",
}
return any(
row["name"] == column
for row in con.execute(table_info_sql[table]).fetchall()
)
def _migrate_identity_keys(self, con: sqlite3.Connection) -> None:
# The first release used globally unique command/execution/event IDs. Keep
# the facts, but move uniqueness to (tenant_id, public_id) so independent
# tenants may safely reuse their own idempotency namespace.
tables = (
"commands",
"executions",
"calls",
"events",
"replays",
"replay_items",
"inbox",
"published_commands",
)
con.execute("PRAGMA foreign_keys=OFF")
con.execute("DROP INDEX IF EXISTS commands_sched_idx")
rename_sql = {
"commands": "ALTER TABLE commands RENAME TO _legacy_commands",
"executions": "ALTER TABLE executions RENAME TO _legacy_executions",
"calls": "ALTER TABLE calls RENAME TO _legacy_calls",
"events": "ALTER TABLE events RENAME TO _legacy_events",
"replays": "ALTER TABLE replays RENAME TO _legacy_replays",
"replay_items": "ALTER TABLE replay_items RENAME TO _legacy_replay_items",
"inbox": "ALTER TABLE inbox RENAME TO _legacy_inbox",
"published_commands": "ALTER TABLE published_commands RENAME TO _legacy_published_commands",
}
for table in tables:
con.execute(rename_sql[table])
con.executescript(SCHEMA_SQL)
# SCHEMA_SQL enables foreign keys; keep them off while the disposable
# tables are rolled back and the legacy names are restored.
con.execute("PRAGMA foreign_keys=OFF")
committed = False
con.execute("BEGIN")
try:
con.execute(
"""INSERT INTO commands(
command_id, tenant_id, tenant_key, command_type, status,
payload_json, payload_hash, operation_type, operation_target,
task_id, execution_id, call_id, trace_id, issued_at, not_after,
admission_deadline, accepted_at, waiting_since,
requested_task_revision, applied_task_revision, task_state,
reason_code, wait_reason_code, snapshot_cutoff, source_type,
source_id, aggregate_version, updated_at)
SELECT command_id, tenant_id, tenant_key, command_type, status,
payload_json, payload_hash, command_type,
COALESCE(task_id, source_id, ''), task_id, execution_id, call_id,
trace_id, issued_at, not_after, admission_deadline, accepted_at,
waiting_since, requested_task_revision, applied_task_revision,
task_state, reason_code, wait_reason_code, snapshot_cutoff,
source_type, source_id, aggregate_version, updated_at
FROM _legacy_commands"""
)
con.execute(
"""INSERT INTO executions(
execution_id, tenant_id, command_id, task_id, task_item_id,
payload_hash, status, call_id, created_at)
SELECT execution_id, tenant_id, command_id, task_id, task_item_id,
payload_hash, status, call_id, created_at
FROM _legacy_executions"""
)
con.execute(
"""INSERT INTO calls(
call_id, tenant_id, execution_id, command_id, task_id,
task_item_id, callee, cell_id, trunk_id, egress_pool_id,
call_state, call_version, outcome, reason_code, started_at,
ended_at, duration_ms, created_at)
SELECT call_id, tenant_id, execution_id, command_id, task_id,
task_item_id, callee, cell_id, trunk_id, egress_pool_id,
call_state, call_version, outcome, reason_code, started_at,
ended_at, duration_ms, created_at
FROM _legacy_calls"""
)
con.execute(
"""INSERT INTO events(
event_id, tenant_id, tenant_key, trace_id, event_type,
aggregate_type, aggregate_id, aggregate_version, occurred_at,
payload_json, body_json, published, broker_status,
broker_attempts, published_at, next_attempt_at, saas_applied,
replay_of)
SELECT event_id, tenant_id, tenant_key, trace_id, event_type,
aggregate_type, aggregate_id, aggregate_version, occurred_at,
payload_json, body_json, published, broker_status,
broker_attempts, published_at, next_attempt_at, saas_applied,
replay_of
FROM _legacy_events"""
)
con.execute(
"""INSERT INTO replays(
replay_id, command_id, tenant_id, source_type, source_id,
snapshot_cutoff, status, sent_count, failure_code, created_at,
updated_at)
SELECT replay_id, command_id, tenant_id, source_type, source_id,
snapshot_cutoff, status, sent_count, failure_code, created_at,
updated_at
FROM _legacy_replays"""
)
con.execute(
"INSERT INTO replay_items(replay_id, event_id, published_at) "
"SELECT replay_id, event_id, published_at FROM _legacy_replay_items"
)
con.execute(
"INSERT INTO inbox(event_id, tenant_id, applied_at, body_json) "
"SELECT event_id, tenant_id, applied_at, body_json FROM _legacy_inbox"
)
con.execute(
"""INSERT INTO published_commands(
command_id, tenant_id, body_json, body_hash, status, created_at)
SELECT command_id, tenant_id, body_json, body_hash, status,
created_at FROM _legacy_published_commands"""
)
con.execute("COMMIT")
committed = True
except Exception:
con.execute("ROLLBACK")
raise
finally:
if committed:
for table in tables:
con.execute(f"DROP TABLE IF EXISTS _legacy_{table}")
else:
# Keep the original tables usable if the copy failed. The new
# schema is disposable because no migration version was saved.
for table in reversed(tables):
con.execute(f"DROP TABLE IF EXISTS {table}")
for table in tables:
con.execute(f"ALTER TABLE _legacy_{table} RENAME TO {table}")
con.execute("PRAGMA foreign_keys=ON")
def _add_compatibility_columns(self, con: sqlite3.Connection) -> None:
additions = {
"published_commands": (
("retry_attempts", "INTEGER NOT NULL DEFAULT 0"),
("next_attempt_at", "TEXT"),
("last_error_code", "TEXT"),
),
"replays": (
("retry_attempts", "INTEGER NOT NULL DEFAULT 0"),
("next_attempt_at", "TEXT"),
),
"leases": (("generation", "INTEGER NOT NULL DEFAULT 0"),),
}
alter_sql = {
(
"published_commands",
"retry_attempts",
): "ALTER TABLE published_commands ADD COLUMN retry_attempts INTEGER NOT NULL DEFAULT 0",
(
"published_commands",
"next_attempt_at",
): "ALTER TABLE published_commands ADD COLUMN next_attempt_at TEXT",
(
"published_commands",
"last_error_code",
): "ALTER TABLE published_commands ADD COLUMN last_error_code TEXT",
(
"replays",
"retry_attempts",
): "ALTER TABLE replays ADD COLUMN retry_attempts INTEGER NOT NULL DEFAULT 0",
(
"replays",
"next_attempt_at",
): "ALTER TABLE replays ADD COLUMN next_attempt_at TEXT",
(
"leases",
"generation",
): "ALTER TABLE leases ADD COLUMN generation INTEGER NOT NULL DEFAULT 0",
}
for table, columns in additions.items():
for column, _definition in columns:
if not self._has_column(con, table, column):
con.execute(alter_sql[(table, column)])
@contextmanager
def tx(self) -> Iterator[sqlite3.Connection]:
con = self.connect()
con.execute("BEGIN IMMEDIATE")
try:
yield con
con.commit()
except Exception:
con.rollback()
raise
finally:
con.close()
def one(self, sql: str, params: tuple[Any, ...] = ()) -> sqlite3.Row:
con = self.connect()
try:
# pi-lens-ignore: python-sql-injection
return cast(sqlite3.Row, con.execute(sql, params).fetchone())
finally:
con.close()
def all(self, sql: str, params: tuple[Any, ...] = ()) -> list[sqlite3.Row]:
con = self.connect()
try:
# pi-lens-ignore: python-sql-injection
return list(con.execute(sql, params).fetchall())
finally:
con.close()
class InMemoryBroker:
"""A strict direct-exchange broker used for deterministic acceptance tests."""
def __init__(self, max_messages: int = 0, max_bytes: int = 0) -> None:
self._queues: dict[str, queue.Queue[dict[str, Any]]] = {}
self._queue_bytes: dict[str, int] = {}
self.max_messages = max_messages
self.max_bytes = max_bytes
self._routes: dict[str, str] = {}
self._lock = threading.Lock()
self.outage = False
self.blocked = False
self.fail_next: str | None = None
self.fail_after: int | None = None
self._successful_publishes = 0
self.published: list[dict[str, Any]] = []
def declare_tenant(self, tenant_key: str) -> None:
name = queue_name(tenant_key)
route = routing_key(tenant_key)
with self._lock:
self._queues.setdefault(name, queue.Queue(maxsize=self.max_messages))
self._queue_bytes.setdefault(name, 0)
self._routes[route] = name
def declare_events(self) -> None:
with self._lock:
self._queues.setdefault(
"agent-call.saas.events.v1", queue.Queue(maxsize=self.max_messages)
)
self._queue_bytes.setdefault("agent-call.saas.events.v1", 0)
def publish(
self,
exchange: str,
route: str,
body: dict[str, Any],
message_id: str | None = None,
headers: dict[str, str] | None = None,
) -> None:
if self.outage:
raise BrokerError()
with self._lock:
if self.blocked:
raise BrokerError("broker is blocked; publication is retained")
if self.fail_next:
reason = self.fail_next
self.fail_next = None
raise BrokerError(reason)
if (
self.fail_after is not None
and self._successful_publishes >= self.fail_after
):
raise BrokerError("publisher confirmation was lost")
if exchange == "agent-call.commands.v1":
target = self._routes.get(route)
else:
target = (
"agent-call.saas.events.v1"
if route.startswith("agent-call.")
else None
)
if target is None or target not in self._queues:
raise BrokerError("mandatory publish was unroutable")
encoded_size = len(canonical(body).encode("utf-8"))
if (
self.max_bytes
and self._queue_bytes[target] + encoded_size > self.max_bytes
):
raise BrokerError("queue byte limit reached; publication is retained")
message = {
"exchange": exchange,
"routing_key": route,
"body": body,
"message_id": message_id,
"headers": headers or {},
"queue_name": target,
"_size": encoded_size,
}
try:
self._queues[target].put_nowait(message)
except queue.Full as exc:
raise BrokerError("queue is full; publication is retained") from exc
self._queue_bytes[target] += encoded_size
self._successful_publishes += 1
self.published.append(message)
def ack(self, message: dict[str, Any]) -> None:
return
def reject(self, message: dict[str, Any]) -> None:
return
@staticmethod
def _message_size(message: dict[str, Any]) -> int:
try:
return max(0, int(message.get("_size", 0)))
except (TypeError, ValueError):
return 0
def requeue(self, message: dict[str, Any]) -> None:
name = message.get("queue_name")
if not name:
raise BrokerError("message has no queue context")
with self._lock:
target = self._queues.get(name)
if target is None:
raise BrokerError("queue is not declared")
try:
target.put_nowait(message)
except queue.Full as exc:
raise BrokerError("queue is full while requeuing") from exc
self._queue_bytes[name] += self._message_size(message)
def consume(self, name: str) -> dict[str, Any] | None:
if self.outage:
raise BrokerError()
with self._lock:
target = self._queues.get(name)
if target is None:
raise BrokerError("queue is not declared")
try:
message = target.get_nowait()
except queue.Empty:
return None
self._queue_bytes[name] = max(
0, self._queue_bytes[name] - self._message_size(message)
)
return message
def depth(self, name: str) -> int:
with self._lock:
target = self._queues.get(name)
return target.qsize() if target is not None else 0
def queue_bytes(self, name: str) -> int:
with self._lock:
return self._queue_bytes.get(name, 0)
def has_route(self, route: str) -> bool:
with self._lock:
return route in self._routes
def ready(self) -> bool:
return not self.outage and not self.blocked
class PikaBroker:
"""RabbitMQ adapter. Import and connection are optional in local unit tests."""
def __init__(
self, url: str, max_messages: int = 1000, max_bytes: int = 16 * 1024 * 1024
) -> None:
try:
pika = importlib.import_module("pika")
except ImportError as exc: # pragma: no cover
raise ConfigurationError(
"BROKER_MODE=rabbit requires the pika package"
) from exc
self.pika = pika
self.max_messages = max_messages
self.max_bytes = max_bytes
self._declared_routes: set[str] = set()
try:
self.connection = pika.BlockingConnection(pika.URLParameters(url))
self.channel = self.connection.channel()
self.channel.confirm_delivery()
self.channel.basic_qos(prefetch_count=1)
self.channel.exchange_declare(
exchange="agent-call.commands.v1", exchange_type="direct", durable=True
)
self.channel.exchange_declare(
exchange="agent-call.dead.v1", exchange_type="direct", durable=True
)
self.channel.exchange_declare(
exchange="agent-call.events.v1", exchange_type="topic", durable=True
)
for queue, dead_route in (
("agent-call.commands.dead.v1", "agent-call.commands.dead"),
("agent-call.events.dead.v1", "agent-call.events.dead"),
):
self.channel.queue_declare(
queue=queue,
durable=True,
arguments={
"x-max-length": self.max_messages,
"x-max-length-bytes": self.max_bytes,
"x-overflow": "reject-publish",
},
)
self.channel.queue_bind(
queue=queue,
exchange="agent-call.dead.v1",
routing_key=dead_route,
)
self.channel.queue_declare(
queue="agent-call.saas.events.v1",
durable=True,
arguments={
"x-max-length": self.max_messages,
"x-max-length-bytes": self.max_bytes,
"x-overflow": "reject-publish",
"x-dead-letter-exchange": "agent-call.dead.v1",
"x-dead-letter-routing-key": "agent-call.events.dead",
},
)
self.channel.queue_bind(
queue="agent-call.saas.events.v1",
exchange="agent-call.events.v1",
routing_key="agent-call.#",
)
self._blocked = False
add_blocked = getattr(
self.connection, "add_on_connection_blocked_callback", None
)
if callable(add_blocked):
add_blocked(
lambda _connection, _reason: setattr(self, "_blocked", True)
)
add_unblocked = getattr(
self.connection, "add_on_connection_unblocked_callback", None
)
if callable(add_unblocked):
add_unblocked(lambda _connection: setattr(self, "_blocked", False))
except Exception as exc: # pragma: no cover
raise BrokerError(
f"RabbitMQ connection failed: {type(exc).__name__}"
) from exc
def declare_tenant(self, tenant_key: str) -> None: # pragma: no cover
name = queue_name(tenant_key)
route = routing_key(tenant_key)
self.channel.queue_declare(
queue=name,
durable=True,
arguments={
"x-max-length": self.max_messages,
"x-max-length-bytes": self.max_bytes,
"x-overflow": "reject-publish",
"x-dead-letter-exchange": "agent-call.dead.v1",
"x-dead-letter-routing-key": "agent-call.commands.dead",
},
)
self.channel.queue_bind(
queue=name, exchange="agent-call.commands.v1", routing_key=route
)
self._declared_routes.add(route)
def declare_events(self) -> None: # pragma: no cover
return
def publish(
self,
exchange: str,
route: str,
body: dict[str, Any],
message_id: str | None = None,
headers: dict[str, str] | None = None,
) -> None: # pragma: no cover
try:
props = self.pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
message_id=message_id,
headers=headers or {},
)
confirmed = self.channel.basic_publish(
exchange=exchange,
routing_key=route,
body=canonical(body).encode("utf-8"),
properties=props,
mandatory=True,
)
if confirmed is not None and not confirmed:
raise BrokerError("publisher confirmation was negative")
except BrokerError:
raise
except Exception as exc:
raise BrokerError(f"RabbitMQ publish failed: {type(exc).__name__}") from exc
def consume(self, name: str) -> dict[str, Any] | None: # pragma: no cover
try:
method, props, body = self.channel.basic_get(queue=name, auto_ack=False)
if method is None or props is None or body is None:
return None
try:
parsed = _loads(body.decode("utf-8"))
except (ServiceError, UnicodeDecodeError):
parsed = {"_invalid_payload": True}
return {
"body": parsed,
"message_id": props.message_id,
"headers": props.headers or {},
"routing_key": getattr(method, "routing_key", None),
"queue_name": name,
"_delivery_tag": method.delivery_tag,
}
except Exception as exc:
raise BrokerError(f"RabbitMQ consume failed: {type(exc).__name__}") from exc
def requeue(self, message: dict[str, Any]) -> None: # pragma: no cover
tag = message.get("_delivery_tag")
if tag is None:
return
try:
self.channel.basic_nack(tag, requeue=True)
except Exception as exc:
raise BrokerError("RabbitMQ requeue failed") from exc
def depth(self, name: str) -> int: # pragma: no cover
try:
return int(
self.channel.queue_declare(
queue=name, passive=True
).method.message_count
)
except Exception as exc:
raise BrokerError("RabbitMQ queue depth failed") from exc
def has_route(self, route: str) -> bool: # pragma: no cover
return route in self._declared_routes
def ack(self, message: dict[str, Any]) -> None: # pragma: no cover
tag = message.get("_delivery_tag")
if tag is None:
return
try:
self.channel.basic_ack(tag)
except Exception as exc:
raise BrokerError("RabbitMQ acknowledgement failed") from exc
def reject(self, message: dict[str, Any]) -> None: # pragma: no cover
tag = message.get("_delivery_tag")
if tag is None:
return
try:
self.channel.basic_nack(tag, requeue=False)
except Exception as exc:
raise BrokerError("RabbitMQ rejection failed") from exc
def ready(self) -> bool: # pragma: no cover
return bool(
self.connection.is_open and self.channel.is_open and not self._blocked
)
class MockObjectStore:
_id_pattern = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
def __init__(self, root: str | os.PathLike[str], secret: str | None = None) -> None:
self.root = Path(root)
self.root.mkdir(parents=True, exist_ok=True)
seed = secret or os.environ.get("MOCK_UPLOAD_SECRET") or self.root.as_posix()
self.secret = seed.encode("utf-8")
@classmethod
def _check_id(cls, upload_id: str) -> None:
if not isinstance(upload_id, str) or not cls._id_pattern.fullmatch(upload_id):
raise ValidationError("UPLOAD_ID_INVALID", "upload id is invalid")
def path_for(self, upload_id: str) -> Path:
self._check_id(upload_id)
return self.root / f"{upload_id}.part"
def final_path_for(self, upload_id: str) -> Path:
self._check_id(upload_id)
return self.root / upload_id
def has_final(self, upload_id: str) -> bool:
return self.final_path_for(upload_id).is_file()
def token(self, upload_id: str, tenant_id: str) -> str:
self._check_id(upload_id)
return hashlib.sha256(
self.secret + b":" + tenant_id.encode() + b":" + upload_id.encode()
).hexdigest()
def put(self, upload_id: str, tenant_id: str, data: bytes, token: str) -> None:
if not secrets.compare_digest(token, self.token(upload_id, tenant_id)):
raise ValidationError("UPLOAD_TOKEN_INVALID", "upload token is invalid")
if not isinstance(data, bytes):
raise ValidationError("OBJECT_BODY_INVALID", "object body must be bytes")
self.path_for(upload_id).write_bytes(data)
def verify_and_promote(
self, upload_id: str, expected_size: int, expected_checksum: str
) -> str:
final = self.final_path_for(upload_id)
path = self.path_for(upload_id)
candidate = final if final.exists() else path
if not candidate.exists():
raise ValidationError("OBJECT_MISSING", "object was not uploaded")
data = candidate.read_bytes()
actual = hashlib.sha256(data).hexdigest()
if len(data) != expected_size or actual != expected_checksum:
raise ValidationError(
"OBJECT_VERIFICATION_FAILED",
"size or checksum does not match the upload contract",
)
if candidate == path:
os.replace(path, final)
return f"oss://mock/{upload_id}"
class AgentCallService:
def __init__(
self,
db_path: str | os.PathLike[str] = ":memory:",
profile_path: str | os.PathLike[str] = DEFAULT_PROFILE,
object_dir: str | os.PathLike[str] | None = None,
broker: InMemoryBroker | PikaBroker | None = None,
mode: str | None = None,
clock: Callable[[], datetime] = utcnow,
start_background: bool = False,
) -> None:
self.clock = clock
self.profile_path = Path(profile_path)
self.profile = _load_json_file(self.profile_path)
self.mq_schema = _load_json_file(DEFAULT_MQ_SCHEMA)
self.mode = mode or self.profile.get("mode", "mock")
self._validate_profile()
self.store = Store(db_path)
self.instance_id = new_id("instance")
object_root = (
Path(object_dir)
if object_dir
else (
Path(db_path).parent / "objects"
if str(db_path) != ":memory:"
else Path("data") / "agent-call-objects"
)
)
self.object_root = object_root
self.object_store = MockObjectStore(
object_root, os.environ.get("MOCK_UPLOAD_SECRET")
)
self.journal = ProtocolJournal(object_root / "protocol.jsonl")
self.mock_sip = MockSIPPeer(self.journal)
self.mock_ari = MockARI(self.journal)
self.mock_rtp = MockRTP(self.journal)
self.mock_ai = MockAI(self.journal)
if broker is None:
broker_mode = self.profile.get("provider_modes", {}).get(
"rabbitmq", "memory"
)
if broker_mode == "rabbit" or os.environ.get("BROKER_MODE") == "rabbit":
broker_url = os.environ.get("RABBITMQ_URL")
if not broker_url:
raise ConfigurationError("BROKER_MODE=rabbit requires RABBITMQ_URL")
broker = PikaBroker(
broker_url,
max_messages=_integer(
self._limits().get("max_queue_messages", 1000)
),
max_bytes=_integer(
self._limits().get("max_queue_bytes", 16 * 1024 * 1024)
),
)
else:
broker = InMemoryBroker(
_integer(self._limits().get("max_queue_messages", 1000)),
_integer(self._limits().get("max_queue_bytes", 16 * 1024 * 1024)),
)
self.broker = broker
self.profile.setdefault("provider_modes", {})["rabbitmq"] = (
"rabbit" if isinstance(broker, PikaBroker) else "memory"
)
self._provider_status = {
str(component): str(provider)
for component, provider in self.profile.get("provider_modes", {}).items()
}
self._unrouteable_tenants: set[str] = set()
self._seed()
self._tenant_order = [
item["tenant_id"] for item in self.profile.get("tenants", [])
]
self._cell_health = {
item["cell_id"]: bool(item.get("enabled", True))
for item in self.profile.get("cells", [])
}
self._cursor = 0
self._failures: dict[str, str] = {}
self._call_threads: dict[str, threading.Thread] = {}
self._stop = threading.Event()
self._background: threading.Thread | None = None
self._last_error: str | None = None
self._last_wall_clock: datetime | None = None
self._last_monotonic: float | None = None
self._clock_jump = False
self._recover_durable_state()
if start_background:
self.start()
def _validate_profile(self) -> None:
if self.mode not in {"mock", "mixed", "real"}:
raise ConfigurationError("mode must be mock, mixed, or real")
providers = self.profile.get("provider_modes", {})
if not isinstance(providers, dict):
raise ConfigurationError("provider_modes must be an object")
mock_values = {
"saas": "mock",
"database": "sqlite",
"rabbitmq": "memory",
"sip": "mock",
"asterisk": "mock",
"asr": "mock",
"llm": "mock",
"tts": "mock",
"oss": "mock",
"cloud": "fake-cli",
}
if self.mode == "real":
raise ConfigurationError(
"real mode is unavailable until every real provider adapter is configured; mock fallback is forbidden"
)
if self.mode == "mixed":
raise ConfigurationError(
"mixed mode requires explicit real adapters; this build has none"
)
unknown = set(providers) - set(mock_values)
if unknown:
raise ConfigurationError(f"unknown provider components: {sorted(unknown)}")
missing = set(mock_values) - set(providers)
if missing:
raise ConfigurationError(f"provider modes are missing: {sorted(missing)}")
for component, expected in mock_values.items():
configured = providers[component]
if configured != expected and not (
component == "rabbitmq" and configured == "rabbit"
):
raise ConfigurationError(
f"mock mode refuses provider {component}={configured!r}"
)
seen_ids: set[str] = set()
seen_keys: set[str] = set()
for tenant in self.profile.get("tenants", []):
tenant_id = tenant.get("tenant_id")
tenant_key = tenant.get("tenant_key")
if not isinstance(tenant_id, str) or not tenant_id:
raise ConfigurationError("tenant_id must be a non-empty string")
if not isinstance(tenant_key, str) or not tenant_key:
raise ConfigurationError("tenant_key must be a non-empty string")
if tenant_id in seen_ids or tenant_key in seen_keys:
raise ConfigurationError(
"tenant identifiers and tenant_key values must be unique"
)
seen_ids.add(tenant_id)
seen_keys.add(tenant_key)
if not self.profile.get("cells"):
raise ConfigurationError("at least one cell is required")
for cell in self.profile["cells"]:
if _integer(cell.get("capacity", 0)) < 1:
raise ConfigurationError("cell capacity must be positive")
def _seed(self) -> None:
now = iso(self.clock())
limits = self.profile.get("limits", {})
with self.store.tx() as con:
for tenant in self.profile.get("tenants", []):
con.execute(
"INSERT OR IGNORE INTO tenants(tenant_id, tenant_key, enabled, max_concurrency, max_cps, created_at) VALUES(?,?,?,?,?,?)",
(
tenant["tenant_id"],
tenant["tenant_key"],
_integer(tenant.get("enabled", True)),
limits.get("tenant_concurrency", 2),
limits.get("tenant_cps", 1),
now,
),
)
for task in self.profile.get("tasks", []):
con.execute(
"INSERT OR IGNORE INTO tasks(task_id, tenant_id, state, revision, updated_at) VALUES(?,?,?,?,?)",
(
task["task_id"],
task["tenant_id"],
task.get("state", "running"),
task.get("revision", 1),
now,
),
)
for tenant in self.profile.get("tenants", []):
try:
self.broker.declare_tenant(tenant["tenant_key"])
except ValidationError:
# Keep an over-budget tenant in durable state. Its original
# command is retained instead of being truncated or rerouted.
self._unrouteable_tenants.add(tenant["tenant_id"])
self.broker.declare_events()
def _recover_durable_state(self) -> None:
"""Fence unfinished calls on restart; never infer that a channel ended."""
with self.store.tx() as con:
calls = con.execute(
"SELECT * FROM calls WHERE call_state NOT IN ('ended', 'failed', 'reconciling')"
).fetchall()
now = iso(self._now())
for call in calls:
con.execute(
"UPDATE calls SET call_state='reconciling', reason_code='RECOVERY_RECONCILING', call_version=call_version+1 WHERE call_id=?",
(call["call_id"],),
)
con.execute(
"UPDATE commands SET status='reconciling', reason_code='RECOVERY_RECONCILING', updated_at=? WHERE tenant_id=? AND command_id=? AND status NOT IN ('completed','failed','rejected','applied')",
(now, call["tenant_id"], call["command_id"]),
)
con.execute(
"UPDATE executions SET status='reconciling' WHERE tenant_id=? AND execution_id=? AND status NOT IN ('completed','failed','rejected')",
(call["tenant_id"], call["execution_id"]),
)
def start(self) -> None:
if self._background and self._background.is_alive():
return
self._stop.clear()
self._background = threading.Thread(
target=self._run_loop, name="agent-call-loop", daemon=True
)
self._background.start()
def stop(self) -> None:
self._stop.set()
deadline = time.monotonic() + 2
if self._background:
self._background.join(timeout=max(0, deadline - time.monotonic()))
for thread in list(self._call_threads.values()):
remaining = deadline - time.monotonic()
if remaining <= 0:
break
thread.join(timeout=remaining)
self.store.close()
def _run_loop(self) -> None:
while not self._stop.is_set():
try:
self.process_once()
except (
ServiceError,
OSError,
RuntimeError,
TypeError,
ValueError,
KeyError,
sqlite3.DatabaseError,
) as exc:
# The next tick retries durable outbox and waiting work. Do not log
# payloads, credentials, or raw broker diagnostics.
self._last_error = type(exc).__name__
self._stop.wait(0.02)
def _now(self) -> datetime:
now = self.clock().astimezone(timezone.utc)
monotonic = time.monotonic()
if self._last_wall_clock is not None and self._last_monotonic is not None:
wall_delta = (now - self._last_wall_clock).total_seconds()
monotonic_delta = monotonic - self._last_monotonic
if abs(wall_delta - monotonic_delta) > 2:
self._clock_jump = True
self._last_wall_clock = now
self._last_monotonic = monotonic
return now
def _limits(self) -> dict[str, Any]:
return self.profile.get("limits", {})
@staticmethod
def _operation_hash(operation: str, target: str, body: dict[str, Any]) -> str:
return digest({"operation": operation, "target": target, "body": body})
def _route_config(self, route_policy_id: str) -> dict[str, Any]:
for route in self.profile.get("routes", []):
if route.get("route_policy_id") == route_policy_id and route.get(
"allowed", True
):
return route
raise ValidationError(
"ROUTE_POLICY_NOT_ALLOWED", "route policy is not configured"
)
def _tenant(self, tenant_id: str, tenant_key: str | None = None) -> sqlite3.Row:
row = self.store.one("SELECT * FROM tenants WHERE tenant_id = ?", (tenant_id,))
if (
row is None
or not row["enabled"]
or (tenant_key is not None and row["tenant_key"] != tenant_key)
):
raise ValidationError(
"TENANT_BINDING_INVALID",
"tenant_id and tenant_key are not an authorized pair",
)
return row
def _validate_message(self, body: dict[str, Any]) -> None:
if not isinstance(body, dict):
raise ValidationError("SCHEMA_INVALID", "MQ message must be a JSON object")
if len(canonical(body).encode("utf-8")) > self._limits().get(
"max_mq_bytes", 262144
):
raise ValidationError(
"MESSAGE_TOO_LARGE", "message exceeds the MQ body limit"
)
if Draft202012Validator is not None:
errors = sorted(
Draft202012Validator(
self.mq_schema,
format_checker=FormatChecker() if FormatChecker else None,
).iter_errors(body),
key=lambda error: list(error.path),
)
if errors:
path = ".".join(str(part) for part in errors[0].path) or "body"
raise ValidationError("SCHEMA_INVALID", f"invalid MQ message at {path}")
else:
required = {
"schema_version",
"command_type",
"command_id",
"tenant_id",
"tenant_key",
"trace_id",
"issued_at",
"not_after",
"payload",
}
if not required.issubset(body):
raise ValidationError(
"SCHEMA_INVALID", "required MQ fields are missing"
)
def _validate_command_times(self, body: dict[str, Any]) -> None:
try:
issued_at = parse_time(body["issued_at"])
not_after = parse_time(body["not_after"])
except (TypeError, ValueError, OverflowError, KeyError) as exc:
raise ValidationError(
"SCHEMA_INVALID", "command timestamps are invalid"
) from exc
now = self._now()
if issued_at > now + timedelta(seconds=5) or not_after <= issued_at:
raise ValidationError(
"SCHEMA_INVALID", "command timestamp window is invalid"
)
if not_after <= now:
raise ConflictError("ADMISSION_EXPIRED", "command is past not_after")
def _validate_event(self, body: dict[str, Any]) -> None:
if Draft202012Validator is not None:
errors = sorted(
Draft202012Validator(
self.mq_schema,
format_checker=FormatChecker() if FormatChecker else None,
).iter_errors(body),
key=lambda error: list(error.path),
)
if errors:
raise ConfigurationError(
"service generated an event outside mq.schema.json"
)
def publish_execute(self, body: dict[str, Any]) -> dict[str, Any]:
"""SaaS-side test publisher. The production ingress is the broker queue."""
self._validate_message(body)
tenant = self._tenant(body["tenant_id"], body["tenant_key"])
body_hash = digest(body)
existing_before_admission = self.store.one(
"SELECT * FROM published_commands WHERE command_id=? AND tenant_id=?",
(body["command_id"], tenant["tenant_id"]),
)
if existing_before_admission:
if existing_before_admission["body_hash"] != body_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"command_id was already published with a different body",
)
if existing_before_admission["status"] not in {"pending", "retained"}:
return {
"command_id": body["command_id"],
"status": existing_before_admission["status"],
"routing_key": routing_key(tenant["tenant_key"]),
}
self._validate_command_times(body)
if existing_before_admission:
task = self.store.one(
"SELECT state FROM tasks WHERE task_id=? AND tenant_id=?",
(body["payload"]["task_id"], tenant["tenant_id"]),
)
if task is None:
raise ValidationError(
"TASK_TENANT_MISMATCH", "task is not owned by the command tenant"
)
if task["state"] != "running":
raise ConflictError(
"TASK_PUBLISH_BLOCKED",
"SaaS persistent task block prevents new publication",
)
rate_limited = False
with self.store.tx() as con:
existing = con.execute(
"SELECT * FROM published_commands WHERE command_id = ? AND tenant_id = ?",
(body["command_id"], tenant["tenant_id"]),
).fetchone()
if existing:
if existing["body_hash"] != body_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"command_id was already published with a different body",
)
if existing["status"] not in {"pending", "retained"}:
return {
"command_id": body["command_id"],
"status": existing["status"],
"routing_key": routing_key(tenant["tenant_key"]),
}
if (
existing["last_error_code"] == "PUBLISH_RATE_LIMIT"
and existing["next_attempt_at"]
and parse_time(existing["next_attempt_at"]) > self._now()
):
raise ServiceError(
"PUBLISH_RATE_LIMIT",
"tenant publication rate limit reached",
429,
True,
)
else:
task = con.execute(
"SELECT * FROM tasks WHERE task_id=? AND tenant_id=?",
(body["payload"]["task_id"], tenant["tenant_id"]),
).fetchone()
if task is None:
raise ValidationError(
"TASK_TENANT_MISMATCH",
"task is not owned by the command tenant",
)
if task["state"] != "running":
raise ConflictError(
"TASK_PUBLISH_BLOCKED",
"SaaS persistent task block prevents new publication",
)
created_at = iso(self._now())
con.execute(
"INSERT INTO published_commands(command_id, tenant_id, body_json, body_hash, status, created_at) VALUES(?,?,?,?,?,?)",
(
body["command_id"],
tenant["tenant_id"],
canonical(body),
body_hash,
"pending",
created_at,
),
)
publish_rate = _integer(self._limits().get("tenant_publish_rate", 0))
if publish_rate:
recent = con.execute(
"SELECT COUNT(*) AS n FROM published_commands WHERE tenant_id=? AND created_at>=?",
(tenant["tenant_id"], iso(self._now() - timedelta(seconds=1))),
).fetchone()["n"]
if _integer(recent) > publish_rate:
con.execute(
"UPDATE published_commands SET status='retained', last_error_code='PUBLISH_RATE_LIMIT', next_attempt_at=? WHERE tenant_id=? AND command_id=?",
(
iso(self._now() + timedelta(seconds=1)),
tenant["tenant_id"],
body["command_id"],
),
)
rate_limited = True
if rate_limited:
raise ServiceError(
"PUBLISH_RATE_LIMIT", "tenant publication rate limit reached", 429, True
)
try:
route = routing_key(tenant["tenant_key"])
self.broker.publish(
"agent-call.commands.v1",
route,
body,
body["command_id"],
{"tenant_id": tenant["tenant_id"], "tenant_key": tenant["tenant_key"]},
)
except ValidationError as exc:
with self.store.tx() as con:
con.execute(
"UPDATE published_commands SET status='retained', last_error_code=? WHERE tenant_id=? AND command_id=?",
(exc.code, tenant["tenant_id"], body["command_id"]),
)
raise
except Exception as exc:
with self.store.tx() as con:
attempts = (
con.execute(
"SELECT retry_attempts FROM published_commands WHERE tenant_id=? AND command_id=?",
(tenant["tenant_id"], body["command_id"]),
).fetchone()["retry_attempts"]
+ 1
)
con.execute(
"UPDATE published_commands SET status='retained', retry_attempts=?, next_attempt_at=?, last_error_code=? WHERE tenant_id=? AND command_id=?",
(
attempts,
iso(
self._now()
+ timedelta(seconds=min(60, 2 ** min(attempts, 5)))
),
type(exc).__name__,
tenant["tenant_id"],
body["command_id"],
),
)
raise
with self.store.tx() as con:
con.execute(
"UPDATE published_commands SET status='broker_confirmed', next_attempt_at=NULL, last_error_code=NULL WHERE tenant_id=? AND command_id=?",
(tenant["tenant_id"], body["command_id"]),
)
return {
"command_id": body["command_id"],
"status": "broker_confirmed",
"routing_key": route,
}
def _tenant_window_full(self, tenant_id: str, tenant_key: str) -> bool:
limits = self._limits()
tenant_limit = _integer(limits.get("pending_window_per_tenant", 16))
global_limit = _integer(limits.get("pending_window_global", 256))
tenant_row = self.store.one(
"SELECT COUNT(*) AS n FROM commands WHERE tenant_id=? AND status NOT IN ('completed','failed','rejected','applied')",
(tenant_id,),
)
global_row = self.store.one(
"SELECT COUNT(*) AS n FROM commands WHERE status NOT IN ('completed','failed','rejected','applied')"
)
queue_depth = getattr(self.broker, "depth", lambda _name: 0)(
queue_name(tenant_key)
)
return (
_integer(tenant_row["n"]) >= tenant_limit
or _integer(global_row["n"]) >= global_limit
or queue_depth >= _integer(limits.get("max_unacked_per_tenant", 32))
)
def _validate_delivery(
self,
body: dict[str, Any],
tenant: sqlite3.Row,
message: dict[str, Any],
) -> None:
expected_route = routing_key(tenant["tenant_key"])
expected_queue = queue_name(tenant["tenant_key"])
if message.get("queue_name") != expected_queue:
raise ValidationError(
"TENANT_ROUTE_MISMATCH", "message arrived on the wrong queue"
)
if message.get("routing_key") != expected_route:
raise ValidationError(
"TENANT_ROUTE_MISMATCH",
"message routing key is not bound to the tenant",
)
headers = message.get("headers") or {}
if not isinstance(headers, dict):
raise ValidationError(
"TENANT_ROUTE_MISMATCH", "message headers are invalid"
)
if "tenant_id" in headers and headers["tenant_id"] != tenant["tenant_id"]:
raise ValidationError(
"TENANT_ROUTE_MISMATCH", "tenant_id header does not match the queue"
)
if "tenant_key" in headers and headers["tenant_key"] != tenant["tenant_key"]:
raise ValidationError(
"TENANT_ROUTE_MISMATCH", "tenant_key header does not match the queue"
)
if message.get("message_id") not in {None, body.get("command_id")}:
raise ValidationError(
"MESSAGE_ID_MISMATCH", "message_id does not match command_id"
)
if (
body.get("tenant_id") != tenant["tenant_id"]
or body.get("tenant_key") != tenant["tenant_key"]
):
raise ValidationError(
"TENANT_BINDING_INVALID", "message tenant does not match the queue"
)
def _dead_letter(self, message: dict[str, Any], body: Any, reason: str) -> None:
body_map = body if isinstance(body, dict) else {}
try:
encoded = canonical(body)
except (TypeError, ValueError):
encoded = "{}"
with self.store.tx() as con:
con.execute(
"INSERT INTO dead_letters(tenant_key, routing_key, body_json, reason_code, created_at) VALUES(?,?,?,?,?)",
(
body_map.get("tenant_key"),
message.get("routing_key"),
encoded,
reason,
iso(self._now()),
),
)
def ingest_commands(self, max_per_tick: int | None = None) -> int:
max_per_tick = (
max_per_tick
if max_per_tick is not None
else max(1, len(self._tenant_order))
)
if not self._tenant_order:
return 0
accepted = 0
for offset in range(min(max_per_tick, len(self._tenant_order))):
tenant_id = self._tenant_order[
(self._cursor + offset) % len(self._tenant_order)
]
tenant = self.store.one(
"SELECT * FROM tenants WHERE tenant_id = ?", (tenant_id,)
)
if tenant is None or tenant_id in self._unrouteable_tenants:
continue
try:
if self._tenant_window_full(tenant_id, tenant["tenant_key"]):
continue
message = self.broker.consume(queue_name(tenant["tenant_key"]))
except (BrokerError, ValidationError):
break
if not message:
continue
body: Any = message.get("body", {})
try:
body_map = body if isinstance(body, dict) else {}
self._validate_delivery(body_map, tenant, message)
self.accept_execute(body_map, delivery_context=message)
accepted += 1
with self.store.tx() as con:
con.execute(
"UPDATE published_commands SET status='accepted' WHERE tenant_id=? AND command_id=?",
(tenant_id, body_map.get("command_id")),
)
self.broker.ack(message)
except ServiceError as exc:
self._dead_letter(message, body, exc.code)
body_map = body if isinstance(body, dict) else {}
if body_map.get("command_id"):
with self.store.tx() as con:
con.execute(
"UPDATE published_commands SET status='rejected', last_error_code=? WHERE tenant_id=? AND command_id=?",
(exc.code, tenant_id, body_map["command_id"]),
)
reject = getattr(self.broker, "reject", None)
if callable(reject):
reject(message)
else:
self.broker.ack(message)
except (
OSError,
RuntimeError,
TypeError,
ValueError,
KeyError,
sqlite3.DatabaseError,
):
# Unknown persistence/broker errors must remain deliverable; an
# invalid message is handled above and sent to the dead-letter log.
with suppress(BrokerError):
self.broker.requeue(message)
break
self._cursor = (self._cursor + max_per_tick) % len(self._tenant_order)
return accepted
def accept_execute(
self,
body: dict[str, Any],
delivery_context: dict[str, Any] | None = None,
) -> dict[str, Any]:
self._validate_message(body)
self._validate_command_times(body)
tenant = self._tenant(body["tenant_id"], body["tenant_key"])
if delivery_context is not None:
self._validate_delivery(body, tenant, delivery_context)
payload = body["payload"]
task = self.store.one(
"SELECT * FROM tasks WHERE task_id = ? AND tenant_id = ?",
(payload["task_id"], tenant["tenant_id"]),
)
if task is None or task["tenant_id"] != tenant["tenant_id"]:
raise ValidationError(
"TASK_TENANT_MISMATCH", "task is not owned by the command tenant"
)
command_hash = digest(body)
now = self._now()
not_after = parse_time(body["not_after"])
parse_time(body["issued_at"])
deadline = min(
not_after,
now + timedelta(seconds=self._limits().get("admission_window_s", 30)),
)
with self.store.tx() as con:
existing = con.execute(
"SELECT * FROM commands WHERE command_id = ? AND tenant_id = ?",
(body["command_id"], tenant["tenant_id"]),
).fetchone()
if existing:
if existing["payload_hash"] != command_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"command_id was already accepted with a different body",
)
return self._command_snapshot(existing)
reason: str | None = None
status = "accepted"
if payload["task_revision"] != task["revision"]:
status, reason = "rejected", "STALE_TASK_REVISION"
elif task["state"] != "running":
status, reason = "rejected", f"TASK_{task['state'].upper()}"
elif not_after <= now:
status, reason = "rejected", "COMMAND_EXPIRED"
elif payload["route_policy_id"] not in {
x["route_policy_id"]
for x in self.profile.get("routes", [])
if x.get("allowed", True)
}:
status, reason = "rejected", "ROUTE_POLICY_NOT_ALLOWED"
elif payload["caller_profile_id"] not in {
x["caller_profile_id"]
for x in self.profile.get("caller_profiles", [])
if x.get("allowed", True)
}:
status, reason = "rejected", "CALLER_PROFILE_NOT_ALLOWED"
elif payload["agent_version_id"] not in {
x["agent_version_id"]
for x in self.profile.get("agents", [])
if x.get("immutable", True)
}:
status, reason = "rejected", "AGENT_VERSION_NOT_ALLOWED"
execution = con.execute(
"SELECT * FROM executions WHERE execution_id = ? AND tenant_id = ?",
(payload["execution_id"], tenant["tenant_id"]),
).fetchone()
if execution and execution["payload_hash"] != digest(payload):
status, reason = "rejected", "EXECUTION_ID_CONFLICT"
elif execution:
status, reason = "completed", "DUPLICATE_EXECUTION"
con.execute(
"""INSERT INTO commands(command_id, tenant_id, tenant_key, command_type, status,
payload_json, payload_hash, task_id, execution_id, call_id, trace_id, issued_at,
not_after, admission_deadline, accepted_at, reason_code, task_state, updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(
body["command_id"],
tenant["tenant_id"],
tenant["tenant_key"],
body["command_type"],
status,
canonical(body),
command_hash,
payload["task_id"],
payload["execution_id"],
execution["call_id"] if execution else None,
body["trace_id"],
body["issued_at"],
body["not_after"],
iso(deadline),
iso(now),
reason,
task["state"],
iso(now),
),
)
if status == "accepted":
con.execute(
"INSERT INTO executions(execution_id, tenant_id, command_id, task_id, task_item_id, payload_hash, status, created_at) VALUES(?,?,?,?,?,?,?,?)",
(
payload["execution_id"],
tenant["tenant_id"],
body["command_id"],
payload["task_id"],
payload["task_item_id"],
digest(payload),
"accepted",
iso(now),
),
)
row = con.execute(
"SELECT * FROM commands WHERE tenant_id = ? AND command_id = ?",
(tenant["tenant_id"], body["command_id"]),
).fetchone()
self._command_event_tx(con, row, reason or "ACCEPTED")
return self._command_snapshot(row)
def _command_event_tx(
self, con: sqlite3.Connection, row: sqlite3.Row, reason: str | None = None
) -> None:
payload = {
"command_id": row["command_id"],
"command_type": row["command_type"],
"status": row["status"],
"reason_code": reason or row["reason_code"],
"task_id": row["task_id"],
"execution_id": row["execution_id"],
"call_id": row["call_id"],
"requested_task_revision": row["requested_task_revision"],
"applied_task_revision": row["applied_task_revision"],
"wait_reason_code": row["wait_reason_code"],
}
self._event_tx(
con,
row["tenant_id"],
row["tenant_key"],
row["trace_id"] or row["command_id"],
"command.result",
"command",
row["command_id"],
row["aggregate_version"],
payload,
row["accepted_at"] or iso(self._now()),
)
def _event_tx(
self,
con: sqlite3.Connection,
tenant_id: str,
tenant_key: str,
trace_id: str,
event_type: str,
aggregate_type: str,
aggregate_id: str,
version: int,
payload: dict[str, Any],
occurred_at: str | None = None,
replay_of: str | None = None,
) -> dict[str, Any]:
body = {
"schema_version": "1.0",
"event_id": new_id("evt"),
"event_type": event_type,
"tenant_id": tenant_id,
"tenant_key": tenant_key,
"trace_id": trace_id,
"occurred_at": occurred_at or iso(self._now()),
"aggregate_type": aggregate_type,
"aggregate_id": aggregate_id,
"aggregate_version": version,
"payload": payload,
}
self._validate_event(body)
con.execute(
"""INSERT INTO events(event_id, tenant_id, tenant_key, trace_id, event_type,
aggregate_type, aggregate_id, aggregate_version, occurred_at, payload_json, body_json, replay_of)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""",
(
body["event_id"],
tenant_id,
tenant_key,
trace_id,
event_type,
aggregate_type,
aggregate_id,
version,
body["occurred_at"],
canonical(payload),
canonical(body),
replay_of,
),
)
return body
def _command_snapshot(self, row: sqlite3.Row | dict[str, Any]) -> dict[str, Any]:
return {
"command_id": row["command_id"],
"command_type": row["command_type"],
"tenant_id": row["tenant_id"],
"tenant_key": row["tenant_key"],
"task_id": row["task_id"],
"execution_id": row["execution_id"],
"call_id": row["call_id"],
"status": row["status"],
"reason_code": row["reason_code"],
"wait_reason_code": row["wait_reason_code"],
"accepted_at": row["accepted_at"],
"waiting_since": row["waiting_since"],
"admission_deadline": row["admission_deadline"],
"requested_task_revision": row["requested_task_revision"],
"applied_task_revision": row["applied_task_revision"],
"task_state": row["task_state"],
"updated_at": row["updated_at"],
"aggregate_version": row["aggregate_version"],
}
def _lease_ttl(self) -> int:
return max(1, _integer(self._limits().get("scheduler_lease_ttl_s", 10)))
def retry_published_commands(self, limit: int = 20) -> int:
now = self._now()
rows = self.store.all(
"SELECT * FROM published_commands WHERE status IN ('pending','retained') AND (next_attempt_at IS NULL OR next_attempt_at<=?) ORDER BY created_at LIMIT ?",
(iso(now), limit),
)
retried = 0
for row in rows:
if row["last_error_code"] == "TENANT_KEY_UNSUPPORTED":
continue
try:
self.publish_execute(_loads(row["body_json"]))
retried += 1
except ServiceError:
continue
return retried
def recover_uploads(self, limit: int = 20) -> int:
rows = self.store.all(
"SELECT * FROM uploads WHERE status IN ('created','uploaded') ORDER BY created_at LIMIT ?",
(limit,),
)
recovered = 0
for upload in rows:
recording = self.store.one(
"SELECT * FROM recordings WHERE recording_id=? AND tenant_id=?",
(upload["recording_id"], upload["tenant_id"]),
)
if recording is None:
continue
try:
if self.object_store.has_final(upload["upload_id"]):
self._complete_upload(
upload["tenant_id"],
upload["upload_id"],
upload["recording_id"],
checksum=upload["expected_checksum"],
size_bytes=upload["expected_size"],
recovery=True,
)
recovered += 1
except ServiceError:
continue
if parse_time(upload["expires_at"]) <= self._now():
with self.store.tx() as con:
con.execute(
"UPDATE uploads SET status='expired' WHERE tenant_id=? AND upload_id=? AND status IN ('created','uploaded')",
(upload["tenant_id"], upload["upload_id"]),
)
return recovered
def _claim_lease(self) -> bool:
if self._clock_jump:
return False
now = self._now()
expires = now + timedelta(seconds=self._lease_ttl())
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM leases WHERE lease_name='scheduler'"
).fetchone()
if (
row
and row["owner_id"] != self.instance_id
and parse_time(row["expires_at"]) > now
):
return False
con.execute(
"""INSERT INTO leases(lease_name, owner_id, expires_at, generation)
VALUES('scheduler', ?, ?, 1)
ON CONFLICT(lease_name) DO UPDATE SET
owner_id=excluded.owner_id, expires_at=excluded.expires_at,
generation=leases.generation+1""",
(self.instance_id, iso(expires)),
)
return True
def _lease_owned_tx(
self, con: sqlite3.Connection, now: datetime | None = None
) -> bool:
now = now or self._now()
row = con.execute(
"SELECT * FROM leases WHERE lease_name='scheduler'"
).fetchone()
return bool(
row
and row["owner_id"] == self.instance_id
and parse_time(row["expires_at"]) > now
)
def process_once(self) -> int:
if not self._claim_lease():
return 0
self.retry_published_commands()
self.ingest_commands()
self.apply_controls()
scheduled = self.schedule_once()
self.dispatch_outbox()
self.consume_saas_events()
self.process_replays()
self.recover_uploads()
return scheduled
def _active_counts(self, tenant_id: str | None = None) -> tuple[int, int]:
if tenant_id:
row = self.store.one(
"SELECT COUNT(*) AS n FROM calls WHERE call_state NOT IN ('ended', 'failed') AND tenant_id = ?",
(tenant_id,),
)
else:
row = self.store.one(
"SELECT COUNT(*) AS n FROM calls WHERE call_state NOT IN ('ended', 'failed')"
)
return _integer(row["n"]), 0
def _resource_cell(self) -> tuple[str, dict[str, Any]] | None:
for cell in self.profile.get("cells", []):
if not self._cell_health.get(cell["cell_id"], False):
continue
row = self.store.one(
"SELECT COUNT(*) AS n FROM calls WHERE cell_id = ? AND call_state NOT IN ('ended','failed')",
(cell["cell_id"],),
)
capacity = min(
_integer(cell.get("capacity", 1)),
_integer(cell.get("media_capacity", cell.get("capacity", 1))),
_integer(cell.get("ai_capacity", cell.get("capacity", 1))),
)
if _integer(row["n"]) < capacity:
return cell["cell_id"], cell
return None
def set_cell_health(self, cell_id: str, healthy: bool) -> None:
if cell_id not in self._cell_health:
raise ValidationError("CELL_NOT_FOUND", "cell is not configured")
self._cell_health[cell_id] = healthy
def _cps_available(self, tenant_id: str) -> bool:
now = self._now()
since = iso(now - timedelta(seconds=1))
global_row = self.store.one(
"SELECT COUNT(*) AS n FROM attempts WHERE created_at >= ?", (since,)
)
tenant_row = self.store.one(
"SELECT COUNT(*) AS n FROM attempts a JOIN calls c ON c.call_id = a.call_id WHERE c.tenant_id = ? AND a.created_at >= ?",
(tenant_id, since),
)
return _integer(global_row["n"]) < _integer(
self._limits().get("global_cps", 3)
) and _integer(tenant_row["n"]) < _integer(self._limits().get("tenant_cps", 1))
def _mark_waiting(self, tenant_id: str, command_id: str, reason: str) -> None:
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, command_id),
).fetchone()
if row is None or row["status"] not in {"accepted", "waiting"}:
return
if row["status"] == "waiting" and row["wait_reason_code"] == reason:
return
version = row["aggregate_version"] + 1
now = iso(self._now())
con.execute(
"UPDATE commands SET status='waiting', waiting_since=COALESCE(waiting_since, ?), wait_reason_code=?, aggregate_version=?, updated_at=? WHERE tenant_id=? AND command_id=?",
(now, reason, version, now, tenant_id, command_id),
)
updated = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, command_id),
).fetchone()
self._command_event_tx(con, updated, reason)
def _reject_command(self, tenant_id: str, command_id: str, reason: str) -> None:
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, command_id),
).fetchone()
if row is None or row["status"] in {
"completed",
"failed",
"rejected",
"applied",
}:
return
now = iso(self._now())
con.execute(
"UPDATE commands SET status='rejected', reason_code=?, aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?",
(reason, now, tenant_id, command_id),
)
updated = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, command_id),
).fetchone()
self._command_event_tx(con, updated, reason)
def schedule_once(self) -> int:
if not self._claim_lease():
return 0
if not self.readiness()["ready"]:
return 0
scheduled = 0
if not self._tenant_order:
return scheduled
order = [
self._tenant_order[(self._cursor + index) % len(self._tenant_order)]
for index in range(len(self._tenant_order))
]
for tenant_id in order:
row = self.store.one(
"SELECT * FROM commands WHERE tenant_id = ? AND status IN ('accepted','waiting') ORDER BY accepted_at, command_id LIMIT 1",
(tenant_id,),
)
if row is None:
continue
now = self._now()
if (
row["admission_deadline"]
and parse_time(row["admission_deadline"]) <= now
):
self._reject_command(tenant_id, row["command_id"], "ADMISSION_TIMEOUT")
continue
task = self.store.one(
"SELECT * FROM tasks WHERE task_id = ? AND tenant_id = ?",
(row["task_id"], tenant_id),
)
if task is None or task["state"] != "running":
self._reject_command(
tenant_id,
row["command_id"],
f"TASK_{task['state'].upper() if task else 'MISSING'}",
)
continue
active_tenant = self._active_counts(tenant_id)[0]
active_global = self._active_counts()[0]
tenant_limit = _integer(
self.store.one(
"SELECT max_concurrency FROM tenants WHERE tenant_id = ?",
(tenant_id,),
)["max_concurrency"]
)
if active_tenant >= tenant_limit or active_global >= _integer(
self._limits().get("global_concurrency", 6)
):
self._mark_waiting(tenant_id, row["command_id"], "CONCURRENCY")
continue
if not self._cps_available(tenant_id):
self._mark_waiting(tenant_id, row["command_id"], "CPS")
continue
cell = self._resource_cell()
if cell is None:
self._mark_waiting(tenant_id, row["command_id"], "CELL_CAPACITY")
continue
if not self._reserve_call(row["command_id"], row, cell):
self._mark_waiting(tenant_id, row["command_id"], "RESOURCE_RACE")
continue
scheduled += 1
self._cursor = (self._cursor + 1) % len(self._tenant_order)
return scheduled
def _reserve_call(
self, command_id: str, row: sqlite3.Row, cell: tuple[str, dict[str, Any]]
) -> bool:
cell_id, cell_cfg = cell
payload = _loads(row["payload_json"])["payload"]
route = self._route_config(payload["route_policy_id"])
prefix = str(route.get("dial_prefix", ""))
target_uri = (
payload["callee"]
if payload["callee"].startswith(prefix)
else prefix + payload["callee"]
)
caller = next(
(
item.get("display", "")
for item in self.profile.get("caller_profiles", [])
if item.get("caller_profile_id") == payload["caller_profile_id"]
),
payload["caller_profile_id"],
)
call_id = new_id("call")
attempt_id = new_id("attempt")
now_dt = self._now()
now = iso(now_dt)
with self.store.tx() as con:
if not self._lease_owned_tx(con, now_dt):
return False
current = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(row["tenant_id"], command_id),
).fetchone()
task = con.execute(
"SELECT * FROM tasks WHERE tenant_id=? AND task_id=?",
(row["tenant_id"], row["task_id"]),
).fetchone()
if (
current is None
or current["status"] not in {"accepted", "waiting"}
or task is None
or task["state"] != "running"
):
return False
tenant_active = con.execute(
"SELECT COUNT(*) AS n FROM calls WHERE tenant_id=? AND call_state NOT IN ('ended','failed')",
(row["tenant_id"],),
).fetchone()["n"]
global_active = con.execute(
"SELECT COUNT(*) AS n FROM calls WHERE call_state NOT IN ('ended','failed')"
).fetchone()["n"]
tenant_limit = con.execute(
"SELECT max_concurrency FROM tenants WHERE tenant_id=?",
(row["tenant_id"],),
).fetchone()["max_concurrency"]
if _integer(tenant_active) >= _integer(tenant_limit) or _integer(
global_active
) >= _integer(self._limits().get("global_concurrency", 6)):
return False
cell_active = con.execute(
"SELECT COUNT(*) AS n FROM calls WHERE cell_id=? AND call_state NOT IN ('ended','failed')",
(cell_id,),
).fetchone()["n"]
cell_capacity = min(
_integer(cell_cfg.get("capacity", 1)),
_integer(cell_cfg.get("media_capacity", cell_cfg.get("capacity", 1))),
_integer(cell_cfg.get("ai_capacity", cell_cfg.get("capacity", 1))),
)
if _integer(cell_active) >= cell_capacity:
return False
route_active = con.execute(
"SELECT COUNT(*) AS n FROM calls WHERE trunk_id=? AND call_state NOT IN ('ended','failed')",
(route.get("trunk_id", "trunk-mock"),),
).fetchone()["n"]
if _integer(route_active) >= _integer(route.get("max_concurrency", 10**9)):
return False
since = iso(now_dt - timedelta(seconds=1))
global_cps = con.execute(
"SELECT COUNT(*) AS n FROM attempts WHERE created_at>=?", (since,)
).fetchone()["n"]
tenant_cps = con.execute(
"SELECT COUNT(*) AS n FROM attempts a JOIN calls c ON c.call_id=a.call_id WHERE c.tenant_id=? AND a.created_at>=?",
(row["tenant_id"], since),
).fetchone()["n"]
route_cps = con.execute(
"SELECT COUNT(*) AS n FROM attempts WHERE route_policy_id=? AND created_at>=?",
(payload["route_policy_id"], since),
).fetchone()["n"]
if (
_integer(global_cps) >= _integer(self._limits().get("global_cps", 3))
or _integer(tenant_cps) >= _integer(self._limits().get("tenant_cps", 1))
or _integer(route_cps) >= _integer(route.get("max_cps", 10**9))
):
return False
version = current["aggregate_version"] + 1
changed = con.execute(
"UPDATE commands SET status='executing', call_id=?, waiting_since=NULL, wait_reason_code=NULL, aggregate_version=?, updated_at=? WHERE tenant_id=? AND command_id=? AND status IN ('accepted','waiting')",
(call_id, version, now, row["tenant_id"], command_id),
).rowcount
if changed != 1:
return False
con.execute(
"INSERT INTO calls(call_id, tenant_id, execution_id, command_id, task_id, task_item_id, callee, cell_id, trunk_id, egress_pool_id, call_state, created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
(
call_id,
current["tenant_id"],
current["execution_id"],
command_id,
current["task_id"],
payload["task_item_id"],
payload["callee"],
cell_id,
route.get("trunk_id", "trunk-mock"),
route.get(
"egress_pool_id", cell_cfg.get("egress_pool_id", "egress-mock")
),
"queued",
now,
),
)
con.execute(
"INSERT INTO attempts(attempt_id, call_id, attempt_number, route_policy_id, caller_profile_id, target_uri, status, created_at) VALUES(?,?,?,?,?,?,?,?)",
(
attempt_id,
call_id,
1,
payload["route_policy_id"],
payload["caller_profile_id"],
target_uri,
"reserved",
now,
),
)
con.execute(
"UPDATE executions SET status='executing', call_id=? WHERE tenant_id=? AND execution_id=?",
(call_id, current["tenant_id"], current["execution_id"]),
)
updated = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(current["tenant_id"], command_id),
).fetchone()
self._command_event_tx(con, updated, "EXECUTING")
self._event_tx(
con,
current["tenant_id"],
current["tenant_key"],
current["trace_id"],
"call.status",
"call",
call_id,
1,
{
"call_id": call_id,
"execution_id": current["execution_id"],
"task_id": current["task_id"],
"call_state": "queued",
"reason_code": None,
},
now,
)
try:
if not self._call_permitted(call_id, cell_id):
self._finish_call(
call_id, "failed", "ADMISSION_REVOKED", notify_protocol=False
)
return True
self.mock_sip.invite(call_id, payload["callee"], target_uri, caller)
self.mock_ari.channel_created(call_id, cell_id)
self.mock_ari.bridge_ready(call_id)
thread = threading.Thread(
target=self._run_call,
args=(call_id,),
name=f"call-{call_id}",
daemon=True,
)
self._call_threads[call_id] = thread
thread.start()
except (
OSError,
RuntimeError,
TypeError,
ValueError,
KeyError,
sqlite3.DatabaseError,
ServiceError,
):
self._finish_call(
call_id, "failed", "PROTOCOL_SETUP_FAILED", notify_protocol=False
)
return True
def _call_permitted(self, call_id: str, cell_id: str) -> bool:
with self.store.tx() as con:
if not self._lease_owned_tx(con):
return False
row = con.execute(
"""SELECT c.call_state, c.tenant_id, cmd.status, t.state
FROM calls c JOIN commands cmd ON cmd.tenant_id=c.tenant_id AND cmd.command_id=c.command_id
JOIN tasks t ON t.tenant_id=c.tenant_id AND t.task_id=c.task_id
WHERE c.call_id=? AND c.cell_id=?""",
(call_id, cell_id),
).fetchone()
return bool(
row
and row["call_state"] == "queued"
and row["status"] == "executing"
and row["state"] == "running"
and self._cell_health.get(cell_id, False)
)
def set_failure(self, command_or_call_id: str, scenario: str) -> None:
allowed = set(self.profile.get("failure_scenarios", []))
if scenario not in allowed:
raise ValidationError(
"SCENARIO_NOT_ALLOWED", "failure scenario is not in the mock profile"
)
self._failures[command_or_call_id] = scenario
def _scenario_for(self, call_id: str) -> str:
row = self.store.one(
"SELECT command_id FROM calls WHERE call_id = ?", (call_id,)
)
return self._failures.get(
call_id, self._failures.get(row["command_id"] if row else "", "success")
)
def _sleep(self, milliseconds: int = 0) -> None:
if milliseconds > 0:
time.sleep(milliseconds / 1000)
def _call_is_ended(self, call_id: str) -> bool:
row = self.store.one(
"SELECT call_state FROM calls WHERE call_id = ?", (call_id,)
)
return row is None or row["call_state"] in {"ended", "failed"}
def _run_call(self, call_id: str) -> None:
scenario = self._scenario_for(call_id)
try:
self._update_call_state(call_id, "dialing")
self._update_attempt(call_id, "dialing", started=True)
self._sleep(1 if scenario == "clock_jump" else 0)
if self._call_is_ended(call_id):
return
self._update_call_state(call_id, "ringing")
self.mock_sip.ringing(call_id)
self.mock_rtp.packet(call_id, "inbound", 1)
if scenario in {"busy", "no_answer"}:
self._finish_call(
call_id,
"busy" if scenario == "busy" else "no_answer",
scenario.upper(),
)
return
self._sleep(1 if scenario == "ai_timeout" else 0)
if self._call_is_ended(call_id):
return
if scenario == "ai_timeout":
self._finish_call(call_id, "failed", "AI_TIMEOUT")
return
self._update_call_state(call_id, "answered", started=True)
self.mock_sip.answered(call_id)
self.mock_rtp.packet(call_id, "outbound", 1)
self._ensure_recording(call_id)
self._sleep(_integer(self._limits().get("hold_ms", 0)))
if scenario == "customer_silent":
self._finish_call(call_id, "failed", "CUSTOMER_SILENT")
return
if scenario == "ari_disconnect":
self._mark_reconciling(call_id, "ARI_DISCONNECTED_RECONCILING")
return
turns = _integer(self._limits().get("turns", 2))
for turn in range(turns):
if self._call_is_ended(call_id):
return
self._add_transcript(
call_id,
turn,
"customer",
f"customer turn {turn + 1}",
"not_applicable",
)
playback = (
"cancelled"
if scenario == "ai_timeout" and turn == turns - 1
else "playback_confirmed"
)
self._add_transcript(
call_id, turn, "agent", f"agent turn {turn + 1}", playback
)
self._finish_call(
call_id, "completed", "CALL_COMPLETED", upload_scenario=scenario
)
except (
OSError,
RuntimeError,
TypeError,
ValueError,
KeyError,
sqlite3.DatabaseError,
ServiceError,
):
if not self._call_is_ended(call_id):
self._finish_call(call_id, "failed", "EXECUTOR_EXCEPTION")
def _mark_reconciling(self, call_id: str, reason: str) -> None:
now = iso(self._now())
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM calls WHERE call_id=?", (call_id,)
).fetchone()
if row is None or row["call_state"] in {"ended", "failed"}:
return
version = row["call_version"] + 1
con.execute(
"UPDATE calls SET call_state='reconciling', call_version=?, reason_code=? WHERE call_id=?",
(version, reason, call_id),
)
con.execute(
"UPDATE commands SET status='reconciling', reason_code=?, aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=? AND status NOT IN ('completed','failed','rejected','applied')",
(reason, now, row["tenant_id"], row["command_id"]),
)
con.execute(
"UPDATE executions SET status='reconciling' WHERE tenant_id=? AND execution_id=? AND status NOT IN ('completed','failed','rejected')",
(row["tenant_id"], row["execution_id"]),
)
tenant = con.execute(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (row["tenant_id"],)
).fetchone()["tenant_key"]
self._event_tx(
con,
row["tenant_id"],
tenant,
row["command_id"],
"call.status",
"call",
call_id,
version,
{
"call_id": call_id,
"execution_id": row["execution_id"],
"call_state": "reconciling",
"reason_code": reason,
},
now,
)
def reconcile_call(
self,
call_id: str,
ended: bool,
outcome: str = "completed",
reason: str = "RECONCILED",
) -> dict[str, Any]:
if not ended:
row = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,))
if row is None:
raise NotFoundError()
return self.get_call(row["tenant_id"], call_id)
self._finish_call(call_id, outcome, reason)
row = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,))
if row is None:
raise NotFoundError()
return self.get_call(row["tenant_id"], call_id)
def _update_attempt(self, call_id: str, status: str, started: bool = False) -> None:
now = iso(self._now())
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM attempts WHERE call_id = ? ORDER BY attempt_number DESC LIMIT 1",
(call_id,),
).fetchone()
if row is None:
return
con.execute(
"UPDATE attempts SET status=?, started_at=COALESCE(started_at, ?), ended_at=? WHERE attempt_id=?",
(
status,
now if started else row["started_at"],
now if status in {"ended", "failed"} else None,
row["attempt_id"],
),
)
def _update_call_state(
self,
call_id: str,
state: str,
outcome: str | None = None,
reason: str | None = None,
started: bool = False,
) -> None:
now = iso(self._now())
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM calls WHERE call_id = ?", (call_id,)
).fetchone()
if row is None or row["call_state"] in {"ended", "failed"}:
return
version = row["call_version"] + 1
started_at = row["started_at"] or (
now if started or state == "answered" else None
)
con.execute(
"UPDATE calls SET call_state=?, call_version=?, outcome=COALESCE(?, outcome), reason_code=COALESCE(?, reason_code), started_at=? WHERE call_id=?",
(state, version, outcome, reason, started_at, call_id),
)
self._event_tx(
con,
row["tenant_id"],
self.store.one(
"SELECT tenant_key FROM tenants WHERE tenant_id = ?",
(row["tenant_id"],),
)["tenant_key"],
row["command_id"],
"call.status",
"call",
call_id,
version,
{
"call_id": call_id,
"execution_id": row["execution_id"],
"call_state": state,
"outcome": outcome,
"reason_code": reason,
},
now,
)
def _add_transcript(
self,
call_id: str,
turn: int,
speaker: str,
text: str,
playback: str,
revision: int | None = None,
is_final: bool = True,
) -> None:
if not isinstance(text, str) or not text:
raise ValidationError("TRANSCRIPT_INVALID", "transcript text is required")
if type(is_final) is not bool:
raise ValidationError("TRANSCRIPT_INVALID", "is_final must be boolean")
try:
final_value = int(is_final)
except (TypeError, ValueError, OverflowError) as exc:
raise ValidationError("TRANSCRIPT_INVALID", "is_final is invalid") from exc
now = iso(self._now())
segment_id = f"seg_{call_id}_{turn}_{speaker}"
with self.store.tx() as con:
call = con.execute(
"SELECT * FROM calls WHERE call_id=?", (call_id,)
).fetchone()
if call is None or call["call_state"] in {"ended", "failed"}:
return
previous = con.execute(
"SELECT * FROM transcripts WHERE segment_id=?", (segment_id,)
).fetchone()
if (
previous
and previous["text"] == text
and bool(previous["is_final"]) == is_final
and previous["playback_state"] == playback
):
return
next_revision = (
_integer(revision)
if revision is not None
else (previous["revision"] + 1 if previous else 1)
)
if next_revision < 1:
raise ValidationError("TRANSCRIPT_INVALID", "revision must be positive")
if previous and previous["is_final"]:
if previous["text"] != text:
raise ConflictError(
"TRANSCRIPT_FINAL_CONFLICT",
"a final transcript segment cannot be overwritten",
)
if not is_final:
return
if previous and next_revision <= previous["revision"]:
return
con.execute(
"""INSERT INTO transcripts(segment_id, call_id, turn_index, speaker, text, revision, is_final, playback_state, updated_at)
VALUES(?,?,?,?,?,?,?,?,?)
ON CONFLICT(segment_id) DO UPDATE SET text=excluded.text, revision=excluded.revision,
is_final=excluded.is_final, playback_state=excluded.playback_state, updated_at=excluded.updated_at""",
(
segment_id,
call_id,
turn,
speaker,
text,
next_revision,
final_value,
playback,
now,
),
)
tenant = con.execute(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (call["tenant_id"],)
).fetchone()["tenant_key"]
self._event_tx(
con,
call["tenant_id"],
tenant,
call["command_id"],
"transcript.updated",
"transcript_segment",
segment_id,
next_revision,
{
"call_id": call_id,
"segment_id": segment_id,
"turn_index": turn,
"speaker": speaker,
"text": text,
"revision": next_revision,
"is_final": is_final,
"playback_state": playback,
},
now,
)
self.mock_ai.turn(call_id, turn, speaker, playback)
def _finish_call(
self,
call_id: str,
outcome: str,
reason: str,
upload_scenario: str | None = None,
notify_protocol: bool = True,
) -> None:
now_dt = self._now()
now = iso(now_dt)
with self.store.tx() as con:
row = con.execute(
"SELECT * FROM calls WHERE call_id = ?", (call_id,)
).fetchone()
if row is None or row["call_state"] in {"ended", "failed"}:
return
version = row["call_version"] + 1
start = parse_time(row["started_at"]) if row["started_at"] else now_dt
duration_ms = max(0, _integer((now_dt - start).total_seconds() * 1000))
state = "failed" if outcome == "failed" else "ended"
con.execute(
"UPDATE calls SET call_state=?, call_version=?, outcome=?, reason_code=?, ended_at=?, duration_ms=? WHERE call_id=?",
(state, version, outcome, reason, now, duration_ms, call_id),
)
con.execute(
"UPDATE attempts SET status=?, ended_at=? WHERE call_id=?",
("failed" if outcome == "failed" else "ended", now, call_id),
)
command = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(row["tenant_id"], row["command_id"]),
).fetchone()
command_status = (
"failed" if outcome == "failed" or outcome == "unknown" else "completed"
)
command_version = command["aggregate_version"] + 1
con.execute(
"UPDATE commands SET status=?, reason_code=?, aggregate_version=?, updated_at=? WHERE tenant_id=? AND command_id=?",
(
command_status,
reason,
command_version,
now,
row["tenant_id"],
row["command_id"],
),
)
con.execute(
"UPDATE executions SET status=?, call_id=? WHERE tenant_id=? AND execution_id=?",
(command_status, call_id, row["tenant_id"], row["execution_id"]),
)
updated = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(row["tenant_id"], row["command_id"]),
).fetchone()
self._command_event_tx(con, updated, reason)
tenant = con.execute(
"SELECT tenant_key FROM tenants WHERE tenant_id = ?",
(row["tenant_id"],),
).fetchone()["tenant_key"]
self._event_tx(
con,
row["tenant_id"],
tenant,
row["command_id"],
"call.finished",
"call",
call_id,
version,
{
"call_id": call_id,
"execution_id": row["execution_id"],
"outcome": outcome,
"reason_code": reason,
"duration_ms": duration_ms,
},
now,
)
if notify_protocol:
self.mock_sip.bye(call_id, reason)
self.mock_ari.media_closed(call_id)
self.mock_ari.cleanup(call_id)
if outcome == "completed" or (row["started_at"] and outcome == "failed"):
self._record_and_upload(call_id, upload_scenario or "success")
def _ensure_recording(self, call_id: str) -> str | None:
call = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,))
if call is None or not call["started_at"]:
return None
existing = self.store.one(
"SELECT recording_id FROM recordings WHERE tenant_id=? AND call_id=? ORDER BY created_at LIMIT 1",
(call["tenant_id"], call_id),
)
if existing:
return existing["recording_id"]
recording_id = new_id("rec")
with self.store.tx() as con:
con.execute(
"""INSERT INTO recordings(
recording_id, call_id, tenant_id, content_type, size_bytes,
checksum_algorithm, checksum, channels, sample_rate_hz,
duration_ms, status, created_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""",
(
recording_id,
call_id,
call["tenant_id"],
"audio/wav",
0,
"SHA-256",
"",
1,
16000,
0,
"recording",
iso(self._now()),
),
)
return recording_id
def _record_and_upload(self, call_id: str, scenario: str) -> None:
call = self.store.one("SELECT * FROM calls WHERE call_id=?", (call_id,))
if call is None or not call["started_at"]:
return
existing = self.store.one(
"SELECT * FROM recordings WHERE tenant_id=? AND call_id=? ORDER BY created_at LIMIT 1",
(call["tenant_id"], call_id),
)
if existing and existing["status"] == "verified":
return
duration_ms = max(
1,
_integer(
call["duration_ms"] or _integer(self._limits().get("turns", 2)) * 100
),
)
data = self._wav_bytes(duration_ms)
checksum = hashlib.sha256(data).hexdigest()
recording_id = (
existing["recording_id"] if existing else self._ensure_recording(call_id)
)
if recording_id is None:
return
with self.store.tx() as con:
con.execute(
"""UPDATE recordings SET content_type='audio/wav', size_bytes=?,
checksum_algorithm='SHA-256', checksum=?, channels=1,
sample_rate_hz=16000, duration_ms=?, status='uploading',
failure_code=NULL WHERE tenant_id=? AND recording_id=?""",
(len(data), checksum, duration_ms, call["tenant_id"], recording_id),
)
if scenario == "upload_missing":
self._recording_failed(recording_id, "OBJECT_MISSING", call["tenant_id"])
return
request = {
"recording_id": recording_id,
"call_id": call_id,
"content_type": "audio/wav",
"size_bytes": len(data),
"checksum_algorithm": "SHA-256",
"checksum": checksum,
"channels": 1,
"sample_rate_hz": 16000,
"duration_ms": duration_ms,
}
try:
_, session = self.create_upload(
call["tenant_id"], request, f"executor:{call_id}", "executor"
)
upload_id = session["upload_id"]
token = session["required_headers"]["X-Mock-Upload-Token"]
upload_data = data
if scenario == "upload_bad_checksum":
upload_data = data[:-1] + bytes([data[-1] ^ 1])
self.put_upload(upload_id, token, upload_data)
self.complete_upload(
call["tenant_id"],
upload_id,
{
"recording_id": recording_id,
"size_bytes": len(data),
"checksum_algorithm": "SHA-256",
"checksum": checksum,
},
f"executor-complete:{call_id}",
"executor",
)
except ServiceError as exc:
self._recording_failed(recording_id, exc.code, call["tenant_id"])
except (OSError, RuntimeError, TypeError, ValueError):
self._recording_failed(
recording_id, "OBJECT_UPLOAD_FAILED", call["tenant_id"]
)
def _wav_bytes(self, duration_ms: int | None = None) -> bytes:
try:
duration_ms = max(
1,
_integer(duration_ms or _integer(self._limits().get("turns", 2)) * 100),
)
frames = max(1, int(16000 * duration_ms / 1000))
output = io.BytesIO()
with wave.open(output, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(16000)
samples = bytearray()
for index in range(frames):
seconds = index / 16000
value = 11000 * math.sin(
2 * math.pi * 440 * seconds
) + 7000 * math.sin(2 * math.pi * 660 * seconds)
samples.extend(
struct.pack("<h", max(-32768, min(32767, int(value))))
)
wav.writeframes(bytes(samples))
return output.getvalue()
except (OSError, OverflowError, TypeError, ValueError, struct.error) as exc:
raise ServiceError(
"RECORDING_GENERATION_FAILED", "recording could not be generated", 500
) from exc
def _recording_failed(self, recording_id: str, code: str, tenant_id: str) -> None:
with self.store.tx() as con:
recording = con.execute(
"SELECT * FROM recordings WHERE tenant_id=? AND recording_id=?",
(tenant_id, recording_id),
).fetchone()
if recording is None or recording["status"] in {"verified", "failed"}:
return
changed = con.execute(
"UPDATE recordings SET status='failed', failure_code=? WHERE tenant_id=? AND recording_id=? AND status NOT IN ('verified','failed')",
(code, recording["tenant_id"], recording_id),
).rowcount
tenant = con.execute(
"SELECT tenant_key FROM tenants WHERE tenant_id = ?",
(recording["tenant_id"],),
).fetchone()["tenant_key"]
call = con.execute(
"SELECT command_id FROM calls WHERE tenant_id=? AND call_id=?",
(recording["tenant_id"], recording["call_id"]),
).fetchone()
if not changed or call is None:
return
self._event_tx(
con,
recording["tenant_id"],
tenant,
call["command_id"],
"recording.failed",
"recording",
recording_id,
1,
{
"recording_id": recording_id,
"call_id": recording["call_id"],
"failure_code": code,
},
iso(self._now()),
)
def create_upload(
self,
tenant_id: str,
request: dict[str, Any],
idempotency_key: str | None = None,
actor_id: str = "internal",
) -> tuple[int, dict[str, Any]]:
self._tenant(tenant_id)
required = {
"recording_id",
"call_id",
"content_type",
"size_bytes",
"checksum_algorithm",
"checksum",
"channels",
"sample_rate_hz",
"duration_ms",
}
if (
set(request) != required
or request.get("content_type") != "audio/wav"
or request.get("checksum_algorithm") != "SHA-256"
):
raise ValidationError(
"SCHEMA_INVALID", "upload request fields do not match the contract"
)
if (
type(request.get("size_bytes")) is not int
or request["size_bytes"] < 1
or request["size_bytes"]
> _integer(self._limits().get("max_recording_bytes", 64 * 1024 * 1024))
):
raise ValidationError(
"SCHEMA_INVALID", "size_bytes is outside the allowed range"
)
if (
type(request.get("channels")) is not int
or request["channels"] != 1
or type(request.get("sample_rate_hz")) is not int
or not 8000 <= request["sample_rate_hz"] <= 48000
or type(request.get("duration_ms")) is not int
or not 1 <= request["duration_ms"] <= 3600000
):
raise ValidationError(
"SCHEMA_INVALID", "audio metadata is outside the allowed range"
)
if not isinstance(request.get("checksum"), str) or not re.fullmatch(
r"[0-9a-f]{64}", request["checksum"]
):
raise ValidationError(
"SCHEMA_INVALID", "checksum must be a SHA-256 hex digest"
)
idempotency_key = idempotency_key or f"internal:{request['recording_id']}"
operation_hash = self._operation_hash(
"recording.create", request["recording_id"], request
)
existing_operation = self.store.one(
"SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?",
(
tenant_id,
actor_id,
"recording.create",
request["recording_id"],
idempotency_key,
),
)
if existing_operation:
if existing_operation["payload_hash"] != operation_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"upload request was already submitted with a different body",
)
return existing_operation["status"], _loads(
existing_operation["response_json"]
)
recording = self.store.one(
"SELECT * FROM recordings WHERE recording_id=? AND tenant_id=?",
(request["recording_id"], tenant_id),
)
if recording is None or recording["call_id"] != request["call_id"]:
raise NotFoundError()
metadata = (
"content_type",
"size_bytes",
"checksum_algorithm",
"checksum",
"channels",
"sample_rate_hz",
"duration_ms",
)
if any(recording[field] != request[field] for field in metadata):
raise ConflictError(
"RECORDING_METADATA_CONFLICT", "recording metadata is immutable"
)
upload = self.store.one(
"SELECT * FROM uploads WHERE tenant_id=? AND recording_id=? ORDER BY created_at DESC LIMIT 1",
(tenant_id, request["recording_id"]),
)
if upload and (
upload["status"] == "verified"
or (
parse_time(upload["expires_at"]) > self._now()
and upload["status"] in {"created", "uploaded"}
)
):
status, upload_id, expires = 200, upload["upload_id"], upload["expires_at"]
else:
status = 201
upload_id = new_id("upload")
expires = iso(
self._now()
+ timedelta(seconds=_integer(self._limits().get("upload_ttl_s", 300)))
)
path = str(self.object_store.path_for(upload_id))
with self.store.tx() as con:
con.execute(
"INSERT INTO uploads(upload_id, recording_id, tenant_id, expected_size, expected_checksum, expires_at, status, object_path, created_at) VALUES(?,?,?,?,?,?,?,?,?)",
(
upload_id,
request["recording_id"],
tenant_id,
request["size_bytes"],
request["checksum"],
expires,
"created",
path,
iso(self._now()),
),
)
con.execute(
"UPDATE recordings SET upload_id=?, status='uploading' WHERE tenant_id=? AND recording_id=?",
(upload_id, tenant_id, request["recording_id"]),
)
response = {
"upload_id": upload_id,
"recording_id": request["recording_id"],
"expires_at": expires,
"upload_method": "PUT",
"upload_url": f"/_mock/oss/{upload_id}",
"required_headers": {
"X-Mock-Upload-Token": self.object_store.token(upload_id, tenant_id)
},
"constraints": {
"size_bytes": request["size_bytes"],
"checksum_algorithm": "SHA-256",
"checksum": request["checksum"],
},
}
cleanup_upload_id: str | None = None
with self.store.tx() as con:
inserted = con.execute(
"""INSERT OR IGNORE INTO storage_operations(tenant_id, actor_id, operation_type, target_id, idempotency_key, payload_hash, status, response_json, created_at, updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?)""",
(
tenant_id,
actor_id,
"recording.create",
request["recording_id"],
idempotency_key,
operation_hash,
status,
canonical(response),
iso(self._now()),
iso(self._now()),
),
).rowcount
if not inserted:
existing_operation = con.execute(
"SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?",
(
tenant_id,
actor_id,
"recording.create",
request["recording_id"],
idempotency_key,
),
).fetchone()
if (
existing_operation is None
or existing_operation["payload_hash"] != operation_hash
):
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"upload request was already submitted with a different body",
)
winner = _loads(existing_operation["response_json"])
if upload_id != winner["upload_id"]:
con.execute(
"DELETE FROM uploads WHERE tenant_id=? AND upload_id=? AND recording_id=? AND status='created'",
(tenant_id, upload_id, request["recording_id"]),
)
con.execute(
"UPDATE recordings SET upload_id=? WHERE tenant_id=? AND recording_id=?",
(winner["upload_id"], tenant_id, request["recording_id"]),
)
cleanup_upload_id = upload_id
result = (existing_operation["status"], winner)
else:
result = (status, response)
if cleanup_upload_id:
with suppress(OSError):
self.object_store.path_for(cleanup_upload_id).unlink()
return result
def put_upload(self, upload_id: str, token: str, data: bytes) -> None:
upload = self.store.one("SELECT * FROM uploads WHERE upload_id=?", (upload_id,))
if upload is None:
raise NotFoundError()
if not isinstance(data, bytes) or len(data) > _integer(
self._limits().get("max_recording_bytes", 64 * 1024 * 1024)
):
raise ValidationError(
"OBJECT_BODY_INVALID", "upload body is outside the allowed range"
)
if parse_time(upload["expires_at"]) <= self._now():
raise ServiceError("UPLOAD_EXPIRED", "upload session has expired", 410)
if upload["status"] == "verified" or self.object_store.has_final(upload_id):
raise ConflictError(
"UPLOAD_ALREADY_FINALIZED", "verified upload objects are immutable"
)
if (
upload["status"] == "uploaded"
and self.object_store.path_for(upload_id).is_file()
):
existing = self.object_store.path_for(upload_id).read_bytes()
if (
len(existing) == len(data)
and hashlib.sha256(existing).hexdigest()
== hashlib.sha256(data).hexdigest()
):
return
raise ConflictError(
"UPLOAD_ALREADY_RECEIVED",
"upload session already contains different bytes",
)
self.object_store.put(upload_id, upload["tenant_id"], data, token)
with self.store.tx() as con:
con.execute(
"UPDATE uploads SET status='uploaded' WHERE tenant_id=? AND upload_id=?",
(upload["tenant_id"], upload_id),
)
def _complete_upload(
self,
tenant_id: str,
upload_id: str,
recording_id: str,
checksum: str,
size_bytes: int,
recovery: bool = False,
) -> dict[str, Any]:
upload = self.store.one(
"SELECT * FROM uploads WHERE upload_id=? AND tenant_id=?",
(upload_id, tenant_id),
)
if upload is None or upload["recording_id"] != recording_id:
raise NotFoundError()
recording = self.store.one(
"SELECT * FROM recordings WHERE recording_id=? AND tenant_id=?",
(recording_id, tenant_id),
)
if recording is None:
raise NotFoundError()
if upload["status"] == "verified" and recording["status"] == "verified":
return {
"upload_id": upload_id,
"recording_id": recording_id,
"status": "verified",
"oss_id": recording["oss_id"],
"verified_at": recording["verified_at"],
}
if parse_time(upload["expires_at"]) <= self._now() and not (
recovery and self.object_store.has_final(upload_id)
):
raise ServiceError("UPLOAD_EXPIRED", "upload session has expired", 410)
if (
type(size_bytes) is not int
or not isinstance(checksum, str)
or size_bytes != upload["expected_size"]
or checksum != upload["expected_checksum"]
):
raise ValidationError(
"RECORDING_METADATA_CONFLICT",
"complete request does not match the reserved metadata",
)
try:
oss_id = self.object_store.verify_and_promote(
upload_id, upload["expected_size"], upload["expected_checksum"]
)
except ValidationError:
self._recording_failed(
recording_id, "OBJECT_VERIFICATION_FAILED", tenant_id
)
raise
now = iso(self._now())
with self.store.tx() as con:
con.execute(
"UPDATE uploads SET status='verified', completed_at=? WHERE tenant_id=? AND upload_id=?",
(now, tenant_id, upload_id),
)
changed = con.execute(
"UPDATE recordings SET status='verified', oss_id=?, verified_at=?, failure_code=NULL WHERE tenant_id=? AND recording_id=? AND status!='verified'",
(oss_id, now, tenant_id, recording_id),
).rowcount
recording = con.execute(
"SELECT * FROM recordings WHERE tenant_id=? AND recording_id=?",
(tenant_id, recording_id),
).fetchone()
call = con.execute(
"SELECT command_id FROM calls WHERE tenant_id=? AND call_id=?",
(tenant_id, recording["call_id"]),
).fetchone()
tenant = con.execute(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,)
).fetchone()["tenant_key"]
if changed:
self._event_tx(
con,
tenant_id,
tenant,
call["command_id"],
"recording.ready",
"recording",
recording_id,
1,
{
"recording_id": recording_id,
"call_id": recording["call_id"],
"oss_id": oss_id,
"size_bytes": recording["size_bytes"],
"checksum": recording["checksum"],
},
now,
)
return {
"upload_id": upload_id,
"recording_id": recording_id,
"status": "verified",
"oss_id": oss_id,
"verified_at": recording["verified_at"],
}
def complete_upload(
self,
tenant_id: str,
upload_id: str,
request: dict[str, Any],
idempotency_key: str | None = None,
actor_id: str = "internal",
) -> dict[str, Any]:
required = {"recording_id", "size_bytes", "checksum_algorithm", "checksum"}
allowed = required | {"etag"}
if (
not required.issubset(request)
or set(request) - allowed
or request.get("checksum_algorithm") != "SHA-256"
or type(request.get("size_bytes")) is not int
or not isinstance(request.get("checksum"), str)
):
raise ValidationError(
"SCHEMA_INVALID", "complete request fields do not match the contract"
)
idempotency_key = idempotency_key or f"internal:{upload_id}"
operation_hash = self._operation_hash("recording.complete", upload_id, request)
existing = self.store.one(
"SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?",
(tenant_id, actor_id, "recording.complete", upload_id, idempotency_key),
)
if existing:
if existing["payload_hash"] != operation_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"upload completion was already submitted with a different body",
)
return _loads(existing["response_json"])
response = self._complete_upload(
tenant_id,
upload_id,
request["recording_id"],
request["checksum"],
request["size_bytes"],
)
with self.store.tx() as con:
inserted = con.execute(
"INSERT OR IGNORE INTO storage_operations(tenant_id, actor_id, operation_type, target_id, idempotency_key, payload_hash, status, response_json, created_at, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)",
(
tenant_id,
actor_id,
"recording.complete",
upload_id,
idempotency_key,
operation_hash,
200,
canonical(response),
iso(self._now()),
iso(self._now()),
),
).rowcount
if not inserted:
existing = con.execute(
"SELECT * FROM storage_operations WHERE tenant_id=? AND actor_id=? AND operation_type=? AND target_id=? AND idempotency_key=?",
(
tenant_id,
actor_id,
"recording.complete",
upload_id,
idempotency_key,
),
).fetchone()
if existing is None or existing["payload_hash"] != operation_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"upload completion was already submitted with a different body",
)
return _loads(existing["response_json"])
return response
def dispatch_outbox(self, limit: int = 100) -> int:
now = iso(self._now())
rows = self.store.all(
"SELECT * FROM events WHERE published=0 AND (next_attempt_at IS NULL OR next_attempt_at <= ?) ORDER BY occurred_at, event_id LIMIT ?",
(now, limit),
)
sent = 0
for row in rows:
body = _loads(row["body_json"])
try:
self.broker.publish(
"agent-call.events.v1",
f"agent-call.{row['event_type']}",
body,
row["event_id"],
)
except BrokerError:
with self.store.tx() as con:
attempts = row["broker_attempts"] + 1
retry_at = iso(
self._now() + timedelta(seconds=min(60, 2 ** min(attempts, 5)))
)
con.execute(
"UPDATE events SET broker_attempts=?, broker_status='failed', next_attempt_at=? WHERE tenant_id=? AND event_id=?",
(attempts, retry_at, row["tenant_id"], row["event_id"]),
)
continue
with self.store.tx() as con:
con.execute(
"UPDATE events SET published=1, broker_status='broker_confirmed', published_at=?, broker_attempts=broker_attempts+1 WHERE tenant_id=? AND event_id=?",
(iso(self._now()), row["tenant_id"], row["event_id"]),
)
sent += 1
return sent
def consume_saas_events(self, limit: int = 100) -> int:
applied = 0
for _ in range(limit):
try:
message = self.broker.consume("agent-call.saas.events.v1")
except BrokerError:
break
if not message:
break
body_raw: Any = message.get("body", {})
body = body_raw if isinstance(body_raw, dict) else {}
event_id = body.get("event_id")
if not event_id:
self._dead_letter(message, body, "INBOX_INVALID")
reject = getattr(self.broker, "reject", None)
if callable(reject):
reject(message)
else:
self.broker.ack(message)
continue
try:
self._validate_event(body)
tenant = self._tenant(body["tenant_id"], body["tenant_key"])
expected_route = f"agent-call.{body['event_type']}"
if message.get("routing_key") not in {None, expected_route}:
raise ValidationError(
"EVENT_ROUTE_MISMATCH", "event routing key is invalid"
)
if message.get("message_id") not in {None, event_id}:
raise ValidationError(
"EVENT_ID_MISMATCH", "event message id is invalid"
)
if body["event_type"] == "recording.ready":
recording_payload = body.get("payload", {})
recording = self.store.one(
"SELECT status, oss_id FROM recordings WHERE tenant_id=? AND recording_id=?",
(tenant["tenant_id"], recording_payload.get("recording_id")),
)
if (
recording is None
or recording["status"] != "verified"
or recording["oss_id"] != recording_payload.get("oss_id")
):
raise ValidationError(
"ASSET_NOT_VERIFIED",
"recording.ready is not independently verified",
)
with self.store.tx() as con:
exists = con.execute(
"SELECT 1 FROM inbox WHERE tenant_id=? AND event_id=?",
(tenant["tenant_id"], event_id),
).fetchone()
if not exists:
con.execute(
"INSERT INTO inbox(event_id, tenant_id, applied_at, body_json) VALUES(?,?,?,?)",
(
event_id,
tenant["tenant_id"],
iso(self._now()),
canonical(body),
),
)
applied += 1
con.execute(
"UPDATE events SET saas_applied=1 WHERE tenant_id=? AND event_id=?",
(tenant["tenant_id"], event_id),
)
self.broker.ack(message)
except (KeyError, ServiceError, sqlite3.IntegrityError) as exc:
self._dead_letter(message, body, getattr(exc, "code", "INBOX_INVALID"))
reject = getattr(self.broker, "reject", None)
if callable(reject):
reject(message)
else:
self.broker.ack(message)
return applied
def apply_controls(self) -> int:
rows = self.store.all(
"SELECT * FROM commands WHERE command_type='task.control' AND status='applying' ORDER BY accepted_at"
)
count = 0
cell_ids = [str(cell["cell_id"]) for cell in self.profile.get("cells", [])]
for row in rows:
body = _loads(row["payload_json"])
action = body["action"]
policy = body.get("active_call_policy", "drain")
if not all(self._cell_health.get(cell_id, False) for cell_id in cell_ids):
continue
if action == "stop" and policy == "hangup":
calls = self.store.all(
"SELECT call_id FROM calls WHERE tenant_id=? AND task_id=? AND call_state NOT IN ('ended','failed')",
(row["tenant_id"], row["task_id"]),
)
for call in calls:
self._finish_controlled_call(call["call_id"])
with self.store.tx() as con:
task = con.execute(
"SELECT * FROM tasks WHERE tenant_id=? AND task_id=?",
(row["tenant_id"], row["task_id"]),
).fetchone()
if task is None:
continue
now = iso(self._now())
for cell_id in cell_ids:
con.execute(
"INSERT INTO control_barriers(command_id, tenant_id, cell_id, status, applied_at) VALUES(?,?,?,?,?) ON CONFLICT(tenant_id, command_id, cell_id) DO UPDATE SET status=excluded.status, applied_at=excluded.applied_at",
(row["command_id"], row["tenant_id"], cell_id, "applied", now),
)
con.execute(
"UPDATE commands SET status='applied', applied_task_revision=?, task_state=?, aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=? AND status='applying'",
(
task["revision"],
task["state"],
now,
row["tenant_id"],
row["command_id"],
),
)
updated = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(row["tenant_id"], row["command_id"]),
).fetchone()
self._command_event_tx(con, updated, "CONTROL_APPLIED")
affected = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND task_id=? AND status IN ('accepted','waiting')",
(row["tenant_id"], row["task_id"]),
).fetchall()
for affected_row in affected:
con.execute(
"UPDATE commands SET status='rejected', reason_code='TASK_CONTROL_BARRIER', aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?",
(now, affected_row["tenant_id"], affected_row["command_id"]),
)
con.execute(
"UPDATE executions SET status='rejected' WHERE tenant_id=? AND execution_id=?",
(affected_row["tenant_id"], affected_row["execution_id"]),
)
barrier_row = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(affected_row["tenant_id"], affected_row["command_id"]),
).fetchone()
self._command_event_tx(con, barrier_row, "TASK_CONTROL_BARRIER")
count += 1
return count
def _finish_controlled_call(self, call_id: str) -> None:
row = self.store.one("SELECT call_state FROM calls WHERE call_id=?", (call_id,))
if row is not None and row["call_state"] == "reconciling":
return
self._finish_call(call_id, "failed", "CONTROL_STOPPED")
def control_task(
self,
tenant_id: str,
task_id: str,
body: dict[str, Any],
idempotency_key: str,
allow_hangup: bool = False,
) -> dict[str, Any]:
tenant = self._tenant(tenant_id)
required = {"command_id", "action", "expected_task_revision", "reason"}
allowed = required | {"active_call_policy"}
if not required.issubset(body) or set(body) - allowed:
raise ValidationError(
"SCHEMA_INVALID", "control request fields do not match the contract"
)
if body["action"] not in {"pause", "resume", "stop"}:
raise ValidationError("SCHEMA_INVALID", "control action is invalid")
if (
type(body["expected_task_revision"]) is not int
or body["expected_task_revision"] < 1
):
raise ValidationError(
"SCHEMA_INVALID", "expected_task_revision must be a positive integer"
)
if (
not isinstance(body["reason"], str)
or not body["reason"]
or len(body["reason"]) > 512
):
raise ValidationError(
"SCHEMA_INVALID", "reason must contain 1..512 characters"
)
if body["action"] != "stop" and "active_call_policy" in body:
raise ValidationError(
"SCHEMA_INVALID", "active_call_policy is only valid for stop"
)
if body.get("active_call_policy", "drain") not in {"drain", "hangup"}:
raise ValidationError("SCHEMA_INVALID", "active_call_policy is invalid")
if body["command_id"] != idempotency_key:
raise ConflictError(
"IDEMPOTENCY_KEY_MISMATCH", "Idempotency-Key must equal command_id"
)
if (
body["action"] == "stop"
and body.get("active_call_policy") == "hangup"
and not allow_hangup
):
raise ServiceError(
"SCOPE_REQUIRED", "hangup control requires outbound.hangup scope", 403
)
task = self.store.one(
"SELECT * FROM tasks WHERE task_id=? AND tenant_id=?", (task_id, tenant_id)
)
if task is None:
raise NotFoundError()
payload_hash = self._operation_hash("task.control", task_id, body)
existing = self.store.one(
"SELECT * FROM commands WHERE command_id=? AND tenant_id=?",
(body["command_id"], tenant_id),
)
if existing:
if existing["payload_hash"] != payload_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"control command was already submitted with a different body",
)
return self._control_response(existing)
if body["expected_task_revision"] != task["revision"]:
raise ConflictError(
"TASK_REVISION_CONFLICT",
"expected_task_revision does not match current task revision",
)
if body["action"] == "resume":
applying = self.store.one(
"SELECT 1 FROM commands WHERE tenant_id=? AND task_id=? AND command_type='task.control' AND status='applying' LIMIT 1",
(tenant_id, task_id),
)
if applying:
raise ConflictError(
"CONTROL_IN_PROGRESS",
"the previous control barrier has not applied",
)
if task["state"] != "paused":
raise ConflictError(
"TASK_NOT_RESUMABLE", "only a paused task can be resumed"
)
if body["action"] in {"pause", "stop"} and task["state"] == "stopped":
raise ConflictError("TASK_STOPPED", "stopped task cannot be changed")
new_state = {"pause": "paused", "resume": "running", "stop": "stopped"}[
body["action"]
]
now = iso(self._now())
with self.store.tx() as con:
con.execute(
"UPDATE tasks SET state=?, revision=revision+1, updated_at=? WHERE tenant_id=? AND task_id=? AND revision=?",
(new_state, now, tenant_id, task_id, body["expected_task_revision"]),
)
if con.execute("SELECT changes()").fetchone()[0] != 1:
raise ConflictError(
"TASK_REVISION_CONFLICT", "task changed concurrently"
)
con.execute(
"""INSERT INTO commands(
command_id, tenant_id, tenant_key, command_type, status,
payload_json, payload_hash, operation_type, operation_target,
task_id, requested_task_revision, task_state, accepted_at,
trace_id, aggregate_version, updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(
body["command_id"],
tenant_id,
tenant["tenant_key"],
"task.control",
"applying",
canonical(body),
payload_hash,
"task.control",
task_id,
task_id,
body["expected_task_revision"],
new_state,
now,
body["command_id"],
1,
now,
),
)
row = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, body["command_id"]),
).fetchone()
self._command_event_tx(con, row, "CONTROL_ACCEPTED")
return self._control_response(
self.store.one(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, body["command_id"]),
)
)
def _control_response(self, row: sqlite3.Row) -> dict[str, Any]:
return {
"command_id": row["command_id"],
"tenant_id": row["tenant_id"],
"tenant_key": row["tenant_key"],
"task_id": row["task_id"],
"status": "accepted",
"requested_task_revision": row["requested_task_revision"],
"accepted_at": row["accepted_at"],
}
def _events_for_call(
self, tenant_id: str, call_id: str, cutoff: str | None = None
) -> list[sqlite3.Row]:
sql = "SELECT * FROM events WHERE tenant_id=?"
params: list[Any] = [tenant_id]
if cutoff is not None:
sql += " AND occurred_at<=?"
params.append(cutoff)
sql += " ORDER BY occurred_at, event_id"
rows = self.store.all(sql, tuple(params))
matched: list[sqlite3.Row] = []
for row in rows:
if row["aggregate_type"] == "call" and row["aggregate_id"] == call_id:
matched.append(row)
continue
payload = _loads(row["payload_json"], {})
if isinstance(payload, dict) and payload.get("call_id") == call_id:
matched.append(row)
return matched
def create_replay(
self,
tenant_id: str,
source_type: str,
source_id: str,
body: dict[str, Any],
idempotency_key: str,
) -> dict[str, Any]:
self._tenant(tenant_id)
if source_type not in {"call", "command"}:
raise ValidationError("SCHEMA_INVALID", "replay source type is invalid")
if (
set(body) != {"command_id", "reason"}
or body["command_id"] != idempotency_key
or not isinstance(body["reason"], str)
or not body["reason"]
or len(body["reason"]) > 512
):
raise ValidationError(
"SCHEMA_INVALID",
"replay request requires command_id and reason matching Idempotency-Key",
)
existing = self.store.one(
"SELECT * FROM commands WHERE command_id=? AND tenant_id=?",
(body["command_id"], tenant_id),
)
operation_hash = self._operation_hash(
"replay", f"{source_type}:{source_id}", body
)
if existing:
if existing["payload_hash"] != operation_hash:
raise ConflictError(
"IDEMPOTENCY_CONFLICT",
"replay command was already submitted with a different body",
)
return {
"command_id": body["command_id"],
"status": "accepted",
"snapshot_cutoff": existing["snapshot_cutoff"],
}
if source_type == "call":
source = self.store.one(
"SELECT * FROM calls WHERE call_id=? AND tenant_id=?",
(source_id, tenant_id),
)
else:
source = self.store.one(
"SELECT * FROM commands WHERE command_id=? AND tenant_id=?",
(source_id, tenant_id),
)
if source is None:
raise NotFoundError()
cutoff = iso(self._now())
retention = self._now() - timedelta(
seconds=self._limits().get("replay_retention_s", 604800)
)
rows = (
self._events_for_call(tenant_id, source_id, cutoff)
if source_type == "call"
else self.store.all(
"SELECT occurred_at FROM events WHERE tenant_id=? AND aggregate_id=? AND occurred_at<=? ORDER BY occurred_at LIMIT 1",
(tenant_id, source_id, cutoff),
)
)
if rows and parse_time(rows[0]["occurred_at"]) < retention:
raise ServiceError(
"REPLAY_EXPIRED", "source is outside the replay retention window", 410
)
replay_id = new_id("replay")
tenant = self.store.one(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,)
)["tenant_key"]
now = iso(self._now())
with self.store.tx() as con:
con.execute(
"""INSERT INTO commands(
command_id, tenant_id, tenant_key, command_type, status,
payload_json, payload_hash, operation_type, operation_target,
snapshot_cutoff, source_type, source_id, accepted_at, trace_id,
aggregate_version, updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(
body["command_id"],
tenant_id,
tenant,
"replay",
"accepted",
canonical(body),
operation_hash,
"replay",
f"{source_type}:{source_id}",
cutoff,
source_type,
source_id,
now,
body["command_id"],
1,
now,
),
)
con.execute(
"INSERT INTO replays(replay_id, command_id, tenant_id, source_type, source_id, snapshot_cutoff, status, created_at, updated_at) VALUES(?,?,?,?,?,?,?,?,?)",
(
replay_id,
body["command_id"],
tenant_id,
source_type,
source_id,
cutoff,
"accepted",
now,
now,
),
)
row = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(tenant_id, body["command_id"]),
).fetchone()
self._command_event_tx(con, row, "REPLAY_ACCEPTED")
return {
"command_id": body["command_id"],
"status": "accepted",
"snapshot_cutoff": cutoff,
}
def process_replays(self) -> int:
now = iso(self._now())
rows = self.store.all(
"SELECT * FROM replays WHERE status='accepted' AND (next_attempt_at IS NULL OR next_attempt_at<=?) ORDER BY created_at LIMIT 10",
(now,),
)
count = 0
max_attempts = max(1, _integer(self._limits().get("max_replay_attempts", 6)))
for replay in rows:
tenant_row = self.store.one(
"SELECT tenant_key FROM tenants WHERE tenant_id=?",
(replay["tenant_id"],),
)
if tenant_row is None:
continue
tenant_key = tenant_row["tenant_key"]
if replay["source_type"] == "call":
candidates = self._events_for_call(
replay["tenant_id"], replay["source_id"], replay["snapshot_cutoff"]
)
else:
candidates = self.store.all(
"SELECT * FROM events WHERE tenant_id=? AND aggregate_id=? AND occurred_at<=? ORDER BY occurred_at, event_id",
(
replay["tenant_id"],
replay["source_id"],
replay["snapshot_cutoff"],
),
)
sent = _integer(replay["sent_count"])
already = {
item["event_id"]
for item in self.store.all(
"SELECT event_id FROM replay_items WHERE replay_id=?",
(replay["replay_id"],),
)
}
try:
for event in candidates:
if event["event_id"] in already:
continue
self.broker.publish(
"agent-call.events.v1",
f"agent-call.{event['event_type']}",
_loads(event["body_json"]),
event["event_id"],
{
"x-replay-id": replay["replay_id"],
"tenant_id": replay["tenant_id"],
"tenant_key": tenant_key,
},
)
with self.store.tx() as con:
con.execute(
"INSERT OR IGNORE INTO replay_items(replay_id, event_id, published_at) VALUES(?,?,?)",
(replay["replay_id"], event["event_id"], iso(self._now())),
)
already.add(event["event_id"])
sent += 1
except BrokerError:
attempts = _integer(replay["retry_attempts"]) + 1
status = "failed" if attempts >= max_attempts else "accepted"
with self.store.tx() as con:
updated_at = iso(self._now())
next_attempt = (
None
if status == "failed"
else iso(
self._now()
+ timedelta(seconds=min(60, 2 ** min(attempts, 5)))
)
)
con.execute(
"UPDATE replays SET status=?, failure_code='BROKER_UNAVAILABLE', retry_attempts=?, sent_count=?, next_attempt_at=?, updated_at=? WHERE tenant_id=? AND replay_id=?",
(
status,
attempts,
sent,
next_attempt,
updated_at,
replay["tenant_id"],
replay["replay_id"],
),
)
if status == "failed":
con.execute(
"UPDATE commands SET status='failed', reason_code='REPLAY_BROKER_EXHAUSTED', aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?",
(updated_at, replay["tenant_id"], replay["command_id"]),
)
failed_command = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(replay["tenant_id"], replay["command_id"]),
).fetchone()
self._command_event_tx(con, failed_command, "REPLAY_FAILED")
continue
with self.store.tx() as con:
con.execute(
"UPDATE replays SET status='completed', failure_code=NULL, sent_count=?, next_attempt_at=NULL, updated_at=? WHERE tenant_id=? AND replay_id=?",
(sent, iso(self._now()), replay["tenant_id"], replay["replay_id"]),
)
con.execute(
"UPDATE commands SET status='completed', reason_code='REPLAY_SENT', aggregate_version=aggregate_version+1, updated_at=? WHERE tenant_id=? AND command_id=?",
(iso(self._now()), replay["tenant_id"], replay["command_id"]),
)
updated = con.execute(
"SELECT * FROM commands WHERE tenant_id=? AND command_id=?",
(replay["tenant_id"], replay["command_id"]),
).fetchone()
self._command_event_tx(con, updated, "REPLAY_SENT")
count += sent - _integer(replay["sent_count"])
return count
def emit_opt_out(self, tenant_id: str, call_id: str, contact_ref: str) -> None:
call = self.store.one(
"SELECT * FROM calls WHERE call_id=? AND tenant_id=?", (call_id, tenant_id)
)
if call is None:
raise NotFoundError()
tenant = self.store.one(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,)
)["tenant_key"]
with self.store.tx() as con:
self._event_tx(
con,
tenant_id,
tenant,
call["command_id"],
"contact.opt_out",
"call",
call_id,
call["call_version"] + 1,
{"call_id": call_id, "contact_ref": contact_ref, "source": "mock"},
)
def get_command(self, tenant_id: str, command_id: str) -> dict[str, Any]:
row = self.store.one(
"SELECT * FROM commands WHERE command_id=? AND tenant_id=?",
(command_id, tenant_id),
)
if row is None:
raise NotFoundError()
return self._command_snapshot(row)
def get_call(self, tenant_id: str, call_id: str) -> dict[str, Any]:
call = self.store.one(
"SELECT * FROM calls WHERE call_id=? AND tenant_id=?", (call_id, tenant_id)
)
if call is None:
raise NotFoundError()
attempts = [
dict(row)
for row in self.store.all(
"""SELECT a.attempt_id, a.attempt_number, a.route_policy_id,
a.caller_profile_id, a.target_uri, a.status, a.started_at, a.ended_at
FROM attempts a JOIN calls c ON c.call_id=a.call_id
WHERE a.call_id=? AND c.tenant_id=? ORDER BY a.attempt_number""",
(call_id, tenant_id),
)
]
segments = [
dict(row)
for row in self.store.all(
"""SELECT tr.segment_id, tr.turn_index, tr.speaker, tr.text,
tr.revision, tr.is_final, tr.playback_state, tr.updated_at
FROM transcripts tr JOIN calls c ON c.call_id=tr.call_id
WHERE tr.call_id=? AND c.tenant_id=? ORDER BY tr.turn_index, tr.speaker""",
(call_id, tenant_id),
)
]
recordings = [
dict(row)
for row in self.store.all(
"""SELECT r.recording_id, r.content_type, r.size_bytes,
r.checksum_algorithm, r.checksum, r.channels, r.sample_rate_hz,
r.duration_ms, r.status, r.oss_id, r.failure_code, r.verified_at
FROM recordings r JOIN calls c ON c.call_id=r.call_id
WHERE r.call_id=? AND c.tenant_id=? ORDER BY r.created_at""",
(call_id, tenant_id),
)
]
event_rows = self._events_for_call(tenant_id, call_id)
confirmed = sum(1 for row in event_rows if row["published"])
pending = sum(
1
for row in event_rows
if not row["published"] or row["broker_status"] == "failed"
)
applied_values = [row["saas_applied"] for row in event_rows if row["published"]]
saas_applied: bool | None = (
None
if not applied_values or any(value is None for value in applied_values)
else all(bool(value) for value in applied_values)
)
return {
"call_id": call["call_id"],
"execution_id": call["execution_id"],
"task_id": call["task_id"],
"task_item_id": call["task_item_id"],
"call_state": call["call_state"],
"call_version": call["call_version"],
"outcome": call["outcome"],
"reason_code": call["reason_code"],
"started_at": call["started_at"],
"ended_at": call["ended_at"],
"duration_ms": call["duration_ms"],
"attempts": attempts,
"transcript": {"segments": segments},
"recordings": recordings,
"delivery": {
"events_total": len(event_rows),
"broker_confirmed": confirmed,
"broker_pending": pending,
"saas_applied": saas_applied,
},
"snapshot_at": iso(self._now()),
}
def metrics(self) -> str:
active = self._active_counts()[0]
pending = self.store.one("SELECT COUNT(*) AS n FROM events WHERE published=0")[
"n"
]
dlq = self.store.one("SELECT COUNT(*) AS n FROM dead_letters")["n"]
waiting = self.store.one(
"SELECT COUNT(*) AS n FROM commands WHERE status='waiting'"
)["n"]
recording_failures = self.store.one(
"SELECT COUNT(*) AS n FROM recordings WHERE status='failed'"
)["n"]
mode = self.mode.replace('"', "")
providers = self._provider_status
try:
ready_value = 1 if self.ready() else 0
except (OSError, BrokerError, sqlite3.DatabaseError, TypeError, ValueError):
ready_value = 0
clock_value = 1 if self._clock_jump else 0
try:
disk = shutil.disk_usage(self.object_root)
disk_ratio = disk.used / disk.total if disk.total else 1.0
except (OSError, TypeError, ZeroDivisionError):
disk_ratio = 1.0
lines = [
"# HELP agent_call_active_calls Active calls including queued and ringing calls.",
"# TYPE agent_call_active_calls gauge",
f"agent_call_active_calls {active}",
"# TYPE agent_call_outbox_pending gauge",
f"agent_call_outbox_pending {pending}",
"# TYPE agent_call_waiting_commands gauge",
f"agent_call_waiting_commands {waiting}",
"# TYPE agent_call_dead_letters_total counter",
f"agent_call_dead_letters_total {dlq}",
"# TYPE agent_call_recording_failures_total counter",
f"agent_call_recording_failures_total {recording_failures}",
"# TYPE agent_call_mode_info gauge",
f'agent_call_mode_info{{mode="{mode}"}} 1',
"# TYPE agent_call_ready gauge",
f"agent_call_ready {ready_value}",
"# TYPE agent_call_clock_jump gauge",
f"agent_call_clock_jump {clock_value}",
"# TYPE agent_call_unknown_calls gauge",
f"agent_call_unknown_calls {self.readiness()['unknown_calls']}",
"# TYPE agent_call_disk_usage_ratio gauge",
f"agent_call_disk_usage_ratio {disk_ratio}",
]
for component, provider_mode in sorted(providers.items()):
safe_component = component.replace('"', "")
safe_mode = str(provider_mode).replace('"', "")
lines.append(
f'agent_call_provider_mode_info{{component="{safe_component}",mode="{safe_mode}"}} 1'
)
for tenant in self.profile.get("tenants", []):
try:
depth = getattr(self.broker, "depth", lambda _name: 0)(
queue_name(tenant["tenant_key"])
)
except (BrokerError, ValidationError):
depth = 0
tenant_label = str(tenant["tenant_id"]).replace('"', "")
lines.append(
f'agent_call_queue_depth{{tenant_id="{tenant_label}"}} {depth}'
)
return "\n".join(lines) + "\n"
def readiness(self) -> dict[str, Any]:
checks: dict[str, bool] = {
"database_read": False,
"database_write": False,
"broker": False,
"routes": not self._unrouteable_tenants,
"cells": bool(self._cell_health) and any(self._cell_health.values()),
"providers": self.mode == "mock"
and all(
value in {"mock", "sqlite", "memory", "fake-cli", "rabbit"}
for value in self._provider_status.values()
),
"clock": not self._clock_jump,
"reconciliation": False,
"disk": False,
}
try:
checks["database_read"] = self.store.one("SELECT 1") is not None
with self.store.tx() as con:
con.execute(
"INSERT INTO readiness_probe(probe_id, touched_at) VALUES(1, ?) ON CONFLICT(probe_id) DO UPDATE SET touched_at=excluded.touched_at",
(iso(self._now()),),
)
con.execute("DELETE FROM readiness_probe WHERE probe_id=1")
checks["database_write"] = True
except (OSError, sqlite3.DatabaseError):
pass
try:
checks["broker"] = bool(self.broker.ready())
has_route = getattr(self.broker, "has_route", None)
if callable(has_route):
checks["routes"] = checks["routes"] and all(
bool(has_route(routing_key(tenant["tenant_key"])))
for tenant in self.profile.get("tenants", [])
)
else:
checks["routes"] = False
except (BrokerError, ValidationError, OSError):
checks["broker"] = False
try:
usage = shutil.disk_usage(self.object_root)
ratio = usage.used / usage.total if usage.total else 1.0
checks["disk"] = ratio < float(self._limits().get("disk_stop_pct", 0.95))
except (OSError, TypeError, ValueError, ZeroDivisionError):
checks["disk"] = False
unknown = self.store.one(
"SELECT COUNT(*) AS n FROM calls WHERE call_state='reconciling'"
)["n"]
checks["reconciliation"] = _integer(unknown) == 0
return {
"ready": all(checks.values()),
"checks": checks,
"unknown_calls": _integer(unknown),
}
def ready(self) -> bool:
return bool(self.readiness()["ready"])
def wait_for_idle(self, timeout: float = 5.0) -> None:
end = time.monotonic() + timeout
while time.monotonic() < end:
self.process_once()
live = [
thread for thread in self._call_threads.values() if thread.is_alive()
]
if (
not live
and not self.store.one(
"SELECT 1 FROM commands WHERE status IN ('pending','retained','broker_confirmed','accepted','waiting','executing','applying','reconciling') LIMIT 1"
)
and not self.store.one(
"SELECT 1 FROM published_commands WHERE status IN ('pending','retained','broker_confirmed') LIMIT 1"
)
):
self.dispatch_outbox()
self.consume_saas_events()
return
time.sleep(0.01)
raise TimeoutError("mock service did not become idle")
Reference in New Issue View Git Blame Copy Permalink