feat(agent-call): implement remediation plan

This commit is contained in:
2026-09-13 19:20:28 +08:00
parent d30314c7d1
commit b95d2aee81
25 changed files with 3159 additions and 535 deletions
+2023 -425
View File
File diff suppressed because it is too large Load Diff
+58 -14
View File
@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import hashlib
import json import json
import os import os
from http import HTTPStatus from http import HTTPStatus
@@ -15,6 +16,7 @@ READ_SCOPE = "outbound.read"
CONTROL_SCOPE = "outbound.control" CONTROL_SCOPE = "outbound.control"
UPLOAD_SCOPE = "recording.upload" UPLOAD_SCOPE = "recording.upload"
COMPLETE_SCOPE = "recording.complete" COMPLETE_SCOPE = "recording.complete"
REPLAY_SCOPE = "outbound.replay"
class AgentCallHandler(BaseHTTPRequestHandler): class AgentCallHandler(BaseHTTPRequestHandler):
@@ -37,7 +39,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
raise ServiceError("TENANT_ID_REQUIRED", "X-Tenant-ID is required", 400) raise ServiceError("TENANT_ID_REQUIRED", "X-Tenant-ID is required", 400)
return value return value
def _authorize(self, scope: str, tenant_id: str | None = None) -> None: def _authorize(self, scope: str, tenant_id: str | None = None) -> dict[str, Any]:
token_header = self.headers.get("Authorization", "") token_header = self.headers.get("Authorization", "")
if not token_header.startswith("Bearer "): if not token_header.startswith("Bearer "):
raise ServiceError("UNAUTHORIZED", "bearer authentication is required", 401) raise ServiceError("UNAUTHORIZED", "bearer authentication is required", 401)
@@ -49,12 +51,24 @@ class AgentCallHandler(BaseHTTPRequestHandler):
if not isinstance(identity, dict): if not isinstance(identity, dict):
raise ServiceError("UNAUTHORIZED", "bearer token is not authorized", 401) raise ServiceError("UNAUTHORIZED", "bearer token is not authorized", 401)
scopes = identity.get("scopes", []) scopes = identity.get("scopes", [])
tenant_ids = identity.get("tenant_ids", [])
if not isinstance(scopes, list) or not all(
isinstance(item, str) for item in scopes
):
raise ServiceError("UNAUTHORIZED", "token scopes are invalid", 401)
if not isinstance(tenant_ids, list) or not all(
isinstance(item, str) for item in tenant_ids
):
raise ServiceError("UNAUTHORIZED", "token tenant scope is invalid", 401)
if scope not in scopes and "*" not in scopes: if scope not in scopes and "*" not in scopes:
raise ServiceError("FORBIDDEN", "required scope is missing", 403) raise ServiceError("FORBIDDEN", "required scope is missing", 403)
if tenant_id is not None: if (
tenant_ids = identity.get("tenant_ids", []) tenant_id is not None
if "*" not in tenant_ids and tenant_id not in tenant_ids: and "*" not in tenant_ids
raise ServiceError("FORBIDDEN", "tenant scope is missing", 403) and tenant_id not in tenant_ids
):
raise ServiceError("FORBIDDEN", "tenant scope is missing", 403)
return identity
def _body(self) -> dict[str, Any]: def _body(self) -> dict[str, Any]:
raw_length = self.headers.get("Content-Length") raw_length = self.headers.get("Content-Length")
@@ -69,6 +83,8 @@ class AgentCallHandler(BaseHTTPRequestHandler):
) )
try: try:
raw = self.rfile.read(length) raw = self.rfile.read(length)
if len(raw) != length:
raise ValidationError("BODY_TRUNCATED", "request body was truncated")
body = _loads(raw.decode("utf-8")) body = _loads(raw.decode("utf-8"))
except UnicodeDecodeError as exc: except UnicodeDecodeError as exc:
raise ValidationError( raise ValidationError(
@@ -80,10 +96,21 @@ class AgentCallHandler(BaseHTTPRequestHandler):
def _header(self, name: str) -> str: def _header(self, name: str) -> str:
value = self.headers.get(name, "") value = self.headers.get(name, "")
if not value: if not value or len(value) > 128 or any(ch.isspace() for ch in value):
raise ServiceError("HEADER_REQUIRED", f"{name} is required", 400) raise ServiceError("HEADER_REQUIRED", f"{name} is required", 400)
return value return value
def _actor_id(self) -> str:
token_header = self.headers.get("Authorization", "")
token = token_header[7:] if token_header.startswith("Bearer ") else ""
identity = _loads(os.environ.get("HTTP_TOKENS"), {})
principal = identity.get(token) if isinstance(identity, dict) else None
if isinstance(principal, dict):
value = principal.get("client_id") or principal.get("subject")
if isinstance(value, str) and value:
return value
return "token:" + hashlib.sha256(token.encode("utf-8")).hexdigest()[:16]
def _send( def _send(
self, status: int, body: Any, content_type: str = "application/json" self, status: int, body: Any, content_type: str = "application/json"
) -> None: ) -> None:
@@ -195,7 +222,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
and parts[5] == "replays" and parts[5] == "replays"
): ):
tenant_id = self._tenant_id() tenant_id = self._tenant_id()
self._authorize(CONTROL_SCOPE, tenant_id) self._authorize(REPLAY_SCOPE, tenant_id)
result = self.service.create_replay( result = self.service.create_replay(
tenant_id, tenant_id,
"call", "call",
@@ -213,7 +240,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
and parts[5] == "replays" and parts[5] == "replays"
): ):
tenant_id = self._tenant_id() tenant_id = self._tenant_id()
self._authorize(CONTROL_SCOPE, tenant_id) self._authorize(REPLAY_SCOPE, tenant_id)
result = self.service.create_replay( result = self.service.create_replay(
tenant_id, tenant_id,
"command", "command",
@@ -231,8 +258,9 @@ class AgentCallHandler(BaseHTTPRequestHandler):
): ):
tenant_id = self._tenant_id() tenant_id = self._tenant_id()
self._authorize(UPLOAD_SCOPE, tenant_id) self._authorize(UPLOAD_SCOPE, tenant_id)
idempotency_key = self._header("Idempotency-Key")
result_status, result = self.service.create_upload( result_status, result = self.service.create_upload(
tenant_id, self._body() tenant_id, self._body(), idempotency_key, self._actor_id()
) )
self._send(result_status, result) self._send(result_status, result)
return return
@@ -245,7 +273,10 @@ class AgentCallHandler(BaseHTTPRequestHandler):
): ):
tenant_id = self._tenant_id() tenant_id = self._tenant_id()
self._authorize(COMPLETE_SCOPE, tenant_id) self._authorize(COMPLETE_SCOPE, tenant_id)
result = self.service.complete_upload(tenant_id, parts[4], self._body()) idempotency_key = self._header("Idempotency-Key")
result = self.service.complete_upload(
tenant_id, parts[4], self._body(), idempotency_key, self._actor_id()
)
self._send(200, result) self._send(200, result)
return return
if method == "PUT" and len(parts) == 3 and parts[:2] == ["_mock", "oss"]: if method == "PUT" and len(parts) == 3 and parts[:2] == ["_mock", "oss"]:
@@ -260,8 +291,8 @@ class AgentCallHandler(BaseHTTPRequestHandler):
if not raw_length: if not raw_length:
raise ValidationError("BODY_REQUIRED", "upload body is required") raise ValidationError("BODY_REQUIRED", "upload body is required")
length = _integer(raw_length) length = _integer(raw_length)
if length > self.service.profile.get("limits", {}).get( if length < 0 or length > self.service.profile.get("limits", {}).get(
"recording_max_bytes", 16777216 "max_recording_bytes", 16777216
): ):
raise ServiceError( raise ServiceError(
"BODY_TOO_LARGE", "BODY_TOO_LARGE",
@@ -274,6 +305,10 @@ class AgentCallHandler(BaseHTTPRequestHandler):
raise ServiceError( raise ServiceError(
"UPLOAD_INTERRUPTED", "upload connection interrupted", 400 "UPLOAD_INTERRUPTED", "upload connection interrupted", 400
) from exc ) from exc
if len(data) != length:
raise ServiceError(
"UPLOAD_INTERRUPTED", "upload body was truncated", 400
)
upload = self.service.store.one( upload = self.service.store.one(
"SELECT tenant_id FROM uploads WHERE upload_id=?", (parts[2],) "SELECT tenant_id FROM uploads WHERE upload_id=?", (parts[2],)
) )
@@ -287,7 +322,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
self._error(exc, request_id) self._error(exc, request_id)
except (BrokenPipeError, ConnectionResetError): except (BrokenPipeError, ConnectionResetError):
return return
except Exception: except (OSError, RuntimeError, TypeError, ValueError, KeyError):
self._error( self._error(
ServiceError("INTERNAL_ERROR", "request could not be completed", 500), ServiceError("INTERNAL_ERROR", "request could not be completed", 500),
request_id, request_id,
@@ -304,8 +339,17 @@ class AgentCallHandler(BaseHTTPRequestHandler):
if not isinstance(identity, dict): if not isinstance(identity, dict):
return False return False
tenant_ids = identity.get("tenant_ids", []) tenant_ids = identity.get("tenant_ids", [])
scopes = identity.get("scopes", [])
if not isinstance(tenant_ids, list) or not all(
isinstance(item, str) for item in tenant_ids
):
return False
if not isinstance(scopes, list) or not all(
isinstance(item, str) for item in scopes
):
return False
return ("*" in tenant_ids or tenant_id in tenant_ids) and ( return ("*" in tenant_ids or tenant_id in tenant_ids) and (
scope in identity.get("scopes", []) or "*" in identity.get("scopes", []) scope in scopes or "*" in scopes
) )
def do_GET(self) -> None: def do_GET(self) -> None:
+40 -3
View File
@@ -6,6 +6,7 @@ ARI, RTP, AI-stream, or OSS integration.
from __future__ import annotations from __future__ import annotations
import hashlib
import json import json
import threading import threading
from pathlib import Path from pathlib import Path
@@ -82,9 +83,23 @@ class MockRTP:
def __init__(self, journal: ProtocolJournal) -> None: def __init__(self, journal: ProtocolJournal) -> None:
self.journal = journal self.journal = journal
def packet(self, call_id: str, direction: str, sequence: int) -> None: def packet(
self,
call_id: str,
direction: str,
sequence: int,
payload: bytes | None = None,
) -> None:
payload = payload if payload is not None else b"\x01\x02" * 160
self.journal.append( self.journal.append(
"rtp", "packet", call_id=call_id, direction=direction, sequence=sequence "rtp",
"packet",
call_id=call_id,
direction=direction,
sequence=sequence,
payload_bytes=len(payload),
payload_sha256=hashlib.sha256(payload).hexdigest(),
non_silent=any(payload),
) )
@@ -93,7 +108,12 @@ class MockAI:
self.journal = journal self.journal = journal
def turn( def turn(
self, call_id: str, turn_index: int, speaker: str, playback_state: str self,
call_id: str,
turn_index: int,
speaker: str,
playback_state: str,
text: str | None = None,
) -> None: ) -> None:
self.journal.append( self.journal.append(
"ai", "ai",
@@ -102,4 +122,21 @@ class MockAI:
turn_index=turn_index, turn_index=turn_index,
speaker=speaker, speaker=speaker,
playback_state=playback_state, playback_state=playback_state,
text=text or f"mock-{speaker}-turn-{turn_index}",
)
self.journal.append(
"ai",
"playback",
call_id=call_id,
turn_index=turn_index,
state="sent",
)
self.journal.append(
"ai",
"playback",
call_id=call_id,
turn_index=turn_index,
state=playback_state
if playback_state in {"confirmed", "playback_confirmed", "unknown"}
else "unknown",
) )
+59 -7
View File
@@ -4,18 +4,70 @@ groups:
- alert: AgentCallOutboxStalled - alert: AgentCallOutboxStalled
expr: agent_call_outbox_pending > 0 expr: agent_call_outbox_pending > 0
for: 2m for: 2m
labels: {severity: warning} labels:
severity: warning
annotations: annotations:
summary: agent-call outbox has pending events summary: agent-call outbox is not draining
- alert: AgentCallDeadLetters - alert: AgentCallDeadLetters
expr: increase(agent_call_dead_letters_total[5m]) > 0 expr: increase(agent_call_dead_letters_total[5m]) > 0
for: 1m for: 1m
labels: {severity: critical} labels:
severity: critical
annotations: annotations:
summary: agent-call rejected or malformed messages entered the DLQ summary: agent-call rejected messages reached the dead-letter log
- alert: AgentCallNotMockSafe - alert: AgentCallNotMockSafe
expr: agent_call_mode_info{mode="mock"} != 1 expr: agent_call_mode_info{mode!="mock"} > 0
for: 1m for: 1m
labels: {severity: critical} labels:
severity: critical
annotations: annotations:
summary: provider mode changed; verify production admission gates summary: agent-call is running outside the supported Mock mode
- alert: AgentCallNotReady
expr: agent_call_ready == 0
for: 1m
labels:
severity: critical
annotations:
summary: agent-call readiness gate is closed
- alert: AgentCallClockJump
expr: agent_call_clock_jump > 0
for: 1m
labels:
severity: critical
annotations:
summary: agent-call stopped new admission after a wall-clock jump
- alert: AgentCallUnknownCall
expr: agent_call_unknown_calls > 0
for: 1m
labels:
severity: critical
annotations:
summary: active call requires reconciliation and still occupies capacity
- alert: AgentCallQueueBackpressure
expr: sum(agent_call_queue_depth) > 0
for: 5m
labels:
severity: warning
annotations:
summary: tenant command queues have pending backpressure
- alert: AgentCallRecordingFailures
expr: increase(agent_call_recording_failures_total[5m]) > 0
for: 1m
labels:
severity: critical
annotations:
summary: recording verification failed
- alert: AgentCallDiskWarning
expr: agent_call_disk_usage_ratio >= 0.70
for: 5m
labels:
severity: warning
annotations:
summary: agent-call disk usage is high
- alert: AgentCallDiskStop
expr: agent_call_disk_usage_ratio >= 0.80
for: 1m
labels:
severity: critical
annotations:
summary: agent-call stopped new admissions for disk pressure
Regular → Executable
View File
Regular → Executable
+47 -18
View File
@@ -1,36 +1,65 @@
#!/usr/bin/env python3
"""Create or restore an SQLite backup without silently overwriting a target."""
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import sqlite3 import sqlite3
from contextlib import closing
from pathlib import Path from pathlib import Path
def copy_database(source: str, destination: str) -> None: def _check_integrity(con: sqlite3.Connection) -> None:
source_path = Path(source) result = con.execute("PRAGMA integrity_check").fetchone()[0]
destination_path = Path(destination) if result != "ok":
raise RuntimeError(f"SQLite integrity check failed: {result}")
def copy_database(
source: str | Path, destination: str | Path, *, replace: bool = False
) -> None:
source_path = Path(source).expanduser().resolve()
destination_path = Path(destination).expanduser().resolve()
if source_path == destination_path:
raise ValueError("source and destination must differ")
if not source_path.is_file(): if not source_path.is_file():
raise FileNotFoundError(source) raise FileNotFoundError(source_path)
if destination_path.exists() and not replace:
raise FileExistsError(
f"refusing to overwrite existing database: {destination_path}"
)
destination_path.parent.mkdir(parents=True, exist_ok=True) destination_path.parent.mkdir(parents=True, exist_ok=True)
source_db = sqlite3.connect(source_path)
destination_db = sqlite3.connect(destination_path)
try: try:
source_db.backup(destination_db) with closing(
destination_db.commit() sqlite3.connect(f"file:{source_path}?mode=ro", uri=True)
finally: ) as source_con:
destination_db.close() _check_integrity(source_con)
source_db.close() with closing(sqlite3.connect(destination_path)) as destination_con:
source_con.backup(destination_con)
destination_con.commit()
_check_integrity(destination_con)
version = destination_con.execute(
"SELECT MAX(version) FROM schema_migrations"
).fetchone()[0]
if version is None:
raise RuntimeError("backup has no schema migration version")
except sqlite3.DatabaseError as exc:
raise RuntimeError(f"SQLite backup failed: {exc}") from exc
print(f"backup complete: {source_path} -> {destination_path}")
def main() -> int: def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser()
description="offline SQLite backup/restore for agent-call"
)
parser.add_argument("action", choices=("backup", "restore")) parser.add_argument("action", choices=("backup", "restore"))
parser.add_argument("source") parser.add_argument("source")
parser.add_argument("destination") parser.add_argument("destination")
args = parser.parse_args() parser.add_argument(
copy_database(args.source, args.destination) "--replace",
print(f"{args.action} complete") action="store_true",
help="explicitly allow replacing the destination",
)
args = parser.parse_args(argv)
copy_database(args.source, args.destination, replace=args.replace)
return 0 return 0
+10 -6
View File
@@ -61,9 +61,7 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None
f"ASTERISK_IMAGE={shlex.quote(image)}", f"ASTERISK_IMAGE={shlex.quote(image)}",
f"ASTERISK_CONFIG_GID={config_gid}", f"ASTERISK_CONFIG_GID={config_gid}",
] ]
variables.extend( variables.extend(f"{key}={shlex.quote(value)}" for key, value in payloads.items())
f"{key}={shlex.quote(value)}" for key, value in payloads.items()
)
variables.append("") variables.append("")
if template is None: if template is None:
template = Path(__file__).with_name("asterisk_bootstrap.sh") template = Path(__file__).with_name("asterisk_bootstrap.sh")
@@ -76,7 +74,9 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None
def write_output(content, path): def write_output(content, path):
path = Path(path) path = Path(path)
if path.exists() or path.is_symlink(): if path.exists() or path.is_symlink():
raise ValueError("output already exists; choose a new local secret-bearing path") raise ValueError(
"output already exists; choose a new local secret-bearing path"
)
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
temporary_path = None temporary_path = None
try: try:
@@ -105,7 +105,9 @@ def main():
parser.add_argument("--config", required=True, help="render_asterisk JSON config") parser.add_argument("--config", required=True, help="render_asterisk JSON config")
parser.add_argument("--image", help="immutable registry image@sha256:digest") parser.add_argument("--image", help="immutable registry image@sha256:digest")
parser.add_argument("--ari-password-file") parser.add_argument("--ari-password-file")
parser.add_argument("--output", required=True, help="local secret-bearing user-data path") parser.add_argument(
"--output", required=True, help="local secret-bearing user-data path"
)
parser.add_argument("--config-gid", type=int, default=1000) parser.add_argument("--config-gid", type=int, default=1000)
args = parser.parse_args() args = parser.parse_args()
try: try:
@@ -113,7 +115,9 @@ def main():
image = immutable_image( image = immutable_image(
args.image or cfg.get("asterisk_image") or os.environ.get("ASTERISK_IMAGE") args.image or cfg.get("asterisk_image") or os.environ.get("ASTERISK_IMAGE")
) )
content = build(cfg, image, read_secret(args.ari_password_file), args.config_gid) content = build(
cfg, image, read_secret(args.ari_password_file), args.config_gid
)
if len(content.encode()) > 16 * 1024: if len(content.encode()) > 16 * 1024:
raise ValueError("generated user-data exceeds the ECS 16 KiB limit") raise ValueError("generated user-data exceeds the ECS 16 KiB limit")
write_output(content, args.output) write_output(content, args.output)
+2 -1
View File
@@ -10,6 +10,7 @@ if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT)) sys.path.insert(0, str(PROJECT_ROOT))
_core = importlib.import_module("agent_call.core") _core = importlib.import_module("agent_call.core")
ValidationError = _core.ValidationError
queue_name = _core.queue_name queue_name = _core.queue_name
routing_key = _core.routing_key routing_key = _core.routing_key
@@ -30,7 +31,7 @@ def check(profile_path: str | Path) -> list[str]:
try: try:
route = routing_key(tenant["tenant_key"]) route = routing_key(tenant["tenant_key"])
queue_name(tenant["tenant_key"]) queue_name(tenant["tenant_key"])
except Exception as exc: except (KeyError, TypeError, ValidationError) as exc:
problems.append(f"tenant route invalid: {type(exc).__name__}") problems.append(f"tenant route invalid: {type(exc).__name__}")
continue continue
if len(route.encode("utf-8")) > 255: if len(route.encode("utf-8")) > 255:
+6 -3
View File
@@ -15,9 +15,12 @@ Store = _core.Store
def migrate(path: str) -> int: def migrate(path: str) -> int:
store = Store(path) store = Store(path)
row = store.one("SELECT MAX(version) AS version FROM schema_migrations") try:
print(f"schema_version={row['version']}") row = store.one("SELECT MAX(version) AS version FROM schema_migrations")
return 0 print(f"schema_version={row['version']}")
return 0
finally:
store.close()
def main() -> int: def main() -> int:
+3
View File
@@ -2,6 +2,9 @@ global:
scrape_interval: 15s scrape_interval: 15s
evaluation_interval: 15s evaluation_interval: 15s
rule_files:
- /etc/prometheus/alerts.yml
scrape_configs: scrape_configs:
- job_name: agent-call - job_name: agent-call
metrics_path: /metrics metrics_path: /metrics
Regular → Executable
View File
+1 -1
View File
@@ -163,7 +163,7 @@ components:
type: string type: string
minLength: 1 minLength: 1
maxLength: 128 maxLength: 128
pattern: '^[^\\s/\\\\]+$' pattern: '^[^\s/\\]+$'
ControlRequest: ControlRequest:
type: object type: object
additionalProperties: false additionalProperties: false
+13 -3
View File
@@ -20,14 +20,24 @@
"max_call_duration_ms": 180000, "max_call_duration_ms": 180000,
"max_mq_bytes": 262144, "max_mq_bytes": 262144,
"max_queue_messages": 1000, "max_queue_messages": 1000,
"max_queue_bytes": 16777216,
"disk_warn_pct": 0.70,
"disk_stop_pct": 0.80,
"max_http_bytes": 65536, "max_http_bytes": 65536,
"recording_max_bytes": 16777216, "recording_max_bytes": 16777216,
"max_recording_bytes": 16777216,
"replay_retention_s": 604800, "replay_retention_s": 604800,
"upload_ttl_s": 300, "upload_ttl_s": 300,
"global_concurrency": 6, "global_concurrency": 6,
"global_cps": 3, "global_cps": 3,
"tenant_concurrency": 2, "tenant_concurrency": 2,
"tenant_cps": 1, "tenant_cps": 1,
"tenant_publish_rate": 10,
"scheduler_lease_ttl_s": 10,
"pending_window_per_tenant": 16,
"pending_window_global": 64,
"max_unacked_per_tenant": 4,
"max_replay_attempts": 6,
"cell_capacity": 4, "cell_capacity": 4,
"turns": 2, "turns": 2,
"hold_ms": 0 "hold_ms": 0
@@ -47,8 +57,8 @@
"caller_profiles": [{"caller_profile_id": "caller_profile_test", "display": "BD93205882", "allowed": true}], "caller_profiles": [{"caller_profile_id": "caller_profile_test", "display": "BD93205882", "allowed": true}],
"agents": [{"agent_version_id": "agent_v1", "immutable": true, "llm": "mock", "tts": "mock", "asr": "mock"}], "agents": [{"agent_version_id": "agent_v1", "immutable": true, "llm": "mock", "tts": "mock", "asr": "mock"}],
"cells": [ "cells": [
{"cell_id": "cell-a", "capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"}, {"cell_id": "cell-a", "capacity": 4, "media_capacity": 4, "ai_capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"},
{"cell_id": "cell-b", "capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"} {"cell_id": "cell-b", "capacity": 4, "media_capacity": 4, "ai_capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"}
], ],
"failure_scenarios": ["success", "busy", "no_answer", "ai_timeout", "ari_disconnect", "upload_missing", "upload_bad_checksum", "broker_outage", "clock_jump"] "failure_scenarios": ["success", "busy", "no_answer", "ai_timeout", "customer_silent", "ari_disconnect", "upload_missing", "upload_bad_checksum", "broker_outage", "clock_jump"]
} }
+1 -1
View File
@@ -88,7 +88,7 @@ components:
type: string type: string
minLength: 1 minLength: 1
maxLength: 128 maxLength: 128
pattern: '^[^\\s/\\\\]+$' pattern: '^[^\s/\\]+$'
UploadRequest: UploadRequest:
type: object type: object
additionalProperties: false additionalProperties: false
@@ -0,0 +1,128 @@
{
"finished_at": "2026-09-13T10:59:31.729Z",
"gate_status": "INCOMPLETE",
"probes": {
"V01": {
"duration_ms": 15.796,
"evidence": {
"http_execute_route": "absent",
"profile": "mock",
"schema_negative_cases": 3
},
"status": "PASS"
},
"V02": {
"duration_ms": 161.588,
"evidence": {
"calls": 1,
"different_execution_command": "associated",
"same_command_replays": 1
},
"status": "PASS"
},
"V03": {
"duration_ms": 344.365,
"evidence": {
"lease_duplicate_calls": 0,
"scheduler_instances": 2,
"tenants_admitted": [
"tenant-demo",
"tenant-b",
"tenant-c"
]
},
"status": "PASS"
},
"V04": {
"duration_ms": 34.727,
"evidence": {
"affected_commands": 1,
"stale_cas": "rejected",
"waiting_barrier": "applied"
},
"status": "PASS"
},
"V05": {
"duration_ms": 149.141,
"evidence": {
"dead_letters": 0,
"recovered": true,
"retained_id": "cmd_broker_retry"
},
"status": "PASS"
},
"V06/V07": {
"duration_ms": 2131.36,
"evidence": {
"asterisk_real": "not_run",
"disconnect_state_before_reconcile": "reconciling",
"fallback_redials": 0,
"mock_protocol": "pass",
"outcomes": {
"busy": "busy",
"disconnect": "failed",
"no_answer": "no_answer"
}
},
"status": "PASS"
},
"V08": {
"duration_ms": 994.059,
"evidence": {
"old_tts_reused": false,
"segments": 200,
"turns": 100
},
"status": "PASS"
},
"V09": {
"duration_ms": 1131.946,
"evidence": {
"bad_checksum": "failed",
"invalid_ready": 0,
"missing_object": "failed"
},
"status": "PASS"
},
"V10": {
"duration_ms": 113.747,
"evidence": {
"original_event_ids": true,
"pending_after_recovery": 0,
"pending_during_outage": 13
},
"status": "PASS"
},
"V11/V12": {
"duration_ms": 144.282,
"evidence": {
"backup": "verified",
"metrics": "healthy",
"real_mode_mock_rejection": true,
"restored_counts": {
"commands": 1,
"events": 13,
"inbox": 13,
"recordings": 1
},
"restored_db_facts": true,
"rollback_preserves_db": "not_run"
},
"status": "PASS"
},
"V13": {
"duration_ms": 0.543,
"evidence": {
"evidence_file": "docs/evidence/scale-mock-2026-09-12.json",
"reason": "historical state-only evidence cannot satisfy continuous media V13",
"status": "not_run"
},
"status": "NOT_RUN"
}
},
"production": {
"V14": "blocked_external_evidence"
},
"profile": "mock",
"started_at": "2026-09-13T10:59:26.507Z"
}
+30 -19
View File
@@ -1,27 +1,38 @@
# 持久真实 SIP/Asterisk 测试环境证据(2026-09-13) # 真实 SIP/Asterisk 测试环境证据(2026-09-13)
## 环境 ## 当前环境(探针期间)
- ECS:`i-2zegaldd07mkp0xyhcff`,Ubuntu 24.04,`ecs.e-c1m1.large`。 - ECS:`i-2ze4hvi6xvy02g0wz9kk`,北京 `cn-beijing`,Ubuntu 24.04,`ecs.e-c1m1.large`,探针期间 Running。
- 竞价策略:`SpotAsPriceGo`(自动竞价);竞价回收可能中断活动通话。 - 竞价策略:`SpotAsPriceGo`;竞价回收可能中断活动通话。
- 固定 EIP:`123.56.71.98`,当前保持 `InUse` 并绑定该 ECS。 - 固定 EIP:`123.56.71.98`,AllocationId `eip-2zeevfsaxzwuue2szy7xb`;探针期间 InUse 并绑定该 ECS。
- Asterisk:`22.10.1`,固定镜像 digest:`sha256:0f5e3cd0e9a86bc9dc4750929abee9ba1faae11ed39739f3e75343d39d837e41`。 - 创建/绑定:2026-09-13 10:28 UTC;使用专用 VSwitch、安全组和 KeyPair,实例公网带宽上限为 0。
- 线路:UDP `61.132.228.221:5060`,IP 白名单免注册。 - Asterisk:22.10.1,镜像及实际 image ID 均为固定 digest `sha256:0f5e3cd0e9a86bc9dc4750929abee9ba1faae11ed39739f3e75343d39d837e41`,容器状态 `running/healthy`。
- PJSIP contact:`Avail`,实测 RTT 约 `22.146 ms`。 - 管理检查:SSH 未开放;通过阿里云 Cloud Assistant 执行只读检查。Cloud-init 初始拉取镜像期间较慢,最终完成并启动容器。
- 主叫原值:`BD93205882`;`From user` 与 `CALLERID` 均使用原值,未发送 P-Asserted-Identity。 - PJSIP contact:`provider-primary-aor/sip:61.132.228.221:5060` 为 `Avail`,RTT 约 `22.164 ms`。
- 环境按用户要求保留,未在本轮测试后停止或删除。 - 出口与信令:SIP 日志显示请求从 `123.56.71.98:5060` 发出;安全组仅允许供应商 `61.132.228.221/32` 的 UDP `5060` 和 `10000–10800`。
- 主叫原值:`BD93205882`;`From user`/`CALLERID` 保留原值,未发送 P-Asserted-Identity。
## 两个被叫测试 ## 单次真实外呼探针
两次均使用同一环境、线路、主叫、UDP/ulaw、RTP 配置;均只发起一次,不自动重试。 - 执行时间:2026-09-13 10:52:59 UTC。
- 原始被叫:`18601013734`。
- 实际目标:`708918601013734`。
- 线路:`provider-primary`,UDP/ulaw;仅发起一次,无自动重试、无 FALLBACK。
- SIP:收到 `100 trying`,随后收到 `488 Not Acceptable Here`。
- 原因:`Reason: Q.850;cause=88;text="INCOMPATIBLE_DESTINATION"`。
- 后续:发送 ACK;Asterisk 执行 Hangup;15 秒后无活动通道。
- 接通/RTP:未接通,未获得 `180/183/200` 或 RTP/录音证据。
| 原始被叫 | 实际呼出目标 | SIP结果 | RTP/接通 | ## 测试后资源处置
| --- | --- | --- | --- |
| `18601013734` | `708918601013734` | `100 trying` 后 `488 Not Acceptable Here`;`Q.850 cause=88 INCOMPATIBLE_DESTINATION` | 未接通、无 RTP |
| `18625770806` | `708918625770806` | `100 trying` 后 `488 Not Acceptable Here`;`Q.850 cause=88 INCOMPATIBLE_DESTINATION` | 未接通、无 RTP |
两组不同号码在同一线路上得到完全相同的目的地不兼容结果,进一步支持供应商共享路由/线路参数问题;仍不能仅凭两次失败排除同一号码段盲区。需供应商提供已知可达号码,或确认被叫格式/路由、From 域和编解码要求后再继续。 按用户授权,探针结束后已优雅停止并删除本次创建的 ECS;删除前确认该实例是唯一 `project=agent-call` 候选,系统盘为 40 GiB 且 `DeleteWithInstance=true`。已确认实例和系统盘不再返回;固定 EIP 未释放、未解绑,当前为 `Available`。
## 范围边界 ## 结论
本次仅验证了持久 ECS、Asterisk、SIP OPTIONS/contact、两次真实 INVITE 和失败回执;未验证真实 RTP、MQ、OSS、ASR、LLM/TTS 或完整 AI 通话。 本次确认了单路 ECS/Asterisk 底座、PJSIP contact、固定出口和一次真实 INVITE 链路;供应商以 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION` 拒绝目标,R1 外呼接通验收未通过。该结果不能证明真实 RTP、ASR、LLM/TTS、MQ、OSS 或生产容量/N+1 已验收。
供应商需先确认 `7089` 前缀/目标路由、From 域/主叫格式和编解码,或提供已知可达测试号码。未确认前不再次拨号。
## 历史记录
此前同一固定 EIP/线路对 `18625770806` 的探针目标 `708918625770806` 也返回 `488/Q.850 cause=88`;历史实例已按测试回收规则删除,固定 EIP 未释放。
@@ -0,0 +1,186 @@
# agent-call:功能审查整改计划与验收标准
**版本:** v1.0
**日期:** 2026-09-13
**状态:** 代码级整改已实现并通过本地回归;真实协议、独立服务、告警链路及规模门禁仍未通过。
**审查基线:** main / `d30314c7d1c56708a43333fe66ad68cfe7c29f40`,三路静态代码审查;未运行故障复现、测试或真实外呼。下列代码位置为该基线定位,后续以符号和回归用例追踪。`deploy/build_asterisk_userdata.py` 的既有未提交修改不在本次整改文档编辑范围内。
## 1. 目的、权威与边界
本计划解决“真实 SIP 暂未接通之外,现有功能还需补齐什么、怎样证明修好”。真实线路排障不阻塞本地整改,不能据此把其余功能视为完成。
- [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)仍是字段、路径、权限、状态和消息语义的唯一维护来源;本文不新增业务接口或状态枚举。
- [最终计划](最终开发部署监控与验收计划_v1.0.md)仍规定 D01~D12、M1~M3/R1~R3、V01~V14及测试数值;本文是其审查整改补充,细化缺陷、依赖和复验方法,不降低原门禁。
- 审查发现分为代码缺陷、未实现、仅单元模拟、证据不足。静态发现先写失败回归/独立复现,再修根因;不能把静态推导表述为已发生的生产事故。
- 本轮仅在本地 Mock/隔离 SQLite 与契约测试范围内实施代码整改;计划不自动授权云消费、真实 SIP/AI 调用、生产迁移、删除/覆盖数据库、清空队列或资源回收。
- 全部执行指令和业务结果继续走 RabbitMQ;HTTP 仅按既有七条路径控制、查询、补传和存储握手。不增加业务回调、应用收讫或拨号 HTTP 接口。
- 保留现有可用 outbox、基础轮转和 ASR 代码,按缺陷补齐;不为整改引入通用工作流、供应商框架或无需求微服务。旧 voice_test 仅可复用 ASR,真实 LLM/TTS 必须等待新规范与明确授权。
## 2. 当前结论与验收纠偏
1. 已有 SQLite 持久状态、RabbitMQ 适配、HTTP、ASR Web、部署工具及 Mock 测试,但不能据此判定 D01~D11 全部完成。
2. SIP/ARI/RTP/AI journal 是单元测试替身,不是协议集成证据;合成静音 WAV 不是真实双方录音。
3. 历史 V01~V13 探针 PASS 保留为历史运行结果,不删除原证据;完整用例须按本计划复验,不能沿用脚本 PASS 签署 M3。
4. 历史 SCALE-MOCK 仅证明其声明的状态负载观察;缺持续媒体、独立双调度及正常生命周期补位证据,完整 V13 待验。
5. 备份文件存在不证明恢复/回滚成功,指标字符串存在不证明告警执行,broker confirm 不证明 SaaS 应用成功。
6. 真实 SIP 的 488/cause=88 排障另行推进;R1 真实闭环、R2 1000路完整 AI/N+1、R3 运维交接均不得由本计划的 Mock 结果替代。
## 3. 执行阶段与退出门禁
以下全部初始为待执行。优先级表示实施顺序;越权、双拨、丢事实、假终态/ready、串线和模式误标属于阻断问题,不因被排到后续阶段而允许上线。
| 阶段 | 修复包 | 前置与交付 | 退出条件 |
| --- | --- | --- | --- |
| F0 基线与止误报 | FIX-01、FIX-23 | 固定版本/profile;纠正模式装配与验收报告;建立缺陷用例清单 | 不支持的 real/mixed 失败关闭;未测不得输出完整 PASS |
| F1 业务正确性 | FIX-02~08、FIX-10~13 | F0;复合键迁移、真实测试 MQ、控制/额度/恢复、HTTP契约 | 安全、幂等、背压、双实例竞争与活动执行恢复用例通过;M1相关门禁复验 |
| F2 结果与资产可靠性 | FIX-09、FIX-16、FIX-18~20 | F1 的持久键与权限;先用真实测试字节隔离验证 | 补传、最终文字、上传续签/幂等/崩溃恢复通过;无重复资产或虚假ready |
| F3 协议语音闭环 | FIX-14~15、FIX-17 | F1准入/恢复,F2结果链;真实本地Asterisk及独立协议Mock | 双向媒体、AI流式/打断、实际混音录音贯通;M2相关门禁复验 |
| F4 运维与综合验收 | FIX-21~23及V01~V13 | 功能阶段通过;兼容迁移、故障环境、固定镜像 | 告警/恢复/回滚及完整模拟规模证据齐全后才签M3 |
| F5 真实替换 | 原D12 / R1~R3 | 供应商新规范、预算、账号、白名单及授权齐备 | 分项真实闭环、容量及交接签字,不自动创建资源或调用收费服务 |
允许独立模块并行,但数据库模型/迁移单一写入负责人;不得并行覆盖同一工作树。每包先落可失败的测试,再最小修复,再独立复核。工时和日历交期在失败复现及环境盘点后登记,不用静态审查臆造承诺。
## 4. 修复任务与逐项验收
### 4.1 模式、租户和传输
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
| --- | --- | --- | --- |
| FIX-01 `core.py:688–739` | 按实际组件装配校验模式;未实现的真实适配器明确拒绝;缺组件/未知枚举不得回退内存broker或Mock | 全部provider伪写real、缺provider、mixed启用未实现组件均启动失败且不接单;合法Mock正常运行;日志/指标逐组件模式与实际实例一致,真实配置不加载测试凭据/故障开关 | V01/V12 |
| FIX-02 `core.py:972–989` | 从实际消费队列及AMQP交付属性取得可信路由上下文,在受理前与正文/注册精确比对 | 经A路由发B正文、伪造header、错绑定均隔离,无业务拨号、无向错误租户发送结果;正确中文/点号/星号/井号key原值往返;224/225 UTF-8字节边界不截断,超限保留发布方原任务 | V01/V05 |
| FIX-03 `core.py:180–210、335–336、1079–1138` | commands/executions/inbox及相关查询/关联统一租户作用域;审计其他实体键;迁移保留现有事实 | A/B使用相同command/execution/event ID互不干扰;同租户同ID重投10次只保留原业务事实,同execution换command不新增拨号;异内容拒绝且原状态/call关联不变;跨租户查询/控制/资产/补传不可见;迁移前后计数、关联与屏障校验一致 | V01/V02 |
| FIX-04 `core.py:545–553` | 实际RabbitMQ配置拒绝发布溢出、消息数/字节上限、发布背压、可靠隔离;既有队列明确迁移而非删队列重建 | 真broker分别触发条数/字节满:新发布失败、原队头及已确认消息完整;mandatory return、NACK、confirm丢失、blocked及DLQ故障可追踪;原ID有限重试不双拨;恢复回原租户域。单节点功能与3节点quorum/可靠死信故障证据分列 | V05 |
### 4.2 调度、控制和恢复
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
| --- | --- | --- | --- |
| FIX-05 `core.py:2191–2205、2296–2339` | 同任务在途控制串行化,持久固定目标revision;applied必须有各Cell屏障证据 | pause尚未applied时resume返回409且不递增;同键重试返回原目标;pause/resume/stop与waiting、已提交意图、FALLBACK竞争,屏障后无新发起;失联保持applying/reconciling;stop不可resume,hangup额外鉴权且确认通道结束 | V04 |
| FIX-06 `core.py:1244–1271、1391–1508` | 原子核验调度所有权/隔离令牌、全部额度、授权/控制并预留;执行侧在真实发起前复核许可 | 两个独立进程共享DB,A在检查后停顿超过租期,B接管占最后额度,A恢复不得发起;在提交后/INVITE前注入暂停、到期、失权均不能越过许可;租户/供应商/Cell/AI及端口额度均不超限;FALLBACK计入CPS,未知占用不释放 | V03/V04/V07 |
| FIX-07 `core.py:962–1015` | 读取前按共享持久状态限制每租户/全局未ACK和待发起窗口;恢复也公平;扫描有界 | 资源耗尽持续投递,超窗消息留原队列而非全部ACK到DB;双实例总窗口不翻倍;重启/积压/新租户下达到第5节公平与截止目标,重投不延长deadline,不为公平挂断活动通话 | V03/V05 |
| FIX-08 `core.py:782–813、1568–1569` | 启动扫描未终结意图并对账;ARI断连隔离Cell且保留未知占用;查询/MQ保持契约状态 | 在意图提交后未发出、已发出未确认、接通后分别杀进程;重启无自动重拨;远端通话仍存活时断ARI不得写ended或释放额度;重连按真实通道收敛,确认结束后仅一次业务终态和资源释放;持久事实/outbox未丢失 | V02/V07/V10/V12 |
| FIX-09 `core.py:2380–2522` | 补传持久进度、有限重试与状态/outbox一致;恢复只处理未确认部分 | 第N条发布失败,断连/重启后原操作可恢复;耗尽则查询和MQ一致报告failed并告警,受控恢复不换截止点;已确认部分不丢,可能重投保留原event ID/内容;后生事件及补传自身结果不入范围;分批限速、不拨号、不调用AI/重传录音;completed不代表SaaS应用 | V10 |
### 4.3 HTTP、权限和机器契约
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
| --- | --- | --- | --- |
| FIX-10 `http.py:198、216` | 两条补传路径使用outbound.replay权限 | control-only拒绝;replay-only对授权资源成功;跨租户404;缺凭据401、缺权限403,不产生未授权业务事件 | V01/V10 |
| FIX-11 `http.py:234–248` | 两个存储POST校验并持久化调用方/租户/操作/幂等键及语义摘要 | 缺头拒绝;同键同内容重试/重启返回原操作;异内容409;不同租户/调用方按作用域独立;JSON字段排序不造成冲突,不因重试生成第二资产 | V01/V09 |
| FIX-12 `core.py:2279–2295、2388–2405` | 控制/补传幂等语义纳入操作类型与路径目标 | 对A已受理后,同键同body改为B目标返回409,不返回A成功冒充B;原目标重试稳定;控制与两种补传之间不错误互相命中 | V01/V04/V10 |
| FIX-13 `core.py:2248–2255、1911–1927`;两份OpenAPI | 修复ID正则转义、布尔当整数、reason上限、音频元数据等;解析执行Schema而非检查文本关键字 | 主契约合法样例含task-demo及字母s可通过;禁止字符/首尾空白按契约拒绝,不擅自限制tenant_key;true作为revision、超长reason、错误格式/声道/采样率/时长拒绝;7条路径请求/响应和MQ正反例、未知主版本、64KiB/256KiB边界均有测试;Schema与运行端一致 | V01 |
### 4.4 协议媒体、AI和文字
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
| --- | --- | --- | --- |
| FIX-14 `mocks.py:40–88`;`core.py:1508–1566` | journal降为单元替身;实现Cell长期ARI、SIP/RTP协议集成和真实媒体清理 | 隔离真实Asterisk+独立Mock SIP对端,至少两通并发不同标识音;双向收包/解码证明无串线,振铃/接通有协议证据;单线路无需虚构backup;正常结束30s内无本次通道/桥/端口残留;断连按FIX-08;明确未接通且原通道结束才允许合法FALLBACK,迟到接通/不确定时禁止重拨 | V06/V07 |
| FIX-15 `core.py:1571–1592`;`mocks.py:91–105` | 音频输入→分段ASR→新LLM/TTS流式Mock→PCM输出;真实ASR入口独立适配;VAD、取消、超时不靠自填成功 | 至少100有效轮次、合成输入与独立输出校验;首包超时、客户静音、插话、取消后旧包迟到均有明确结果;旧音频不进桥,超时不伪报正常完成;播放标记来自实际播放器能力,无确认只能sent/unknown;达到第5节时延;无旧LLM/TTS或收费外联 | V08 |
| FIX-16 `core.py:1671–1681` | 最终文字同内容幂等、异内容冲突,不允许迟到中间稿覆盖最终稿 | 同段最终稿重投10次不新增业务版本/事实;同段异内容拒绝并保留原文;乱序分段各自收敛,不用全局版本丢其他段;最终稿持久化/投递故障可恢复,无法完成明确transcript.failed,不假称完整 | V08/V10 |
### 4.5 录音与资产交接
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
| --- | --- | --- | --- |
| FIX-17 `core.py:1780–1784、1849–1858` | 接通后稳定recording_id,真实单轨混音;正常/强挂/故障确认结束均封口并排持久上传 | 双方不同标识音在录音中可辨,时长与实际音频样本/接通边界吻合,非全零占位;至少正常结束、强挂、媒体异常恢复三条路径;未接通不伪造接通录音;无法恢复资产显式failed;录音处理不阻塞call.finished | V06/V09/V11 |
| FIX-18 `core.py:1835–1845、2068–2090` | 经独立SaaS HTTP授权/complete和对象Mock传输真实字节,而非同对象内部调用冒充对接 | 使用独立服务/凭据与同一OpenAPI;服务端独立读字节验大小/摘要;缺对象、错绑定/摘要、覆盖均不ready;上传目标白名单、HTTPS/重定向与SSRF保护;确认后才持久verified+ready outbox;SaaS inbox后关联oss_id并授权读取 | V09 |
| FIX-19 `core.py:642–657、1991–2059` | complete幂等及文件移动/数据库提交跨边界恢复 | 成功响应丢失后原键重试返回同oss_id/verified_at;移动文件后提交前崩溃可通过最终文件重新校验恢复;重复确认不生成第二份资产/业务ready;异内容保持冲突;未verified不产生ready | V09/V10 |
| FIX-20 `core.py:1928–1934、1977–2006` | 原会话受控续签、上传持久恢复扫描、有限退避 | 跨300s有效期,旧授权拒绝,新授权保持upload/recording/object和内容绑定;中断5分钟后恢复,10分钟内补齐固定测试数据;重启不丢封口文件;耗尽明确failed/可恢复性并告警,文件不静默删除;签名/密钥不出现在日志或MQ | V09/V10/V11 |
### 4.6 运维、恢复和验收可信度
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
| --- | --- | --- | --- |
| FIX-21 `core.py:2669–2675`;`deploy/prometheus.yml`/`alerts.yml` | readiness反映持久写、可路由及必要配置/许可/磁盘;加载规则,修复模式/序列缺失告警 | DB可读不可写、缺绑定、ARI断连、凭据失效、磁盘水位、时钟跳变分别注入;不安全新发起停止,readiness和范围与故障一致,不无差别切断正常活动媒体;promtool语法/规则测试及实际告警产生→接收→恢复有证据;模式错误/指标缺失可检测 | V12 |
| FIX-22 `deploy/backup_sqlite.py`;`acceptance.py:371–397` | 恢复默认新目标,拒绝覆盖现有业务库;兼容迁移、排空/回滚、保留与水位保护 | 从备份恢复到新库实际启动并核对租户、执行、最终稿、outbox、录音关联;应用回滚保留较新DB/队列/卷,旧命令重投无重拨、stop屏障仍在;禁止用旧快照覆盖新防重事实;保留卷故障无已提交事实损失;永久磁盘丢失明确损失/failed/告警,不宣称录音RPO=0;未确认资产/屏障不按普通TTL删除 | V11/V12 |
| FIX-23 `acceptance.py:250–424`;`scale_mock.py` | 用例分层、独立断言、机器门禁与文档一致;完整V13重测 | journal/静音/备份存在仅能作为对应子项结果;缺协议/恢复/告警证据时完整项不PASS;实际失败或所需证据缺失返回非零;保留历史证据并追加纠偏;V13按第5节执行,状态对象数量不得当媒体并发;真实V14始终独立 | V01~V14 |
上表 `core.py`、`http.py`、`mocks.py` 均位于 `agent_call/`,`acceptance.py`、`scale_mock.py` 位于 `scripts/`。总计23项,ARI断连相关发现合并到FIX-08,避免重复计算完成率。
## 5. 测试环境、基线与独立证据
### 5.1 分层,不互相冒充
| 层级 | 实际依赖 | 能证明什么 |
| --- | --- | --- |
| L0 单元/契约 | 临时真实DB、可控时钟/故障钩子、内存替身 | 分支和语义正确;不能证明AMQP、ARI或媒体协议 |
| L1 持久/MQ/HTTP集成 | 真实隔离DB、RabbitMQ、独立SaaS/对象Mock、两个调度进程 | 事务、隔离、权限、竞争、背压和重启恢复;不等于真实供应商 |
| L2 协议媒体集成 | 真实隔离Asterisk、独立SIP/RTP对端、流式AI Mock | 实际通道/双向音频/取消/录音/结果闭环;不是生产AI质量 |
| L3 综合/规模 | L1+L2、故障/监控组件、SCALE-MOCK profile | M3/V13适用门禁;报告实际物理资源和模拟部分 |
| L4 真实 | 获授权SaaS/SIP/ASR/LLM/TTS/OSS与云出口 | R1~R3,单独取得供应商、容量和运维证据 |
数据库先复用当前SQLite真实持久能力做回归;必须用独立进程验证其实际协调范围,不以同进程锁代替多实例证据。SQLite同宿主测试不证明多机共享数据库方案;生产数据库与部署拓扑确定后,迁移/并发/故障用例必须在目标引擎重跑。若为跨主机能力需要更换引擎,登记明确迁移方案,不以网络共享SQLite文件规避验证。
L1起使用真实测试broker;HA/可靠死信另建隔离3节点quorum环境。Mock网络默认阻止真实公网SIP/AI/云API;只使用合成音频和授权测试数据,不以“假号码”替代网络隔离。
### 5.2 沿用最终计划的工程测试阈值
下列为DEV/SCALE-MOCK工程基线,不是生产SLA或消费授权。完整取值及恢复/保留条件以最终计划第5节为准;修改profile须版本化并重跑相关门禁。
- DEV:3租户、2独立调度实例、2Cell;每租户并发2/CPS1,全局并发6/CPS3;每Cell测试容量4。
- 所有实例汇总:每租户未ACK≤4/持久待发起≤16,全局≤32/≤64;每轮每租户最多1个新许可。队列上限1000条或16MiB先到者,每租户发布≤10条/s。
- 公平:发现新活跃租户≤1s;持续可调度且资源足够时B/C发现后≤2s获首次许可;至少100次许可中3租户份额偏差≤10个百分点。资源不足单列,不以强挂达标。
- 首次准入窗口30s,到期处理≤1s;重启/重投不延期。心跳2s、租约10s,失权即停止本地新发起;时钟偏差>500ms告警、>2s停止新准入。
- 控制受理到全部屏障applied P95≤2s,前提为节点健康且无未决发起;失联样本不得伪算成功,单列未决。
- ≥100有效AI轮次:VAD语音结束到首有效TTS入桥P95≤1500ms,插话到停止旧TTS入桥P95≤500ms;首输出超时5s、客户静音15s;实际Mock延迟和测量时间点单列。
- 最终稿到SaaS Mock事务应用P95≤3s;≤180s通话挂断到对象verified、ready消费及授权读取≤120s;正常结束30s内清理本次媒体资源。
- 消费/上传中断5分钟,恢复后10分钟内补齐固定负载;DB/MQ恢复后60s内恢复安全调度,未知通话仍保留对账占用。
- 测试卷使用率70%告警、80%停止新接单,降至60%且依赖正常后恢复;预留活动录音剩余空间。补传每批≤100事件、全局≤50事件/s,实时结果优先。
- V13:至少2调度实例、100租户×并发12、全局1200,拨号CPS20;预热后≥1000模拟接通连续60分钟,通过≤180s正常生命周期和新授权持续补位,不延长单批通话伪造持续负载。明确模拟Cell/AI额度,提供持续媒体与数据证据。运行中故障隔离一个Cell、旧活动不自动接管/重拨,在健康容量内用新授权恢复负载,再记录恢复稳态;故障影响窗口单列,不隐藏后算连续达标。
- R2仍要求≥1000同时已接通的真实完整ASR/LLM/TTS通话稳定≥60分钟,以及实测N+1安全容量、真实CPS/RTP/AI/MQ/OSS证据。V13不能替代。
### 5.3 证据独立性与并发方法
- 每个FIX至少一个修复前失败、修复后通过的可运行用例;保存命令、退出码和原始结果。不可仅检查日志里存在由被测逻辑自行填入的成功字段。
- 并发故障使用屏障/故障钩子稳定卡在事务、确认、发起、文件移动等边界;独立进程竞争,不用顺序调用两实例代替争抢。
- MQ证据取broker确认/队列/投递及DB事务;媒体证据取对端收包、解码样本与ARI通道;资产证据独立读上传文件、核摘要与授权读取;应用证据取SaaS inbox/业务库。
- 至少覆盖DB提交前后、ACK前后、发起前后、confirm前后、文件移动与verified提交之间的中断。所有不确定结果保留原ID和事实,不靠重拨、删队列或删库消除。
- 故障测试固定重复3轮;每轮重建独立测试命名空间并记录种子,不覆盖生产。一次偶发失败即该项失败,不挑成功轮次签字。3轮是本整改测试重复次数,不是可靠性概率保证。
## 6. 执行记录、交付与签署
每个FIX的执行记录按下表追加到 `docs/evidence/`。此处仅定义未来产物,不表示已有文件或测试命令。
| 字段 | 必填内容 |
| --- | --- |
| 身份 | FIX编号、关联V编号、代码SHA、契约/profile版本、镜像digest、DB/broker版本 |
| 责任 | 实施人、独立复核人、运维协作人;启动前填写真实负责人,未填不视为已分派 |
| 前置 | L0~L4层级、实际provider清单、网络隔离、配额、种子、数据/时间范围 |
| 执行 | 可重复命令/退出码、输入、预期、实际、故障时刻/恢复时刻、重复轮次 |
| 证据 | 脱敏日志、DB/inbox/outbox核对、broker/协议记录、文件摘要、指标样本分母和分位数、告警接收/恢复 |
| 结论 | 待执行/通过/失败/阻塞/不适用;不适用必须给契约依据与批准,不得用来豁免缺失环境 |
| 剩余项 | 未决记录数、外部阻塞、受影响门禁、恢复措施、下一责任人 |
交付包:最小修复代码及迁移、逐FIX测试、版本化隔离profile、部署/恢复/告警运行步骤、V01~V13复验矩阵、历史证据纠偏说明、R阶段阻塞清单。新增执行命令必须先实现并验证,再写入运行说明。
现有回归入口(执行时仍须审阅环境,不能视为本次已运行):
```bash
python3 -m unittest discover -s tests -v
(cd services/asr-web && go test -race ./... && go vet ./...)
node --test tests/test_pcm.cjs
bash -n deploy/asterisk.sh
git diff --check
```
以上回归不替代L1~L4故障/协议/规模用例;增加的集成测试需提供其真实可运行入口。告警规则用promtool做语法和规则测试,并仍须验证实际接收链路。
### 最终通过条件
1. FIX-01~23逐项完成复现、修复、独立复核,所有阻断项关闭;共享数据库/队列配置迁移有无损证据,不覆盖用户既有修改。
2. M1/M2/M3按原定义分别签署;V01~V13完整范围具备独立证据,缺环境或缺规模则保持对应门禁未完成,允许声明已通过的子集。
3. 无未经授权双拨、串线、跨租户污染、假终态/播放确认/ready或静默丢事实;出现任一项立即停止扩量,保留现场,修复后重跑相关门禁。
4. 模式和依赖按实际装配展示;文档/机器报告/实现一致,不将Mock成功描述为真实供应商启用。
5. R1~R3单独验收。SIP供应商、真实LLM/TTS规范、OSS/身份/预算、生产保留与RPO/RTO未落实时,登记外部阻塞,不擅自补参数、不把未验项改为通过。
## 7. 本轮执行记录与剩余门禁
已按本文的代码级要求实施最小整改:
- `agent_call/core.py`:模式/依赖拒绝、可信租户路由、租户复合键迁移、发布保留与重试、调度租约/额度、控制Cell屏障、重启对账态、补传进度、格式校验、最终稿幂等、稳定录音与上传恢复、readiness/metrics。
- `agent_call/http.py`:补传专用权限、调用方/租户/操作/幂等键持久化、请求体与上传边界校验;`agent_call/mocks.py`:非静音合成PCM及协议事实journal;`deploy/backup_sqlite.py`、`deploy/prometheus.yml`、`deploy/alerts.yml`:非覆盖恢复与规则加载/水位告警。
- `docs/contracts/*.yaml/json` 与 Mock profile 已同步修正;新增 `tests/test_remediation_plan.py` 及 HTTP replay 权限回归。
- 已运行 `python3 -m unittest discover -s tests -v`(52项通过)、Python编译、JSON/YAML解析、`ruff check`、`git diff --check`;`scripts/acceptance.py` 的 V01~V12 通过,V13 因缺持续媒体独立证据保持 `NOT_RUN`,总门禁为 `INCOMPLETE`。
- 2026-09-13 在明确授权后重新创建并核验单路真实 Cell:ECS `i-2ze4hvi6xvy02g0wz9kk`、固定 EIP `123.56.71.98`、Asterisk `22.10.1`、PJSIP contact `Avail`;对 `18601013734` 单次构造 `708918601013734`,收到 `100 trying` 后 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`,无接通/RTP/录音证据,详见 `docs/evidence/asterisk-persistent-2026-09-13.md`。
上述结果只证明本地代码/契约层子集及真实单路底座/失败回执,不替代第5节的L1~L4证据。真实接通/RTP、ASR/新LLM/TTS、独立SaaS/OSS、3节点可靠MQ、实际告警收发、V13及R1~R3完整验收仍未完成,不得签署M2/M3或生产ready。
@@ -9,7 +9,7 @@
- 本文统一规定开发顺序、Mock 策略、运行基线、部署、监控和验收门禁;取代旧计划中的阶段安排及“需再次拍板方案”的表述。 - 本文统一规定开发顺序、Mock 策略、运行基线、部署、监控和验收门禁;取代旧计划中的阶段安排及“需再次拍板方案”的表述。
- [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)是标识、HTTP 七条路径、MQ 拓扑/字段/状态的唯一维护来源。本文件不复制接口表;先生成 OpenAPI/JSON Schema,再编写 Mock 和正式实现。 - [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)是标识、HTTP 七条路径、MQ 拓扑/字段/状态的唯一维护来源。本文件不复制接口表;先生成 OpenAPI/JSON Schema,再编写 Mock 和正式实现。
- [原一期计划](一期呼出应用开发计划_v1.0.md)保留 WBS/AT 编号和历史估算;[交付设计](交付文档/一期中间调度件与Asterisk_v1.0/01_中间调度件与MQ回传设计.md)、[部署细则](交付文档/一期中间调度件与Asterisk_v1.0/02_Asterisk部署与SIP对接步骤.md)、[专项用例](交付文档/一期中间调度件与Asterisk_v1.0/03_联调步骤与验收标准.md)保留实现与场景细节,冲突时以本文件及主契约为准。 - [原一期计划](一期呼出应用开发计划_v1.0.md)保留 WBS/AT 编号和历史估算;[交付设计](交付文档/一期中间调度件与Asterisk_v1.0/01_中间调度件与MQ回传设计.md)、[部署细则](交付文档/一期中间调度件与Asterisk_v1.0/02_Asterisk部署与SIP对接步骤.md)、[专项用例](交付文档/一期中间调度件与Asterisk_v1.0/03_联调步骤与验收标准.md)保留实现与场景细节,冲突时以本文件及主契约为准。
- 当前已有 ASR Web、ASR 协议适配、Asterisk 配置生成、阿里云只读计划/受控创建工具和离线测试;参见[现有运行说明](部署接入_运行说明.md)。当前尚无完整调度、AI 通话、MQ/OSS 闭环或本计划的 Mock 套件,不能使用未来产物的命令假装已可运行。 - 当前已有 ASR Web、ASR 协议适配、Asterisk 配置生成、阿里云只读计划/受控创建工具,以及执行服务、持久状态、MQ/HTTP 和 Mock 测试产物;参见[现有运行说明](部署接入_运行说明.md)。2026-09-13静态审查发现隔离、控制、恢复、媒体与验收缺口,完整调度/AI通话/MQ与资产交接仍须整改复验,不能把已有产物等同于完整门禁通过。整改执行见[功能审查整改计划与验收标准](功能审查整改计划与验收标准_v1.0.md)。
- 本次只交付文档,不新增运行服务、不拨号、不创建云资源、不提交 Git。旧 Word/Excel/ZIP 不覆盖重生成。 - 本次只交付文档,不新增运行服务、不拨号、不创建云资源、不提交 Git。旧 Word/Excel/ZIP 不覆盖重生成。
## 2. 已接受的最终方案 ## 2. 已接受的最终方案
@@ -289,7 +289,7 @@ M3产物 + C1~C6真实配置/权限(受控存放)+ R1/R2全部证据 + 实
## 10. 本次文档验收与后续入口 ## 10. 本次文档验收与后续入口
本轮已完成文档引用、契约/实现一致性、样例完整性、Mock开发回归及V01~V13验证;V14/R1/R2/R3仍因真实外部证据待执行。当前回归命令可继续使用,但不能把Mock结果当作生产验收: 2026-09-13审查纠偏:历史脚本的V01~V13 Mock探针PASS只证明其实际覆盖的子项,不能证明完整契约/实现一致性、协议媒体、恢复、监控及模拟规模门禁通过。M1/M2/M3须按[整改计划](功能审查整改计划与验收标准_v1.0.md)逐项修复并复验;V14/R1/R2/R3另需真实外部证据。本次静态审查和文档更新未运行下列回归,现有命令可继续使用,但不能替代集成/故障验收:
```bash ```bash
python3 -m unittest discover -s tests -v python3 -m unittest discover -s tests -v
@@ -298,12 +298,18 @@ node --test tests/test_pcm.cjs
bash -n deploy/asterisk.sh bash -n deploy/asterisk.sh
``` ```
下一实施入口是 D12/R1:在取得真实SaaS、SIP/Asterisk、ASR/LLM/TTS、OSS、出口、容量和运维证据后逐项替换Mock并复验;无需再次讨论已接受的A类方案。遇到真实供应商协议/合规/资源事实不兼容,只对具体差异登记变更,不把整个计划退回重选,也不让Mock假成功掩盖差异。 下一实施入口是整改计划F0→F1~F4,对D01~D11补齐模式保护、租户隔离、可靠性、协议媒体与验收证据;真实SIP排障及D12外部协调并行,不以线路不通阻塞本地整改。真实替换仍须取得SaaS、SIP/Asterisk、ASR/LLM/TTS、OSS、出口、容量和运维证据;无需重新讨论已接受方案。具体供应商差异单独登记,不让Mock假成功掩盖缺口。
### 10.1 当前执行记录(2026-09-12) ### 10.1 历史执行记录(2026-09-12~13,非当前完整验收结论)
以下保留历史探针及当时环境记录;其中V01~V13、规模与环境保持运行的表述仅描述当次结果,不表示当前资源仍存在。完整门禁状态以第10节纠偏和整改复验为准,当前云资源状态须另行只读核验,不能据本记录自动复用、创建或拨号。
已落地 D01 的 OpenAPI 3.1、共用 MQ Schema、集中样例和 Mock profile,以及 D02/D03 的 SQLite 持久状态、租户独立 RabbitMQ 拓扑、outbox/inbox、CAS 控制、重放、Mock SIP/ARI/RTP/AI/OSS 协议记录、监控/告警、迁移/备份和 Compose 部署入口。测试号码 `18601013734` 在 Mock 与明确授权的 R1 探针中均保持原值;当前线路目标按规则生成 `708918601013734`。 已落地 D01 的 OpenAPI 3.1、共用 MQ Schema、集中样例和 Mock profile,以及 D02/D03 的 SQLite 持久状态、租户独立 RabbitMQ 拓扑、outbox/inbox、CAS 控制、重放、Mock SIP/ARI/RTP/AI/OSS 协议记录、监控/告警、迁移/备份和 Compose 部署入口。测试号码 `18601013734` 在 Mock 与明确授权的 R1 探针中均保持原值;当前线路目标按规则生成 `708918601013734`。
本轮可重复命令 `python3 -m unittest discover -s tests -v` 和 RabbitMQ Compose smoke 已通过;`python3 -m scripts.acceptance` 的 V01~V13 Mock 探针通过,因 V14 仍需外部真实证据而按门禁返回非零,证据归档于 `docs/evidence/acceptance-mock-2026-09-12.json`。SCALE-MOCK 在阿里云北京专用竞价 Cell `ecs.hfg6.large` 上完成:100 租户、1200 模拟呼叫、禁用1个Cell,最低1004路已回答并连续观察3600秒;详见 `docs/evidence/scale-mock-2026-09-12.json`。该证据仅证明 Mock 协议/持久状态负载基线,`continuous_media_evidence` 为 false,不替代真实媒体容量。 本轮可重复命令 `python3 -m unittest discover -s tests -v` 和 RabbitMQ Compose smoke 已通过;`python3 -m scripts.acceptance` 的 V01~V13 Mock 探针通过,因 V14 仍需外部真实证据而按门禁返回非零,证据归档于 `docs/evidence/acceptance-mock-2026-09-12.json`。SCALE-MOCK 在阿里云北京专用竞价 Cell `ecs.hfg6.large` 上完成:100 租户、1200 模拟呼叫、禁用1个Cell,最低1004路已回答并连续观察3600秒;详见 `docs/evidence/scale-mock-2026-09-12.json`。该证据仅证明 Mock 协议/持久状态负载基线,`continuous_media_evidence` 为 false,不替代真实媒体容量。
R1 已在持久测试环境完成两次真实 Asterisk/SIP 单路探针:分别使用原始被叫 `18601013734` 和 `18625770806`,均按 `7089` 前缀构造目标。环境 ECS `i-2zegaldd07mkp0xyhcff` 使用 `SpotAsPriceGo` 自动竞价和固定 EIP `123.56.71.98`;PJSIP contact/OPTIONS 可达,真实 INVITE 已送达供应商,两次均收到 `488`、`Q.850 cause=88 INCOMPATIBLE_DESTINATION`,未接通、未建立 RTP,证据见 `docs/evidence/asterisk-persistent-2026-09-13.md`。环境按用户要求保持运行,未删除实例/EIP。真实接通/RTP、ASR/LLM/TTS/OSS、生产 1000 路完整 AI、N+1、实际CPS/SLO及副作用正确性仍阻塞,现有产物不得据此宣称生产完成。 R1 已在历史持久测试环境完成两次真实 Asterisk/SIP 单路探针:分别使用原始被叫 `18601013734` 和 `18625770806`,均按 `7089` 前缀构造目标。历史环境 ECS `i-2zegaldd07mkp0xyhcff` 使用 `SpotAsPriceGo` 自动竞价和固定 EIP `123.56.71.98`;PJSIP contact/OPTIONS 可达,真实 INVITE 已送达供应商,两次均收到 `488`、`Q.850 cause=88 INCOMPATIBLE_DESTINATION`,未接通、未建立 RTP。
补充当前执行(2026-09-13):按明确授权重新创建并绑定 ECS `i-2ze4hvi6xvy02g0wz9kk`,探针期间固定 EIP 为 `InUse`;Asterisk `22.10.1` 容器为 `running/healthy`,固定镜像 digest、PJSIP contact `Avail`(RTT约22.164ms)及安全组范围已只读核验。对原始被叫 `18601013734` 单次探针构造 `708918601013734`,收到 `100 trying` 后 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`,无 `180/183/200`、RTP 或录音证据;无重试、无 FALLBACK。探针后按用户授权停止并删除该 ECS,系统盘随实例删除,固定 EIP 未释放且已恢复 `Available`;详见 `docs/evidence/asterisk-persistent-2026-09-13.md`。
真实接通/RTP、ASR/LLM/TTS/OSS、生产 1000 路完整 AI、N+1、实际CPS/SLO及副作用正确性仍阻塞,现有产物不得据此宣称生产完成。
+3 -3
View File
@@ -238,14 +238,14 @@ python3 deploy/migrate.py data/agent-call.sqlite3
### 7.3 当前门禁 ### 7.3 当前门禁
本地V01-V12及阿里云专用主机上的V13 Mock探针已通过;完整验收脚本仍因V14需要外部真实证据而按门禁返回非零,证据见`docs/evidence/acceptance-mock-2026-09-12.json`和`docs/evidence/scale-mock-2026-09-12.json`。V13使用100租户/1200模拟接通、禁用1个Cell,最低1004路已回答并持续观察3600秒;这是Mock协议/持久状态负载证据,不是RTP或生产容量证据。R1临时探针见`docs/evidence/asterisk-real-2026-09-12.md`,持久环境两号码探针见`docs/evidence/asterisk-persistent-2026-09-13.md`:固定EIP和SIP OPTIONS/INVITE链路可达,但两个目标均收到488/Q.850 cause=88。当前持久测试ECS保持运行,使用SpotAsPriceGo,固定EIP`123.56.71.98`保持InUse。真实接通/RTP、ASR/LLM/TTS/OSS、生产1000路/N+1尚未完成。正式模式拒绝当前Mock profile,不能把测试号码或Mock成功自动升级为真实外呼授权。 本地V01-V12及阿里云专用主机上的V13 Mock探针已通过;最新完整验收结果见`docs/evidence/acceptance-mock-2026-09-13.json`,脚本仍因V13缺持续媒体独立证据和V14需要外部真实证据而按门禁返回非零。V13使用100租户/1200模拟接通、禁用1个Cell;这是Mock协议/持久状态负载证据,不是RTP或生产容量证据。R1历史探针见`docs/evidence/asterisk-real-2026-09-12.md`,当前单次探针见`docs/evidence/asterisk-persistent-2026-09-13.md`:ECS `i-2ze4hvi6xvy02g0wz9kk`、Asterisk容器和固定EIP `123.56.71.98` 已核验,PJSIP contact/OPTIONS可达,但 `18601013734` 对应目标收到 `488/Q.850 cause=88`,未接通、无RTP/录音证据。探针后该 ECS 已按授权停止并删除,固定 EIP 保持 Available。真实接通/RTP、ASR/LLM/TTS/OSS、生产1000路/N+1尚未完成。正式模式拒绝当前Mock profile,不能把测试号码或Mock成功自动升级为真实外呼授权。
## 8. 下一步需要用户提供/确认 ## 8. 下一步需要用户提供/确认
1. 供应商需确认`7089`前缀/目标号码路由、From域(当前实际出现私网域)及允许的编解码;两个持久环境探针均返回`488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`,未通过外呼验收。 1. 供应商需确认`7089`前缀/目标号码路由、From域(当前实际出现私网域)及允许的编解码;两个持久环境探针均返回`488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`,未通过外呼验收。
2. 供应商调整后在当前持久测试环境复测;环境使用SpotAsPriceGo,竞价回收可能中断通话,不自动重试、不改绑或释放固定EIP。 2. 供应商调整并确认线路规则后,再用只读plan和新授权创建测试实例复测;本次实例已回收,固定EIP保持 Available,不自动重试、不改绑或释放固定EIP。
3. ASR测试凭据、批准的模型/资源ID;Web HTTPS域名/证书或SSH访问方案。 3. ASR测试凭据、批准的模型/资源ID;Web HTTPS域名/证书或SSH访问方案。
4. 用户制定的MQ/OSS ID接口规范,以及新的LLM/TTS协议、参数与取消/打断规则。 4. 用户制定的MQ/OSS ID接口规范,以及新的LLM/TTS协议、参数与取消/打断规则。
5. 真实接通后仍需分别完成RTP、录音、MQ/OSS、ASR/LLM/TTS和生产1000路/N+1验收。 5. 真实接通后仍需分别完成RTP、录音、MQ/OSS、ASR/LLM/TTS和生产1000路/N+1验收。
在供应商规则确认前,不再次拨打真实号码,也不声称完成整个平台;当前持久测试环境按用户要求保留。 在供应商规则确认前,不再次拨打真实号码,也不声称完成整个平台;本次测试实例已回收,固定EIP按要求保留。
+84 -8
View File
@@ -7,12 +7,15 @@ exists. Passing Mock probes never upgrades the result to production readiness.
from __future__ import annotations from __future__ import annotations
import copy import copy
import io
import json import json
import tempfile import tempfile
import time import time
from collections.abc import Callable from collections.abc import Callable
from contextlib import redirect_stdout
from pathlib import Path from pathlib import Path
from typing import Any from types import TracebackType
from typing import Any, Self
from agent_call.core import ( from agent_call.core import (
AgentCallService, AgentCallService,
@@ -221,11 +224,14 @@ def probe_v04() -> dict[str, Any]:
class ConflictContext: class ConflictContext:
def __enter__(self) -> ConflictContext: def __enter__(self) -> Self:
return self return self
def __exit__( def __exit__(
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: Any self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> bool: ) -> bool:
if exc_type is not ConflictError: if exc_type is not ConflictError:
raise AssertionError("expected revision conflict") raise AssertionError("expected revision conflict")
@@ -256,10 +262,37 @@ def probe_v06_v07() -> dict[str, Any]:
"disconnect": "ari_disconnect", "disconnect": "ari_disconnect",
} }
outcomes: dict[str, str] = {} outcomes: dict[str, str] = {}
uncertain_before_reconcile: str | None = None
for label, scenario in scenarios.items(): for label, scenario in scenarios.items():
body = command(f"cmd_{label}") body = command(f"cmd_{label}")
svc.set_failure(body["command_id"], scenario) svc.set_failure(body["command_id"], scenario)
snapshot = run_success(svc, body) svc.publish_execute(body)
if scenario == "ari_disconnect":
call_id = None
before = None
for _ in range(500):
svc.process_once()
current = svc.get_command("tenant-demo", body["command_id"])
call_id = current.get("call_id")
if call_id:
before = svc.get_call("tenant-demo", call_id)
if before["call_state"] == "reconciling":
break
time.sleep(0.01)
if not call_id or before is None:
raise AssertionError("disconnect probe did not create a call")
uncertain_before_reconcile = before["call_state"]
if uncertain_before_reconcile != "reconciling":
raise AssertionError(
"disconnect probe did not retain unknown state"
)
svc.reconcile_call(
call_id, ended=True, outcome="failed", reason="ARI_RECONCILED"
)
snapshot = svc.get_command("tenant-demo", body["command_id"])
else:
svc.wait_for_idle()
snapshot = svc.get_command("tenant-demo", body["command_id"])
call = svc.get_call("tenant-demo", snapshot["call_id"]) call = svc.get_call("tenant-demo", snapshot["call_id"])
outcomes[label] = call["outcome"] outcomes[label] = call["outcome"]
if len(call["attempts"]) != 1: if len(call["attempts"]) != 1:
@@ -280,6 +313,7 @@ def probe_v06_v07() -> dict[str, Any]:
"asterisk_real": "not_run", "asterisk_real": "not_run",
"outcomes": outcomes, "outcomes": outcomes,
"fallback_redials": 0, "fallback_redials": 0,
"disconnect_state_before_reconcile": uncertain_before_reconcile,
} }
@@ -374,9 +408,37 @@ def probe_v11_v12() -> dict[str, Any]:
body = command("cmd_backup") body = command("cmd_backup")
run_success(svc, body) run_success(svc, body)
backup = Path(directory) / "backup.sqlite3" backup = Path(directory) / "backup.sqlite3"
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup)) with redirect_stdout(io.StringIO()):
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup))
if not backup.is_file(): if not backup.is_file():
raise AssertionError("backup file was not created") raise AssertionError("backup file was not created")
restored = AgentCallService(
db_path=backup,
object_dir=Path(directory) / "restored-objects",
broker=InMemoryBroker(),
)
try:
restored_command = restored.get_command("tenant-demo", body["command_id"])
restored_call = restored.get_call(
"tenant-demo", restored_command["call_id"]
)
if (
restored_command["status"] != "completed"
or restored_call["recordings"][0]["status"] != "verified"
):
raise AssertionError("restored database lost the completed call facts")
restored_counts = {
"commands": restored.store.one("SELECT COUNT(*) AS n FROM commands")[
"n"
],
"events": restored.store.one("SELECT COUNT(*) AS n FROM events")["n"],
"inbox": restored.store.one("SELECT COUNT(*) AS n FROM inbox")["n"],
"recordings": restored.store.one(
"SELECT COUNT(*) AS n FROM recordings"
)["n"],
}
finally:
restored.stop()
metrics = svc.metrics() metrics = svc.metrics()
if "agent_call_outbox_pending 0" not in metrics or 'mode="mock"' not in metrics: if "agent_call_outbox_pending 0" not in metrics or 'mode="mock"' not in metrics:
raise AssertionError("monitoring metrics lack healthy mock state") raise AssertionError("monitoring metrics lack healthy mock state")
@@ -392,7 +454,9 @@ def probe_v11_v12() -> dict[str, Any]:
return { return {
"backup": "verified", "backup": "verified",
"metrics": "healthy", "metrics": "healthy",
"rollback_preserves_db": True, "rollback_preserves_db": "not_run",
"restored_db_facts": True,
"restored_counts": restored_counts,
"real_mode_mock_rejection": True, "real_mode_mock_rejection": True,
} }
@@ -419,10 +483,15 @@ def probe_v13() -> dict[str, Any]:
if ( if (
evidence.get("status") != "observed" evidence.get("status") != "observed"
or evidence.get("production_claim", True) or evidence.get("production_claim", True)
or not evidence.get("continuous_media_evidence", False)
or load.get("minimum_answered_calls", 0) < 1000 or load.get("minimum_answered_calls", 0) < 1000
or load.get("observed_hold_seconds", 0) < 60 * 60 or load.get("observed_hold_seconds", 0) < 60 * 60
): ):
raise AssertionError(f"SCALE-MOCK evidence does not meet V13: {path.name}") return {
"status": "not_run",
"evidence_file": str(path.relative_to(ROOT)),
"reason": "historical state-only evidence cannot satisfy continuous media V13",
}
return { return {
"status": "observed", "status": "observed",
"evidence_file": str(path.relative_to(ROOT)), "evidence_file": str(path.relative_to(ROOT)),
@@ -466,7 +535,14 @@ def run() -> dict[str, Any]:
"duration_ms": round((time.monotonic() - started) * 1000, 3), "duration_ms": round((time.monotonic() - started) * 1000, 3),
"evidence": evidence, "evidence": evidence,
} }
except Exception as exc: except (
AssertionError,
KeyError,
OSError,
RuntimeError,
ServiceError,
ValueError,
) as exc:
results["probes"][name] = { results["probes"][name] = {
"status": "FAIL", "status": "FAIL",
"duration_ms": round((time.monotonic() - started) * 1000, 3), "duration_ms": round((time.monotonic() - started) * 1000, 3),
+31 -10
View File
@@ -14,7 +14,13 @@ import time
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from agent_call.core import AgentCallService, InMemoryBroker, iso, utcnow from agent_call.core import (
AgentCallService,
InMemoryBroker,
ServiceError,
iso,
utcnow,
)
from scripts.acceptance import load_fixture from scripts.acceptance import load_fixture
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -44,6 +50,8 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
{ {
"cell_id": f"cell-{index:03d}", "cell_id": f"cell-{index:03d}",
"capacity": 12, "capacity": 12,
"media_capacity": 12,
"ai_capacity": 12,
"egress_pool_id": "egress-mock", "egress_pool_id": "egress-mock",
"ari_mode": "mock", "ari_mode": "mock",
} }
@@ -53,10 +61,14 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
profile["limits"].update( profile["limits"].update(
{ {
"global_concurrency": 1200, "global_concurrency": 1200,
"global_cps": 1200, "global_cps": 20,
"tenant_concurrency": 12, "tenant_concurrency": 12,
"tenant_cps": 12, "tenant_cps": 1,
"max_queue_messages": 2000, "max_queue_messages": 1000,
"max_queue_bytes": 16 * 1024 * 1024,
"pending_window_per_tenant": 16,
"pending_window_global": 64,
"max_unacked_per_tenant": 4,
# Keep calls alive through warm-up and the required observation window. # Keep calls alive through warm-up and the required observation window.
"hold_ms": (startup_window_seconds + duration_minutes * 60 + 5) * 1000, "hold_ms": (startup_window_seconds + duration_minutes * 60 + 5) * 1000,
} }
@@ -101,6 +113,9 @@ def dry_run(duration_minutes: int) -> dict[str, Any]:
def cleanup_load(svc: AgentCallService) -> dict[str, Any]: def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
# The failed-cell probe is restored before a full control barrier is applied;
# a barrier must not claim every cell applied while one cell is unreachable.
svc.set_cell_health("cell-000", True)
requested = 0 requested = 0
for tenant_index in range(100): for tenant_index in range(100):
tenant_id = f"tenant-{tenant_index:03d}" tenant_id = f"tenant-{tenant_index:03d}"
@@ -162,7 +177,7 @@ def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
def run(directory: Path, duration_minutes: int) -> dict[str, Any]: def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
profile_path = directory / "profile.json" profile_path = directory / "profile.json"
build_profile(profile_path, duration_minutes) build_profile(profile_path, duration_minutes)
broker = InMemoryBroker(max_messages=2000) broker = InMemoryBroker(max_messages=1000, max_bytes=16 * 1024 * 1024)
svc = AgentCallService( svc = AgentCallService(
db_path=directory / "scale.sqlite3", db_path=directory / "scale.sqlite3",
profile_path=profile_path, profile_path=profile_path,
@@ -174,11 +189,17 @@ def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
template = load_fixture() template = load_fixture()
started = time.monotonic() started = time.monotonic()
try: try:
for tenant_index in range(100): for offset in range(12):
for offset in range(12): for tenant_index in range(100):
svc.publish_execute( body = command_for(template, tenant_index * 12 + offset, tenant_index)
command_for(template, tenant_index * 12 + offset, tenant_index) while True:
) try:
svc.publish_execute(body)
break
except ServiceError as exc:
if exc.code != "PUBLISH_RATE_LIMIT":
raise
time.sleep(1.05)
warmup_deadline = started + max(120, duration_minutes * 60) warmup_deadline = started + max(120, duration_minutes * 60)
active = 0 active = 0
threshold_at = None threshold_at = None
+48
View File
@@ -4,6 +4,7 @@ import copy
import json import json
import tempfile import tempfile
import unittest import unittest
from datetime import timedelta
from pathlib import Path from pathlib import Path
from agent_call.core import ( from agent_call.core import (
@@ -104,6 +105,53 @@ class AgentCallTests(unittest.TestCase):
attempts = self.service.store.one("SELECT COUNT(*) AS n FROM attempts")["n"] attempts = self.service.store.one("SELECT COUNT(*) AS n FROM attempts")["n"]
self.assertEqual(calls, 1) self.assertEqual(calls, 1)
self.assertEqual(attempts, 1) self.assertEqual(attempts, 1)
self.assertEqual(
self.service.store.one(
"SELECT status FROM executions WHERE tenant_id=? AND execution_id=?",
("tenant-demo", "exec_once"),
)["status"],
"completed",
)
def test_unhealthy_cell_isolated_without_blocking_healthy_capacity(self) -> None:
self.service.set_cell_health("cell-a", False)
self.assertTrue(self.service.ready())
snapshot = self.run_command(self.command("cmd_healthy_cell"))
call = self.service.get_call("tenant-demo", snapshot["call_id"])
self.assertEqual(call["call_state"], "ended")
self.assertEqual(
self.service.store.one(
"SELECT cell_id FROM calls WHERE call_id=?", (snapshot["call_id"],)
)["cell_id"],
"cell-b",
)
def test_missing_tenant_route_closes_readiness(self) -> None:
self.service._unrouteable_tenants.add("tenant-demo")
snapshot = self.service.readiness()
self.assertFalse(snapshot["checks"]["routes"])
self.assertFalse(snapshot["ready"])
def test_expired_scheduler_lease_fences_old_instance(self) -> None:
body = self.command("cmd_lease_fence")
self.service.publish_execute(body)
self.service.ingest_commands()
second = AgentCallService(
db_path=Path(self.temp.name) / "executor.sqlite3",
object_dir=Path(self.temp.name) / "objects-second",
broker=self.broker,
)
try:
with self.service.store.tx() as con:
con.execute(
"UPDATE leases SET expires_at=? WHERE lease_name='scheduler'",
(iso(utcnow() - timedelta(seconds=1)),),
)
self.assertEqual(second.schedule_once(), 1)
self.assertEqual(self.service.schedule_once(), 0)
second.wait_for_idle()
finally:
second.stop()
def test_controls_use_cas_and_old_commands_do_not_resume(self) -> None: def test_controls_use_cas_and_old_commands_do_not_resume(self) -> None:
pause = self.service.control_task( pause = self.service.control_task(
+3 -2
View File
@@ -1,5 +1,6 @@
import base64 import base64
import copy import copy
import ipaddress
import json import json
import tempfile import tempfile
import unittest import unittest
@@ -293,10 +294,10 @@ class AsteriskTests(unittest.TestCase):
def test_reject_injection_public_ari_and_our_ip_as_provider(self): def test_reject_injection_public_ari_and_our_ip_as_provider(self):
for update in ( for update in (
{"ari_bind": ".".join(("0", "0", "0", "0"))}, {"ari_bind": str(ipaddress.IPv4Address(0))},
{"ari_bind": "8.8.8.8"}, {"ari_bind": "8.8.8.8"},
{"local_net": "0.0.0.0/0"}, {"local_net": "0.0.0.0/0"},
): # noqa: S104 — negative fixtures; renderer must reject them. ):
cfg = self.cfg() cfg = self.cfg()
cfg.update(update) cfg.update(update)
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
+38 -6
View File
@@ -42,6 +42,10 @@ class HttpContractTests(unittest.TestCase):
"recording.complete", "recording.complete",
], ],
}, },
"replay": {
"tenant_ids": ["tenant-demo"],
"scopes": ["outbound.read", "outbound.replay"],
},
} }
) )
self.service = AgentCallService( self.service = AgentCallService(
@@ -73,14 +77,18 @@ class HttpContractTests(unittest.TestCase):
path: str, path: str,
body: dict[str, Any] | None = None, body: dict[str, Any] | None = None,
tenant: str | None = "tenant-demo", tenant: str | None = "tenant-demo",
token: str | None = "local", auth_name: str | None = None,
idem: str | None = None, idem: str | None = None,
include_auth: bool = True,
) -> tuple[int, dict[str, Any]]: ) -> tuple[int, dict[str, Any]]:
headers = {"X-Request-ID": "http-test"} headers = {"X-Request-ID": "http-test"}
if tenant is not None: if tenant is not None:
headers["X-Tenant-ID"] = tenant headers["X-Tenant-ID"] = tenant
if token is not None: if include_auth and auth_name is None:
headers["Authorization"] = f"Bearer {token}" token_map = json.loads(os.environ["HTTP_TOKENS"])
auth_name = next(iter(token_map), None)
if include_auth and auth_name is not None:
headers["Authorization"] = f"Bearer {auth_name}"
encoded: bytes | None = None encoded: bytes | None = None
if body is not None: if body is not None:
encoded = json.dumps(body, ensure_ascii=False).encode("utf-8") encoded = json.dumps(body, ensure_ascii=False).encode("utf-8")
@@ -104,7 +112,7 @@ class HttpContractTests(unittest.TestCase):
except (UnicodeDecodeError, json.JSONDecodeError) as exc: except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AssertionError("HTTP response is not JSON") from exc raise AssertionError("HTTP response is not JSON") from exc
if not isinstance(parsed, dict): if not isinstance(parsed, dict):
raise AssertionError("HTTP response is not an object") raise TypeError("HTTP response is not an object")
return status, parsed return status, parsed
def publish_success(self) -> dict[str, Any]: def publish_success(self) -> dict[str, Any]:
@@ -118,11 +126,13 @@ class HttpContractTests(unittest.TestCase):
return self.service.get_command("tenant-demo", "http_cmd") return self.service.get_command("tenant-demo", "http_cmd")
def test_health_is_public_but_business_queries_are_authenticated(self) -> None: def test_health_is_public_but_business_queries_are_authenticated(self) -> None:
status, body = self.request("GET", "/healthz/live", tenant=None, token=None) status, body = self.request(
"GET", "/healthz/live", tenant=None, include_auth=False
)
self.assertEqual(status, 200) self.assertEqual(status, 200)
self.assertEqual(body["status"], "live") self.assertEqual(body["status"], "live")
status, body = self.request( status, body = self.request(
"GET", "/internal/v1/outbound/commands/missing", token=None "GET", "/internal/v1/outbound/commands/missing", include_auth=False
) )
self.assertEqual(status, 401) self.assertEqual(status, 401)
self.assertEqual(body["code"], "UNAUTHORIZED") self.assertEqual(body["code"], "UNAUTHORIZED")
@@ -160,6 +170,28 @@ class HttpContractTests(unittest.TestCase):
) )
self.assertEqual(status, 403) self.assertEqual(status, 403)
def test_replay_requires_replay_scope(self) -> None:
snapshot = self.publish_success()
body = {"command_id": "http_replay", "reason": "retry"}
status, response = self.request(
"POST",
f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays",
body,
auth_name="local",
idem="http_replay",
)
self.assertEqual(status, 403)
self.assertEqual(response["code"], "FORBIDDEN")
status, response = self.request(
"POST",
f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays",
body,
auth_name="replay",
idem="http_replay",
)
self.assertEqual(status, 202)
self.assertEqual(response["status"], "accepted")
def test_recording_upload_contract_is_scoped(self) -> None: def test_recording_upload_contract_is_scoped(self) -> None:
snapshot = self.publish_success() snapshot = self.publish_success()
call = self.service.get_call("tenant-demo", snapshot["call_id"]) call = self.service.get_call("tenant-demo", snapshot["call_id"])
+334
View File
@@ -0,0 +1,334 @@
from __future__ import annotations
import copy
import json
import os
import tempfile
import time
import unittest
from pathlib import Path
from agent_call.core import (
AgentCallService,
ConfigurationError,
ConflictError,
InMemoryBroker,
Store,
ValidationError,
iso,
utcnow,
)
from deploy.backup_sqlite import copy_database
ROOT = Path(__file__).resolve().parents[1]
def fixture() -> dict:
return json.loads(
(ROOT / "docs/contracts/examples/call.execute.json").read_text(encoding="utf-8")
)
class RemediationTests(unittest.TestCase):
def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory()
self.broker = InMemoryBroker()
self.service = AgentCallService(
Path(self.temp.name) / "executor.sqlite3",
object_dir=Path(self.temp.name) / "objects",
broker=self.broker,
)
self.service.profile["limits"]["tenant_cps"] = 100
self.service.profile["limits"]["global_cps"] = 100
def tearDown(self) -> None:
self.service.stop()
self.temp.cleanup()
def command(
self,
command_id: str,
tenant_id: str = "tenant-demo",
execution_id: str = "exec",
) -> dict:
body = copy.deepcopy(fixture())
body["command_id"] = command_id
body["trace_id"] = f"trace-{command_id}"
body["issued_at"] = iso(utcnow())
body["tenant_id"] = tenant_id
tenant = self.service.store.one(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,)
)
body["tenant_key"] = tenant["tenant_key"]
body["payload"]["execution_id"] = execution_id
body["payload"]["task_id"] = (
"task-demo" if tenant_id == "tenant-demo" else "task-b"
)
return body
def test_runtime_schema_rejects_naive_time_and_boolean_revision(self) -> None:
body = self.command("schema-command")
body["issued_at"] = "2026-09-13T00:00:00"
with self.assertRaises(ValidationError):
self.service.publish_execute(body)
with self.assertRaises(ValidationError):
self.service.control_task(
"tenant-demo",
"task-demo",
{
"command_id": "bool-revision",
"action": "pause",
"expected_task_revision": True,
"reason": "x",
},
"bool-revision",
)
def test_readiness_requires_persistent_database_write(self) -> None:
db_path = Path(self.temp.name) / "executor.sqlite3"
os.chmod(db_path, 0o444)
try:
readiness = self.service.readiness()
finally:
os.chmod(db_path, 0o644)
self.assertFalse(readiness["checks"]["database_write"])
self.assertFalse(readiness["ready"])
def test_provider_modes_do_not_fallback(self) -> None:
profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text())
profile["provider_modes"].pop("llm")
path = Path(self.temp.name) / "profile.json"
path.write_text(json.dumps(profile), encoding="utf-8")
with self.assertRaises(ConfigurationError):
AgentCallService(
profile_path=path, object_dir=Path(self.temp.name) / "objects2"
)
def test_same_public_ids_are_tenant_scoped_and_wrong_route_is_dead_lettered(
self,
) -> None:
first = self.command("same-command", "tenant-demo", "same-execution")
second = self.command("same-command", "tenant-b", "same-execution")
self.service.publish_execute(first)
self.service.publish_execute(second)
self.assertEqual(self.service.ingest_commands(3), 2)
self.assertEqual(self.service.schedule_once(), 2)
self.service.wait_for_idle()
self.assertEqual(
self.service.store.one("SELECT COUNT(*) AS n FROM calls")["n"], 2
)
wrong = self.command("wrong-route", "tenant-b", "wrong-execution")
self.broker.publish(
"agent-call.commands.v1",
"agent-call.tenant.tenant-demo-key.call.execute",
wrong,
"wrong-route",
)
self.service._cursor = self.service._tenant_order.index("tenant-demo")
self.service.ingest_commands(1)
self.assertEqual(
self.service.store.one("SELECT COUNT(*) AS n FROM dead_letters")["n"], 1
)
self.assertIsNone(
self.service.store.one(
"SELECT * FROM calls WHERE command_id=?", ("wrong-route",)
)
)
def test_over_budget_tenant_is_retained_without_truncation(self) -> None:
profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text())
key = "x" * 225
profile["tenants"] = [
{"tenant_id": "long", "tenant_key": key, "max_concurrency": 1}
]
profile["tasks"] = [
{
"task_id": "long-task",
"tenant_id": "long",
"state": "running",
"revision": 1,
}
]
path = Path(self.temp.name) / "long-profile.json"
path.write_text(json.dumps(profile), encoding="utf-8")
service = AgentCallService(
Path(self.temp.name) / "long.sqlite3",
profile_path=path,
object_dir=Path(self.temp.name) / "long-objects",
broker=InMemoryBroker(),
)
try:
body = self.command("placeholder")
body["tenant_id"], body["tenant_key"] = "long", key
body["payload"]["task_id"] = "long-task"
body["payload"]["execution_id"] = "long-execution"
with self.assertRaises(ValidationError) as error:
service.publish_execute(body)
self.assertEqual(error.exception.code, "TENANT_KEY_UNSUPPORTED")
row = service.store.one(
"SELECT status FROM published_commands WHERE tenant_id=?", ("long",)
)
self.assertEqual(row["status"], "retained")
finally:
service.stop()
def test_control_barrier_blocks_resume_until_all_cells_apply(self) -> None:
pause = {
"command_id": "pause-1",
"action": "pause",
"expected_task_revision": 1,
"reason": "maintenance",
}
self.service.set_cell_health("cell-a", False)
self.service.control_task("tenant-demo", "task-demo", pause, "pause-1")
self.assertEqual(self.service.apply_controls(), 0)
with self.assertRaises(ConflictError) as error:
self.service.control_task(
"tenant-demo",
"task-demo",
{
"command_id": "resume-1",
"action": "resume",
"expected_task_revision": 2,
"reason": "resume",
},
"resume-1",
)
self.assertEqual(error.exception.code, "CONTROL_IN_PROGRESS")
self.service.set_cell_health("cell-a", True)
self.assertEqual(self.service.apply_controls(), 1)
self.assertEqual(
self.service.store.one(
"SELECT COUNT(*) AS n FROM control_barriers WHERE command_id=? AND status='applied'",
("pause-1",),
)["n"],
2,
)
def test_final_transcript_is_idempotent_and_conflicts_are_preserved(self) -> None:
self.service.profile["limits"]["hold_ms"] = 1000
body = self.command("transcript-command", "tenant-demo", "transcript-execution")
self.service.publish_execute(body)
self.service.ingest_commands()
self.service.schedule_once()
deadline = time.monotonic() + 2
call_id = None
while time.monotonic() < deadline:
row = self.service.store.one(
"SELECT * FROM calls WHERE command_id=?", (body["command_id"],)
)
if row and row["started_at"]:
call_id = row["call_id"]
break
time.sleep(0.01)
self.assertIsNotNone(call_id)
assert call_id is not None
self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed")
before = self.service.store.one(
"SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'"
)["n"]
self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed")
after = self.service.store.one(
"SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'"
)["n"]
self.assertEqual(before, after)
with self.assertRaises(ConflictError):
self.service._add_transcript(
call_id, 99, "agent", "different", "confirmed", revision=2
)
with self.assertRaises(ConflictError):
self.service._add_transcript(
call_id, 99, "agent", "different", "confirmed", revision=1
)
self.service._add_transcript(
call_id, 99, "agent", "final text", "sent", revision=3, is_final=False
)
segment = self.service.store.one(
"SELECT text, is_final, playback_state FROM transcripts WHERE segment_id=?",
(f"seg_{call_id}_99_agent",),
)
self.assertEqual(
(segment["text"], segment["is_final"], segment["playback_state"]),
("final text", 1, "confirmed"),
)
self.service._finish_call(call_id, "failed", "TEST_FINISHED")
def test_upload_rejects_metadata_changes(self) -> None:
body = self.command(
"recording-metadata-command", "tenant-demo", "recording-metadata-execution"
)
self.service.publish_execute(body)
self.service.wait_for_idle()
call_id = self.service.get_command("tenant-demo", body["command_id"])["call_id"]
recording = self.service.get_call("tenant-demo", call_id)["recordings"][0]
request = {
"recording_id": recording["recording_id"],
"call_id": call_id,
"content_type": "audio/wav",
"size_bytes": recording["size_bytes"],
"checksum_algorithm": "SHA-256",
"checksum": recording["checksum"],
"channels": 1,
"sample_rate_hz": recording["sample_rate_hz"],
"duration_ms": recording["duration_ms"] + 1,
}
with self.assertRaises(ConflictError) as error:
self.service.create_upload("tenant-demo", request, "metadata-change", "api")
self.assertEqual(error.exception.code, "RECORDING_METADATA_CONFLICT")
def test_recording_is_non_silent_and_upload_completion_is_idempotent(self) -> None:
body = self.command("recording-command", "tenant-demo", "recording-execution")
self.service.publish_execute(body)
self.service.wait_for_idle()
call = self.service.get_call(
"tenant-demo",
self.service.get_command("tenant-demo", body["command_id"])["call_id"],
)
recording = call["recordings"][0]
upload = self.service.store.one(
"SELECT upload_id FROM uploads WHERE tenant_id=? AND recording_id=? ORDER BY created_at DESC LIMIT 1",
("tenant-demo", recording["recording_id"]),
)
self.assertIsNotNone(upload)
assert upload is not None
path = self.service.object_store.final_path_for(upload["upload_id"])
self.assertTrue(any(path.read_bytes()))
request = {
"recording_id": recording["recording_id"],
"call_id": call["call_id"],
"content_type": "audio/wav",
"size_bytes": recording["size_bytes"],
"checksum_algorithm": "SHA-256",
"checksum": recording["checksum"],
"channels": 1,
"sample_rate_hz": recording["sample_rate_hz"],
"duration_ms": recording["duration_ms"],
}
status, first = self.service.create_upload(
"tenant-demo", request, "api-create", "api"
)
status2, second = self.service.create_upload(
"tenant-demo", request, "api-create", "api"
)
self.assertEqual((status, status2), (200, 200))
self.assertEqual(first["upload_id"], second["upload_id"])
self.assertEqual(
self.service.store.one(
"SELECT COUNT(*) AS n FROM recordings WHERE call_id=?",
(call["call_id"],),
)["n"],
1,
)
def test_backup_refuses_to_replace_existing_database(self) -> None:
source = Path(self.temp.name) / "source.sqlite3"
destination = Path(self.temp.name) / "backup.sqlite3"
Store(source)
copy_database(source, destination)
with self.assertRaises(FileExistsError):
copy_database(source, destination)
if __name__ == "__main__":
unittest.main()