feat(agent-call): implement remediation plan
This commit is contained in:
+2023
-425
File diff suppressed because it is too large
Load Diff
+58
-14
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from http import HTTPStatus
|
||||
@@ -15,6 +16,7 @@ READ_SCOPE = "outbound.read"
|
||||
CONTROL_SCOPE = "outbound.control"
|
||||
UPLOAD_SCOPE = "recording.upload"
|
||||
COMPLETE_SCOPE = "recording.complete"
|
||||
REPLAY_SCOPE = "outbound.replay"
|
||||
|
||||
|
||||
class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
@@ -37,7 +39,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
raise ServiceError("TENANT_ID_REQUIRED", "X-Tenant-ID is required", 400)
|
||||
return value
|
||||
|
||||
def _authorize(self, scope: str, tenant_id: str | None = None) -> None:
|
||||
def _authorize(self, scope: str, tenant_id: str | None = None) -> dict[str, Any]:
|
||||
token_header = self.headers.get("Authorization", "")
|
||||
if not token_header.startswith("Bearer "):
|
||||
raise ServiceError("UNAUTHORIZED", "bearer authentication is required", 401)
|
||||
@@ -49,12 +51,24 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
if not isinstance(identity, dict):
|
||||
raise ServiceError("UNAUTHORIZED", "bearer token is not authorized", 401)
|
||||
scopes = identity.get("scopes", [])
|
||||
tenant_ids = identity.get("tenant_ids", [])
|
||||
if not isinstance(scopes, list) or not all(
|
||||
isinstance(item, str) for item in scopes
|
||||
):
|
||||
raise ServiceError("UNAUTHORIZED", "token scopes are invalid", 401)
|
||||
if not isinstance(tenant_ids, list) or not all(
|
||||
isinstance(item, str) for item in tenant_ids
|
||||
):
|
||||
raise ServiceError("UNAUTHORIZED", "token tenant scope is invalid", 401)
|
||||
if scope not in scopes and "*" not in scopes:
|
||||
raise ServiceError("FORBIDDEN", "required scope is missing", 403)
|
||||
if tenant_id is not None:
|
||||
tenant_ids = identity.get("tenant_ids", [])
|
||||
if "*" not in tenant_ids and tenant_id not in tenant_ids:
|
||||
raise ServiceError("FORBIDDEN", "tenant scope is missing", 403)
|
||||
if (
|
||||
tenant_id is not None
|
||||
and "*" not in tenant_ids
|
||||
and tenant_id not in tenant_ids
|
||||
):
|
||||
raise ServiceError("FORBIDDEN", "tenant scope is missing", 403)
|
||||
return identity
|
||||
|
||||
def _body(self) -> dict[str, Any]:
|
||||
raw_length = self.headers.get("Content-Length")
|
||||
@@ -69,6 +83,8 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
)
|
||||
try:
|
||||
raw = self.rfile.read(length)
|
||||
if len(raw) != length:
|
||||
raise ValidationError("BODY_TRUNCATED", "request body was truncated")
|
||||
body = _loads(raw.decode("utf-8"))
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValidationError(
|
||||
@@ -80,10 +96,21 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def _header(self, name: str) -> str:
|
||||
value = self.headers.get(name, "")
|
||||
if not value:
|
||||
if not value or len(value) > 128 or any(ch.isspace() for ch in value):
|
||||
raise ServiceError("HEADER_REQUIRED", f"{name} is required", 400)
|
||||
return value
|
||||
|
||||
def _actor_id(self) -> str:
|
||||
token_header = self.headers.get("Authorization", "")
|
||||
token = token_header[7:] if token_header.startswith("Bearer ") else ""
|
||||
identity = _loads(os.environ.get("HTTP_TOKENS"), {})
|
||||
principal = identity.get(token) if isinstance(identity, dict) else None
|
||||
if isinstance(principal, dict):
|
||||
value = principal.get("client_id") or principal.get("subject")
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return "token:" + hashlib.sha256(token.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
def _send(
|
||||
self, status: int, body: Any, content_type: str = "application/json"
|
||||
) -> None:
|
||||
@@ -195,7 +222,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
and parts[5] == "replays"
|
||||
):
|
||||
tenant_id = self._tenant_id()
|
||||
self._authorize(CONTROL_SCOPE, tenant_id)
|
||||
self._authorize(REPLAY_SCOPE, tenant_id)
|
||||
result = self.service.create_replay(
|
||||
tenant_id,
|
||||
"call",
|
||||
@@ -213,7 +240,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
and parts[5] == "replays"
|
||||
):
|
||||
tenant_id = self._tenant_id()
|
||||
self._authorize(CONTROL_SCOPE, tenant_id)
|
||||
self._authorize(REPLAY_SCOPE, tenant_id)
|
||||
result = self.service.create_replay(
|
||||
tenant_id,
|
||||
"command",
|
||||
@@ -231,8 +258,9 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
):
|
||||
tenant_id = self._tenant_id()
|
||||
self._authorize(UPLOAD_SCOPE, tenant_id)
|
||||
idempotency_key = self._header("Idempotency-Key")
|
||||
result_status, result = self.service.create_upload(
|
||||
tenant_id, self._body()
|
||||
tenant_id, self._body(), idempotency_key, self._actor_id()
|
||||
)
|
||||
self._send(result_status, result)
|
||||
return
|
||||
@@ -245,7 +273,10 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
):
|
||||
tenant_id = self._tenant_id()
|
||||
self._authorize(COMPLETE_SCOPE, tenant_id)
|
||||
result = self.service.complete_upload(tenant_id, parts[4], self._body())
|
||||
idempotency_key = self._header("Idempotency-Key")
|
||||
result = self.service.complete_upload(
|
||||
tenant_id, parts[4], self._body(), idempotency_key, self._actor_id()
|
||||
)
|
||||
self._send(200, result)
|
||||
return
|
||||
if method == "PUT" and len(parts) == 3 and parts[:2] == ["_mock", "oss"]:
|
||||
@@ -260,8 +291,8 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
if not raw_length:
|
||||
raise ValidationError("BODY_REQUIRED", "upload body is required")
|
||||
length = _integer(raw_length)
|
||||
if length > self.service.profile.get("limits", {}).get(
|
||||
"recording_max_bytes", 16777216
|
||||
if length < 0 or length > self.service.profile.get("limits", {}).get(
|
||||
"max_recording_bytes", 16777216
|
||||
):
|
||||
raise ServiceError(
|
||||
"BODY_TOO_LARGE",
|
||||
@@ -274,6 +305,10 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
raise ServiceError(
|
||||
"UPLOAD_INTERRUPTED", "upload connection interrupted", 400
|
||||
) from exc
|
||||
if len(data) != length:
|
||||
raise ServiceError(
|
||||
"UPLOAD_INTERRUPTED", "upload body was truncated", 400
|
||||
)
|
||||
upload = self.service.store.one(
|
||||
"SELECT tenant_id FROM uploads WHERE upload_id=?", (parts[2],)
|
||||
)
|
||||
@@ -287,7 +322,7 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
self._error(exc, request_id)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
return
|
||||
except Exception:
|
||||
except (OSError, RuntimeError, TypeError, ValueError, KeyError):
|
||||
self._error(
|
||||
ServiceError("INTERNAL_ERROR", "request could not be completed", 500),
|
||||
request_id,
|
||||
@@ -304,8 +339,17 @@ class AgentCallHandler(BaseHTTPRequestHandler):
|
||||
if not isinstance(identity, dict):
|
||||
return False
|
||||
tenant_ids = identity.get("tenant_ids", [])
|
||||
scopes = identity.get("scopes", [])
|
||||
if not isinstance(tenant_ids, list) or not all(
|
||||
isinstance(item, str) for item in tenant_ids
|
||||
):
|
||||
return False
|
||||
if not isinstance(scopes, list) or not all(
|
||||
isinstance(item, str) for item in scopes
|
||||
):
|
||||
return False
|
||||
return ("*" in tenant_ids or tenant_id in tenant_ids) and (
|
||||
scope in identity.get("scopes", []) or "*" in identity.get("scopes", [])
|
||||
scope in scopes or "*" in scopes
|
||||
)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
|
||||
+40
-3
@@ -6,6 +6,7 @@ ARI, RTP, AI-stream, or OSS integration.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@@ -82,9 +83,23 @@ class MockRTP:
|
||||
def __init__(self, journal: ProtocolJournal) -> None:
|
||||
self.journal = journal
|
||||
|
||||
def packet(self, call_id: str, direction: str, sequence: int) -> None:
|
||||
def packet(
|
||||
self,
|
||||
call_id: str,
|
||||
direction: str,
|
||||
sequence: int,
|
||||
payload: bytes | None = None,
|
||||
) -> None:
|
||||
payload = payload if payload is not None else b"\x01\x02" * 160
|
||||
self.journal.append(
|
||||
"rtp", "packet", call_id=call_id, direction=direction, sequence=sequence
|
||||
"rtp",
|
||||
"packet",
|
||||
call_id=call_id,
|
||||
direction=direction,
|
||||
sequence=sequence,
|
||||
payload_bytes=len(payload),
|
||||
payload_sha256=hashlib.sha256(payload).hexdigest(),
|
||||
non_silent=any(payload),
|
||||
)
|
||||
|
||||
|
||||
@@ -93,7 +108,12 @@ class MockAI:
|
||||
self.journal = journal
|
||||
|
||||
def turn(
|
||||
self, call_id: str, turn_index: int, speaker: str, playback_state: str
|
||||
self,
|
||||
call_id: str,
|
||||
turn_index: int,
|
||||
speaker: str,
|
||||
playback_state: str,
|
||||
text: str | None = None,
|
||||
) -> None:
|
||||
self.journal.append(
|
||||
"ai",
|
||||
@@ -102,4 +122,21 @@ class MockAI:
|
||||
turn_index=turn_index,
|
||||
speaker=speaker,
|
||||
playback_state=playback_state,
|
||||
text=text or f"mock-{speaker}-turn-{turn_index}",
|
||||
)
|
||||
self.journal.append(
|
||||
"ai",
|
||||
"playback",
|
||||
call_id=call_id,
|
||||
turn_index=turn_index,
|
||||
state="sent",
|
||||
)
|
||||
self.journal.append(
|
||||
"ai",
|
||||
"playback",
|
||||
call_id=call_id,
|
||||
turn_index=turn_index,
|
||||
state=playback_state
|
||||
if playback_state in {"confirmed", "playback_confirmed", "unknown"}
|
||||
else "unknown",
|
||||
)
|
||||
|
||||
+59
-7
@@ -4,18 +4,70 @@ groups:
|
||||
- alert: AgentCallOutboxStalled
|
||||
expr: agent_call_outbox_pending > 0
|
||||
for: 2m
|
||||
labels: {severity: warning}
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: agent-call outbox has pending events
|
||||
summary: agent-call outbox is not draining
|
||||
- alert: AgentCallDeadLetters
|
||||
expr: increase(agent_call_dead_letters_total[5m]) > 0
|
||||
for: 1m
|
||||
labels: {severity: critical}
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: agent-call rejected or malformed messages entered the DLQ
|
||||
summary: agent-call rejected messages reached the dead-letter log
|
||||
- alert: AgentCallNotMockSafe
|
||||
expr: agent_call_mode_info{mode="mock"} != 1
|
||||
expr: agent_call_mode_info{mode!="mock"} > 0
|
||||
for: 1m
|
||||
labels: {severity: critical}
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: provider mode changed; verify production admission gates
|
||||
summary: agent-call is running outside the supported Mock mode
|
||||
- alert: AgentCallNotReady
|
||||
expr: agent_call_ready == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: agent-call readiness gate is closed
|
||||
- alert: AgentCallClockJump
|
||||
expr: agent_call_clock_jump > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: agent-call stopped new admission after a wall-clock jump
|
||||
- alert: AgentCallUnknownCall
|
||||
expr: agent_call_unknown_calls > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: active call requires reconciliation and still occupies capacity
|
||||
- alert: AgentCallQueueBackpressure
|
||||
expr: sum(agent_call_queue_depth) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: tenant command queues have pending backpressure
|
||||
- alert: AgentCallRecordingFailures
|
||||
expr: increase(agent_call_recording_failures_total[5m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: recording verification failed
|
||||
- alert: AgentCallDiskWarning
|
||||
expr: agent_call_disk_usage_ratio >= 0.70
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: agent-call disk usage is high
|
||||
- alert: AgentCallDiskStop
|
||||
expr: agent_call_disk_usage_ratio >= 0.80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: agent-call stopped new admissions for disk pressure
|
||||
|
||||
Regular → Executable
Regular → Executable
+47
-18
@@ -1,36 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create or restore an SQLite backup without silently overwriting a target."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def copy_database(source: str, destination: str) -> None:
|
||||
source_path = Path(source)
|
||||
destination_path = Path(destination)
|
||||
def _check_integrity(con: sqlite3.Connection) -> None:
|
||||
result = con.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
raise RuntimeError(f"SQLite integrity check failed: {result}")
|
||||
|
||||
|
||||
def copy_database(
|
||||
source: str | Path, destination: str | Path, *, replace: bool = False
|
||||
) -> None:
|
||||
source_path = Path(source).expanduser().resolve()
|
||||
destination_path = Path(destination).expanduser().resolve()
|
||||
if source_path == destination_path:
|
||||
raise ValueError("source and destination must differ")
|
||||
if not source_path.is_file():
|
||||
raise FileNotFoundError(source)
|
||||
raise FileNotFoundError(source_path)
|
||||
if destination_path.exists() and not replace:
|
||||
raise FileExistsError(
|
||||
f"refusing to overwrite existing database: {destination_path}"
|
||||
)
|
||||
destination_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
source_db = sqlite3.connect(source_path)
|
||||
destination_db = sqlite3.connect(destination_path)
|
||||
try:
|
||||
source_db.backup(destination_db)
|
||||
destination_db.commit()
|
||||
finally:
|
||||
destination_db.close()
|
||||
source_db.close()
|
||||
with closing(
|
||||
sqlite3.connect(f"file:{source_path}?mode=ro", uri=True)
|
||||
) as source_con:
|
||||
_check_integrity(source_con)
|
||||
with closing(sqlite3.connect(destination_path)) as destination_con:
|
||||
source_con.backup(destination_con)
|
||||
destination_con.commit()
|
||||
_check_integrity(destination_con)
|
||||
version = destination_con.execute(
|
||||
"SELECT MAX(version) FROM schema_migrations"
|
||||
).fetchone()[0]
|
||||
if version is None:
|
||||
raise RuntimeError("backup has no schema migration version")
|
||||
except sqlite3.DatabaseError as exc:
|
||||
raise RuntimeError(f"SQLite backup failed: {exc}") from exc
|
||||
print(f"backup complete: {source_path} -> {destination_path}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="offline SQLite backup/restore for agent-call"
|
||||
)
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=("backup", "restore"))
|
||||
parser.add_argument("source")
|
||||
parser.add_argument("destination")
|
||||
args = parser.parse_args()
|
||||
copy_database(args.source, args.destination)
|
||||
print(f"{args.action} complete")
|
||||
parser.add_argument(
|
||||
"--replace",
|
||||
action="store_true",
|
||||
help="explicitly allow replacing the destination",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
copy_database(args.source, args.destination, replace=args.replace)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Regular → Executable
+10
-6
@@ -61,9 +61,7 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None
|
||||
f"ASTERISK_IMAGE={shlex.quote(image)}",
|
||||
f"ASTERISK_CONFIG_GID={config_gid}",
|
||||
]
|
||||
variables.extend(
|
||||
f"{key}={shlex.quote(value)}" for key, value in payloads.items()
|
||||
)
|
||||
variables.extend(f"{key}={shlex.quote(value)}" for key, value in payloads.items())
|
||||
variables.append("")
|
||||
if template is None:
|
||||
template = Path(__file__).with_name("asterisk_bootstrap.sh")
|
||||
@@ -76,7 +74,9 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None
|
||||
def write_output(content, path):
|
||||
path = Path(path)
|
||||
if path.exists() or path.is_symlink():
|
||||
raise ValueError("output already exists; choose a new local secret-bearing path")
|
||||
raise ValueError(
|
||||
"output already exists; choose a new local secret-bearing path"
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_path = None
|
||||
try:
|
||||
@@ -105,7 +105,9 @@ def main():
|
||||
parser.add_argument("--config", required=True, help="render_asterisk JSON config")
|
||||
parser.add_argument("--image", help="immutable registry image@sha256:digest")
|
||||
parser.add_argument("--ari-password-file")
|
||||
parser.add_argument("--output", required=True, help="local secret-bearing user-data path")
|
||||
parser.add_argument(
|
||||
"--output", required=True, help="local secret-bearing user-data path"
|
||||
)
|
||||
parser.add_argument("--config-gid", type=int, default=1000)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
@@ -113,7 +115,9 @@ def main():
|
||||
image = immutable_image(
|
||||
args.image or cfg.get("asterisk_image") or os.environ.get("ASTERISK_IMAGE")
|
||||
)
|
||||
content = build(cfg, image, read_secret(args.ari_password_file), args.config_gid)
|
||||
content = build(
|
||||
cfg, image, read_secret(args.ari_password_file), args.config_gid
|
||||
)
|
||||
if len(content.encode()) > 16 * 1024:
|
||||
raise ValueError("generated user-data exceeds the ECS 16 KiB limit")
|
||||
write_output(content, args.output)
|
||||
|
||||
@@ -10,6 +10,7 @@ if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
_core = importlib.import_module("agent_call.core")
|
||||
ValidationError = _core.ValidationError
|
||||
queue_name = _core.queue_name
|
||||
routing_key = _core.routing_key
|
||||
|
||||
@@ -30,7 +31,7 @@ def check(profile_path: str | Path) -> list[str]:
|
||||
try:
|
||||
route = routing_key(tenant["tenant_key"])
|
||||
queue_name(tenant["tenant_key"])
|
||||
except Exception as exc:
|
||||
except (KeyError, TypeError, ValidationError) as exc:
|
||||
problems.append(f"tenant route invalid: {type(exc).__name__}")
|
||||
continue
|
||||
if len(route.encode("utf-8")) > 255:
|
||||
|
||||
+6
-3
@@ -15,9 +15,12 @@ Store = _core.Store
|
||||
|
||||
def migrate(path: str) -> int:
|
||||
store = Store(path)
|
||||
row = store.one("SELECT MAX(version) AS version FROM schema_migrations")
|
||||
print(f"schema_version={row['version']}")
|
||||
return 0
|
||||
try:
|
||||
row = store.one("SELECT MAX(version) AS version FROM schema_migrations")
|
||||
print(f"schema_version={row['version']}")
|
||||
return 0
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -2,6 +2,9 @@ global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: agent-call
|
||||
metrics_path: /metrics
|
||||
|
||||
Regular → Executable
@@ -163,7 +163,7 @@ components:
|
||||
type: string
|
||||
minLength: 1
|
||||
maxLength: 128
|
||||
pattern: '^[^\\s/\\\\]+$'
|
||||
pattern: '^[^\s/\\]+$'
|
||||
ControlRequest:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
||||
@@ -20,14 +20,24 @@
|
||||
"max_call_duration_ms": 180000,
|
||||
"max_mq_bytes": 262144,
|
||||
"max_queue_messages": 1000,
|
||||
"max_queue_bytes": 16777216,
|
||||
"disk_warn_pct": 0.70,
|
||||
"disk_stop_pct": 0.80,
|
||||
"max_http_bytes": 65536,
|
||||
"recording_max_bytes": 16777216,
|
||||
"max_recording_bytes": 16777216,
|
||||
"replay_retention_s": 604800,
|
||||
"upload_ttl_s": 300,
|
||||
"global_concurrency": 6,
|
||||
"global_cps": 3,
|
||||
"tenant_concurrency": 2,
|
||||
"tenant_cps": 1,
|
||||
"tenant_publish_rate": 10,
|
||||
"scheduler_lease_ttl_s": 10,
|
||||
"pending_window_per_tenant": 16,
|
||||
"pending_window_global": 64,
|
||||
"max_unacked_per_tenant": 4,
|
||||
"max_replay_attempts": 6,
|
||||
"cell_capacity": 4,
|
||||
"turns": 2,
|
||||
"hold_ms": 0
|
||||
@@ -47,8 +57,8 @@
|
||||
"caller_profiles": [{"caller_profile_id": "caller_profile_test", "display": "BD93205882", "allowed": true}],
|
||||
"agents": [{"agent_version_id": "agent_v1", "immutable": true, "llm": "mock", "tts": "mock", "asr": "mock"}],
|
||||
"cells": [
|
||||
{"cell_id": "cell-a", "capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"},
|
||||
{"cell_id": "cell-b", "capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"}
|
||||
{"cell_id": "cell-a", "capacity": 4, "media_capacity": 4, "ai_capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"},
|
||||
{"cell_id": "cell-b", "capacity": 4, "media_capacity": 4, "ai_capacity": 4, "egress_pool_id": "egress-mock", "ari_mode": "mock"}
|
||||
],
|
||||
"failure_scenarios": ["success", "busy", "no_answer", "ai_timeout", "ari_disconnect", "upload_missing", "upload_bad_checksum", "broker_outage", "clock_jump"]
|
||||
"failure_scenarios": ["success", "busy", "no_answer", "ai_timeout", "customer_silent", "ari_disconnect", "upload_missing", "upload_bad_checksum", "broker_outage", "clock_jump"]
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ components:
|
||||
type: string
|
||||
minLength: 1
|
||||
maxLength: 128
|
||||
pattern: '^[^\\s/\\\\]+$'
|
||||
pattern: '^[^\s/\\]+$'
|
||||
UploadRequest:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"finished_at": "2026-09-13T10:59:31.729Z",
|
||||
"gate_status": "INCOMPLETE",
|
||||
"probes": {
|
||||
"V01": {
|
||||
"duration_ms": 15.796,
|
||||
"evidence": {
|
||||
"http_execute_route": "absent",
|
||||
"profile": "mock",
|
||||
"schema_negative_cases": 3
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V02": {
|
||||
"duration_ms": 161.588,
|
||||
"evidence": {
|
||||
"calls": 1,
|
||||
"different_execution_command": "associated",
|
||||
"same_command_replays": 1
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V03": {
|
||||
"duration_ms": 344.365,
|
||||
"evidence": {
|
||||
"lease_duplicate_calls": 0,
|
||||
"scheduler_instances": 2,
|
||||
"tenants_admitted": [
|
||||
"tenant-demo",
|
||||
"tenant-b",
|
||||
"tenant-c"
|
||||
]
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V04": {
|
||||
"duration_ms": 34.727,
|
||||
"evidence": {
|
||||
"affected_commands": 1,
|
||||
"stale_cas": "rejected",
|
||||
"waiting_barrier": "applied"
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V05": {
|
||||
"duration_ms": 149.141,
|
||||
"evidence": {
|
||||
"dead_letters": 0,
|
||||
"recovered": true,
|
||||
"retained_id": "cmd_broker_retry"
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V06/V07": {
|
||||
"duration_ms": 2131.36,
|
||||
"evidence": {
|
||||
"asterisk_real": "not_run",
|
||||
"disconnect_state_before_reconcile": "reconciling",
|
||||
"fallback_redials": 0,
|
||||
"mock_protocol": "pass",
|
||||
"outcomes": {
|
||||
"busy": "busy",
|
||||
"disconnect": "failed",
|
||||
"no_answer": "no_answer"
|
||||
}
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V08": {
|
||||
"duration_ms": 994.059,
|
||||
"evidence": {
|
||||
"old_tts_reused": false,
|
||||
"segments": 200,
|
||||
"turns": 100
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V09": {
|
||||
"duration_ms": 1131.946,
|
||||
"evidence": {
|
||||
"bad_checksum": "failed",
|
||||
"invalid_ready": 0,
|
||||
"missing_object": "failed"
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V10": {
|
||||
"duration_ms": 113.747,
|
||||
"evidence": {
|
||||
"original_event_ids": true,
|
||||
"pending_after_recovery": 0,
|
||||
"pending_during_outage": 13
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V11/V12": {
|
||||
"duration_ms": 144.282,
|
||||
"evidence": {
|
||||
"backup": "verified",
|
||||
"metrics": "healthy",
|
||||
"real_mode_mock_rejection": true,
|
||||
"restored_counts": {
|
||||
"commands": 1,
|
||||
"events": 13,
|
||||
"inbox": 13,
|
||||
"recordings": 1
|
||||
},
|
||||
"restored_db_facts": true,
|
||||
"rollback_preserves_db": "not_run"
|
||||
},
|
||||
"status": "PASS"
|
||||
},
|
||||
"V13": {
|
||||
"duration_ms": 0.543,
|
||||
"evidence": {
|
||||
"evidence_file": "docs/evidence/scale-mock-2026-09-12.json",
|
||||
"reason": "historical state-only evidence cannot satisfy continuous media V13",
|
||||
"status": "not_run"
|
||||
},
|
||||
"status": "NOT_RUN"
|
||||
}
|
||||
},
|
||||
"production": {
|
||||
"V14": "blocked_external_evidence"
|
||||
},
|
||||
"profile": "mock",
|
||||
"started_at": "2026-09-13T10:59:26.507Z"
|
||||
}
|
||||
@@ -1,27 +1,38 @@
|
||||
# 持久真实 SIP/Asterisk 测试环境证据（2026-09-13）
|
||||
# 真实 SIP/Asterisk 测试环境证据（2026-09-13）
|
||||
|
||||
## 环境
|
||||
## 当前环境（探针期间）
|
||||
|
||||
- ECS：`i-2zegaldd07mkp0xyhcff`，Ubuntu 24.04，`ecs.e-c1m1.large`。
|
||||
- 竞价策略：`SpotAsPriceGo`（自动竞价）；竞价回收可能中断活动通话。
|
||||
- 固定 EIP：`123.56.71.98`，当前保持 `InUse` 并绑定该 ECS。
|
||||
- Asterisk：`22.10.1`，固定镜像 digest：`sha256:0f5e3cd0e9a86bc9dc4750929abee9ba1faae11ed39739f3e75343d39d837e41`。
|
||||
- 线路：UDP `61.132.228.221:5060`，IP 白名单免注册。
|
||||
- PJSIP contact：`Avail`，实测 RTT 约 `22.146 ms`。
|
||||
- 主叫原值：`BD93205882`；`From user` 与 `CALLERID` 均使用原值，未发送 P-Asserted-Identity。
|
||||
- 环境按用户要求保留，未在本轮测试后停止或删除。
|
||||
- ECS：`i-2ze4hvi6xvy02g0wz9kk`，北京 `cn-beijing`，Ubuntu 24.04，`ecs.e-c1m1.large`，探针期间 Running。
|
||||
- 竞价策略：`SpotAsPriceGo`；竞价回收可能中断活动通话。
|
||||
- 固定 EIP：`123.56.71.98`，AllocationId `eip-2zeevfsaxzwuue2szy7xb`；探针期间 InUse 并绑定该 ECS。
|
||||
- 创建/绑定：2026-09-13 10:28 UTC；使用专用 VSwitch、安全组和 KeyPair，实例公网带宽上限为 0。
|
||||
- Asterisk：22.10.1，镜像及实际 image ID 均为固定 digest `sha256:0f5e3cd0e9a86bc9dc4750929abee9ba1faae11ed39739f3e75343d39d837e41`，容器状态 `running/healthy`。
|
||||
- 管理检查：SSH 未开放；通过阿里云 Cloud Assistant 执行只读检查。Cloud-init 初始拉取镜像期间较慢，最终完成并启动容器。
|
||||
- PJSIP contact：`provider-primary-aor/sip:61.132.228.221:5060` 为 `Avail`，RTT 约 `22.164 ms`。
|
||||
- 出口与信令：SIP 日志显示请求从 `123.56.71.98:5060` 发出；安全组仅允许供应商 `61.132.228.221/32` 的 UDP `5060` 和 `10000–10800`。
|
||||
- 主叫原值：`BD93205882`；`From user`/`CALLERID` 保留原值，未发送 P-Asserted-Identity。
|
||||
|
||||
## 两个被叫测试
|
||||
## 单次真实外呼探针
|
||||
|
||||
两次均使用同一环境、线路、主叫、UDP/ulaw、RTP 配置；均只发起一次，不自动重试。
|
||||
- 执行时间：2026-09-13 10:52:59 UTC。
|
||||
- 原始被叫：`18601013734`。
|
||||
- 实际目标：`708918601013734`。
|
||||
- 线路：`provider-primary`，UDP/ulaw；仅发起一次，无自动重试、无 FALLBACK。
|
||||
- SIP：收到 `100 trying`，随后收到 `488 Not Acceptable Here`。
|
||||
- 原因：`Reason: Q.850;cause=88;text="INCOMPATIBLE_DESTINATION"`。
|
||||
- 后续：发送 ACK；Asterisk 执行 Hangup；15 秒后无活动通道。
|
||||
- 接通/RTP：未接通，未获得 `180/183/200` 或 RTP/录音证据。
|
||||
|
||||
| 原始被叫 | 实际呼出目标 | SIP结果 | RTP/接通 |
|
||||
| --- | --- | --- | --- |
|
||||
| `18601013734` | `708918601013734` | `100 trying` 后 `488 Not Acceptable Here`；`Q.850 cause=88 INCOMPATIBLE_DESTINATION` | 未接通、无 RTP |
|
||||
| `18625770806` | `708918625770806` | `100 trying` 后 `488 Not Acceptable Here`；`Q.850 cause=88 INCOMPATIBLE_DESTINATION` | 未接通、无 RTP |
|
||||
## 测试后资源处置
|
||||
|
||||
两组不同号码在同一线路上得到完全相同的目的地不兼容结果，进一步支持供应商共享路由/线路参数问题；仍不能仅凭两次失败排除同一号码段盲区。需供应商提供已知可达号码，或确认被叫格式/路由、From 域和编解码要求后再继续。
|
||||
按用户授权，探针结束后已优雅停止并删除本次创建的 ECS；删除前确认该实例是唯一 `project=agent-call` 候选，系统盘为 40 GiB 且 `DeleteWithInstance=true`。已确认实例和系统盘不再返回；固定 EIP 未释放、未解绑，当前为 `Available`。
|
||||
|
||||
## 范围边界
|
||||
## 结论
|
||||
|
||||
本次仅验证了持久 ECS、Asterisk、SIP OPTIONS/contact、两次真实 INVITE 和失败回执；未验证真实 RTP、MQ、OSS、ASR、LLM/TTS 或完整 AI 通话。
|
||||
本次确认了单路 ECS/Asterisk 底座、PJSIP contact、固定出口和一次真实 INVITE 链路；供应商以 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION` 拒绝目标，R1 外呼接通验收未通过。该结果不能证明真实 RTP、ASR、LLM/TTS、MQ、OSS 或生产容量/N+1 已验收。
|
||||
|
||||
供应商需先确认 `7089` 前缀/目标路由、From 域/主叫格式和编解码，或提供已知可达测试号码。未确认前不再次拨号。
|
||||
|
||||
## 历史记录
|
||||
|
||||
此前同一固定 EIP/线路对 `18625770806` 的探针目标 `708918625770806` 也返回 `488/Q.850 cause=88`；历史实例已按测试回收规则删除，固定 EIP 未释放。
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
# agent-call：功能审查整改计划与验收标准
|
||||
|
||||
**版本：** v1.0
|
||||
**日期：** 2026-09-13
|
||||
**状态：** 代码级整改已实现并通过本地回归；真实协议、独立服务、告警链路及规模门禁仍未通过。
|
||||
**审查基线：** main / `d30314c7d1c56708a43333fe66ad68cfe7c29f40`，三路静态代码审查；未运行故障复现、测试或真实外呼。下列代码位置为该基线定位，后续以符号和回归用例追踪。`deploy/build_asterisk_userdata.py` 的既有未提交修改不在本次整改文档编辑范围内。
|
||||
|
||||
## 1. 目的、权威与边界
|
||||
|
||||
本计划解决“真实 SIP 暂未接通之外，现有功能还需补齐什么、怎样证明修好”。真实线路排障不阻塞本地整改，不能据此把其余功能视为完成。
|
||||
|
||||
- [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)仍是字段、路径、权限、状态和消息语义的唯一维护来源；本文不新增业务接口或状态枚举。
|
||||
- [最终计划](最终开发部署监控与验收计划_v1.0.md)仍规定 D01～D12、M1～M3/R1～R3、V01～V14及测试数值；本文是其审查整改补充，细化缺陷、依赖和复验方法，不降低原门禁。
|
||||
- 审查发现分为代码缺陷、未实现、仅单元模拟、证据不足。静态发现先写失败回归/独立复现，再修根因；不能把静态推导表述为已发生的生产事故。
|
||||
- 本轮仅在本地 Mock/隔离 SQLite 与契约测试范围内实施代码整改；计划不自动授权云消费、真实 SIP/AI 调用、生产迁移、删除/覆盖数据库、清空队列或资源回收。
|
||||
- 全部执行指令和业务结果继续走 RabbitMQ；HTTP 仅按既有七条路径控制、查询、补传和存储握手。不增加业务回调、应用收讫或拨号 HTTP 接口。
|
||||
- 保留现有可用 outbox、基础轮转和 ASR 代码，按缺陷补齐；不为整改引入通用工作流、供应商框架或无需求微服务。旧 voice_test 仅可复用 ASR，真实 LLM/TTS 必须等待新规范与明确授权。
|
||||
|
||||
## 2. 当前结论与验收纠偏
|
||||
|
||||
1. 已有 SQLite 持久状态、RabbitMQ 适配、HTTP、ASR Web、部署工具及 Mock 测试，但不能据此判定 D01～D11 全部完成。
|
||||
2. SIP/ARI/RTP/AI journal 是单元测试替身，不是协议集成证据；合成静音 WAV 不是真实双方录音。
|
||||
3. 历史 V01～V13 探针 PASS 保留为历史运行结果，不删除原证据；完整用例须按本计划复验，不能沿用脚本 PASS 签署 M3。
|
||||
4. 历史 SCALE-MOCK 仅证明其声明的状态负载观察；缺持续媒体、独立双调度及正常生命周期补位证据，完整 V13 待验。
|
||||
5. 备份文件存在不证明恢复/回滚成功，指标字符串存在不证明告警执行，broker confirm 不证明 SaaS 应用成功。
|
||||
6. 真实 SIP 的 488/cause=88 排障另行推进；R1 真实闭环、R2 1000路完整 AI/N+1、R3 运维交接均不得由本计划的 Mock 结果替代。
|
||||
|
||||
## 3. 执行阶段与退出门禁
|
||||
|
||||
以下全部初始为待执行。优先级表示实施顺序；越权、双拨、丢事实、假终态/ready、串线和模式误标属于阻断问题，不因被排到后续阶段而允许上线。
|
||||
|
||||
| 阶段 | 修复包 | 前置与交付 | 退出条件 |
|
||||
| --- | --- | --- | --- |
|
||||
| F0 基线与止误报 | FIX-01、FIX-23 | 固定版本/profile；纠正模式装配与验收报告；建立缺陷用例清单 | 不支持的 real/mixed 失败关闭；未测不得输出完整 PASS |
|
||||
| F1 业务正确性 | FIX-02～08、FIX-10～13 | F0；复合键迁移、真实测试 MQ、控制/额度/恢复、HTTP契约 | 安全、幂等、背压、双实例竞争与活动执行恢复用例通过；M1相关门禁复验 |
|
||||
| F2 结果与资产可靠性 | FIX-09、FIX-16、FIX-18～20 | F1 的持久键与权限；先用真实测试字节隔离验证 | 补传、最终文字、上传续签/幂等/崩溃恢复通过；无重复资产或虚假ready |
|
||||
| F3 协议语音闭环 | FIX-14～15、FIX-17 | F1准入/恢复，F2结果链；真实本地Asterisk及独立协议Mock | 双向媒体、AI流式/打断、实际混音录音贯通；M2相关门禁复验 |
|
||||
| F4 运维与综合验收 | FIX-21～23及V01～V13 | 功能阶段通过；兼容迁移、故障环境、固定镜像 | 告警/恢复/回滚及完整模拟规模证据齐全后才签M3 |
|
||||
| F5 真实替换 | 原D12 / R1～R3 | 供应商新规范、预算、账号、白名单及授权齐备 | 分项真实闭环、容量及交接签字，不自动创建资源或调用收费服务 |
|
||||
|
||||
允许独立模块并行，但数据库模型/迁移单一写入负责人；不得并行覆盖同一工作树。每包先落可失败的测试，再最小修复，再独立复核。工时和日历交期在失败复现及环境盘点后登记，不用静态审查臆造承诺。
|
||||
|
||||
## 4. 修复任务与逐项验收
|
||||
|
||||
### 4.1 模式、租户和传输
|
||||
|
||||
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
|
||||
| --- | --- | --- | --- |
|
||||
| FIX-01 `core.py:688–739` | 按实际组件装配校验模式；未实现的真实适配器明确拒绝；缺组件/未知枚举不得回退内存broker或Mock | 全部provider伪写real、缺provider、mixed启用未实现组件均启动失败且不接单；合法Mock正常运行；日志/指标逐组件模式与实际实例一致，真实配置不加载测试凭据/故障开关 | V01/V12 |
|
||||
| FIX-02 `core.py:972–989` | 从实际消费队列及AMQP交付属性取得可信路由上下文，在受理前与正文/注册精确比对 | 经A路由发B正文、伪造header、错绑定均隔离，无业务拨号、无向错误租户发送结果；正确中文/点号/星号/井号key原值往返；224/225 UTF-8字节边界不截断，超限保留发布方原任务 | V01/V05 |
|
||||
| FIX-03 `core.py:180–210、335–336、1079–1138` | commands/executions/inbox及相关查询/关联统一租户作用域；审计其他实体键；迁移保留现有事实 | A/B使用相同command/execution/event ID互不干扰；同租户同ID重投10次只保留原业务事实，同execution换command不新增拨号；异内容拒绝且原状态/call关联不变；跨租户查询/控制/资产/补传不可见；迁移前后计数、关联与屏障校验一致 | V01/V02 |
|
||||
| FIX-04 `core.py:545–553` | 实际RabbitMQ配置拒绝发布溢出、消息数/字节上限、发布背压、可靠隔离；既有队列明确迁移而非删队列重建 | 真broker分别触发条数/字节满：新发布失败、原队头及已确认消息完整；mandatory return、NACK、confirm丢失、blocked及DLQ故障可追踪；原ID有限重试不双拨；恢复回原租户域。单节点功能与3节点quorum/可靠死信故障证据分列 | V05 |
|
||||
|
||||
### 4.2 调度、控制和恢复
|
||||
|
||||
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
|
||||
| --- | --- | --- | --- |
|
||||
| FIX-05 `core.py:2191–2205、2296–2339` | 同任务在途控制串行化，持久固定目标revision；applied必须有各Cell屏障证据 | pause尚未applied时resume返回409且不递增；同键重试返回原目标；pause/resume/stop与waiting、已提交意图、FALLBACK竞争，屏障后无新发起；失联保持applying/reconciling；stop不可resume，hangup额外鉴权且确认通道结束 | V04 |
|
||||
| FIX-06 `core.py:1244–1271、1391–1508` | 原子核验调度所有权/隔离令牌、全部额度、授权/控制并预留；执行侧在真实发起前复核许可 | 两个独立进程共享DB，A在检查后停顿超过租期，B接管占最后额度，A恢复不得发起；在提交后/INVITE前注入暂停、到期、失权均不能越过许可；租户/供应商/Cell/AI及端口额度均不超限；FALLBACK计入CPS，未知占用不释放 | V03/V04/V07 |
|
||||
| FIX-07 `core.py:962–1015` | 读取前按共享持久状态限制每租户/全局未ACK和待发起窗口；恢复也公平；扫描有界 | 资源耗尽持续投递，超窗消息留原队列而非全部ACK到DB；双实例总窗口不翻倍；重启/积压/新租户下达到第5节公平与截止目标，重投不延长deadline，不为公平挂断活动通话 | V03/V05 |
|
||||
| FIX-08 `core.py:782–813、1568–1569` | 启动扫描未终结意图并对账；ARI断连隔离Cell且保留未知占用；查询/MQ保持契约状态 | 在意图提交后未发出、已发出未确认、接通后分别杀进程；重启无自动重拨；远端通话仍存活时断ARI不得写ended或释放额度；重连按真实通道收敛，确认结束后仅一次业务终态和资源释放；持久事实/outbox未丢失 | V02/V07/V10/V12 |
|
||||
| FIX-09 `core.py:2380–2522` | 补传持久进度、有限重试与状态/outbox一致；恢复只处理未确认部分 | 第N条发布失败，断连/重启后原操作可恢复；耗尽则查询和MQ一致报告failed并告警，受控恢复不换截止点；已确认部分不丢，可能重投保留原event ID/内容；后生事件及补传自身结果不入范围；分批限速、不拨号、不调用AI/重传录音；completed不代表SaaS应用 | V10 |
|
||||
|
||||
### 4.3 HTTP、权限和机器契约
|
||||
|
||||
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
|
||||
| --- | --- | --- | --- |
|
||||
| FIX-10 `http.py:198、216` | 两条补传路径使用outbound.replay权限 | control-only拒绝；replay-only对授权资源成功；跨租户404；缺凭据401、缺权限403，不产生未授权业务事件 | V01/V10 |
|
||||
| FIX-11 `http.py:234–248` | 两个存储POST校验并持久化调用方/租户/操作/幂等键及语义摘要 | 缺头拒绝；同键同内容重试/重启返回原操作；异内容409；不同租户/调用方按作用域独立；JSON字段排序不造成冲突，不因重试生成第二资产 | V01/V09 |
|
||||
| FIX-12 `core.py:2279–2295、2388–2405` | 控制/补传幂等语义纳入操作类型与路径目标 | 对A已受理后，同键同body改为B目标返回409，不返回A成功冒充B；原目标重试稳定；控制与两种补传之间不错误互相命中 | V01/V04/V10 |
|
||||
| FIX-13 `core.py:2248–2255、1911–1927`；两份OpenAPI | 修复ID正则转义、布尔当整数、reason上限、音频元数据等；解析执行Schema而非检查文本关键字 | 主契约合法样例含task-demo及字母s可通过；禁止字符/首尾空白按契约拒绝，不擅自限制tenant_key；true作为revision、超长reason、错误格式/声道/采样率/时长拒绝；7条路径请求/响应和MQ正反例、未知主版本、64KiB/256KiB边界均有测试；Schema与运行端一致 | V01 |
|
||||
|
||||
### 4.4 协议媒体、AI和文字
|
||||
|
||||
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
|
||||
| --- | --- | --- | --- |
|
||||
| FIX-14 `mocks.py:40–88`；`core.py:1508–1566` | journal降为单元替身；实现Cell长期ARI、SIP/RTP协议集成和真实媒体清理 | 隔离真实Asterisk+独立Mock SIP对端，至少两通并发不同标识音；双向收包/解码证明无串线，振铃/接通有协议证据；单线路无需虚构backup；正常结束30s内无本次通道/桥/端口残留；断连按FIX-08；明确未接通且原通道结束才允许合法FALLBACK，迟到接通/不确定时禁止重拨 | V06/V07 |
|
||||
| FIX-15 `core.py:1571–1592`；`mocks.py:91–105` | 音频输入→分段ASR→新LLM/TTS流式Mock→PCM输出；真实ASR入口独立适配；VAD、取消、超时不靠自填成功 | 至少100有效轮次、合成输入与独立输出校验；首包超时、客户静音、插话、取消后旧包迟到均有明确结果；旧音频不进桥，超时不伪报正常完成；播放标记来自实际播放器能力，无确认只能sent/unknown；达到第5节时延；无旧LLM/TTS或收费外联 | V08 |
|
||||
| FIX-16 `core.py:1671–1681` | 最终文字同内容幂等、异内容冲突，不允许迟到中间稿覆盖最终稿 | 同段最终稿重投10次不新增业务版本/事实；同段异内容拒绝并保留原文；乱序分段各自收敛，不用全局版本丢其他段；最终稿持久化/投递故障可恢复，无法完成明确transcript.failed，不假称完整 | V08/V10 |
|
||||
|
||||
### 4.5 录音与资产交接
|
||||
|
||||
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
|
||||
| --- | --- | --- | --- |
|
||||
| FIX-17 `core.py:1780–1784、1849–1858` | 接通后稳定recording_id，真实单轨混音；正常/强挂/故障确认结束均封口并排持久上传 | 双方不同标识音在录音中可辨，时长与实际音频样本/接通边界吻合，非全零占位；至少正常结束、强挂、媒体异常恢复三条路径；未接通不伪造接通录音；无法恢复资产显式failed；录音处理不阻塞call.finished | V06/V09/V11 |
|
||||
| FIX-18 `core.py:1835–1845、2068–2090` | 经独立SaaS HTTP授权/complete和对象Mock传输真实字节，而非同对象内部调用冒充对接 | 使用独立服务/凭据与同一OpenAPI；服务端独立读字节验大小/摘要；缺对象、错绑定/摘要、覆盖均不ready；上传目标白名单、HTTPS/重定向与SSRF保护；确认后才持久verified+ready outbox；SaaS inbox后关联oss_id并授权读取 | V09 |
|
||||
| FIX-19 `core.py:642–657、1991–2059` | complete幂等及文件移动/数据库提交跨边界恢复 | 成功响应丢失后原键重试返回同oss_id/verified_at；移动文件后提交前崩溃可通过最终文件重新校验恢复；重复确认不生成第二份资产/业务ready；异内容保持冲突；未verified不产生ready | V09/V10 |
|
||||
| FIX-20 `core.py:1928–1934、1977–2006` | 原会话受控续签、上传持久恢复扫描、有限退避 | 跨300s有效期，旧授权拒绝，新授权保持upload/recording/object和内容绑定；中断5分钟后恢复，10分钟内补齐固定测试数据；重启不丢封口文件；耗尽明确failed/可恢复性并告警，文件不静默删除；签名/密钥不出现在日志或MQ | V09/V10/V11 |
|
||||
|
||||
### 4.6 运维、恢复和验收可信度
|
||||
|
||||
| 编号/定位 | 修复范围 | 必测场景与通过标准 | 原门禁 |
|
||||
| --- | --- | --- | --- |
|
||||
| FIX-21 `core.py:2669–2675`；`deploy/prometheus.yml`/`alerts.yml` | readiness反映持久写、可路由及必要配置/许可/磁盘；加载规则，修复模式/序列缺失告警 | DB可读不可写、缺绑定、ARI断连、凭据失效、磁盘水位、时钟跳变分别注入；不安全新发起停止，readiness和范围与故障一致，不无差别切断正常活动媒体；promtool语法/规则测试及实际告警产生→接收→恢复有证据；模式错误/指标缺失可检测 | V12 |
|
||||
| FIX-22 `deploy/backup_sqlite.py`；`acceptance.py:371–397` | 恢复默认新目标，拒绝覆盖现有业务库；兼容迁移、排空/回滚、保留与水位保护 | 从备份恢复到新库实际启动并核对租户、执行、最终稿、outbox、录音关联；应用回滚保留较新DB/队列/卷，旧命令重投无重拨、stop屏障仍在；禁止用旧快照覆盖新防重事实；保留卷故障无已提交事实损失；永久磁盘丢失明确损失/failed/告警，不宣称录音RPO=0；未确认资产/屏障不按普通TTL删除 | V11/V12 |
|
||||
| FIX-23 `acceptance.py:250–424`；`scale_mock.py` | 用例分层、独立断言、机器门禁与文档一致；完整V13重测 | journal/静音/备份存在仅能作为对应子项结果；缺协议/恢复/告警证据时完整项不PASS；实际失败或所需证据缺失返回非零；保留历史证据并追加纠偏；V13按第5节执行，状态对象数量不得当媒体并发；真实V14始终独立 | V01～V14 |
|
||||
|
||||
上表 `core.py`、`http.py`、`mocks.py` 均位于 `agent_call/`，`acceptance.py`、`scale_mock.py` 位于 `scripts/`。总计23项，ARI断连相关发现合并到FIX-08，避免重复计算完成率。
|
||||
|
||||
## 5. 测试环境、基线与独立证据
|
||||
|
||||
### 5.1 分层，不互相冒充
|
||||
|
||||
| 层级 | 实际依赖 | 能证明什么 |
|
||||
| --- | --- | --- |
|
||||
| L0 单元/契约 | 临时真实DB、可控时钟/故障钩子、内存替身 | 分支和语义正确；不能证明AMQP、ARI或媒体协议 |
|
||||
| L1 持久/MQ/HTTP集成 | 真实隔离DB、RabbitMQ、独立SaaS/对象Mock、两个调度进程 | 事务、隔离、权限、竞争、背压和重启恢复；不等于真实供应商 |
|
||||
| L2 协议媒体集成 | 真实隔离Asterisk、独立SIP/RTP对端、流式AI Mock | 实际通道/双向音频/取消/录音/结果闭环；不是生产AI质量 |
|
||||
| L3 综合/规模 | L1+L2、故障/监控组件、SCALE-MOCK profile | M3/V13适用门禁；报告实际物理资源和模拟部分 |
|
||||
| L4 真实 | 获授权SaaS/SIP/ASR/LLM/TTS/OSS与云出口 | R1～R3，单独取得供应商、容量和运维证据 |
|
||||
|
||||
数据库先复用当前SQLite真实持久能力做回归；必须用独立进程验证其实际协调范围，不以同进程锁代替多实例证据。SQLite同宿主测试不证明多机共享数据库方案；生产数据库与部署拓扑确定后，迁移/并发/故障用例必须在目标引擎重跑。若为跨主机能力需要更换引擎，登记明确迁移方案，不以网络共享SQLite文件规避验证。
|
||||
|
||||
L1起使用真实测试broker；HA/可靠死信另建隔离3节点quorum环境。Mock网络默认阻止真实公网SIP/AI/云API；只使用合成音频和授权测试数据，不以“假号码”替代网络隔离。
|
||||
|
||||
### 5.2 沿用最终计划的工程测试阈值
|
||||
|
||||
下列为DEV/SCALE-MOCK工程基线，不是生产SLA或消费授权。完整取值及恢复/保留条件以最终计划第5节为准；修改profile须版本化并重跑相关门禁。
|
||||
|
||||
- DEV：3租户、2独立调度实例、2Cell；每租户并发2/CPS1，全局并发6/CPS3；每Cell测试容量4。
|
||||
- 所有实例汇总：每租户未ACK≤4/持久待发起≤16，全局≤32/≤64；每轮每租户最多1个新许可。队列上限1000条或16MiB先到者，每租户发布≤10条/s。
|
||||
- 公平：发现新活跃租户≤1s；持续可调度且资源足够时B/C发现后≤2s获首次许可；至少100次许可中3租户份额偏差≤10个百分点。资源不足单列，不以强挂达标。
|
||||
- 首次准入窗口30s，到期处理≤1s；重启/重投不延期。心跳2s、租约10s，失权即停止本地新发起；时钟偏差>500ms告警、>2s停止新准入。
|
||||
- 控制受理到全部屏障applied P95≤2s，前提为节点健康且无未决发起；失联样本不得伪算成功，单列未决。
|
||||
- ≥100有效AI轮次：VAD语音结束到首有效TTS入桥P95≤1500ms，插话到停止旧TTS入桥P95≤500ms；首输出超时5s、客户静音15s；实际Mock延迟和测量时间点单列。
|
||||
- 最终稿到SaaS Mock事务应用P95≤3s；≤180s通话挂断到对象verified、ready消费及授权读取≤120s；正常结束30s内清理本次媒体资源。
|
||||
- 消费/上传中断5分钟，恢复后10分钟内补齐固定负载；DB/MQ恢复后60s内恢复安全调度，未知通话仍保留对账占用。
|
||||
- 测试卷使用率70%告警、80%停止新接单，降至60%且依赖正常后恢复；预留活动录音剩余空间。补传每批≤100事件、全局≤50事件/s，实时结果优先。
|
||||
- V13：至少2调度实例、100租户×并发12、全局1200，拨号CPS20；预热后≥1000模拟接通连续60分钟，通过≤180s正常生命周期和新授权持续补位，不延长单批通话伪造持续负载。明确模拟Cell/AI额度，提供持续媒体与数据证据。运行中故障隔离一个Cell、旧活动不自动接管/重拨，在健康容量内用新授权恢复负载，再记录恢复稳态；故障影响窗口单列，不隐藏后算连续达标。
|
||||
- R2仍要求≥1000同时已接通的真实完整ASR/LLM/TTS通话稳定≥60分钟，以及实测N+1安全容量、真实CPS/RTP/AI/MQ/OSS证据。V13不能替代。
|
||||
|
||||
### 5.3 证据独立性与并发方法
|
||||
|
||||
- 每个FIX至少一个修复前失败、修复后通过的可运行用例；保存命令、退出码和原始结果。不可仅检查日志里存在由被测逻辑自行填入的成功字段。
|
||||
- 并发故障使用屏障/故障钩子稳定卡在事务、确认、发起、文件移动等边界；独立进程竞争，不用顺序调用两实例代替争抢。
|
||||
- MQ证据取broker确认/队列/投递及DB事务；媒体证据取对端收包、解码样本与ARI通道；资产证据独立读上传文件、核摘要与授权读取；应用证据取SaaS inbox/业务库。
|
||||
- 至少覆盖DB提交前后、ACK前后、发起前后、confirm前后、文件移动与verified提交之间的中断。所有不确定结果保留原ID和事实，不靠重拨、删队列或删库消除。
|
||||
- 故障测试固定重复3轮；每轮重建独立测试命名空间并记录种子，不覆盖生产。一次偶发失败即该项失败，不挑成功轮次签字。3轮是本整改测试重复次数，不是可靠性概率保证。
|
||||
|
||||
## 6. 执行记录、交付与签署
|
||||
|
||||
每个FIX的执行记录按下表追加到 `docs/evidence/`。此处仅定义未来产物，不表示已有文件或测试命令。
|
||||
|
||||
| 字段 | 必填内容 |
|
||||
| --- | --- |
|
||||
| 身份 | FIX编号、关联V编号、代码SHA、契约/profile版本、镜像digest、DB/broker版本 |
|
||||
| 责任 | 实施人、独立复核人、运维协作人；启动前填写真实负责人，未填不视为已分派 |
|
||||
| 前置 | L0～L4层级、实际provider清单、网络隔离、配额、种子、数据/时间范围 |
|
||||
| 执行 | 可重复命令/退出码、输入、预期、实际、故障时刻/恢复时刻、重复轮次 |
|
||||
| 证据 | 脱敏日志、DB/inbox/outbox核对、broker/协议记录、文件摘要、指标样本分母和分位数、告警接收/恢复 |
|
||||
| 结论 | 待执行/通过/失败/阻塞/不适用；不适用必须给契约依据与批准，不得用来豁免缺失环境 |
|
||||
| 剩余项 | 未决记录数、外部阻塞、受影响门禁、恢复措施、下一责任人 |
|
||||
|
||||
交付包：最小修复代码及迁移、逐FIX测试、版本化隔离profile、部署/恢复/告警运行步骤、V01～V13复验矩阵、历史证据纠偏说明、R阶段阻塞清单。新增执行命令必须先实现并验证，再写入运行说明。
|
||||
|
||||
现有回归入口（执行时仍须审阅环境，不能视为本次已运行）：
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s tests -v
|
||||
(cd services/asr-web && go test -race ./... && go vet ./...)
|
||||
node --test tests/test_pcm.cjs
|
||||
bash -n deploy/asterisk.sh
|
||||
git diff --check
|
||||
```
|
||||
|
||||
以上回归不替代L1～L4故障/协议/规模用例；增加的集成测试需提供其真实可运行入口。告警规则用promtool做语法和规则测试，并仍须验证实际接收链路。
|
||||
|
||||
### 最终通过条件
|
||||
|
||||
1. FIX-01～23逐项完成复现、修复、独立复核，所有阻断项关闭；共享数据库/队列配置迁移有无损证据，不覆盖用户既有修改。
|
||||
2. M1/M2/M3按原定义分别签署；V01～V13完整范围具备独立证据，缺环境或缺规模则保持对应门禁未完成，允许声明已通过的子集。
|
||||
3. 无未经授权双拨、串线、跨租户污染、假终态/播放确认/ready或静默丢事实；出现任一项立即停止扩量，保留现场，修复后重跑相关门禁。
|
||||
4. 模式和依赖按实际装配展示；文档/机器报告/实现一致，不将Mock成功描述为真实供应商启用。
|
||||
5. R1～R3单独验收。SIP供应商、真实LLM/TTS规范、OSS/身份/预算、生产保留与RPO/RTO未落实时，登记外部阻塞，不擅自补参数、不把未验项改为通过。
|
||||
|
||||
## 7. 本轮执行记录与剩余门禁
|
||||
|
||||
已按本文的代码级要求实施最小整改：
|
||||
|
||||
- `agent_call/core.py`：模式/依赖拒绝、可信租户路由、租户复合键迁移、发布保留与重试、调度租约/额度、控制Cell屏障、重启对账态、补传进度、格式校验、最终稿幂等、稳定录音与上传恢复、readiness/metrics。
|
||||
- `agent_call/http.py`：补传专用权限、调用方/租户/操作/幂等键持久化、请求体与上传边界校验；`agent_call/mocks.py`：非静音合成PCM及协议事实journal；`deploy/backup_sqlite.py`、`deploy/prometheus.yml`、`deploy/alerts.yml`：非覆盖恢复与规则加载/水位告警。
|
||||
- `docs/contracts/*.yaml/json` 与 Mock profile 已同步修正；新增 `tests/test_remediation_plan.py` 及 HTTP replay 权限回归。
|
||||
- 已运行 `python3 -m unittest discover -s tests -v`（52项通过）、Python编译、JSON/YAML解析、`ruff check`、`git diff --check`；`scripts/acceptance.py` 的 V01～V12 通过，V13 因缺持续媒体独立证据保持 `NOT_RUN`，总门禁为 `INCOMPLETE`。
|
||||
- 2026-09-13 在明确授权后重新创建并核验单路真实 Cell：ECS `i-2ze4hvi6xvy02g0wz9kk`、固定 EIP `123.56.71.98`、Asterisk `22.10.1`、PJSIP contact `Avail`；对 `18601013734` 单次构造 `708918601013734`，收到 `100 trying` 后 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`，无接通/RTP/录音证据，详见 `docs/evidence/asterisk-persistent-2026-09-13.md`。
|
||||
|
||||
上述结果只证明本地代码/契约层子集及真实单路底座/失败回执，不替代第5节的L1～L4证据。真实接通/RTP、ASR/新LLM/TTS、独立SaaS/OSS、3节点可靠MQ、实际告警收发、V13及R1～R3完整验收仍未完成，不得签署M2/M3或生产ready。
|
||||
@@ -9,7 +9,7 @@
|
||||
- 本文统一规定开发顺序、Mock 策略、运行基线、部署、监控和验收门禁；取代旧计划中的阶段安排及“需再次拍板方案”的表述。
|
||||
- [主契约](SaaS交互_OpenAPI与MQ契约规划_v0.1.md)是标识、HTTP 七条路径、MQ 拓扑/字段/状态的唯一维护来源。本文件不复制接口表；先生成 OpenAPI/JSON Schema，再编写 Mock 和正式实现。
|
||||
- [原一期计划](一期呼出应用开发计划_v1.0.md)保留 WBS/AT 编号和历史估算；[交付设计](交付文档/一期中间调度件与Asterisk_v1.0/01_中间调度件与MQ回传设计.md)、[部署细则](交付文档/一期中间调度件与Asterisk_v1.0/02_Asterisk部署与SIP对接步骤.md)、[专项用例](交付文档/一期中间调度件与Asterisk_v1.0/03_联调步骤与验收标准.md)保留实现与场景细节，冲突时以本文件及主契约为准。
|
||||
- 当前已有 ASR Web、ASR 协议适配、Asterisk 配置生成、阿里云只读计划/受控创建工具和离线测试；参见[现有运行说明](部署接入_运行说明.md)。当前尚无完整调度、AI 通话、MQ/OSS 闭环或本计划的 Mock 套件，不能使用未来产物的命令假装已可运行。
|
||||
- 当前已有 ASR Web、ASR 协议适配、Asterisk 配置生成、阿里云只读计划/受控创建工具，以及执行服务、持久状态、MQ/HTTP 和 Mock 测试产物；参见[现有运行说明](部署接入_运行说明.md)。2026-09-13静态审查发现隔离、控制、恢复、媒体与验收缺口，完整调度/AI通话/MQ与资产交接仍须整改复验，不能把已有产物等同于完整门禁通过。整改执行见[功能审查整改计划与验收标准](功能审查整改计划与验收标准_v1.0.md)。
|
||||
- 本次只交付文档，不新增运行服务、不拨号、不创建云资源、不提交 Git。旧 Word/Excel/ZIP 不覆盖重生成。
|
||||
|
||||
## 2. 已接受的最终方案
|
||||
@@ -289,7 +289,7 @@ M3产物 + C1～C6真实配置/权限（受控存放）+ R1/R2全部证据 + 实
|
||||
|
||||
## 10. 本次文档验收与后续入口
|
||||
|
||||
本轮已完成文档引用、契约/实现一致性、样例完整性、Mock开发回归及V01～V13验证；V14/R1/R2/R3仍因真实外部证据待执行。当前回归命令可继续使用，但不能把Mock结果当作生产验收：
|
||||
2026-09-13审查纠偏：历史脚本的V01～V13 Mock探针PASS只证明其实际覆盖的子项，不能证明完整契约/实现一致性、协议媒体、恢复、监控及模拟规模门禁通过。M1/M2/M3须按[整改计划](功能审查整改计划与验收标准_v1.0.md)逐项修复并复验；V14/R1/R2/R3另需真实外部证据。本次静态审查和文档更新未运行下列回归，现有命令可继续使用，但不能替代集成/故障验收：
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s tests -v
|
||||
@@ -298,12 +298,18 @@ node --test tests/test_pcm.cjs
|
||||
bash -n deploy/asterisk.sh
|
||||
```
|
||||
|
||||
下一实施入口是 D12/R1：在取得真实SaaS、SIP/Asterisk、ASR/LLM/TTS、OSS、出口、容量和运维证据后逐项替换Mock并复验；无需再次讨论已接受的A类方案。遇到真实供应商协议/合规/资源事实不兼容，只对具体差异登记变更，不把整个计划退回重选，也不让Mock假成功掩盖差异。
|
||||
下一实施入口是整改计划F0→F1～F4，对D01～D11补齐模式保护、租户隔离、可靠性、协议媒体与验收证据；真实SIP排障及D12外部协调并行，不以线路不通阻塞本地整改。真实替换仍须取得SaaS、SIP/Asterisk、ASR/LLM/TTS、OSS、出口、容量和运维证据；无需重新讨论已接受方案。具体供应商差异单独登记，不让Mock假成功掩盖缺口。
|
||||
|
||||
### 10.1 当前执行记录（2026-09-12）
|
||||
### 10.1 历史执行记录（2026-09-12～13，非当前完整验收结论）
|
||||
|
||||
以下保留历史探针及当时环境记录；其中V01～V13、规模与环境保持运行的表述仅描述当次结果，不表示当前资源仍存在。完整门禁状态以第10节纠偏和整改复验为准，当前云资源状态须另行只读核验，不能据本记录自动复用、创建或拨号。
|
||||
|
||||
已落地 D01 的 OpenAPI 3.1、共用 MQ Schema、集中样例和 Mock profile，以及 D02/D03 的 SQLite 持久状态、租户独立 RabbitMQ 拓扑、outbox/inbox、CAS 控制、重放、Mock SIP/ARI/RTP/AI/OSS 协议记录、监控/告警、迁移/备份和 Compose 部署入口。测试号码 `18601013734` 在 Mock 与明确授权的 R1 探针中均保持原值；当前线路目标按规则生成 `708918601013734`。
|
||||
|
||||
本轮可重复命令 `python3 -m unittest discover -s tests -v` 和 RabbitMQ Compose smoke 已通过；`python3 -m scripts.acceptance` 的 V01～V13 Mock 探针通过，因 V14 仍需外部真实证据而按门禁返回非零，证据归档于 `docs/evidence/acceptance-mock-2026-09-12.json`。SCALE-MOCK 在阿里云北京专用竞价 Cell `ecs.hfg6.large` 上完成：100 租户、1200 模拟呼叫、禁用1个Cell，最低1004路已回答并连续观察3600秒；详见 `docs/evidence/scale-mock-2026-09-12.json`。该证据仅证明 Mock 协议/持久状态负载基线，`continuous_media_evidence` 为 false，不替代真实媒体容量。
|
||||
|
||||
R1 已在持久测试环境完成两次真实 Asterisk/SIP 单路探针：分别使用原始被叫 `18601013734` 和 `18625770806`，均按 `7089` 前缀构造目标。环境 ECS `i-2zegaldd07mkp0xyhcff` 使用 `SpotAsPriceGo` 自动竞价和固定 EIP `123.56.71.98`；PJSIP contact/OPTIONS 可达，真实 INVITE 已送达供应商，两次均收到 `488`、`Q.850 cause=88 INCOMPATIBLE_DESTINATION`，未接通、未建立 RTP，证据见 `docs/evidence/asterisk-persistent-2026-09-13.md`。环境按用户要求保持运行，未删除实例/EIP。真实接通/RTP、ASR/LLM/TTS/OSS、生产 1000 路完整 AI、N+1、实际CPS/SLO及副作用正确性仍阻塞，现有产物不得据此宣称生产完成。
|
||||
R1 已在历史持久测试环境完成两次真实 Asterisk/SIP 单路探针：分别使用原始被叫 `18601013734` 和 `18625770806`，均按 `7089` 前缀构造目标。历史环境 ECS `i-2zegaldd07mkp0xyhcff` 使用 `SpotAsPriceGo` 自动竞价和固定 EIP `123.56.71.98`；PJSIP contact/OPTIONS 可达，真实 INVITE 已送达供应商，两次均收到 `488`、`Q.850 cause=88 INCOMPATIBLE_DESTINATION`，未接通、未建立 RTP。
|
||||
|
||||
补充当前执行（2026-09-13）：按明确授权重新创建并绑定 ECS `i-2ze4hvi6xvy02g0wz9kk`，探针期间固定 EIP 为 `InUse`；Asterisk `22.10.1` 容器为 `running/healthy`，固定镜像 digest、PJSIP contact `Avail`（RTT约22.164ms）及安全组范围已只读核验。对原始被叫 `18601013734` 单次探针构造 `708918601013734`，收到 `100 trying` 后 `488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`，无 `180/183/200`、RTP 或录音证据；无重试、无 FALLBACK。探针后按用户授权停止并删除该 ECS，系统盘随实例删除，固定 EIP 未释放且已恢复 `Available`；详见 `docs/evidence/asterisk-persistent-2026-09-13.md`。
|
||||
|
||||
真实接通/RTP、ASR/LLM/TTS/OSS、生产 1000 路完整 AI、N+1、实际CPS/SLO及副作用正确性仍阻塞，现有产物不得据此宣称生产完成。
|
||||
|
||||
+3
-3
@@ -238,14 +238,14 @@ python3 deploy/migrate.py data/agent-call.sqlite3
|
||||
|
||||
### 7.3 当前门禁
|
||||
|
||||
本地V01-V12及阿里云专用主机上的V13 Mock探针已通过；完整验收脚本仍因V14需要外部真实证据而按门禁返回非零，证据见`docs/evidence/acceptance-mock-2026-09-12.json`和`docs/evidence/scale-mock-2026-09-12.json`。V13使用100租户/1200模拟接通、禁用1个Cell，最低1004路已回答并持续观察3600秒；这是Mock协议/持久状态负载证据，不是RTP或生产容量证据。R1临时探针见`docs/evidence/asterisk-real-2026-09-12.md`，持久环境两号码探针见`docs/evidence/asterisk-persistent-2026-09-13.md`：固定EIP和SIP OPTIONS/INVITE链路可达，但两个目标均收到488/Q.850 cause=88。当前持久测试ECS保持运行，使用SpotAsPriceGo，固定EIP`123.56.71.98`保持InUse。真实接通/RTP、ASR/LLM/TTS/OSS、生产1000路/N+1尚未完成。正式模式拒绝当前Mock profile，不能把测试号码或Mock成功自动升级为真实外呼授权。
|
||||
本地V01-V12及阿里云专用主机上的V13 Mock探针已通过；最新完整验收结果见`docs/evidence/acceptance-mock-2026-09-13.json`，脚本仍因V13缺持续媒体独立证据和V14需要外部真实证据而按门禁返回非零。V13使用100租户/1200模拟接通、禁用1个Cell；这是Mock协议/持久状态负载证据，不是RTP或生产容量证据。R1历史探针见`docs/evidence/asterisk-real-2026-09-12.md`，当前单次探针见`docs/evidence/asterisk-persistent-2026-09-13.md`：ECS `i-2ze4hvi6xvy02g0wz9kk`、Asterisk容器和固定EIP `123.56.71.98` 已核验，PJSIP contact/OPTIONS可达，但 `18601013734` 对应目标收到 `488/Q.850 cause=88`，未接通、无RTP/录音证据。探针后该 ECS 已按授权停止并删除，固定 EIP 保持 Available。真实接通/RTP、ASR/LLM/TTS/OSS、生产1000路/N+1尚未完成。正式模式拒绝当前Mock profile，不能把测试号码或Mock成功自动升级为真实外呼授权。
|
||||
|
||||
## 8. 下一步需要用户提供/确认
|
||||
|
||||
1. 供应商需确认`7089`前缀/目标号码路由、From域（当前实际出现私网域）及允许的编解码；两个持久环境探针均返回`488/Q.850 cause=88 INCOMPATIBLE_DESTINATION`，未通过外呼验收。
|
||||
2. 供应商调整后在当前持久测试环境复测；环境使用SpotAsPriceGo，竞价回收可能中断通话，不自动重试、不改绑或释放固定EIP。
|
||||
2. 供应商调整并确认线路规则后，再用只读plan和新授权创建测试实例复测；本次实例已回收，固定EIP保持 Available，不自动重试、不改绑或释放固定EIP。
|
||||
3. ASR测试凭据、批准的模型/资源ID；Web HTTPS域名/证书或SSH访问方案。
|
||||
4. 用户制定的MQ/OSS ID接口规范，以及新的LLM/TTS协议、参数与取消/打断规则。
|
||||
5. 真实接通后仍需分别完成RTP、录音、MQ/OSS、ASR/LLM/TTS和生产1000路/N+1验收。
|
||||
|
||||
在供应商规则确认前，不再次拨打真实号码，也不声称完成整个平台；当前持久测试环境按用户要求保留。
|
||||
在供应商规则确认前，不再次拨打真实号码，也不声称完成整个平台；本次测试实例已回收，固定EIP按要求保留。
|
||||
|
||||
+84
-8
@@ -7,12 +7,15 @@ exists. Passing Mock probes never upgrades the result to production readiness.
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from types import TracebackType
|
||||
from typing import Any, Self
|
||||
|
||||
from agent_call.core import (
|
||||
AgentCallService,
|
||||
@@ -221,11 +224,14 @@ def probe_v04() -> dict[str, Any]:
|
||||
|
||||
|
||||
class ConflictContext:
|
||||
def __enter__(self) -> ConflictContext:
|
||||
def __enter__(self) -> Self:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: Any
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
tb: TracebackType | None,
|
||||
) -> bool:
|
||||
if exc_type is not ConflictError:
|
||||
raise AssertionError("expected revision conflict")
|
||||
@@ -256,10 +262,37 @@ def probe_v06_v07() -> dict[str, Any]:
|
||||
"disconnect": "ari_disconnect",
|
||||
}
|
||||
outcomes: dict[str, str] = {}
|
||||
uncertain_before_reconcile: str | None = None
|
||||
for label, scenario in scenarios.items():
|
||||
body = command(f"cmd_{label}")
|
||||
svc.set_failure(body["command_id"], scenario)
|
||||
snapshot = run_success(svc, body)
|
||||
svc.publish_execute(body)
|
||||
if scenario == "ari_disconnect":
|
||||
call_id = None
|
||||
before = None
|
||||
for _ in range(500):
|
||||
svc.process_once()
|
||||
current = svc.get_command("tenant-demo", body["command_id"])
|
||||
call_id = current.get("call_id")
|
||||
if call_id:
|
||||
before = svc.get_call("tenant-demo", call_id)
|
||||
if before["call_state"] == "reconciling":
|
||||
break
|
||||
time.sleep(0.01)
|
||||
if not call_id or before is None:
|
||||
raise AssertionError("disconnect probe did not create a call")
|
||||
uncertain_before_reconcile = before["call_state"]
|
||||
if uncertain_before_reconcile != "reconciling":
|
||||
raise AssertionError(
|
||||
"disconnect probe did not retain unknown state"
|
||||
)
|
||||
svc.reconcile_call(
|
||||
call_id, ended=True, outcome="failed", reason="ARI_RECONCILED"
|
||||
)
|
||||
snapshot = svc.get_command("tenant-demo", body["command_id"])
|
||||
else:
|
||||
svc.wait_for_idle()
|
||||
snapshot = svc.get_command("tenant-demo", body["command_id"])
|
||||
call = svc.get_call("tenant-demo", snapshot["call_id"])
|
||||
outcomes[label] = call["outcome"]
|
||||
if len(call["attempts"]) != 1:
|
||||
@@ -280,6 +313,7 @@ def probe_v06_v07() -> dict[str, Any]:
|
||||
"asterisk_real": "not_run",
|
||||
"outcomes": outcomes,
|
||||
"fallback_redials": 0,
|
||||
"disconnect_state_before_reconcile": uncertain_before_reconcile,
|
||||
}
|
||||
|
||||
|
||||
@@ -374,9 +408,37 @@ def probe_v11_v12() -> dict[str, Any]:
|
||||
body = command("cmd_backup")
|
||||
run_success(svc, body)
|
||||
backup = Path(directory) / "backup.sqlite3"
|
||||
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup))
|
||||
with redirect_stdout(io.StringIO()):
|
||||
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup))
|
||||
if not backup.is_file():
|
||||
raise AssertionError("backup file was not created")
|
||||
restored = AgentCallService(
|
||||
db_path=backup,
|
||||
object_dir=Path(directory) / "restored-objects",
|
||||
broker=InMemoryBroker(),
|
||||
)
|
||||
try:
|
||||
restored_command = restored.get_command("tenant-demo", body["command_id"])
|
||||
restored_call = restored.get_call(
|
||||
"tenant-demo", restored_command["call_id"]
|
||||
)
|
||||
if (
|
||||
restored_command["status"] != "completed"
|
||||
or restored_call["recordings"][0]["status"] != "verified"
|
||||
):
|
||||
raise AssertionError("restored database lost the completed call facts")
|
||||
restored_counts = {
|
||||
"commands": restored.store.one("SELECT COUNT(*) AS n FROM commands")[
|
||||
"n"
|
||||
],
|
||||
"events": restored.store.one("SELECT COUNT(*) AS n FROM events")["n"],
|
||||
"inbox": restored.store.one("SELECT COUNT(*) AS n FROM inbox")["n"],
|
||||
"recordings": restored.store.one(
|
||||
"SELECT COUNT(*) AS n FROM recordings"
|
||||
)["n"],
|
||||
}
|
||||
finally:
|
||||
restored.stop()
|
||||
metrics = svc.metrics()
|
||||
if "agent_call_outbox_pending 0" not in metrics or 'mode="mock"' not in metrics:
|
||||
raise AssertionError("monitoring metrics lack healthy mock state")
|
||||
@@ -392,7 +454,9 @@ def probe_v11_v12() -> dict[str, Any]:
|
||||
return {
|
||||
"backup": "verified",
|
||||
"metrics": "healthy",
|
||||
"rollback_preserves_db": True,
|
||||
"rollback_preserves_db": "not_run",
|
||||
"restored_db_facts": True,
|
||||
"restored_counts": restored_counts,
|
||||
"real_mode_mock_rejection": True,
|
||||
}
|
||||
|
||||
@@ -419,10 +483,15 @@ def probe_v13() -> dict[str, Any]:
|
||||
if (
|
||||
evidence.get("status") != "observed"
|
||||
or evidence.get("production_claim", True)
|
||||
or not evidence.get("continuous_media_evidence", False)
|
||||
or load.get("minimum_answered_calls", 0) < 1000
|
||||
or load.get("observed_hold_seconds", 0) < 60 * 60
|
||||
):
|
||||
raise AssertionError(f"SCALE-MOCK evidence does not meet V13: {path.name}")
|
||||
return {
|
||||
"status": "not_run",
|
||||
"evidence_file": str(path.relative_to(ROOT)),
|
||||
"reason": "historical state-only evidence cannot satisfy continuous media V13",
|
||||
}
|
||||
return {
|
||||
"status": "observed",
|
||||
"evidence_file": str(path.relative_to(ROOT)),
|
||||
@@ -466,7 +535,14 @@ def run() -> dict[str, Any]:
|
||||
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
||||
"evidence": evidence,
|
||||
}
|
||||
except Exception as exc:
|
||||
except (
|
||||
AssertionError,
|
||||
KeyError,
|
||||
OSError,
|
||||
RuntimeError,
|
||||
ServiceError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
results["probes"][name] = {
|
||||
"status": "FAIL",
|
||||
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
||||
|
||||
+31
-10
@@ -14,7 +14,13 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from agent_call.core import AgentCallService, InMemoryBroker, iso, utcnow
|
||||
from agent_call.core import (
|
||||
AgentCallService,
|
||||
InMemoryBroker,
|
||||
ServiceError,
|
||||
iso,
|
||||
utcnow,
|
||||
)
|
||||
from scripts.acceptance import load_fixture
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -44,6 +50,8 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
|
||||
{
|
||||
"cell_id": f"cell-{index:03d}",
|
||||
"capacity": 12,
|
||||
"media_capacity": 12,
|
||||
"ai_capacity": 12,
|
||||
"egress_pool_id": "egress-mock",
|
||||
"ari_mode": "mock",
|
||||
}
|
||||
@@ -53,10 +61,14 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
|
||||
profile["limits"].update(
|
||||
{
|
||||
"global_concurrency": 1200,
|
||||
"global_cps": 1200,
|
||||
"global_cps": 20,
|
||||
"tenant_concurrency": 12,
|
||||
"tenant_cps": 12,
|
||||
"max_queue_messages": 2000,
|
||||
"tenant_cps": 1,
|
||||
"max_queue_messages": 1000,
|
||||
"max_queue_bytes": 16 * 1024 * 1024,
|
||||
"pending_window_per_tenant": 16,
|
||||
"pending_window_global": 64,
|
||||
"max_unacked_per_tenant": 4,
|
||||
# Keep calls alive through warm-up and the required observation window.
|
||||
"hold_ms": (startup_window_seconds + duration_minutes * 60 + 5) * 1000,
|
||||
}
|
||||
@@ -101,6 +113,9 @@ def dry_run(duration_minutes: int) -> dict[str, Any]:
|
||||
|
||||
|
||||
def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
|
||||
# The failed-cell probe is restored before a full control barrier is applied;
|
||||
# a barrier must not claim every cell applied while one cell is unreachable.
|
||||
svc.set_cell_health("cell-000", True)
|
||||
requested = 0
|
||||
for tenant_index in range(100):
|
||||
tenant_id = f"tenant-{tenant_index:03d}"
|
||||
@@ -162,7 +177,7 @@ def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
|
||||
def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
|
||||
profile_path = directory / "profile.json"
|
||||
build_profile(profile_path, duration_minutes)
|
||||
broker = InMemoryBroker(max_messages=2000)
|
||||
broker = InMemoryBroker(max_messages=1000, max_bytes=16 * 1024 * 1024)
|
||||
svc = AgentCallService(
|
||||
db_path=directory / "scale.sqlite3",
|
||||
profile_path=profile_path,
|
||||
@@ -174,11 +189,17 @@ def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
|
||||
template = load_fixture()
|
||||
started = time.monotonic()
|
||||
try:
|
||||
for tenant_index in range(100):
|
||||
for offset in range(12):
|
||||
svc.publish_execute(
|
||||
command_for(template, tenant_index * 12 + offset, tenant_index)
|
||||
)
|
||||
for offset in range(12):
|
||||
for tenant_index in range(100):
|
||||
body = command_for(template, tenant_index * 12 + offset, tenant_index)
|
||||
while True:
|
||||
try:
|
||||
svc.publish_execute(body)
|
||||
break
|
||||
except ServiceError as exc:
|
||||
if exc.code != "PUBLISH_RATE_LIMIT":
|
||||
raise
|
||||
time.sleep(1.05)
|
||||
warmup_deadline = started + max(120, duration_minutes * 60)
|
||||
active = 0
|
||||
threshold_at = None
|
||||
|
||||
@@ -4,6 +4,7 @@ import copy
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from agent_call.core import (
|
||||
@@ -104,6 +105,53 @@ class AgentCallTests(unittest.TestCase):
|
||||
attempts = self.service.store.one("SELECT COUNT(*) AS n FROM attempts")["n"]
|
||||
self.assertEqual(calls, 1)
|
||||
self.assertEqual(attempts, 1)
|
||||
self.assertEqual(
|
||||
self.service.store.one(
|
||||
"SELECT status FROM executions WHERE tenant_id=? AND execution_id=?",
|
||||
("tenant-demo", "exec_once"),
|
||||
)["status"],
|
||||
"completed",
|
||||
)
|
||||
|
||||
def test_unhealthy_cell_isolated_without_blocking_healthy_capacity(self) -> None:
|
||||
self.service.set_cell_health("cell-a", False)
|
||||
self.assertTrue(self.service.ready())
|
||||
snapshot = self.run_command(self.command("cmd_healthy_cell"))
|
||||
call = self.service.get_call("tenant-demo", snapshot["call_id"])
|
||||
self.assertEqual(call["call_state"], "ended")
|
||||
self.assertEqual(
|
||||
self.service.store.one(
|
||||
"SELECT cell_id FROM calls WHERE call_id=?", (snapshot["call_id"],)
|
||||
)["cell_id"],
|
||||
"cell-b",
|
||||
)
|
||||
|
||||
def test_missing_tenant_route_closes_readiness(self) -> None:
|
||||
self.service._unrouteable_tenants.add("tenant-demo")
|
||||
snapshot = self.service.readiness()
|
||||
self.assertFalse(snapshot["checks"]["routes"])
|
||||
self.assertFalse(snapshot["ready"])
|
||||
|
||||
def test_expired_scheduler_lease_fences_old_instance(self) -> None:
|
||||
body = self.command("cmd_lease_fence")
|
||||
self.service.publish_execute(body)
|
||||
self.service.ingest_commands()
|
||||
second = AgentCallService(
|
||||
db_path=Path(self.temp.name) / "executor.sqlite3",
|
||||
object_dir=Path(self.temp.name) / "objects-second",
|
||||
broker=self.broker,
|
||||
)
|
||||
try:
|
||||
with self.service.store.tx() as con:
|
||||
con.execute(
|
||||
"UPDATE leases SET expires_at=? WHERE lease_name='scheduler'",
|
||||
(iso(utcnow() - timedelta(seconds=1)),),
|
||||
)
|
||||
self.assertEqual(second.schedule_once(), 1)
|
||||
self.assertEqual(self.service.schedule_once(), 0)
|
||||
second.wait_for_idle()
|
||||
finally:
|
||||
second.stop()
|
||||
|
||||
def test_controls_use_cas_and_old_commands_do_not_resume(self) -> None:
|
||||
pause = self.service.control_task(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import base64
|
||||
import copy
|
||||
import ipaddress
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -293,10 +294,10 @@ class AsteriskTests(unittest.TestCase):
|
||||
|
||||
def test_reject_injection_public_ari_and_our_ip_as_provider(self):
|
||||
for update in (
|
||||
{"ari_bind": ".".join(("0", "0", "0", "0"))},
|
||||
{"ari_bind": str(ipaddress.IPv4Address(0))},
|
||||
{"ari_bind": "8.8.8.8"},
|
||||
{"local_net": "0.0.0.0/0"},
|
||||
): # noqa: S104 — negative fixtures; renderer must reject them.
|
||||
):
|
||||
cfg = self.cfg()
|
||||
cfg.update(update)
|
||||
with self.assertRaises(ValueError):
|
||||
|
||||
+38
-6
@@ -42,6 +42,10 @@ class HttpContractTests(unittest.TestCase):
|
||||
"recording.complete",
|
||||
],
|
||||
},
|
||||
"replay": {
|
||||
"tenant_ids": ["tenant-demo"],
|
||||
"scopes": ["outbound.read", "outbound.replay"],
|
||||
},
|
||||
}
|
||||
)
|
||||
self.service = AgentCallService(
|
||||
@@ -73,14 +77,18 @@ class HttpContractTests(unittest.TestCase):
|
||||
path: str,
|
||||
body: dict[str, Any] | None = None,
|
||||
tenant: str | None = "tenant-demo",
|
||||
token: str | None = "local",
|
||||
auth_name: str | None = None,
|
||||
idem: str | None = None,
|
||||
include_auth: bool = True,
|
||||
) -> tuple[int, dict[str, Any]]:
|
||||
headers = {"X-Request-ID": "http-test"}
|
||||
if tenant is not None:
|
||||
headers["X-Tenant-ID"] = tenant
|
||||
if token is not None:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
if include_auth and auth_name is None:
|
||||
token_map = json.loads(os.environ["HTTP_TOKENS"])
|
||||
auth_name = next(iter(token_map), None)
|
||||
if include_auth and auth_name is not None:
|
||||
headers["Authorization"] = f"Bearer {auth_name}"
|
||||
encoded: bytes | None = None
|
||||
if body is not None:
|
||||
encoded = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
@@ -104,7 +112,7 @@ class HttpContractTests(unittest.TestCase):
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise AssertionError("HTTP response is not JSON") from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise AssertionError("HTTP response is not an object")
|
||||
raise TypeError("HTTP response is not an object")
|
||||
return status, parsed
|
||||
|
||||
def publish_success(self) -> dict[str, Any]:
|
||||
@@ -118,11 +126,13 @@ class HttpContractTests(unittest.TestCase):
|
||||
return self.service.get_command("tenant-demo", "http_cmd")
|
||||
|
||||
def test_health_is_public_but_business_queries_are_authenticated(self) -> None:
|
||||
status, body = self.request("GET", "/healthz/live", tenant=None, token=None)
|
||||
status, body = self.request(
|
||||
"GET", "/healthz/live", tenant=None, include_auth=False
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["status"], "live")
|
||||
status, body = self.request(
|
||||
"GET", "/internal/v1/outbound/commands/missing", token=None
|
||||
"GET", "/internal/v1/outbound/commands/missing", include_auth=False
|
||||
)
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(body["code"], "UNAUTHORIZED")
|
||||
@@ -160,6 +170,28 @@ class HttpContractTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(status, 403)
|
||||
|
||||
def test_replay_requires_replay_scope(self) -> None:
|
||||
snapshot = self.publish_success()
|
||||
body = {"command_id": "http_replay", "reason": "retry"}
|
||||
status, response = self.request(
|
||||
"POST",
|
||||
f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays",
|
||||
body,
|
||||
auth_name="local",
|
||||
idem="http_replay",
|
||||
)
|
||||
self.assertEqual(status, 403)
|
||||
self.assertEqual(response["code"], "FORBIDDEN")
|
||||
status, response = self.request(
|
||||
"POST",
|
||||
f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays",
|
||||
body,
|
||||
auth_name="replay",
|
||||
idem="http_replay",
|
||||
)
|
||||
self.assertEqual(status, 202)
|
||||
self.assertEqual(response["status"], "accepted")
|
||||
|
||||
def test_recording_upload_contract_is_scoped(self) -> None:
|
||||
snapshot = self.publish_success()
|
||||
call = self.service.get_call("tenant-demo", snapshot["call_id"])
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from agent_call.core import (
|
||||
AgentCallService,
|
||||
ConfigurationError,
|
||||
ConflictError,
|
||||
InMemoryBroker,
|
||||
Store,
|
||||
ValidationError,
|
||||
iso,
|
||||
utcnow,
|
||||
)
|
||||
from deploy.backup_sqlite import copy_database
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def fixture() -> dict:
|
||||
return json.loads(
|
||||
(ROOT / "docs/contracts/examples/call.execute.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
class RemediationTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.broker = InMemoryBroker()
|
||||
self.service = AgentCallService(
|
||||
Path(self.temp.name) / "executor.sqlite3",
|
||||
object_dir=Path(self.temp.name) / "objects",
|
||||
broker=self.broker,
|
||||
)
|
||||
self.service.profile["limits"]["tenant_cps"] = 100
|
||||
self.service.profile["limits"]["global_cps"] = 100
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.service.stop()
|
||||
self.temp.cleanup()
|
||||
|
||||
def command(
|
||||
self,
|
||||
command_id: str,
|
||||
tenant_id: str = "tenant-demo",
|
||||
execution_id: str = "exec",
|
||||
) -> dict:
|
||||
body = copy.deepcopy(fixture())
|
||||
body["command_id"] = command_id
|
||||
body["trace_id"] = f"trace-{command_id}"
|
||||
body["issued_at"] = iso(utcnow())
|
||||
body["tenant_id"] = tenant_id
|
||||
tenant = self.service.store.one(
|
||||
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,)
|
||||
)
|
||||
body["tenant_key"] = tenant["tenant_key"]
|
||||
body["payload"]["execution_id"] = execution_id
|
||||
body["payload"]["task_id"] = (
|
||||
"task-demo" if tenant_id == "tenant-demo" else "task-b"
|
||||
)
|
||||
return body
|
||||
|
||||
def test_runtime_schema_rejects_naive_time_and_boolean_revision(self) -> None:
|
||||
body = self.command("schema-command")
|
||||
body["issued_at"] = "2026-09-13T00:00:00"
|
||||
with self.assertRaises(ValidationError):
|
||||
self.service.publish_execute(body)
|
||||
with self.assertRaises(ValidationError):
|
||||
self.service.control_task(
|
||||
"tenant-demo",
|
||||
"task-demo",
|
||||
{
|
||||
"command_id": "bool-revision",
|
||||
"action": "pause",
|
||||
"expected_task_revision": True,
|
||||
"reason": "x",
|
||||
},
|
||||
"bool-revision",
|
||||
)
|
||||
|
||||
def test_readiness_requires_persistent_database_write(self) -> None:
|
||||
db_path = Path(self.temp.name) / "executor.sqlite3"
|
||||
os.chmod(db_path, 0o444)
|
||||
try:
|
||||
readiness = self.service.readiness()
|
||||
finally:
|
||||
os.chmod(db_path, 0o644)
|
||||
self.assertFalse(readiness["checks"]["database_write"])
|
||||
self.assertFalse(readiness["ready"])
|
||||
|
||||
def test_provider_modes_do_not_fallback(self) -> None:
|
||||
profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text())
|
||||
profile["provider_modes"].pop("llm")
|
||||
path = Path(self.temp.name) / "profile.json"
|
||||
path.write_text(json.dumps(profile), encoding="utf-8")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
AgentCallService(
|
||||
profile_path=path, object_dir=Path(self.temp.name) / "objects2"
|
||||
)
|
||||
|
||||
def test_same_public_ids_are_tenant_scoped_and_wrong_route_is_dead_lettered(
|
||||
self,
|
||||
) -> None:
|
||||
first = self.command("same-command", "tenant-demo", "same-execution")
|
||||
second = self.command("same-command", "tenant-b", "same-execution")
|
||||
self.service.publish_execute(first)
|
||||
self.service.publish_execute(second)
|
||||
self.assertEqual(self.service.ingest_commands(3), 2)
|
||||
self.assertEqual(self.service.schedule_once(), 2)
|
||||
self.service.wait_for_idle()
|
||||
self.assertEqual(
|
||||
self.service.store.one("SELECT COUNT(*) AS n FROM calls")["n"], 2
|
||||
)
|
||||
|
||||
wrong = self.command("wrong-route", "tenant-b", "wrong-execution")
|
||||
self.broker.publish(
|
||||
"agent-call.commands.v1",
|
||||
"agent-call.tenant.tenant-demo-key.call.execute",
|
||||
wrong,
|
||||
"wrong-route",
|
||||
)
|
||||
self.service._cursor = self.service._tenant_order.index("tenant-demo")
|
||||
self.service.ingest_commands(1)
|
||||
self.assertEqual(
|
||||
self.service.store.one("SELECT COUNT(*) AS n FROM dead_letters")["n"], 1
|
||||
)
|
||||
self.assertIsNone(
|
||||
self.service.store.one(
|
||||
"SELECT * FROM calls WHERE command_id=?", ("wrong-route",)
|
||||
)
|
||||
)
|
||||
|
||||
def test_over_budget_tenant_is_retained_without_truncation(self) -> None:
|
||||
profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text())
|
||||
key = "x" * 225
|
||||
profile["tenants"] = [
|
||||
{"tenant_id": "long", "tenant_key": key, "max_concurrency": 1}
|
||||
]
|
||||
profile["tasks"] = [
|
||||
{
|
||||
"task_id": "long-task",
|
||||
"tenant_id": "long",
|
||||
"state": "running",
|
||||
"revision": 1,
|
||||
}
|
||||
]
|
||||
path = Path(self.temp.name) / "long-profile.json"
|
||||
path.write_text(json.dumps(profile), encoding="utf-8")
|
||||
service = AgentCallService(
|
||||
Path(self.temp.name) / "long.sqlite3",
|
||||
profile_path=path,
|
||||
object_dir=Path(self.temp.name) / "long-objects",
|
||||
broker=InMemoryBroker(),
|
||||
)
|
||||
try:
|
||||
body = self.command("placeholder")
|
||||
body["tenant_id"], body["tenant_key"] = "long", key
|
||||
body["payload"]["task_id"] = "long-task"
|
||||
body["payload"]["execution_id"] = "long-execution"
|
||||
with self.assertRaises(ValidationError) as error:
|
||||
service.publish_execute(body)
|
||||
self.assertEqual(error.exception.code, "TENANT_KEY_UNSUPPORTED")
|
||||
row = service.store.one(
|
||||
"SELECT status FROM published_commands WHERE tenant_id=?", ("long",)
|
||||
)
|
||||
self.assertEqual(row["status"], "retained")
|
||||
finally:
|
||||
service.stop()
|
||||
|
||||
def test_control_barrier_blocks_resume_until_all_cells_apply(self) -> None:
|
||||
pause = {
|
||||
"command_id": "pause-1",
|
||||
"action": "pause",
|
||||
"expected_task_revision": 1,
|
||||
"reason": "maintenance",
|
||||
}
|
||||
self.service.set_cell_health("cell-a", False)
|
||||
self.service.control_task("tenant-demo", "task-demo", pause, "pause-1")
|
||||
self.assertEqual(self.service.apply_controls(), 0)
|
||||
with self.assertRaises(ConflictError) as error:
|
||||
self.service.control_task(
|
||||
"tenant-demo",
|
||||
"task-demo",
|
||||
{
|
||||
"command_id": "resume-1",
|
||||
"action": "resume",
|
||||
"expected_task_revision": 2,
|
||||
"reason": "resume",
|
||||
},
|
||||
"resume-1",
|
||||
)
|
||||
self.assertEqual(error.exception.code, "CONTROL_IN_PROGRESS")
|
||||
self.service.set_cell_health("cell-a", True)
|
||||
self.assertEqual(self.service.apply_controls(), 1)
|
||||
self.assertEqual(
|
||||
self.service.store.one(
|
||||
"SELECT COUNT(*) AS n FROM control_barriers WHERE command_id=? AND status='applied'",
|
||||
("pause-1",),
|
||||
)["n"],
|
||||
2,
|
||||
)
|
||||
|
||||
def test_final_transcript_is_idempotent_and_conflicts_are_preserved(self) -> None:
|
||||
self.service.profile["limits"]["hold_ms"] = 1000
|
||||
body = self.command("transcript-command", "tenant-demo", "transcript-execution")
|
||||
self.service.publish_execute(body)
|
||||
self.service.ingest_commands()
|
||||
self.service.schedule_once()
|
||||
deadline = time.monotonic() + 2
|
||||
call_id = None
|
||||
while time.monotonic() < deadline:
|
||||
row = self.service.store.one(
|
||||
"SELECT * FROM calls WHERE command_id=?", (body["command_id"],)
|
||||
)
|
||||
if row and row["started_at"]:
|
||||
call_id = row["call_id"]
|
||||
break
|
||||
time.sleep(0.01)
|
||||
self.assertIsNotNone(call_id)
|
||||
assert call_id is not None
|
||||
self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed")
|
||||
before = self.service.store.one(
|
||||
"SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'"
|
||||
)["n"]
|
||||
self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed")
|
||||
after = self.service.store.one(
|
||||
"SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'"
|
||||
)["n"]
|
||||
self.assertEqual(before, after)
|
||||
with self.assertRaises(ConflictError):
|
||||
self.service._add_transcript(
|
||||
call_id, 99, "agent", "different", "confirmed", revision=2
|
||||
)
|
||||
with self.assertRaises(ConflictError):
|
||||
self.service._add_transcript(
|
||||
call_id, 99, "agent", "different", "confirmed", revision=1
|
||||
)
|
||||
self.service._add_transcript(
|
||||
call_id, 99, "agent", "final text", "sent", revision=3, is_final=False
|
||||
)
|
||||
segment = self.service.store.one(
|
||||
"SELECT text, is_final, playback_state FROM transcripts WHERE segment_id=?",
|
||||
(f"seg_{call_id}_99_agent",),
|
||||
)
|
||||
self.assertEqual(
|
||||
(segment["text"], segment["is_final"], segment["playback_state"]),
|
||||
("final text", 1, "confirmed"),
|
||||
)
|
||||
self.service._finish_call(call_id, "failed", "TEST_FINISHED")
|
||||
|
||||
def test_upload_rejects_metadata_changes(self) -> None:
|
||||
body = self.command(
|
||||
"recording-metadata-command", "tenant-demo", "recording-metadata-execution"
|
||||
)
|
||||
self.service.publish_execute(body)
|
||||
self.service.wait_for_idle()
|
||||
call_id = self.service.get_command("tenant-demo", body["command_id"])["call_id"]
|
||||
recording = self.service.get_call("tenant-demo", call_id)["recordings"][0]
|
||||
request = {
|
||||
"recording_id": recording["recording_id"],
|
||||
"call_id": call_id,
|
||||
"content_type": "audio/wav",
|
||||
"size_bytes": recording["size_bytes"],
|
||||
"checksum_algorithm": "SHA-256",
|
||||
"checksum": recording["checksum"],
|
||||
"channels": 1,
|
||||
"sample_rate_hz": recording["sample_rate_hz"],
|
||||
"duration_ms": recording["duration_ms"] + 1,
|
||||
}
|
||||
with self.assertRaises(ConflictError) as error:
|
||||
self.service.create_upload("tenant-demo", request, "metadata-change", "api")
|
||||
self.assertEqual(error.exception.code, "RECORDING_METADATA_CONFLICT")
|
||||
|
||||
def test_recording_is_non_silent_and_upload_completion_is_idempotent(self) -> None:
|
||||
body = self.command("recording-command", "tenant-demo", "recording-execution")
|
||||
self.service.publish_execute(body)
|
||||
self.service.wait_for_idle()
|
||||
call = self.service.get_call(
|
||||
"tenant-demo",
|
||||
self.service.get_command("tenant-demo", body["command_id"])["call_id"],
|
||||
)
|
||||
recording = call["recordings"][0]
|
||||
upload = self.service.store.one(
|
||||
"SELECT upload_id FROM uploads WHERE tenant_id=? AND recording_id=? ORDER BY created_at DESC LIMIT 1",
|
||||
("tenant-demo", recording["recording_id"]),
|
||||
)
|
||||
self.assertIsNotNone(upload)
|
||||
assert upload is not None
|
||||
path = self.service.object_store.final_path_for(upload["upload_id"])
|
||||
self.assertTrue(any(path.read_bytes()))
|
||||
request = {
|
||||
"recording_id": recording["recording_id"],
|
||||
"call_id": call["call_id"],
|
||||
"content_type": "audio/wav",
|
||||
"size_bytes": recording["size_bytes"],
|
||||
"checksum_algorithm": "SHA-256",
|
||||
"checksum": recording["checksum"],
|
||||
"channels": 1,
|
||||
"sample_rate_hz": recording["sample_rate_hz"],
|
||||
"duration_ms": recording["duration_ms"],
|
||||
}
|
||||
status, first = self.service.create_upload(
|
||||
"tenant-demo", request, "api-create", "api"
|
||||
)
|
||||
status2, second = self.service.create_upload(
|
||||
"tenant-demo", request, "api-create", "api"
|
||||
)
|
||||
self.assertEqual((status, status2), (200, 200))
|
||||
self.assertEqual(first["upload_id"], second["upload_id"])
|
||||
self.assertEqual(
|
||||
self.service.store.one(
|
||||
"SELECT COUNT(*) AS n FROM recordings WHERE call_id=?",
|
||||
(call["call_id"],),
|
||||
)["n"],
|
||||
1,
|
||||
)
|
||||
|
||||
def test_backup_refuses_to_replace_existing_database(self) -> None:
|
||||
source = Path(self.temp.name) / "source.sqlite3"
|
||||
destination = Path(self.temp.name) / "backup.sqlite3"
|
||||
Store(source)
|
||||
copy_database(source, destination)
|
||||
with self.assertRaises(FileExistsError):
|
||||
copy_database(source, destination)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user