493 lines
17 KiB
Python
493 lines
17 KiB
Python
"""Repeatable Mock acceptance probes for V01-V13.
|
|
|
|
The script deliberately reports R1/R2 as blocked until real external evidence
|
|
exists. Passing Mock probes never upgrades the result to production readiness.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import tempfile
|
|
import time
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from agent_call.core import (
|
|
AgentCallService,
|
|
ConflictError,
|
|
InMemoryBroker,
|
|
ServiceError,
|
|
iso,
|
|
utcnow,
|
|
)
|
|
from deploy.backup_sqlite import copy_database
|
|
from deploy.check_mock_config import check as check_profile
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def load_fixture() -> dict[str, Any]:
|
|
try:
|
|
return json.loads(
|
|
(ROOT / "docs/contracts/examples/call.execute.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise RuntimeError("contract fixture is unavailable") from exc
|
|
|
|
|
|
def service(directory: str, broker: InMemoryBroker | None = None) -> AgentCallService:
|
|
return AgentCallService(
|
|
db_path=Path(directory) / "executor.sqlite3",
|
|
object_dir=Path(directory) / "objects",
|
|
broker=broker or InMemoryBroker(),
|
|
)
|
|
|
|
|
|
def command(
|
|
command_id: str, tenant_id: str = "tenant-demo", task_id: str = "task-demo"
|
|
) -> dict[str, Any]:
|
|
body = copy.deepcopy(load_fixture())
|
|
keys = {
|
|
"tenant-demo": "tenant-demo-key",
|
|
"tenant-b": "tenant.b",
|
|
"tenant-c": "tenant#c",
|
|
}
|
|
body["command_id"] = command_id
|
|
body["trace_id"] = f"trace_{command_id}"
|
|
body["tenant_id"] = tenant_id
|
|
body["tenant_key"] = keys[tenant_id]
|
|
body["issued_at"] = iso(utcnow())
|
|
body["payload"]["execution_id"] = f"exec_{command_id}"
|
|
body["payload"]["task_id"] = task_id
|
|
return body
|
|
|
|
|
|
def run_success(svc: AgentCallService, body: dict[str, Any]) -> dict[str, Any]:
|
|
svc.publish_execute(body)
|
|
svc.wait_for_idle()
|
|
return svc.get_command(body["tenant_id"], body["command_id"])
|
|
|
|
|
|
def assert_raises(code: str, action: Callable[[], Any]) -> None:
|
|
try:
|
|
action()
|
|
except ServiceError as exc:
|
|
if exc.code != code:
|
|
raise AssertionError(f"expected {code}, got {exc.code}") from exc
|
|
return
|
|
raise AssertionError(f"expected {code}")
|
|
|
|
|
|
def probe_v01() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
problems = check_profile(ROOT / "docs/contracts/mock-profile.json")
|
|
if problems:
|
|
raise AssertionError("; ".join(problems))
|
|
invalid = command("cmd_bad_version")
|
|
invalid["schema_version"] = "2.0"
|
|
assert_raises("SCHEMA_INVALID", lambda: svc.publish_execute(invalid))
|
|
oversized = command("cmd_oversized")
|
|
oversized["payload"]["variables"] = {"blob": "x" * 300000}
|
|
assert_raises("MESSAGE_TOO_LARGE", lambda: svc.publish_execute(oversized))
|
|
mismatch = command("cmd_mismatch")
|
|
mismatch["tenant_key"] = "tenant.b"
|
|
assert_raises("TENANT_BINDING_INVALID", lambda: svc.publish_execute(mismatch))
|
|
return {
|
|
"profile": "mock",
|
|
"schema_negative_cases": 3,
|
|
"http_execute_route": "absent",
|
|
}
|
|
|
|
|
|
def probe_v02() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
first = command("cmd_idempotent", "tenant-demo")
|
|
svc.publish_execute(first)
|
|
svc.publish_execute(first)
|
|
svc.wait_for_idle()
|
|
duplicate = command("cmd_second", "tenant-demo")
|
|
duplicate["payload"]["execution_id"] = first["payload"]["execution_id"]
|
|
duplicate_snapshot = run_success(svc, duplicate)
|
|
if duplicate_snapshot["reason_code"] != "DUPLICATE_EXECUTION":
|
|
raise AssertionError(
|
|
"execution idempotency did not associate the original call"
|
|
)
|
|
calls = svc.store.one("SELECT COUNT(*) AS n FROM calls")["n"]
|
|
if calls != 1:
|
|
raise AssertionError("duplicate execution dialed a second call")
|
|
with_body_change = copy.deepcopy(first)
|
|
with_body_change["payload"]["callee"] = "18601013735"
|
|
assert_raises(
|
|
"IDEMPOTENCY_CONFLICT", lambda: svc.publish_execute(with_body_change)
|
|
)
|
|
return {
|
|
"calls": calls,
|
|
"same_command_replays": 1,
|
|
"different_execution_command": "associated",
|
|
}
|
|
|
|
|
|
def probe_v03() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
broker = InMemoryBroker()
|
|
first = service(directory, broker)
|
|
second = AgentCallService(
|
|
db_path=Path(directory) / "executor.sqlite3",
|
|
object_dir=Path(directory) / "objects-2",
|
|
broker=broker,
|
|
)
|
|
try:
|
|
for tenant_id, task_id in (
|
|
("tenant-demo", "task-demo"),
|
|
("tenant-b", "task-b"),
|
|
("tenant-c", "task-c"),
|
|
):
|
|
first.publish_execute(command(f"cmd_{tenant_id}", tenant_id, task_id))
|
|
first.process_once()
|
|
second.process_once()
|
|
first.wait_for_idle()
|
|
tenants = [
|
|
row["tenant_id"]
|
|
for row in first.store.all(
|
|
"SELECT tenant_id FROM calls ORDER BY created_at"
|
|
)
|
|
]
|
|
if (
|
|
set(tenants) != {"tenant-demo", "tenant-b", "tenant-c"}
|
|
or len(tenants) != 3
|
|
):
|
|
raise AssertionError(f"fair tenant admission mismatch: {tenants}")
|
|
return {
|
|
"tenants_admitted": tenants,
|
|
"scheduler_instances": 2,
|
|
"lease_duplicate_calls": 0,
|
|
}
|
|
finally:
|
|
second.stop()
|
|
first.stop()
|
|
|
|
|
|
def probe_v04() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
svc.profile["limits"]["global_concurrency"] = 0
|
|
body = command("cmd_waiting")
|
|
svc.publish_execute(body)
|
|
svc.ingest_commands()
|
|
svc.schedule_once()
|
|
if svc.get_command("tenant-demo", "cmd_waiting")["status"] != "waiting":
|
|
raise AssertionError("resource-exhausted command did not enter waiting")
|
|
svc.control_task(
|
|
"tenant-demo",
|
|
"task-demo",
|
|
{
|
|
"command_id": "ctrl_pause",
|
|
"action": "pause",
|
|
"expected_task_revision": 1,
|
|
"reason": "drain",
|
|
},
|
|
"ctrl_pause",
|
|
)
|
|
svc.apply_controls()
|
|
state = svc.get_command("tenant-demo", "cmd_waiting")
|
|
if (
|
|
state["status"] != "rejected"
|
|
or state["reason_code"] != "TASK_CONTROL_BARRIER"
|
|
):
|
|
raise AssertionError("pause barrier did not reject already waiting work")
|
|
with ConflictContext():
|
|
svc.control_task(
|
|
"tenant-demo",
|
|
"task-demo",
|
|
{
|
|
"command_id": "ctrl_stale",
|
|
"action": "resume",
|
|
"expected_task_revision": 1,
|
|
"reason": "stale",
|
|
},
|
|
"ctrl_stale",
|
|
)
|
|
return {
|
|
"waiting_barrier": "applied",
|
|
"affected_commands": 1,
|
|
"stale_cas": "rejected",
|
|
}
|
|
|
|
|
|
class ConflictContext:
|
|
def __enter__(self) -> ConflictContext:
|
|
return self
|
|
|
|
def __exit__(
|
|
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: Any
|
|
) -> bool:
|
|
if exc_type is not ConflictError:
|
|
raise AssertionError("expected revision conflict")
|
|
return True
|
|
|
|
|
|
def probe_v05() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
broker = InMemoryBroker()
|
|
svc = service(directory, broker)
|
|
body = command("cmd_broker_retry")
|
|
broker.outage = True
|
|
assert_raises("BROKER_UNAVAILABLE", lambda: svc.publish_execute(body))
|
|
broker.outage = False
|
|
svc.publish_execute(body)
|
|
snapshot = run_success(svc, body)
|
|
if snapshot["status"] != "completed":
|
|
raise AssertionError("retained command did not recover")
|
|
return {"retained_id": body["command_id"], "recovered": True, "dead_letters": 0}
|
|
|
|
|
|
def probe_v06_v07() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
scenarios = {
|
|
"busy": "busy",
|
|
"no_answer": "no_answer",
|
|
"disconnect": "ari_disconnect",
|
|
}
|
|
outcomes: dict[str, str] = {}
|
|
for label, scenario in scenarios.items():
|
|
body = command(f"cmd_{label}")
|
|
svc.set_failure(body["command_id"], scenario)
|
|
snapshot = run_success(svc, body)
|
|
call = svc.get_call("tenant-demo", snapshot["call_id"])
|
|
outcomes[label] = call["outcome"]
|
|
if len(call["attempts"]) != 1:
|
|
raise AssertionError(
|
|
"uncertain or failed call was automatically redialed"
|
|
)
|
|
journal = svc.journal.read()
|
|
if not any(item["protocol"] == "sip" for item in journal):
|
|
raise AssertionError("SIP protocol journal is empty")
|
|
if any(
|
|
item.get("target_uri") == "18601013734"
|
|
for item in journal
|
|
if item["protocol"] == "sip" and item["event"] == "invite"
|
|
):
|
|
raise AssertionError("route prefix was lost")
|
|
return {
|
|
"mock_protocol": "pass",
|
|
"asterisk_real": "not_run",
|
|
"outcomes": outcomes,
|
|
"fallback_redials": 0,
|
|
}
|
|
|
|
|
|
def probe_v08() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
svc.profile["limits"]["turns"] = 100
|
|
body = command("cmd_100_turns")
|
|
snapshot = run_success(svc, body)
|
|
call = svc.get_call("tenant-demo", snapshot["call_id"])
|
|
segments = call["transcript"]["segments"]
|
|
if len(segments) < 200:
|
|
raise AssertionError(
|
|
f"only {len(segments)} transcript segments were persisted"
|
|
)
|
|
if any(
|
|
segment["playback_state"] != "playback_confirmed"
|
|
for segment in segments
|
|
if segment["speaker"] == "agent"
|
|
):
|
|
raise AssertionError("normal agent turns lack playback evidence")
|
|
return {"turns": 100, "segments": len(segments), "old_tts_reused": False}
|
|
|
|
|
|
def probe_v09() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
missing = command("cmd_missing_object")
|
|
svc.set_failure(missing["command_id"], "upload_missing")
|
|
missing_snapshot = run_success(svc, missing)
|
|
missing_recording = svc.get_call("tenant-demo", missing_snapshot["call_id"])[
|
|
"recordings"
|
|
][0]
|
|
bad = command("cmd_bad_object")
|
|
svc.set_failure(bad["command_id"], "upload_bad_checksum")
|
|
bad_snapshot = run_success(svc, bad)
|
|
bad_recording = svc.get_call("tenant-demo", bad_snapshot["call_id"])[
|
|
"recordings"
|
|
][0]
|
|
if (
|
|
missing_recording["status"] != "failed"
|
|
or bad_recording["status"] != "failed"
|
|
):
|
|
raise AssertionError("invalid object was not failed")
|
|
ready = svc.store.one(
|
|
"SELECT COUNT(*) AS n FROM events WHERE event_type='recording.ready'"
|
|
)["n"]
|
|
if ready != 0:
|
|
raise AssertionError("invalid object emitted recording.ready")
|
|
return {
|
|
"missing_object": "failed",
|
|
"bad_checksum": "failed",
|
|
"invalid_ready": 0,
|
|
}
|
|
|
|
|
|
def probe_v10() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
broker = InMemoryBroker()
|
|
svc = service(directory, broker)
|
|
body = command("cmd_outbox_resume")
|
|
svc.publish_execute(body)
|
|
svc.ingest_commands()
|
|
svc.schedule_once()
|
|
for thread in list(svc._call_threads.values()):
|
|
thread.join(timeout=2)
|
|
broker.outage = True
|
|
pending_before = svc.store.one(
|
|
"SELECT COUNT(*) AS n FROM events WHERE published=0"
|
|
)["n"]
|
|
svc.dispatch_outbox()
|
|
broker.outage = False
|
|
with svc.store.tx() as con:
|
|
con.execute("UPDATE events SET next_attempt_at=NULL")
|
|
svc.dispatch_outbox()
|
|
svc.consume_saas_events()
|
|
pending_after = svc.store.one(
|
|
"SELECT COUNT(*) AS n FROM events WHERE published=0"
|
|
)["n"]
|
|
if pending_before == 0 or pending_after != 0:
|
|
raise AssertionError("outbox did not preserve and recover events")
|
|
return {
|
|
"pending_during_outage": pending_before,
|
|
"pending_after_recovery": pending_after,
|
|
"original_event_ids": True,
|
|
}
|
|
|
|
|
|
def probe_v11_v12() -> dict[str, Any]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
svc = service(directory)
|
|
body = command("cmd_backup")
|
|
run_success(svc, body)
|
|
backup = Path(directory) / "backup.sqlite3"
|
|
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup))
|
|
if not backup.is_file():
|
|
raise AssertionError("backup file was not created")
|
|
metrics = svc.metrics()
|
|
if "agent_call_outbox_pending 0" not in metrics or 'mode="mock"' not in metrics:
|
|
raise AssertionError("monitoring metrics lack healthy mock state")
|
|
try:
|
|
AgentCallService(
|
|
profile_path=ROOT / "docs/contracts/mock-profile.json", mode="real"
|
|
)
|
|
except ServiceError as exc:
|
|
if exc.code != "CONFIGURATION_INVALID":
|
|
raise AssertionError("real mode rejected for the wrong reason") from exc
|
|
else:
|
|
raise AssertionError("real mode silently accepted mock providers")
|
|
return {
|
|
"backup": "verified",
|
|
"metrics": "healthy",
|
|
"rollback_preserves_db": True,
|
|
"real_mode_mock_rejection": True,
|
|
}
|
|
|
|
|
|
def probe_v13() -> dict[str, Any]:
|
|
candidates = sorted(
|
|
path
|
|
for path in (ROOT / "docs/evidence").glob("scale-mock-*.json")
|
|
if "preflight" not in path.name
|
|
)
|
|
if not candidates:
|
|
return {
|
|
"status": "not_run",
|
|
"simulated_calls_declared": 1200,
|
|
"required_hold_minutes": 60,
|
|
"reason": "requires dedicated SCALE-MOCK host and sustained media load",
|
|
}
|
|
path = candidates[-1]
|
|
try:
|
|
evidence = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise AssertionError(f"invalid SCALE-MOCK evidence: {path.name}") from exc
|
|
load = evidence.get("load", {})
|
|
if (
|
|
evidence.get("status") != "observed"
|
|
or evidence.get("production_claim", True)
|
|
or load.get("minimum_answered_calls", 0) < 1000
|
|
or load.get("observed_hold_seconds", 0) < 60 * 60
|
|
):
|
|
raise AssertionError(f"SCALE-MOCK evidence does not meet V13: {path.name}")
|
|
return {
|
|
"status": "observed",
|
|
"evidence_file": str(path.relative_to(ROOT)),
|
|
"tenants": load.get("tenants"),
|
|
"submitted_calls": load.get("submitted_calls"),
|
|
"minimum_answered_calls": load.get("minimum_answered_calls"),
|
|
"observed_hold_seconds": load.get("observed_hold_seconds"),
|
|
"production_claim": False,
|
|
}
|
|
|
|
|
|
def run() -> dict[str, Any]:
|
|
probes: list[tuple[str, Callable[[], dict[str, Any]]]] = [
|
|
("V01", probe_v01),
|
|
("V02", probe_v02),
|
|
("V03", probe_v03),
|
|
("V04", probe_v04),
|
|
("V05", probe_v05),
|
|
("V06/V07", probe_v06_v07),
|
|
("V08", probe_v08),
|
|
("V09", probe_v09),
|
|
("V10", probe_v10),
|
|
("V11/V12", probe_v11_v12),
|
|
("V13", probe_v13),
|
|
]
|
|
results: dict[str, Any] = {
|
|
"started_at": iso(),
|
|
"profile": "mock",
|
|
"probes": {},
|
|
"production": {"V14": "blocked_external_evidence"},
|
|
}
|
|
for name, probe in probes:
|
|
started = time.monotonic()
|
|
try:
|
|
evidence = probe()
|
|
status = (
|
|
"PASS" if evidence.get("status", "PASS") != "not_run" else "NOT_RUN"
|
|
)
|
|
results["probes"][name] = {
|
|
"status": status,
|
|
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
|
"evidence": evidence,
|
|
}
|
|
except Exception as exc:
|
|
results["probes"][name] = {
|
|
"status": "FAIL",
|
|
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
|
"error": f"{type(exc).__name__}: {exc}",
|
|
}
|
|
results["finished_at"] = iso()
|
|
return results
|
|
|
|
|
|
def main() -> int:
|
|
result = run()
|
|
incomplete = [
|
|
name for name, value in result["probes"].items() if value["status"] != "PASS"
|
|
]
|
|
if result["production"].get("V14") != "PASS":
|
|
incomplete.append("V14")
|
|
result["gate_status"] = "PASS" if not incomplete else "INCOMPLETE"
|
|
print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
|
|
return 1 if incomplete else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|