rogee/agent-call
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent-call): implement remediation plan

Browse Source
This commit is contained in:
rogee
2026-09-13 19:20:28 +08:00
parent d30314c7d1
commit b95d2aee81
25 changed files with 3159 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/acceptance.py
+84 -8
View File
@@ -7,12 +7,15 @@ exists. Passing Mock probes never upgrades the result to production readiness.
from __future__ import annotations
import copy
import io
import json
import tempfile
import time
from collections.abc import Callable
from contextlib import redirect_stdout
from pathlib import Path
from typing import Any
from types import TracebackType
from typing import Any, Self
from agent_call.core import (
AgentCallService,
@@ -221,11 +224,14 @@ def probe_v04() -> dict[str, Any]:
class ConflictContext:
def __enter__(self) -> ConflictContext:
def __enter__(self) -> Self:
return self
def __exit__(
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: Any
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> bool:
if exc_type is not ConflictError:
raise AssertionError("expected revision conflict")
@@ -256,10 +262,37 @@ def probe_v06_v07() -> dict[str, Any]:
"disconnect": "ari_disconnect",
}
outcomes: dict[str, str] = {}
uncertain_before_reconcile: str | None = None
for label, scenario in scenarios.items():
body = command(f"cmd_{label}")
svc.set_failure(body["command_id"], scenario)
snapshot = run_success(svc, body)
svc.publish_execute(body)
if scenario == "ari_disconnect":
call_id = None
before = None
for _ in range(500):
svc.process_once()
current = svc.get_command("tenant-demo", body["command_id"])
call_id = current.get("call_id")
if call_id:
before = svc.get_call("tenant-demo", call_id)
if before["call_state"] == "reconciling":
break
time.sleep(0.01)
if not call_id or before is None:
raise AssertionError("disconnect probe did not create a call")
uncertain_before_reconcile = before["call_state"]
if uncertain_before_reconcile != "reconciling":
raise AssertionError(
"disconnect probe did not retain unknown state"
)
svc.reconcile_call(
call_id, ended=True, outcome="failed", reason="ARI_RECONCILED"
)
snapshot = svc.get_command("tenant-demo", body["command_id"])
else:
svc.wait_for_idle()
snapshot = svc.get_command("tenant-demo", body["command_id"])
call = svc.get_call("tenant-demo", snapshot["call_id"])
outcomes[label] = call["outcome"]
if len(call["attempts"]) != 1:
@@ -280,6 +313,7 @@ def probe_v06_v07() -> dict[str, Any]:
"asterisk_real": "not_run",
"outcomes": outcomes,
"fallback_redials": 0,
"disconnect_state_before_reconcile": uncertain_before_reconcile,
}
@@ -374,9 +408,37 @@ def probe_v11_v12() -> dict[str, Any]:
body = command("cmd_backup")
run_success(svc, body)
backup = Path(directory) / "backup.sqlite3"
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup))
with redirect_stdout(io.StringIO()):
copy_database(str(Path(directory) / "executor.sqlite3"), str(backup))
if not backup.is_file():
raise AssertionError("backup file was not created")
restored = AgentCallService(
db_path=backup,
object_dir=Path(directory) / "restored-objects",
broker=InMemoryBroker(),
)
try:
restored_command = restored.get_command("tenant-demo", body["command_id"])
restored_call = restored.get_call(
"tenant-demo", restored_command["call_id"]
)
if (
restored_command["status"] != "completed"
or restored_call["recordings"][0]["status"] != "verified"
):
raise AssertionError("restored database lost the completed call facts")
restored_counts = {
"commands": restored.store.one("SELECT COUNT(*) AS n FROM commands")[
"n"
],
"events": restored.store.one("SELECT COUNT(*) AS n FROM events")["n"],
"inbox": restored.store.one("SELECT COUNT(*) AS n FROM inbox")["n"],
"recordings": restored.store.one(
"SELECT COUNT(*) AS n FROM recordings"
)["n"],
}
finally:
restored.stop()
metrics = svc.metrics()
if "agent_call_outbox_pending 0" not in metrics or 'mode="mock"' not in metrics:
raise AssertionError("monitoring metrics lack healthy mock state")
@@ -392,7 +454,9 @@ def probe_v11_v12() -> dict[str, Any]:
return {
"backup": "verified",
"metrics": "healthy",
"rollback_preserves_db": True,
"rollback_preserves_db": "not_run",
"restored_db_facts": True,
"restored_counts": restored_counts,
"real_mode_mock_rejection": True,
}
@@ -419,10 +483,15 @@ def probe_v13() -> dict[str, Any]:
if (
evidence.get("status") != "observed"
or evidence.get("production_claim", True)
or not evidence.get("continuous_media_evidence", False)
or load.get("minimum_answered_calls", 0) < 1000
or load.get("observed_hold_seconds", 0) < 60 * 60
):
raise AssertionError(f"SCALE-MOCK evidence does not meet V13: {path.name}")
return {
"status": "not_run",
"evidence_file": str(path.relative_to(ROOT)),
"reason": "historical state-only evidence cannot satisfy continuous media V13",
}
return {
"status": "observed",
"evidence_file": str(path.relative_to(ROOT)),
@@ -466,7 +535,14 @@ def run() -> dict[str, Any]:
"duration_ms": round((time.monotonic() - started) * 1000, 3),
"evidence": evidence,
}
except Exception as exc:
except (
AssertionError,
KeyError,
OSError,
RuntimeError,
ServiceError,
ValueError,
) as exc:
results["probes"][name] = {
"status": "FAIL",
"duration_ms": round((time.monotonic() - started) * 1000, 3),
scripts/scale_mock.py
+31 -10
View File
@@ -14,7 +14,13 @@ import time
from pathlib import Path
from typing import Any
from agent_call.core import AgentCallService, InMemoryBroker, iso, utcnow
from agent_call.core import (
AgentCallService,
InMemoryBroker,
ServiceError,
iso,
utcnow,
)
from scripts.acceptance import load_fixture
ROOT = Path(__file__).resolve().parents[1]
@@ -44,6 +50,8 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
{
"cell_id": f"cell-{index:03d}",
"capacity": 12,
"media_capacity": 12,
"ai_capacity": 12,
"egress_pool_id": "egress-mock",
"ari_mode": "mock",
}
@@ -53,10 +61,14 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
profile["limits"].update(
{
"global_concurrency": 1200,
"global_cps": 1200,
"global_cps": 20,
"tenant_concurrency": 12,
"tenant_cps": 12,
"max_queue_messages": 2000,
"tenant_cps": 1,
"max_queue_messages": 1000,
"max_queue_bytes": 16 * 1024 * 1024,
"pending_window_per_tenant": 16,
"pending_window_global": 64,
"max_unacked_per_tenant": 4,
# Keep calls alive through warm-up and the required observation window.
"hold_ms": (startup_window_seconds + duration_minutes * 60 + 5) * 1000,
}
@@ -101,6 +113,9 @@ def dry_run(duration_minutes: int) -> dict[str, Any]:
def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
# The failed-cell probe is restored before a full control barrier is applied;
# a barrier must not claim every cell applied while one cell is unreachable.
svc.set_cell_health("cell-000", True)
requested = 0
for tenant_index in range(100):
tenant_id = f"tenant-{tenant_index:03d}"
@@ -162,7 +177,7 @@ def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
profile_path = directory / "profile.json"
build_profile(profile_path, duration_minutes)
broker = InMemoryBroker(max_messages=2000)
broker = InMemoryBroker(max_messages=1000, max_bytes=16 * 1024 * 1024)
svc = AgentCallService(
db_path=directory / "scale.sqlite3",
profile_path=profile_path,
@@ -174,11 +189,17 @@ def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
template = load_fixture()
started = time.monotonic()
try:
for tenant_index in range(100):
for offset in range(12):
svc.publish_execute(
command_for(template, tenant_index * 12 + offset, tenant_index)
)
for offset in range(12):
for tenant_index in range(100):
body = command_for(template, tenant_index * 12 + offset, tenant_index)
while True:
try:
svc.publish_execute(body)
break
except ServiceError as exc:
if exc.code != "PUBLISH_RATE_LIMIT":
raise
time.sleep(1.05)
warmup_deadline = started + max(120, duration_minutes * 60)
active = 0
threshold_at = None