feat(agent-call): implement remediation plan

scripts/scale_mock.py

@@ -14,7 +14,13 @@ import time
 from pathlib import Path
 from typing import Any
 
-from agent_call.core import AgentCallService, InMemoryBroker, iso, utcnow
+from agent_call.core import (
+    AgentCallService,
+    InMemoryBroker,
+    ServiceError,
+    iso,
+    utcnow,
+)
 from scripts.acceptance import load_fixture
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -44,6 +50,8 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
         {
             "cell_id": f"cell-{index:03d}",
             "capacity": 12,
+            "media_capacity": 12,
+            "ai_capacity": 12,
             "egress_pool_id": "egress-mock",
             "ari_mode": "mock",
         }
@@ -53,10 +61,14 @@ def build_profile(path: Path, duration_minutes: int) -> dict[str, Any]:
     profile["limits"].update(
         {
             "global_concurrency": 1200,
-            "global_cps": 1200,
+            "global_cps": 20,
             "tenant_concurrency": 12,
-            "tenant_cps": 12,
-            "max_queue_messages": 2000,
+            "tenant_cps": 1,
+            "max_queue_messages": 1000,
+            "max_queue_bytes": 16 * 1024 * 1024,
+            "pending_window_per_tenant": 16,
+            "pending_window_global": 64,
+            "max_unacked_per_tenant": 4,
             # Keep calls alive through warm-up and the required observation window.
             "hold_ms": (startup_window_seconds + duration_minutes * 60 + 5) * 1000,
         }
@@ -101,6 +113,9 @@ def dry_run(duration_minutes: int) -> dict[str, Any]:
 
 
 def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
+    # The failed-cell probe is restored before a full control barrier is applied;
+    # a barrier must not claim every cell applied while one cell is unreachable.
+    svc.set_cell_health("cell-000", True)
     requested = 0
     for tenant_index in range(100):
         tenant_id = f"tenant-{tenant_index:03d}"
@@ -162,7 +177,7 @@ def cleanup_load(svc: AgentCallService) -> dict[str, Any]:
 def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
     profile_path = directory / "profile.json"
     build_profile(profile_path, duration_minutes)
-    broker = InMemoryBroker(max_messages=2000)
+    broker = InMemoryBroker(max_messages=1000, max_bytes=16 * 1024 * 1024)
     svc = AgentCallService(
         db_path=directory / "scale.sqlite3",
         profile_path=profile_path,
@@ -174,11 +189,17 @@ def run(directory: Path, duration_minutes: int) -> dict[str, Any]:
     template = load_fixture()
     started = time.monotonic()
     try:
-        for tenant_index in range(100):
-            for offset in range(12):
-                svc.publish_execute(
-                    command_for(template, tenant_index * 12 + offset, tenant_index)
-                )
+        for offset in range(12):
+            for tenant_index in range(100):
+                body = command_for(template, tenant_index * 12 + offset, tenant_index)
+                while True:
+                    try:
+                        svc.publish_execute(body)
+                        break
+                    except ServiceError as exc:
+                        if exc.code != "PUBLISH_RATE_LIMIT":
+                            raise
+                        time.sleep(1.05)
         warmup_deadline = started + max(120, duration_minutes * 60)
         active = 0
         threshold_at = None