rogee/agent-call
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent-call): implement remediation plan

Browse Source
This commit is contained in:
rogee
2026-09-13 19:20:28 +08:00
parent d30314c7d1
commit b95d2aee81
25 changed files with 3159 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_agent_call.py
+48
View File
@@ -4,6 +4,7 @@ import copy
import json
import tempfile
import unittest
from datetime import timedelta
from pathlib import Path
from agent_call.core import (
@@ -104,6 +105,53 @@ class AgentCallTests(unittest.TestCase):
attempts = self.service.store.one("SELECT COUNT(*) AS n FROM attempts")["n"]
self.assertEqual(calls, 1)
self.assertEqual(attempts, 1)
self.assertEqual(
self.service.store.one(
"SELECT status FROM executions WHERE tenant_id=? AND execution_id=?",
("tenant-demo", "exec_once"),
)["status"],
"completed",
)
def test_unhealthy_cell_isolated_without_blocking_healthy_capacity(self) -> None:
self.service.set_cell_health("cell-a", False)
self.assertTrue(self.service.ready())
snapshot = self.run_command(self.command("cmd_healthy_cell"))
call = self.service.get_call("tenant-demo", snapshot["call_id"])
self.assertEqual(call["call_state"], "ended")
self.assertEqual(
self.service.store.one(
"SELECT cell_id FROM calls WHERE call_id=?", (snapshot["call_id"],)
)["cell_id"],
"cell-b",
)
def test_missing_tenant_route_closes_readiness(self) -> None:
self.service._unrouteable_tenants.add("tenant-demo")
snapshot = self.service.readiness()
self.assertFalse(snapshot["checks"]["routes"])
self.assertFalse(snapshot["ready"])
def test_expired_scheduler_lease_fences_old_instance(self) -> None:
body = self.command("cmd_lease_fence")
self.service.publish_execute(body)
self.service.ingest_commands()
second = AgentCallService(
db_path=Path(self.temp.name) / "executor.sqlite3",
object_dir=Path(self.temp.name) / "objects-second",
broker=self.broker,
)
try:
with self.service.store.tx() as con:
con.execute(
"UPDATE leases SET expires_at=? WHERE lease_name='scheduler'",
(iso(utcnow() - timedelta(seconds=1)),),
)
self.assertEqual(second.schedule_once(), 1)
self.assertEqual(self.service.schedule_once(), 0)
second.wait_for_idle()
finally:
second.stop()
def test_controls_use_cas_and_old_commands_do_not_resume(self) -> None:
pause = self.service.control_task(
tests/test_deployment.py
+3 -2
View File
@@ -1,5 +1,6 @@
import base64
import copy
import ipaddress
import json
import tempfile
import unittest
@@ -293,10 +294,10 @@ class AsteriskTests(unittest.TestCase):
def test_reject_injection_public_ari_and_our_ip_as_provider(self):
for update in (
{"ari_bind": ".".join(("0", "0", "0", "0"))},
{"ari_bind": str(ipaddress.IPv4Address(0))},
{"ari_bind": "8.8.8.8"},
{"local_net": "0.0.0.0/0"},
): # noqa: S104 — negative fixtures; renderer must reject them.
):
cfg = self.cfg()
cfg.update(update)
with self.assertRaises(ValueError):
tests/test_http.py
+38 -6
View File
@@ -42,6 +42,10 @@ class HttpContractTests(unittest.TestCase):
"recording.complete",
],
},
"replay": {
"tenant_ids": ["tenant-demo"],
"scopes": ["outbound.read", "outbound.replay"],
},
}
)
self.service = AgentCallService(
@@ -73,14 +77,18 @@ class HttpContractTests(unittest.TestCase):
path: str,
body: dict[str, Any] | None = None,
tenant: str | None = "tenant-demo",
token: str | None = "local",
auth_name: str | None = None,
idem: str | None = None,
include_auth: bool = True,
) -> tuple[int, dict[str, Any]]:
headers = {"X-Request-ID": "http-test"}
if tenant is not None:
headers["X-Tenant-ID"] = tenant
if token is not None:
headers["Authorization"] = f"Bearer {token}"
if include_auth and auth_name is None:
token_map = json.loads(os.environ["HTTP_TOKENS"])
auth_name = next(iter(token_map), None)
if include_auth and auth_name is not None:
headers["Authorization"] = f"Bearer {auth_name}"
encoded: bytes | None = None
if body is not None:
encoded = json.dumps(body, ensure_ascii=False).encode("utf-8")
@@ -104,7 +112,7 @@ class HttpContractTests(unittest.TestCase):
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AssertionError("HTTP response is not JSON") from exc
if not isinstance(parsed, dict):
raise AssertionError("HTTP response is not an object")
raise TypeError("HTTP response is not an object")
return status, parsed
def publish_success(self) -> dict[str, Any]:
@@ -118,11 +126,13 @@ class HttpContractTests(unittest.TestCase):
return self.service.get_command("tenant-demo", "http_cmd")
def test_health_is_public_but_business_queries_are_authenticated(self) -> None:
status, body = self.request("GET", "/healthz/live", tenant=None, token=None)
status, body = self.request(
"GET", "/healthz/live", tenant=None, include_auth=False
)
self.assertEqual(status, 200)
self.assertEqual(body["status"], "live")
status, body = self.request(
"GET", "/internal/v1/outbound/commands/missing", token=None
"GET", "/internal/v1/outbound/commands/missing", include_auth=False
)
self.assertEqual(status, 401)
self.assertEqual(body["code"], "UNAUTHORIZED")
@@ -160,6 +170,28 @@ class HttpContractTests(unittest.TestCase):
)
self.assertEqual(status, 403)
def test_replay_requires_replay_scope(self) -> None:
snapshot = self.publish_success()
body = {"command_id": "http_replay", "reason": "retry"}
status, response = self.request(
"POST",
f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays",
body,
auth_name="local",
idem="http_replay",
)
self.assertEqual(status, 403)
self.assertEqual(response["code"], "FORBIDDEN")
status, response = self.request(
"POST",
f"/internal/v1/outbound/commands/{snapshot['command_id']}/replays",
body,
auth_name="replay",
idem="http_replay",
)
self.assertEqual(status, 202)
self.assertEqual(response["status"], "accepted")
def test_recording_upload_contract_is_scoped(self) -> None:
snapshot = self.publish_success()
call = self.service.get_call("tenant-demo", snapshot["call_id"])
tests/test_remediation_plan.py
+334
View File
@@ -0,0 +1,334 @@
from __future__ import annotations
import copy
import json
import os
import tempfile
import time
import unittest
from pathlib import Path
from agent_call.core import (
AgentCallService,
ConfigurationError,
ConflictError,
InMemoryBroker,
Store,
ValidationError,
iso,
utcnow,
)
from deploy.backup_sqlite import copy_database
ROOT = Path(__file__).resolve().parents[1]
def fixture() -> dict:
return json.loads(
(ROOT / "docs/contracts/examples/call.execute.json").read_text(encoding="utf-8")
)
class RemediationTests(unittest.TestCase):
def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory()
self.broker = InMemoryBroker()
self.service = AgentCallService(
Path(self.temp.name) / "executor.sqlite3",
object_dir=Path(self.temp.name) / "objects",
broker=self.broker,
)
self.service.profile["limits"]["tenant_cps"] = 100
self.service.profile["limits"]["global_cps"] = 100
def tearDown(self) -> None:
self.service.stop()
self.temp.cleanup()
def command(
self,
command_id: str,
tenant_id: str = "tenant-demo",
execution_id: str = "exec",
) -> dict:
body = copy.deepcopy(fixture())
body["command_id"] = command_id
body["trace_id"] = f"trace-{command_id}"
body["issued_at"] = iso(utcnow())
body["tenant_id"] = tenant_id
tenant = self.service.store.one(
"SELECT tenant_key FROM tenants WHERE tenant_id=?", (tenant_id,)
)
body["tenant_key"] = tenant["tenant_key"]
body["payload"]["execution_id"] = execution_id
body["payload"]["task_id"] = (
"task-demo" if tenant_id == "tenant-demo" else "task-b"
)
return body
def test_runtime_schema_rejects_naive_time_and_boolean_revision(self) -> None:
body = self.command("schema-command")
body["issued_at"] = "2026-09-13T00:00:00"
with self.assertRaises(ValidationError):
self.service.publish_execute(body)
with self.assertRaises(ValidationError):
self.service.control_task(
"tenant-demo",
"task-demo",
{
"command_id": "bool-revision",
"action": "pause",
"expected_task_revision": True,
"reason": "x",
},
"bool-revision",
)
def test_readiness_requires_persistent_database_write(self) -> None:
db_path = Path(self.temp.name) / "executor.sqlite3"
os.chmod(db_path, 0o444)
try:
readiness = self.service.readiness()
finally:
os.chmod(db_path, 0o644)
self.assertFalse(readiness["checks"]["database_write"])
self.assertFalse(readiness["ready"])
def test_provider_modes_do_not_fallback(self) -> None:
profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text())
profile["provider_modes"].pop("llm")
path = Path(self.temp.name) / "profile.json"
path.write_text(json.dumps(profile), encoding="utf-8")
with self.assertRaises(ConfigurationError):
AgentCallService(
profile_path=path, object_dir=Path(self.temp.name) / "objects2"
)
def test_same_public_ids_are_tenant_scoped_and_wrong_route_is_dead_lettered(
self,
) -> None:
first = self.command("same-command", "tenant-demo", "same-execution")
second = self.command("same-command", "tenant-b", "same-execution")
self.service.publish_execute(first)
self.service.publish_execute(second)
self.assertEqual(self.service.ingest_commands(3), 2)
self.assertEqual(self.service.schedule_once(), 2)
self.service.wait_for_idle()
self.assertEqual(
self.service.store.one("SELECT COUNT(*) AS n FROM calls")["n"], 2
)
wrong = self.command("wrong-route", "tenant-b", "wrong-execution")
self.broker.publish(
"agent-call.commands.v1",
"agent-call.tenant.tenant-demo-key.call.execute",
wrong,
"wrong-route",
)
self.service._cursor = self.service._tenant_order.index("tenant-demo")
self.service.ingest_commands(1)
self.assertEqual(
self.service.store.one("SELECT COUNT(*) AS n FROM dead_letters")["n"], 1
)
self.assertIsNone(
self.service.store.one(
"SELECT * FROM calls WHERE command_id=?", ("wrong-route",)
)
)
def test_over_budget_tenant_is_retained_without_truncation(self) -> None:
profile = json.loads((ROOT / "docs/contracts/mock-profile.json").read_text())
key = "x" * 225
profile["tenants"] = [
{"tenant_id": "long", "tenant_key": key, "max_concurrency": 1}
]
profile["tasks"] = [
{
"task_id": "long-task",
"tenant_id": "long",
"state": "running",
"revision": 1,
}
]
path = Path(self.temp.name) / "long-profile.json"
path.write_text(json.dumps(profile), encoding="utf-8")
service = AgentCallService(
Path(self.temp.name) / "long.sqlite3",
profile_path=path,
object_dir=Path(self.temp.name) / "long-objects",
broker=InMemoryBroker(),
)
try:
body = self.command("placeholder")
body["tenant_id"], body["tenant_key"] = "long", key
body["payload"]["task_id"] = "long-task"
body["payload"]["execution_id"] = "long-execution"
with self.assertRaises(ValidationError) as error:
service.publish_execute(body)
self.assertEqual(error.exception.code, "TENANT_KEY_UNSUPPORTED")
row = service.store.one(
"SELECT status FROM published_commands WHERE tenant_id=?", ("long",)
)
self.assertEqual(row["status"], "retained")
finally:
service.stop()
def test_control_barrier_blocks_resume_until_all_cells_apply(self) -> None:
pause = {
"command_id": "pause-1",
"action": "pause",
"expected_task_revision": 1,
"reason": "maintenance",
}
self.service.set_cell_health("cell-a", False)
self.service.control_task("tenant-demo", "task-demo", pause, "pause-1")
self.assertEqual(self.service.apply_controls(), 0)
with self.assertRaises(ConflictError) as error:
self.service.control_task(
"tenant-demo",
"task-demo",
{
"command_id": "resume-1",
"action": "resume",
"expected_task_revision": 2,
"reason": "resume",
},
"resume-1",
)
self.assertEqual(error.exception.code, "CONTROL_IN_PROGRESS")
self.service.set_cell_health("cell-a", True)
self.assertEqual(self.service.apply_controls(), 1)
self.assertEqual(
self.service.store.one(
"SELECT COUNT(*) AS n FROM control_barriers WHERE command_id=? AND status='applied'",
("pause-1",),
)["n"],
2,
)
def test_final_transcript_is_idempotent_and_conflicts_are_preserved(self) -> None:
self.service.profile["limits"]["hold_ms"] = 1000
body = self.command("transcript-command", "tenant-demo", "transcript-execution")
self.service.publish_execute(body)
self.service.ingest_commands()
self.service.schedule_once()
deadline = time.monotonic() + 2
call_id = None
while time.monotonic() < deadline:
row = self.service.store.one(
"SELECT * FROM calls WHERE command_id=?", (body["command_id"],)
)
if row and row["started_at"]:
call_id = row["call_id"]
break
time.sleep(0.01)
self.assertIsNotNone(call_id)
assert call_id is not None
self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed")
before = self.service.store.one(
"SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'"
)["n"]
self.service._add_transcript(call_id, 99, "agent", "final text", "confirmed")
after = self.service.store.one(
"SELECT COUNT(*) AS n FROM events WHERE event_type='transcript.updated'"
)["n"]
self.assertEqual(before, after)
with self.assertRaises(ConflictError):
self.service._add_transcript(
call_id, 99, "agent", "different", "confirmed", revision=2
)
with self.assertRaises(ConflictError):
self.service._add_transcript(
call_id, 99, "agent", "different", "confirmed", revision=1
)
self.service._add_transcript(
call_id, 99, "agent", "final text", "sent", revision=3, is_final=False
)
segment = self.service.store.one(
"SELECT text, is_final, playback_state FROM transcripts WHERE segment_id=?",
(f"seg_{call_id}_99_agent",),
)
self.assertEqual(
(segment["text"], segment["is_final"], segment["playback_state"]),
("final text", 1, "confirmed"),
)
self.service._finish_call(call_id, "failed", "TEST_FINISHED")
def test_upload_rejects_metadata_changes(self) -> None:
body = self.command(
"recording-metadata-command", "tenant-demo", "recording-metadata-execution"
)
self.service.publish_execute(body)
self.service.wait_for_idle()
call_id = self.service.get_command("tenant-demo", body["command_id"])["call_id"]
recording = self.service.get_call("tenant-demo", call_id)["recordings"][0]
request = {
"recording_id": recording["recording_id"],
"call_id": call_id,
"content_type": "audio/wav",
"size_bytes": recording["size_bytes"],
"checksum_algorithm": "SHA-256",
"checksum": recording["checksum"],
"channels": 1,
"sample_rate_hz": recording["sample_rate_hz"],
"duration_ms": recording["duration_ms"] + 1,
}
with self.assertRaises(ConflictError) as error:
self.service.create_upload("tenant-demo", request, "metadata-change", "api")
self.assertEqual(error.exception.code, "RECORDING_METADATA_CONFLICT")
def test_recording_is_non_silent_and_upload_completion_is_idempotent(self) -> None:
body = self.command("recording-command", "tenant-demo", "recording-execution")
self.service.publish_execute(body)
self.service.wait_for_idle()
call = self.service.get_call(
"tenant-demo",
self.service.get_command("tenant-demo", body["command_id"])["call_id"],
)
recording = call["recordings"][0]
upload = self.service.store.one(
"SELECT upload_id FROM uploads WHERE tenant_id=? AND recording_id=? ORDER BY created_at DESC LIMIT 1",
("tenant-demo", recording["recording_id"]),
)
self.assertIsNotNone(upload)
assert upload is not None
path = self.service.object_store.final_path_for(upload["upload_id"])
self.assertTrue(any(path.read_bytes()))
request = {
"recording_id": recording["recording_id"],
"call_id": call["call_id"],
"content_type": "audio/wav",
"size_bytes": recording["size_bytes"],
"checksum_algorithm": "SHA-256",
"checksum": recording["checksum"],
"channels": 1,
"sample_rate_hz": recording["sample_rate_hz"],
"duration_ms": recording["duration_ms"],
}
status, first = self.service.create_upload(
"tenant-demo", request, "api-create", "api"
)
status2, second = self.service.create_upload(
"tenant-demo", request, "api-create", "api"
)
self.assertEqual((status, status2), (200, 200))
self.assertEqual(first["upload_id"], second["upload_id"])
self.assertEqual(
self.service.store.one(
"SELECT COUNT(*) AS n FROM recordings WHERE call_id=?",
(call["call_id"],),
)["n"],
1,
)
def test_backup_refuses_to_replace_existing_database(self) -> None:
source = Path(self.temp.name) / "source.sqlite3"
destination = Path(self.temp.name) / "backup.sqlite3"
Store(source)
copy_database(source, destination)
with self.assertRaises(FileExistsError):
copy_database(source, destination)
if __name__ == "__main__":
unittest.main()