feat(agent-call): implement remediation plan

deploy/alerts.yml

@@ -4,18 +4,70 @@ groups:
       - alert: AgentCallOutboxStalled
         expr: agent_call_outbox_pending > 0
         for: 2m
-        labels: {severity: warning}
+        labels:
+          severity: warning
         annotations:
-          summary: agent-call outbox has pending events
+          summary: agent-call outbox is not draining
       - alert: AgentCallDeadLetters
         expr: increase(agent_call_dead_letters_total[5m]) > 0
         for: 1m
-        labels: {severity: critical}
+        labels:
+          severity: critical
         annotations:
-          summary: agent-call rejected or malformed messages entered the DLQ
+          summary: agent-call rejected messages reached the dead-letter log
       - alert: AgentCallNotMockSafe
-        expr: agent_call_mode_info{mode="mock"} != 1
+        expr: agent_call_mode_info{mode!="mock"} > 0
         for: 1m
-        labels: {severity: critical}
+        labels:
+          severity: critical
         annotations:
-          summary: provider mode changed; verify production admission gates
+          summary: agent-call is running outside the supported Mock mode
+      - alert: AgentCallNotReady
+        expr: agent_call_ready == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: agent-call readiness gate is closed
+      - alert: AgentCallClockJump
+        expr: agent_call_clock_jump > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: agent-call stopped new admission after a wall-clock jump
+      - alert: AgentCallUnknownCall
+        expr: agent_call_unknown_calls > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: active call requires reconciliation and still occupies capacity
+      - alert: AgentCallQueueBackpressure
+        expr: sum(agent_call_queue_depth) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: tenant command queues have pending backpressure
+      - alert: AgentCallRecordingFailures
+        expr: increase(agent_call_recording_failures_total[5m]) > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: recording verification failed
+      - alert: AgentCallDiskWarning
+        expr: agent_call_disk_usage_ratio >= 0.70
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: agent-call disk usage is high
+      - alert: AgentCallDiskStop
+        expr: agent_call_disk_usage_ratio >= 0.80
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: agent-call stopped new admissions for disk pressure

deploy/backup_sqlite.py

@@ -1,36 +1,65 @@
+#!/usr/bin/env python3
+"""Create or restore an SQLite backup without silently overwriting a target."""
+
 from __future__ import annotations
 
 import argparse
 import sqlite3
+from contextlib import closing
 from pathlib import Path
 
 
-def copy_database(source: str, destination: str) -> None:
-    source_path = Path(source)
-    destination_path = Path(destination)
+def _check_integrity(con: sqlite3.Connection) -> None:
+    result = con.execute("PRAGMA integrity_check").fetchone()[0]
+    if result != "ok":
+        raise RuntimeError(f"SQLite integrity check failed: {result}")
+
+
+def copy_database(
+    source: str | Path, destination: str | Path, *, replace: bool = False
+) -> None:
+    source_path = Path(source).expanduser().resolve()
+    destination_path = Path(destination).expanduser().resolve()
+    if source_path == destination_path:
+        raise ValueError("source and destination must differ")
     if not source_path.is_file():
-        raise FileNotFoundError(source)
+        raise FileNotFoundError(source_path)
+    if destination_path.exists() and not replace:
+        raise FileExistsError(
+            f"refusing to overwrite existing database: {destination_path}"
+        )
     destination_path.parent.mkdir(parents=True, exist_ok=True)
-    source_db = sqlite3.connect(source_path)
-    destination_db = sqlite3.connect(destination_path)
     try:
-        source_db.backup(destination_db)
-        destination_db.commit()
-    finally:
-        destination_db.close()
-        source_db.close()
+        with closing(
+            sqlite3.connect(f"file:{source_path}?mode=ro", uri=True)
+        ) as source_con:
+            _check_integrity(source_con)
+            with closing(sqlite3.connect(destination_path)) as destination_con:
+                source_con.backup(destination_con)
+                destination_con.commit()
+                _check_integrity(destination_con)
+                version = destination_con.execute(
+                    "SELECT MAX(version) FROM schema_migrations"
+                ).fetchone()[0]
+                if version is None:
+                    raise RuntimeError("backup has no schema migration version")
+    except sqlite3.DatabaseError as exc:
+        raise RuntimeError(f"SQLite backup failed: {exc}") from exc
+    print(f"backup complete: {source_path} -> {destination_path}")
 
 
-def main() -> int:
-    parser = argparse.ArgumentParser(
-        description="offline SQLite backup/restore for agent-call"
-    )
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
     parser.add_argument("action", choices=("backup", "restore"))
     parser.add_argument("source")
     parser.add_argument("destination")
-    args = parser.parse_args()
-    copy_database(args.source, args.destination)
-    print(f"{args.action} complete")
+    parser.add_argument(
+        "--replace",
+        action="store_true",
+        help="explicitly allow replacing the destination",
+    )
+    args = parser.parse_args(argv)
+    copy_database(args.source, args.destination, replace=args.replace)
     return 0
 
 

deploy/build_asterisk_userdata.py

@@ -61,9 +61,7 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None
         f"ASTERISK_IMAGE={shlex.quote(image)}",
         f"ASTERISK_CONFIG_GID={config_gid}",
     ]
-    variables.extend(
-        f"{key}={shlex.quote(value)}" for key, value in payloads.items()
-    )
+    variables.extend(f"{key}={shlex.quote(value)}" for key, value in payloads.items())
     variables.append("")
     if template is None:
         template = Path(__file__).with_name("asterisk_bootstrap.sh")
@@ -76,7 +74,9 @@ def build(cfg, image, password, config_gid=1000, template=None, environment=None
 def write_output(content, path):
     path = Path(path)
     if path.exists() or path.is_symlink():
-        raise ValueError("output already exists; choose a new local secret-bearing path")
+        raise ValueError(
+            "output already exists; choose a new local secret-bearing path"
+        )
     path.parent.mkdir(parents=True, exist_ok=True)
     temporary_path = None
     try:
@@ -105,7 +105,9 @@ def main():
     parser.add_argument("--config", required=True, help="render_asterisk JSON config")
     parser.add_argument("--image", help="immutable registry image@sha256:digest")
     parser.add_argument("--ari-password-file")
-    parser.add_argument("--output", required=True, help="local secret-bearing user-data path")
+    parser.add_argument(
+        "--output", required=True, help="local secret-bearing user-data path"
+    )
     parser.add_argument("--config-gid", type=int, default=1000)
     args = parser.parse_args()
     try:
@@ -113,7 +115,9 @@ def main():
         image = immutable_image(
             args.image or cfg.get("asterisk_image") or os.environ.get("ASTERISK_IMAGE")
         )
-        content = build(cfg, image, read_secret(args.ari_password_file), args.config_gid)
+        content = build(
+            cfg, image, read_secret(args.ari_password_file), args.config_gid
+        )
         if len(content.encode()) > 16 * 1024:
             raise ValueError("generated user-data exceeds the ECS 16 KiB limit")
         write_output(content, args.output)

deploy/check_mock_config.py

@@ -10,6 +10,7 @@ if str(PROJECT_ROOT) not in sys.path:
     sys.path.insert(0, str(PROJECT_ROOT))
 
 _core = importlib.import_module("agent_call.core")
+ValidationError = _core.ValidationError
 queue_name = _core.queue_name
 routing_key = _core.routing_key
 
@@ -30,7 +31,7 @@ def check(profile_path: str | Path) -> list[str]:
         try:
             route = routing_key(tenant["tenant_key"])
             queue_name(tenant["tenant_key"])
-        except Exception as exc:
+        except (KeyError, TypeError, ValidationError) as exc:
             problems.append(f"tenant route invalid: {type(exc).__name__}")
             continue
         if len(route.encode("utf-8")) > 255:

deploy/migrate.py

@@ -15,9 +15,12 @@ Store = _core.Store
 
 def migrate(path: str) -> int:
     store = Store(path)
-    row = store.one("SELECT MAX(version) AS version FROM schema_migrations")
-    print(f"schema_version={row['version']}")
-    return 0
+    try:
+        row = store.one("SELECT MAX(version) AS version FROM schema_migrations")
+        print(f"schema_version={row['version']}")
+        return 0
+    finally:
+        store.close()
 
 
 def main() -> int:

deploy/prometheus.yml

@@ -2,6 +2,9 @@ global:
   scrape_interval: 15s
   evaluation_interval: 15s
 
+rule_files:
+  - /etc/prometheus/alerts.yml
+
 scrape_configs:
   - job_name: agent-call
     metrics_path: /metrics