feat(agent-call): implement remediation plan

deploy/alerts.yml

@@ -4,18 +4,70 @@ groups:
       - alert: AgentCallOutboxStalled
         expr: agent_call_outbox_pending > 0
         for: 2m
-        labels: {severity: warning}
+        labels:
+          severity: warning
         annotations:
-          summary: agent-call outbox has pending events
+          summary: agent-call outbox is not draining
       - alert: AgentCallDeadLetters
         expr: increase(agent_call_dead_letters_total[5m]) > 0
         for: 1m
-        labels: {severity: critical}
+        labels:
+          severity: critical
         annotations:
-          summary: agent-call rejected or malformed messages entered the DLQ
+          summary: agent-call rejected messages reached the dead-letter log
       - alert: AgentCallNotMockSafe
-        expr: agent_call_mode_info{mode="mock"} != 1
+        expr: agent_call_mode_info{mode!="mock"} > 0
         for: 1m
-        labels: {severity: critical}
+        labels:
+          severity: critical
         annotations:
-          summary: provider mode changed; verify production admission gates
+          summary: agent-call is running outside the supported Mock mode
+      - alert: AgentCallNotReady
+        expr: agent_call_ready == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: agent-call readiness gate is closed
+      - alert: AgentCallClockJump
+        expr: agent_call_clock_jump > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: agent-call stopped new admission after a wall-clock jump
+      - alert: AgentCallUnknownCall
+        expr: agent_call_unknown_calls > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: active call requires reconciliation and still occupies capacity
+      - alert: AgentCallQueueBackpressure
+        expr: sum(agent_call_queue_depth) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: tenant command queues have pending backpressure
+      - alert: AgentCallRecordingFailures
+        expr: increase(agent_call_recording_failures_total[5m]) > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: recording verification failed
+      - alert: AgentCallDiskWarning
+        expr: agent_call_disk_usage_ratio >= 0.70
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: agent-call disk usage is high
+      - alert: AgentCallDiskStop
+        expr: agent_call_disk_usage_ratio >= 0.80
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: agent-call stopped new admissions for disk pressure