HH-445: deploy production observability and runbooks (#96)

* HH-445: deploy production observability and runbooks

* fix(ops): share production database DSN

* fix(HH-445): enforce database TLS gate

* fix(HH-445): preserve production serve command

* fix(prod): require external database dependencies

* fix(prod): unify database host rejection gates

* test(prod): enforce exact database TLS runbook contract

---------

Co-authored-by: Rogee <rogee@ipao.vip>

deploy/prometheus/postgres_queries.yml

@@ -0,0 +1,26 @@
+gochat_background_jobs:
+  query: |
+    SELECT
+      queue,
+      status,
+      COUNT(*)::double precision AS total,
+      COALESCE(MAX(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - CASE
+        WHEN status = 'running' THEN COALESCE(locked_at, updated_at)
+        ELSE scheduled_at
+      END))), 0)::double precision AS oldest_seconds
+    FROM background_jobs
+    WHERE status IN ('queued', 'retrying', 'running')
+    GROUP BY queue, status
+  metrics:
+    - queue:
+        usage: LABEL
+        description: Background job queue.
+    - status:
+        usage: LABEL
+        description: Background job state.
+    - total:
+        usage: GAUGE
+        description: Current background jobs by queue and state.
+    - oldest_seconds:
+        usage: GAUGE
+        description: Age of the oldest background job by queue and state.