From fb8328561782a26c3b8aaf1680a0d59dee287abe Mon Sep 17 00:00:00 2001
From: Rogee <rogeecn@users.noreply.github.com>
Date: Sat, 22 Aug 2026 19:39:57 +0800
Subject: [PATCH] HH-445: deploy production observability and runbooks (#96)

* HH-445: deploy production observability and runbooks

* fix(ops): share production database DSN

* fix(HH-445): enforce database TLS gate

* fix(HH-445): preserve production serve command

* fix(prod): require external database dependencies

* fix(prod): unify database host rejection gates

* test(prod): enforce exact database TLS runbook contract

---------

Co-authored-by: Rogee <rogee@ipao.vip>
---
 .env.example                                  |  22 +-
 .github/workflows/ci.yml                      |  26 +-
 backend/cmd/migrate/main.go                   |   6 +
 backend/configs/prometheus_alerts.yml         | 143 +++++++---
 backend/configs/prometheus_alerts_test.yml    | 138 +++++++++-
 backend/internal/config/config_test.go        |  35 ++-
 backend/internal/config/validator.go          |  52 +++-
 backend/scripts/db_backup.sh                  |  19 ++
 backend/scripts/db_backup_test.sh             |  65 +++++
 backend/scripts/db_restore.sh                 |   3 +
 deploy/docker/Dockerfile                      |   2 +-
 deploy/docker/database_client_entrypoint.sh   | 125 +++++++++
 .../docker/database_client_entrypoint_test.sh | 143 ++++++++++
 .../docker/database_host_rejection_cases.txt  |  15 ++
 deploy/docker/docker-compose.prod-smoke.yml   |  82 ++++++
 deploy/docker/docker-compose.prod.yml         | 246 +++++++++++++-----
 deploy/docker/observability_test.sh           | 162 ++++++++++++
 deploy/docker/preflight.sh                    |  41 ++-
 deploy/docker/preflight_test.sh               |  42 +++
 deploy/fluentd/fluent.conf                    | 117 ++-------
 deploy/prometheus/alertmanager.yml            |  17 ++
 deploy/prometheus/blackbox.yml                |   7 +
 deploy/prometheus/postgres_queries.yml        |  26 ++
 deploy/prometheus/prometheus.yml              |  79 ++++++
 docs/README.md                                |   2 +
 docs/ops/01-rolling-upgrade.md                |  16 ++
 docs/ops/02-production-operations.md          | 111 ++++++++
 docs/ops/03-observability-drill.md            | 100 +++++++
 28 files changed, 1622 insertions(+), 220 deletions(-)
 create mode 100755 backend/scripts/db_backup_test.sh
 create mode 100755 deploy/docker/database_client_entrypoint.sh
 create mode 100755 deploy/docker/database_client_entrypoint_test.sh
 create mode 100644 deploy/docker/database_host_rejection_cases.txt
 create mode 100755 deploy/docker/observability_test.sh
 create mode 100644 deploy/prometheus/alertmanager.yml
 create mode 100644 deploy/prometheus/blackbox.yml
 create mode 100644 deploy/prometheus/postgres_queries.yml
 create mode 100644 deploy/prometheus/prometheus.yml
 create mode 100644 docs/ops/02-production-operations.md
 create mode 100644 docs/ops/03-observability-drill.md

diff --git a/.env.example b/.env.example
index 5bf6f4f5..12865908 100644
--- a/.env.example
+++ b/.env.example
@@ -9,16 +9,20 @@ GOCHAT_SERVER_TRUSTED_PROXIES=10.0.0.0/8
 GOCHAT_DATABASE_DSN=postgres://gochat:CHANGE_ME@db.CHANGE_ME.example.com:5432/gochat_production?sslmode=verify-full
 GOCHAT_REDIS_DSN=rediss://:CHANGE_ME@redis.CHANGE_ME.example.com:6380/0
 
-POSTGRES_DB=gochat_production
-POSTGRES_USER=gochat
-POSTGRES_PASSWORD=CHANGE_ME
 # Release mode requires external PostgreSQL/Redis endpoints with verified TLS.
+# Set a complete GOCHAT_DATABASE_DSN with sslmode=verify-ca or verify-full.
+# Compose passes this value unchanged to every database client. Keep these fixed
+# container paths in the DSN and point the host variables at untracked files.
+# GOCHAT_DATABASE_DSN='postgres://user:pass@db.example.com:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+# GOCHAT_DATABASE_TLS_CA_FILE=../../.secrets/external-db-ca.crt
+# GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE=../../.secrets/external-db-client.crt
+# GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE=../../.secrets/external-db-client.key
+# Set all three files to this group and grant group read permission (0640 is
+# suitable for the private key). Compose adds the group to every DB client.
+# GOCHAT_DATABASE_TLS_GID=65534
 
-POSTGRES_IMAGE_REF=pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
-REDIS_IMAGE_REF=redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf
 MEILI_IMAGE_REF=getmeili/meilisearch:v1.13@sha256:bed3fb650e62da53145777204891159242f6ea4ce69e215b36223af4aa64a0ae
 
-REDIS_PASSWORD=CHANGE_ME
 MEILI_MASTER_KEY=CHANGE_ME
 GOCHAT_JWT_SECRET=CHANGE_ME_WITH_AT_LEAST_32_RANDOM_CHARACTERS
 # Optional during a bounded rotation window; comma-separated old 32+ byte secrets.
@@ -33,6 +37,12 @@ GOCHAT_ENCRYPTION_ENABLED=true
 GOCHAT_ENCRYPTION_CURRENT_KEY_VERSION=1
 GOCHAT_ENCRYPTION_AES_KEY=CHANGE_ME
 
+# Alertmanager reads the HTTPS receiver from this untracked one-line file.
+ALERTMANAGER_WEBHOOK_URL_FILE=../../.secrets/alertmanager-webhook-url
+PROMETHEUS_PORT=9090
+ALERTMANAGER_PORT=9093
+PROMETHEUS_RETENTION=30d
+
 # Optional connector. Supply the digest published by its release pipeline.
 SHANGWUTONG_IMAGE_REF=ghcr.io/rogeecn/shangwutong@sha256:CHANGE_ME
 GOCHAT_CONNECTOR_SERVICE_TOKEN=
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 07ae5e6d..58b99e42 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,8 +39,8 @@ jobs:
           chrome-version: 152.0.7977.54
       - name: Install browser harness
         run: python -m pip install --require-hashes -r .github/requirements-browser-harness.txt
-      - name: Test Prometheus alert rules
-        run: docker run --rm --entrypoint promtool -v "$PWD/backend/configs:/configs:ro" prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 test rules /configs/prometheus_alerts_test.yml
+      - name: Test production observability configuration
+        run: deploy/docker/observability_test.sh
       - name: Test quality gate failure contracts
         working-directory: backend
         run: python3 scripts/quality_gate_test.py
@@ -389,7 +389,10 @@ jobs:
       GOCHAT_IMAGE_REF: gochat:production-smoke
       GOCHAT_PORT: "38080"
       GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: https://chat.ci.rogeecn.com
-      GOCHAT_DATABASE_DSN: "postgres://gochat:ci-postgres-secret@postgres:5432/gochat_production?sslmode=verify-full&sslrootcert=/run/tls/ca.crt"
+      GOCHAT_DATABASE_DSN: "postgres://gochat:ci-postgres-secret@db.smoke.test:5432/gochat_production?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key"
+      GOCHAT_DATABASE_TLS_CA_FILE: ${{ github.workspace }}/.tmp/gochat-tls/ca.crt
+      GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE: ${{ github.workspace }}/.tmp/gochat-tls/postgres.crt
+      GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE: ${{ github.workspace }}/.tmp/gochat-tls/postgres.key
       GOCHAT_REDIS_DSN: "rediss://:ci-redis-secret@redis:6379/0"
       GOCHAT_ENCRYPTION_AES_KEY: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
       GOCHAT_TLS_DIR: ${{ github.workspace }}/.tmp/gochat-tls
@@ -444,20 +447,25 @@ jobs:
           openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=GoChat CI CA' \
             -keyout "$GOCHAT_TLS_DIR/ca.key" -out "$GOCHAT_TLS_DIR/ca.crt"
           for service in postgres redis; do
-            openssl req -newkey rsa:2048 -nodes -subj "/CN=$service" \
-              -addext "subjectAltName=DNS:$service" \
+            hostname=$service
+            [[ $service == postgres ]] && hostname=db.smoke.test
+            openssl req -newkey rsa:2048 -nodes -subj "/CN=$hostname" \
+              -addext "subjectAltName=DNS:$hostname" \
               -keyout "$GOCHAT_TLS_DIR/$service.key" -out "$GOCHAT_TLS_DIR/$service.csr"
-            printf 'subjectAltName=DNS:%s\n' "$service" > "$GOCHAT_TLS_DIR/$service.ext"
+            printf 'subjectAltName=DNS:%s\n' "$hostname" > "$GOCHAT_TLS_DIR/$service.ext"
             openssl x509 -req -days 1 -CA "$GOCHAT_TLS_DIR/ca.crt" -CAkey "$GOCHAT_TLS_DIR/ca.key" \
               -CAcreateserial -extfile "$GOCHAT_TLS_DIR/$service.ext" \
               -in "$GOCHAT_TLS_DIR/$service.csr" -out "$GOCHAT_TLS_DIR/$service.crt"
           done
-          chmod 600 "$GOCHAT_TLS_DIR"/*.key
+          chmod 640 "$GOCHAT_TLS_DIR"/postgres.key
+          chmod 600 "$GOCHAT_TLS_DIR"/ca.key "$GOCHAT_TLS_DIR"/redis.key
           chmod 644 "$GOCHAT_TLS_DIR"/*.crt
+          echo "GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$GOCHAT_TLS_DIR/postgres.key")" >> "$GITHUB_ENV"
       - name: Start production Compose and smoke core pages
         run: |
           deploy/docker/preflight_test.sh
-          docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
+          GOCHAT_TEST_IMAGE="$GOCHAT_IMAGE_REF" deploy/docker/database_client_entrypoint_test.sh
+          docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert config["services"]["gochat"]["command"] == ["serve"]; assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
           compose=(docker compose -f deploy/docker/docker-compose.prod.yml -f deploy/docker/docker-compose.prod-smoke.yml)
           "${compose[@]}" --profile ops run --rm migrate
           "${compose[@]}" up -d --wait gochat shangwutong
@@ -473,6 +481,8 @@ jobs:
           done
           curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" > "$RUNNER_TEMP/health.json"
           python3 backend/scripts/validate_health_json.py "$RUNNER_TEMP/health.json"
+          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/ready" | grep -q '"ready":true'
+          ! "${compose[@]}" logs --no-color gochat | grep -F 'slice bounds out of range'
           curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'
           curl -fsS "http://127.0.0.1:$GOCHAT_PORT/runtime-config.js" | grep -q 'window.__GOCHAT_CONFIG__'
           curl -fsS -o "$RUNNER_TEMP/favicon-32x32.png" "http://127.0.0.1:$GOCHAT_PORT/favicon-32x32.png"
diff --git a/backend/cmd/migrate/main.go b/backend/cmd/migrate/main.go
index 4e702407..bc532df9 100644
--- a/backend/cmd/migrate/main.go
+++ b/backend/cmd/migrate/main.go
@@ -32,6 +32,12 @@ func main() {
 	}
 
 	dbURL := cfg.Database.MigrateDSN()
+	if cfg.Server.Mode == "release" {
+		if err := config.ValidateProductionDatabaseDSN(dbURL); err != nil {
+			fmt.Fprintf(os.Stderr, "Error validating database DSN: %v\n", err)
+			os.Exit(1)
+		}
+	}
 	migrationsPath := cfg.Database.GetMigrationsPath()
 
 	switch command {
diff --git a/backend/configs/prometheus_alerts.yml b/backend/configs/prometheus_alerts.yml
index fef7c9ed..0b4ec439 100644
--- a/backend/configs/prometheus_alerts.yml
+++ b/backend/configs/prometheus_alerts.yml
@@ -1,84 +1,159 @@
-# GoChat Prometheus Alert Rules
-# Reference: Chatwoot production monitoring with Sidekiq queue alerts
-# Adjust thresholds based on your deployment scale
-
 groups:
-  - name: gochat-app
+  - name: gochat-application
     rules:
-      # Application down
       - alert: GoChatAppDown
         expr: up{job="gochat"} == 0
         for: 1m
         labels:
           severity: critical
         annotations:
-          summary: "GoChat application is down"
-          description: "GoChat instance {{ $labels.instance }} has been down for more than 1 minute."
+          summary: GoChat application is down
+          description: GoChat metrics have been unreachable for more than one minute.
+
+      - alert: GoChatDatabaseReadinessFailed
+        expr: probe_success{job="gochat-database-readiness"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: GoChat database readiness failed
+          description: The application cannot complete its PostgreSQL dependency check.
+
+      - alert: GoChatRedisReadinessFailed
+        expr: probe_success{job="gochat-redis-readiness"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: GoChat Redis readiness failed
+          description: The application cannot complete its Redis dependency check.
 
-      # High error rate
       - alert: GoChatHighErrorRate
-        expr: sum by (job, instance) (rate(http_requests_total{job="gochat", status=~"5.."}[5m])) / sum by (job, instance) (rate(http_requests_total{job="gochat"}[5m])) > 0.05
+        expr: |
+          sum by (job, instance) (rate(http_request_errors_total{job="gochat"}[5m]))
+            / sum by (job, instance) (rate(http_requests_total{job="gochat"}[5m])) > 0.05
+          and sum by (job, instance) (rate(http_requests_total{job="gochat"}[5m])) > 0
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "GoChat error rate above 5%"
-          description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
+          summary: GoChat error rate above 5%
+          description: Error rate is {{ $value | humanizePercentage }} over the last 5 minutes.
 
-      # High memory usage
       - alert: GoChatHighMemory
-        expr: gochat_go_memory_alloc_bytes / (1024 * 1024) > 400
+        expr: gochat_go_memory_alloc_bytes / 1024 / 1024 > 400
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "GoChat memory usage above 400MB"
-          description: "Memory allocation is {{ $value }}MB."
+          summary: GoChat memory usage above 400 MB
+          description: Allocated heap is {{ $value | humanize }} MB.
 
-      # Too many goroutines
       - alert: GoChatHighGoroutines
         expr: gochat_go_goroutines > 1000
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "GoChat goroutine count above 1000"
-          description: "{{ $value }} goroutines running."
+          summary: GoChat goroutine count above 1000
+          description: GoChat has {{ $value | humanize }} goroutines.
 
-  - name: gochat-infra
+  - name: gochat-workers
     rules:
-      # PostgreSQL down
-      - alert: GoChatPostgresDown
-        expr: up{job="gochat-postgres"} == 0
+      - alert: GoChatWorkerDown
+        expr: |
+          time() - max(container_last_seen{container_label_com_docker_compose_service="worker"}) > 60
+          or absent(container_last_seen{container_label_com_docker_compose_service="worker"})
         for: 1m
         labels:
           severity: critical
         annotations:
-          summary: "PostgreSQL is down"
+          summary: GoChat worker is down
+          description: cAdvisor has not observed a production worker container for more than one minute.
 
-      # Redis down
-      - alert: GoChatRedisDown
-        expr: up{job="gochat-redis"} == 0
+      - alert: GoChatCriticalQueueBacklog
+        expr: |
+          sum by (queue) (gochat_background_jobs_total{queue=~"critical|high",status=~"queued|retrying"}) > 25
+          or max by (queue) (gochat_background_jobs_oldest_seconds{queue=~"critical|high",status=~"queued|retrying"}) > 300
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Critical background queue is delayed
+          description: Queue {{ $labels.queue }} exceeds 25 ready jobs or its oldest job is over five minutes old.
+
+      - alert: GoChatWorkerJobsStuck
+        expr: max by (queue) (gochat_background_jobs_oldest_seconds{status="running"}) > 900
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Background jobs are stuck
+          description: Queue {{ $labels.queue }} has a running job older than 15 minutes.
+
+  - name: gochat-infrastructure
+    rules:
+      - alert: GoChatPostgresExporterDown
+        expr: up{job="postgres-exporter"} == 0
         for: 1m
         labels:
           severity: critical
         annotations:
-          summary: "Redis is down"
+          summary: PostgreSQL exporter is down
+          description: Prometheus cannot scrape the PostgreSQL exporter.
+
+      - alert: GoChatRedisExporterDown
+        expr: up{job="redis-exporter"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Redis exporter is down
+          description: Prometheus cannot scrape the Redis exporter.
 
-      # Redis memory approaching limit
       - alert: GoChatRedisMemoryHigh
-        expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.8
+        expr: redis_memory_max_bytes > 0 and redis_memory_used_bytes / redis_memory_max_bytes > 0.8
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Redis memory usage above 80%"
+          summary: Redis memory usage above 80%
+          description: Redis is approaching its configured memory ceiling.
 
-      # PostgreSQL connections exhausted
       - alert: GoChatPostgresConnectionsHigh
-        expr: pg_stat_activity_count / pg_settings_max_connections > 0.8
+        expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections) > 0.8
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "PostgreSQL connection usage above 80%"
+          summary: PostgreSQL connection usage above 80%
+          description: PostgreSQL is approaching its connection limit.
+
+      - alert: GoChatBackupStale
+        expr: |
+          time() - gochat_backup_last_success_timestamp_seconds > gochat_backup_rpo_target_seconds
+          or absent(gochat_backup_last_success_timestamp_seconds)
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: GoChat backup is stale
+          description: No successful encrypted off-site backup exists inside the declared RPO.
+
+      - alert: GoChatAlertmanagerDown
+        expr: up{job="alertmanager"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Alertmanager is down
+          description: Prometheus cannot deliver notifications to Alertmanager.
+
+      - alert: ShangwutongReadinessFailed
+        expr: probe_success{job="shangwutong-readiness"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Shangwutong readiness failed
+          description: The production Connector readiness endpoint is failing.
diff --git a/backend/configs/prometheus_alerts_test.yml b/backend/configs/prometheus_alerts_test.yml
index 420c9561..d391b1a4 100644
--- a/backend/configs/prometheus_alerts_test.yml
+++ b/backend/configs/prometheus_alerts_test.yml
@@ -4,15 +4,16 @@ rule_files:
 evaluation_interval: 1m
 
 tests:
-  - interval: 1m
+  - name: HTTP error rate only fires above five percent
+    interval: 1m
     input_series:
-      - series: 'http_requests_total{job="gochat",instance="below-threshold",method="GET",route="/ok",status="200"}'
-        values: '0+96x12'
-      - series: 'http_requests_total{job="gochat",instance="below-threshold",method="GET",route="/error",status="500"}'
-        values: '0+4x12'
-      - series: 'http_requests_total{job="gochat",instance="above-threshold",method="GET",route="/ok",status="200"}'
-        values: '0+94x12'
-      - series: 'http_requests_total{job="gochat",instance="above-threshold",method="GET",route="/error",status="500"}'
+      - series: 'http_requests_total{job="gochat",instance="below-threshold"}'
+        values: '0+100x12'
+      - series: 'http_request_errors_total{job="gochat",instance="below-threshold"}'
+        values: '0+5x12'
+      - series: 'http_requests_total{job="gochat",instance="above-threshold"}'
+        values: '0+100x12'
+      - series: 'http_request_errors_total{job="gochat",instance="above-threshold"}'
         values: '0+6x12'
     alert_rule_test:
       - eval_time: 10m
@@ -25,3 +26,124 @@ tests:
             exp_annotations:
               summary: GoChat error rate above 5%
               description: Error rate is 6% over the last 5 minutes.
+
+  - name: Dependency failures page after one minute
+    interval: 1m
+    input_series:
+      - series: 'probe_success{job="gochat-database-readiness",instance="database"}'
+        values: '0x5'
+      - series: 'probe_success{job="gochat-redis-readiness",instance="redis"}'
+        values: '0x5'
+      - series: 'up{job="gochat",instance="gochat:3000"}'
+        values: '0x5'
+      - series: 'up{job="alertmanager",instance="alertmanager:9093"}'
+        values: '0x5'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: GoChatDatabaseReadinessFailed
+        exp_alerts:
+          - exp_labels:
+              instance: database
+              job: gochat-database-readiness
+              severity: critical
+            exp_annotations:
+              summary: GoChat database readiness failed
+              description: The application cannot complete its PostgreSQL dependency check.
+      - eval_time: 2m
+        alertname: GoChatRedisReadinessFailed
+        exp_alerts:
+          - exp_labels:
+              instance: redis
+              job: gochat-redis-readiness
+              severity: critical
+            exp_annotations:
+              summary: GoChat Redis readiness failed
+              description: The application cannot complete its Redis dependency check.
+      - eval_time: 2m
+        alertname: GoChatAppDown
+        exp_alerts:
+          - exp_labels:
+              instance: gochat:3000
+              job: gochat
+              severity: critical
+            exp_annotations:
+              summary: GoChat application is down
+              description: GoChat metrics have been unreachable for more than one minute.
+      - eval_time: 2m
+        alertname: GoChatAlertmanagerDown
+        exp_alerts:
+          - exp_labels:
+              instance: alertmanager:9093
+              job: alertmanager
+              severity: critical
+            exp_annotations:
+              summary: Alertmanager is down
+              description: Prometheus cannot deliver notifications to Alertmanager.
+
+  - name: Worker and critical queues page
+    interval: 1m
+    input_series:
+      - series: 'container_last_seen{container_label_com_docker_compose_service="worker"}'
+        values: '0x10'
+      - series: 'gochat_background_jobs_total{queue="critical",status="queued"}'
+        values: '30x10'
+      - series: 'gochat_background_jobs_oldest_seconds{queue="critical",status="queued"}'
+        values: '600x10'
+      - series: 'gochat_background_jobs_oldest_seconds{queue="default",status="running"}'
+        values: '1000x10'
+    alert_rule_test:
+      - eval_time: 6m
+        alertname: GoChatWorkerDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: GoChat worker is down
+              description: cAdvisor has not observed a production worker container for more than one minute.
+      - eval_time: 6m
+        alertname: GoChatCriticalQueueBacklog
+        exp_alerts:
+          - exp_labels:
+              queue: critical
+              severity: critical
+            exp_annotations:
+              summary: Critical background queue is delayed
+              description: Queue critical exceeds 25 ready jobs or its oldest job is over five minutes old.
+      - eval_time: 6m
+        alertname: GoChatWorkerJobsStuck
+        exp_alerts:
+          - exp_labels:
+              queue: default
+              severity: critical
+            exp_annotations:
+              summary: Background jobs are stuck
+              description: Queue default has a running job older than 15 minutes.
+
+  - name: Backup RPO pages and clears
+    interval: 1h
+    input_series:
+      - series: gochat_backup_last_success_timestamp_seconds
+        values: '0x4'
+      - series: gochat_backup_rpo_target_seconds
+        values: '3600x4'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: GoChatBackupStale
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: GoChat backup is stale
+              description: No successful encrypted off-site backup exists inside the declared RPO.
+
+  - name: Fresh backup does not page
+    interval: 1h
+    input_series:
+      - series: gochat_backup_last_success_timestamp_seconds
+        values: '7000x4'
+      - series: gochat_backup_rpo_target_seconds
+        values: '86400x4'
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: GoChatBackupStale
+        exp_alerts: []
diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go
index 5921ee75..2f086586 100644
--- a/backend/internal/config/config_test.go
+++ b/backend/internal/config/config_test.go
@@ -1,9 +1,11 @@
 package config
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -292,15 +294,18 @@ func TestValidate_ReleaseDatabaseTLS(t *testing.T) {
 	}{
 		{"external disable", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=disable", true},
 		{"external missing sslmode", "postgres://gochat:database-secret@db.example.test:5432/gochat", true},
+		{"external duplicate downgrade", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=disable&sslmode=verify-full", true},
+		{"external duplicate allowed", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full&sslmode=verify-full", true},
+		{"external non-fixed certificate path", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/tmp/ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key", true},
 		{"external require", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=require", true},
 		{"external verify ca", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-ca", false},
 		{"external verify full", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full", false},
-		{"built-in compose disable", "postgres://gochat:database-secret@postgres:5432/gochat?sslmode=disable", true},
+		{"external fixed certificate paths", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key", false},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg.Database.DSN = tt.dsn
 			if tt.wantErr {
-				assert.ErrorContains(t, Validate(cfg), "production database DSN must use sslmode")
+				assert.Error(t, Validate(cfg))
 			} else {
 				assert.NoError(t, Validate(cfg))
 			}
@@ -308,6 +313,32 @@ func TestValidate_ReleaseDatabaseTLS(t *testing.T) {
 	}
 }
 
+func TestValidateProductionDatabaseDSN_RejectsHostMatrix(t *testing.T) {
+	file, err := os.Open("../../../deploy/docker/database_host_rejection_cases.txt")
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, file.Close()) })
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		name, dsn, ok := strings.Cut(scanner.Text(), "|")
+		require.True(t, ok)
+		t.Run(name, func(t *testing.T) {
+			assert.ErrorContains(t, ValidateProductionDatabaseDSN(dsn), "must use an external PostgreSQL host")
+		})
+	}
+	require.NoError(t, scanner.Err())
+}
+
+func TestProductionDatabaseTLSRunbookContract(t *testing.T) {
+	runbook, err := os.ReadFile("../../../docs/ops/02-production-operations.md")
+	require.NoError(t, err)
+	runbookText := string(runbook)
+	assert.Equal(t, 1, strings.Count(runbookText, "`sslmode=verify-ca|verify-full`"))
+	assert.Equal(t, 1, strings.Count(runbookText, "sslmode="))
+	assert.NotContains(t, runbookText, "sslmode=disable")
+	assert.NotContains(t, runbookText, "sslmode=require")
+}
+
 func TestLoadWithEnv_ProductionRequiresOverlay(t *testing.T) {
 	tmpDir := t.TempDir()
 	require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "configs"), 0o755))
diff --git a/backend/internal/config/validator.go b/backend/internal/config/validator.go
index 5ee72941..4aed0387 100644
--- a/backend/internal/config/validator.go
+++ b/backend/internal/config/validator.go
@@ -125,9 +125,8 @@ func Validate(cfg *Config) error {
 		if password, ok := dbURL.User.Password(); !ok || password == "" || containsPlaceholder(password) {
 			return fmt.Errorf("production database password is required and must not contain placeholders")
 		}
-		sslMode := dbURL.Query().Get("sslmode")
-		if sslMode != "verify-full" && sslMode != "verify-ca" {
-			return fmt.Errorf("production database DSN must use sslmode=verify-full or verify-ca")
+		if err := ValidateProductionDatabaseDSN(cfg.Database.DSN); err != nil {
+			return err
 		}
 		if redisURL.User == nil {
 			return fmt.Errorf("production Redis credentials are required")
@@ -183,6 +182,53 @@ func Validate(cfg *Config) error {
 	return nil
 }
 
+// ValidateProductionDatabaseDSN protects every direct Go database client from
+// local targets, duplicate sslmode downgrades, and unsafe certificate paths.
+func ValidateProductionDatabaseDSN(dsn string) error {
+	dbURL, err := url.Parse(dsn)
+	if err != nil {
+		return fmt.Errorf("invalid production database DSN: %w", err)
+	}
+	hostname := strings.TrimSuffix(strings.ToLower(dbURL.Hostname()), ".")
+	if zone := strings.LastIndexByte(hostname, '%'); zone >= 0 {
+		hostname = hostname[:zone]
+	}
+	ip := net.ParseIP(hostname)
+	if hostname == "" || hostname == "postgres" || hostname == "localhost" || ip != nil && ip.IsLoopback() {
+		return fmt.Errorf("production database DSN must use an external PostgreSQL host")
+	}
+	query, err := url.ParseQuery(dbURL.RawQuery)
+	if err != nil {
+		return fmt.Errorf("invalid production database DSN query: %w", err)
+	}
+	modes := query["sslmode"]
+	if len(modes) != 1 {
+		return fmt.Errorf("production database DSN must contain exactly one sslmode")
+	}
+	if modes[0] != "verify-ca" && modes[0] != "verify-full" {
+		return fmt.Errorf("production database DSN must use sslmode=verify-full or verify-ca")
+	}
+
+	fixedPaths := map[string]string{
+		"sslrootcert": "/run/secrets/external-db-ca.crt",
+		"sslcert":     "/run/secrets/external-db-client.crt",
+		"sslkey":      "/run/secrets/external-db-client.key",
+	}
+	usesCertificateFiles := false
+	for parameter := range fixedPaths {
+		usesCertificateFiles = usesCertificateFiles || len(query[parameter]) > 0
+	}
+	if usesCertificateFiles {
+		for parameter, path := range fixedPaths {
+			values := query[parameter]
+			if len(values) != 1 || values[0] != path {
+				return fmt.Errorf("production database DSN %s must appear exactly once and use %s", parameter, path)
+			}
+		}
+	}
+	return nil
+}
+
 func containsPlaceholder(value string) bool {
 	value = strings.ToLower(value)
 	return strings.Contains(value, "change_me") || strings.Contains(value, "change-me") || strings.Contains(value, "changeme")
diff --git a/backend/scripts/db_backup.sh b/backend/scripts/db_backup.sh
index ed360ee8..77332d3f 100755
--- a/backend/scripts/db_backup.sh
+++ b/backend/scripts/db_backup.sh
@@ -5,6 +5,9 @@
 set -euo pipefail
 umask 077
 
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+"$script_dir/database_client_entrypoint.sh" --check
+
 dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
 storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
 connector=${GOCHAT_CONNECTOR_BACKUP_FILE:?GOCHAT_CONNECTOR_BACKUP_FILE is required}
@@ -12,6 +15,7 @@ backup_dir=${GOCHAT_BACKUP_DIR:-/var/backups/gochat}
 offsite_dir=${GOCHAT_BACKUP_OFFSITE_DIR:?GOCHAT_BACKUP_OFFSITE_DIR is required}
 passphrase_file=${GOCHAT_BACKUP_PASSPHRASE_FILE:?GOCHAT_BACKUP_PASSPHRASE_FILE is required}
 retention_days=${GOCHAT_BACKUP_RETENTION_DAYS:-30}
+metrics_file=${GOCHAT_BACKUP_METRICS_FILE:-}
 version=${GOCHAT_VERSION:-unknown}
 timestamp=$(date -u +%Y%m%dT%H%M%SZ)
 
@@ -59,4 +63,19 @@ openssl enc -d -aes-256-cbc -pbkdf2 -pass "file:$passphrase_file" -in "$bundle"
 cp "$bundle" "$bundle.sha256" "$offsite_dir/"
 find "$backup_dir" "$offsite_dir" -maxdepth 1 -type f -name 'gochat-*.tar.enc*' -mtime "+$retention_days" -delete
 
+if [[ -n $metrics_file ]]; then
+  install -d -m 0755 "$(dirname "$metrics_file")"
+  metrics_tmp=$metrics_file.tmp
+  {
+    echo '# HELP gochat_backup_last_success_timestamp_seconds Unix time of the last verified off-site backup.'
+    echo '# TYPE gochat_backup_last_success_timestamp_seconds gauge'
+    echo "gochat_backup_last_success_timestamp_seconds $created_at_epoch"
+    echo '# HELP gochat_backup_rpo_target_seconds Maximum allowed age of the latest backup.'
+    echo '# TYPE gochat_backup_rpo_target_seconds gauge'
+    echo 'gochat_backup_rpo_target_seconds 86400'
+  } >"$metrics_tmp"
+  chmod 0644 "$metrics_tmp"
+  mv "$metrics_tmp" "$metrics_file"
+fi
+
 echo "backup=$bundle offsite=$offsite_dir/$(basename "$bundle") version=$version created_at=$timestamp"
diff --git a/backend/scripts/db_backup_test.sh b/backend/scripts/db_backup_test.sh
new file mode 100755
index 00000000..81c5df47
--- /dev/null
+++ b/backend/scripts/db_backup_test.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin" "$tmp/scripts" "$tmp/storage" "$tmp/local" "$tmp/offsite" "$tmp/metrics"
+cp "$script_dir/db_backup.sh" "$script_dir/../../deploy/docker/database_client_entrypoint.sh" "$tmp/scripts/"
+printf 'attachment\n' >"$tmp/storage/file.txt"
+printf 'connector\n' >"$tmp/connector.db"
+printf 'test-passphrase\n' >"$tmp/passphrase"
+
+cat >"$tmp/bin/psql" <<'EOF'
+#!/usr/bin/env bash
+echo 160000
+EOF
+cat >"$tmp/bin/pg_dump" <<'EOF'
+#!/usr/bin/env bash
+if [[ $1 == --version ]]; then
+  echo 'pg_dump (PostgreSQL) 16.0'
+  exit
+fi
+while (($#)); do
+  if [[ $1 == --file ]]; then
+    printf 'dump\n' >"$2"
+    exit
+  fi
+  shift
+done
+exit 1
+EOF
+cat >"$tmp/bin/pg_restore" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+chmod +x "$tmp/bin/psql" "$tmp/bin/pg_dump" "$tmp/bin/pg_restore"
+
+if GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=disable' "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
+  echo 'backup gate accepted sslmode=disable' >&2
+  exit 1
+fi
+grep -F 'sslmode must be verify-ca or verify-full' "$tmp/rejected" >/dev/null
+while IFS='|' read -r name dsn; do
+  if GOCHAT_DATABASE_DSN=$dsn "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
+    echo "backup gate accepted $name" >&2
+    exit 1
+  fi
+  grep -F 'must use an external PostgreSQL host' "$tmp/rejected" >/dev/null
+done < "$script_dir/../../deploy/docker/database_host_rejection_cases.txt"
+
+PATH="$tmp/bin:$PATH" \
+GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=verify-full' \
+GOCHAT_STORAGE_PATH="$tmp/storage" \
+GOCHAT_CONNECTOR_BACKUP_FILE="$tmp/connector.db" \
+GOCHAT_BACKUP_DIR="$tmp/local" \
+GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite" \
+GOCHAT_BACKUP_PASSPHRASE_FILE="$tmp/passphrase" \
+GOCHAT_BACKUP_METRICS_FILE="$tmp/metrics/gochat_backup.prom" \
+  "$tmp/scripts/db_backup.sh" >"$tmp/output"
+
+grep -Eq '^gochat_backup_last_success_timestamp_seconds [0-9]+$' "$tmp/metrics/gochat_backup.prom"
+grep -Fx 'gochat_backup_rpo_target_seconds 86400' "$tmp/metrics/gochat_backup.prom" >/dev/null
+test "$(find "$tmp/offsite" -name 'gochat-*.tar.enc' | wc -l)" -eq 1
+grep -F 'backup=' "$tmp/output" >/dev/null
+echo 'backup metric test passed'
diff --git a/backend/scripts/db_restore.sh b/backend/scripts/db_restore.sh
index d4f742be..2faad48e 100755
--- a/backend/scripts/db_restore.sh
+++ b/backend/scripts/db_restore.sh
@@ -5,6 +5,9 @@
 set -euo pipefail
 umask 077
 
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+"$script_dir/database_client_entrypoint.sh" --check
+
 bundle=${1:?usage: db_restore.sh /path/to/gochat-*.tar.enc}
 dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
 storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 7d3a3fbc..6fe22a9f 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -78,7 +78,7 @@ COPY --from=builder /migrate /app/migrate
 COPY --chown=gochat:gochat --from=frontend-builder /app/frontend/dist /app/frontend/dist
 COPY backend/configs/ /app/configs/
 COPY backend/migrations/ /app/migrations/
-COPY backend/scripts/db_backup.sh backend/scripts/db_restore.sh /app/scripts/
+COPY backend/scripts/db_backup.sh backend/scripts/db_restore.sh deploy/docker/database_client_entrypoint.sh /app/scripts/
 
 ENV GOCHAT_FRONTEND_DIST=/app/frontend/dist
 
diff --git a/deploy/docker/database_client_entrypoint.sh b/deploy/docker/database_client_entrypoint.sh
new file mode 100755
index 00000000..42d18bac
--- /dev/null
+++ b/deploy/docker/database_client_entrypoint.sh
@@ -0,0 +1,125 @@
+#!/bin/sh
+set -eu
+
+is_local_database_host() {
+  database_host_value=$1
+  case $database_host_value in
+    '' | postgres | localhost | 127.*) return 0 ;;
+  esac
+
+  if [ "$(printf '%s' "$database_host_value" | tr -d '0:')" = 1 ]; then
+    return 0
+  fi
+
+  database_ipv6_prefix=${database_host_value%:*}
+  database_ipv6_tail=${database_host_value##*:}
+  database_ipv6_marker=$(printf '%s' "$database_ipv6_prefix" | tr -d '0:')
+  case $database_ipv6_tail in
+    0.0.0.1) [ -z "$database_ipv6_marker" ] && return 0 ;;
+    127.*) [ "$database_ipv6_marker" = ffff ] && return 0 ;;
+  esac
+
+  database_ipv6_high=${database_ipv6_prefix##*:}
+  database_ipv6_prefix=${database_ipv6_prefix%:*}
+  database_ipv6_marker=$(printf '%s' "$database_ipv6_prefix" | tr -d '0:')
+  case $database_ipv6_high in
+    7f[0-9a-f][0-9a-f]) [ "$database_ipv6_marker" = ffff ] && return 0 ;;
+  esac
+  return 1
+}
+
+validate_production_database_dsn() {
+  database_dsn=$1
+  database_dsn_label=${2:-production database DSN}
+  database_dsn_uses_certificate_files=0
+  if [ -z "$database_dsn" ]; then
+    echo "$database_dsn_label is required" >&2
+    return 1
+  fi
+
+  database_authority=${database_dsn#*://}
+  database_authority=${database_authority%%/*}
+  database_hostport=${database_authority##*@}
+  case $database_hostport in
+    \[* ) database_host=${database_hostport#\[}; database_host=${database_host%%\]*} ;;
+    * ) database_host=${database_hostport%%:*} ;;
+  esac
+  database_host=$(printf '%s' "$database_host" | tr '[:upper:]' '[:lower:]')
+  database_host=${database_host%.}
+  database_host=${database_host%%%*}
+  if is_local_database_host "$database_host"; then
+    echo "$database_dsn_label must use an external PostgreSQL host" >&2
+    return 1
+  fi
+
+  database_query=
+  case $database_dsn in
+    *\?*) database_query=${database_dsn#*\?}; database_query=${database_query%%#*} ;;
+  esac
+
+  sslmode_count=0
+  sslmode=
+  sslrootcert_count=0
+  sslcert_count=0
+  sslkey_count=0
+  database_parameters=$database_query
+  while [ -n "$database_parameters" ]; do
+    case $database_parameters in
+      *'&'*) database_parameter=${database_parameters%%&*}; database_parameters=${database_parameters#*&} ;;
+      *) database_parameter=$database_parameters; database_parameters= ;;
+    esac
+    database_key=${database_parameter%%=*}
+    database_value=${database_parameter#*=}
+    case $database_key in
+      sslmode) sslmode_count=$((sslmode_count + 1)); sslmode=$database_value ;;
+      sslrootcert)
+        sslrootcert_count=$((sslrootcert_count + 1))
+        [ "$database_value" = /run/secrets/external-db-ca.crt ] || { echo "$database_dsn_label sslrootcert must use /run/secrets/external-db-ca.crt" >&2; return 1; }
+        ;;
+      sslcert)
+        sslcert_count=$((sslcert_count + 1))
+        [ "$database_value" = /run/secrets/external-db-client.crt ] || { echo "$database_dsn_label sslcert must use /run/secrets/external-db-client.crt" >&2; return 1; }
+        ;;
+      sslkey)
+        sslkey_count=$((sslkey_count + 1))
+        [ "$database_value" = /run/secrets/external-db-client.key ] || { echo "$database_dsn_label sslkey must use /run/secrets/external-db-client.key" >&2; return 1; }
+        ;;
+    esac
+  done
+
+  if [ "$sslmode_count" -ne 1 ]; then
+    echo "$database_dsn_label must contain exactly one sslmode" >&2
+    return 1
+  fi
+  if [ "$sslmode" != verify-ca ] && [ "$sslmode" != verify-full ]; then
+    echo "$database_dsn_label sslmode must be verify-ca or verify-full" >&2
+    return 1
+  fi
+
+  tls_count=$((sslrootcert_count + sslcert_count + sslkey_count))
+  if [ "$tls_count" -ne 0 ]; then
+    database_dsn_uses_certificate_files=1
+    if [ "$sslrootcert_count" -ne 1 ] || [ "$sslcert_count" -ne 1 ] || [ "$sslkey_count" -ne 1 ]; then
+      echo "$database_dsn_label must contain each fixed TLS certificate parameter exactly once" >&2
+      return 1
+    fi
+  fi
+}
+
+dsn=${GOCHAT_DATABASE_DSN:-${DATA_SOURCE_NAME:-}}
+validate_production_database_dsn "$dsn"
+
+if [ "${1:-}" = --preflight ]; then
+  exit 0
+fi
+
+if [ "$database_dsn_uses_certificate_files" -eq 1 ]; then
+  for file in /run/secrets/external-db-ca.crt /run/secrets/external-db-client.crt /run/secrets/external-db-client.key; do
+    [ -f "$file" ] && [ -r "$file" ] || { echo "database TLS file is missing or unreadable: $file" >&2; exit 1; }
+  done
+fi
+
+if [ "${1:-}" = --check ]; then
+  exit 0
+fi
+exec "$@"
diff --git a/deploy/docker/database_client_entrypoint_test.sh b/deploy/docker/database_client_entrypoint_test.sh
new file mode 100755
index 00000000..d6f1ddcf
--- /dev/null
+++ b/deploy/docker/database_client_entrypoint_test.sh
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+gate=$root/deploy/docker/database_client_entrypoint.sh
+gochat_image=${GOCHAT_TEST_IMAGE:-gochat:production-smoke}
+postgres_image='pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b'
+exporter_image='quay.io/prometheuscommunity/postgres-exporter:v0.17.1@sha256:38606faa38c54787525fb0ff2fd6b41b4cfb75d455c1df294927c5f611699b17'
+tmp=$(mktemp -d)
+network=gochat-db-gate-$$
+database=gochat-db-gate-postgres-$$
+exporter=gochat-db-gate-exporter-$$
+
+cleanup() {
+  docker rm -f "$exporter" "$database" >/dev/null 2>&1 || true
+  docker network rm "$network" >/dev/null 2>&1 || true
+  rm -rf "$tmp"
+}
+trap cleanup EXIT
+
+mkdir -p "$tmp/ca" "$tmp/server" "$tmp/client" "$tmp/denied"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=GoChat test CA' \
+  -keyout "$tmp/ca/ca.key" -out "$tmp/ca/ca.crt" >/dev/null 2>&1
+openssl req -newkey rsa:2048 -nodes -subj '/CN=tls-db' \
+  -keyout "$tmp/server/server.key" -out "$tmp/server/server.csr" >/dev/null 2>&1
+printf 'subjectAltName=DNS:tls-db\n' >"$tmp/server/server.ext"
+openssl x509 -req -days 1 -in "$tmp/server/server.csr" -CA "$tmp/ca/ca.crt" -CAkey "$tmp/ca/ca.key" \
+  -CAcreateserial -extfile "$tmp/server/server.ext" -out "$tmp/server/server.crt" >/dev/null 2>&1
+openssl req -newkey rsa:2048 -nodes -subj '/CN=gochat-client' \
+  -keyout "$tmp/client/external-db-client.key" -out "$tmp/client/client.csr" >/dev/null 2>&1
+openssl x509 -req -days 1 -in "$tmp/client/client.csr" -CA "$tmp/ca/ca.crt" -CAkey "$tmp/ca/ca.key" \
+  -CAcreateserial -out "$tmp/client/external-db-client.crt" >/dev/null 2>&1
+cp "$tmp/ca/ca.crt" "$tmp/client/external-db-ca.crt"
+cp "$tmp/client/external-db-client.key" "$tmp/denied/external-db-client.key"
+
+tls_gid=4242
+postgres_uid=$(docker run --rm --entrypoint id "$postgres_image" -u postgres)
+postgres_gid=$(docker run --rm --entrypoint id "$postgres_image" -g postgres)
+docker run --rm --entrypoint sh -v "$tmp:/work" "$postgres_image" -c \
+  "chown $postgres_uid:$postgres_gid /work/server/server.crt /work/server/server.key && chmod 0644 /work/server/server.crt && chmod 0600 /work/server/server.key && chown 0:$tls_gid /work/client/external-db-*.crt /work/client/external-db-client.key && chmod 0644 /work/client/external-db-*.crt && chmod 0640 /work/client/external-db-client.key && chown 0:0 /work/denied/external-db-client.key && chmod 0600 /work/denied/external-db-client.key"
+
+bad_dsn='postgres://postgres:test-password@tls-db:5432/postgres?sslmode=disable&sslmode=verify-full'
+reject_duplicate() {
+  local label=$1 image=$2 variable=$3 user=$4
+  shift 4
+  local args=(docker run --rm -e "$variable=$bad_dsn" -v "$gate:/usr/local/bin/database-client-entrypoint:ro" --entrypoint /usr/local/bin/database-client-entrypoint)
+  [[ -z $user ]] || args+=(--user "$user")
+  if "${args[@]}" "$image" "$@" >"$tmp/$label.log" 2>&1; then
+    echo "$label accepted duplicate sslmode" >&2
+    exit 1
+  fi
+  grep -F 'exactly one sslmode' "$tmp/$label.log" >/dev/null
+}
+
+reject_duplicate gochat "$gochat_image" GOCHAT_DATABASE_DSN '' /app/gochat --help
+reject_duplicate worker "$gochat_image" GOCHAT_DATABASE_DSN '' /app/gochat worker
+reject_duplicate migrate "$gochat_image" GOCHAT_DATABASE_DSN '' /app/migrate version
+reject_duplicate backup "$gochat_image" GOCHAT_DATABASE_DSN 0:0 /app/scripts/db_backup.sh
+reject_duplicate restore "$gochat_image" GOCHAT_DATABASE_DSN 0:0 /app/scripts/db_restore.sh /tmp/missing.tar.enc
+reject_duplicate postgres-exporter "$exporter_image" DATA_SOURCE_NAME '' /bin/postgres_exporter --version
+
+production_env=(
+  -e GOCHAT_ENV=production
+  -e GOCHAT_DATABASE_DSN="$bad_dsn"
+  -e GOCHAT_REDIS_DSN=redis://:test-password@redis:6379
+  -e GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
+  -e GOCHAT_SEARCH_API_KEY=ci-meili-secret-16
+  -e GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
+)
+for command in serve worker; do
+  if docker run --rm "${production_env[@]}" --entrypoint /app/gochat "$gochat_image" "$command" >"$tmp/direct-$command.log" 2>&1; then
+    echo "direct GoChat $command accepted duplicate sslmode" >&2
+    exit 1
+  fi
+  grep -F 'production database DSN must contain exactly one sslmode' "$tmp/direct-$command.log" >/dev/null
+done
+if docker run --rm "${production_env[@]}" --entrypoint /app/migrate "$gochat_image" version >"$tmp/direct-migrate.log" 2>&1; then
+  echo 'direct migrate accepted duplicate sslmode' >&2
+  exit 1
+fi
+grep -F 'production database DSN must contain exactly one sslmode' "$tmp/direct-migrate.log" >/dev/null
+for client in backup restore; do
+  script=/app/scripts/db_${client}.sh
+  args=()
+  [[ $client == backup ]] || args=(/tmp/missing.tar.enc)
+  if docker run --rm --user 0:0 -e GOCHAT_DATABASE_DSN="$bad_dsn" --entrypoint "$script" "$gochat_image" "${args[@]}" >"$tmp/direct-$client.log" 2>&1; then
+    echo "direct $client accepted duplicate sslmode" >&2
+    exit 1
+  fi
+  grep -F 'database DSN must contain exactly one sslmode' "$tmp/direct-$client.log" >/dev/null
+done
+
+test "$(docker run --rm --entrypoint id "$gochat_image" -u)" -ne 0
+test "$(docker run --rm --entrypoint id "$exporter_image" -u)" -ne 0
+
+dsn='postgres://postgres:test-password@tls-db:5432/postgres?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+if docker run --rm --group-add "$tls_gid" -e GOCHAT_DATABASE_DSN="$dsn" \
+  -v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
+  -v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
+  -v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
+  -v "$tmp/denied/external-db-client.key:/run/secrets/external-db-client.key:ro" \
+  --entrypoint /usr/local/bin/database-client-entrypoint "$gochat_image" /bin/true >"$tmp/permissions.log" 2>&1; then
+  echo 'non-root GoChat client read a root-owned 0600 private key' >&2
+  exit 1
+fi
+grep -F 'database TLS file is missing or unreadable' "$tmp/permissions.log" >/dev/null
+
+docker network create "$network" >/dev/null
+docker run -d --name "$database" --network "$network" --network-alias tls-db \
+  -e POSTGRES_PASSWORD=test-password \
+  -v "$tmp/server/server.crt:/certs/server.crt:ro" \
+  -v "$tmp/server/server.key:/certs/server.key:ro" \
+  "$postgres_image" -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key >/dev/null
+for _ in $(seq 1 30); do
+  docker exec "$database" pg_isready -U postgres >/dev/null 2>&1 && break
+  sleep 1
+done
+docker exec "$database" pg_isready -U postgres >/dev/null
+
+psql_result=$(docker run --rm --network "$network" --group-add "$tls_gid" -e GOCHAT_DATABASE_DSN="$dsn" \
+  -v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
+  -v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
+  -v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
+  -v "$tmp/client/external-db-client.key:/run/secrets/external-db-client.key:ro" \
+  --entrypoint /usr/local/bin/database-client-entrypoint "$gochat_image" psql "$dsn" -Atqc 'SELECT 1')
+test "$psql_result" = 1
+
+docker run -d --name "$exporter" --network "$network" --group-add "$tls_gid" -e DATA_SOURCE_NAME="$dsn" \
+  -v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
+  -v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
+  -v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
+  -v "$tmp/client/external-db-client.key:/run/secrets/external-db-client.key:ro" \
+  --entrypoint /usr/local/bin/database-client-entrypoint "$exporter_image" /bin/postgres_exporter >/dev/null
+for _ in $(seq 1 20); do
+  if docker run --rm --network "$network" --entrypoint sh "$gochat_image" -c \
+    'curl -fsS http://gochat-db-gate-exporter-'"$$"':9187/metrics' >"$tmp/metrics" 2>/dev/null; then
+    break
+  fi
+  sleep 1
+done
+grep -E '^pg_up 1$' "$tmp/metrics" >/dev/null
+
+echo 'database client entrypoint tests passed'
diff --git a/deploy/docker/database_host_rejection_cases.txt b/deploy/docker/database_host_rejection_cases.txt
new file mode 100644
index 00000000..d70e2e99
--- /dev/null
+++ b/deploy/docker/database_host_rejection_cases.txt
@@ -0,0 +1,15 @@
+empty host|postgres://gochat:database-secret@/gochat?sslmode=verify-full
+built-in Compose host|postgres://gochat:database-secret@postgres:5432/gochat?sslmode=verify-full
+localhost|postgres://gochat:database-secret@localhost:5432/gochat?sslmode=verify-full
+IPv4 loopback start|postgres://gochat:database-secret@127.0.0.1:5432/gochat?sslmode=verify-full
+IPv4 loopback end|postgres://gochat:database-secret@127.255.255.255:5432/gochat?sslmode=verify-full
+compressed IPv6 loopback|postgres://gochat:database-secret@[::1]:5432/gochat?sslmode=verify-full
+partially compressed IPv6 loopback|postgres://gochat:database-secret@[0::1]:5432/gochat?sslmode=verify-full
+expanded IPv6 loopback|postgres://gochat:database-secret@[0:0:0:0:0:0:0:01]:5432/gochat?sslmode=verify-full
+IPv4-embedded IPv6 loopback|postgres://gochat:database-secret@[::0.0.0.1]:5432/gochat?sslmode=verify-full
+zoned IPv6 loopback|postgres://gochat:database-secret@[::1%25lo]:5432/gochat?sslmode=verify-full
+IPv4-mapped dotted loopback|postgres://gochat:database-secret@[::ffff:127.0.0.1]:5432/gochat?sslmode=verify-full
+IPv4-mapped dotted loopback end|postgres://gochat:database-secret@[::ffff:127.255.255.255]:5432/gochat?sslmode=verify-full
+IPv4-mapped hexadecimal loopback|postgres://gochat:database-secret@[::ffff:7f00:1]:5432/gochat?sslmode=verify-full
+IPv4-mapped partially expanded loopback|postgres://gochat:database-secret@[0:0::ffff:7f01:203]:5432/gochat?sslmode=verify-full
+IPv4-mapped expanded loopback|postgres://gochat:database-secret@[0:0:0:0:0:ffff:7fff:ffff]:5432/gochat?sslmode=verify-full
diff --git a/deploy/docker/docker-compose.prod-smoke.yml b/deploy/docker/docker-compose.prod-smoke.yml
index 1112c41c..6435d825 100644
--- a/deploy/docker/docker-compose.prod-smoke.yml
+++ b/deploy/docker/docker-compose.prod-smoke.yml
@@ -1,5 +1,7 @@
 services:
   postgres:
+    image: ${POSTGRES_IMAGE_REF:-pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b}
+    restart: always
     entrypoint:
       - /bin/sh
       - -ec
@@ -8,10 +10,36 @@ services:
         install -o postgres -g postgres -m 644 /run/tls/postgres.crt /var/lib/postgresql/server.crt
         exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key
     command: []
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-gochat_production}
+      POSTGRES_USER: ${POSTGRES_USER:-gochat}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for the smoke database}
     volumes:
+      - postgres_data:/var/lib/postgresql/data
       - ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
+    networks:
+      default:
+        aliases: [db.smoke.test]
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: 127.0.0.1:24224
+        fluentd-async: "true"
+        fluentd-buffer-limit: "65536"
+        tag: gochat.{{.Name}}
 
   redis:
+    image: ${REDIS_IMAGE_REF:-redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf}
+    restart: always
     entrypoint:
       - /bin/sh
       - -ec
@@ -20,7 +48,10 @@ services:
         install -o redis -g redis -m 644 /run/tls/redis.crt /data/redis.crt
         exec /usr/bin/setpriv --reuid redis --regid redis --clear-groups redis-server --port 0 --tls-port 6379 --tls-cert-file /data/redis.crt --tls-key-file /data/redis.key --tls-ca-cert-file /run/tls/ca.crt --tls-auth-clients no --requirepass "$${REDIS_PASSWORD}" --appendonly yes
     command: []
+    environment:
+      REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD for smoke Redis}
     volumes:
+      - redis_data:/data
       - ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
     healthcheck:
       test:
@@ -28,21 +59,72 @@ services:
           "CMD-SHELL",
           "redis-cli --tls --cacert /run/tls/ca.crt -h redis -a '$${REDIS_PASSWORD}' ping",
         ]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: 127.0.0.1:24224
+        fluentd-async: "true"
+        fluentd-buffer-limit: "65536"
+        tag: gochat.{{.Name}}
 
   gochat:
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
     environment:
       SSL_CERT_FILE: /run/tls/ca.crt
     volumes:
       - ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
 
   worker:
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
     environment:
       SSL_CERT_FILE: /run/tls/ca.crt
     volumes:
       - ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
 
   migrate:
+    depends_on:
+      postgres:
+        condition: service_healthy
     environment:
       SSL_CERT_FILE: /run/tls/ca.crt
     volumes:
       - ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
+
+  backup:
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  restore:
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  postgres-exporter:
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  redis-exporter:
+    depends_on:
+      redis:
+        condition: service_healthy
+
+volumes:
+  postgres_data:
+  redis_data:
diff --git a/deploy/docker/docker-compose.prod.yml b/deploy/docker/docker-compose.prod.yml
index 2d97c5f4..4776bce1 100644
--- a/deploy/docker/docker-compose.prod.yml
+++ b/deploy/docker/docker-compose.prod.yml
@@ -1,13 +1,48 @@
 name: gochat-production
 
 x-gochat-image: &gochat-image ${GOCHAT_IMAGE_REF:?set GOCHAT_IMAGE_REF to an immutable image digest}
+x-postgres-tls-ca: &postgres-tls-ca
+  type: bind
+  source: ${GOCHAT_DATABASE_TLS_CA_FILE:-/dev/null}
+  target: /run/secrets/external-db-ca.crt
+  read_only: true
+  bind:
+    create_host_path: false
+x-postgres-tls-client-cert: &postgres-tls-client-cert
+  type: bind
+  source: ${GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE:-/dev/null}
+  target: /run/secrets/external-db-client.crt
+  read_only: true
+  bind:
+    create_host_path: false
+x-postgres-tls-client-key: &postgres-tls-client-key
+  type: bind
+  source: ${GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE:-/dev/null}
+  target: /run/secrets/external-db-client.key
+  read_only: true
+  bind:
+    create_host_path: false
+x-database-client-entrypoint: &database-client-entrypoint
+  type: bind
+  source: ./database_client_entrypoint.sh
+  target: /usr/local/bin/database-client-entrypoint
+  read_only: true
+  bind:
+    create_host_path: false
+x-gochat-logging: &gochat-logging
+  driver: fluentd
+  options:
+    fluentd-address: 127.0.0.1:24224
+    fluentd-async: "true"
+    fluentd-buffer-limit: "65536"
+    tag: gochat.{{.Name}}
 x-gochat-environment: &gochat-environment
   GOCHAT_ENV: production
   GOCHAT_SERVER_HOST: 0.0.0.0
   GOCHAT_SERVER_PORT: 3000
   GOCHAT_SERVER_MODE: release
   GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: ${GOCHAT_SERVER_CORS_ALLOWED_ORIGINS:?set production CORS origins}
-  GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
+  GOCHAT_DATABASE_DSN: &gochat-database-dsn ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
   GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
   GOCHAT_DATABASE_MIGRATIONS_PATH: /app/migrations
   GOCHAT_REDIS_DSN: ${GOCHAT_REDIS_DSN:?set an external Redis rediss:// DSN}
@@ -25,43 +60,6 @@ x-gochat-environment: &gochat-environment
   GOCHAT_STORAGE_LOCAL_PATH: /app/storage/uploads
 
 services:
-  postgres:
-    image: ${POSTGRES_IMAGE_REF:-pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b}
-    restart: always
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB:-gochat_production}
-      POSTGRES_USER: ${POSTGRES_USER:-gochat}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-    deploy:
-      resources:
-        limits:
-          memory: 1G
-
-  redis:
-    image: ${REDIS_IMAGE_REF:-redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf}
-    restart: always
-    command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?set REDIS_PASSWORD}", "--appendonly", "yes", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
-    environment:
-      REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
-    volumes:
-      - redis_data:/data
-    healthcheck:
-      test: ["CMD-SHELL", "redis-cli -a '$${REDIS_PASSWORD}' ping"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-    deploy:
-      resources:
-        limits:
-          memory: 512M
-
   meilisearch:
     image: ${MEILI_IMAGE_REF:-getmeili/meilisearch:v1.13@sha256:bed3fb650e62da53145777204891159242f6ea4ce69e215b36223af4aa64a0ae}
     restart: always
@@ -76,16 +74,16 @@ services:
       interval: 5s
       timeout: 5s
       retries: 20
+    logging: *gochat-logging
 
   gochat:
     image: *gochat-image
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/gochat"]
+    command: ["serve"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
     restart: always
     stop_grace_period: ${GOCHAT_STOP_GRACE_PERIOD:-35s}
     depends_on:
-      postgres:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
       meilisearch:
         condition: service_healthy
     environment: *gochat-environment
@@ -93,6 +91,10 @@ services:
       - "127.0.0.1:${GOCHAT_PORT:-3000}:3000"
     volumes:
       - gochat_storage:/app/storage
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
     healthcheck:
       test: ["CMD", "wget", "-q", "-T", "3", "-O", "/dev/null", "http://127.0.0.1:3000/ready"]
       interval: 10s
@@ -107,18 +109,17 @@ services:
         reservations:
           memory: 256M
           cpus: "0.5"
+    logging: *gochat-logging
 
   worker:
     image: *gochat-image
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/gochat"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
     restart: always
     stop_grace_period: ${GOCHAT_STOP_GRACE_PERIOD:-35s}
     depends_on:
       gochat:
         condition: service_healthy
-      postgres:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
       meilisearch:
         condition: service_healthy
     command: ["worker"]
@@ -129,42 +130,51 @@ services:
       GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
     volumes:
       - gochat_storage:/app/storage
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
     deploy:
       resources:
         limits:
           memory: 512M
           cpus: "1.0"
+    logging: *gochat-logging
 
   migrate:
     image: *gochat-image
     profiles: ["ops"]
-    depends_on:
-      postgres:
-        condition: service_healthy
-    entrypoint: ["/app/migrate"]
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/migrate"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
     command: ["up"]
     restart: "no"
     environment:
       <<: *gochat-environment
       GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
+      PGOPTIONS: -c lock_timeout=5000 -c statement_timeout=900000
+    volumes:
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging
 
   backup:
     image: *gochat-image
     user: "0:0"
     profiles: ["ops"]
-    depends_on:
-      postgres:
-        condition: service_healthy
-    entrypoint: ["/app/scripts/db_backup.sh"]
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/scripts/db_backup.sh"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
     restart: "no"
     environment:
-      GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
+      GOCHAT_DATABASE_DSN: *gochat-database-dsn
       GOCHAT_STORAGE_PATH: /source/storage/uploads
       GOCHAT_CONNECTOR_BACKUP_FILE: /source/connector/${GOCHAT_CONNECTOR_BACKUP_NAME:-latest.db}
       GOCHAT_BACKUP_DIR: /backup/local
       GOCHAT_BACKUP_OFFSITE_DIR: /backup/offsite
       GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
       GOCHAT_BACKUP_RETENTION_DAYS: ${GOCHAT_BACKUP_RETENTION_DAYS:-30}
+      GOCHAT_BACKUP_METRICS_FILE: /metrics/gochat_backup.prom
       GOCHAT_VERSION: ${GOCHAT_IMAGE_REF}
     volumes:
       - gochat_storage:/source/storage:ro
@@ -176,19 +186,23 @@ services:
         bind:
           create_host_path: false
       - ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
+      - backup_metrics:/metrics
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging
 
   restore:
     image: *gochat-image
     user: "0:0"
     profiles: ["ops"]
-    depends_on:
-      postgres:
-        condition: service_healthy
-    entrypoint: ["/app/scripts/db_restore.sh"]
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/scripts/db_restore.sh"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
     command: ["/backup/offsite/${GOCHAT_RESTORE_BUNDLE:-missing.tar.enc}"]
     restart: "no"
     environment:
-      GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
+      GOCHAT_DATABASE_DSN: *gochat-database-dsn
       GOCHAT_STORAGE_PATH: /restore/storage/uploads
       GOCHAT_CONNECTOR_DB_PATH: /restore/connector/connector.db
       GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
@@ -202,6 +216,11 @@ services:
         bind:
           create_host_path: false
       - ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging
 
   shangwutong:
     image: ${SHANGWUTONG_IMAGE_REF:?set SHANGWUTONG_IMAGE_REF to an immutable image digest}
@@ -233,11 +252,114 @@ services:
         reservations:
           memory: 128M
           cpus: "0.25"
+    logging: *gochat-logging
+
+  fluentd:
+    image: fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440
+    restart: always
+    ports:
+      - "127.0.0.1:24224:24224"
+    volumes:
+      - ../fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro
+      - fluentd_logs:/fluentd/log
+      - fluentd_buffer:/fluentd/buffer
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter:v0.17.1@sha256:38606faa38c54787525fb0ff2fd6b41b4cfb75d455c1df294927c5f611699b17
+    restart: always
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/bin/postgres_exporter"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
+    command: ["--config.file=/dev/null", "--extend.query-path=/etc/postgres-exporter/queries.yml"]
+    environment:
+      DATA_SOURCE_NAME: *gochat-database-dsn
+    volumes:
+      - ../prometheus/postgres_queries.yml:/etc/postgres-exporter/queries.yml:ro
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging
+
+  redis-exporter:
+    image: oliver006/redis_exporter:v1.72.1@sha256:f90cae1e7ecc6ac223d04bdb0c95e084918baced9f81f08c3d01c2f11bff72bf
+    restart: always
+    environment:
+      REDIS_ADDR: ${GOCHAT_REDIS_DSN:?set an external Redis rediss:// DSN}
+    logging: *gochat-logging
+
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d
+    restart: always
+    command: ["--config.file=/etc/blackbox_exporter/config.yml"]
+    volumes:
+      - ../prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+    logging: *gochat-logging
+
+  node-exporter:
+    image: prom/node-exporter:v1.9.1@sha256:d00a542e409ee618a4edc67da14dd48c5da66726bbd5537ab2af9c1dfc442c8a
+    restart: always
+    command: ["--collector.disable-defaults", "--collector.textfile", "--collector.textfile.directory=/var/lib/node_exporter/textfile_collector"]
+    volumes:
+      - backup_metrics:/var/lib/node_exporter/textfile_collector:ro
+    logging: *gochat-logging
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.52.1@sha256:f40e65878e25c2e78ea037f73a449527a0fb994e303dc3e34cb6b187b4b91435
+    restart: always
+    privileged: true
+    devices:
+      - /dev/kmsg:/dev/kmsg
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+      - /dev/disk:/dev/disk:ro
+    logging: *gochat-logging
+
+  alertmanager:
+    image: prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba
+    restart: always
+    command: ["--config.file=/etc/alertmanager/alertmanager.yml", "--storage.path=/alertmanager"]
+    ports:
+      - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
+    volumes:
+      - ../prometheus/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - type: bind
+        source: ${ALERTMANAGER_WEBHOOK_URL_FILE:-../../.secrets/alertmanager-webhook-url}
+        target: /run/secrets/alertmanager-webhook-url
+        read_only: true
+        bind:
+          create_host_path: false
+      - alertmanager_data:/alertmanager
+    logging: *gochat-logging
+
+  prometheus:
+    image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996
+    restart: always
+    depends_on:
+      - alertmanager
+      - blackbox-exporter
+      - cadvisor
+      - node-exporter
+      - postgres-exporter
+      - redis-exporter
+    command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}"]
+    ports:
+      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../../backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro
+      - prometheus_data:/prometheus
+    logging: *gochat-logging
 
 volumes:
-  postgres_data:
-  redis_data:
   meili_data:
   gochat_storage:
   shangwutong_data:
   shangwutong_backups:
+  backup_metrics:
+  prometheus_data:
+  alertmanager_data:
+  fluentd_logs:
+  fluentd_buffer:
diff --git a/deploy/docker/observability_test.sh b/deploy/docker/observability_test.sh
new file mode 100755
index 00000000..e6b0cd7b
--- /dev/null
+++ b/deploy/docker/observability_test.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-observability-test' \
+  -keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
+cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
+chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
+chmod 0640 "$tmp/external-db-client.key"
+
+export GOCHAT_IMAGE_REF='gochat.example.test/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
+export SHANGWUTONG_IMAGE_REF='gochat.example.test/shangwutong@sha256:1111111111111111111111111111111111111111111111111111111111111111'
+export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
+export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
+export GOCHAT_ENCRYPTION_AES_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
+export POSTGRES_PASSWORD=ci-postgres-secret
+export REDIS_PASSWORD=ci-redis-secret
+export MEILI_MASTER_KEY=ci-meili-secret-16
+export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
+export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
+export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
+mkdir "$GOCHAT_BACKUP_OFFSITE_DIR"
+
+baseline_dsn='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
+baseline_compose_json=$tmp/baseline-compose.json
+mtls_compose_json=$tmp/mtls-compose.json
+smoke_compose_json=$tmp/smoke-compose.json
+export GOCHAT_DATABASE_DSN=$baseline_dsn
+docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$baseline_compose_json"
+export GOCHAT_TLS_DIR=$tmp
+docker compose -f "$root/deploy/docker/docker-compose.prod.yml" -f "$root/deploy/docker/docker-compose.prod-smoke.yml" --profile '*' config --format json >"$smoke_compose_json"
+export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
+export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
+export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
+export GOCHAT_DATABASE_TLS_GID
+GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
+export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$mtls_compose_json"
+python3 - "$baseline_compose_json" "$mtls_compose_json" "$smoke_compose_json" "$baseline_dsn" "$GOCHAT_DATABASE_DSN" <<'PY'
+import json
+import os
+import sys
+from urllib.parse import parse_qs, urlsplit
+
+baseline = json.load(open(sys.argv[1], encoding="utf-8"))["services"]
+mtls = json.load(open(sys.argv[2], encoding="utf-8"))["services"]
+smoke = json.load(open(sys.argv[3], encoding="utf-8"))["services"]
+baseline_dsn = sys.argv[4]
+mtls_dsn = sys.argv[5]
+services = baseline
+required = {
+    "alertmanager", "blackbox-exporter", "cadvisor", "fluentd", "node-exporter",
+    "postgres-exporter", "prometheus", "redis-exporter",
+}
+assert required <= services.keys(), required - services.keys()
+assert not {"postgres", "redis"} & services.keys()
+for service in ("gochat", "worker", "migrate", "backup", "shangwutong"):
+    assert services[service]["logging"]["driver"] == "fluentd", service
+for service in ("postgres", "redis"):
+    assert smoke[service]["logging"]["driver"] == "fluentd", service
+for service, dependencies in {
+    "gochat": {"postgres", "redis"},
+    "worker": {"postgres", "redis"},
+    "migrate": {"postgres"},
+    "backup": {"postgres"},
+    "restore": {"postgres"},
+    "postgres-exporter": {"postgres"},
+    "redis-exporter": {"redis"},
+}.items():
+    assert not dependencies & baseline[service].get("depends_on", {}).keys(), service
+    assert dependencies <= smoke[service]["depends_on"].keys(), service
+assert baseline["redis-exporter"]["environment"]["REDIS_ADDR"] == os.environ["GOCHAT_REDIS_DSN"]
+assert any(volume["target"] == "/etc/prometheus/rules/gochat.yml" for volume in services["prometheus"]["volumes"])
+assert any(volume["target"] == "/fluentd/etc/fluent.conf" for volume in services["fluentd"]["volumes"])
+assert services["backup"]["environment"]["GOCHAT_BACKUP_METRICS_FILE"] == "/metrics/gochat_backup.prom"
+
+def database_dsns(config):
+    return {
+        "gochat": config["gochat"]["environment"]["GOCHAT_DATABASE_DSN"],
+        "worker": config["worker"]["environment"]["GOCHAT_DATABASE_DSN"],
+        "migrate": config["migrate"]["environment"]["GOCHAT_DATABASE_DSN"],
+        "backup": config["backup"]["environment"]["GOCHAT_DATABASE_DSN"],
+        "restore": config["restore"]["environment"]["GOCHAT_DATABASE_DSN"],
+        "postgres-exporter": config["postgres-exporter"]["environment"]["DATA_SOURCE_NAME"],
+    }
+
+client_commands = {
+    "gochat": "/app/gochat",
+    "worker": "/app/gochat",
+    "migrate": "/app/migrate",
+    "backup": "/app/scripts/db_backup.sh",
+    "restore": "/app/scripts/db_restore.sh",
+    "postgres-exporter": "/bin/postgres_exporter",
+}
+for config, expected_gid in ((baseline, "65534"), (mtls, os.environ["GOCHAT_DATABASE_TLS_GID"])):
+    for service, command in client_commands.items():
+        assert config[service]["entrypoint"] == ["/usr/local/bin/database-client-entrypoint", command], service
+        assert expected_gid in map(str, config[service]["group_add"]), service
+        gate = [volume for volume in config[service]["volumes"] if volume["target"] == "/usr/local/bin/database-client-entrypoint"]
+        assert len(gate) == 1 and gate[0]["read_only"] is True, service
+
+baseline_dsns = database_dsns(baseline)
+assert set(baseline_dsns.values()) == {baseline_dsn}, baseline_dsns
+mtls_dsns = database_dsns(mtls)
+assert set(mtls_dsns.values()) == {mtls_dsn}, mtls_dsns
+
+pgoptions = "-c lock_timeout=5000 -c statement_timeout=900000"
+assert baseline["migrate"]["environment"]["PGOPTIONS"] == pgoptions
+assert mtls["migrate"]["environment"]["PGOPTIONS"] == pgoptions
+
+tls_paths = {
+    "sslrootcert": "/run/secrets/external-db-ca.crt",
+    "sslcert": "/run/secrets/external-db-client.crt",
+    "sslkey": "/run/secrets/external-db-client.key",
+}
+tls_sources = {
+    tls_paths["sslrootcert"]: os.environ["GOCHAT_DATABASE_TLS_CA_FILE"],
+    tls_paths["sslcert"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE"],
+    tls_paths["sslkey"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"],
+}
+query = parse_qs(urlsplit(mtls_dsn).query)
+assert query["sslmode"] == ["verify-full"]
+for parameter, target in tls_paths.items():
+    assert query[parameter] == [target], (parameter, query)
+
+for config, expected_sources in ((baseline, {target: "/dev/null" for target in tls_paths.values()}), (mtls, tls_sources)):
+    for service in database_dsns(config):
+        mounts = {volume["target"]: volume for volume in config[service]["volumes"] if volume["target"] in tls_paths.values()}
+        assert mounts.keys() == expected_sources.keys(), (service, mounts)
+        for target, source in expected_sources.items():
+            assert mounts[target]["source"] == source, (service, target, mounts[target])
+            assert mounts[target]["read_only"] is True, (service, target, mounts[target])
+PY
+
+prometheus_image='prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996'
+alertmanager_image='prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba'
+blackbox_image='prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d'
+fluentd_image='fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440'
+
+docker run --rm --entrypoint promtool \
+  -v "$root/deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \
+  -v "$root/backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro" \
+  "$prometheus_image" check config /etc/prometheus/prometheus.yml
+docker run --rm --entrypoint promtool \
+  -v "$root/backend/configs:/configs:ro" \
+  "$prometheus_image" test rules /configs/prometheus_alerts_test.yml
+docker run --rm --entrypoint amtool \
+  -v "$root/deploy/prometheus:/etc/alertmanager:ro" \
+  -v "$tmp/alertmanager-webhook-url:/run/secrets/alertmanager-webhook-url:ro" \
+  "$alertmanager_image" check-config /etc/alertmanager/alertmanager.yml
+docker run --rm \
+  -v "$root/deploy/prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro" \
+  "$blackbox_image" --config.file=/etc/blackbox_exporter/config.yml --config.check
+docker run --rm \
+  -v "$root/deploy/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro" \
+  "$fluentd_image" fluentd --dry-run -c /fluentd/etc/fluent.conf
+"$root/backend/scripts/db_backup_test.sh"
+
+echo 'observability configuration tests passed'
diff --git a/deploy/docker/preflight.sh b/deploy/docker/preflight.sh
index 0c278cee..505cfe28 100755
--- a/deploy/docker/preflight.sh
+++ b/deploy/docker/preflight.sh
@@ -12,7 +12,7 @@ if (($#)); then
   compose_args=(--env-file "$env_file" "${compose_args[@]}")
 fi
 
-required=(GOCHAT_IMAGE_REF SHANGWUTONG_IMAGE_REF GOCHAT_SERVER_CORS_ALLOWED_ORIGINS POSTGRES_PASSWORD REDIS_PASSWORD MEILI_MASTER_KEY GOCHAT_JWT_SECRET GOCHAT_BACKUP_OFFSITE_DIR GOCHAT_BACKUP_OFFSITE_SOURCE GOCHAT_BACKUP_OFFSITE_FSTYPE)
+required=(GOCHAT_IMAGE_REF SHANGWUTONG_IMAGE_REF GOCHAT_SERVER_CORS_ALLOWED_ORIGINS GOCHAT_DATABASE_DSN GOCHAT_REDIS_DSN MEILI_MASTER_KEY GOCHAT_JWT_SECRET GOCHAT_BACKUP_OFFSITE_DIR GOCHAT_BACKUP_OFFSITE_SOURCE GOCHAT_BACKUP_OFFSITE_FSTYPE)
 for name in "${required[@]}"; do
   value=${!name:-}
   if [[ -z $value || ${value^^} == *CHANGE_ME* ]]; then
@@ -67,10 +67,45 @@ if ((${#MEILI_MASTER_KEY} < 16)); then
   echo "MEILI_MASTER_KEY must be at least 16 bytes" >&2
   exit 1
 fi
-if [[ -n ${GOCHAT_DATABASE_DSN:-} && ! $GOCHAT_DATABASE_DSN =~ (^|[?&])sslmode=(require|verify-ca|verify-full)(&|$) ]]; then
-  echo "GOCHAT_DATABASE_DSN must explicitly require TLS for an external database" >&2
+alertmanager_webhook_file=${ALERTMANAGER_WEBHOOK_URL_FILE:-../../.secrets/alertmanager-webhook-url}
+if [[ $alertmanager_webhook_file != /* ]]; then
+  alertmanager_webhook_file=$script_dir/$alertmanager_webhook_file
+fi
+if [[ ! -r $alertmanager_webhook_file ]] || ! IFS= read -r alertmanager_webhook_url <"$alertmanager_webhook_file" || [[ ! $alertmanager_webhook_url =~ ^https://[^[:space:]]+$ ]]; then
+  echo "ALERTMANAGER_WEBHOOK_URL_FILE must contain one HTTPS URL" >&2
   exit 1
 fi
+if [[ -n ${GOCHAT_DATABASE_DSN:-} ]]; then
+  "$script_dir/database_client_entrypoint.sh" --preflight
+  if [[ $GOCHAT_DATABASE_DSN == *sslrootcert=* ]]; then
+    tls_gid=${GOCHAT_DATABASE_TLS_GID:-}
+    if [[ ! $tls_gid =~ ^[1-9][0-9]*$ ]]; then
+      echo "GOCHAT_DATABASE_TLS_GID must be set to a positive numeric group ID" >&2
+      exit 1
+    fi
+    for name in GOCHAT_DATABASE_TLS_CA_FILE GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE; do
+      host_file=${!name:-}
+      if [[ -z $host_file ]]; then
+        echo "$name must be set when the database DSN uses certificate files" >&2
+        exit 1
+      fi
+      if [[ $host_file != /* ]]; then
+        host_file=$script_dir/$host_file
+      fi
+      if [[ ! -f $host_file ]]; then
+        echo "$name must point to an existing regular file" >&2
+        exit 1
+      fi
+      file_gid=$(stat -c %g -- "$host_file")
+      file_mode=$(stat -c %a -- "$host_file")
+      group_digit=${file_mode: -2:1}
+      if [[ $file_gid != "$tls_gid" ]] || (((8#$group_digit & 4) == 0)); then
+        echo "$name must be readable by GOCHAT_DATABASE_TLS_GID" >&2
+        exit 1
+      fi
+    done
+  fi
+fi
 
 images=$(docker compose "${compose_args[@]}" config --images)
 while IFS= read -r image; do
diff --git a/deploy/docker/preflight_test.sh b/deploy/docker/preflight_test.sh
index 7af8ce4f..431b8bcc 100755
--- a/deploy/docker/preflight_test.sh
+++ b/deploy/docker/preflight_test.sh
@@ -5,6 +5,12 @@ root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
 tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
 mkdir -p "$tmp/bin" "$tmp/local" "$tmp/offsite"
+printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-preflight-test' \
+  -keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
+cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
+chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
+chmod 0640 "$tmp/external-db-client.key"
 
 cat > "$tmp/bin/findmnt" <<'EOF'
 #!/usr/bin/env bash
@@ -32,10 +38,13 @@ export PATH="$tmp/bin:$PATH"
 export GOCHAT_IMAGE_REF='gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
 export SHANGWUTONG_IMAGE_REF='gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
 export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.ci.rogeecn.com
+export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
+export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
 export POSTGRES_PASSWORD=ci-postgres-secret
 export REDIS_PASSWORD=ci-redis-secret
 export MEILI_MASTER_KEY=ci-meili-secret-16
 export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
+export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
 export GOCHAT_BACKUP_DIR="$tmp/local"
 export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
 export GOCHAT_BACKUP_OFFSITE_SOURCE='backup.example.test:/gochat'
@@ -86,11 +95,44 @@ MEILI_MASTER_KEY=too-short
 expect_failure 'a short Meilisearch key' 'must be at least 16 bytes'
 MEILI_MASTER_KEY=ci-meili-secret-16
 
+printf '%s\n' 'http://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
+expect_failure 'a plaintext alert receiver' 'must contain one HTTPS URL'
+printf '%s\n' 'https://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
+
 TEST_MUTABLE_IMAGE=1
 export TEST_MUTABLE_IMAGE
 expect_failure 'a mutable production image' 'must be pinned to a sha256 digest'
 unset TEST_MUTABLE_IMAGE
 
+export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable'
+expect_failure 'an external database without verified TLS' 'sslmode must be verify-ca or verify-full'
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=require'
+expect_failure 'an external database without certificate verification' 'sslmode must be verify-ca or verify-full'
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable&sslmode=verify-full'
+expect_failure 'duplicate sslmode parameters with a disabling first value' 'must contain exactly one sslmode'
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslmode=verify-full'
+expect_failure 'duplicate allowed sslmode parameters' 'must contain exactly one sslmode'
+while IFS='|' read -r name GOCHAT_DATABASE_DSN; do
+  expect_failure "$name" 'must use an external PostgreSQL host'
+done < "$root/deploy/docker/database_host_rejection_cases.txt"
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/tmp/ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+expect_failure 'a non-fixed certificate path' 'sslrootcert must use /run/secrets/external-db-ca.crt'
+
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+export GOCHAT_DATABASE_TLS_GID
+GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
+unset GOCHAT_DATABASE_TLS_CA_FILE GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE
+expect_failure 'missing host TLS variables' 'GOCHAT_DATABASE_TLS_CA_FILE must be set'
+
+export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/missing-ca.crt"
+export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
+export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
+expect_failure 'a missing host TLS file' 'GOCHAT_DATABASE_TLS_CA_FILE must point to an existing regular file'
+
+GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
+chmod 0600 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
+expect_failure 'a private key without group read permission' 'GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE must be readable by GOCHAT_DATABASE_TLS_GID'
+chmod 0640 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
 run_preflight
 grep -F 'production preflight passed' "$tmp/output" >/dev/null
 echo 'preflight tests passed'
diff --git a/deploy/fluentd/fluent.conf b/deploy/fluentd/fluent.conf
index fdc656b8..20743de6 100644
--- a/deploy/fluentd/fluent.conf
+++ b/deploy/fluentd/fluent.conf
@@ -1,110 +1,41 @@
-# GoChat Fluentd Configuration
-# Reference: Chatwoot logging infrastructure pattern
-# Collects structured JSON logs from GoChat containers and sends to Elasticsearch
-#
-# Deployment: deploy/fluentd/ — apply as ConfigMap + Deployment in K8s
-# Usage: kubectl apply -k deploy/fluentd/
-
-# ---- Source: tail GoChat container logs ----
+# Production Docker Compose log collector. Docker's non-blocking Fluentd driver
+# sends every service with a gochat.<container> tag to this forward input.
 <source>
-  @type tail
-  path /var/log/containers/gochat*.log
-  pos_file /var/log/fluentd-gochat.pos
-  tag gochat.app
-  read_from_head true
-  <parse>
-    @type json
-    time_key timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%NZ
-    keep_time_key true
-  </parse>
+  @type forward
+  bind 0.0.0.0
+  port 24224
 </source>
 
-# ---- Source: tail GoChat worker logs ----
-<source>
-  @type tail
-  path /var/log/containers/gochat-worker*.log
-  pos_file /var/log/fluentd-gochat-worker.pos
-  tag gochat.worker
-  read_from_head true
-  <parse>
-    @type json
-    time_key timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%NZ
-    keep_time_key true
-  </parse>
-</source>
-
-# ---- Filter: Add Kubernetes metadata (pod name, namespace, labels) ----
-<filter gochat.**>
-  @type kubernetes_metadata
-  @id filter_kube_metadata
-</filter>
-
-# ---- Filter: Parse log level for Elasticsearch routing ----
 <filter gochat.**>
   @type record_transformer
   <record>
-    # Add searchable fields from GoChat structured logs
-    log_level ${record["level"]}
-    component ${record["component"]}
-    environment ${record["GOCHAT_ENV"]}
-    # Flatten error details for Kibana searching
-    error_message ${record["error"]}
+    service ${tag_parts[1]}
+    environment production
   </record>
 </filter>
 
-# ---- Output: Elasticsearch (primary) ----
+# Local, disk-buffered JSON is the reliable baseline and remains searchable
+# without an external logging vendor. Ship these files onward if required.
 <match gochat.**>
-  @type elasticsearch
-  @id out_es_gochat
-  @log_level info
-
-  # Elasticsearch connection
-  host ${ELASTICSEARCH_HOST}
-  port ${ELASTICSEARCH_PORT}
-  scheme https
-  ssl_version TLSv1_2
-  # Authentication
-  user ${ELASTICSEARCH_USER}
-  password ${ELASTICSEARCH_PASSWORD}
-
-  # Index naming: gochat-YYYY.MM.dd (daily rotation)
-  index_name gochat
-  template_name gochat
-  template_file /fluentd/etc/gochat-index-template.json
-
-  # ILM (Index Lifecycle Management) for automatic rotation
-  ilm_policy_name gochat-log-policy
-  ilm_policy_id gochat-log-policy
-
-  # Bulk indexing for performance
-  bulk_request_timeout 10s
-  flush_interval 5s
-  retry_max_interval 30s
-  retry_forever true
-
-  # Buffer configuration (disk-backed for reliability)
-  <buffer>
+  @type file
+  path /fluentd/log/gochat
+  append true
+  compress gzip
+  <buffer time>
     @type file
-    path /var/log/fluentd/buffers/gochat
+    path /fluentd/buffer/gochat
+    timekey 60
+    timekey_wait 10s
+    timekey_use_utc true
+    flush_mode interval
     flush_interval 5s
-    flush_thread_interval 1s
-    flush_mode lazy
+    chunk_limit_size 16m
+    total_limit_size 8g
     retry_type exponential_backoff
     retry_forever true
     overflow_action block
-    chunk_limit_size 16M
-    total_limit_size 8G
   </buffer>
-
-  # Time-based index naming
-  time_key timestamp
-  time_slice_format %Y.%m.%d
-  time_slice_wait 10m
+  <format>
+    @type json
+  </format>
 </match>
-
-# ---- Output: Stdout for debugging ----
-<match gochat.debug.**>
-  @type stdout
-</match>
\ No newline at end of file
diff --git a/deploy/prometheus/alertmanager.yml b/deploy/prometheus/alertmanager.yml
new file mode 100644
index 00000000..d77db25d
--- /dev/null
+++ b/deploy/prometheus/alertmanager.yml
@@ -0,0 +1,17 @@
+route:
+  receiver: operations-webhook
+  group_by: [alertname, job]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  routes:
+    - receiver: operations-webhook
+      matchers:
+        - severity="critical"
+      repeat_interval: 30m
+
+receivers:
+  - name: operations-webhook
+    webhook_configs:
+      - url_file: /run/secrets/alertmanager-webhook-url
+        send_resolved: true
diff --git a/deploy/prometheus/blackbox.yml b/deploy/prometheus/blackbox.yml
new file mode 100644
index 00000000..2750aa9f
--- /dev/null
+++ b/deploy/prometheus/blackbox.yml
@@ -0,0 +1,7 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: ip4
+      valid_status_codes: [200]
diff --git a/deploy/prometheus/postgres_queries.yml b/deploy/prometheus/postgres_queries.yml
new file mode 100644
index 00000000..d9314ea2
--- /dev/null
+++ b/deploy/prometheus/postgres_queries.yml
@@ -0,0 +1,26 @@
+gochat_background_jobs:
+  query: |
+    SELECT
+      queue,
+      status,
+      COUNT(*)::double precision AS total,
+      COALESCE(MAX(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - CASE
+        WHEN status = 'running' THEN COALESCE(locked_at, updated_at)
+        ELSE scheduled_at
+      END))), 0)::double precision AS oldest_seconds
+    FROM background_jobs
+    WHERE status IN ('queued', 'retrying', 'running')
+    GROUP BY queue, status
+  metrics:
+    - queue:
+        usage: LABEL
+        description: Background job queue.
+    - status:
+        usage: LABEL
+        description: Background job state.
+    - total:
+        usage: GAUGE
+        description: Current background jobs by queue and state.
+    - oldest_seconds:
+        usage: GAUGE
+        description: Age of the oldest background job by queue and state.
diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml
new file mode 100644
index 00000000..2a2c2160
--- /dev/null
+++ b/deploy/prometheus/prometheus.yml
@@ -0,0 +1,79 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: [alertmanager:9093]
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: [prometheus:9090]
+
+  - job_name: alertmanager
+    static_configs:
+      - targets: [alertmanager:9093]
+
+  - job_name: gochat
+    metrics_path: /metrics
+    static_configs:
+      - targets: [gochat:3000]
+
+  - job_name: postgres-exporter
+    static_configs:
+      - targets: [postgres-exporter:9187]
+
+  - job_name: redis-exporter
+    static_configs:
+      - targets: [redis-exporter:9121]
+
+  - job_name: cadvisor
+    static_configs:
+      - targets: [cadvisor:8080]
+
+  - job_name: node-exporter
+    static_configs:
+      - targets: [node-exporter:9100]
+
+  - job_name: gochat-readiness
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ["http://gochat:3000/ready"]
+    relabel_configs: &blackbox-relabel
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+
+  - job_name: gochat-database-readiness
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ["http://gochat:3000/health?check=database"]
+    relabel_configs: *blackbox-relabel
+
+  - job_name: gochat-redis-readiness
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ["http://gochat:3000/health?check=redis"]
+    relabel_configs: *blackbox-relabel
+
+  - job_name: shangwutong-readiness
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ["http://shangwutong:9100/readyz"]
+    relabel_configs: *blackbox-relabel
diff --git a/docs/README.md b/docs/README.md
index eb2433c8..41162ef6 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -120,6 +120,8 @@ M01-M12 模块的 Chatwoot 功能梳理文档，基于 Chatwoot 源码深度阅
 | 文档 | 说明 |
 |------|------|
 | [ops/01-rolling-upgrade.md](ops/01-rolling-upgrade.md) | 滚动升级策略 — 蓝绿部署、数据库迁移、健康检查 |
+| [ops/02-production-operations.md](ops/02-production-operations.md) | 生产预检、TLS、容量、轮换、事故、日志、停服与灾备 runbooks |
+| [ops/03-observability-drill.md](ops/03-observability-drill.md) | 告警/恢复/日志链路演练与非作者验收证据模板 |
 
 ---
 
diff --git a/docs/ops/01-rolling-upgrade.md b/docs/ops/01-rolling-upgrade.md
index 18ef5a11..39d3f528 100644
--- a/docs/ops/01-rolling-upgrade.md
+++ b/docs/ops/01-rolling-upgrade.md
@@ -1,5 +1,11 @@
 # Production backup, restore, and upgrade runbook
 
+- **Owner:** release engineer and database SRE; a non-author SRE executes quarterly restore drills.
+- **Prerequisites:** immutable old/new image digests, green release gate, isolated restore target, external off-site mount and secret-manager access.
+- **Success / failure:** backup RPO is at most 24 hours, restore RTO at most 4 hours, migration is clean and all reconciliation/health checks pass; any mismatch or dirty migration fails the run.
+- **Rollback:** redeploy the recorded old application digest; restore the verified bundle into a clean target if the schema is unusable. Never run production schema down.
+- **Drill cadence:** backup daily, staging upgrade/rollback every release, clean restore quarterly.
+
 Production web and worker processes never migrate on startup. The Compose file
 also requires `GOCHAT_IMAGE_REF` to be an immutable `image@sha256:digest`; tags such
 as `latest` are not an acceptable rollback record.
@@ -35,6 +41,10 @@ export GOCHAT_CONNECTOR_BACKUP_NAME="connector-$(date -u +%Y%m%dT%H%M%SZ).db"
 
 Backup/restore are audited one-shot containers and run as root only to read or
 rebuild Docker volumes; web and worker remain non-root.
+For an external database using certificate files, keep the three DSN paths
+fixed as documented in `.env.example`. Set the host files' group to
+`GOCHAT_DATABASE_TLS_GID` and grant that group read permission (`0640` for the
+private key); Compose adds this supplemental group to all six database clients.
 `GOCHAT_BACKUP_OFFSITE_DIR` must be an existing external mount point provisioned
 outside this Compose project. Set its approved source and filesystem type to the
 exact values reported by `findmnt -M "$GOCHAT_BACKUP_OFFSITE_DIR"`; the preflight
@@ -47,6 +57,7 @@ encrypted bundle containing PostgreSQL, attachments, and that Connector copy:
 ```bash
 docker compose -f deploy/docker/docker-compose.prod.yml exec shangwutong \
   shangwutong backup --output "/backup/$GOCHAT_CONNECTOR_BACKUP_NAME"
+deploy/docker/preflight.sh
 docker compose -f deploy/docker/docker-compose.prod.yml --profile ops run --rm backup
 ```
 
@@ -62,6 +73,7 @@ Do not point this procedure at the live project.
 ```bash
 export COMPOSE_PROJECT_NAME=gochat-restore-$(date +%Y%m%d)
 export GOCHAT_RESTORE_BUNDLE='gochat-<timestamp>.tar.enc'
+deploy/docker/preflight.sh
 docker compose -f deploy/docker/docker-compose.prod.yml --profile ops run --rm restore
 ```
 
@@ -108,6 +120,7 @@ migration runners, but the deployment pipeline must contain only this one step.
 
 ```bash
 export GOCHAT_IMAGE_REF="$NEW_IMAGE"
+deploy/docker/preflight.sh
 time docker compose -f deploy/docker/docker-compose.prod.yml --profile ops \
   up --abort-on-container-exit --exit-code-from migrate migrate
 docker compose -f deploy/docker/docker-compose.prod.yml logs migrate
@@ -115,6 +128,9 @@ docker compose -f deploy/docker/docker-compose.prod.yml up -d --no-deps gochat w
 curl -fsS http://127.0.0.1:3000/health
 ```
 
+Do not override the database client entrypoint or use `docker run` for these
+operations; that bypasses the shared production DSN and certificate gate.
+
 Post-migration checks:
 
 ```sql
diff --git a/docs/ops/02-production-operations.md b/docs/ops/02-production-operations.md
new file mode 100644
index 00000000..0be14e3c
--- /dev/null
+++ b/docs/ops/02-production-operations.md
@@ -0,0 +1,111 @@
+# Production operations runbooks
+
+All commands run from the repository root. Set `COMPOSE='docker compose -f deploy/docker/docker-compose.prod.yml'` and load the approved production environment before use. Never paste secrets into tickets or terminal transcripts.
+
+## Install and preflight
+
+- **Owner:** release engineer; SRE on-call approves the target host.
+- **Prerequisites:** Docker Compose 2.26+, immutable GoChat/Connector digests, external backup mount, untracked backup passphrase and Alertmanager HTTPS webhook files. For external PostgreSQL, set one complete `GOCHAT_DATABASE_DSN` with exactly one `sslmode=verify-ca|verify-full`. When using a private CA or client certificate, keep the DSN paths fixed at `/run/secrets/external-db-ca.crt`, `/run/secrets/external-db-client.crt`, and `/run/secrets/external-db-client.key`; set the three corresponding untracked host-file variables plus `GOCHAT_DATABASE_TLS_GID`, assign that group to the files, and grant group read permission (`0640` for the key). Compose mounts the files read-only and adds the group to every database client.
+- **Commands:** `deploy/docker/preflight.sh .env`; then `$COMPOSE config --quiet`; then `$COMPOSE --profile ops run --rm migrate`; finally `$COMPOSE up -d --wait`.
+- **Success / failure:** success means preflight exits 0, every non-profile service is healthy/running, and rendered `gochat`, `migrate`, `backup`, `restore`, and `postgres-exporter` configuration uses the same database DSN; any mutable image, missing secret file, DSN mismatch, migration error, or unhealthy service is failure.
+- **Rollback:** `$COMPOSE down` leaves named data volumes intact; if migration ran, follow `01-rolling-upgrade.md` and restore the preflight backup instead of running schema down.
+- **Drill cadence:** before every release and quarterly on a clean host.
+
+## TLS and reverse proxy
+
+- **Owner:** SRE on-call.
+- **Prerequisites:** approved DNS, certificate and key in the secret manager; only `127.0.0.1:${GOCHAT_PORT}` is exposed by Compose.
+- **Commands:** configure the platform proxy to terminate TLS, proxy HTTP/WebSocket traffic to `127.0.0.1:${GOCHAT_PORT}`, set `X-Forwarded-Proto https`, and run `curl --fail --proto '=https' --tlsv1.2 https://$GOCHAT_HOST/health` plus a WebSocket handshake through the public hostname.
+- **Success / failure:** success is a trusted certificate, HTTP-to-HTTPS redirect, healthy JSON and a `101` WebSocket response; direct public access to ports 3000, 9090, 9093 or 24224 is failure.
+- **Rollback:** restore the previous proxy configuration and certificate, reload the proxy, then repeat the probes.
+- **Drill cadence:** on certificate/proxy changes and monthly certificate-expiry review.
+
+## Scale out and in
+
+- **Owner:** SRE on-call; application owner approves capacity.
+- **Prerequisites:** external PostgreSQL/Redis/Meilisearch endpoints, shared attachment storage for multi-host web replicas, and a reverse proxy target per host. A single Compose host may scale workers directly.
+- **Commands:** worker scale-out: `$COMPOSE up -d --scale worker=2 worker`; confirm both with `$COMPOSE ps worker`. For web scale-out, provision another preflighted host with the same immutable digest and shared dependencies, add `https://<new-host>/ready` to the load balancer, then drain/remove the old target. Scale in only after `gochat_background_jobs_total` and in-flight requests are stable.
+- **Success / failure:** success is all replicas present, readiness 200, no duplicate/lost jobs and stable queue age; any readiness failure, storage mismatch or rising backlog is failure.
+- **Rollback:** remove the new load-balancer target and restore the previous worker replica count.
+- **Drill cadence:** semi-annually and before forecast traffic peaks.
+
+## Capacity thresholds
+
+- **Owner:** SRE on-call.
+- **Prerequisites:** Prometheus targets are up and at least seven days of representative data exist.
+- **Commands:** review `http_request_duration_seconds`, `gochat_go_memory_alloc_bytes`, `redis_memory_used_bytes / redis_memory_max_bytes`, `pg_stat_activity_count / pg_settings_max_connections`, `gochat_background_jobs_total`, Fluentd buffer use and filesystem free space. Increase capacity before sustained 70%; page at the repository rules' 80%, 400 MB, 25 critical jobs, five-minute critical queue age and 15-minute running-job age thresholds.
+- **Success / failure:** success is 30% headroom and no firing capacity alert; missing series or sustained warning thresholds are failure.
+- **Rollback:** revert the last resource/replica change if latency, restarts or errors worsen, and return traffic to the prior capacity.
+- **Drill cadence:** weekly review; threshold calibration quarterly.
+
+## Secret and key rotation
+
+- **Owner:** security on-call with SRE operator.
+- **Prerequisites:** two-person approval, new values in the secret manager, verified backup and recorded old image digest; never print either value.
+- **Commands:** for JWT, set the new `GOCHAT_JWT_SECRET`, put the old value in `GOCHAT_JWT_PREVIOUS_SECRETS`, run preflight and roll web/worker; after the maximum token lifetime, clear the previous value and roll again. Rotate Connector/Alertmanager credentials by updating their secret files/variables and recreating only affected services. Rotate the backup passphrase only after creating and restoring one bundle with the new passphrase; retain the old key until old bundles expire.
+- **Success / failure:** new credentials work, old JWTs work only during the bounded overlap, revoked credentials fail, logs contain no secret and backup restore succeeds; otherwise failure.
+- **Rollback:** restore the previous secret version and immutable image, recreate affected services, and invalidate the failed new credential.
+- **Drill cadence:** quarterly and immediately after suspected exposure.
+
+## Incident response
+
+- **Owner:** primary SRE on-call; incident commander for critical incidents.
+- **Prerequisites:** alert payload, dashboard access, immutable deployment record and this repository checkout.
+- **Commands:** acknowledge the alert; record UTC start time; run `$COMPOSE ps`, `curl -fsS http://127.0.0.1:9090/api/v1/alerts`, and `$COMPOSE logs --since 15m`; classify dependency, application, queue or capacity failure; mitigate with the matching runbook; record each command and result without secrets.
+- **Success / failure:** success is customer impact stopped, alert resolved and evidence preserved; recurring/reopened alerts or unknown data integrity are failure and require escalation.
+- **Rollback:** revert the most recent change or isolate the failed dependency; never run ad-hoc schema down.
+- **Drill cadence:** quarterly game day and after each Sev-1/Sev-2 retrospective.
+
+## On-call escalation
+
+- **Owner:** primary SRE on-call.
+- **Prerequisites:** current contact rota and severity policy in the paging system.
+- **Commands:** acknowledge warning within 15 minutes and critical within 5; page secondary SRE after 5 unacknowledged minutes; page application/data owner for 15 minutes of unresolved impact; appoint incident commander and notify product owner at 30 minutes or on confirmed data/security impact.
+- **Success / failure:** success is a named owner, incident channel, next update time and acknowledged handoff; no acknowledgement inside the window is failure.
+- **Rollback:** if escalation was false, resolve the page with reason and tune only after evidence review.
+- **Drill cadence:** monthly paging test and quarterly rota failover.
+
+## Log search and retention
+
+- **Owner:** SRE on-call; security approves retention.
+- **Prerequisites:** `fluentd` running, `fluentd_logs` and `fluentd_buffer` below 70% capacity.
+- **Commands:** search recent compressed output with `$COMPOSE exec -T fluentd sh -c 'zgrep -h "request_id_or_error" /fluentd/log/gochat*.gz'`; preserve incident files before cleanup; delete only flushed files older than the approved 14-day baseline with `$COMPOSE exec -T fluentd sh -c 'find /fluentd/log -type f -name "gochat*.gz" -mtime +14 -delete'`.
+- **Success / failure:** success is searchable JSON containing `service` and `environment`, with disk below 70%; missing incident logs, full buffers or deletion inside retention are failure.
+- **Rollback:** restore preserved logs from the incident archive and stop cleanup scheduling while retention is investigated.
+- **Drill cadence:** weekly capacity/retention check and quarterly search drill.
+
+## Backup and restore
+
+- **Owner:** database SRE.
+- **Prerequisites:** dedicated local/off-site mounts, passphrase file, Connector online backup and clean isolated restore project.
+- **Commands:** follow `01-rolling-upgrade.md` daily backup and clean-environment restore commands; verify `gochat_backup_last_success_timestamp_seconds` is fresh in Prometheus.
+- **Success / failure:** success is verified encryption/checksums, off-site copy, RPO at most 24 hours and isolated RTO at most 4 hours; any missing metric, checksum/decrypt error or non-empty restore target is failure.
+- **Rollback:** do not overwrite production; discard the isolated target, restore mount/key access, and retry from the last verified bundle.
+- **Drill cadence:** backup daily; non-author restore rehearsal quarterly.
+
+## Upgrade and rollback
+
+- **Owner:** release engineer with database SRE.
+- **Prerequisites:** approved immutable old/new digests, green release gate, verified backup/restore and maintenance window.
+- **Commands:** execute `01-rolling-upgrade.md` preflight, one-shot migration, post-check and application rollout exactly as written.
+- **Success / failure:** success is one migration run, clean schema version, ready web/worker and unchanged reconciliation counts; any dirty migration, lock timeout or failed health check is failure.
+- **Rollback:** redeploy the recorded old application digest; if schema is unusable, stop writes and restore the verified bundle into a clean target. Never run production `migrate down`.
+- **Drill cadence:** every release in staging; quarterly old-digest rollback drill.
+
+## Planned shutdown
+
+- **Owner:** incident commander or release engineer.
+- **Prerequisites:** approved change window, customer notice, verified backup and rollback owner.
+- **Commands:** remove web from the load balancer; wait for active requests; run `$COMPOSE stop -t 35 gochat worker shangwutong`; stop monitoring last with `$COMPOSE stop prometheus alertmanager fluentd`; leave data services running unless full host maintenance requires them stopped.
+- **Success / failure:** success is graceful exit inside timeout, no running jobs left locked and no data-volume deletion; forced kill or new accepted traffic is failure.
+- **Rollback:** `$COMPOSE up -d --wait`, restore the load-balancer target only after readiness succeeds.
+- **Drill cadence:** semi-annually.
+
+## Disaster failover
+
+- **Owner:** incident commander and database SRE.
+- **Prerequisites:** declared disaster, last verified off-site bundle, alternate host/network/DNS, secret-manager access and immutable image digests.
+- **Commands:** fence the failed site; restore to a clean alternate environment via `01-rolling-upgrade.md`; validate schema/accounts/attachments/Connector; start monitoring, dependencies, web/worker; lower DNS/load-balancer TTL and shift traffic only after `/ready` and smoke pass.
+- **Success / failure:** success is a single writable site, RPO/RTO recorded, alerts/logs working and customer smoke passing; split brain, checksum mismatch or failed reconciliation is failure.
+- **Rollback:** stop the alternate writers and return traffic only if the original site is proven authoritative and reconciled; otherwise keep it fenced.
+- **Drill cadence:** semi-annual full failover and quarterly tabletop.
diff --git a/docs/ops/03-observability-drill.md b/docs/ops/03-observability-drill.md
new file mode 100644
index 00000000..e2f020f1
--- /dev/null
+++ b/docs/ops/03-observability-drill.md
@@ -0,0 +1,100 @@
+# Observability alert and recovery drill
+
+- **Owner:** a non-author SRE on-call performs and signs the drill; the change author may observe only.
+- **Prerequisites:** isolated staging project, approved paging receiver, no production traffic, current immutable images, successful backup, and `PROMETHEUS_PORT`/`ALERTMANAGER_PORT` reachable only on localhost.
+- **Success / failure:** every scenario must produce a firing notification, a resolved notification and a matching searchable Fluentd record. Missing any of the three is failure.
+- **Rollback:** each scenario includes recovery; if recovery fails, run `$COMPOSE up -d --wait` and restore the latest verified staging bundle.
+- **Drill cadence:** before first production release, quarterly, and after alert-route changes.
+
+Set `COMPOSE='docker compose -f deploy/docker/docker-compose.prod.yml'`. Record UTC timestamps, Alertmanager payload IDs and redacted log excerpts; never record environment files, URLs carrying tokens or secret values.
+
+## Baseline
+
+```bash
+$COMPOSE up -d --wait
+curl -fsS http://127.0.0.1:${PROMETHEUS_PORT:-9090}/-/ready
+curl -fsS http://127.0.0.1:${ALERTMANAGER_PORT:-9093}/-/ready
+curl -fsS http://127.0.0.1:${PROMETHEUS_PORT:-9090}/api/v1/targets
+```
+
+Pass when every configured target is healthy. Fix baseline failures before injecting faults.
+
+## Database readiness failure and recovery
+
+Use the external PostgreSQL provider's approved staging-only fault mechanism to
+block GoChat's staging clients while preserving operator access. Wait for
+`GoChatDatabaseReadinessFailed`, record its notification ID, restore access,
+then run `$COMPOSE up -d --wait gochat` and record the resolved notification.
+Include the provider change/audit ID in the evidence; production Compose does
+not run a local PostgreSQL service.
+
+Search logs with `$COMPOSE exec -T fluentd sh -c 'zgrep -h -E "database|unhealthy" /fluentd/log/gochat*.gz'` and record a redacted excerpt.
+
+## Redis readiness failure and recovery
+
+Use the external Redis provider's approved staging-only fault mechanism to
+block GoChat's staging clients while preserving operator access. Wait for
+`GoChatRedisReadinessFailed`, record its notification ID, restore access, then
+run `$COMPOSE up -d --wait gochat worker` and record the resolved notification.
+Include the provider change/audit ID in the evidence; production Compose does
+not run a local Redis service.
+
+Search logs with `$COMPOSE exec -T fluentd sh -c 'zgrep -h -E "redis|unhealthy" /fluentd/log/gochat*.gz'`.
+
+## HTTP 5xx rate and recovery
+
+The unknown dependency selector intentionally returns 503 and is recorded by the normal HTTP metrics middleware.
+
+```bash
+end=$((SECONDS + 360))
+while ((SECONDS < end)); do
+  curl -sS 'http://127.0.0.1:3000/health?check=drill-unknown' >/dev/null || true
+  sleep 0.2
+done
+# Wait for GoChatHighErrorRate, stop injection, then wait for its resolved notification.
+```
+
+Search logs for `drill-unknown` or the matching request IDs. Failure to resolve after the five-minute rate window plus route delay fails the drill.
+
+## Process and worker failure recovery
+
+```bash
+$COMPOSE stop gochat worker
+# Record GoChatAppDown and GoChatWorkerDown notifications.
+$COMPOSE up -d --wait gochat worker
+# Record both resolved notifications.
+```
+
+Confirm Fluentd contains the shutdown and startup records for both services.
+
+## Stale backup and recovery
+
+Use the existing backup service so the test writes the same shared textfile volume as production:
+
+```bash
+$COMPOSE --profile ops run --rm --entrypoint sh backup -c \
+  'printf "%s\n" "gochat_backup_last_success_timestamp_seconds 0" "gochat_backup_rpo_target_seconds 1" > /metrics/gochat_backup.prom'
+# Wait for GoChatBackupStale and record its notification ID.
+$COMPOSE --profile ops run --rm backup
+# Wait for the resolved notification and confirm the new metric timestamp.
+```
+
+Search Fluentd for the backup service's `backup=... created_at=...` record without copying bundle paths or secrets into the evidence.
+
+## Evidence record
+
+Commit a completed copy of this table or attach it to HH-445. The executor must not be the implementation author.
+
+| Field | Recorded value |
+|---|---|
+| Date / staging revision | |
+| Non-author executor | |
+| Baseline targets healthy | |
+| DB firing / resolved IDs | |
+| Redis firing / resolved IDs | |
+| 5xx firing / resolved IDs | |
+| process + worker firing / resolved IDs | |
+| backup stale firing / resolved IDs | |
+| redacted Fluentd evidence references | |
+| RPO / RTO observed | |
+| Result and follow-ups | |