HH-445: deploy production observability and runbooks (#96)

* HH-445: deploy production observability and runbooks * fix(ops): share production database DSN * fix(HH-445): enforce database TLS gate * fix(HH-445): preserve production serve command * fix(prod): require external database dependencies * fix(prod): unify database host rejection gates * test(prod): enforce exact database TLS runbook contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
2026-08-22 19:39:57 +08:00
parent 61376a57fd
commit fb83285617
28 changed files with 1622 additions and 220 deletions
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin" "$tmp/scripts" "$tmp/storage" "$tmp/local" "$tmp/offsite" "$tmp/metrics"
+cp "$script_dir/db_backup.sh" "$script_dir/../../deploy/docker/database_client_entrypoint.sh" "$tmp/scripts/"
+printf 'attachment\n' >"$tmp/storage/file.txt"
+printf 'connector\n' >"$tmp/connector.db"
+printf 'test-passphrase\n' >"$tmp/passphrase"
+
+cat >"$tmp/bin/psql" <<'EOF'
+#!/usr/bin/env bash
+echo 160000
+EOF
+cat >"$tmp/bin/pg_dump" <<'EOF'
+#!/usr/bin/env bash
+if [[ $1 == --version ]]; then
+  echo 'pg_dump (PostgreSQL) 16.0'
+  exit
+fi
+while (($#)); do
+  if [[ $1 == --file ]]; then
+    printf 'dump\n' >"$2"
+    exit
+  fi
+  shift
+done
+exit 1
+EOF
+cat >"$tmp/bin/pg_restore" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+chmod +x "$tmp/bin/psql" "$tmp/bin/pg_dump" "$tmp/bin/pg_restore"
+
+if GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=disable' "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
+  echo 'backup gate accepted sslmode=disable' >&2
+  exit 1
+fi
+grep -F 'sslmode must be verify-ca or verify-full' "$tmp/rejected" >/dev/null
+while IFS='|' read -r name dsn; do
+  if GOCHAT_DATABASE_DSN=$dsn "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
+    echo "backup gate accepted $name" >&2
+    exit 1
+  fi
+  grep -F 'must use an external PostgreSQL host' "$tmp/rejected" >/dev/null
+done < "$script_dir/../../deploy/docker/database_host_rejection_cases.txt"
+
+PATH="$tmp/bin:$PATH" \
+GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=verify-full' \
+GOCHAT_STORAGE_PATH="$tmp/storage" \
+GOCHAT_CONNECTOR_BACKUP_FILE="$tmp/connector.db" \
+GOCHAT_BACKUP_DIR="$tmp/local" \
+GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite" \
+GOCHAT_BACKUP_PASSPHRASE_FILE="$tmp/passphrase" \
+GOCHAT_BACKUP_METRICS_FILE="$tmp/metrics/gochat_backup.prom" \
+  "$tmp/scripts/db_backup.sh" >"$tmp/output"
+
+grep -Eq '^gochat_backup_last_success_timestamp_seconds [0-9]+$' "$tmp/metrics/gochat_backup.prom"
+grep -Fx 'gochat_backup_rpo_target_seconds 86400' "$tmp/metrics/gochat_backup.prom" >/dev/null
+test "$(find "$tmp/offsite" -name 'gochat-*.tar.enc' | wc -l)" -eq 1
+grep -F 'backup=' "$tmp/output" >/dev/null
+echo 'backup metric test passed'