HH-445: deploy production observability and runbooks (#96)

* HH-445: deploy production observability and runbooks

* fix(ops): share production database DSN

* fix(HH-445): enforce database TLS gate

* fix(HH-445): preserve production serve command

* fix(prod): require external database dependencies

* fix(prod): unify database host rejection gates

* test(prod): enforce exact database TLS runbook contract

---------

Co-authored-by: Rogee <rogee@ipao.vip>

backend/scripts/db_backup.sh

@@ -5,6 +5,9 @@
 set -euo pipefail
 umask 077
 
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+"$script_dir/database_client_entrypoint.sh" --check
+
 dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
 storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
 connector=${GOCHAT_CONNECTOR_BACKUP_FILE:?GOCHAT_CONNECTOR_BACKUP_FILE is required}
@@ -12,6 +15,7 @@ backup_dir=${GOCHAT_BACKUP_DIR:-/var/backups/gochat}
 offsite_dir=${GOCHAT_BACKUP_OFFSITE_DIR:?GOCHAT_BACKUP_OFFSITE_DIR is required}
 passphrase_file=${GOCHAT_BACKUP_PASSPHRASE_FILE:?GOCHAT_BACKUP_PASSPHRASE_FILE is required}
 retention_days=${GOCHAT_BACKUP_RETENTION_DAYS:-30}
+metrics_file=${GOCHAT_BACKUP_METRICS_FILE:-}
 version=${GOCHAT_VERSION:-unknown}
 timestamp=$(date -u +%Y%m%dT%H%M%SZ)
 
@@ -59,4 +63,19 @@ openssl enc -d -aes-256-cbc -pbkdf2 -pass "file:$passphrase_file" -in "$bundle"
 cp "$bundle" "$bundle.sha256" "$offsite_dir/"
 find "$backup_dir" "$offsite_dir" -maxdepth 1 -type f -name 'gochat-*.tar.enc*' -mtime "+$retention_days" -delete
 
+if [[ -n $metrics_file ]]; then
+  install -d -m 0755 "$(dirname "$metrics_file")"
+  metrics_tmp=$metrics_file.tmp
+  {
+    echo '# HELP gochat_backup_last_success_timestamp_seconds Unix time of the last verified off-site backup.'
+    echo '# TYPE gochat_backup_last_success_timestamp_seconds gauge'
+    echo "gochat_backup_last_success_timestamp_seconds $created_at_epoch"
+    echo '# HELP gochat_backup_rpo_target_seconds Maximum allowed age of the latest backup.'
+    echo '# TYPE gochat_backup_rpo_target_seconds gauge'
+    echo 'gochat_backup_rpo_target_seconds 86400'
+  } >"$metrics_tmp"
+  chmod 0644 "$metrics_tmp"
+  mv "$metrics_tmp" "$metrics_file"
+fi
+
 echo "backup=$bundle offsite=$offsite_dir/$(basename "$bundle") version=$version created_at=$timestamp"

backend/scripts/db_backup_test.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin" "$tmp/scripts" "$tmp/storage" "$tmp/local" "$tmp/offsite" "$tmp/metrics"
+cp "$script_dir/db_backup.sh" "$script_dir/../../deploy/docker/database_client_entrypoint.sh" "$tmp/scripts/"
+printf 'attachment\n' >"$tmp/storage/file.txt"
+printf 'connector\n' >"$tmp/connector.db"
+printf 'test-passphrase\n' >"$tmp/passphrase"
+
+cat >"$tmp/bin/psql" <<'EOF'
+#!/usr/bin/env bash
+echo 160000
+EOF
+cat >"$tmp/bin/pg_dump" <<'EOF'
+#!/usr/bin/env bash
+if [[ $1 == --version ]]; then
+  echo 'pg_dump (PostgreSQL) 16.0'
+  exit
+fi
+while (($#)); do
+  if [[ $1 == --file ]]; then
+    printf 'dump\n' >"$2"
+    exit
+  fi
+  shift
+done
+exit 1
+EOF
+cat >"$tmp/bin/pg_restore" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+chmod +x "$tmp/bin/psql" "$tmp/bin/pg_dump" "$tmp/bin/pg_restore"
+
+if GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=disable' "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
+  echo 'backup gate accepted sslmode=disable' >&2
+  exit 1
+fi
+grep -F 'sslmode must be verify-ca or verify-full' "$tmp/rejected" >/dev/null
+while IFS='|' read -r name dsn; do
+  if GOCHAT_DATABASE_DSN=$dsn "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
+    echo "backup gate accepted $name" >&2
+    exit 1
+  fi
+  grep -F 'must use an external PostgreSQL host' "$tmp/rejected" >/dev/null
+done < "$script_dir/../../deploy/docker/database_host_rejection_cases.txt"
+
+PATH="$tmp/bin:$PATH" \
+GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=verify-full' \
+GOCHAT_STORAGE_PATH="$tmp/storage" \
+GOCHAT_CONNECTOR_BACKUP_FILE="$tmp/connector.db" \
+GOCHAT_BACKUP_DIR="$tmp/local" \
+GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite" \
+GOCHAT_BACKUP_PASSPHRASE_FILE="$tmp/passphrase" \
+GOCHAT_BACKUP_METRICS_FILE="$tmp/metrics/gochat_backup.prom" \
+  "$tmp/scripts/db_backup.sh" >"$tmp/output"
+
+grep -Eq '^gochat_backup_last_success_timestamp_seconds [0-9]+$' "$tmp/metrics/gochat_backup.prom"
+grep -Fx 'gochat_backup_rpo_target_seconds 86400' "$tmp/metrics/gochat_backup.prom" >/dev/null
+test "$(find "$tmp/offsite" -name 'gochat-*.tar.enc' | wc -l)" -eq 1
+grep -F 'backup=' "$tmp/output" >/dev/null
+echo 'backup metric test passed'

backend/scripts/db_restore.sh

@@ -5,6 +5,9 @@
 set -euo pipefail
 umask 077
 
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+"$script_dir/database_client_entrypoint.sh" --check
+
 bundle=${1:?usage: db_restore.sh /path/to/gochat-*.tar.enc}
 dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
 storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}