HH-445: deploy production observability and runbooks (#96)

* HH-445: deploy production observability and runbooks

* fix(ops): share production database DSN

* fix(HH-445): enforce database TLS gate

* fix(HH-445): preserve production serve command

* fix(prod): require external database dependencies

* fix(prod): unify database host rejection gates

* test(prod): enforce exact database TLS runbook contract

---------

Co-authored-by: Rogee <rogee@ipao.vip>

deploy/docker/preflight_test.sh

@@ -5,6 +5,12 @@ root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
 tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
 mkdir -p "$tmp/bin" "$tmp/local" "$tmp/offsite"
+printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-preflight-test' \
+  -keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
+cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
+chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
+chmod 0640 "$tmp/external-db-client.key"
 
 cat > "$tmp/bin/findmnt" <<'EOF'
 #!/usr/bin/env bash
@@ -32,10 +38,13 @@ export PATH="$tmp/bin:$PATH"
 export GOCHAT_IMAGE_REF='gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
 export SHANGWUTONG_IMAGE_REF='gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
 export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.ci.rogeecn.com
+export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
+export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
 export POSTGRES_PASSWORD=ci-postgres-secret
 export REDIS_PASSWORD=ci-redis-secret
 export MEILI_MASTER_KEY=ci-meili-secret-16
 export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
+export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
 export GOCHAT_BACKUP_DIR="$tmp/local"
 export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
 export GOCHAT_BACKUP_OFFSITE_SOURCE='backup.example.test:/gochat'
@@ -86,11 +95,44 @@ MEILI_MASTER_KEY=too-short
 expect_failure 'a short Meilisearch key' 'must be at least 16 bytes'
 MEILI_MASTER_KEY=ci-meili-secret-16
 
+printf '%s\n' 'http://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
+expect_failure 'a plaintext alert receiver' 'must contain one HTTPS URL'
+printf '%s\n' 'https://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
+
 TEST_MUTABLE_IMAGE=1
 export TEST_MUTABLE_IMAGE
 expect_failure 'a mutable production image' 'must be pinned to a sha256 digest'
 unset TEST_MUTABLE_IMAGE
 
+export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable'
+expect_failure 'an external database without verified TLS' 'sslmode must be verify-ca or verify-full'
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=require'
+expect_failure 'an external database without certificate verification' 'sslmode must be verify-ca or verify-full'
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable&sslmode=verify-full'
+expect_failure 'duplicate sslmode parameters with a disabling first value' 'must contain exactly one sslmode'
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslmode=verify-full'
+expect_failure 'duplicate allowed sslmode parameters' 'must contain exactly one sslmode'
+while IFS='|' read -r name GOCHAT_DATABASE_DSN; do
+  expect_failure "$name" 'must use an external PostgreSQL host'
+done < "$root/deploy/docker/database_host_rejection_cases.txt"
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/tmp/ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+expect_failure 'a non-fixed certificate path' 'sslrootcert must use /run/secrets/external-db-ca.crt'
+
+GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
+export GOCHAT_DATABASE_TLS_GID
+GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
+unset GOCHAT_DATABASE_TLS_CA_FILE GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE
+expect_failure 'missing host TLS variables' 'GOCHAT_DATABASE_TLS_CA_FILE must be set'
+
+export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/missing-ca.crt"
+export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
+export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
+expect_failure 'a missing host TLS file' 'GOCHAT_DATABASE_TLS_CA_FILE must point to an existing regular file'
+
+GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
+chmod 0600 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
+expect_failure 'a private key without group read permission' 'GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE must be readable by GOCHAT_DATABASE_TLS_GID'
+chmod 0640 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
 run_preflight
 grep -F 'production preflight passed' "$tmp/output" >/dev/null
 echo 'preflight tests passed'