HH-445: deploy production observability and runbooks (#96)
* HH-445: deploy production observability and runbooks * fix(ops): share production database DSN * fix(HH-445): enforce database TLS gate * fix(HH-445): preserve production serve command * fix(prod): require external database dependencies * fix(prod): unify database host rejection gates * test(prod): enforce exact database TLS runbook contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
gate=$root/deploy/docker/database_client_entrypoint.sh
|
||||
gochat_image=${GOCHAT_TEST_IMAGE:-gochat:production-smoke}
|
||||
postgres_image='pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b'
|
||||
exporter_image='quay.io/prometheuscommunity/postgres-exporter:v0.17.1@sha256:38606faa38c54787525fb0ff2fd6b41b4cfb75d455c1df294927c5f611699b17'
|
||||
tmp=$(mktemp -d)
|
||||
network=gochat-db-gate-$$
|
||||
database=gochat-db-gate-postgres-$$
|
||||
exporter=gochat-db-gate-exporter-$$
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$exporter" "$database" >/dev/null 2>&1 || true
|
||||
docker network rm "$network" >/dev/null 2>&1 || true
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$tmp/ca" "$tmp/server" "$tmp/client" "$tmp/denied"
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=GoChat test CA' \
|
||||
-keyout "$tmp/ca/ca.key" -out "$tmp/ca/ca.crt" >/dev/null 2>&1
|
||||
openssl req -newkey rsa:2048 -nodes -subj '/CN=tls-db' \
|
||||
-keyout "$tmp/server/server.key" -out "$tmp/server/server.csr" >/dev/null 2>&1
|
||||
printf 'subjectAltName=DNS:tls-db\n' >"$tmp/server/server.ext"
|
||||
openssl x509 -req -days 1 -in "$tmp/server/server.csr" -CA "$tmp/ca/ca.crt" -CAkey "$tmp/ca/ca.key" \
|
||||
-CAcreateserial -extfile "$tmp/server/server.ext" -out "$tmp/server/server.crt" >/dev/null 2>&1
|
||||
openssl req -newkey rsa:2048 -nodes -subj '/CN=gochat-client' \
|
||||
-keyout "$tmp/client/external-db-client.key" -out "$tmp/client/client.csr" >/dev/null 2>&1
|
||||
openssl x509 -req -days 1 -in "$tmp/client/client.csr" -CA "$tmp/ca/ca.crt" -CAkey "$tmp/ca/ca.key" \
|
||||
-CAcreateserial -out "$tmp/client/external-db-client.crt" >/dev/null 2>&1
|
||||
cp "$tmp/ca/ca.crt" "$tmp/client/external-db-ca.crt"
|
||||
cp "$tmp/client/external-db-client.key" "$tmp/denied/external-db-client.key"
|
||||
|
||||
tls_gid=4242
|
||||
postgres_uid=$(docker run --rm --entrypoint id "$postgres_image" -u postgres)
|
||||
postgres_gid=$(docker run --rm --entrypoint id "$postgres_image" -g postgres)
|
||||
docker run --rm --entrypoint sh -v "$tmp:/work" "$postgres_image" -c \
|
||||
"chown $postgres_uid:$postgres_gid /work/server/server.crt /work/server/server.key && chmod 0644 /work/server/server.crt && chmod 0600 /work/server/server.key && chown 0:$tls_gid /work/client/external-db-*.crt /work/client/external-db-client.key && chmod 0644 /work/client/external-db-*.crt && chmod 0640 /work/client/external-db-client.key && chown 0:0 /work/denied/external-db-client.key && chmod 0600 /work/denied/external-db-client.key"
|
||||
|
||||
bad_dsn='postgres://postgres:test-password@tls-db:5432/postgres?sslmode=disable&sslmode=verify-full'
|
||||
reject_duplicate() {
|
||||
local label=$1 image=$2 variable=$3 user=$4
|
||||
shift 4
|
||||
local args=(docker run --rm -e "$variable=$bad_dsn" -v "$gate:/usr/local/bin/database-client-entrypoint:ro" --entrypoint /usr/local/bin/database-client-entrypoint)
|
||||
[[ -z $user ]] || args+=(--user "$user")
|
||||
if "${args[@]}" "$image" "$@" >"$tmp/$label.log" 2>&1; then
|
||||
echo "$label accepted duplicate sslmode" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'exactly one sslmode' "$tmp/$label.log" >/dev/null
|
||||
}
|
||||
|
||||
reject_duplicate gochat "$gochat_image" GOCHAT_DATABASE_DSN '' /app/gochat --help
|
||||
reject_duplicate worker "$gochat_image" GOCHAT_DATABASE_DSN '' /app/gochat worker
|
||||
reject_duplicate migrate "$gochat_image" GOCHAT_DATABASE_DSN '' /app/migrate version
|
||||
reject_duplicate backup "$gochat_image" GOCHAT_DATABASE_DSN 0:0 /app/scripts/db_backup.sh
|
||||
reject_duplicate restore "$gochat_image" GOCHAT_DATABASE_DSN 0:0 /app/scripts/db_restore.sh /tmp/missing.tar.enc
|
||||
reject_duplicate postgres-exporter "$exporter_image" DATA_SOURCE_NAME '' /bin/postgres_exporter --version
|
||||
|
||||
production_env=(
|
||||
-e GOCHAT_ENV=production
|
||||
-e GOCHAT_DATABASE_DSN="$bad_dsn"
|
||||
-e GOCHAT_REDIS_DSN=redis://:test-password@redis:6379
|
||||
-e GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
||||
-e GOCHAT_SEARCH_API_KEY=ci-meili-secret-16
|
||||
-e GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
|
||||
)
|
||||
for command in serve worker; do
|
||||
if docker run --rm "${production_env[@]}" --entrypoint /app/gochat "$gochat_image" "$command" >"$tmp/direct-$command.log" 2>&1; then
|
||||
echo "direct GoChat $command accepted duplicate sslmode" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'production database DSN must contain exactly one sslmode' "$tmp/direct-$command.log" >/dev/null
|
||||
done
|
||||
if docker run --rm "${production_env[@]}" --entrypoint /app/migrate "$gochat_image" version >"$tmp/direct-migrate.log" 2>&1; then
|
||||
echo 'direct migrate accepted duplicate sslmode' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'production database DSN must contain exactly one sslmode' "$tmp/direct-migrate.log" >/dev/null
|
||||
for client in backup restore; do
|
||||
script=/app/scripts/db_${client}.sh
|
||||
args=()
|
||||
[[ $client == backup ]] || args=(/tmp/missing.tar.enc)
|
||||
if docker run --rm --user 0:0 -e GOCHAT_DATABASE_DSN="$bad_dsn" --entrypoint "$script" "$gochat_image" "${args[@]}" >"$tmp/direct-$client.log" 2>&1; then
|
||||
echo "direct $client accepted duplicate sslmode" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'database DSN must contain exactly one sslmode' "$tmp/direct-$client.log" >/dev/null
|
||||
done
|
||||
|
||||
test "$(docker run --rm --entrypoint id "$gochat_image" -u)" -ne 0
|
||||
test "$(docker run --rm --entrypoint id "$exporter_image" -u)" -ne 0
|
||||
|
||||
dsn='postgres://postgres:test-password@tls-db:5432/postgres?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
||||
if docker run --rm --group-add "$tls_gid" -e GOCHAT_DATABASE_DSN="$dsn" \
|
||||
-v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
|
||||
-v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
|
||||
-v "$tmp/denied/external-db-client.key:/run/secrets/external-db-client.key:ro" \
|
||||
--entrypoint /usr/local/bin/database-client-entrypoint "$gochat_image" /bin/true >"$tmp/permissions.log" 2>&1; then
|
||||
echo 'non-root GoChat client read a root-owned 0600 private key' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'database TLS file is missing or unreadable' "$tmp/permissions.log" >/dev/null
|
||||
|
||||
docker network create "$network" >/dev/null
|
||||
docker run -d --name "$database" --network "$network" --network-alias tls-db \
|
||||
-e POSTGRES_PASSWORD=test-password \
|
||||
-v "$tmp/server/server.crt:/certs/server.crt:ro" \
|
||||
-v "$tmp/server/server.key:/certs/server.key:ro" \
|
||||
"$postgres_image" -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key >/dev/null
|
||||
for _ in $(seq 1 30); do
|
||||
docker exec "$database" pg_isready -U postgres >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$database" pg_isready -U postgres >/dev/null
|
||||
|
||||
psql_result=$(docker run --rm --network "$network" --group-add "$tls_gid" -e GOCHAT_DATABASE_DSN="$dsn" \
|
||||
-v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
|
||||
-v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.key:/run/secrets/external-db-client.key:ro" \
|
||||
--entrypoint /usr/local/bin/database-client-entrypoint "$gochat_image" psql "$dsn" -Atqc 'SELECT 1')
|
||||
test "$psql_result" = 1
|
||||
|
||||
docker run -d --name "$exporter" --network "$network" --group-add "$tls_gid" -e DATA_SOURCE_NAME="$dsn" \
|
||||
-v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
|
||||
-v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.key:/run/secrets/external-db-client.key:ro" \
|
||||
--entrypoint /usr/local/bin/database-client-entrypoint "$exporter_image" /bin/postgres_exporter >/dev/null
|
||||
for _ in $(seq 1 20); do
|
||||
if docker run --rm --network "$network" --entrypoint sh "$gochat_image" -c \
|
||||
'curl -fsS http://gochat-db-gate-exporter-'"$$"':9187/metrics' >"$tmp/metrics" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
grep -E '^pg_up 1$' "$tmp/metrics" >/dev/null
|
||||
|
||||
echo 'database client entrypoint tests passed'
|
||||
Reference in New Issue
Block a user