Rogeeandrogee
fb83285617
* HH-445: deploy production observability and runbooks * fix(ops): share production database DSN * fix(HH-445): enforce database TLS gate * fix(HH-445): preserve production serve command * fix(prod): require external database dependencies * fix(prod): unify database host rejection gates * test(prod): enforce exact database TLS runbook contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
139 lines
6.7 KiB
Bash
Executable File
139 lines
6.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
mkdir -p "$tmp/bin" "$tmp/local" "$tmp/offsite"
|
|
printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
|
|
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-preflight-test' \
|
|
-keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
|
|
cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
|
|
chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
|
|
chmod 0640 "$tmp/external-db-client.key"
|
|
|
|
cat > "$tmp/bin/findmnt" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
if [[ $1 == -M && $2 == "$GOCHAT_BACKUP_OFFSITE_DIR" ]]; then
|
|
printf '%s %s %s\n' "$TEST_OFFSITE_SOURCE" "$TEST_OFFSITE_FSTYPE" "$TEST_OFFSITE_DEVICE"
|
|
elif [[ $1 == -T ]]; then
|
|
printf '%s %s\n' "$TEST_LOCAL_SOURCE" "$TEST_LOCAL_DEVICE"
|
|
else
|
|
exit 1
|
|
fi
|
|
EOF
|
|
cat > "$tmp/bin/docker" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
if [[ -n ${TEST_MUTABLE_IMAGE:-} ]]; then
|
|
echo 'pgvector/pgvector:pg16'
|
|
else
|
|
printf '%s\n' \
|
|
'gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000' \
|
|
'gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
|
fi
|
|
EOF
|
|
chmod +x "$tmp/bin/findmnt" "$tmp/bin/docker"
|
|
|
|
export PATH="$tmp/bin:$PATH"
|
|
export GOCHAT_IMAGE_REF='gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
|
|
export SHANGWUTONG_IMAGE_REF='gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
|
export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.ci.rogeecn.com
|
|
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
|
|
export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
|
|
export POSTGRES_PASSWORD=ci-postgres-secret
|
|
export REDIS_PASSWORD=ci-redis-secret
|
|
export MEILI_MASTER_KEY=ci-meili-secret-16
|
|
export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
|
export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
|
|
export GOCHAT_BACKUP_DIR="$tmp/local"
|
|
export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
|
|
export GOCHAT_BACKUP_OFFSITE_SOURCE='backup.example.test:/gochat'
|
|
export GOCHAT_BACKUP_OFFSITE_FSTYPE=nfs4
|
|
export TEST_OFFSITE_SOURCE=$GOCHAT_BACKUP_OFFSITE_SOURCE
|
|
export TEST_OFFSITE_FSTYPE=$GOCHAT_BACKUP_OFFSITE_FSTYPE
|
|
export TEST_OFFSITE_DEVICE=0:42
|
|
export TEST_LOCAL_SOURCE=/dev/sda1
|
|
export TEST_LOCAL_DEVICE=8:1
|
|
|
|
run_preflight() {
|
|
"$root/deploy/docker/preflight.sh" > "$tmp/output" 2>&1
|
|
}
|
|
|
|
expect_failure() {
|
|
local name=$1 expected=$2
|
|
if run_preflight; then
|
|
echo "preflight accepted $name" >&2
|
|
exit 1
|
|
fi
|
|
grep -F "$expected" "$tmp/output" >/dev/null
|
|
}
|
|
|
|
GOCHAT_BACKUP_OFFSITE_DIR=/
|
|
expect_failure 'the root filesystem' 'must not be /'
|
|
|
|
GOCHAT_BACKUP_OFFSITE_DIR=/tmp
|
|
TEST_OFFSITE_SOURCE=tmpfs
|
|
TEST_OFFSITE_FSTYPE=tmpfs
|
|
GOCHAT_BACKUP_OFFSITE_SOURCE=tmpfs
|
|
GOCHAT_BACKUP_OFFSITE_FSTYPE=tmpfs
|
|
expect_failure 'tmpfs' 'must not use an in-memory filesystem'
|
|
|
|
GOCHAT_BACKUP_OFFSITE_DIR=$tmp/offsite
|
|
TEST_OFFSITE_SOURCE='backup.example.test:/gochat'
|
|
TEST_OFFSITE_FSTYPE=nfs4
|
|
GOCHAT_BACKUP_OFFSITE_SOURCE=$TEST_OFFSITE_SOURCE
|
|
GOCHAT_BACKUP_OFFSITE_FSTYPE=$TEST_OFFSITE_FSTYPE
|
|
TEST_OFFSITE_DEVICE=$TEST_LOCAL_DEVICE
|
|
expect_failure 'the local backup device' 'must use a different source/device'
|
|
|
|
TEST_OFFSITE_DEVICE=0:42
|
|
GOCHAT_BACKUP_OFFSITE_SOURCE='other.example.test:/gochat'
|
|
expect_failure 'an unapproved mount source' 'does not match the approved values'
|
|
|
|
GOCHAT_BACKUP_OFFSITE_SOURCE=$TEST_OFFSITE_SOURCE
|
|
MEILI_MASTER_KEY=too-short
|
|
expect_failure 'a short Meilisearch key' 'must be at least 16 bytes'
|
|
MEILI_MASTER_KEY=ci-meili-secret-16
|
|
|
|
printf '%s\n' 'http://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
|
|
expect_failure 'a plaintext alert receiver' 'must contain one HTTPS URL'
|
|
printf '%s\n' 'https://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
|
|
|
|
TEST_MUTABLE_IMAGE=1
|
|
export TEST_MUTABLE_IMAGE
|
|
expect_failure 'a mutable production image' 'must be pinned to a sha256 digest'
|
|
unset TEST_MUTABLE_IMAGE
|
|
|
|
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable'
|
|
expect_failure 'an external database without verified TLS' 'sslmode must be verify-ca or verify-full'
|
|
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=require'
|
|
expect_failure 'an external database without certificate verification' 'sslmode must be verify-ca or verify-full'
|
|
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable&sslmode=verify-full'
|
|
expect_failure 'duplicate sslmode parameters with a disabling first value' 'must contain exactly one sslmode'
|
|
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslmode=verify-full'
|
|
expect_failure 'duplicate allowed sslmode parameters' 'must contain exactly one sslmode'
|
|
while IFS='|' read -r name GOCHAT_DATABASE_DSN; do
|
|
expect_failure "$name" 'must use an external PostgreSQL host'
|
|
done < "$root/deploy/docker/database_host_rejection_cases.txt"
|
|
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/tmp/ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
|
expect_failure 'a non-fixed certificate path' 'sslrootcert must use /run/secrets/external-db-ca.crt'
|
|
|
|
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
|
export GOCHAT_DATABASE_TLS_GID
|
|
GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
|
|
unset GOCHAT_DATABASE_TLS_CA_FILE GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE
|
|
expect_failure 'missing host TLS variables' 'GOCHAT_DATABASE_TLS_CA_FILE must be set'
|
|
|
|
export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/missing-ca.crt"
|
|
export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
|
|
export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
|
|
expect_failure 'a missing host TLS file' 'GOCHAT_DATABASE_TLS_CA_FILE must point to an existing regular file'
|
|
|
|
GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
|
|
chmod 0600 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
|
|
expect_failure 'a private key without group read permission' 'GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE must be readable by GOCHAT_DATABASE_TLS_GID'
|
|
chmod 0640 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
|
|
run_preflight
|
|
grep -F 'production preflight passed' "$tmp/output" >/dev/null
|
|
echo 'preflight tests passed'
|