HH-445: deploy production observability and runbooks (#96)
* HH-445: deploy production observability and runbooks * fix(ops): share production database DSN * fix(HH-445): enforce database TLS gate * fix(HH-445): preserve production serve command * fix(prod): require external database dependencies * fix(prod): unify database host rejection gates * test(prod): enforce exact database TLS runbook contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
@@ -78,7 +78,7 @@ COPY --from=builder /migrate /app/migrate
|
||||
COPY --chown=gochat:gochat --from=frontend-builder /app/frontend/dist /app/frontend/dist
|
||||
COPY backend/configs/ /app/configs/
|
||||
COPY backend/migrations/ /app/migrations/
|
||||
COPY backend/scripts/db_backup.sh backend/scripts/db_restore.sh /app/scripts/
|
||||
COPY backend/scripts/db_backup.sh backend/scripts/db_restore.sh deploy/docker/database_client_entrypoint.sh /app/scripts/
|
||||
|
||||
ENV GOCHAT_FRONTEND_DIST=/app/frontend/dist
|
||||
|
||||
|
||||
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
is_local_database_host() {
|
||||
database_host_value=$1
|
||||
case $database_host_value in
|
||||
'' | postgres | localhost | 127.*) return 0 ;;
|
||||
esac
|
||||
|
||||
if [ "$(printf '%s' "$database_host_value" | tr -d '0:')" = 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
database_ipv6_prefix=${database_host_value%:*}
|
||||
database_ipv6_tail=${database_host_value##*:}
|
||||
database_ipv6_marker=$(printf '%s' "$database_ipv6_prefix" | tr -d '0:')
|
||||
case $database_ipv6_tail in
|
||||
0.0.0.1) [ -z "$database_ipv6_marker" ] && return 0 ;;
|
||||
127.*) [ "$database_ipv6_marker" = ffff ] && return 0 ;;
|
||||
esac
|
||||
|
||||
database_ipv6_high=${database_ipv6_prefix##*:}
|
||||
database_ipv6_prefix=${database_ipv6_prefix%:*}
|
||||
database_ipv6_marker=$(printf '%s' "$database_ipv6_prefix" | tr -d '0:')
|
||||
case $database_ipv6_high in
|
||||
7f[0-9a-f][0-9a-f]) [ "$database_ipv6_marker" = ffff ] && return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
validate_production_database_dsn() {
|
||||
database_dsn=$1
|
||||
database_dsn_label=${2:-production database DSN}
|
||||
database_dsn_uses_certificate_files=0
|
||||
if [ -z "$database_dsn" ]; then
|
||||
echo "$database_dsn_label is required" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
database_authority=${database_dsn#*://}
|
||||
database_authority=${database_authority%%/*}
|
||||
database_hostport=${database_authority##*@}
|
||||
case $database_hostport in
|
||||
\[* ) database_host=${database_hostport#\[}; database_host=${database_host%%\]*} ;;
|
||||
* ) database_host=${database_hostport%%:*} ;;
|
||||
esac
|
||||
database_host=$(printf '%s' "$database_host" | tr '[:upper:]' '[:lower:]')
|
||||
database_host=${database_host%.}
|
||||
database_host=${database_host%%%*}
|
||||
if is_local_database_host "$database_host"; then
|
||||
echo "$database_dsn_label must use an external PostgreSQL host" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
database_query=
|
||||
case $database_dsn in
|
||||
*\?*) database_query=${database_dsn#*\?}; database_query=${database_query%%#*} ;;
|
||||
esac
|
||||
|
||||
sslmode_count=0
|
||||
sslmode=
|
||||
sslrootcert_count=0
|
||||
sslcert_count=0
|
||||
sslkey_count=0
|
||||
database_parameters=$database_query
|
||||
while [ -n "$database_parameters" ]; do
|
||||
case $database_parameters in
|
||||
*'&'*) database_parameter=${database_parameters%%&*}; database_parameters=${database_parameters#*&} ;;
|
||||
*) database_parameter=$database_parameters; database_parameters= ;;
|
||||
esac
|
||||
database_key=${database_parameter%%=*}
|
||||
database_value=${database_parameter#*=}
|
||||
case $database_key in
|
||||
sslmode) sslmode_count=$((sslmode_count + 1)); sslmode=$database_value ;;
|
||||
sslrootcert)
|
||||
sslrootcert_count=$((sslrootcert_count + 1))
|
||||
[ "$database_value" = /run/secrets/external-db-ca.crt ] || { echo "$database_dsn_label sslrootcert must use /run/secrets/external-db-ca.crt" >&2; return 1; }
|
||||
;;
|
||||
sslcert)
|
||||
sslcert_count=$((sslcert_count + 1))
|
||||
[ "$database_value" = /run/secrets/external-db-client.crt ] || { echo "$database_dsn_label sslcert must use /run/secrets/external-db-client.crt" >&2; return 1; }
|
||||
;;
|
||||
sslkey)
|
||||
sslkey_count=$((sslkey_count + 1))
|
||||
[ "$database_value" = /run/secrets/external-db-client.key ] || { echo "$database_dsn_label sslkey must use /run/secrets/external-db-client.key" >&2; return 1; }
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$sslmode_count" -ne 1 ]; then
|
||||
echo "$database_dsn_label must contain exactly one sslmode" >&2
|
||||
return 1
|
||||
fi
|
||||
if [ "$sslmode" != verify-ca ] && [ "$sslmode" != verify-full ]; then
|
||||
echo "$database_dsn_label sslmode must be verify-ca or verify-full" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
tls_count=$((sslrootcert_count + sslcert_count + sslkey_count))
|
||||
if [ "$tls_count" -ne 0 ]; then
|
||||
database_dsn_uses_certificate_files=1
|
||||
if [ "$sslrootcert_count" -ne 1 ] || [ "$sslcert_count" -ne 1 ] || [ "$sslkey_count" -ne 1 ]; then
|
||||
echo "$database_dsn_label must contain each fixed TLS certificate parameter exactly once" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
dsn=${GOCHAT_DATABASE_DSN:-${DATA_SOURCE_NAME:-}}
|
||||
validate_production_database_dsn "$dsn"
|
||||
|
||||
if [ "${1:-}" = --preflight ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$database_dsn_uses_certificate_files" -eq 1 ]; then
|
||||
for file in /run/secrets/external-db-ca.crt /run/secrets/external-db-client.crt /run/secrets/external-db-client.key; do
|
||||
[ -f "$file" ] && [ -r "$file" ] || { echo "database TLS file is missing or unreadable: $file" >&2; exit 1; }
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${1:-}" = --check ]; then
|
||||
exit 0
|
||||
fi
|
||||
exec "$@"
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
gate=$root/deploy/docker/database_client_entrypoint.sh
|
||||
gochat_image=${GOCHAT_TEST_IMAGE:-gochat:production-smoke}
|
||||
postgres_image='pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b'
|
||||
exporter_image='quay.io/prometheuscommunity/postgres-exporter:v0.17.1@sha256:38606faa38c54787525fb0ff2fd6b41b4cfb75d455c1df294927c5f611699b17'
|
||||
tmp=$(mktemp -d)
|
||||
network=gochat-db-gate-$$
|
||||
database=gochat-db-gate-postgres-$$
|
||||
exporter=gochat-db-gate-exporter-$$
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$exporter" "$database" >/dev/null 2>&1 || true
|
||||
docker network rm "$network" >/dev/null 2>&1 || true
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$tmp/ca" "$tmp/server" "$tmp/client" "$tmp/denied"
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=GoChat test CA' \
|
||||
-keyout "$tmp/ca/ca.key" -out "$tmp/ca/ca.crt" >/dev/null 2>&1
|
||||
openssl req -newkey rsa:2048 -nodes -subj '/CN=tls-db' \
|
||||
-keyout "$tmp/server/server.key" -out "$tmp/server/server.csr" >/dev/null 2>&1
|
||||
printf 'subjectAltName=DNS:tls-db\n' >"$tmp/server/server.ext"
|
||||
openssl x509 -req -days 1 -in "$tmp/server/server.csr" -CA "$tmp/ca/ca.crt" -CAkey "$tmp/ca/ca.key" \
|
||||
-CAcreateserial -extfile "$tmp/server/server.ext" -out "$tmp/server/server.crt" >/dev/null 2>&1
|
||||
openssl req -newkey rsa:2048 -nodes -subj '/CN=gochat-client' \
|
||||
-keyout "$tmp/client/external-db-client.key" -out "$tmp/client/client.csr" >/dev/null 2>&1
|
||||
openssl x509 -req -days 1 -in "$tmp/client/client.csr" -CA "$tmp/ca/ca.crt" -CAkey "$tmp/ca/ca.key" \
|
||||
-CAcreateserial -out "$tmp/client/external-db-client.crt" >/dev/null 2>&1
|
||||
cp "$tmp/ca/ca.crt" "$tmp/client/external-db-ca.crt"
|
||||
cp "$tmp/client/external-db-client.key" "$tmp/denied/external-db-client.key"
|
||||
|
||||
tls_gid=4242
|
||||
postgres_uid=$(docker run --rm --entrypoint id "$postgres_image" -u postgres)
|
||||
postgres_gid=$(docker run --rm --entrypoint id "$postgres_image" -g postgres)
|
||||
docker run --rm --entrypoint sh -v "$tmp:/work" "$postgres_image" -c \
|
||||
"chown $postgres_uid:$postgres_gid /work/server/server.crt /work/server/server.key && chmod 0644 /work/server/server.crt && chmod 0600 /work/server/server.key && chown 0:$tls_gid /work/client/external-db-*.crt /work/client/external-db-client.key && chmod 0644 /work/client/external-db-*.crt && chmod 0640 /work/client/external-db-client.key && chown 0:0 /work/denied/external-db-client.key && chmod 0600 /work/denied/external-db-client.key"
|
||||
|
||||
bad_dsn='postgres://postgres:test-password@tls-db:5432/postgres?sslmode=disable&sslmode=verify-full'
|
||||
reject_duplicate() {
|
||||
local label=$1 image=$2 variable=$3 user=$4
|
||||
shift 4
|
||||
local args=(docker run --rm -e "$variable=$bad_dsn" -v "$gate:/usr/local/bin/database-client-entrypoint:ro" --entrypoint /usr/local/bin/database-client-entrypoint)
|
||||
[[ -z $user ]] || args+=(--user "$user")
|
||||
if "${args[@]}" "$image" "$@" >"$tmp/$label.log" 2>&1; then
|
||||
echo "$label accepted duplicate sslmode" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'exactly one sslmode' "$tmp/$label.log" >/dev/null
|
||||
}
|
||||
|
||||
reject_duplicate gochat "$gochat_image" GOCHAT_DATABASE_DSN '' /app/gochat --help
|
||||
reject_duplicate worker "$gochat_image" GOCHAT_DATABASE_DSN '' /app/gochat worker
|
||||
reject_duplicate migrate "$gochat_image" GOCHAT_DATABASE_DSN '' /app/migrate version
|
||||
reject_duplicate backup "$gochat_image" GOCHAT_DATABASE_DSN 0:0 /app/scripts/db_backup.sh
|
||||
reject_duplicate restore "$gochat_image" GOCHAT_DATABASE_DSN 0:0 /app/scripts/db_restore.sh /tmp/missing.tar.enc
|
||||
reject_duplicate postgres-exporter "$exporter_image" DATA_SOURCE_NAME '' /bin/postgres_exporter --version
|
||||
|
||||
production_env=(
|
||||
-e GOCHAT_ENV=production
|
||||
-e GOCHAT_DATABASE_DSN="$bad_dsn"
|
||||
-e GOCHAT_REDIS_DSN=redis://:test-password@redis:6379
|
||||
-e GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
||||
-e GOCHAT_SEARCH_API_KEY=ci-meili-secret-16
|
||||
-e GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
|
||||
)
|
||||
for command in serve worker; do
|
||||
if docker run --rm "${production_env[@]}" --entrypoint /app/gochat "$gochat_image" "$command" >"$tmp/direct-$command.log" 2>&1; then
|
||||
echo "direct GoChat $command accepted duplicate sslmode" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'production database DSN must contain exactly one sslmode' "$tmp/direct-$command.log" >/dev/null
|
||||
done
|
||||
if docker run --rm "${production_env[@]}" --entrypoint /app/migrate "$gochat_image" version >"$tmp/direct-migrate.log" 2>&1; then
|
||||
echo 'direct migrate accepted duplicate sslmode' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'production database DSN must contain exactly one sslmode' "$tmp/direct-migrate.log" >/dev/null
|
||||
for client in backup restore; do
|
||||
script=/app/scripts/db_${client}.sh
|
||||
args=()
|
||||
[[ $client == backup ]] || args=(/tmp/missing.tar.enc)
|
||||
if docker run --rm --user 0:0 -e GOCHAT_DATABASE_DSN="$bad_dsn" --entrypoint "$script" "$gochat_image" "${args[@]}" >"$tmp/direct-$client.log" 2>&1; then
|
||||
echo "direct $client accepted duplicate sslmode" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'database DSN must contain exactly one sslmode' "$tmp/direct-$client.log" >/dev/null
|
||||
done
|
||||
|
||||
test "$(docker run --rm --entrypoint id "$gochat_image" -u)" -ne 0
|
||||
test "$(docker run --rm --entrypoint id "$exporter_image" -u)" -ne 0
|
||||
|
||||
dsn='postgres://postgres:test-password@tls-db:5432/postgres?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
||||
if docker run --rm --group-add "$tls_gid" -e GOCHAT_DATABASE_DSN="$dsn" \
|
||||
-v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
|
||||
-v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
|
||||
-v "$tmp/denied/external-db-client.key:/run/secrets/external-db-client.key:ro" \
|
||||
--entrypoint /usr/local/bin/database-client-entrypoint "$gochat_image" /bin/true >"$tmp/permissions.log" 2>&1; then
|
||||
echo 'non-root GoChat client read a root-owned 0600 private key' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'database TLS file is missing or unreadable' "$tmp/permissions.log" >/dev/null
|
||||
|
||||
docker network create "$network" >/dev/null
|
||||
docker run -d --name "$database" --network "$network" --network-alias tls-db \
|
||||
-e POSTGRES_PASSWORD=test-password \
|
||||
-v "$tmp/server/server.crt:/certs/server.crt:ro" \
|
||||
-v "$tmp/server/server.key:/certs/server.key:ro" \
|
||||
"$postgres_image" -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key >/dev/null
|
||||
for _ in $(seq 1 30); do
|
||||
docker exec "$database" pg_isready -U postgres >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$database" pg_isready -U postgres >/dev/null
|
||||
|
||||
psql_result=$(docker run --rm --network "$network" --group-add "$tls_gid" -e GOCHAT_DATABASE_DSN="$dsn" \
|
||||
-v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
|
||||
-v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.key:/run/secrets/external-db-client.key:ro" \
|
||||
--entrypoint /usr/local/bin/database-client-entrypoint "$gochat_image" psql "$dsn" -Atqc 'SELECT 1')
|
||||
test "$psql_result" = 1
|
||||
|
||||
docker run -d --name "$exporter" --network "$network" --group-add "$tls_gid" -e DATA_SOURCE_NAME="$dsn" \
|
||||
-v "$gate:/usr/local/bin/database-client-entrypoint:ro" \
|
||||
-v "$tmp/client/external-db-ca.crt:/run/secrets/external-db-ca.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.crt:/run/secrets/external-db-client.crt:ro" \
|
||||
-v "$tmp/client/external-db-client.key:/run/secrets/external-db-client.key:ro" \
|
||||
--entrypoint /usr/local/bin/database-client-entrypoint "$exporter_image" /bin/postgres_exporter >/dev/null
|
||||
for _ in $(seq 1 20); do
|
||||
if docker run --rm --network "$network" --entrypoint sh "$gochat_image" -c \
|
||||
'curl -fsS http://gochat-db-gate-exporter-'"$$"':9187/metrics' >"$tmp/metrics" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
grep -E '^pg_up 1$' "$tmp/metrics" >/dev/null
|
||||
|
||||
echo 'database client entrypoint tests passed'
|
||||
@@ -0,0 +1,15 @@
|
||||
empty host|postgres://gochat:database-secret@/gochat?sslmode=verify-full
|
||||
built-in Compose host|postgres://gochat:database-secret@postgres:5432/gochat?sslmode=verify-full
|
||||
localhost|postgres://gochat:database-secret@localhost:5432/gochat?sslmode=verify-full
|
||||
IPv4 loopback start|postgres://gochat:database-secret@127.0.0.1:5432/gochat?sslmode=verify-full
|
||||
IPv4 loopback end|postgres://gochat:database-secret@127.255.255.255:5432/gochat?sslmode=verify-full
|
||||
compressed IPv6 loopback|postgres://gochat:database-secret@[::1]:5432/gochat?sslmode=verify-full
|
||||
partially compressed IPv6 loopback|postgres://gochat:database-secret@[0::1]:5432/gochat?sslmode=verify-full
|
||||
expanded IPv6 loopback|postgres://gochat:database-secret@[0:0:0:0:0:0:0:01]:5432/gochat?sslmode=verify-full
|
||||
IPv4-embedded IPv6 loopback|postgres://gochat:database-secret@[::0.0.0.1]:5432/gochat?sslmode=verify-full
|
||||
zoned IPv6 loopback|postgres://gochat:database-secret@[::1%25lo]:5432/gochat?sslmode=verify-full
|
||||
IPv4-mapped dotted loopback|postgres://gochat:database-secret@[::ffff:127.0.0.1]:5432/gochat?sslmode=verify-full
|
||||
IPv4-mapped dotted loopback end|postgres://gochat:database-secret@[::ffff:127.255.255.255]:5432/gochat?sslmode=verify-full
|
||||
IPv4-mapped hexadecimal loopback|postgres://gochat:database-secret@[::ffff:7f00:1]:5432/gochat?sslmode=verify-full
|
||||
IPv4-mapped partially expanded loopback|postgres://gochat:database-secret@[0:0::ffff:7f01:203]:5432/gochat?sslmode=verify-full
|
||||
IPv4-mapped expanded loopback|postgres://gochat:database-secret@[0:0:0:0:0:ffff:7fff:ffff]:5432/gochat?sslmode=verify-full
|
||||
@@ -1,5 +1,7 @@
|
||||
services:
|
||||
postgres:
|
||||
image: ${POSTGRES_IMAGE_REF:-pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b}
|
||||
restart: always
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
@@ -8,10 +10,36 @@ services:
|
||||
install -o postgres -g postgres -m 644 /run/tls/postgres.crt /var/lib/postgresql/server.crt
|
||||
exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key
|
||||
command: []
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-gochat_production}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-gochat}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD for the smoke database}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
|
||||
networks:
|
||||
default:
|
||||
aliases: [db.smoke.test]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
logging:
|
||||
driver: fluentd
|
||||
options:
|
||||
fluentd-address: 127.0.0.1:24224
|
||||
fluentd-async: "true"
|
||||
fluentd-buffer-limit: "65536"
|
||||
tag: gochat.{{.Name}}
|
||||
|
||||
redis:
|
||||
image: ${REDIS_IMAGE_REF:-redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf}
|
||||
restart: always
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
@@ -20,7 +48,10 @@ services:
|
||||
install -o redis -g redis -m 644 /run/tls/redis.crt /data/redis.crt
|
||||
exec /usr/bin/setpriv --reuid redis --regid redis --clear-groups redis-server --port 0 --tls-port 6379 --tls-cert-file /data/redis.crt --tls-key-file /data/redis.key --tls-ca-cert-file /run/tls/ca.crt --tls-auth-clients no --requirepass "$${REDIS_PASSWORD}" --appendonly yes
|
||||
command: []
|
||||
environment:
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD for smoke Redis}
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
- ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
|
||||
healthcheck:
|
||||
test:
|
||||
@@ -28,21 +59,72 @@ services:
|
||||
"CMD-SHELL",
|
||||
"redis-cli --tls --cacert /run/tls/ca.crt -h redis -a '$${REDIS_PASSWORD}' ping",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
logging:
|
||||
driver: fluentd
|
||||
options:
|
||||
fluentd-address: 127.0.0.1:24224
|
||||
fluentd-async: "true"
|
||||
fluentd-buffer-limit: "65536"
|
||||
tag: gochat.{{.Name}}
|
||||
|
||||
gochat:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
SSL_CERT_FILE: /run/tls/ca.crt
|
||||
volumes:
|
||||
- ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
|
||||
|
||||
worker:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
SSL_CERT_FILE: /run/tls/ca.crt
|
||||
volumes:
|
||||
- ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
|
||||
|
||||
migrate:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
SSL_CERT_FILE: /run/tls/ca.crt
|
||||
volumes:
|
||||
- ${GOCHAT_TLS_DIR:?set GOCHAT_TLS_DIR}:/run/tls:ro
|
||||
|
||||
backup:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
restore:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
postgres-exporter:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
redis-exporter:
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
@@ -1,13 +1,48 @@
|
||||
name: gochat-production
|
||||
|
||||
x-gochat-image: &gochat-image ${GOCHAT_IMAGE_REF:?set GOCHAT_IMAGE_REF to an immutable image digest}
|
||||
x-postgres-tls-ca: &postgres-tls-ca
|
||||
type: bind
|
||||
source: ${GOCHAT_DATABASE_TLS_CA_FILE:-/dev/null}
|
||||
target: /run/secrets/external-db-ca.crt
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
x-postgres-tls-client-cert: &postgres-tls-client-cert
|
||||
type: bind
|
||||
source: ${GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE:-/dev/null}
|
||||
target: /run/secrets/external-db-client.crt
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
x-postgres-tls-client-key: &postgres-tls-client-key
|
||||
type: bind
|
||||
source: ${GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE:-/dev/null}
|
||||
target: /run/secrets/external-db-client.key
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
x-database-client-entrypoint: &database-client-entrypoint
|
||||
type: bind
|
||||
source: ./database_client_entrypoint.sh
|
||||
target: /usr/local/bin/database-client-entrypoint
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
x-gochat-logging: &gochat-logging
|
||||
driver: fluentd
|
||||
options:
|
||||
fluentd-address: 127.0.0.1:24224
|
||||
fluentd-async: "true"
|
||||
fluentd-buffer-limit: "65536"
|
||||
tag: gochat.{{.Name}}
|
||||
x-gochat-environment: &gochat-environment
|
||||
GOCHAT_ENV: production
|
||||
GOCHAT_SERVER_HOST: 0.0.0.0
|
||||
GOCHAT_SERVER_PORT: 3000
|
||||
GOCHAT_SERVER_MODE: release
|
||||
GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: ${GOCHAT_SERVER_CORS_ALLOWED_ORIGINS:?set production CORS origins}
|
||||
GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
|
||||
GOCHAT_DATABASE_DSN: &gochat-database-dsn ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
|
||||
GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
|
||||
GOCHAT_DATABASE_MIGRATIONS_PATH: /app/migrations
|
||||
GOCHAT_REDIS_DSN: ${GOCHAT_REDIS_DSN:?set an external Redis rediss:// DSN}
|
||||
@@ -25,43 +60,6 @@ x-gochat-environment: &gochat-environment
|
||||
GOCHAT_STORAGE_LOCAL_PATH: /app/storage/uploads
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: ${POSTGRES_IMAGE_REF:-pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b}
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-gochat_production}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-gochat}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
|
||||
redis:
|
||||
image: ${REDIS_IMAGE_REF:-redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf}
|
||||
restart: always
|
||||
command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?set REDIS_PASSWORD}", "--appendonly", "yes", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
environment:
|
||||
REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli -a '$${REDIS_PASSWORD}' ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
|
||||
meilisearch:
|
||||
image: ${MEILI_IMAGE_REF:-getmeili/meilisearch:v1.13@sha256:bed3fb650e62da53145777204891159242f6ea4ce69e215b36223af4aa64a0ae}
|
||||
restart: always
|
||||
@@ -76,16 +74,16 @@ services:
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
logging: *gochat-logging
|
||||
|
||||
gochat:
|
||||
image: *gochat-image
|
||||
entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/gochat"]
|
||||
command: ["serve"]
|
||||
group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
|
||||
restart: always
|
||||
stop_grace_period: ${GOCHAT_STOP_GRACE_PERIOD:-35s}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
environment: *gochat-environment
|
||||
@@ -93,6 +91,10 @@ services:
|
||||
- "127.0.0.1:${GOCHAT_PORT:-3000}:3000"
|
||||
volumes:
|
||||
- gochat_storage:/app/storage
|
||||
- *database-client-entrypoint
|
||||
- *postgres-tls-ca
|
||||
- *postgres-tls-client-cert
|
||||
- *postgres-tls-client-key
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "3", "-O", "/dev/null", "http://127.0.0.1:3000/ready"]
|
||||
interval: 10s
|
||||
@@ -107,18 +109,17 @@ services:
|
||||
reservations:
|
||||
memory: 256M
|
||||
cpus: "0.5"
|
||||
logging: *gochat-logging
|
||||
|
||||
worker:
|
||||
image: *gochat-image
|
||||
entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/gochat"]
|
||||
group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
|
||||
restart: always
|
||||
stop_grace_period: ${GOCHAT_STOP_GRACE_PERIOD:-35s}
|
||||
depends_on:
|
||||
gochat:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
command: ["worker"]
|
||||
@@ -129,42 +130,51 @@ services:
|
||||
GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
|
||||
volumes:
|
||||
- gochat_storage:/app/storage
|
||||
- *database-client-entrypoint
|
||||
- *postgres-tls-ca
|
||||
- *postgres-tls-client-cert
|
||||
- *postgres-tls-client-key
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 512M
|
||||
cpus: "1.0"
|
||||
logging: *gochat-logging
|
||||
|
||||
migrate:
|
||||
image: *gochat-image
|
||||
profiles: ["ops"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/app/migrate"]
|
||||
entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/migrate"]
|
||||
group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
|
||||
command: ["up"]
|
||||
restart: "no"
|
||||
environment:
|
||||
<<: *gochat-environment
|
||||
GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
|
||||
PGOPTIONS: -c lock_timeout=5000 -c statement_timeout=900000
|
||||
volumes:
|
||||
- *database-client-entrypoint
|
||||
- *postgres-tls-ca
|
||||
- *postgres-tls-client-cert
|
||||
- *postgres-tls-client-key
|
||||
logging: *gochat-logging
|
||||
|
||||
backup:
|
||||
image: *gochat-image
|
||||
user: "0:0"
|
||||
profiles: ["ops"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/app/scripts/db_backup.sh"]
|
||||
entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/scripts/db_backup.sh"]
|
||||
group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
|
||||
restart: "no"
|
||||
environment:
|
||||
GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
|
||||
GOCHAT_DATABASE_DSN: *gochat-database-dsn
|
||||
GOCHAT_STORAGE_PATH: /source/storage/uploads
|
||||
GOCHAT_CONNECTOR_BACKUP_FILE: /source/connector/${GOCHAT_CONNECTOR_BACKUP_NAME:-latest.db}
|
||||
GOCHAT_BACKUP_DIR: /backup/local
|
||||
GOCHAT_BACKUP_OFFSITE_DIR: /backup/offsite
|
||||
GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
|
||||
GOCHAT_BACKUP_RETENTION_DAYS: ${GOCHAT_BACKUP_RETENTION_DAYS:-30}
|
||||
GOCHAT_BACKUP_METRICS_FILE: /metrics/gochat_backup.prom
|
||||
GOCHAT_VERSION: ${GOCHAT_IMAGE_REF}
|
||||
volumes:
|
||||
- gochat_storage:/source/storage:ro
|
||||
@@ -176,19 +186,23 @@ services:
|
||||
bind:
|
||||
create_host_path: false
|
||||
- ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
|
||||
- backup_metrics:/metrics
|
||||
- *database-client-entrypoint
|
||||
- *postgres-tls-ca
|
||||
- *postgres-tls-client-cert
|
||||
- *postgres-tls-client-key
|
||||
logging: *gochat-logging
|
||||
|
||||
restore:
|
||||
image: *gochat-image
|
||||
user: "0:0"
|
||||
profiles: ["ops"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/app/scripts/db_restore.sh"]
|
||||
entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/scripts/db_restore.sh"]
|
||||
group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
|
||||
command: ["/backup/offsite/${GOCHAT_RESTORE_BUNDLE:-missing.tar.enc}"]
|
||||
restart: "no"
|
||||
environment:
|
||||
GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
|
||||
GOCHAT_DATABASE_DSN: *gochat-database-dsn
|
||||
GOCHAT_STORAGE_PATH: /restore/storage/uploads
|
||||
GOCHAT_CONNECTOR_DB_PATH: /restore/connector/connector.db
|
||||
GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
|
||||
@@ -202,6 +216,11 @@ services:
|
||||
bind:
|
||||
create_host_path: false
|
||||
- ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
|
||||
- *database-client-entrypoint
|
||||
- *postgres-tls-ca
|
||||
- *postgres-tls-client-cert
|
||||
- *postgres-tls-client-key
|
||||
logging: *gochat-logging
|
||||
|
||||
shangwutong:
|
||||
image: ${SHANGWUTONG_IMAGE_REF:?set SHANGWUTONG_IMAGE_REF to an immutable image digest}
|
||||
@@ -233,11 +252,114 @@ services:
|
||||
reservations:
|
||||
memory: 128M
|
||||
cpus: "0.25"
|
||||
logging: *gochat-logging
|
||||
|
||||
fluentd:
|
||||
image: fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440
|
||||
restart: always
|
||||
ports:
|
||||
- "127.0.0.1:24224:24224"
|
||||
volumes:
|
||||
- ../fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro
|
||||
- fluentd_logs:/fluentd/log
|
||||
- fluentd_buffer:/fluentd/buffer
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:v0.17.1@sha256:38606faa38c54787525fb0ff2fd6b41b4cfb75d455c1df294927c5f611699b17
|
||||
restart: always
|
||||
entrypoint: ["/usr/local/bin/database-client-entrypoint", "/bin/postgres_exporter"]
|
||||
group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
|
||||
command: ["--config.file=/dev/null", "--extend.query-path=/etc/postgres-exporter/queries.yml"]
|
||||
environment:
|
||||
DATA_SOURCE_NAME: *gochat-database-dsn
|
||||
volumes:
|
||||
- ../prometheus/postgres_queries.yml:/etc/postgres-exporter/queries.yml:ro
|
||||
- *database-client-entrypoint
|
||||
- *postgres-tls-ca
|
||||
- *postgres-tls-client-cert
|
||||
- *postgres-tls-client-key
|
||||
logging: *gochat-logging
|
||||
|
||||
redis-exporter:
|
||||
image: oliver006/redis_exporter:v1.72.1@sha256:f90cae1e7ecc6ac223d04bdb0c95e084918baced9f81f08c3d01c2f11bff72bf
|
||||
restart: always
|
||||
environment:
|
||||
REDIS_ADDR: ${GOCHAT_REDIS_DSN:?set an external Redis rediss:// DSN}
|
||||
logging: *gochat-logging
|
||||
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d
|
||||
restart: always
|
||||
command: ["--config.file=/etc/blackbox_exporter/config.yml"]
|
||||
volumes:
|
||||
- ../prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
|
||||
logging: *gochat-logging
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.9.1@sha256:d00a542e409ee618a4edc67da14dd48c5da66726bbd5537ab2af9c1dfc442c8a
|
||||
restart: always
|
||||
command: ["--collector.disable-defaults", "--collector.textfile", "--collector.textfile.directory=/var/lib/node_exporter/textfile_collector"]
|
||||
volumes:
|
||||
- backup_metrics:/var/lib/node_exporter/textfile_collector:ro
|
||||
logging: *gochat-logging
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:v0.52.1@sha256:f40e65878e25c2e78ea037f73a449527a0fb994e303dc3e34cb6b187b4b91435
|
||||
restart: always
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg:/dev/kmsg
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker:/var/lib/docker:ro
|
||||
- /dev/disk:/dev/disk:ro
|
||||
logging: *gochat-logging
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba
|
||||
restart: always
|
||||
command: ["--config.file=/etc/alertmanager/alertmanager.yml", "--storage.path=/alertmanager"]
|
||||
ports:
|
||||
- "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
|
||||
volumes:
|
||||
- ../prometheus/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- type: bind
|
||||
source: ${ALERTMANAGER_WEBHOOK_URL_FILE:-../../.secrets/alertmanager-webhook-url}
|
||||
target: /run/secrets/alertmanager-webhook-url
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
- alertmanager_data:/alertmanager
|
||||
logging: *gochat-logging
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996
|
||||
restart: always
|
||||
depends_on:
|
||||
- alertmanager
|
||||
- blackbox-exporter
|
||||
- cadvisor
|
||||
- node-exporter
|
||||
- postgres-exporter
|
||||
- redis-exporter
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}"]
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
|
||||
volumes:
|
||||
- ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ../../backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
logging: *gochat-logging
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
meili_data:
|
||||
gochat_storage:
|
||||
shangwutong_data:
|
||||
shangwutong_backups:
|
||||
backup_metrics:
|
||||
prometheus_data:
|
||||
alertmanager_data:
|
||||
fluentd_logs:
|
||||
fluentd_buffer:
|
||||
|
||||
Executable
+162
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-observability-test' \
|
||||
-keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
|
||||
cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
|
||||
chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
|
||||
chmod 0640 "$tmp/external-db-client.key"
|
||||
|
||||
export GOCHAT_IMAGE_REF='gochat.example.test/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
|
||||
export SHANGWUTONG_IMAGE_REF='gochat.example.test/shangwutong@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
||||
export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
|
||||
export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
|
||||
export GOCHAT_ENCRYPTION_AES_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
|
||||
export POSTGRES_PASSWORD=ci-postgres-secret
|
||||
export REDIS_PASSWORD=ci-redis-secret
|
||||
export MEILI_MASTER_KEY=ci-meili-secret-16
|
||||
export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
||||
export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
|
||||
export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
|
||||
mkdir "$GOCHAT_BACKUP_OFFSITE_DIR"
|
||||
|
||||
baseline_dsn='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
|
||||
baseline_compose_json=$tmp/baseline-compose.json
|
||||
mtls_compose_json=$tmp/mtls-compose.json
|
||||
smoke_compose_json=$tmp/smoke-compose.json
|
||||
export GOCHAT_DATABASE_DSN=$baseline_dsn
|
||||
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$baseline_compose_json"
|
||||
export GOCHAT_TLS_DIR=$tmp
|
||||
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" -f "$root/deploy/docker/docker-compose.prod-smoke.yml" --profile '*' config --format json >"$smoke_compose_json"
|
||||
export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
|
||||
export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
|
||||
export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
|
||||
export GOCHAT_DATABASE_TLS_GID
|
||||
GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
|
||||
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
||||
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$mtls_compose_json"
|
||||
python3 - "$baseline_compose_json" "$mtls_compose_json" "$smoke_compose_json" "$baseline_dsn" "$GOCHAT_DATABASE_DSN" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
baseline = json.load(open(sys.argv[1], encoding="utf-8"))["services"]
|
||||
mtls = json.load(open(sys.argv[2], encoding="utf-8"))["services"]
|
||||
smoke = json.load(open(sys.argv[3], encoding="utf-8"))["services"]
|
||||
baseline_dsn = sys.argv[4]
|
||||
mtls_dsn = sys.argv[5]
|
||||
services = baseline
|
||||
required = {
|
||||
"alertmanager", "blackbox-exporter", "cadvisor", "fluentd", "node-exporter",
|
||||
"postgres-exporter", "prometheus", "redis-exporter",
|
||||
}
|
||||
assert required <= services.keys(), required - services.keys()
|
||||
assert not {"postgres", "redis"} & services.keys()
|
||||
for service in ("gochat", "worker", "migrate", "backup", "shangwutong"):
|
||||
assert services[service]["logging"]["driver"] == "fluentd", service
|
||||
for service in ("postgres", "redis"):
|
||||
assert smoke[service]["logging"]["driver"] == "fluentd", service
|
||||
for service, dependencies in {
|
||||
"gochat": {"postgres", "redis"},
|
||||
"worker": {"postgres", "redis"},
|
||||
"migrate": {"postgres"},
|
||||
"backup": {"postgres"},
|
||||
"restore": {"postgres"},
|
||||
"postgres-exporter": {"postgres"},
|
||||
"redis-exporter": {"redis"},
|
||||
}.items():
|
||||
assert not dependencies & baseline[service].get("depends_on", {}).keys(), service
|
||||
assert dependencies <= smoke[service]["depends_on"].keys(), service
|
||||
assert baseline["redis-exporter"]["environment"]["REDIS_ADDR"] == os.environ["GOCHAT_REDIS_DSN"]
|
||||
assert any(volume["target"] == "/etc/prometheus/rules/gochat.yml" for volume in services["prometheus"]["volumes"])
|
||||
assert any(volume["target"] == "/fluentd/etc/fluent.conf" for volume in services["fluentd"]["volumes"])
|
||||
assert services["backup"]["environment"]["GOCHAT_BACKUP_METRICS_FILE"] == "/metrics/gochat_backup.prom"
|
||||
|
||||
def database_dsns(config):
|
||||
return {
|
||||
"gochat": config["gochat"]["environment"]["GOCHAT_DATABASE_DSN"],
|
||||
"worker": config["worker"]["environment"]["GOCHAT_DATABASE_DSN"],
|
||||
"migrate": config["migrate"]["environment"]["GOCHAT_DATABASE_DSN"],
|
||||
"backup": config["backup"]["environment"]["GOCHAT_DATABASE_DSN"],
|
||||
"restore": config["restore"]["environment"]["GOCHAT_DATABASE_DSN"],
|
||||
"postgres-exporter": config["postgres-exporter"]["environment"]["DATA_SOURCE_NAME"],
|
||||
}
|
||||
|
||||
client_commands = {
|
||||
"gochat": "/app/gochat",
|
||||
"worker": "/app/gochat",
|
||||
"migrate": "/app/migrate",
|
||||
"backup": "/app/scripts/db_backup.sh",
|
||||
"restore": "/app/scripts/db_restore.sh",
|
||||
"postgres-exporter": "/bin/postgres_exporter",
|
||||
}
|
||||
for config, expected_gid in ((baseline, "65534"), (mtls, os.environ["GOCHAT_DATABASE_TLS_GID"])):
|
||||
for service, command in client_commands.items():
|
||||
assert config[service]["entrypoint"] == ["/usr/local/bin/database-client-entrypoint", command], service
|
||||
assert expected_gid in map(str, config[service]["group_add"]), service
|
||||
gate = [volume for volume in config[service]["volumes"] if volume["target"] == "/usr/local/bin/database-client-entrypoint"]
|
||||
assert len(gate) == 1 and gate[0]["read_only"] is True, service
|
||||
|
||||
baseline_dsns = database_dsns(baseline)
|
||||
assert set(baseline_dsns.values()) == {baseline_dsn}, baseline_dsns
|
||||
mtls_dsns = database_dsns(mtls)
|
||||
assert set(mtls_dsns.values()) == {mtls_dsn}, mtls_dsns
|
||||
|
||||
pgoptions = "-c lock_timeout=5000 -c statement_timeout=900000"
|
||||
assert baseline["migrate"]["environment"]["PGOPTIONS"] == pgoptions
|
||||
assert mtls["migrate"]["environment"]["PGOPTIONS"] == pgoptions
|
||||
|
||||
tls_paths = {
|
||||
"sslrootcert": "/run/secrets/external-db-ca.crt",
|
||||
"sslcert": "/run/secrets/external-db-client.crt",
|
||||
"sslkey": "/run/secrets/external-db-client.key",
|
||||
}
|
||||
tls_sources = {
|
||||
tls_paths["sslrootcert"]: os.environ["GOCHAT_DATABASE_TLS_CA_FILE"],
|
||||
tls_paths["sslcert"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE"],
|
||||
tls_paths["sslkey"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"],
|
||||
}
|
||||
query = parse_qs(urlsplit(mtls_dsn).query)
|
||||
assert query["sslmode"] == ["verify-full"]
|
||||
for parameter, target in tls_paths.items():
|
||||
assert query[parameter] == [target], (parameter, query)
|
||||
|
||||
for config, expected_sources in ((baseline, {target: "/dev/null" for target in tls_paths.values()}), (mtls, tls_sources)):
|
||||
for service in database_dsns(config):
|
||||
mounts = {volume["target"]: volume for volume in config[service]["volumes"] if volume["target"] in tls_paths.values()}
|
||||
assert mounts.keys() == expected_sources.keys(), (service, mounts)
|
||||
for target, source in expected_sources.items():
|
||||
assert mounts[target]["source"] == source, (service, target, mounts[target])
|
||||
assert mounts[target]["read_only"] is True, (service, target, mounts[target])
|
||||
PY
|
||||
|
||||
prometheus_image='prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996'
|
||||
alertmanager_image='prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba'
|
||||
blackbox_image='prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d'
|
||||
fluentd_image='fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440'
|
||||
|
||||
docker run --rm --entrypoint promtool \
|
||||
-v "$root/deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \
|
||||
-v "$root/backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro" \
|
||||
"$prometheus_image" check config /etc/prometheus/prometheus.yml
|
||||
docker run --rm --entrypoint promtool \
|
||||
-v "$root/backend/configs:/configs:ro" \
|
||||
"$prometheus_image" test rules /configs/prometheus_alerts_test.yml
|
||||
docker run --rm --entrypoint amtool \
|
||||
-v "$root/deploy/prometheus:/etc/alertmanager:ro" \
|
||||
-v "$tmp/alertmanager-webhook-url:/run/secrets/alertmanager-webhook-url:ro" \
|
||||
"$alertmanager_image" check-config /etc/alertmanager/alertmanager.yml
|
||||
docker run --rm \
|
||||
-v "$root/deploy/prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro" \
|
||||
"$blackbox_image" --config.file=/etc/blackbox_exporter/config.yml --config.check
|
||||
docker run --rm \
|
||||
-v "$root/deploy/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro" \
|
||||
"$fluentd_image" fluentd --dry-run -c /fluentd/etc/fluent.conf
|
||||
"$root/backend/scripts/db_backup_test.sh"
|
||||
|
||||
echo 'observability configuration tests passed'
|
||||
@@ -12,7 +12,7 @@ if (($#)); then
|
||||
compose_args=(--env-file "$env_file" "${compose_args[@]}")
|
||||
fi
|
||||
|
||||
required=(GOCHAT_IMAGE_REF SHANGWUTONG_IMAGE_REF GOCHAT_SERVER_CORS_ALLOWED_ORIGINS POSTGRES_PASSWORD REDIS_PASSWORD MEILI_MASTER_KEY GOCHAT_JWT_SECRET GOCHAT_BACKUP_OFFSITE_DIR GOCHAT_BACKUP_OFFSITE_SOURCE GOCHAT_BACKUP_OFFSITE_FSTYPE)
|
||||
required=(GOCHAT_IMAGE_REF SHANGWUTONG_IMAGE_REF GOCHAT_SERVER_CORS_ALLOWED_ORIGINS GOCHAT_DATABASE_DSN GOCHAT_REDIS_DSN MEILI_MASTER_KEY GOCHAT_JWT_SECRET GOCHAT_BACKUP_OFFSITE_DIR GOCHAT_BACKUP_OFFSITE_SOURCE GOCHAT_BACKUP_OFFSITE_FSTYPE)
|
||||
for name in "${required[@]}"; do
|
||||
value=${!name:-}
|
||||
if [[ -z $value || ${value^^} == *CHANGE_ME* ]]; then
|
||||
@@ -67,10 +67,45 @@ if ((${#MEILI_MASTER_KEY} < 16)); then
|
||||
echo "MEILI_MASTER_KEY must be at least 16 bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n ${GOCHAT_DATABASE_DSN:-} && ! $GOCHAT_DATABASE_DSN =~ (^|[?&])sslmode=(require|verify-ca|verify-full)(&|$) ]]; then
|
||||
echo "GOCHAT_DATABASE_DSN must explicitly require TLS for an external database" >&2
|
||||
alertmanager_webhook_file=${ALERTMANAGER_WEBHOOK_URL_FILE:-../../.secrets/alertmanager-webhook-url}
|
||||
if [[ $alertmanager_webhook_file != /* ]]; then
|
||||
alertmanager_webhook_file=$script_dir/$alertmanager_webhook_file
|
||||
fi
|
||||
if [[ ! -r $alertmanager_webhook_file ]] || ! IFS= read -r alertmanager_webhook_url <"$alertmanager_webhook_file" || [[ ! $alertmanager_webhook_url =~ ^https://[^[:space:]]+$ ]]; then
|
||||
echo "ALERTMANAGER_WEBHOOK_URL_FILE must contain one HTTPS URL" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n ${GOCHAT_DATABASE_DSN:-} ]]; then
|
||||
"$script_dir/database_client_entrypoint.sh" --preflight
|
||||
if [[ $GOCHAT_DATABASE_DSN == *sslrootcert=* ]]; then
|
||||
tls_gid=${GOCHAT_DATABASE_TLS_GID:-}
|
||||
if [[ ! $tls_gid =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "GOCHAT_DATABASE_TLS_GID must be set to a positive numeric group ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
for name in GOCHAT_DATABASE_TLS_CA_FILE GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE; do
|
||||
host_file=${!name:-}
|
||||
if [[ -z $host_file ]]; then
|
||||
echo "$name must be set when the database DSN uses certificate files" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ $host_file != /* ]]; then
|
||||
host_file=$script_dir/$host_file
|
||||
fi
|
||||
if [[ ! -f $host_file ]]; then
|
||||
echo "$name must point to an existing regular file" >&2
|
||||
exit 1
|
||||
fi
|
||||
file_gid=$(stat -c %g -- "$host_file")
|
||||
file_mode=$(stat -c %a -- "$host_file")
|
||||
group_digit=${file_mode: -2:1}
|
||||
if [[ $file_gid != "$tls_gid" ]] || (((8#$group_digit & 4) == 0)); then
|
||||
echo "$name must be readable by GOCHAT_DATABASE_TLS_GID" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
images=$(docker compose "${compose_args[@]}" config --images)
|
||||
while IFS= read -r image; do
|
||||
|
||||
@@ -5,6 +5,12 @@ root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
mkdir -p "$tmp/bin" "$tmp/local" "$tmp/offsite"
|
||||
printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-preflight-test' \
|
||||
-keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
|
||||
cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
|
||||
chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
|
||||
chmod 0640 "$tmp/external-db-client.key"
|
||||
|
||||
cat > "$tmp/bin/findmnt" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
@@ -32,10 +38,13 @@ export PATH="$tmp/bin:$PATH"
|
||||
export GOCHAT_IMAGE_REF='gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
|
||||
export SHANGWUTONG_IMAGE_REF='gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
||||
export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.ci.rogeecn.com
|
||||
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
|
||||
export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
|
||||
export POSTGRES_PASSWORD=ci-postgres-secret
|
||||
export REDIS_PASSWORD=ci-redis-secret
|
||||
export MEILI_MASTER_KEY=ci-meili-secret-16
|
||||
export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
||||
export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
|
||||
export GOCHAT_BACKUP_DIR="$tmp/local"
|
||||
export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
|
||||
export GOCHAT_BACKUP_OFFSITE_SOURCE='backup.example.test:/gochat'
|
||||
@@ -86,11 +95,44 @@ MEILI_MASTER_KEY=too-short
|
||||
expect_failure 'a short Meilisearch key' 'must be at least 16 bytes'
|
||||
MEILI_MASTER_KEY=ci-meili-secret-16
|
||||
|
||||
printf '%s\n' 'http://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
|
||||
expect_failure 'a plaintext alert receiver' 'must contain one HTTPS URL'
|
||||
printf '%s\n' 'https://alerts.example.test/gochat' >"$ALERTMANAGER_WEBHOOK_URL_FILE"
|
||||
|
||||
TEST_MUTABLE_IMAGE=1
|
||||
export TEST_MUTABLE_IMAGE
|
||||
expect_failure 'a mutable production image' 'must be pinned to a sha256 digest'
|
||||
unset TEST_MUTABLE_IMAGE
|
||||
|
||||
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable'
|
||||
expect_failure 'an external database without verified TLS' 'sslmode must be verify-ca or verify-full'
|
||||
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=require'
|
||||
expect_failure 'an external database without certificate verification' 'sslmode must be verify-ca or verify-full'
|
||||
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=disable&sslmode=verify-full'
|
||||
expect_failure 'duplicate sslmode parameters with a disabling first value' 'must contain exactly one sslmode'
|
||||
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslmode=verify-full'
|
||||
expect_failure 'duplicate allowed sslmode parameters' 'must contain exactly one sslmode'
|
||||
while IFS='|' read -r name GOCHAT_DATABASE_DSN; do
|
||||
expect_failure "$name" 'must use an external PostgreSQL host'
|
||||
done < "$root/deploy/docker/database_host_rejection_cases.txt"
|
||||
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/tmp/ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
||||
expect_failure 'a non-fixed certificate path' 'sslrootcert must use /run/secrets/external-db-ca.crt'
|
||||
|
||||
GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
||||
export GOCHAT_DATABASE_TLS_GID
|
||||
GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
|
||||
unset GOCHAT_DATABASE_TLS_CA_FILE GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE
|
||||
expect_failure 'missing host TLS variables' 'GOCHAT_DATABASE_TLS_CA_FILE must be set'
|
||||
|
||||
export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/missing-ca.crt"
|
||||
export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
|
||||
export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
|
||||
expect_failure 'a missing host TLS file' 'GOCHAT_DATABASE_TLS_CA_FILE must point to an existing regular file'
|
||||
|
||||
GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
|
||||
chmod 0600 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
|
||||
expect_failure 'a private key without group read permission' 'GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE must be readable by GOCHAT_DATABASE_TLS_GID'
|
||||
chmod 0640 "$GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"
|
||||
run_preflight
|
||||
grep -F 'production preflight passed' "$tmp/output" >/dev/null
|
||||
echo 'preflight tests passed'
|
||||
|
||||
Reference in New Issue
Block a user