#!/usr/bin/env bash set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url" openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-observability-test' \ -keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1 cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt" chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt" chmod 0640 "$tmp/external-db-client.key" export GOCHAT_IMAGE_REF='gochat.example.test/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000' export SHANGWUTONG_IMAGE_REF='gochat.example.test/shangwutong@sha256:1111111111111111111111111111111111111111111111111111111111111111' export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0' export GOCHAT_ENCRYPTION_AES_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY= export POSTGRES_PASSWORD=ci-postgres-secret export REDIS_PASSWORD=ci-redis-secret export MEILI_MASTER_KEY=ci-meili-secret-16 export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite" export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url" mkdir "$GOCHAT_BACKUP_OFFSITE_DIR" baseline_dsn='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full' baseline_compose_json=$tmp/baseline-compose.json mtls_compose_json=$tmp/mtls-compose.json smoke_compose_json=$tmp/smoke-compose.json export GOCHAT_DATABASE_DSN=$baseline_dsn docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$baseline_compose_json" export GOCHAT_TLS_DIR=$tmp docker compose -f "$root/deploy/docker/docker-compose.prod.yml" -f "$root/deploy/docker/docker-compose.prod-smoke.yml" --profile '*' config --format json >"$smoke_compose_json" export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt" export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt" export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key" export GOCHAT_DATABASE_TLS_GID GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key") export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key' docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$mtls_compose_json" python3 - "$baseline_compose_json" "$mtls_compose_json" "$smoke_compose_json" "$baseline_dsn" "$GOCHAT_DATABASE_DSN" <<'PY' import json import os import sys from urllib.parse import parse_qs, urlsplit baseline = json.load(open(sys.argv[1], encoding="utf-8"))["services"] mtls = json.load(open(sys.argv[2], encoding="utf-8"))["services"] smoke = json.load(open(sys.argv[3], encoding="utf-8"))["services"] baseline_dsn = sys.argv[4] mtls_dsn = sys.argv[5] services = baseline required = { "alertmanager", "blackbox-exporter", "cadvisor", "fluentd", "node-exporter", "postgres-exporter", "prometheus", "redis-exporter", } assert required <= services.keys(), required - services.keys() assert not {"postgres", "redis"} & services.keys() for service in ("gochat", "worker", "migrate", "backup", "shangwutong"): assert services[service]["logging"]["driver"] == "fluentd", service for service in ("postgres", "redis"): assert smoke[service]["logging"]["driver"] == "fluentd", service for service, dependencies in { "gochat": {"postgres", "redis"}, "worker": {"postgres", "redis"}, "migrate": {"postgres"}, "backup": {"postgres"}, "restore": {"postgres"}, "postgres-exporter": {"postgres"}, "redis-exporter": {"redis"}, }.items(): assert not dependencies & baseline[service].get("depends_on", {}).keys(), service assert dependencies <= smoke[service]["depends_on"].keys(), service assert baseline["redis-exporter"]["environment"]["REDIS_ADDR"] == os.environ["GOCHAT_REDIS_DSN"] assert any(volume["target"] == "/etc/prometheus/rules/gochat.yml" for volume in services["prometheus"]["volumes"]) assert any(volume["target"] == "/fluentd/etc/fluent.conf" for volume in services["fluentd"]["volumes"]) assert services["backup"]["environment"]["GOCHAT_BACKUP_METRICS_FILE"] == "/metrics/gochat_backup.prom" def database_dsns(config): return { "gochat": config["gochat"]["environment"]["GOCHAT_DATABASE_DSN"], "worker": config["worker"]["environment"]["GOCHAT_DATABASE_DSN"], "migrate": config["migrate"]["environment"]["GOCHAT_DATABASE_DSN"], "backup": config["backup"]["environment"]["GOCHAT_DATABASE_DSN"], "restore": config["restore"]["environment"]["GOCHAT_DATABASE_DSN"], "postgres-exporter": config["postgres-exporter"]["environment"]["DATA_SOURCE_NAME"], } client_commands = { "gochat": "/app/gochat", "worker": "/app/gochat", "migrate": "/app/migrate", "backup": "/app/scripts/db_backup.sh", "restore": "/app/scripts/db_restore.sh", "postgres-exporter": "/bin/postgres_exporter", } for config, expected_gid in ((baseline, "65534"), (mtls, os.environ["GOCHAT_DATABASE_TLS_GID"])): for service, command in client_commands.items(): assert config[service]["entrypoint"] == ["/usr/local/bin/database-client-entrypoint", command], service assert expected_gid in map(str, config[service]["group_add"]), service gate = [volume for volume in config[service]["volumes"] if volume["target"] == "/usr/local/bin/database-client-entrypoint"] assert len(gate) == 1 and gate[0]["read_only"] is True, service baseline_dsns = database_dsns(baseline) assert set(baseline_dsns.values()) == {baseline_dsn}, baseline_dsns mtls_dsns = database_dsns(mtls) assert set(mtls_dsns.values()) == {mtls_dsn}, mtls_dsns pgoptions = "-c lock_timeout=5000 -c statement_timeout=900000" assert baseline["migrate"]["environment"]["PGOPTIONS"] == pgoptions assert mtls["migrate"]["environment"]["PGOPTIONS"] == pgoptions tls_paths = { "sslrootcert": "/run/secrets/external-db-ca.crt", "sslcert": "/run/secrets/external-db-client.crt", "sslkey": "/run/secrets/external-db-client.key", } tls_sources = { tls_paths["sslrootcert"]: os.environ["GOCHAT_DATABASE_TLS_CA_FILE"], tls_paths["sslcert"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE"], tls_paths["sslkey"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"], } query = parse_qs(urlsplit(mtls_dsn).query) assert query["sslmode"] == ["verify-full"] for parameter, target in tls_paths.items(): assert query[parameter] == [target], (parameter, query) for config, expected_sources in ((baseline, {target: "/dev/null" for target in tls_paths.values()}), (mtls, tls_sources)): for service in database_dsns(config): mounts = {volume["target"]: volume for volume in config[service]["volumes"] if volume["target"] in tls_paths.values()} assert mounts.keys() == expected_sources.keys(), (service, mounts) for target, source in expected_sources.items(): assert mounts[target]["source"] == source, (service, target, mounts[target]) assert mounts[target]["read_only"] is True, (service, target, mounts[target]) PY prometheus_image='prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996' alertmanager_image='prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba' blackbox_image='prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d' fluentd_image='fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440' docker run --rm --entrypoint promtool \ -v "$root/deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \ -v "$root/backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro" \ "$prometheus_image" check config /etc/prometheus/prometheus.yml docker run --rm --entrypoint promtool \ -v "$root/backend/configs:/configs:ro" \ "$prometheus_image" test rules /configs/prometheus_alerts_test.yml docker run --rm --entrypoint amtool \ -v "$root/deploy/prometheus:/etc/alertmanager:ro" \ -v "$tmp/alertmanager-webhook-url:/run/secrets/alertmanager-webhook-url:ro" \ "$alertmanager_image" check-config /etc/alertmanager/alertmanager.yml docker run --rm \ -v "$root/deploy/prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro" \ "$blackbox_image" --config.file=/etc/blackbox_exporter/config.yml --config.check docker run --rm \ -v "$root/deploy/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro" \ "$fluentd_image" fluentd --dry-run -c /fluentd/etc/fluent.conf "$root/backend/scripts/db_backup_test.sh" echo 'observability configuration tests passed'