gochat/deploy/docker/observability_test.sh

#!/usr/bin/env bash
set -euo pipefail

root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-observability-test' \
  -keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
chmod 0640 "$tmp/external-db-client.key"

export GOCHAT_IMAGE_REF='gochat.example.test/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
export SHANGWUTONG_IMAGE_REF='gochat.example.test/shangwutong@sha256:1111111111111111111111111111111111111111111111111111111111111111'
export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
export GOCHAT_ENCRYPTION_AES_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
export POSTGRES_PASSWORD=ci-postgres-secret
export REDIS_PASSWORD=ci-redis-secret
export MEILI_MASTER_KEY=ci-meili-secret-16
export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
mkdir "$GOCHAT_BACKUP_OFFSITE_DIR"

baseline_dsn='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
baseline_compose_json=$tmp/baseline-compose.json
mtls_compose_json=$tmp/mtls-compose.json
smoke_compose_json=$tmp/smoke-compose.json
export GOCHAT_DATABASE_DSN=$baseline_dsn
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$baseline_compose_json"
export GOCHAT_TLS_DIR=$tmp
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" -f "$root/deploy/docker/docker-compose.prod-smoke.yml" --profile '*' config --format json >"$smoke_compose_json"
export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
export GOCHAT_DATABASE_TLS_GID
GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$mtls_compose_json"
python3 - "$baseline_compose_json" "$mtls_compose_json" "$smoke_compose_json" "$baseline_dsn" "$GOCHAT_DATABASE_DSN" <<'PY'
import json
import os
import sys
from urllib.parse import parse_qs, urlsplit

baseline = json.load(open(sys.argv[1], encoding="utf-8"))["services"]
mtls = json.load(open(sys.argv[2], encoding="utf-8"))["services"]
smoke = json.load(open(sys.argv[3], encoding="utf-8"))["services"]
baseline_dsn = sys.argv[4]
mtls_dsn = sys.argv[5]
services = baseline
required = {
    "alertmanager", "blackbox-exporter", "cadvisor", "fluentd", "node-exporter",
    "postgres-exporter", "prometheus", "redis-exporter",
}
assert required <= services.keys(), required - services.keys()
assert not {"postgres", "redis"} & services.keys()
for service in ("gochat", "worker", "migrate", "backup", "shangwutong"):
    assert services[service]["logging"]["driver"] == "fluentd", service
for service in ("postgres", "redis"):
    assert smoke[service]["logging"]["driver"] == "fluentd", service
for service, dependencies in {
    "gochat": {"postgres", "redis"},
    "worker": {"postgres", "redis"},
    "migrate": {"postgres"},
    "backup": {"postgres"},
    "restore": {"postgres"},
    "postgres-exporter": {"postgres"},
    "redis-exporter": {"redis"},
}.items():
    assert not dependencies & baseline[service].get("depends_on", {}).keys(), service
    assert dependencies <= smoke[service]["depends_on"].keys(), service
assert baseline["redis-exporter"]["environment"]["REDIS_ADDR"] == os.environ["GOCHAT_REDIS_DSN"]
assert any(volume["target"] == "/etc/prometheus/rules/gochat.yml" for volume in services["prometheus"]["volumes"])
assert any(volume["target"] == "/fluentd/etc/fluent.conf" for volume in services["fluentd"]["volumes"])
assert services["backup"]["environment"]["GOCHAT_BACKUP_METRICS_FILE"] == "/metrics/gochat_backup.prom"

def database_dsns(config):
    return {
        "gochat": config["gochat"]["environment"]["GOCHAT_DATABASE_DSN"],
        "worker": config["worker"]["environment"]["GOCHAT_DATABASE_DSN"],
        "migrate": config["migrate"]["environment"]["GOCHAT_DATABASE_DSN"],
        "backup": config["backup"]["environment"]["GOCHAT_DATABASE_DSN"],
        "restore": config["restore"]["environment"]["GOCHAT_DATABASE_DSN"],
        "postgres-exporter": config["postgres-exporter"]["environment"]["DATA_SOURCE_NAME"],
    }

client_commands = {
    "gochat": "/app/gochat",
    "worker": "/app/gochat",
    "migrate": "/app/migrate",
    "backup": "/app/scripts/db_backup.sh",
    "restore": "/app/scripts/db_restore.sh",
    "postgres-exporter": "/bin/postgres_exporter",
}
for config, expected_gid in ((baseline, "65534"), (mtls, os.environ["GOCHAT_DATABASE_TLS_GID"])):
    for service, command in client_commands.items():
        assert config[service]["entrypoint"] == ["/usr/local/bin/database-client-entrypoint", command], service
        assert expected_gid in map(str, config[service]["group_add"]), service
        gate = [volume for volume in config[service]["volumes"] if volume["target"] == "/usr/local/bin/database-client-entrypoint"]
        assert len(gate) == 1 and gate[0]["read_only"] is True, service

baseline_dsns = database_dsns(baseline)
assert set(baseline_dsns.values()) == {baseline_dsn}, baseline_dsns
mtls_dsns = database_dsns(mtls)
assert set(mtls_dsns.values()) == {mtls_dsn}, mtls_dsns

pgoptions = "-c lock_timeout=5000 -c statement_timeout=900000"
assert baseline["migrate"]["environment"]["PGOPTIONS"] == pgoptions
assert mtls["migrate"]["environment"]["PGOPTIONS"] == pgoptions

tls_paths = {
    "sslrootcert": "/run/secrets/external-db-ca.crt",
    "sslcert": "/run/secrets/external-db-client.crt",
    "sslkey": "/run/secrets/external-db-client.key",
}
tls_sources = {
    tls_paths["sslrootcert"]: os.environ["GOCHAT_DATABASE_TLS_CA_FILE"],
    tls_paths["sslcert"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE"],
    tls_paths["sslkey"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"],
}
query = parse_qs(urlsplit(mtls_dsn).query)
assert query["sslmode"] == ["verify-full"]
for parameter, target in tls_paths.items():
    assert query[parameter] == [target], (parameter, query)

for config, expected_sources in ((baseline, {target: "/dev/null" for target in tls_paths.values()}), (mtls, tls_sources)):
    for service in database_dsns(config):
        mounts = {volume["target"]: volume for volume in config[service]["volumes"] if volume["target"] in tls_paths.values()}
        assert mounts.keys() == expected_sources.keys(), (service, mounts)
        for target, source in expected_sources.items():
            assert mounts[target]["source"] == source, (service, target, mounts[target])
            assert mounts[target]["read_only"] is True, (service, target, mounts[target])
PY

prometheus_image='prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996'
alertmanager_image='prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba'
blackbox_image='prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d'
fluentd_image='fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440'

docker run --rm --entrypoint promtool \
  -v "$root/deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \
  -v "$root/backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro" \
  "$prometheus_image" check config /etc/prometheus/prometheus.yml
docker run --rm --entrypoint promtool \
  -v "$root/backend/configs:/configs:ro" \
  "$prometheus_image" test rules /configs/prometheus_alerts_test.yml
docker run --rm --entrypoint amtool \
  -v "$root/deploy/prometheus:/etc/alertmanager:ro" \
  -v "$tmp/alertmanager-webhook-url:/run/secrets/alertmanager-webhook-url:ro" \
  "$alertmanager_image" check-config /etc/alertmanager/alertmanager.yml
docker run --rm \
  -v "$root/deploy/prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro" \
  "$blackbox_image" --config.file=/etc/blackbox_exporter/config.yml --config.check
docker run --rm \
  -v "$root/deploy/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro" \
  "$fluentd_image" fluentd --dry-run -c /fluentd/etc/fluent.conf
"$root/backend/scripts/db_backup_test.sh"

echo 'observability configuration tests passed'