162 lines
8.8 KiB
Bash
Executable File
162 lines
8.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
printf '%s\n' 'https://alerts.example.test/gochat' >"$tmp/alertmanager-webhook-url"
|
|
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=gochat-observability-test' \
|
|
-keyout "$tmp/external-db-client.key" -out "$tmp/external-db-client.crt" >/dev/null 2>&1
|
|
cp "$tmp/external-db-client.crt" "$tmp/external-db-ca.crt"
|
|
chmod 0644 "$tmp/external-db-ca.crt" "$tmp/external-db-client.crt"
|
|
chmod 0640 "$tmp/external-db-client.key"
|
|
|
|
export GOCHAT_IMAGE_REF='gochat.example.test/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
|
|
export SHANGWUTONG_IMAGE_REF='gochat.example.test/shangwutong@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
|
export GOCHAT_REDIS_DSN='rediss://:ci-redis-secret@redis.example.test:6379/0'
|
|
export GOCHAT_ENCRYPTION_AES_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
|
|
export POSTGRES_PASSWORD=ci-postgres-secret
|
|
export REDIS_PASSWORD=ci-redis-secret
|
|
export MEILI_MASTER_KEY=ci-meili-secret-16
|
|
export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
|
export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
|
|
export ALERTMANAGER_WEBHOOK_URL_FILE="$tmp/alertmanager-webhook-url"
|
|
mkdir "$GOCHAT_BACKUP_OFFSITE_DIR"
|
|
|
|
baseline_dsn='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full'
|
|
baseline_compose_json=$tmp/baseline-compose.json
|
|
mtls_compose_json=$tmp/mtls-compose.json
|
|
smoke_compose_json=$tmp/smoke-compose.json
|
|
export GOCHAT_DATABASE_DSN=$baseline_dsn
|
|
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$baseline_compose_json"
|
|
export GOCHAT_TLS_DIR=$tmp
|
|
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" -f "$root/deploy/docker/docker-compose.prod-smoke.yml" --profile '*' config --format json >"$smoke_compose_json"
|
|
export GOCHAT_DATABASE_TLS_CA_FILE="$tmp/external-db-ca.crt"
|
|
export GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE="$tmp/external-db-client.crt"
|
|
export GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE="$tmp/external-db-client.key"
|
|
export GOCHAT_DATABASE_TLS_GID
|
|
GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$tmp/external-db-client.key")
|
|
export GOCHAT_DATABASE_DSN='postgres://external_user:external_password@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key'
|
|
docker compose -f "$root/deploy/docker/docker-compose.prod.yml" --profile '*' config --format json >"$mtls_compose_json"
|
|
python3 - "$baseline_compose_json" "$mtls_compose_json" "$smoke_compose_json" "$baseline_dsn" "$GOCHAT_DATABASE_DSN" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
baseline = json.load(open(sys.argv[1], encoding="utf-8"))["services"]
|
|
mtls = json.load(open(sys.argv[2], encoding="utf-8"))["services"]
|
|
smoke = json.load(open(sys.argv[3], encoding="utf-8"))["services"]
|
|
baseline_dsn = sys.argv[4]
|
|
mtls_dsn = sys.argv[5]
|
|
services = baseline
|
|
required = {
|
|
"alertmanager", "blackbox-exporter", "cadvisor", "fluentd", "node-exporter",
|
|
"postgres-exporter", "prometheus", "redis-exporter",
|
|
}
|
|
assert required <= services.keys(), required - services.keys()
|
|
assert not {"postgres", "redis"} & services.keys()
|
|
for service in ("gochat", "worker", "migrate", "backup", "shangwutong"):
|
|
assert services[service]["logging"]["driver"] == "fluentd", service
|
|
for service in ("postgres", "redis"):
|
|
assert smoke[service]["logging"]["driver"] == "fluentd", service
|
|
for service, dependencies in {
|
|
"gochat": {"postgres", "redis"},
|
|
"worker": {"postgres", "redis"},
|
|
"migrate": {"postgres"},
|
|
"backup": {"postgres"},
|
|
"restore": {"postgres"},
|
|
"postgres-exporter": {"postgres"},
|
|
"redis-exporter": {"redis"},
|
|
}.items():
|
|
assert not dependencies & baseline[service].get("depends_on", {}).keys(), service
|
|
assert dependencies <= smoke[service]["depends_on"].keys(), service
|
|
assert baseline["redis-exporter"]["environment"]["REDIS_ADDR"] == os.environ["GOCHAT_REDIS_DSN"]
|
|
assert any(volume["target"] == "/etc/prometheus/rules/gochat.yml" for volume in services["prometheus"]["volumes"])
|
|
assert any(volume["target"] == "/fluentd/etc/fluent.conf" for volume in services["fluentd"]["volumes"])
|
|
assert services["backup"]["environment"]["GOCHAT_BACKUP_METRICS_FILE"] == "/metrics/gochat_backup.prom"
|
|
|
|
def database_dsns(config):
|
|
return {
|
|
"gochat": config["gochat"]["environment"]["GOCHAT_DATABASE_DSN"],
|
|
"worker": config["worker"]["environment"]["GOCHAT_DATABASE_DSN"],
|
|
"migrate": config["migrate"]["environment"]["GOCHAT_DATABASE_DSN"],
|
|
"backup": config["backup"]["environment"]["GOCHAT_DATABASE_DSN"],
|
|
"restore": config["restore"]["environment"]["GOCHAT_DATABASE_DSN"],
|
|
"postgres-exporter": config["postgres-exporter"]["environment"]["DATA_SOURCE_NAME"],
|
|
}
|
|
|
|
client_commands = {
|
|
"gochat": "/app/gochat",
|
|
"worker": "/app/gochat",
|
|
"migrate": "/app/migrate",
|
|
"backup": "/app/scripts/db_backup.sh",
|
|
"restore": "/app/scripts/db_restore.sh",
|
|
"postgres-exporter": "/bin/postgres_exporter",
|
|
}
|
|
for config, expected_gid in ((baseline, "65534"), (mtls, os.environ["GOCHAT_DATABASE_TLS_GID"])):
|
|
for service, command in client_commands.items():
|
|
assert config[service]["entrypoint"] == ["/usr/local/bin/database-client-entrypoint", command], service
|
|
assert expected_gid in map(str, config[service]["group_add"]), service
|
|
gate = [volume for volume in config[service]["volumes"] if volume["target"] == "/usr/local/bin/database-client-entrypoint"]
|
|
assert len(gate) == 1 and gate[0]["read_only"] is True, service
|
|
|
|
baseline_dsns = database_dsns(baseline)
|
|
assert set(baseline_dsns.values()) == {baseline_dsn}, baseline_dsns
|
|
mtls_dsns = database_dsns(mtls)
|
|
assert set(mtls_dsns.values()) == {mtls_dsn}, mtls_dsns
|
|
|
|
pgoptions = "-c lock_timeout=5000 -c statement_timeout=900000"
|
|
assert baseline["migrate"]["environment"]["PGOPTIONS"] == pgoptions
|
|
assert mtls["migrate"]["environment"]["PGOPTIONS"] == pgoptions
|
|
|
|
tls_paths = {
|
|
"sslrootcert": "/run/secrets/external-db-ca.crt",
|
|
"sslcert": "/run/secrets/external-db-client.crt",
|
|
"sslkey": "/run/secrets/external-db-client.key",
|
|
}
|
|
tls_sources = {
|
|
tls_paths["sslrootcert"]: os.environ["GOCHAT_DATABASE_TLS_CA_FILE"],
|
|
tls_paths["sslcert"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE"],
|
|
tls_paths["sslkey"]: os.environ["GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE"],
|
|
}
|
|
query = parse_qs(urlsplit(mtls_dsn).query)
|
|
assert query["sslmode"] == ["verify-full"]
|
|
for parameter, target in tls_paths.items():
|
|
assert query[parameter] == [target], (parameter, query)
|
|
|
|
for config, expected_sources in ((baseline, {target: "/dev/null" for target in tls_paths.values()}), (mtls, tls_sources)):
|
|
for service in database_dsns(config):
|
|
mounts = {volume["target"]: volume for volume in config[service]["volumes"] if volume["target"] in tls_paths.values()}
|
|
assert mounts.keys() == expected_sources.keys(), (service, mounts)
|
|
for target, source in expected_sources.items():
|
|
assert mounts[target]["source"] == source, (service, target, mounts[target])
|
|
assert mounts[target]["read_only"] is True, (service, target, mounts[target])
|
|
PY
|
|
|
|
prometheus_image='prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996'
|
|
alertmanager_image='prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba'
|
|
blackbox_image='prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d'
|
|
fluentd_image='fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440'
|
|
|
|
docker run --rm --entrypoint promtool \
|
|
-v "$root/deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \
|
|
-v "$root/backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro" \
|
|
"$prometheus_image" check config /etc/prometheus/prometheus.yml
|
|
docker run --rm --entrypoint promtool \
|
|
-v "$root/backend/configs:/configs:ro" \
|
|
"$prometheus_image" test rules /configs/prometheus_alerts_test.yml
|
|
docker run --rm --entrypoint amtool \
|
|
-v "$root/deploy/prometheus:/etc/alertmanager:ro" \
|
|
-v "$tmp/alertmanager-webhook-url:/run/secrets/alertmanager-webhook-url:ro" \
|
|
"$alertmanager_image" check-config /etc/alertmanager/alertmanager.yml
|
|
docker run --rm \
|
|
-v "$root/deploy/prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro" \
|
|
"$blackbox_image" --config.file=/etc/blackbox_exporter/config.yml --config.check
|
|
docker run --rm \
|
|
-v "$root/deploy/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro" \
|
|
"$fluentd_image" fluentd --dry-run -c /fluentd/etc/fluent.conf
|
|
"$root/backend/scripts/db_backup_test.sh"
|
|
|
|
echo 'observability configuration tests passed'
|