rogee/gochat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HH-445: deploy production observability and runbooks (#96)

Browse Source 
* HH-445: deploy production observability and runbooks

* fix(ops): share production database DSN

* fix(HH-445): enforce database TLS gate

* fix(HH-445): preserve production serve command

* fix(prod): require external database dependencies

* fix(prod): unify database host rejection gates

* test(prod): enforce exact database TLS runbook contract

---------

Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
2026-08-22 19:39:57 +08:00
committed by GitHub
co-authored by rogee
parent 61376a57fd
commit fb83285617
28 changed files with 1622 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+18 -8
View File
@@ -39,8 +39,8 @@ jobs:
chrome-version: 152.0.7977.54
- name: Install browser harness
run: python -m pip install --require-hashes -r .github/requirements-browser-harness.txt
- name: Test Prometheus alert rules
run: docker run --rm --entrypoint promtool -v "$PWD/backend/configs:/configs:ro" prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 test rules /configs/prometheus_alerts_test.yml
- name: Test production observability configuration
run: deploy/docker/observability_test.sh
- name: Test quality gate failure contracts
working-directory: backend
run: python3 scripts/quality_gate_test.py
@@ -389,7 +389,10 @@ jobs:
GOCHAT_IMAGE_REF: gochat:production-smoke
GOCHAT_PORT: "38080"
GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: https://chat.ci.rogeecn.com
GOCHAT_DATABASE_DSN: "postgres://gochat:ci-postgres-secret@postgres:5432/gochat_production?sslmode=verify-full&sslrootcert=/run/tls/ca.crt"
GOCHAT_DATABASE_DSN: "postgres://gochat:ci-postgres-secret@db.smoke.test:5432/gochat_production?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key"
GOCHAT_DATABASE_TLS_CA_FILE: ${{ github.workspace }}/.tmp/gochat-tls/ca.crt
GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE: ${{ github.workspace }}/.tmp/gochat-tls/postgres.crt
GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE: ${{ github.workspace }}/.tmp/gochat-tls/postgres.key
GOCHAT_REDIS_DSN: "rediss://:ci-redis-secret@redis:6379/0"
GOCHAT_ENCRYPTION_AES_KEY: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
GOCHAT_TLS_DIR: ${{ github.workspace }}/.tmp/gochat-tls
@@ -444,20 +447,25 @@ jobs:
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=GoChat CI CA' \
-keyout "$GOCHAT_TLS_DIR/ca.key" -out "$GOCHAT_TLS_DIR/ca.crt"
for service in postgres redis; do
openssl req -newkey rsa:2048 -nodes -subj "/CN=$service" \
-addext "subjectAltName=DNS:$service" \
hostname=$service
[[ $service == postgres ]] && hostname=db.smoke.test
openssl req -newkey rsa:2048 -nodes -subj "/CN=$hostname" \
-addext "subjectAltName=DNS:$hostname" \
-keyout "$GOCHAT_TLS_DIR/$service.key" -out "$GOCHAT_TLS_DIR/$service.csr"
printf 'subjectAltName=DNS:%s\n' "$service" > "$GOCHAT_TLS_DIR/$service.ext"
printf 'subjectAltName=DNS:%s\n' "$hostname" > "$GOCHAT_TLS_DIR/$service.ext"
openssl x509 -req -days 1 -CA "$GOCHAT_TLS_DIR/ca.crt" -CAkey "$GOCHAT_TLS_DIR/ca.key" \
-CAcreateserial -extfile "$GOCHAT_TLS_DIR/$service.ext" \
-in "$GOCHAT_TLS_DIR/$service.csr" -out "$GOCHAT_TLS_DIR/$service.crt"
done
chmod 600 "$GOCHAT_TLS_DIR"/*.key
chmod 640 "$GOCHAT_TLS_DIR"/postgres.key
chmod 600 "$GOCHAT_TLS_DIR"/ca.key "$GOCHAT_TLS_DIR"/redis.key
chmod 644 "$GOCHAT_TLS_DIR"/*.crt
echo "GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$GOCHAT_TLS_DIR/postgres.key")" >> "$GITHUB_ENV"
- name: Start production Compose and smoke core pages
run: |
deploy/docker/preflight_test.sh
docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
GOCHAT_TEST_IMAGE="$GOCHAT_IMAGE_REF" deploy/docker/database_client_entrypoint_test.sh
docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert config["services"]["gochat"]["command"] == ["serve"]; assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
compose=(docker compose -f deploy/docker/docker-compose.prod.yml -f deploy/docker/docker-compose.prod-smoke.yml)
"${compose[@]}" --profile ops run --rm migrate
"${compose[@]}" up -d --wait gochat shangwutong
@@ -473,6 +481,8 @@ jobs:
done
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" > "$RUNNER_TEMP/health.json"
python3 backend/scripts/validate_health_json.py "$RUNNER_TEMP/health.json"
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/ready" | grep -q '"ready":true'
! "${compose[@]}" logs --no-color gochat | grep -F 'slice bounds out of range'
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'
curl -fsS "http://127.0.0.1:$GOCHAT_PORT/runtime-config.js" | grep -q 'window.__GOCHAT_CONFIG__'
curl -fsS -o "$RUNNER_TEMP/favicon-32x32.png" "http://127.0.0.1:$GOCHAT_PORT/favicon-32x32.png"