HH-445: deploy production observability and runbooks (#96)

* HH-445: deploy production observability and runbooks * fix(ops): share production database DSN * fix(HH-445): enforce database TLS gate * fix(HH-445): preserve production serve command * fix(prod): require external database dependencies * fix(prod): unify database host rejection gates * test(prod): enforce exact database TLS runbook contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
2026-08-22 19:39:57 +08:00
parent 61376a57fd
commit fb83285617
28 changed files with 1622 additions and 220 deletions
@@ -1,13 +1,48 @@
 name: gochat-production

 x-gochat-image: &gochat-image ${GOCHAT_IMAGE_REF:?set GOCHAT_IMAGE_REF to an immutable image digest}
+x-postgres-tls-ca: &postgres-tls-ca
+  type: bind
+  source: ${GOCHAT_DATABASE_TLS_CA_FILE:-/dev/null}
+  target: /run/secrets/external-db-ca.crt
+  read_only: true
+  bind:
+    create_host_path: false
+x-postgres-tls-client-cert: &postgres-tls-client-cert
+  type: bind
+  source: ${GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE:-/dev/null}
+  target: /run/secrets/external-db-client.crt
+  read_only: true
+  bind:
+    create_host_path: false
+x-postgres-tls-client-key: &postgres-tls-client-key
+  type: bind
+  source: ${GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE:-/dev/null}
+  target: /run/secrets/external-db-client.key
+  read_only: true
+  bind:
+    create_host_path: false
+x-database-client-entrypoint: &database-client-entrypoint
+  type: bind
+  source: ./database_client_entrypoint.sh
+  target: /usr/local/bin/database-client-entrypoint
+  read_only: true
+  bind:
+    create_host_path: false
+x-gochat-logging: &gochat-logging
+  driver: fluentd
+  options:
+    fluentd-address: 127.0.0.1:24224
+    fluentd-async: "true"
+    fluentd-buffer-limit: "65536"
+    tag: gochat.{{.Name}}
 x-gochat-environment: &gochat-environment
  GOCHAT_ENV: production
  GOCHAT_SERVER_HOST: 0.0.0.0
  GOCHAT_SERVER_PORT: 3000
  GOCHAT_SERVER_MODE: release
  GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: ${GOCHAT_SERVER_CORS_ALLOWED_ORIGINS:?set production CORS origins}
-  GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
+  GOCHAT_DATABASE_DSN: &gochat-database-dsn ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
  GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
  GOCHAT_DATABASE_MIGRATIONS_PATH: /app/migrations
  GOCHAT_REDIS_DSN: ${GOCHAT_REDIS_DSN:?set an external Redis rediss:// DSN}
@@ -25,43 +60,6 @@ x-gochat-environment: &gochat-environment
  GOCHAT_STORAGE_LOCAL_PATH: /app/storage/uploads

 services:
-  postgres:
-    image: ${POSTGRES_IMAGE_REF:-pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b}
-    restart: always
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB:-gochat_production}
-      POSTGRES_USER: ${POSTGRES_USER:-gochat}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-    deploy:
-      resources:
-        limits:
-          memory: 1G
-
-  redis:
-    image: ${REDIS_IMAGE_REF:-redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf}
-    restart: always
-    command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?set REDIS_PASSWORD}", "--appendonly", "yes", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
-    environment:
-      REDIS_PASSWORD: ${REDIS_PASSWORD:?set REDIS_PASSWORD}
-    volumes:
-      - redis_data:/data
-    healthcheck:
-      test: ["CMD-SHELL", "redis-cli -a '$${REDIS_PASSWORD}' ping"]
-      interval: 5s
-      timeout: 5s
-      retries: 20
-    deploy:
-      resources:
-        limits:
-          memory: 512M
-
  meilisearch:
    image: ${MEILI_IMAGE_REF:-getmeili/meilisearch:v1.13@sha256:bed3fb650e62da53145777204891159242f6ea4ce69e215b36223af4aa64a0ae}
    restart: always
@@ -76,16 +74,16 @@ services:
      interval: 5s
      timeout: 5s
      retries: 20
+    logging: *gochat-logging

  gochat:
    image: *gochat-image
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/gochat"]
+    command: ["serve"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
    restart: always
    stop_grace_period: ${GOCHAT_STOP_GRACE_PERIOD:-35s}
    depends_on:
-      postgres:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
      meilisearch:
        condition: service_healthy
    environment: *gochat-environment
@@ -93,6 +91,10 @@ services:
      - "127.0.0.1:${GOCHAT_PORT:-3000}:3000"
    volumes:
      - gochat_storage:/app/storage
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
    healthcheck:
      test: ["CMD", "wget", "-q", "-T", "3", "-O", "/dev/null", "http://127.0.0.1:3000/ready"]
      interval: 10s
@@ -107,18 +109,17 @@ services:
        reservations:
          memory: 256M
          cpus: "0.5"
+    logging: *gochat-logging

  worker:
    image: *gochat-image
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/gochat"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
    restart: always
    stop_grace_period: ${GOCHAT_STOP_GRACE_PERIOD:-35s}
    depends_on:
      gochat:
        condition: service_healthy
-      postgres:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
      meilisearch:
        condition: service_healthy
    command: ["worker"]
@@ -129,42 +130,51 @@ services:
      GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
    volumes:
      - gochat_storage:/app/storage
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: "1.0"
+    logging: *gochat-logging

  migrate:
    image: *gochat-image
    profiles: ["ops"]
-    depends_on:
-      postgres:
-        condition: service_healthy
-    entrypoint: ["/app/migrate"]
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/migrate"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
    command: ["up"]
    restart: "no"
    environment:
      <<: *gochat-environment
      GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
+      PGOPTIONS: -c lock_timeout=5000 -c statement_timeout=900000
+    volumes:
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging

  backup:
    image: *gochat-image
    user: "0:0"
    profiles: ["ops"]
-    depends_on:
-      postgres:
-        condition: service_healthy
-    entrypoint: ["/app/scripts/db_backup.sh"]
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/scripts/db_backup.sh"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
    restart: "no"
    environment:
-      GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
+      GOCHAT_DATABASE_DSN: *gochat-database-dsn
      GOCHAT_STORAGE_PATH: /source/storage/uploads
      GOCHAT_CONNECTOR_BACKUP_FILE: /source/connector/${GOCHAT_CONNECTOR_BACKUP_NAME:-latest.db}
      GOCHAT_BACKUP_DIR: /backup/local
      GOCHAT_BACKUP_OFFSITE_DIR: /backup/offsite
      GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
      GOCHAT_BACKUP_RETENTION_DAYS: ${GOCHAT_BACKUP_RETENTION_DAYS:-30}
+      GOCHAT_BACKUP_METRICS_FILE: /metrics/gochat_backup.prom
      GOCHAT_VERSION: ${GOCHAT_IMAGE_REF}
    volumes:
      - gochat_storage:/source/storage:ro
@@ -176,19 +186,23 @@ services:
        bind:
          create_host_path: false
      - ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
+      - backup_metrics:/metrics
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging

  restore:
    image: *gochat-image
    user: "0:0"
    profiles: ["ops"]
-    depends_on:
-      postgres:
-        condition: service_healthy
-    entrypoint: ["/app/scripts/db_restore.sh"]
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/app/scripts/db_restore.sh"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
    command: ["/backup/offsite/${GOCHAT_RESTORE_BUNDLE:-missing.tar.enc}"]
    restart: "no"
    environment:
-      GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:?set an external PostgreSQL DSN with sslmode=verify-ca or verify-full}
+      GOCHAT_DATABASE_DSN: *gochat-database-dsn
      GOCHAT_STORAGE_PATH: /restore/storage/uploads
      GOCHAT_CONNECTOR_DB_PATH: /restore/connector/connector.db
      GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
@@ -202,6 +216,11 @@ services:
        bind:
          create_host_path: false
      - ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging

  shangwutong:
    image: ${SHANGWUTONG_IMAGE_REF:?set SHANGWUTONG_IMAGE_REF to an immutable image digest}
@@ -233,11 +252,114 @@ services:
        reservations:
          memory: 128M
          cpus: "0.25"
+    logging: *gochat-logging
+
+  fluentd:
+    image: fluent/fluentd:v1.18-debian-1@sha256:f8d26db76ba06ce96e8d402119675071624dab724af49be40fd34641c347c440
+    restart: always
+    ports:
+      - "127.0.0.1:24224:24224"
+    volumes:
+      - ../fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro
+      - fluentd_logs:/fluentd/log
+      - fluentd_buffer:/fluentd/buffer
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter:v0.17.1@sha256:38606faa38c54787525fb0ff2fd6b41b4cfb75d455c1df294927c5f611699b17
+    restart: always
+    entrypoint: ["/usr/local/bin/database-client-entrypoint", "/bin/postgres_exporter"]
+    group_add: ["${GOCHAT_DATABASE_TLS_GID:-65534}"]
+    command: ["--config.file=/dev/null", "--extend.query-path=/etc/postgres-exporter/queries.yml"]
+    environment:
+      DATA_SOURCE_NAME: *gochat-database-dsn
+    volumes:
+      - ../prometheus/postgres_queries.yml:/etc/postgres-exporter/queries.yml:ro
+      - *database-client-entrypoint
+      - *postgres-tls-ca
+      - *postgres-tls-client-cert
+      - *postgres-tls-client-key
+    logging: *gochat-logging
+
+  redis-exporter:
+    image: oliver006/redis_exporter:v1.72.1@sha256:f90cae1e7ecc6ac223d04bdb0c95e084918baced9f81f08c3d01c2f11bff72bf
+    restart: always
+    environment:
+      REDIS_ADDR: ${GOCHAT_REDIS_DSN:?set an external Redis rediss:// DSN}
+    logging: *gochat-logging
+
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.27.0@sha256:a50c4c0eda297baa1678cd4dc4712a67fdea713b832d43ce7fcc5f9bea05094d
+    restart: always
+    command: ["--config.file=/etc/blackbox_exporter/config.yml"]
+    volumes:
+      - ../prometheus/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+    logging: *gochat-logging
+
+  node-exporter:
+    image: prom/node-exporter:v1.9.1@sha256:d00a542e409ee618a4edc67da14dd48c5da66726bbd5537ab2af9c1dfc442c8a
+    restart: always
+    command: ["--collector.disable-defaults", "--collector.textfile", "--collector.textfile.directory=/var/lib/node_exporter/textfile_collector"]
+    volumes:
+      - backup_metrics:/var/lib/node_exporter/textfile_collector:ro
+    logging: *gochat-logging
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.52.1@sha256:f40e65878e25c2e78ea037f73a449527a0fb994e303dc3e34cb6b187b4b91435
+    restart: always
+    privileged: true
+    devices:
+      - /dev/kmsg:/dev/kmsg
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+      - /dev/disk:/dev/disk:ro
+    logging: *gochat-logging
+
+  alertmanager:
+    image: prom/alertmanager:v0.28.1@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba
+    restart: always
+    command: ["--config.file=/etc/alertmanager/alertmanager.yml", "--storage.path=/alertmanager"]
+    ports:
+      - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
+    volumes:
+      - ../prometheus/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - type: bind
+        source: ${ALERTMANAGER_WEBHOOK_URL_FILE:-../../.secrets/alertmanager-webhook-url}
+        target: /run/secrets/alertmanager-webhook-url
+        read_only: true
+        bind:
+          create_host_path: false
+      - alertmanager_data:/alertmanager
+    logging: *gochat-logging
+
+  prometheus:
+    image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996
+    restart: always
+    depends_on:
+      - alertmanager
+      - blackbox-exporter
+      - cadvisor
+      - node-exporter
+      - postgres-exporter
+      - redis-exporter
+    command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}"]
+    ports:
+      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../../backend/configs/prometheus_alerts.yml:/etc/prometheus/rules/gochat.yml:ro
+      - prometheus_data:/prometheus
+    logging: *gochat-logging

 volumes:
-  postgres_data:
-  redis_data:
  meili_data:
  gochat_storage:
  shangwutong_data:
  shangwutong_backups:
+  backup_metrics:
+  prometheus_data:
+  alertmanager_data:
+  fluentd_logs:
+  fluentd_buffer: