HH-441: harden durable storage and recovery (#92)

* HH-441: harden durable storage and recovery

* HH-441: clear recovery review blockers

* HH-441: enforce offsite backup failure domain

---------

Co-authored-by: Rogee <rogee@ipao.vip>

deploy/docker/docker-compose.prod.yml

@@ -8,7 +8,7 @@ x-gochat-environment: &gochat-environment
   GOCHAT_SERVER_MODE: release
   GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: ${GOCHAT_SERVER_CORS_ALLOWED_ORIGINS:?set production CORS origins}
   GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:-postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable}
-  GOCHAT_DATABASE_RUN_MIGRATIONS: "true"
+  GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
   GOCHAT_DATABASE_MIGRATIONS_PATH: /app/migrations
   GOCHAT_REDIS_DSN: redis://:${REDIS_PASSWORD:?set REDIS_PASSWORD}@redis:6379
   GOCHAT_SEARCH_ENGINE: meilisearch
@@ -126,6 +126,75 @@ services:
           memory: 512M
           cpus: "1.0"
 
+  migrate:
+    image: *gochat-image
+    profiles: ["ops"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    entrypoint: ["/app/migrate"]
+    command: ["up"]
+    restart: "no"
+    environment:
+      <<: *gochat-environment
+      GOCHAT_DATABASE_DSN: postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable&lock_timeout=5000&statement_timeout=900000
+      GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
+
+  backup:
+    image: *gochat-image
+    user: "0:0"
+    profiles: ["ops"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    entrypoint: ["/app/scripts/db_backup.sh"]
+    restart: "no"
+    environment:
+      GOCHAT_DATABASE_DSN: postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable
+      GOCHAT_STORAGE_PATH: /source/storage/uploads
+      GOCHAT_CONNECTOR_BACKUP_FILE: /source/connector/${GOCHAT_CONNECTOR_BACKUP_NAME:-latest.db}
+      GOCHAT_BACKUP_DIR: /backup/local
+      GOCHAT_BACKUP_OFFSITE_DIR: /backup/offsite
+      GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
+      GOCHAT_BACKUP_RETENTION_DAYS: ${GOCHAT_BACKUP_RETENTION_DAYS:-30}
+      GOCHAT_VERSION: ${GOCHAT_IMAGE_REF}
+    volumes:
+      - gochat_storage:/source/storage:ro
+      - shangwutong_backups:/source/connector:ro
+      - ${GOCHAT_BACKUP_DIR:-./backups/local}:/backup/local
+      - type: bind
+        source: ${GOCHAT_BACKUP_OFFSITE_DIR:?set an existing external off-site mount point}
+        target: /backup/offsite
+        bind:
+          create_host_path: false
+      - ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
+
+  restore:
+    image: *gochat-image
+    user: "0:0"
+    profiles: ["ops"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    entrypoint: ["/app/scripts/db_restore.sh"]
+    command: ["/backup/offsite/${GOCHAT_RESTORE_BUNDLE:-missing.tar.enc}"]
+    restart: "no"
+    environment:
+      GOCHAT_DATABASE_DSN: postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable
+      GOCHAT_STORAGE_PATH: /restore/storage/uploads
+      GOCHAT_CONNECTOR_DB_PATH: /restore/connector/connector.db
+      GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
+    volumes:
+      - gochat_storage:/restore/storage
+      - shangwutong_data:/restore/connector
+      - type: bind
+        source: ${GOCHAT_BACKUP_OFFSITE_DIR:?set an existing external off-site mount point}
+        target: /backup/offsite
+        read_only: true
+        bind:
+          create_host_path: false
+      - ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
+
   shangwutong:
     image: ${SHANGWUTONG_IMAGE_REF:?set SHANGWUTONG_IMAGE_REF to an immutable image digest}
     restart: always