HH-441: harden durable storage and recovery (#92)

* HH-441: harden durable storage and recovery

* HH-441: clear recovery review blockers

* HH-441: enforce offsite backup failure domain

---------

Co-authored-by: Rogee <rogee@ipao.vip>

.github/workflows/ci.yml

@@ -178,6 +178,9 @@ jobs:
       MEILI_MASTER_KEY: ci-meili-secret-16
       GOCHAT_JWT_SECRET: ci-smoke-jwt-secret-at-least-32-characters
       GOCHAT_JWT_PREVIOUS_SECRETS: ci-previous-jwt-secret-at-least-32-characters
+      GOCHAT_BACKUP_OFFSITE_DIR: /mnt/gochat-offsite
+      GOCHAT_BACKUP_OFFSITE_SOURCE: backup.example.test:/gochat
+      GOCHAT_BACKUP_OFFSITE_FSTYPE: nfs4
     steps:
       - uses: actions/checkout@v4
       - name: Build production images from repository root
@@ -200,16 +203,9 @@ jobs:
           test -s /app/frontend/dist/index.html'
       - name: Start production Compose and smoke core pages
         run: |
-          if MEILI_MASTER_KEY=too-short deploy/docker/preflight.sh; then
-            echo "preflight accepted a short Meilisearch key" >&2
-            exit 1
-          fi
-          if POSTGRES_IMAGE_REF=pgvector/pgvector:pg16 deploy/docker/preflight.sh; then
-            echo "preflight accepted a mutable PostgreSQL image" >&2
-            exit 1
-          fi
-          deploy/docker/preflight.sh
+          deploy/docker/preflight_test.sh
           docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
+          docker compose -f deploy/docker/docker-compose.prod.yml --profile ops run --rm migrate
           docker compose -f deploy/docker/docker-compose.prod.yml up -d --wait gochat
           curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" | grep -q '"status":"ok"'
           curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'