HH-441: harden durable storage and recovery (#92)

* HH-441: harden durable storage and recovery

* HH-441: clear recovery review blockers

* HH-441: enforce offsite backup failure domain

---------

Co-authored-by: Rogee <rogee@ipao.vip>

deploy/docker/preflight_test.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin" "$tmp/local" "$tmp/offsite"
+
+cat > "$tmp/bin/findmnt" <<'EOF'
+#!/usr/bin/env bash
+if [[ $1 == -M && $2 == "$GOCHAT_BACKUP_OFFSITE_DIR" ]]; then
+  printf '%s %s %s\n' "$TEST_OFFSITE_SOURCE" "$TEST_OFFSITE_FSTYPE" "$TEST_OFFSITE_DEVICE"
+elif [[ $1 == -T ]]; then
+  printf '%s %s\n' "$TEST_LOCAL_SOURCE" "$TEST_LOCAL_DEVICE"
+else
+  exit 1
+fi
+EOF
+cat > "$tmp/bin/docker" <<'EOF'
+#!/usr/bin/env bash
+if [[ -n ${TEST_MUTABLE_IMAGE:-} ]]; then
+  echo 'pgvector/pgvector:pg16'
+else
+  printf '%s\n' \
+    'gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000' \
+    'gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
+fi
+EOF
+chmod +x "$tmp/bin/findmnt" "$tmp/bin/docker"
+
+export PATH="$tmp/bin:$PATH"
+export GOCHAT_IMAGE_REF='gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
+export SHANGWUTONG_IMAGE_REF='gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
+export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
+export POSTGRES_PASSWORD=ci-postgres-secret
+export REDIS_PASSWORD=ci-redis-secret
+export MEILI_MASTER_KEY=ci-meili-secret-16
+export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
+export GOCHAT_BACKUP_DIR="$tmp/local"
+export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
+export GOCHAT_BACKUP_OFFSITE_SOURCE='backup.example.test:/gochat'
+export GOCHAT_BACKUP_OFFSITE_FSTYPE=nfs4
+export TEST_OFFSITE_SOURCE=$GOCHAT_BACKUP_OFFSITE_SOURCE
+export TEST_OFFSITE_FSTYPE=$GOCHAT_BACKUP_OFFSITE_FSTYPE
+export TEST_OFFSITE_DEVICE=0:42
+export TEST_LOCAL_SOURCE=/dev/sda1
+export TEST_LOCAL_DEVICE=8:1
+
+run_preflight() {
+  "$root/deploy/docker/preflight.sh" > "$tmp/output" 2>&1
+}
+
+expect_failure() {
+  local name=$1 expected=$2
+  if run_preflight; then
+    echo "preflight accepted $name" >&2
+    exit 1
+  fi
+  grep -F "$expected" "$tmp/output" >/dev/null
+}
+
+GOCHAT_BACKUP_OFFSITE_DIR=/
+expect_failure 'the root filesystem' 'must not be /'
+
+GOCHAT_BACKUP_OFFSITE_DIR=/tmp
+TEST_OFFSITE_SOURCE=tmpfs
+TEST_OFFSITE_FSTYPE=tmpfs
+GOCHAT_BACKUP_OFFSITE_SOURCE=tmpfs
+GOCHAT_BACKUP_OFFSITE_FSTYPE=tmpfs
+expect_failure 'tmpfs' 'must not use an in-memory filesystem'
+
+GOCHAT_BACKUP_OFFSITE_DIR=$tmp/offsite
+TEST_OFFSITE_SOURCE='backup.example.test:/gochat'
+TEST_OFFSITE_FSTYPE=nfs4
+GOCHAT_BACKUP_OFFSITE_SOURCE=$TEST_OFFSITE_SOURCE
+GOCHAT_BACKUP_OFFSITE_FSTYPE=$TEST_OFFSITE_FSTYPE
+TEST_OFFSITE_DEVICE=$TEST_LOCAL_DEVICE
+expect_failure 'the local backup device' 'must use a different source/device'
+
+TEST_OFFSITE_DEVICE=0:42
+GOCHAT_BACKUP_OFFSITE_SOURCE='other.example.test:/gochat'
+expect_failure 'an unapproved mount source' 'does not match the approved values'
+
+GOCHAT_BACKUP_OFFSITE_SOURCE=$TEST_OFFSITE_SOURCE
+MEILI_MASTER_KEY=too-short
+expect_failure 'a short Meilisearch key' 'must be at least 16 bytes'
+MEILI_MASTER_KEY=ci-meili-secret-16
+
+TEST_MUTABLE_IMAGE=1
+export TEST_MUTABLE_IMAGE
+expect_failure 'a mutable production image' 'must be pinned to a sha256 digest'
+unset TEST_MUTABLE_IMAGE
+
+run_preflight
+grep -F 'production preflight passed' "$tmp/output" >/dev/null
+echo 'preflight tests passed'