HH-441: harden durable storage and recovery (#92)
* HH-441: harden durable storage and recovery * HH-441: clear recovery review blockers * HH-441: enforce offsite backup failure domain --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
@@ -50,6 +50,9 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
-X github.com/gochat/gochat/internal/config.BuildDate=${BUILD_DATE}" \
|
||||
-o /gochat-worker ./cmd/gochat/
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build -ldflags="-s -w" -o /migrate ./cmd/migrate/
|
||||
|
||||
# ========== Production Stage ==========
|
||||
FROM alpine:3.21
|
||||
|
||||
@@ -60,17 +63,20 @@ LABEL org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.revision="${COMMIT_SHA}" \
|
||||
org.opencontainers.image.created="${BUILD_DATE}"
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apk --no-cache add ca-certificates tzdata curl && addgroup -S gochat && adduser -S gochat -G gochat
|
||||
# Install runtime and recovery dependencies.
|
||||
RUN apk --no-cache add bash ca-certificates tzdata curl openssl postgresql16-client && \
|
||||
addgroup -S gochat && adduser -S gochat -G gochat
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy binary and configs from builder
|
||||
COPY --from=builder /gochat /app/gochat
|
||||
COPY --from=builder /gochat-worker /app/gochat-worker
|
||||
COPY --from=builder /migrate /app/migrate
|
||||
COPY --chown=gochat:gochat --from=frontend-builder /app/frontend/dist /app/frontend/dist
|
||||
COPY backend/configs/ /app/configs/
|
||||
COPY backend/migrations/ /app/migrations/
|
||||
COPY backend/scripts/db_backup.sh backend/scripts/db_restore.sh /app/scripts/
|
||||
|
||||
ENV GOCHAT_FRONTEND_DIST=/app/frontend/dist
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ x-gochat-environment: &gochat-environment
|
||||
GOCHAT_SERVER_MODE: release
|
||||
GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: ${GOCHAT_SERVER_CORS_ALLOWED_ORIGINS:?set production CORS origins}
|
||||
GOCHAT_DATABASE_DSN: ${GOCHAT_DATABASE_DSN:-postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable}
|
||||
GOCHAT_DATABASE_RUN_MIGRATIONS: "true"
|
||||
GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
|
||||
GOCHAT_DATABASE_MIGRATIONS_PATH: /app/migrations
|
||||
GOCHAT_REDIS_DSN: redis://:${REDIS_PASSWORD:?set REDIS_PASSWORD}@redis:6379
|
||||
GOCHAT_SEARCH_ENGINE: meilisearch
|
||||
@@ -126,6 +126,75 @@ services:
|
||||
memory: 512M
|
||||
cpus: "1.0"
|
||||
|
||||
migrate:
|
||||
image: *gochat-image
|
||||
profiles: ["ops"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/app/migrate"]
|
||||
command: ["up"]
|
||||
restart: "no"
|
||||
environment:
|
||||
<<: *gochat-environment
|
||||
GOCHAT_DATABASE_DSN: postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable&lock_timeout=5000&statement_timeout=900000
|
||||
GOCHAT_DATABASE_RUN_MIGRATIONS: "false"
|
||||
|
||||
backup:
|
||||
image: *gochat-image
|
||||
user: "0:0"
|
||||
profiles: ["ops"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/app/scripts/db_backup.sh"]
|
||||
restart: "no"
|
||||
environment:
|
||||
GOCHAT_DATABASE_DSN: postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable
|
||||
GOCHAT_STORAGE_PATH: /source/storage/uploads
|
||||
GOCHAT_CONNECTOR_BACKUP_FILE: /source/connector/${GOCHAT_CONNECTOR_BACKUP_NAME:-latest.db}
|
||||
GOCHAT_BACKUP_DIR: /backup/local
|
||||
GOCHAT_BACKUP_OFFSITE_DIR: /backup/offsite
|
||||
GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
|
||||
GOCHAT_BACKUP_RETENTION_DAYS: ${GOCHAT_BACKUP_RETENTION_DAYS:-30}
|
||||
GOCHAT_VERSION: ${GOCHAT_IMAGE_REF}
|
||||
volumes:
|
||||
- gochat_storage:/source/storage:ro
|
||||
- shangwutong_backups:/source/connector:ro
|
||||
- ${GOCHAT_BACKUP_DIR:-./backups/local}:/backup/local
|
||||
- type: bind
|
||||
source: ${GOCHAT_BACKUP_OFFSITE_DIR:?set an existing external off-site mount point}
|
||||
target: /backup/offsite
|
||||
bind:
|
||||
create_host_path: false
|
||||
- ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
|
||||
|
||||
restore:
|
||||
image: *gochat-image
|
||||
user: "0:0"
|
||||
profiles: ["ops"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/app/scripts/db_restore.sh"]
|
||||
command: ["/backup/offsite/${GOCHAT_RESTORE_BUNDLE:-missing.tar.enc}"]
|
||||
restart: "no"
|
||||
environment:
|
||||
GOCHAT_DATABASE_DSN: postgres://${POSTGRES_USER:-gochat}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gochat_production}?sslmode=disable
|
||||
GOCHAT_STORAGE_PATH: /restore/storage/uploads
|
||||
GOCHAT_CONNECTOR_DB_PATH: /restore/connector/connector.db
|
||||
GOCHAT_BACKUP_PASSPHRASE_FILE: /run/secrets/backup-passphrase
|
||||
volumes:
|
||||
- gochat_storage:/restore/storage
|
||||
- shangwutong_data:/restore/connector
|
||||
- type: bind
|
||||
source: ${GOCHAT_BACKUP_OFFSITE_DIR:?set an existing external off-site mount point}
|
||||
target: /backup/offsite
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
- ${GOCHAT_BACKUP_PASSPHRASE_FILE:-./.secrets/backup-passphrase}:/run/secrets/backup-passphrase:ro
|
||||
|
||||
shangwutong:
|
||||
image: ${SHANGWUTONG_IMAGE_REF:?set SHANGWUTONG_IMAGE_REF to an immutable image digest}
|
||||
restart: always
|
||||
|
||||
@@ -12,7 +12,7 @@ if (($#)); then
|
||||
compose_args=(--env-file "$env_file" "${compose_args[@]}")
|
||||
fi
|
||||
|
||||
required=(GOCHAT_IMAGE_REF SHANGWUTONG_IMAGE_REF GOCHAT_SERVER_CORS_ALLOWED_ORIGINS POSTGRES_PASSWORD REDIS_PASSWORD MEILI_MASTER_KEY GOCHAT_JWT_SECRET)
|
||||
required=(GOCHAT_IMAGE_REF SHANGWUTONG_IMAGE_REF GOCHAT_SERVER_CORS_ALLOWED_ORIGINS POSTGRES_PASSWORD REDIS_PASSWORD MEILI_MASTER_KEY GOCHAT_JWT_SECRET GOCHAT_BACKUP_OFFSITE_DIR GOCHAT_BACKUP_OFFSITE_SOURCE GOCHAT_BACKUP_OFFSITE_FSTYPE)
|
||||
for name in "${required[@]}"; do
|
||||
value=${!name:-}
|
||||
if [[ -z $value || ${value^^} == *CHANGE_ME* ]]; then
|
||||
@@ -21,6 +21,44 @@ for name in "${required[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $GOCHAT_BACKUP_OFFSITE_DIR != /* ]] || [[ ! -d $GOCHAT_BACKUP_OFFSITE_DIR ]]; then
|
||||
echo "GOCHAT_BACKUP_OFFSITE_DIR must be an existing external mount point" >&2
|
||||
exit 1
|
||||
fi
|
||||
offsite_dir=$(realpath -e -- "$GOCHAT_BACKUP_OFFSITE_DIR")
|
||||
if [[ $offsite_dir == / ]]; then
|
||||
echo "GOCHAT_BACKUP_OFFSITE_DIR must not be /" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! offsite_info=$(findmnt -M "$offsite_dir" -n -o SOURCE,FSTYPE,MAJ:MIN); then
|
||||
echo "GOCHAT_BACKUP_OFFSITE_DIR must be an existing external mount point" >&2
|
||||
exit 1
|
||||
fi
|
||||
read -r offsite_source offsite_type offsite_device <<< "$offsite_info"
|
||||
if [[ $offsite_source != "$GOCHAT_BACKUP_OFFSITE_SOURCE" || $offsite_type != "$GOCHAT_BACKUP_OFFSITE_FSTYPE" ]]; then
|
||||
echo "GOCHAT_BACKUP_OFFSITE_DIR mount source/type does not match the approved values" >&2
|
||||
exit 1
|
||||
fi
|
||||
case $offsite_type in
|
||||
tmpfs | devtmpfs | ramfs)
|
||||
echo "GOCHAT_BACKUP_OFFSITE_DIR must not use an in-memory filesystem" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
local_probe=${GOCHAT_BACKUP_DIR:-./backups/local}
|
||||
if [[ $local_probe != /* ]]; then
|
||||
local_probe=$script_dir/$local_probe
|
||||
fi
|
||||
while [[ ! -e $local_probe && $local_probe != / ]]; do
|
||||
local_probe=$(dirname -- "$local_probe")
|
||||
done
|
||||
read -r local_source local_device < <(findmnt -T "$local_probe" -n -o SOURCE,MAJ:MIN)
|
||||
if [[ $offsite_source == "$local_source" || $offsite_device == "$local_device" ]]; then
|
||||
echo "GOCHAT_BACKUP_OFFSITE_DIR must use a different source/device than GOCHAT_BACKUP_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ((${#GOCHAT_JWT_SECRET} < 32)); then
|
||||
echo "GOCHAT_JWT_SECRET must be at least 32 characters" >&2
|
||||
exit 1
|
||||
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
mkdir -p "$tmp/bin" "$tmp/local" "$tmp/offsite"
|
||||
|
||||
cat > "$tmp/bin/findmnt" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
if [[ $1 == -M && $2 == "$GOCHAT_BACKUP_OFFSITE_DIR" ]]; then
|
||||
printf '%s %s %s\n' "$TEST_OFFSITE_SOURCE" "$TEST_OFFSITE_FSTYPE" "$TEST_OFFSITE_DEVICE"
|
||||
elif [[ $1 == -T ]]; then
|
||||
printf '%s %s\n' "$TEST_LOCAL_SOURCE" "$TEST_LOCAL_DEVICE"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
EOF
|
||||
cat > "$tmp/bin/docker" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
if [[ -n ${TEST_MUTABLE_IMAGE:-} ]]; then
|
||||
echo 'pgvector/pgvector:pg16'
|
||||
else
|
||||
printf '%s\n' \
|
||||
'gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000' \
|
||||
'gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
||||
fi
|
||||
EOF
|
||||
chmod +x "$tmp/bin/findmnt" "$tmp/bin/docker"
|
||||
|
||||
export PATH="$tmp/bin:$PATH"
|
||||
export GOCHAT_IMAGE_REF='gochat@example.invalid/gochat@sha256:0000000000000000000000000000000000000000000000000000000000000000'
|
||||
export SHANGWUTONG_IMAGE_REF='gochat@example.invalid/connector@sha256:1111111111111111111111111111111111111111111111111111111111111111'
|
||||
export GOCHAT_SERVER_CORS_ALLOWED_ORIGINS=https://chat.example.test
|
||||
export POSTGRES_PASSWORD=ci-postgres-secret
|
||||
export REDIS_PASSWORD=ci-redis-secret
|
||||
export MEILI_MASTER_KEY=ci-meili-secret-16
|
||||
export GOCHAT_JWT_SECRET=ci-smoke-jwt-secret-at-least-32-characters
|
||||
export GOCHAT_BACKUP_DIR="$tmp/local"
|
||||
export GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite"
|
||||
export GOCHAT_BACKUP_OFFSITE_SOURCE='backup.example.test:/gochat'
|
||||
export GOCHAT_BACKUP_OFFSITE_FSTYPE=nfs4
|
||||
export TEST_OFFSITE_SOURCE=$GOCHAT_BACKUP_OFFSITE_SOURCE
|
||||
export TEST_OFFSITE_FSTYPE=$GOCHAT_BACKUP_OFFSITE_FSTYPE
|
||||
export TEST_OFFSITE_DEVICE=0:42
|
||||
export TEST_LOCAL_SOURCE=/dev/sda1
|
||||
export TEST_LOCAL_DEVICE=8:1
|
||||
|
||||
run_preflight() {
|
||||
"$root/deploy/docker/preflight.sh" > "$tmp/output" 2>&1
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
local name=$1 expected=$2
|
||||
if run_preflight; then
|
||||
echo "preflight accepted $name" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F "$expected" "$tmp/output" >/dev/null
|
||||
}
|
||||
|
||||
GOCHAT_BACKUP_OFFSITE_DIR=/
|
||||
expect_failure 'the root filesystem' 'must not be /'
|
||||
|
||||
GOCHAT_BACKUP_OFFSITE_DIR=/tmp
|
||||
TEST_OFFSITE_SOURCE=tmpfs
|
||||
TEST_OFFSITE_FSTYPE=tmpfs
|
||||
GOCHAT_BACKUP_OFFSITE_SOURCE=tmpfs
|
||||
GOCHAT_BACKUP_OFFSITE_FSTYPE=tmpfs
|
||||
expect_failure 'tmpfs' 'must not use an in-memory filesystem'
|
||||
|
||||
GOCHAT_BACKUP_OFFSITE_DIR=$tmp/offsite
|
||||
TEST_OFFSITE_SOURCE='backup.example.test:/gochat'
|
||||
TEST_OFFSITE_FSTYPE=nfs4
|
||||
GOCHAT_BACKUP_OFFSITE_SOURCE=$TEST_OFFSITE_SOURCE
|
||||
GOCHAT_BACKUP_OFFSITE_FSTYPE=$TEST_OFFSITE_FSTYPE
|
||||
TEST_OFFSITE_DEVICE=$TEST_LOCAL_DEVICE
|
||||
expect_failure 'the local backup device' 'must use a different source/device'
|
||||
|
||||
TEST_OFFSITE_DEVICE=0:42
|
||||
GOCHAT_BACKUP_OFFSITE_SOURCE='other.example.test:/gochat'
|
||||
expect_failure 'an unapproved mount source' 'does not match the approved values'
|
||||
|
||||
GOCHAT_BACKUP_OFFSITE_SOURCE=$TEST_OFFSITE_SOURCE
|
||||
MEILI_MASTER_KEY=too-short
|
||||
expect_failure 'a short Meilisearch key' 'must be at least 16 bytes'
|
||||
MEILI_MASTER_KEY=ci-meili-secret-16
|
||||
|
||||
TEST_MUTABLE_IMAGE=1
|
||||
export TEST_MUTABLE_IMAGE
|
||||
expect_failure 'a mutable production image' 'must be pinned to a sha256 digest'
|
||||
unset TEST_MUTABLE_IMAGE
|
||||
|
||||
run_preflight
|
||||
grep -F 'production preflight passed' "$tmp/output" >/dev/null
|
||||
echo 'preflight tests passed'
|
||||
Reference in New Issue
Block a user