HH-441: harden durable storage and recovery (#92)
* HH-441: harden durable storage and recovery * HH-441: clear recovery review blockers * HH-441: enforce offsite backup failure domain --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
@@ -1,54 +1,62 @@
|
||||
#!/bin/bash
|
||||
# GoChat Database Backup Script
|
||||
# Reference: Chatwoot uses pg_dump for backup in production deployments
|
||||
# Usage: ./scripts/db_backup.sh [env]
|
||||
#!/usr/bin/env bash
|
||||
# Create one encrypted, off-site copy containing PostgreSQL, attachments, and
|
||||
# the Connector's online SQLite backup.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
ENV="${1:-production}"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
BACKUP_DIR="${GOCHAT_BACKUP_DIR:-/var/backups/gochat}"
|
||||
DSN="${GOCHAT_DATABASE_DSN:-}"
|
||||
RETENTION_DAYS="${GOCHAT_BACKUP_RETENTION_DAYS:-30}"
|
||||
dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
|
||||
storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
|
||||
connector=${GOCHAT_CONNECTOR_BACKUP_FILE:?GOCHAT_CONNECTOR_BACKUP_FILE is required}
|
||||
backup_dir=${GOCHAT_BACKUP_DIR:-/var/backups/gochat}
|
||||
offsite_dir=${GOCHAT_BACKUP_OFFSITE_DIR:?GOCHAT_BACKUP_OFFSITE_DIR is required}
|
||||
passphrase_file=${GOCHAT_BACKUP_PASSPHRASE_FILE:?GOCHAT_BACKUP_PASSPHRASE_FILE is required}
|
||||
retention_days=${GOCHAT_BACKUP_RETENTION_DAYS:-30}
|
||||
version=${GOCHAT_VERSION:-unknown}
|
||||
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
|
||||
if [[ -z "$DSN" ]]; then
|
||||
echo "[$(date)] ERROR: GOCHAT_DATABASE_DSN is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Parse DSN: postgres://user:password@host:port/dbname?sslmode=...
|
||||
DB_USER=$(echo "$DSN" | sed 's|.*://||; s|:.*||')
|
||||
DB_PASS=$(echo "$DSN" | sed 's|.*://[^:]*:||; s|@.*||')
|
||||
DB_HOST=$(echo "$DSN" | sed 's|.*@||; s|:.*||')
|
||||
DB_PORT=$(echo "$DSN" | sed 's|.*@.*:||; s|/.*||')
|
||||
DB_NAME=$(echo "$DSN" | sed 's|.*/||; s|\?.*||')
|
||||
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
|
||||
BACKUP_FILE="${BACKUP_DIR}/${DB_NAME}_${TIMESTAMP}.sql.gz"
|
||||
|
||||
echo "[$(date)] Starting backup of ${DB_NAME} on ${DB_HOST}:${DB_PORT}"
|
||||
|
||||
# pg_dump with compression — mirrors Chatwoot backup approach
|
||||
PGPASSWORD="${DB_PASS}" pg_dump \
|
||||
-h "${DB_HOST}" \
|
||||
-p "${DB_PORT}" \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
--format=custom \
|
||||
--compress=9 \
|
||||
| gzip > "${BACKUP_FILE}"
|
||||
|
||||
BACKUP_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1)
|
||||
echo "[$(date)] Backup complete: ${BACKUP_FILE} (${BACKUP_SIZE})"
|
||||
|
||||
# Prune old backups beyond retention period
|
||||
find "${BACKUP_DIR}" -name "*.sql.gz" -mtime +"${RETENTION_DAYS}" -delete
|
||||
echo "[$(date)] Pruned backups older than ${RETENTION_DAYS} days"
|
||||
|
||||
# Verify backup integrity
|
||||
gunzip -t "${BACKUP_FILE}" && echo "[$(date)] Backup integrity verified" || {
|
||||
echo "[$(date)] ERROR: Backup integrity check failed!"
|
||||
rm -f "${BACKUP_FILE}"
|
||||
exit 1
|
||||
[[ -d "$storage" ]] || { echo "storage path does not exist: $storage" >&2; exit 1; }
|
||||
[[ -z "$(find "$storage" -type l -print -quit)" ]] || { echo "storage must not contain symbolic links" >&2; exit 1; }
|
||||
[[ -f "$connector" ]] || { echo "connector backup does not exist: $connector" >&2; exit 1; }
|
||||
[[ -r "$passphrase_file" ]] || { echo "backup passphrase file is not readable" >&2; exit 1; }
|
||||
server_major=$(($(psql "$dsn" -Atqc "SHOW server_version_num") / 10000))
|
||||
client_major=$(pg_dump --version | awk '{print $NF}' | cut -d. -f1)
|
||||
[[ "$client_major" == "$server_major" ]] || {
|
||||
echo "pg_dump major $client_major must match PostgreSQL major $server_major" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
install -d -m 0700 "$backup_dir" "$offsite_dir"
|
||||
[[ "$(realpath "$backup_dir")" != "$(realpath "$offsite_dir")" ]] || {
|
||||
echo "off-site directory must use a different path/failure domain" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
work_dir=$(mktemp -d "$backup_dir/.backup.XXXXXX")
|
||||
trap 'rm -rf "$work_dir"' EXIT
|
||||
stage=$work_dir/snapshot
|
||||
mkdir "$stage"
|
||||
|
||||
pg_dump --dbname "$dsn" --format=custom --compress=9 --file "$stage/postgres.dump"
|
||||
pg_restore --list "$stage/postgres.dump" >/dev/null
|
||||
tar -C "$storage" -cf "$stage/attachments.tar" .
|
||||
cp "$connector" "$stage/connector.db"
|
||||
|
||||
created_at_epoch=$(date -u +%s)
|
||||
{
|
||||
echo "created_at=$timestamp"
|
||||
echo "created_at_epoch=$created_at_epoch"
|
||||
echo "version=$version"
|
||||
echo "rpo_target_seconds=86400"
|
||||
} >"$stage/manifest"
|
||||
(cd "$stage" && sha256sum postgres.dump attachments.tar connector.db manifest >SHA256SUMS)
|
||||
|
||||
bundle=$backup_dir/gochat-$timestamp.tar.enc
|
||||
tar -C "$stage" -cf - . | openssl enc -aes-256-cbc -pbkdf2 -salt -pass "file:$passphrase_file" -out "$bundle"
|
||||
(cd "$backup_dir" && sha256sum "$(basename "$bundle")" >"$(basename "$bundle").sha256")
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass "file:$passphrase_file" -in "$bundle" | tar -tf - >/dev/null
|
||||
|
||||
cp "$bundle" "$bundle.sha256" "$offsite_dir/"
|
||||
find "$backup_dir" "$offsite_dir" -maxdepth 1 -type f -name 'gochat-*.tar.enc*' -mtime "+$retention_days" -delete
|
||||
|
||||
echo "backup=$bundle offsite=$offsite_dir/$(basename "$bundle") version=$version created_at=$timestamp"
|
||||
|
||||
Reference in New Issue
Block a user