HH-441: harden durable storage and recovery (#92)
* HH-441: harden durable storage and recovery * HH-441: clear recovery review blockers * HH-441: enforce offsite backup failure domain --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
@@ -1,54 +1,62 @@
|
||||
#!/bin/bash
|
||||
# GoChat Database Backup Script
|
||||
# Reference: Chatwoot uses pg_dump for backup in production deployments
|
||||
# Usage: ./scripts/db_backup.sh [env]
|
||||
#!/usr/bin/env bash
|
||||
# Create one encrypted, off-site copy containing PostgreSQL, attachments, and
|
||||
# the Connector's online SQLite backup.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
ENV="${1:-production}"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
BACKUP_DIR="${GOCHAT_BACKUP_DIR:-/var/backups/gochat}"
|
||||
DSN="${GOCHAT_DATABASE_DSN:-}"
|
||||
RETENTION_DAYS="${GOCHAT_BACKUP_RETENTION_DAYS:-30}"
|
||||
dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
|
||||
storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
|
||||
connector=${GOCHAT_CONNECTOR_BACKUP_FILE:?GOCHAT_CONNECTOR_BACKUP_FILE is required}
|
||||
backup_dir=${GOCHAT_BACKUP_DIR:-/var/backups/gochat}
|
||||
offsite_dir=${GOCHAT_BACKUP_OFFSITE_DIR:?GOCHAT_BACKUP_OFFSITE_DIR is required}
|
||||
passphrase_file=${GOCHAT_BACKUP_PASSPHRASE_FILE:?GOCHAT_BACKUP_PASSPHRASE_FILE is required}
|
||||
retention_days=${GOCHAT_BACKUP_RETENTION_DAYS:-30}
|
||||
version=${GOCHAT_VERSION:-unknown}
|
||||
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
|
||||
if [[ -z "$DSN" ]]; then
|
||||
echo "[$(date)] ERROR: GOCHAT_DATABASE_DSN is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Parse DSN: postgres://user:password@host:port/dbname?sslmode=...
|
||||
DB_USER=$(echo "$DSN" | sed 's|.*://||; s|:.*||')
|
||||
DB_PASS=$(echo "$DSN" | sed 's|.*://[^:]*:||; s|@.*||')
|
||||
DB_HOST=$(echo "$DSN" | sed 's|.*@||; s|:.*||')
|
||||
DB_PORT=$(echo "$DSN" | sed 's|.*@.*:||; s|/.*||')
|
||||
DB_NAME=$(echo "$DSN" | sed 's|.*/||; s|\?.*||')
|
||||
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
|
||||
BACKUP_FILE="${BACKUP_DIR}/${DB_NAME}_${TIMESTAMP}.sql.gz"
|
||||
|
||||
echo "[$(date)] Starting backup of ${DB_NAME} on ${DB_HOST}:${DB_PORT}"
|
||||
|
||||
# pg_dump with compression — mirrors Chatwoot backup approach
|
||||
PGPASSWORD="${DB_PASS}" pg_dump \
|
||||
-h "${DB_HOST}" \
|
||||
-p "${DB_PORT}" \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
--format=custom \
|
||||
--compress=9 \
|
||||
| gzip > "${BACKUP_FILE}"
|
||||
|
||||
BACKUP_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1)
|
||||
echo "[$(date)] Backup complete: ${BACKUP_FILE} (${BACKUP_SIZE})"
|
||||
|
||||
# Prune old backups beyond retention period
|
||||
find "${BACKUP_DIR}" -name "*.sql.gz" -mtime +"${RETENTION_DAYS}" -delete
|
||||
echo "[$(date)] Pruned backups older than ${RETENTION_DAYS} days"
|
||||
|
||||
# Verify backup integrity
|
||||
gunzip -t "${BACKUP_FILE}" && echo "[$(date)] Backup integrity verified" || {
|
||||
echo "[$(date)] ERROR: Backup integrity check failed!"
|
||||
rm -f "${BACKUP_FILE}"
|
||||
exit 1
|
||||
[[ -d "$storage" ]] || { echo "storage path does not exist: $storage" >&2; exit 1; }
|
||||
[[ -z "$(find "$storage" -type l -print -quit)" ]] || { echo "storage must not contain symbolic links" >&2; exit 1; }
|
||||
[[ -f "$connector" ]] || { echo "connector backup does not exist: $connector" >&2; exit 1; }
|
||||
[[ -r "$passphrase_file" ]] || { echo "backup passphrase file is not readable" >&2; exit 1; }
|
||||
server_major=$(($(psql "$dsn" -Atqc "SHOW server_version_num") / 10000))
|
||||
client_major=$(pg_dump --version | awk '{print $NF}' | cut -d. -f1)
|
||||
[[ "$client_major" == "$server_major" ]] || {
|
||||
echo "pg_dump major $client_major must match PostgreSQL major $server_major" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
install -d -m 0700 "$backup_dir" "$offsite_dir"
|
||||
[[ "$(realpath "$backup_dir")" != "$(realpath "$offsite_dir")" ]] || {
|
||||
echo "off-site directory must use a different path/failure domain" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
work_dir=$(mktemp -d "$backup_dir/.backup.XXXXXX")
|
||||
trap 'rm -rf "$work_dir"' EXIT
|
||||
stage=$work_dir/snapshot
|
||||
mkdir "$stage"
|
||||
|
||||
pg_dump --dbname "$dsn" --format=custom --compress=9 --file "$stage/postgres.dump"
|
||||
pg_restore --list "$stage/postgres.dump" >/dev/null
|
||||
tar -C "$storage" -cf "$stage/attachments.tar" .
|
||||
cp "$connector" "$stage/connector.db"
|
||||
|
||||
created_at_epoch=$(date -u +%s)
|
||||
{
|
||||
echo "created_at=$timestamp"
|
||||
echo "created_at_epoch=$created_at_epoch"
|
||||
echo "version=$version"
|
||||
echo "rpo_target_seconds=86400"
|
||||
} >"$stage/manifest"
|
||||
(cd "$stage" && sha256sum postgres.dump attachments.tar connector.db manifest >SHA256SUMS)
|
||||
|
||||
bundle=$backup_dir/gochat-$timestamp.tar.enc
|
||||
tar -C "$stage" -cf - . | openssl enc -aes-256-cbc -pbkdf2 -salt -pass "file:$passphrase_file" -out "$bundle"
|
||||
(cd "$backup_dir" && sha256sum "$(basename "$bundle")" >"$(basename "$bundle").sha256")
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass "file:$passphrase_file" -in "$bundle" | tar -tf - >/dev/null
|
||||
|
||||
cp "$bundle" "$bundle.sha256" "$offsite_dir/"
|
||||
find "$backup_dir" "$offsite_dir" -maxdepth 1 -type f -name 'gochat-*.tar.enc*' -mtime "+$retention_days" -delete
|
||||
|
||||
echo "backup=$bundle offsite=$offsite_dir/$(basename "$bundle") version=$version created_at=$timestamp"
|
||||
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
# Restore an encrypted GoChat backup into an empty database/storage target and
|
||||
# print auditable RPO/RTO and business-integrity evidence.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
bundle=${1:?usage: db_restore.sh /path/to/gochat-*.tar.enc}
|
||||
dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
|
||||
storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
|
||||
connector=${GOCHAT_CONNECTOR_DB_PATH:?GOCHAT_CONNECTOR_DB_PATH is required}
|
||||
passphrase_file=${GOCHAT_BACKUP_PASSPHRASE_FILE:?GOCHAT_BACKUP_PASSPHRASE_FILE is required}
|
||||
started_at=$(date -u +%s)
|
||||
|
||||
[[ -f "$bundle" ]] || { echo "backup bundle does not exist: $bundle" >&2; exit 1; }
|
||||
[[ -f "$bundle.sha256" ]] || { echo "backup checksum does not exist: $bundle.sha256" >&2; exit 1; }
|
||||
(cd "$(dirname "$bundle")" && sha256sum -c "$(basename "$bundle").sha256")
|
||||
|
||||
server_major=$(($(psql "$dsn" -Atqc "SHOW server_version_num") / 10000))
|
||||
client_major=$(pg_restore --version | awk '{print $NF}' | cut -d. -f1)
|
||||
[[ "$client_major" == "$server_major" ]] || {
|
||||
echo "pg_restore major $client_major must match PostgreSQL major $server_major" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
table_count=$(psql "$dsn" -Atqc "SELECT count(*) FROM pg_tables WHERE schemaname = 'public'")
|
||||
[[ "$table_count" == "0" ]] || { echo "target database is not empty" >&2; exit 1; }
|
||||
install -d -m 0700 "$storage" "$(dirname "$connector")"
|
||||
[[ -z "$(find "$storage" -mindepth 1 -maxdepth 1 -print -quit)" ]] || { echo "target storage is not empty" >&2; exit 1; }
|
||||
[[ ! -e "$connector" ]] || { echo "target connector database already exists" >&2; exit 1; }
|
||||
|
||||
work_dir=$(mktemp -d "$(dirname "$bundle")/.restore.XXXXXX")
|
||||
trap 'rm -rf "$work_dir"' EXIT
|
||||
archive=$work_dir/snapshot.tar
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass "file:$passphrase_file" -in "$bundle" -out "$archive"
|
||||
while IFS= read -r path; do
|
||||
case "$path" in
|
||||
/*|../*|*/../*|*/..) echo "unsafe archive path: $path" >&2; exit 1 ;;
|
||||
esac
|
||||
done < <(tar -tf "$archive")
|
||||
if tar -tvf "$archive" | awk '{type = substr($1, 1, 1); if (type != "-" && type != "d") found = 1} END {exit(found ? 0 : 1)}'; then
|
||||
echo "backup archive contains unsupported entry types" >&2
|
||||
exit 1
|
||||
fi
|
||||
tar -C "$work_dir" -xf "$archive"
|
||||
rm "$archive"
|
||||
(cd "$work_dir" && sha256sum -c SHA256SUMS)
|
||||
pg_restore --list "$work_dir/postgres.dump" >/dev/null
|
||||
|
||||
pg_restore --exit-on-error --no-owner --no-privileges --dbname "$dsn" "$work_dir/postgres.dump"
|
||||
while IFS= read -r path; do
|
||||
case "$path" in
|
||||
/*|../*|*/../*|*/..) echo "unsafe attachment path: $path" >&2; exit 1 ;;
|
||||
esac
|
||||
done < <(tar -tf "$work_dir/attachments.tar")
|
||||
if tar -tvf "$work_dir/attachments.tar" | awk '{type = substr($1, 1, 1); if (type != "-" && type != "d") found = 1} END {exit(found ? 0 : 1)}'; then
|
||||
echo "attachment archive contains unsupported entry types" >&2
|
||||
exit 1
|
||||
fi
|
||||
tar -C "$storage" -xf "$work_dir/attachments.tar"
|
||||
install -m 0600 "$work_dir/connector.db" "$connector"
|
||||
|
||||
migration_version=$(psql "$dsn" -Atqc "SELECT COALESCE(MAX(version), 0) FROM schema_migrations WHERE NOT dirty")
|
||||
accounts=$(psql "$dsn" -Atqc "SELECT count(*) FROM accounts WHERE deleted_at IS NULL")
|
||||
attachments=$(psql "$dsn" -Atqc "SELECT count(*) FROM attachments WHERE deleted_at IS NULL")
|
||||
backup_epoch=$(awk -F= '$1 == "created_at_epoch" {print $2}' "$work_dir/manifest")
|
||||
version=$(awk -F= '$1 == "version" {print substr($0, index($0, "=") + 1)}' "$work_dir/manifest")
|
||||
finished_at=$(date -u +%s)
|
||||
|
||||
echo "restore=ok version=$version migration=$migration_version accounts=$accounts attachments=$attachments rpo_seconds=$((started_at-backup_epoch)) rto_seconds=$((finished_at-started_at))"
|
||||
Reference in New Issue
Block a user