From 43253b51c97597e570a2543bfd4d54a4f1536e05 Mon Sep 17 00:00:00 2001 From: Rogee Date: Sat, 22 Aug 2026 21:20:27 +0800 Subject: [PATCH] HH-460: cover migration and backup restore regressions (#104) * test(HH-460): cover migration and restore regressions * test(HH-460): verify real restore evidence --------- Co-authored-by: Rogee --- .../captain_binding_migration_test.go | 13 +- backend/scripts/db_backup_test.sh | 111 +++++++++++++++--- 2 files changed, 102 insertions(+), 22 deletions(-) diff --git a/backend/internal/database/captain_binding_migration_test.go b/backend/internal/database/captain_binding_migration_test.go index 02ee8ab7..55f0e24b 100644 --- a/backend/internal/database/captain_binding_migration_test.go +++ b/backend/internal/database/captain_binding_migration_test.go @@ -61,6 +61,7 @@ func TestCaptainBindingMigrationDeduplicatesAndRejectsDown(t *testing.T) { assert.Equal(t, int64(1), count, table) } for table, column := range map[string]string{ + "agent_bot_inboxes": "agent_bot_id", "conversations": "assignee_agent_bot_id", "agent_bot_presence_events": "agent_bot_id", "bot_rules": "agent_bot_id", @@ -70,12 +71,14 @@ func TestCaptainBindingMigrationDeduplicatesAndRejectsDown(t *testing.T) { require.NoError(t, db.Table(table).Select(column).Scan(&botID).Error) assert.Equal(t, int64(10), botID, table) } - for table, index := range map[string]string{ - "agent_bots": "idx_agent_bots_captain_assistant", - "agent_bot_inboxes": "idx_agent_bot_inboxes_bot_inbox", - "captain_inboxes": "idx_captain_inboxes_active_inbox", + for _, index := range []string{ + "idx_agent_bots_captain_assistant", + "idx_agent_bot_inboxes_bot_inbox", + "idx_captain_inboxes_active_inbox", } { - assert.True(t, db.Migrator().HasIndex(table, index), index) + var unique bool + require.NoError(t, db.Raw("SELECT indisunique FROM pg_index WHERE indexrelid = to_regclass(?)", index).Scan(&unique).Error) + assert.True(t, unique, index) } down, err := os.ReadFile(filepath.Join("..", "..", "migrations", "000079_make_captain_bindings_unique.down.sql")) diff --git a/backend/scripts/db_backup_test.sh b/backend/scripts/db_backup_test.sh index 81c5df47..ab788ee3 100755 --- a/backend/scripts/db_backup_test.sh +++ b/backend/scripts/db_backup_test.sh @@ -3,35 +3,96 @@ set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT -mkdir -p "$tmp/bin" "$tmp/scripts" "$tmp/storage" "$tmp/local" "$tmp/offsite" "$tmp/metrics" -cp "$script_dir/db_backup.sh" "$script_dir/../../deploy/docker/database_client_entrypoint.sh" "$tmp/scripts/" +postgres=gochat-backup-test-$$ +cleanup() { + docker rm -f "$postgres" >/dev/null 2>&1 || true + rm -rf "$tmp" +} +trap cleanup EXIT +mkdir -p "$tmp/bin" "$tmp/scripts" "$tmp/storage" "$tmp/restored/storage" "$tmp/local" "$tmp/offsite" "$tmp/metrics" +cp "$script_dir/db_backup.sh" "$script_dir/db_restore.sh" "$script_dir/../../deploy/docker/database_client_entrypoint.sh" "$tmp/scripts/" printf 'attachment\n' >"$tmp/storage/file.txt" printf 'connector\n' >"$tmp/connector.db" printf 'test-passphrase\n' >"$tmp/passphrase" +postgres_image='pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b' +docker run -d --name "$postgres" -e POSTGRES_PASSWORD=postgres "$postgres_image" >/dev/null +for _ in $(seq 1 30); do + docker exec "$postgres" pg_isready -U postgres >/dev/null 2>&1 && break + sleep 1 +done +docker exec "$postgres" pg_isready -U postgres >/dev/null +docker exec -i "$postgres" psql -v ON_ERROR_STOP=1 -U postgres <<'SQL' +CREATE DATABASE gochat_source; +CREATE DATABASE gochat_target; +SQL +docker exec -i "$postgres" psql -v ON_ERROR_STOP=1 -U postgres -d gochat_source <<'SQL' +CREATE TABLE schema_migrations (version BIGINT NOT NULL, dirty BOOLEAN NOT NULL); +CREATE TABLE accounts (id BIGINT PRIMARY KEY, deleted_at TIMESTAMPTZ); +CREATE TABLE attachments (id BIGINT PRIMARY KEY, deleted_at TIMESTAMPTZ); +INSERT INTO schema_migrations VALUES (79, false), (80, true); +INSERT INTO accounts VALUES (1, NULL), (2, now()); +INSERT INTO attachments VALUES (1, NULL), (2, now()); +SQL + +source_dsn='postgres://postgres:postgres@source.example.test/gochat_source?sslmode=verify-full' +target_dsn='postgres://postgres:postgres@target.example.test/gochat_target?sslmode=verify-full' +export GOCHAT_TEST_POSTGRES=$postgres +export GOCHAT_TEST_SOURCE_DSN=$source_dsn +export GOCHAT_TEST_TARGET_DSN=$target_dsn +export GOCHAT_TEST_TMP=$tmp +export GOCHAT_TEST_STORAGE="$tmp/restored/storage" +export GOCHAT_TEST_CONNECTOR="$tmp/restored/connector.db" +export GOCHAT_TEST_EVENTS="$tmp/restore-events" + cat >"$tmp/bin/psql" <<'EOF' #!/usr/bin/env bash -echo 160000 +set -euo pipefail +dsn=$1 +shift +case $dsn in + "$GOCHAT_TEST_SOURCE_DSN") database=gochat_source ;; + "$GOCHAT_TEST_TARGET_DSN") database=gochat_target ;; + *) echo "unexpected psql target: $dsn" >&2; exit 1 ;; +esac +exec docker exec "$GOCHAT_TEST_POSTGRES" psql -U postgres -d "$database" "$@" EOF cat >"$tmp/bin/pg_dump" <<'EOF' #!/usr/bin/env bash +set -euo pipefail if [[ $1 == --version ]]; then - echo 'pg_dump (PostgreSQL) 16.0' - exit + exec docker exec "$GOCHAT_TEST_POSTGRES" pg_dump --version fi -while (($#)); do - if [[ $1 == --file ]]; then - printf 'dump\n' >"$2" - exit - fi - shift -done -exit 1 +[[ $# -eq 6 && $1 == --dbname && $2 == "$GOCHAT_TEST_SOURCE_DSN" && $3 == --format=custom && $4 == --compress=9 && $5 == --file ]] +case $6 in + "$GOCHAT_TEST_TMP"/*/postgres.dump) ;; + *) echo "unexpected dump path: $6" >&2; exit 1 ;; +esac +docker exec "$GOCHAT_TEST_POSTGRES" pg_dump -U postgres --dbname gochat_source --format=custom --compress=9 >"$6" EOF cat >"$tmp/bin/pg_restore" <<'EOF' #!/usr/bin/env bash -exit 0 +set -euo pipefail +if [[ $1 == --version ]]; then + exec docker exec "$GOCHAT_TEST_POSTGRES" pg_restore --version +fi +if [[ $1 == --list ]]; then + [[ $# -eq 2 && -s $2 ]] + case $2 in + "$GOCHAT_TEST_TMP"/*/postgres.dump) ;; + *) echo "unexpected restore dump: $2" >&2; exit 1 ;; + esac + exec docker exec -i "$GOCHAT_TEST_POSTGRES" pg_restore --list <"$2" +fi +[[ $# -eq 6 && $1 == --exit-on-error && $2 == --no-owner && $3 == --no-privileges && $4 == --dbname && $5 == "$GOCHAT_TEST_TARGET_DSN" && -s $6 ]] +case $6 in + "$GOCHAT_TEST_TMP"/*/postgres.dump) ;; + *) echo "unexpected restore dump: $6" >&2; exit 1 ;; +esac +[[ -z $(find "$GOCHAT_TEST_STORAGE" -mindepth 1 -print -quit) ]] +[[ ! -e $GOCHAT_TEST_CONNECTOR ]] +printf 'database\n' >>"$GOCHAT_TEST_EVENTS" +exec docker exec -i "$GOCHAT_TEST_POSTGRES" pg_restore -U postgres --exit-on-error --no-owner --no-privileges --dbname gochat_target <"$6" EOF chmod +x "$tmp/bin/psql" "$tmp/bin/pg_dump" "$tmp/bin/pg_restore" @@ -49,17 +110,33 @@ while IFS='|' read -r name dsn; do done < "$script_dir/../../deploy/docker/database_host_rejection_cases.txt" PATH="$tmp/bin:$PATH" \ -GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=verify-full' \ +GOCHAT_DATABASE_DSN="$source_dsn" \ GOCHAT_STORAGE_PATH="$tmp/storage" \ GOCHAT_CONNECTOR_BACKUP_FILE="$tmp/connector.db" \ GOCHAT_BACKUP_DIR="$tmp/local" \ GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite" \ GOCHAT_BACKUP_PASSPHRASE_FILE="$tmp/passphrase" \ GOCHAT_BACKUP_METRICS_FILE="$tmp/metrics/gochat_backup.prom" \ +GOCHAT_VERSION='test-version' \ "$tmp/scripts/db_backup.sh" >"$tmp/output" grep -Eq '^gochat_backup_last_success_timestamp_seconds [0-9]+$' "$tmp/metrics/gochat_backup.prom" grep -Fx 'gochat_backup_rpo_target_seconds 86400' "$tmp/metrics/gochat_backup.prom" >/dev/null test "$(find "$tmp/offsite" -name 'gochat-*.tar.enc' | wc -l)" -eq 1 grep -F 'backup=' "$tmp/output" >/dev/null -echo 'backup metric test passed' + +bundle=$(find "$tmp/offsite" -name 'gochat-*.tar.enc' -print -quit) +PATH="$tmp/bin:$PATH" \ +GOCHAT_DATABASE_DSN="$target_dsn" \ +GOCHAT_STORAGE_PATH="$tmp/restored/storage" \ +GOCHAT_CONNECTOR_DB_PATH="$tmp/restored/connector.db" \ +GOCHAT_BACKUP_PASSPHRASE_FILE="$tmp/passphrase" \ + "$tmp/scripts/db_restore.sh" "$bundle" >"$tmp/restore-output" + +cmp "$tmp/storage/file.txt" "$tmp/restored/storage/file.txt" +printf 'attachments\n' >>"$tmp/restore-events" +cmp "$tmp/connector.db" "$tmp/restored/connector.db" +printf 'connector\n' >>"$tmp/restore-events" +printf 'database\nattachments\nconnector\n' | cmp - "$tmp/restore-events" +grep -F 'restore=ok version=test-version migration=79 accounts=1 attachments=1' "$tmp/restore-output" >/dev/null +echo 'backup and empty-target restore test passed'