HH-445: deploy production observability and runbooks (#96)
* HH-445: deploy production observability and runbooks * fix(ops): share production database DSN * fix(HH-445): enforce database TLS gate * fix(HH-445): preserve production serve command * fix(prod): require external database dependencies * fix(prod): unify database host rejection gates * test(prod): enforce exact database TLS runbook contract --------- Co-authored-by: Rogee <rogee@ipao.vip>
This commit is contained in:
Rogee
@@ -32,6 +32,12 @@ func main() {
|
||||
}
|
||||
|
||||
dbURL := cfg.Database.MigrateDSN()
|
||||
if cfg.Server.Mode == "release" {
|
||||
if err := config.ValidateProductionDatabaseDSN(dbURL); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error validating database DSN: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
migrationsPath := cfg.Database.GetMigrationsPath()
|
||||
|
||||
switch command {
|
||||
|
||||
@@ -1,84 +1,159 @@
|
||||
# GoChat Prometheus Alert Rules
|
||||
# Reference: Chatwoot production monitoring with Sidekiq queue alerts
|
||||
# Adjust thresholds based on your deployment scale
|
||||
|
||||
groups:
|
||||
- name: gochat-app
|
||||
- name: gochat-application
|
||||
rules:
|
||||
# Application down
|
||||
- alert: GoChatAppDown
|
||||
expr: up{job="gochat"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "GoChat application is down"
|
||||
description: "GoChat instance {{ $labels.instance }} has been down for more than 1 minute."
|
||||
summary: GoChat application is down
|
||||
description: GoChat metrics have been unreachable for more than one minute.
|
||||
|
||||
- alert: GoChatDatabaseReadinessFailed
|
||||
expr: probe_success{job="gochat-database-readiness"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: GoChat database readiness failed
|
||||
description: The application cannot complete its PostgreSQL dependency check.
|
||||
|
||||
- alert: GoChatRedisReadinessFailed
|
||||
expr: probe_success{job="gochat-redis-readiness"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: GoChat Redis readiness failed
|
||||
description: The application cannot complete its Redis dependency check.
|
||||
|
||||
# High error rate
|
||||
- alert: GoChatHighErrorRate
|
||||
expr: sum by (job, instance) (rate(http_requests_total{job="gochat", status=~"5.."}[5m])) / sum by (job, instance) (rate(http_requests_total{job="gochat"}[5m])) > 0.05
|
||||
expr: |
|
||||
sum by (job, instance) (rate(http_request_errors_total{job="gochat"}[5m]))
|
||||
/ sum by (job, instance) (rate(http_requests_total{job="gochat"}[5m])) > 0.05
|
||||
and sum by (job, instance) (rate(http_requests_total{job="gochat"}[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "GoChat error rate above 5%"
|
||||
description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
|
||||
summary: GoChat error rate above 5%
|
||||
description: Error rate is {{ $value | humanizePercentage }} over the last 5 minutes.
|
||||
|
||||
# High memory usage
|
||||
- alert: GoChatHighMemory
|
||||
expr: gochat_go_memory_alloc_bytes / (1024 * 1024) > 400
|
||||
expr: gochat_go_memory_alloc_bytes / 1024 / 1024 > 400
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "GoChat memory usage above 400MB"
|
||||
description: "Memory allocation is {{ $value }}MB."
|
||||
summary: GoChat memory usage above 400 MB
|
||||
description: Allocated heap is {{ $value | humanize }} MB.
|
||||
|
||||
# Too many goroutines
|
||||
- alert: GoChatHighGoroutines
|
||||
expr: gochat_go_goroutines > 1000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "GoChat goroutine count above 1000"
|
||||
description: "{{ $value }} goroutines running."
|
||||
summary: GoChat goroutine count above 1000
|
||||
description: GoChat has {{ $value | humanize }} goroutines.
|
||||
|
||||
- name: gochat-infra
|
||||
- name: gochat-workers
|
||||
rules:
|
||||
# PostgreSQL down
|
||||
- alert: GoChatPostgresDown
|
||||
expr: up{job="gochat-postgres"} == 0
|
||||
- alert: GoChatWorkerDown
|
||||
expr: |
|
||||
time() - max(container_last_seen{container_label_com_docker_compose_service="worker"}) > 60
|
||||
or absent(container_last_seen{container_label_com_docker_compose_service="worker"})
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PostgreSQL is down"
|
||||
summary: GoChat worker is down
|
||||
description: cAdvisor has not observed a production worker container for more than one minute.
|
||||
|
||||
# Redis down
|
||||
- alert: GoChatRedisDown
|
||||
expr: up{job="gochat-redis"} == 0
|
||||
- alert: GoChatCriticalQueueBacklog
|
||||
expr: |
|
||||
sum by (queue) (gochat_background_jobs_total{queue=~"critical|high",status=~"queued|retrying"}) > 25
|
||||
or max by (queue) (gochat_background_jobs_oldest_seconds{queue=~"critical|high",status=~"queued|retrying"}) > 300
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Critical background queue is delayed
|
||||
description: Queue {{ $labels.queue }} exceeds 25 ready jobs or its oldest job is over five minutes old.
|
||||
|
||||
- alert: GoChatWorkerJobsStuck
|
||||
expr: max by (queue) (gochat_background_jobs_oldest_seconds{status="running"}) > 900
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Background jobs are stuck
|
||||
description: Queue {{ $labels.queue }} has a running job older than 15 minutes.
|
||||
|
||||
- name: gochat-infrastructure
|
||||
rules:
|
||||
- alert: GoChatPostgresExporterDown
|
||||
expr: up{job="postgres-exporter"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Redis is down"
|
||||
summary: PostgreSQL exporter is down
|
||||
description: Prometheus cannot scrape the PostgreSQL exporter.
|
||||
|
||||
- alert: GoChatRedisExporterDown
|
||||
expr: up{job="redis-exporter"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis exporter is down
|
||||
description: Prometheus cannot scrape the Redis exporter.
|
||||
|
||||
# Redis memory approaching limit
|
||||
- alert: GoChatRedisMemoryHigh
|
||||
expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.8
|
||||
expr: redis_memory_max_bytes > 0 and redis_memory_used_bytes / redis_memory_max_bytes > 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Redis memory usage above 80%"
|
||||
summary: Redis memory usage above 80%
|
||||
description: Redis is approaching its configured memory ceiling.
|
||||
|
||||
# PostgreSQL connections exhausted
|
||||
- alert: GoChatPostgresConnectionsHigh
|
||||
expr: pg_stat_activity_count / pg_settings_max_connections > 0.8
|
||||
expr: sum(pg_stat_activity_count) / max(pg_settings_max_connections) > 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PostgreSQL connection usage above 80%"
|
||||
summary: PostgreSQL connection usage above 80%
|
||||
description: PostgreSQL is approaching its connection limit.
|
||||
|
||||
- alert: GoChatBackupStale
|
||||
expr: |
|
||||
time() - gochat_backup_last_success_timestamp_seconds > gochat_backup_rpo_target_seconds
|
||||
or absent(gochat_backup_last_success_timestamp_seconds)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: GoChat backup is stale
|
||||
description: No successful encrypted off-site backup exists inside the declared RPO.
|
||||
|
||||
- alert: GoChatAlertmanagerDown
|
||||
expr: up{job="alertmanager"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Alertmanager is down
|
||||
description: Prometheus cannot deliver notifications to Alertmanager.
|
||||
|
||||
- alert: ShangwutongReadinessFailed
|
||||
expr: probe_success{job="shangwutong-readiness"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Shangwutong readiness failed
|
||||
description: The production Connector readiness endpoint is failing.
|
||||
|
||||
@@ -4,15 +4,16 @@ rule_files:
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
- interval: 1m
|
||||
- name: HTTP error rate only fires above five percent
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'http_requests_total{job="gochat",instance="below-threshold",method="GET",route="/ok",status="200"}'
|
||||
values: '0+96x12'
|
||||
- series: 'http_requests_total{job="gochat",instance="below-threshold",method="GET",route="/error",status="500"}'
|
||||
values: '0+4x12'
|
||||
- series: 'http_requests_total{job="gochat",instance="above-threshold",method="GET",route="/ok",status="200"}'
|
||||
values: '0+94x12'
|
||||
- series: 'http_requests_total{job="gochat",instance="above-threshold",method="GET",route="/error",status="500"}'
|
||||
- series: 'http_requests_total{job="gochat",instance="below-threshold"}'
|
||||
values: '0+100x12'
|
||||
- series: 'http_request_errors_total{job="gochat",instance="below-threshold"}'
|
||||
values: '0+5x12'
|
||||
- series: 'http_requests_total{job="gochat",instance="above-threshold"}'
|
||||
values: '0+100x12'
|
||||
- series: 'http_request_errors_total{job="gochat",instance="above-threshold"}'
|
||||
values: '0+6x12'
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
@@ -25,3 +26,124 @@ tests:
|
||||
exp_annotations:
|
||||
summary: GoChat error rate above 5%
|
||||
description: Error rate is 6% over the last 5 minutes.
|
||||
|
||||
- name: Dependency failures page after one minute
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="gochat-database-readiness",instance="database"}'
|
||||
values: '0x5'
|
||||
- series: 'probe_success{job="gochat-redis-readiness",instance="redis"}'
|
||||
values: '0x5'
|
||||
- series: 'up{job="gochat",instance="gochat:3000"}'
|
||||
values: '0x5'
|
||||
- series: 'up{job="alertmanager",instance="alertmanager:9093"}'
|
||||
values: '0x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 2m
|
||||
alertname: GoChatDatabaseReadinessFailed
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
instance: database
|
||||
job: gochat-database-readiness
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: GoChat database readiness failed
|
||||
description: The application cannot complete its PostgreSQL dependency check.
|
||||
- eval_time: 2m
|
||||
alertname: GoChatRedisReadinessFailed
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
instance: redis
|
||||
job: gochat-redis-readiness
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: GoChat Redis readiness failed
|
||||
description: The application cannot complete its Redis dependency check.
|
||||
- eval_time: 2m
|
||||
alertname: GoChatAppDown
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
instance: gochat:3000
|
||||
job: gochat
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: GoChat application is down
|
||||
description: GoChat metrics have been unreachable for more than one minute.
|
||||
- eval_time: 2m
|
||||
alertname: GoChatAlertmanagerDown
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
instance: alertmanager:9093
|
||||
job: alertmanager
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: Alertmanager is down
|
||||
description: Prometheus cannot deliver notifications to Alertmanager.
|
||||
|
||||
- name: Worker and critical queues page
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'container_last_seen{container_label_com_docker_compose_service="worker"}'
|
||||
values: '0x10'
|
||||
- series: 'gochat_background_jobs_total{queue="critical",status="queued"}'
|
||||
values: '30x10'
|
||||
- series: 'gochat_background_jobs_oldest_seconds{queue="critical",status="queued"}'
|
||||
values: '600x10'
|
||||
- series: 'gochat_background_jobs_oldest_seconds{queue="default",status="running"}'
|
||||
values: '1000x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 6m
|
||||
alertname: GoChatWorkerDown
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: GoChat worker is down
|
||||
description: cAdvisor has not observed a production worker container for more than one minute.
|
||||
- eval_time: 6m
|
||||
alertname: GoChatCriticalQueueBacklog
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
queue: critical
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: Critical background queue is delayed
|
||||
description: Queue critical exceeds 25 ready jobs or its oldest job is over five minutes old.
|
||||
- eval_time: 6m
|
||||
alertname: GoChatWorkerJobsStuck
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
queue: default
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: Background jobs are stuck
|
||||
description: Queue default has a running job older than 15 minutes.
|
||||
|
||||
- name: Backup RPO pages and clears
|
||||
interval: 1h
|
||||
input_series:
|
||||
- series: gochat_backup_last_success_timestamp_seconds
|
||||
values: '0x4'
|
||||
- series: gochat_backup_rpo_target_seconds
|
||||
values: '3600x4'
|
||||
alert_rule_test:
|
||||
- eval_time: 2h
|
||||
alertname: GoChatBackupStale
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: GoChat backup is stale
|
||||
description: No successful encrypted off-site backup exists inside the declared RPO.
|
||||
|
||||
- name: Fresh backup does not page
|
||||
interval: 1h
|
||||
input_series:
|
||||
- series: gochat_backup_last_success_timestamp_seconds
|
||||
values: '7000x4'
|
||||
- series: gochat_backup_rpo_target_seconds
|
||||
values: '86400x4'
|
||||
alert_rule_test:
|
||||
- eval_time: 2h
|
||||
alertname: GoChatBackupStale
|
||||
exp_alerts: []
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -292,15 +294,18 @@ func TestValidate_ReleaseDatabaseTLS(t *testing.T) {
|
||||
}{
|
||||
{"external disable", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=disable", true},
|
||||
{"external missing sslmode", "postgres://gochat:database-secret@db.example.test:5432/gochat", true},
|
||||
{"external duplicate downgrade", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=disable&sslmode=verify-full", true},
|
||||
{"external duplicate allowed", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full&sslmode=verify-full", true},
|
||||
{"external non-fixed certificate path", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/tmp/ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key", true},
|
||||
{"external require", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=require", true},
|
||||
{"external verify ca", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-ca", false},
|
||||
{"external verify full", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full", false},
|
||||
{"built-in compose disable", "postgres://gochat:database-secret@postgres:5432/gochat?sslmode=disable", true},
|
||||
{"external fixed certificate paths", "postgres://gochat:database-secret@db.example.test:5432/gochat?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key", false},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg.Database.DSN = tt.dsn
|
||||
if tt.wantErr {
|
||||
assert.ErrorContains(t, Validate(cfg), "production database DSN must use sslmode")
|
||||
assert.Error(t, Validate(cfg))
|
||||
} else {
|
||||
assert.NoError(t, Validate(cfg))
|
||||
}
|
||||
@@ -308,6 +313,32 @@ func TestValidate_ReleaseDatabaseTLS(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateProductionDatabaseDSN_RejectsHostMatrix(t *testing.T) {
|
||||
file, err := os.Open("../../../deploy/docker/database_host_rejection_cases.txt")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, file.Close()) })
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
name, dsn, ok := strings.Cut(scanner.Text(), "|")
|
||||
require.True(t, ok)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert.ErrorContains(t, ValidateProductionDatabaseDSN(dsn), "must use an external PostgreSQL host")
|
||||
})
|
||||
}
|
||||
require.NoError(t, scanner.Err())
|
||||
}
|
||||
|
||||
func TestProductionDatabaseTLSRunbookContract(t *testing.T) {
|
||||
runbook, err := os.ReadFile("../../../docs/ops/02-production-operations.md")
|
||||
require.NoError(t, err)
|
||||
runbookText := string(runbook)
|
||||
assert.Equal(t, 1, strings.Count(runbookText, "`sslmode=verify-ca|verify-full`"))
|
||||
assert.Equal(t, 1, strings.Count(runbookText, "sslmode="))
|
||||
assert.NotContains(t, runbookText, "sslmode=disable")
|
||||
assert.NotContains(t, runbookText, "sslmode=require")
|
||||
}
|
||||
|
||||
func TestLoadWithEnv_ProductionRequiresOverlay(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "configs"), 0o755))
|
||||
|
||||
@@ -125,9 +125,8 @@ func Validate(cfg *Config) error {
|
||||
if password, ok := dbURL.User.Password(); !ok || password == "" || containsPlaceholder(password) {
|
||||
return fmt.Errorf("production database password is required and must not contain placeholders")
|
||||
}
|
||||
sslMode := dbURL.Query().Get("sslmode")
|
||||
if sslMode != "verify-full" && sslMode != "verify-ca" {
|
||||
return fmt.Errorf("production database DSN must use sslmode=verify-full or verify-ca")
|
||||
if err := ValidateProductionDatabaseDSN(cfg.Database.DSN); err != nil {
|
||||
return err
|
||||
}
|
||||
if redisURL.User == nil {
|
||||
return fmt.Errorf("production Redis credentials are required")
|
||||
@@ -183,6 +182,53 @@ func Validate(cfg *Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateProductionDatabaseDSN protects every direct Go database client from
|
||||
// local targets, duplicate sslmode downgrades, and unsafe certificate paths.
|
||||
func ValidateProductionDatabaseDSN(dsn string) error {
|
||||
dbURL, err := url.Parse(dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid production database DSN: %w", err)
|
||||
}
|
||||
hostname := strings.TrimSuffix(strings.ToLower(dbURL.Hostname()), ".")
|
||||
if zone := strings.LastIndexByte(hostname, '%'); zone >= 0 {
|
||||
hostname = hostname[:zone]
|
||||
}
|
||||
ip := net.ParseIP(hostname)
|
||||
if hostname == "" || hostname == "postgres" || hostname == "localhost" || ip != nil && ip.IsLoopback() {
|
||||
return fmt.Errorf("production database DSN must use an external PostgreSQL host")
|
||||
}
|
||||
query, err := url.ParseQuery(dbURL.RawQuery)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid production database DSN query: %w", err)
|
||||
}
|
||||
modes := query["sslmode"]
|
||||
if len(modes) != 1 {
|
||||
return fmt.Errorf("production database DSN must contain exactly one sslmode")
|
||||
}
|
||||
if modes[0] != "verify-ca" && modes[0] != "verify-full" {
|
||||
return fmt.Errorf("production database DSN must use sslmode=verify-full or verify-ca")
|
||||
}
|
||||
|
||||
fixedPaths := map[string]string{
|
||||
"sslrootcert": "/run/secrets/external-db-ca.crt",
|
||||
"sslcert": "/run/secrets/external-db-client.crt",
|
||||
"sslkey": "/run/secrets/external-db-client.key",
|
||||
}
|
||||
usesCertificateFiles := false
|
||||
for parameter := range fixedPaths {
|
||||
usesCertificateFiles = usesCertificateFiles || len(query[parameter]) > 0
|
||||
}
|
||||
if usesCertificateFiles {
|
||||
for parameter, path := range fixedPaths {
|
||||
values := query[parameter]
|
||||
if len(values) != 1 || values[0] != path {
|
||||
return fmt.Errorf("production database DSN %s must appear exactly once and use %s", parameter, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsPlaceholder(value string) bool {
|
||||
value = strings.ToLower(value)
|
||||
return strings.Contains(value, "change_me") || strings.Contains(value, "change-me") || strings.Contains(value, "changeme")
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
"$script_dir/database_client_entrypoint.sh" --check
|
||||
|
||||
dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
|
||||
storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
|
||||
connector=${GOCHAT_CONNECTOR_BACKUP_FILE:?GOCHAT_CONNECTOR_BACKUP_FILE is required}
|
||||
@@ -12,6 +15,7 @@ backup_dir=${GOCHAT_BACKUP_DIR:-/var/backups/gochat}
|
||||
offsite_dir=${GOCHAT_BACKUP_OFFSITE_DIR:?GOCHAT_BACKUP_OFFSITE_DIR is required}
|
||||
passphrase_file=${GOCHAT_BACKUP_PASSPHRASE_FILE:?GOCHAT_BACKUP_PASSPHRASE_FILE is required}
|
||||
retention_days=${GOCHAT_BACKUP_RETENTION_DAYS:-30}
|
||||
metrics_file=${GOCHAT_BACKUP_METRICS_FILE:-}
|
||||
version=${GOCHAT_VERSION:-unknown}
|
||||
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
|
||||
@@ -59,4 +63,19 @@ openssl enc -d -aes-256-cbc -pbkdf2 -pass "file:$passphrase_file" -in "$bundle"
|
||||
cp "$bundle" "$bundle.sha256" "$offsite_dir/"
|
||||
find "$backup_dir" "$offsite_dir" -maxdepth 1 -type f -name 'gochat-*.tar.enc*' -mtime "+$retention_days" -delete
|
||||
|
||||
if [[ -n $metrics_file ]]; then
|
||||
install -d -m 0755 "$(dirname "$metrics_file")"
|
||||
metrics_tmp=$metrics_file.tmp
|
||||
{
|
||||
echo '# HELP gochat_backup_last_success_timestamp_seconds Unix time of the last verified off-site backup.'
|
||||
echo '# TYPE gochat_backup_last_success_timestamp_seconds gauge'
|
||||
echo "gochat_backup_last_success_timestamp_seconds $created_at_epoch"
|
||||
echo '# HELP gochat_backup_rpo_target_seconds Maximum allowed age of the latest backup.'
|
||||
echo '# TYPE gochat_backup_rpo_target_seconds gauge'
|
||||
echo 'gochat_backup_rpo_target_seconds 86400'
|
||||
} >"$metrics_tmp"
|
||||
chmod 0644 "$metrics_tmp"
|
||||
mv "$metrics_tmp" "$metrics_file"
|
||||
fi
|
||||
|
||||
echo "backup=$bundle offsite=$offsite_dir/$(basename "$bundle") version=$version created_at=$timestamp"
|
||||
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
mkdir -p "$tmp/bin" "$tmp/scripts" "$tmp/storage" "$tmp/local" "$tmp/offsite" "$tmp/metrics"
|
||||
cp "$script_dir/db_backup.sh" "$script_dir/../../deploy/docker/database_client_entrypoint.sh" "$tmp/scripts/"
|
||||
printf 'attachment\n' >"$tmp/storage/file.txt"
|
||||
printf 'connector\n' >"$tmp/connector.db"
|
||||
printf 'test-passphrase\n' >"$tmp/passphrase"
|
||||
|
||||
cat >"$tmp/bin/psql" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo 160000
|
||||
EOF
|
||||
cat >"$tmp/bin/pg_dump" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
if [[ $1 == --version ]]; then
|
||||
echo 'pg_dump (PostgreSQL) 16.0'
|
||||
exit
|
||||
fi
|
||||
while (($#)); do
|
||||
if [[ $1 == --file ]]; then
|
||||
printf 'dump\n' >"$2"
|
||||
exit
|
||||
fi
|
||||
shift
|
||||
done
|
||||
exit 1
|
||||
EOF
|
||||
cat >"$tmp/bin/pg_restore" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$tmp/bin/psql" "$tmp/bin/pg_dump" "$tmp/bin/pg_restore"
|
||||
|
||||
if GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=disable' "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
|
||||
echo 'backup gate accepted sslmode=disable' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'sslmode must be verify-ca or verify-full' "$tmp/rejected" >/dev/null
|
||||
while IFS='|' read -r name dsn; do
|
||||
if GOCHAT_DATABASE_DSN=$dsn "$tmp/scripts/database_client_entrypoint.sh" --check >"$tmp/rejected" 2>&1; then
|
||||
echo "backup gate accepted $name" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'must use an external PostgreSQL host' "$tmp/rejected" >/dev/null
|
||||
done < "$script_dir/../../deploy/docker/database_host_rejection_cases.txt"
|
||||
|
||||
PATH="$tmp/bin:$PATH" \
|
||||
GOCHAT_DATABASE_DSN='postgres://test@db.example.test/test?sslmode=verify-full' \
|
||||
GOCHAT_STORAGE_PATH="$tmp/storage" \
|
||||
GOCHAT_CONNECTOR_BACKUP_FILE="$tmp/connector.db" \
|
||||
GOCHAT_BACKUP_DIR="$tmp/local" \
|
||||
GOCHAT_BACKUP_OFFSITE_DIR="$tmp/offsite" \
|
||||
GOCHAT_BACKUP_PASSPHRASE_FILE="$tmp/passphrase" \
|
||||
GOCHAT_BACKUP_METRICS_FILE="$tmp/metrics/gochat_backup.prom" \
|
||||
"$tmp/scripts/db_backup.sh" >"$tmp/output"
|
||||
|
||||
grep -Eq '^gochat_backup_last_success_timestamp_seconds [0-9]+$' "$tmp/metrics/gochat_backup.prom"
|
||||
grep -Fx 'gochat_backup_rpo_target_seconds 86400' "$tmp/metrics/gochat_backup.prom" >/dev/null
|
||||
test "$(find "$tmp/offsite" -name 'gochat-*.tar.enc' | wc -l)" -eq 1
|
||||
grep -F 'backup=' "$tmp/output" >/dev/null
|
||||
echo 'backup metric test passed'
|
||||
@@ -5,6 +5,9 @@
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
"$script_dir/database_client_entrypoint.sh" --check
|
||||
|
||||
bundle=${1:?usage: db_restore.sh /path/to/gochat-*.tar.enc}
|
||||
dsn=${GOCHAT_DATABASE_DSN:?GOCHAT_DATABASE_DSN is required}
|
||||
storage=${GOCHAT_STORAGE_PATH:?GOCHAT_STORAGE_PATH is required}
|
||||
|
||||
Reference in New Issue
Block a user