gochat/backend/configs/prometheus_alerts_test.yml

rule_files:
  - prometheus_alerts.yml

evaluation_interval: 1m

tests:
  - name: HTTP error rate only fires above five percent
    interval: 1m
    input_series:
      - series: 'http_requests_total{job="gochat",instance="below-threshold"}'
        values: '0+100x12'
      - series: 'http_request_errors_total{job="gochat",instance="below-threshold"}'
        values: '0+5x12'
      - series: 'http_requests_total{job="gochat",instance="above-threshold"}'
        values: '0+100x12'
      - series: 'http_request_errors_total{job="gochat",instance="above-threshold"}'
        values: '0+6x12'
    alert_rule_test:
      - eval_time: 10m
        alertname: GoChatHighErrorRate
        exp_alerts:
          - exp_labels:
              instance: above-threshold
              job: gochat
              severity: warning
            exp_annotations:
              summary: GoChat error rate above 5%
              description: Error rate is 6% over the last 5 minutes.

  - name: Dependency failures page after one minute
    interval: 1m
    input_series:
      - series: 'probe_success{job="gochat-database-readiness",instance="database"}'
        values: '0x5'
      - series: 'probe_success{job="gochat-redis-readiness",instance="redis"}'
        values: '0x5'
      - series: 'up{job="gochat",instance="gochat:3000"}'
        values: '0x5'
      - series: 'up{job="alertmanager",instance="alertmanager:9093"}'
        values: '0x5'
    alert_rule_test:
      - eval_time: 2m
        alertname: GoChatDatabaseReadinessFailed
        exp_alerts:
          - exp_labels:
              instance: database
              job: gochat-database-readiness
              severity: critical
            exp_annotations:
              summary: GoChat database readiness failed
              description: The application cannot complete its PostgreSQL dependency check.
      - eval_time: 2m
        alertname: GoChatRedisReadinessFailed
        exp_alerts:
          - exp_labels:
              instance: redis
              job: gochat-redis-readiness
              severity: critical
            exp_annotations:
              summary: GoChat Redis readiness failed
              description: The application cannot complete its Redis dependency check.
      - eval_time: 2m
        alertname: GoChatAppDown
        exp_alerts:
          - exp_labels:
              instance: gochat:3000
              job: gochat
              severity: critical
            exp_annotations:
              summary: GoChat application is down
              description: GoChat metrics have been unreachable for more than one minute.
      - eval_time: 2m
        alertname: GoChatAlertmanagerDown
        exp_alerts:
          - exp_labels:
              instance: alertmanager:9093
              job: alertmanager
              severity: critical
            exp_annotations:
              summary: Alertmanager is down
              description: Prometheus cannot deliver notifications to Alertmanager.

  - name: Worker and critical queues page
    interval: 1m
    input_series:
      - series: 'container_last_seen{container_label_com_docker_compose_service="worker"}'
        values: '0x10'
      - series: 'gochat_background_jobs_total{queue="critical",status="queued"}'
        values: '30x10'
      - series: 'gochat_background_jobs_oldest_seconds{queue="critical",status="queued"}'
        values: '600x10'
      - series: 'gochat_background_jobs_oldest_seconds{queue="default",status="running"}'
        values: '1000x10'
    alert_rule_test:
      - eval_time: 6m
        alertname: GoChatWorkerDown
        exp_alerts:
          - exp_labels:
              severity: critical
            exp_annotations:
              summary: GoChat worker is down
              description: cAdvisor has not observed a production worker container for more than one minute.
      - eval_time: 6m
        alertname: GoChatCriticalQueueBacklog
        exp_alerts:
          - exp_labels:
              queue: critical
              severity: critical
            exp_annotations:
              summary: Critical background queue is delayed
              description: Queue critical exceeds 25 ready jobs or its oldest job is over five minutes old.
      - eval_time: 6m
        alertname: GoChatWorkerJobsStuck
        exp_alerts:
          - exp_labels:
              queue: default
              severity: critical
            exp_annotations:
              summary: Background jobs are stuck
              description: Queue default has a running job older than 15 minutes.

  - name: Backup RPO pages and clears
    interval: 1h
    input_series:
      - series: gochat_backup_last_success_timestamp_seconds
        values: '0x4'
      - series: gochat_backup_rpo_target_seconds
        values: '3600x4'
    alert_rule_test:
      - eval_time: 2h
        alertname: GoChatBackupStale
        exp_alerts:
          - exp_labels:
              severity: critical
            exp_annotations:
              summary: GoChat backup is stale
              description: No successful encrypted off-site backup exists inside the declared RPO.

  - name: Fresh backup does not page
    interval: 1h
    input_series:
      - series: gochat_backup_last_success_timestamp_seconds
        values: '7000x4'
      - series: gochat_backup_rpo_target_seconds
        values: '86400x4'
    alert_rule_test:
      - eval_time: 2h
        alertname: GoChatBackupStale
        exp_alerts: []