name: GoChat CI/CD Pipeline

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]
    tags: ['v*.*.*']

permissions:
  contents: read

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  SHANGWUTONG_IMAGE_NAME: ${{ github.repository_owner }}/shangwutong
  GOSEC_VERSION: v2.28.0
  GOVULNCHECK_VERSION: v1.1.4
  TRIVY_VERSION: v0.74.0
  COSIGN_VERSION: v3.1.3
  CRANE_VERSION: v0.21.9

jobs:
  backend:
    name: Backend (SQLite)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.12.11"
      - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2
        id: chrome
        with:
          chrome-version: 152.0.7977.54
      - name: Install browser harness
        run: python -m pip install --require-hashes -r .github/requirements-browser-harness.txt
      - name: Test production observability configuration
        run: deploy/docker/observability_test.sh
      - name: Test quality gate failure contracts
        working-directory: backend
        env:
          GOCHAT_QUALITY_DIR: ${{ github.workspace }}/backend/quality/sqlite/failure-injection
        run: python3 scripts/quality_gate_test.py
      - name: Test
        working-directory: backend
        env:
          GOCHAT_TEST_DB: sqlite
          GOCHAT_BROWSER_SMOKE: "1"
          BU_CDP_URL: http://127.0.0.1:9222
        run: |
          "${{ steps.chrome.outputs.chrome-path }}" --headless=new --no-sandbox --disable-dev-shm-usage --remote-debugging-port=9222 --user-data-dir="$RUNNER_TEMP/chrome" about:blank &
          chrome_pid=$!
          trap 'kill "$chrome_pid" 2>/dev/null || true; browser-harness --reload > /dev/null 2>&1 || true' EXIT
          curl --fail --silent --show-error --retry 10 --retry-connrefused --retry-delay 1 http://127.0.0.1:9222/json/version > /dev/null
          browser-harness <<'PY'
          print(page_info())
          PY
          GOCHAT_RACE=false \
          GOCHAT_COLD_CACHE=true \
          GOCHAT_MAX_DURATION_SECONDS=480 \
          GOCHAT_QUALITY_DIR="$GITHUB_WORKSPACE/backend/quality/sqlite/cold-cache" \
          ./scripts/ci_quality_gate.sh
      - name: Test critical race suites
        working-directory: backend
        env:
          GOCHAT_TEST_DB: sqlite
        run: |
          GOCHAT_RACE=true \
          GOCHAT_COLD_CACHE=true \
          GOCHAT_TEST_PACKAGES='./internal/ws ./internal/handler/ws ./internal/worker ./internal/channel/...' \
          GOCHAT_TEST_TIMEOUT=180s \
          GOCHAT_MAX_DURATION_SECONDS=180 \
          GOCHAT_QUALITY_DIR="$GITHUB_WORKSPACE/backend/quality/sqlite/race" \
          ./scripts/ci_quality_gate.sh
      - name: Upload SQLite quality evidence
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: backend-quality-sqlite
          path: backend/quality/sqlite
          if-no-files-found: error
      - name: Build
        working-directory: backend
        run: go build ./...
      - name: Vet
        working-directory: backend
        run: go vet ./...

  backend-postgres:
    name: Backend (PostgreSQL)
    runs-on: ubuntu-latest
    services:
      postgres:
        image: pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
        env:
          POSTGRES_DB: gochat_test
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d gochat_test"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis:7.4.2-alpine@sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
      - name: Test auto-assignment concurrency
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: |
          set -euo pipefail
          output="$RUNNER_TEMP/autoassignment-concurrency.json"
          go test -json ./internal/autoassignment -run '^TestAssignmentServiceOnlyOneConcurrentWorkerWinsPostgres$' -count=3 | tee "$output"
          grep -Eq '"Action":"pass".*"Test":"TestAssignmentServiceOnlyOneConcurrentWorkerWinsPostgres"' "$output" || {
            echo "PostgreSQL auto-assignment concurrency test did not run" >&2
            exit 1
          }
      - name: Test backend against PostgreSQL
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: go test -count=1 -timeout 10m ./internal/... ./pkg/... ./cmd/...
      - name: Test controlled service race budget
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: |
          set -euo pipefail
          evidence="$GITHUB_WORKSPACE/backend/quality/postgres/service-race"
          mkdir -p "$evidence"
          start=$SECONDS
          set +e
          timeout 180s go test -race -count=1 -json -timeout 170s \
            -run '^(TestCaptainPreferenceService_(Create|Create_DuplicateAccount|Get|Get_NotFound|Update|Delete)|TestEnsureCaptainAgentBotBindingConcurrentCallsStayUnique|TestTagCreate7_Cov35)$' \
            ./internal/service | tee "$evidence/test.jsonl"
          test_status=${PIPESTATUS[0]}
          set -e
          elapsed=$((SECONDS - start))
          python3 - "$evidence/test.jsonl" "$evidence/summary.json" "$test_status" "$elapsed" <<'PY'
          import json, sys

          required = {
              "TestCaptainPreferenceService_Create",
              "TestCaptainPreferenceService_Create_DuplicateAccount",
              "TestCaptainPreferenceService_Get",
              "TestCaptainPreferenceService_Get_NotFound",
              "TestCaptainPreferenceService_Update",
              "TestCaptainPreferenceService_Delete",
              "TestEnsureCaptainAgentBotBindingConcurrentCallsStayUnique",
              "TestTagCreate7_Cov35",
          }
          results = {}
          with open(sys.argv[1], encoding="utf-8") as events:
              for line in events:
                  event = json.loads(line)
                  if event.get("Test") in required and event.get("Action") in {"pass", "fail", "skip"}:
                      results[event["Test"]] = event["Action"]
          status, elapsed = int(sys.argv[3]), int(sys.argv[4])
          failures = [name for name in sorted(required) if results.get(name) != "pass"]
          report = {
              "status": "passed" if status == 0 and elapsed <= 180 and not failures else "failed",
              "duration_seconds": elapsed,
              "budget_seconds": 180,
              "test_exit_code": status,
              "required_tests": {name: results.get(name, "missing") for name in sorted(required)},
          }
          with open(sys.argv[2], "w", encoding="utf-8") as output:
              json.dump(report, output, indent=2, sort_keys=True)
              output.write("\n")
          print(json.dumps(report, sort_keys=True))
          if report["status"] != "passed":
              raise SystemExit(1)
          PY
      - name: Test production upload migrations
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: go test ./internal/database -run '^TestUploadPostgresProductionMigration' -count=1
      - name: Test PostgreSQL E2E
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: |
          set -euo pipefail
          output="$RUNNER_TEMP/postgres-e2e.json"
          go test -json -count=1 -timeout 10m ./tests/e2e/... | tee "$output"

          python3 - "$output" <<'PY'
          import json
          import sys

          required = {
              "TestAuthE2ESuite",
              "TestAccountCRUDE2ESuite",
              "TestConversationE2ESuite",
              "TestCRME2ESuite",
              "TestCSRFE2ETestSuite",
              "TestRBACE2ESuite",
          }
          results = {}
          with open(sys.argv[1], encoding="utf-8") as events:
              for line in events:
                  event = json.loads(line)
                  if event.get("Test") and event.get("Action") in {"pass", "skip", "fail"}:
                      results[event["Test"]] = event["Action"]

          missing = sorted(suite for suite in required if results.get(suite) != "pass")
          passed_subtests = {suite: 0 for suite in required}
          skipped_subtests = {suite: 0 for suite in required}
          for test, action in results.items():
              suite, separator, _ = test.partition("/")
              if separator and suite in required:
                  if action == "pass":
                      passed_subtests[suite] += 1
                  elif action == "skip":
                      skipped_subtests[suite] += 1

          empty = sorted(suite for suite, count in passed_subtests.items() if count == 0)
          for suite in sorted(required):
              print(f"{suite}: {passed_subtests[suite]} passed, {skipped_subtests[suite]} skipped")
          print(
              "E2E totals: "
              f"{sum(action == 'pass' for action in results.values())} passed, "
              f"{sum(action == 'skip' for action in results.values())} skipped"
          )
          if missing:
              raise SystemExit(f"required E2E suites did not pass: {', '.join(missing)}")
          if empty:
              raise SystemExit(f"required E2E suites ran no passing subtests: {', '.join(empty)}")
          PY
      - name: Test multi-instance WebSocket fanout
        working-directory: backend
        env:
          GOCHAT_MULTI_INSTANCE_E2E: "1"
          GOCHAT_MULTI_INSTANCE_DATABASE_DSN: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
          GOCHAT_REDIS_DSN: redis://localhost:6379
          GOCHAT_WS_E2E_EVIDENCE_DIR: ${{ github.workspace }}/backend/quality/postgres/websocket-multi-instance
        run: |
          set -euo pipefail
          mkdir -p "$GOCHAT_WS_E2E_EVIDENCE_DIR"
          go test -json -count=3 -timeout 6m ./tests/e2e -run '^TestWebSocketMultiInstanceFanout$' \
            | tee "$GOCHAT_WS_E2E_EVIDENCE_DIR/test.jsonl"
          grep -Eq '"Action":"pass".*"Test":"TestWebSocketMultiInstanceFanout"' \
            "$GOCHAT_WS_E2E_EVIDENCE_DIR/test.jsonl"
      - name: Upload PostgreSQL quality evidence
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: backend-quality-postgres
          path: backend/quality/postgres
          if-no-files-found: error

  frontend:
    name: Frontend Build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 20.19.5
      - run: corepack enable && corepack prepare pnpm@10.2.0 --activate
      - run: pnpm install --frozen-lockfile
      - name: Test
        run: pnpm --dir frontend test
      - name: ESLint representative JS/Vue files
        run: pnpm --dir frontend exec eslint app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.spec.js app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.vue
      - name: Prettier representative JS/TS/Vue files
        run: pnpm --dir frontend exec prettier --check app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.spec.js app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.vue vite.config.ts app/javascript/histoire.setup.ts
      - name: Build and verify artifacts
        run: pnpm --dir frontend test:build

  frontend-smoke:
    name: Browser core smoke
    needs: [frontend]
    runs-on: ubuntu-latest
    services:
      postgres:
        image: pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
        env:
          POSTGRES_DB: gochat_test
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d gochat_test"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis:7.4.2-alpine@sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 20.19.5
      - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2
        id: chrome
        with:
          chrome-version: 152.0.7977.54
      - name: Test browser smoke failure contracts
        run: node backend/scripts/parity_frontend_browser_smoke.mjs --self-test
      - run: corepack enable && corepack prepare pnpm@10.2.0 --activate
      - run: pnpm install --frozen-lockfile
      - name: Run browser smoke
        env:
          CI: "true"
          GOCHAT_ENV: development
          GOCHAT_DATABASE_DSN: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
          GOCHAT_REDIS_DSN: redis://localhost:6379
          GOCHAT_JWT_SECRET: gochat-smoke-jwt-secret-at-least-32-characters
          CHATWOOT_DIR: ${{ github.workspace }}/frontend
          GOCHAT_SMOKE_SEARCH_ENGINE: db
          GOCHAT_SMOKE_LOG_DIR: ${{ github.workspace }}/backend/.tmp/frontend-smoke
          GOCHAT_SMOKE_REPORT: ${{ github.workspace }}/docs/parity/frontend-smoke-report.md
          GOCHAT_SMOKE_CHROME: ${{ steps.chrome.outputs.chrome-path }}
        run: |
          mkdir -p "$GOCHAT_SMOKE_LOG_DIR"
          (cd backend && go run ./cmd/gochat serve >"$GOCHAT_SMOKE_LOG_DIR/gochat.log" 2>&1) &
          backend_pid=$!
          (cd frontend && VITE_API_HOST=http://127.0.0.1:3000 pnpm exec vite --host localhost --port 3036 >"$GOCHAT_SMOKE_LOG_DIR/vite.log" 2>&1) &
          frontend_pid=$!
          trap 'kill "$backend_pid" "$frontend_pid" 2>/dev/null || true' EXIT
          curl --fail --silent --show-error --retry 60 --retry-connrefused --retry-delay 1 \
            http://127.0.0.1:3000/health > /dev/null
          backend/scripts/parity_frontend_smoke.sh --browser-smoke
      - name: Upload browser smoke evidence
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: frontend-browser-smoke
          path: |
            backend/.tmp/frontend-smoke
            docs/parity/frontend-smoke-report.md
          if-no-files-found: error

  security:
    name: Security (SAST/SCA)
    needs: [frontend-smoke]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - name: Test release promotion safeguards
        run: .github/scripts/test_release_promotion.sh
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: |
            backend/go.sum
            channels/shangwutong/go.sum
      - name: Install pinned Go security tools
        env:
          GOBIN: ${{ runner.temp }}/security-tools
        run: |
          go install github.com/securego/gosec/v2/cmd/gosec@"${GOSEC_VERSION}"
          GOTOOLCHAIN=go1.26.6+auto go install golang.org/x/vuln/cmd/govulncheck@"${GOVULNCHECK_VERSION}"
          echo "$GOBIN" >> "$GITHUB_PATH"
      - name: Scan Go code and dependencies
        run: |
          (cd backend && gosec -quiet -severity high -confidence high ./... && govulncheck ./...)
          (cd channels/shangwutong && GOTOOLCHAIN=go1.26.6+auto gosec -quiet -severity high -confidence high ./... && GOTOOLCHAIN=go1.26.6+auto govulncheck ./...)
      - name: Scan repository dependencies
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: fs
          scan-ref: .
          scanners: vuln
          version: ${{ env.TRIVY_VERSION }}
          severity: HIGH,CRITICAL
          exit-code: 1

  production-smoke:
    name: Production image smoke
    if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')
    needs: [frontend-smoke]
    runs-on: ubuntu-latest
    env:
      GOCHAT_IMAGE_REF: gochat:production-smoke
      GOCHAT_PORT: "38080"
      GOCHAT_DATABASE_DSN: "postgres://gochat:ci-postgres-secret@db.smoke.test:5432/gochat_production?sslmode=verify-full&sslrootcert=/run/secrets/external-db-ca.crt&sslcert=/run/secrets/external-db-client.crt&sslkey=/run/secrets/external-db-client.key"
      GOCHAT_DATABASE_TLS_CA_FILE: ${{ github.workspace }}/.tmp/gochat-tls/ca.crt
      GOCHAT_DATABASE_TLS_CLIENT_CERT_FILE: ${{ github.workspace }}/.tmp/gochat-tls/postgres.crt
      GOCHAT_DATABASE_TLS_CLIENT_KEY_FILE: ${{ github.workspace }}/.tmp/gochat-tls/postgres.key
      GOCHAT_REDIS_DSN: "rediss://:ci-redis-secret@redis:6379/0"
      GOCHAT_ENCRYPTION_AES_KEY: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
      GOCHAT_TLS_DIR: ${{ github.workspace }}/.tmp/gochat-tls
      POSTGRES_PASSWORD: ci-postgres-secret
      REDIS_PASSWORD: ci-redis-secret
      MEILI_MASTER_KEY: ci-meili-secret-16
      GOCHAT_JWT_SECRET: ci-smoke-jwt-secret-at-least-32-characters
      GOCHAT_JWT_PREVIOUS_SECRETS: ci-previous-jwt-secret-at-least-32-characters
      GOCHAT_CONNECTOR_SERVICE_TOKEN: ci-connector-service-token
      GOCHAT_BACKUP_OFFSITE_DIR: /mnt/gochat-offsite
      GOCHAT_BACKUP_OFFSITE_SOURCE: backup.example.test:/gochat
      GOCHAT_BACKUP_OFFSITE_FSTYPE: nfs4
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - name: Build production images from repository root
        run: |
          docker build -t "$GOCHAT_IMAGE_REF" -f deploy/docker/Dockerfile .
          docker build -t shangwutong:production-smoke -f channels/shangwutong/Dockerfile .
          docker run -d --name gochat-ci-registry -p 127.0.0.1:5000:5000 registry:2@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
          timeout 30 sh -c 'until curl -fsS http://127.0.0.1:5000/v2/; do sleep 1; done'
          docker tag "$GOCHAT_IMAGE_REF" localhost:5000/gochat:production-smoke
          docker tag shangwutong:production-smoke localhost:5000/shangwutong:production-smoke
          docker push localhost:5000/gochat:production-smoke
          docker push localhost:5000/shangwutong:production-smoke
          echo "GOCHAT_IMAGE_REF=$(docker inspect --format '{{index .RepoDigests 0}}' localhost:5000/gochat:production-smoke)" >> "$GITHUB_ENV"
          echo "SHANGWUTONG_IMAGE_REF=$(docker inspect --format '{{index .RepoDigests 0}}' localhost:5000/shangwutong:production-smoke)" >> "$GITHUB_ENV"
      - name: Scan GoChat image
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: gochat:production-smoke
          scanners: vuln
          version: ${{ env.TRIVY_VERSION }}
          severity: HIGH,CRITICAL
          exit-code: 1
      - name: Scan Shangwutong image
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: shangwutong:production-smoke
          scanners: vuln
          version: ${{ env.TRIVY_VERSION }}
          severity: HIGH,CRITICAL
          exit-code: 1
      - name: Verify artifact contents
        run: |
          docker run --rm --entrypoint sh "$GOCHAT_IMAGE_REF" -c \
            'test -s /app/configs/config.production.yaml && find /app/migrations -maxdepth 1 -type f -name "*.sql" -print -quit | grep -q . && test -s /app/frontend/dist/index.html && test -s /app/frontend/dist/favicon-32x32.png'
          docker run --rm --entrypoint sh "$SHANGWUTONG_IMAGE_REF" -c \
            'test -x /usr/local/bin/shangwutong && test -d /data && test -d /backup && test "$(id -u)" -eq 10001'
      - name: Create trusted TLS fixtures
        run: |
          mkdir -p "$GOCHAT_TLS_DIR"
          openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=GoChat CI CA' \
            -keyout "$GOCHAT_TLS_DIR/ca.key" -out "$GOCHAT_TLS_DIR/ca.crt"
          for service in postgres redis; do
            hostname=$service
            [[ $service == postgres ]] && hostname=db.smoke.test
            openssl req -newkey rsa:2048 -nodes -subj "/CN=$hostname" \
              -addext "subjectAltName=DNS:$hostname" \
              -keyout "$GOCHAT_TLS_DIR/$service.key" -out "$GOCHAT_TLS_DIR/$service.csr"
            printf 'subjectAltName=DNS:%s\n' "$hostname" > "$GOCHAT_TLS_DIR/$service.ext"
            openssl x509 -req -days 1 -CA "$GOCHAT_TLS_DIR/ca.crt" -CAkey "$GOCHAT_TLS_DIR/ca.key" \
              -CAcreateserial -extfile "$GOCHAT_TLS_DIR/$service.ext" \
              -in "$GOCHAT_TLS_DIR/$service.csr" -out "$GOCHAT_TLS_DIR/$service.crt"
          done
          chmod 640 "$GOCHAT_TLS_DIR"/postgres.key
          chmod 600 "$GOCHAT_TLS_DIR"/ca.key "$GOCHAT_TLS_DIR"/redis.key
          chmod 644 "$GOCHAT_TLS_DIR"/*.crt
          echo "GOCHAT_DATABASE_TLS_GID=$(stat -c %g "$GOCHAT_TLS_DIR/postgres.key")" >> "$GITHUB_ENV"
      - name: Start production Compose and smoke core pages
        run: |
          deploy/docker/preflight_test.sh
          GOCHAT_TEST_IMAGE="$GOCHAT_IMAGE_REF" deploy/docker/database_client_entrypoint_test.sh
          docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert config["services"]["gochat"]["command"] == ["serve"]; assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
          compose=(docker compose -f deploy/docker/docker-compose.prod.yml -f deploy/docker/docker-compose.prod-smoke.yml)
          "${compose[@]}" --profile ops run --rm migrate
          "${compose[@]}" up -d --wait gochat shangwutong
          "${compose[@]}" up -d worker
          for service in gochat worker shangwutong; do
            container_id="$("${compose[@]}" ps -q "$service")"
            test -n "$container_id"
            test "$(docker inspect --format '{{.State.Status}}' "$container_id")" = running
          done
          for service in gochat shangwutong; do
            container_id="$("${compose[@]}" ps -q "$service")"
            test "$(docker inspect --format '{{.State.Health.Status}}' "$container_id")" = healthy
          done
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" > "$RUNNER_TEMP/health.json"
          python3 backend/scripts/validate_health_json.py "$RUNNER_TEMP/health.json"
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/ready" | grep -q '"ready":true'
          ! "${compose[@]}" logs --no-color gochat | grep -F 'slice bounds out of range'
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/runtime-config.js" | grep -q 'window.__GOCHAT_CONFIG__'
          curl -fsS -o "$RUNNER_TEMP/favicon-32x32.png" "http://127.0.0.1:$GOCHAT_PORT/favicon-32x32.png"
          test -s "$RUNNER_TEMP/favicon-32x32.png"
          "${compose[@]}" exec -T shangwutong \
            wget -q -T 3 -O - http://127.0.0.1:9100/readyz | grep -q '"status":"ready"'
      - name: Stop production Compose
        if: always()
        run: |
          docker compose -f deploy/docker/docker-compose.prod.yml -f deploy/docker/docker-compose.prod-smoke.yml down -v
          docker rm -f gochat-ci-registry || true

  release:
    name: Publish signed immutable images
    if: startsWith(github.ref, 'refs/tags/v')
    needs: [backend, backend-postgres, frontend, security, production-smoke]
    runs-on: ubuntu-latest
    concurrency:
      group: release-promotion
      cancel-in-progress: false
    permissions:
      contents: read
      packages: write
      id-token: write
    outputs:
      image_ref: ${{ steps.ref.outputs.image_ref }}
      image_digest: ${{ steps.build.outputs.digest }}
      shangwutong_image_ref: ${{ steps.ref.outputs.shangwutong_image_ref }}
      shangwutong_image_digest: ${{ steps.shangwutong-build.outputs.digest }}
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Extract immutable tags
        id: meta
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}}
            type=sha
      - name: Extract Shangwutong immutable tags
        id: shangwutong-meta
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.SHANGWUTONG_IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}}
            type=sha
      - name: Capture reproducible build metadata
        id: date
        run: |
          epoch=$(git show -s --format=%ct "$GITHUB_SHA")
          echo "epoch=$epoch" >> "$GITHUB_OUTPUT"
          echo "value=$(date -u -d "@$epoch" +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT"
      - name: Define isolated staging references
        id: staging
        run: |
          suffix="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          echo "gochat=${REGISTRY}/${IMAGE_NAME}:$suffix" >> "$GITHUB_OUTPUT"
          echo "shangwutong=${REGISTRY}/${SHANGWUTONG_IMAGE_NAME}:$suffix" >> "$GITHUB_OUTPUT"
      - name: Build and push GoChat staging image
        id: build
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: .
          file: deploy/docker/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          tags: ${{ steps.staging.outputs.gochat }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            VERSION=${{ github.ref_name }}
            COMMIT_SHA=${{ github.sha }}
            BUILD_DATE=${{ steps.date.outputs.value }}
            SOURCE_DATE_EPOCH=${{ steps.date.outputs.epoch }}
          cache-from: type=gha,scope=gochat
          cache-to: type=gha,mode=max,scope=gochat
      - name: Build and push Shangwutong staging image
        id: shangwutong-build
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: .
          file: channels/shangwutong/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          tags: ${{ steps.staging.outputs.shangwutong }}
          labels: ${{ steps.shangwutong-meta.outputs.labels }}
          build-args: |
            VERSION=${{ github.ref_name }}
            COMMIT_SHA=${{ github.sha }}
            BUILD_DATE=${{ steps.date.outputs.value }}
            SOURCE_DATE_EPOCH=${{ steps.date.outputs.epoch }}
          cache-from: type=gha,scope=shangwutong
          cache-to: type=gha,mode=max,scope=shangwutong
      - name: Record staging digest references
        id: ref
        run: |
          image_ref="${REGISTRY}/${IMAGE_NAME}@${{ steps.build.outputs.digest }}"
          shangwutong_image_ref="${REGISTRY}/${SHANGWUTONG_IMAGE_NAME}@${{ steps.shangwutong-build.outputs.digest }}"
          echo "image_ref=$image_ref" >> "$GITHUB_OUTPUT"
          echo "shangwutong_image_ref=$shangwutong_image_ref" >> "$GITHUB_OUTPUT"
      - name: Install Cosign
        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
        with:
          cosign-release: ${{ env.COSIGN_VERSION }}
      - name: Install crane
        uses: imjasonh/setup-crane@feee3b6bb0d4c68370f256a4502498c9227e5c6b # v0.7
        with:
          version: ${{ env.CRANE_VERSION }}
      - name: Sign image digests
        run: |
          cosign sign --yes "${{ steps.ref.outputs.image_ref }}"
          cosign sign --yes "${{ steps.ref.outputs.shangwutong_image_ref }}"
      - name: Attest SBOM and provenance
        env:
          SOURCE_URI: git+https://github.com/${{ github.repository }}.git
          SOURCE_REF: ${{ github.ref }}
          BUILDER_ID: https://github.com/${{ github.repository }}/.github/workflows/ci.yml@${{ github.ref }}
          INVOCATION_ID: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
        run: |
          set -euo pipefail
          .github/scripts/release_attestation.sh attest-images \
            "$SOURCE_URI" "$SOURCE_REF" "$GITHUB_SHA" "$BUILDER_ID" "$INVOCATION_ID" \
            "${{ steps.ref.outputs.image_ref }}" "${{ steps.ref.outputs.shangwutong_image_ref }}"
      - name: Verify signatures and attestations
        env:
          SOURCE_URI: git+https://github.com/${{ github.repository }}.git
          SOURCE_REF: ${{ github.ref }}
          BUILDER_ID: https://github.com/${{ github.repository }}/.github/workflows/ci.yml@${{ github.ref }}
          INVOCATION_ID: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
        run: |
          set -euo pipefail
          identity="https://github.com/${GITHUB_REPOSITORY}/.github/workflows/ci.yml@${GITHUB_REF}"
          .github/scripts/release_attestation.sh verify-images \
            "$identity" "$SOURCE_URI" "$SOURCE_REF" "$GITHUB_SHA" "$BUILDER_ID" "$INVOCATION_ID" \
            "${{ steps.ref.outputs.image_ref }}" "${{ steps.ref.outputs.shangwutong_image_ref }}"
      - name: Promote and reverify release tags
        env:
          GOCHAT_RELEASE_TAGS: ${{ steps.meta.outputs.tags }}
          SHANGWUTONG_RELEASE_TAGS: ${{ steps.shangwutong-meta.outputs.tags }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PACKAGE_OWNER: ${{ github.repository_owner }}
          PACKAGE_OWNER_TYPE: ${{ github.event.repository.owner.type }}
          SOURCE_URI: git+https://github.com/${{ github.repository }}.git
          SOURCE_REF: ${{ github.ref }}
          BUILDER_ID: https://github.com/${{ github.repository }}/.github/workflows/ci.yml@${{ github.ref }}
          INVOCATION_ID: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
        run: |
          set -euo pipefail
          source .github/scripts/release_helpers.sh

          declare -a targets=() created=()
          declare -A cleanup_refs=()
          cleanup_tag_name="rollback-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"

          add_targets() {
            local source_ref=$1 tags=$2 repo=${1%@*} target count=0
            while IFS= read -r target; do
              [[ -n $target ]] || continue
              [[ $target == "$repo:"* ]] || {
                echo "Release tag is outside source repository: $target" >&2
                return 1
              }
              targets+=("$source_ref|$target")
              ((++count))
            done <<< "$tags"
            ((count > 0))
          }

          delete_cleanup_version() {
            local repo=$1 cleanup_ref=${cleanup_refs[$1]} package endpoint
            package=${repo#"ghcr.io/$PACKAGE_OWNER/"}
            [[ $package != "$repo" ]] || return 1
            package=$(jq -rn --arg value "$package" '$value | @uri')
            case $PACKAGE_OWNER_TYPE in
              User) endpoint="/users/$PACKAGE_OWNER/packages/container/$package/versions" ;;
              Organization) endpoint="/orgs/$PACKAGE_OWNER/packages/container/$package/versions" ;;
              *) return 1 ;;
            esac
            delete_package_version "$endpoint" "$cleanup_tag_name" "${cleanup_ref##*@}"
          }

          rollback() {
            local status=$? entry source_ref target repo tag cleanup_ref rollback_failed=0
            ((status != 0)) || return
            trap - EXIT INT TERM
            set +e
            echo "Promotion failed; removing created release tags" >&2
            for entry in "${created[@]}"; do
              source_ref=${entry%%|*}
              target=${entry#*|}
              repo=${source_ref%@*}
              tag=${target#"$repo:"}
              cleanup_ref=${cleanup_refs[$repo]}
              crane tag "$cleanup_ref" "$tag" || rollback_failed=1
            done
            for repo in "${!cleanup_refs[@]}"; do
              delete_cleanup_version "$repo" || rollback_failed=1
              wait_for_tag_absent "$repo:$cleanup_tag_name" || rollback_failed=1
            done
            for entry in "${created[@]}"; do
              target=${entry#*|}
              wait_for_tag_absent "$target" || rollback_failed=1
            done
            ((rollback_failed == 0)) || echo "Release-tag rollback was incomplete" >&2
            exit "$status"
          }

          add_targets "${{ steps.ref.outputs.image_ref }}" "$GOCHAT_RELEASE_TAGS"
          add_targets "${{ steps.ref.outputs.shangwutong_image_ref }}" "$SHANGWUTONG_RELEASE_TAGS"

          trap rollback EXIT
          trap 'exit 1' INT TERM
          cleanup_dir=$(mktemp -d)
          printf '%s\n' "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" > "$cleanup_dir/rollback-marker"
          tar -C "$cleanup_dir" -cf "$cleanup_dir/rollback.tar" rollback-marker
          for source_ref in \
            "${{ steps.ref.outputs.image_ref }}" \
            "${{ steps.ref.outputs.shangwutong_image_ref }}"; do
            repo=${source_ref%@*}
            cleanup_tag="$repo:$cleanup_tag_name"
            cleanup_refs["$repo"]=$(crane append \
              --oci-empty-base \
              --new_layer "$cleanup_dir/rollback.tar" \
              --new_tag "$cleanup_tag")
          done

          error_file="$RUNNER_TEMP/crane-digest.err"
          for entry in "${targets[@]}"; do
            target=${entry#*|}
            if digest=$(crane digest "$target" 2> "$error_file"); then
              echo "Refusing to overwrite existing release tag $target ($digest)" >&2
              exit 1
            fi
            grep -Eiq 'MANIFEST_UNKNOWN|manifest unknown|not found|404' "$error_file" || {
              cat "$error_file" >&2
              exit 1
            }
          done

          for entry in "${targets[@]}"; do
            source_ref=${entry%%|*}
            target=${entry#*|}
            repo=${source_ref%@*}
            tag=${target#"$repo:"}
            created+=("$entry")
            crane tag "$source_ref" "$tag"
          done

          identity="https://github.com/${GITHUB_REPOSITORY}/.github/workflows/ci.yml@${GITHUB_REF}"
          .github/scripts/release_attestation.sh verify-promoted \
            "$identity" "$SOURCE_URI" "$SOURCE_REF" "$GITHUB_SHA" "$BUILDER_ID" "$INVOCATION_ID" \
            "${targets[@]}"

          trap - EXIT INT TERM
          for repo in "${!cleanup_refs[@]}"; do
            delete_cleanup_version "$repo"
            wait_for_tag_absent "$repo:$cleanup_tag_name"
          done

          echo "GoChat image: \`${{ steps.ref.outputs.image_ref }}\`" >> "$GITHUB_STEP_SUMMARY"
          echo "Shangwutong image: \`${{ steps.ref.outputs.shangwutong_image_ref }}\`" >> "$GITHUB_STEP_SUMMARY"