gochat/.github/workflows/ci.yml

name: GoChat CI/CD Pipeline

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]
    tags: ['v*.*.*']

permissions:
  contents: read

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  SHANGWUTONG_IMAGE_NAME: ${{ github.repository_owner }}/shangwutong
  GOSEC_VERSION: v2.28.0
  GOVULNCHECK_VERSION: v1.1.4
  TRIVY_VERSION: v0.74.0
  COSIGN_VERSION: v3.1.3
  CRANE_VERSION: v0.21.9

jobs:
  backend:
    name: Backend (SQLite)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.12"
      - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2
        id: chrome
      - name: Install browser harness
        run: python -m pip install browser-harness==0.1.9
      - name: Test Prometheus alert rules
        run: docker run --rm --entrypoint promtool -v "$PWD/backend/configs:/configs:ro" prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 test rules /configs/prometheus_alerts_test.yml
      - name: Test
        working-directory: backend
        env:
          GOCHAT_TEST_DB: sqlite
          GOCHAT_BROWSER_SMOKE: "1"
          BU_CDP_URL: http://127.0.0.1:9222
        run: |
          "${{ steps.chrome.outputs.chrome-path }}" --headless=new --no-sandbox --disable-dev-shm-usage --remote-debugging-port=9222 --user-data-dir="$RUNNER_TEMP/chrome" about:blank &
          chrome_pid=$!
          trap 'kill "$chrome_pid" 2>/dev/null || true; browser-harness --reload > /dev/null 2>&1 || true' EXIT
          curl --fail --silent --show-error --retry 10 --retry-connrefused --retry-delay 1 http://127.0.0.1:9222/json/version > /dev/null
          browser-harness <<'PY'
          print(page_info())
          PY
          go test ./internal/... ./pkg/... ./cmd/...
      - name: Build
        working-directory: backend
        run: go build ./...
      - name: Vet
        working-directory: backend
        run: go vet ./...

  backend-postgres:
    name: Backend (PostgreSQL)
    runs-on: ubuntu-latest
    services:
      postgres:
        image: pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
        env:
          POSTGRES_DB: gochat_test
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d gochat_test"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
      - name: Test auto-assignment concurrency
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: |
          set -euo pipefail
          output="$RUNNER_TEMP/autoassignment-concurrency.json"
          go test -json ./internal/autoassignment -run '^TestAssignmentServiceOnlyOneConcurrentWorkerWinsPostgres$' -count=3 | tee "$output"
          grep -Eq '"Action":"pass".*"Test":"TestAssignmentServiceOnlyOneConcurrentWorkerWinsPostgres"' "$output" || {
            echo "PostgreSQL auto-assignment concurrency test did not run" >&2
            exit 1
          }
      - name: Test backend against PostgreSQL
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: go test -count=1 -timeout 10m ./internal/... ./pkg/... ./cmd/...
      - name: Test production upload migrations
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: go test ./internal/database -run '^TestUploadPostgresProductionMigration' -count=1
      - name: Test PostgreSQL E2E
        working-directory: backend
        env:
          GOCHAT_TEST_DB: postgres
          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
        run: |
          set -euo pipefail
          output="$RUNNER_TEMP/postgres-e2e.json"
          go test -json -count=1 -timeout 10m ./tests/e2e/... | tee "$output"

          python3 - "$output" <<'PY'
          import json
          import sys

          required = {
              "TestAuthE2ESuite",
              "TestAccountCRUDE2ESuite",
              "TestConversationE2ESuite",
              "TestCRME2ESuite",
              "TestCSRFE2ETestSuite",
              "TestRBACE2ESuite",
          }
          results = {}
          with open(sys.argv[1], encoding="utf-8") as events:
              for line in events:
                  event = json.loads(line)
                  if event.get("Test") and event.get("Action") in {"pass", "skip", "fail"}:
                      results[event["Test"]] = event["Action"]

          missing = sorted(suite for suite in required if results.get(suite) != "pass")
          passed_subtests = {suite: 0 for suite in required}
          skipped_subtests = {suite: 0 for suite in required}
          for test, action in results.items():
              suite, separator, _ = test.partition("/")
              if separator and suite in required:
                  if action == "pass":
                      passed_subtests[suite] += 1
                  elif action == "skip":
                      skipped_subtests[suite] += 1

          empty = sorted(suite for suite, count in passed_subtests.items() if count == 0)
          for suite in sorted(required):
              print(f"{suite}: {passed_subtests[suite]} passed, {skipped_subtests[suite]} skipped")
          print(
              "E2E totals: "
              f"{sum(action == 'pass' for action in results.values())} passed, "
              f"{sum(action == 'skip' for action in results.values())} skipped"
          )
          if missing:
              raise SystemExit(f"required E2E suites did not pass: {', '.join(missing)}")
          if empty:
              raise SystemExit(f"required E2E suites ran no passing subtests: {', '.join(empty)}")
          PY

  frontend:
    name: Frontend Build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 20
      - run: corepack enable && corepack prepare pnpm@10.2.0 --activate
      - run: pnpm install --frozen-lockfile
      - name: Test
        run: pnpm --dir frontend test
      - name: ESLint representative JS/Vue files
        run: pnpm --dir frontend exec eslint app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.spec.js app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.vue
      - name: Prettier representative JS/TS/Vue files
        run: pnpm --dir frontend exec prettier --check app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.spec.js app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.vue vite.config.ts app/javascript/histoire.setup.ts
      - name: Build and verify artifacts
        run: pnpm --dir frontend test:build

  security:
    name: Security (SAST/SCA)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - name: Test release promotion safeguards
        run: .github/scripts/test_release_promotion.sh
      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 1.25.13
          cache-dependency-path: |
            backend/go.sum
            channels/shangwutong/go.sum
      - name: Install pinned Go security tools
        env:
          GOBIN: ${{ runner.temp }}/security-tools
        run: |
          go install github.com/securego/gosec/v2/cmd/gosec@"${GOSEC_VERSION}"
          GOTOOLCHAIN=go1.26.6+auto go install golang.org/x/vuln/cmd/govulncheck@"${GOVULNCHECK_VERSION}"
          echo "$GOBIN" >> "$GITHUB_PATH"
      - name: Scan Go code and dependencies
        run: |
          (cd backend && gosec -quiet -severity high -confidence high ./... && govulncheck ./...)
          (cd channels/shangwutong && GOTOOLCHAIN=go1.26.6+auto gosec -quiet -severity high -confidence high ./... && GOTOOLCHAIN=go1.26.6+auto govulncheck ./...)
      - name: Scan repository dependencies
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: fs
          scan-ref: .
          scanners: vuln
          version: ${{ env.TRIVY_VERSION }}
          severity: HIGH,CRITICAL
          exit-code: 1

  production-smoke:
    name: Production image smoke
    if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    env:
      GOCHAT_IMAGE_REF: gochat:production-smoke
      GOCHAT_PORT: "38080"
      GOCHAT_SERVER_CORS_ALLOWED_ORIGINS: https://chat.example.test
      POSTGRES_PASSWORD: ci-postgres-secret
      REDIS_PASSWORD: ci-redis-secret
      MEILI_MASTER_KEY: ci-meili-secret-16
      GOCHAT_JWT_SECRET: ci-smoke-jwt-secret-at-least-32-characters
      GOCHAT_JWT_PREVIOUS_SECRETS: ci-previous-jwt-secret-at-least-32-characters
      GOCHAT_CONNECTOR_SERVICE_TOKEN: ci-connector-service-token
      GOCHAT_BACKUP_OFFSITE_DIR: /mnt/gochat-offsite
      GOCHAT_BACKUP_OFFSITE_SOURCE: backup.example.test:/gochat
      GOCHAT_BACKUP_OFFSITE_FSTYPE: nfs4
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - name: Build production images from repository root
        run: |
          docker build -t "$GOCHAT_IMAGE_REF" -f deploy/docker/Dockerfile .
          docker build -t shangwutong:production-smoke -f channels/shangwutong/Dockerfile .
          docker run -d --name gochat-ci-registry -p 127.0.0.1:5000:5000 registry:2
          timeout 30 sh -c 'until curl -fsS http://127.0.0.1:5000/v2/; do sleep 1; done'
          docker tag "$GOCHAT_IMAGE_REF" localhost:5000/gochat:production-smoke
          docker tag shangwutong:production-smoke localhost:5000/shangwutong:production-smoke
          docker push localhost:5000/gochat:production-smoke
          docker push localhost:5000/shangwutong:production-smoke
          echo "GOCHAT_IMAGE_REF=$(docker inspect --format '{{index .RepoDigests 0}}' localhost:5000/gochat:production-smoke)" >> "$GITHUB_ENV"
          echo "SHANGWUTONG_IMAGE_REF=$(docker inspect --format '{{index .RepoDigests 0}}' localhost:5000/shangwutong:production-smoke)" >> "$GITHUB_ENV"
      - name: Scan GoChat image
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: gochat:production-smoke
          scanners: vuln
          version: ${{ env.TRIVY_VERSION }}
          severity: HIGH,CRITICAL
          exit-code: 1
      - name: Scan Shangwutong image
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: shangwutong:production-smoke
          scanners: vuln
          version: ${{ env.TRIVY_VERSION }}
          severity: HIGH,CRITICAL
          exit-code: 1
      - name: Verify artifact contents
        run: |
          docker run --rm --entrypoint sh "$GOCHAT_IMAGE_REF" -c \
            'test -s /app/configs/config.production.yaml && test -d /app/migrations && test -s /app/frontend/dist/index.html'
          docker run --rm --entrypoint sh "$SHANGWUTONG_IMAGE_REF" -c \
            'test -x /usr/local/bin/shangwutong && test -d /data && test -d /backup && test "$(id -u)" -eq 10001'
      - name: Start production Compose and smoke core pages
        run: |
          deploy/docker/preflight_test.sh
          docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
          docker compose -f deploy/docker/docker-compose.prod.yml --profile ops run --rm migrate
          docker compose -f deploy/docker/docker-compose.prod.yml up -d --wait gochat shangwutong
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" | grep -q '"status":"healthy"'
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'
          docker compose -f deploy/docker/docker-compose.prod.yml exec -T shangwutong \
            wget -q -T 3 -O - http://127.0.0.1:9100/readyz | grep -q '"status":"ready"'
      - name: Stop production Compose
        if: always()
        run: |
          docker compose -f deploy/docker/docker-compose.prod.yml down -v
          docker rm -f gochat-ci-registry || true

  release:
    name: Publish signed immutable images
    if: startsWith(github.ref, 'refs/tags/v')
    needs: [backend, backend-postgres, frontend, security, production-smoke]
    runs-on: ubuntu-latest
    concurrency:
      group: release-promotion
      cancel-in-progress: false
    permissions:
      contents: read
      packages: write
      id-token: write
    outputs:
      image_ref: ${{ steps.ref.outputs.image_ref }}
      image_digest: ${{ steps.build.outputs.digest }}
      shangwutong_image_ref: ${{ steps.ref.outputs.shangwutong_image_ref }}
      shangwutong_image_digest: ${{ steps.shangwutong-build.outputs.digest }}
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Extract immutable tags
        id: meta
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}}
            type=sha
      - name: Extract Shangwutong immutable tags
        id: shangwutong-meta
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.SHANGWUTONG_IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}}
            type=sha
      - name: Capture reproducible build metadata
        id: date
        run: |
          epoch=$(git show -s --format=%ct "$GITHUB_SHA")
          echo "epoch=$epoch" >> "$GITHUB_OUTPUT"
          echo "value=$(date -u -d "@$epoch" +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT"
      - name: Define isolated staging references
        id: staging
        run: |
          suffix="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          echo "gochat=${REGISTRY}/${IMAGE_NAME}:$suffix" >> "$GITHUB_OUTPUT"
          echo "shangwutong=${REGISTRY}/${SHANGWUTONG_IMAGE_NAME}:$suffix" >> "$GITHUB_OUTPUT"
      - name: Build and push GoChat staging image
        id: build
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: .
          file: deploy/docker/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          tags: ${{ steps.staging.outputs.gochat }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            VERSION=${{ github.ref_name }}
            COMMIT_SHA=${{ github.sha }}
            BUILD_DATE=${{ steps.date.outputs.value }}
            SOURCE_DATE_EPOCH=${{ steps.date.outputs.epoch }}
          cache-from: type=gha,scope=gochat
          cache-to: type=gha,mode=max,scope=gochat
      - name: Build and push Shangwutong staging image
        id: shangwutong-build
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: .
          file: channels/shangwutong/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          tags: ${{ steps.staging.outputs.shangwutong }}
          labels: ${{ steps.shangwutong-meta.outputs.labels }}
          build-args: |
            VERSION=${{ github.ref_name }}
            COMMIT_SHA=${{ github.sha }}
            BUILD_DATE=${{ steps.date.outputs.value }}
            SOURCE_DATE_EPOCH=${{ steps.date.outputs.epoch }}
          cache-from: type=gha,scope=shangwutong
          cache-to: type=gha,mode=max,scope=shangwutong
      - name: Record staging digest references
        id: ref
        run: |
          image_ref="${REGISTRY}/${IMAGE_NAME}@${{ steps.build.outputs.digest }}"
          shangwutong_image_ref="${REGISTRY}/${SHANGWUTONG_IMAGE_NAME}@${{ steps.shangwutong-build.outputs.digest }}"
          echo "image_ref=$image_ref" >> "$GITHUB_OUTPUT"
          echo "shangwutong_image_ref=$shangwutong_image_ref" >> "$GITHUB_OUTPUT"
      - name: Install Cosign
        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
        with:
          cosign-release: ${{ env.COSIGN_VERSION }}
      - name: Install crane
        uses: imjasonh/setup-crane@feee3b6bb0d4c68370f256a4502498c9227e5c6b # v0.7
        with:
          version: ${{ env.CRANE_VERSION }}
      - name: Sign image digests
        run: |
          cosign sign --yes "${{ steps.ref.outputs.image_ref }}"
          cosign sign --yes "${{ steps.ref.outputs.shangwutong_image_ref }}"
      - name: Verify signatures and attestations
        run: |
          for image_ref in "${{ steps.ref.outputs.image_ref }}" "${{ steps.ref.outputs.shangwutong_image_ref }}"; do
            cosign verify \
              --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/ci.yml@refs/tags/v" \
              --certificate-oidc-issuer https://token.actions.githubusercontent.com \
              "$image_ref"
            docker buildx imagetools inspect "$image_ref" --format '{{ json .SBOM }}' | grep -q '"SPDXID"'
            provenance=$(docker buildx imagetools inspect "$image_ref" --format '{{ json .Provenance }}')
            grep -q '"SLSA"' <<< "$provenance"
            grep -q "$GITHUB_SHA" <<< "$provenance"
          done
      - name: Promote and reverify release tags
        env:
          GOCHAT_RELEASE_TAGS: ${{ steps.meta.outputs.tags }}
          SHANGWUTONG_RELEASE_TAGS: ${{ steps.shangwutong-meta.outputs.tags }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PACKAGE_OWNER: ${{ github.repository_owner }}
          PACKAGE_OWNER_TYPE: ${{ github.event.repository.owner.type }}
        run: |
          set -euo pipefail
          source .github/scripts/release_helpers.sh

          declare -a targets=() created=()
          declare -A cleanup_refs=()
          cleanup_tag_name="rollback-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"

          add_targets() {
            local source_ref=$1 tags=$2 repo=${1%@*} target count=0
            while IFS= read -r target; do
              [[ -n $target ]] || continue
              [[ $target == "$repo:"* ]] || {
                echo "Release tag is outside source repository: $target" >&2
                return 1
              }
              targets+=("$source_ref|$target")
              ((++count))
            done <<< "$tags"
            ((count > 0))
          }

          delete_cleanup_version() {
            local repo=$1 cleanup_ref=${cleanup_refs[$1]} package endpoint
            package=${repo#"ghcr.io/$PACKAGE_OWNER/"}
            [[ $package != "$repo" ]] || return 1
            package=$(jq -rn --arg value "$package" '$value | @uri')
            case $PACKAGE_OWNER_TYPE in
              User) endpoint="/users/$PACKAGE_OWNER/packages/container/$package/versions" ;;
              Organization) endpoint="/orgs/$PACKAGE_OWNER/packages/container/$package/versions" ;;
              *) return 1 ;;
            esac
            delete_package_version "$endpoint" "$cleanup_tag_name" "${cleanup_ref##*@}"
          }

          rollback() {
            local status=$? entry source_ref target repo tag cleanup_ref rollback_failed=0
            ((status != 0)) || return
            trap - EXIT INT TERM
            set +e
            echo "Promotion failed; removing created release tags" >&2
            for entry in "${created[@]}"; do
              source_ref=${entry%%|*}
              target=${entry#*|}
              repo=${source_ref%@*}
              tag=${target#"$repo:"}
              cleanup_ref=${cleanup_refs[$repo]}
              crane tag "$cleanup_ref" "$tag" || rollback_failed=1
            done
            for repo in "${!cleanup_refs[@]}"; do
              delete_cleanup_version "$repo" || rollback_failed=1
              wait_for_tag_absent "$repo:$cleanup_tag_name" || rollback_failed=1
            done
            for entry in "${created[@]}"; do
              target=${entry#*|}
              wait_for_tag_absent "$target" || rollback_failed=1
            done
            ((rollback_failed == 0)) || echo "Release-tag rollback was incomplete" >&2
            exit "$status"
          }

          add_targets "${{ steps.ref.outputs.image_ref }}" "$GOCHAT_RELEASE_TAGS"
          add_targets "${{ steps.ref.outputs.shangwutong_image_ref }}" "$SHANGWUTONG_RELEASE_TAGS"

          trap rollback EXIT
          trap 'exit 1' INT TERM
          cleanup_dir=$(mktemp -d)
          printf '%s\n' "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" > "$cleanup_dir/rollback-marker"
          tar -C "$cleanup_dir" -cf "$cleanup_dir/rollback.tar" rollback-marker
          for source_ref in \
            "${{ steps.ref.outputs.image_ref }}" \
            "${{ steps.ref.outputs.shangwutong_image_ref }}"; do
            repo=${source_ref%@*}
            cleanup_tag="$repo:$cleanup_tag_name"
            cleanup_refs["$repo"]=$(crane append \
              --oci-empty-base \
              --new_layer "$cleanup_dir/rollback.tar" \
              --new_tag "$cleanup_tag")
          done

          error_file="$RUNNER_TEMP/crane-digest.err"
          for entry in "${targets[@]}"; do
            target=${entry#*|}
            if digest=$(crane digest "$target" 2> "$error_file"); then
              echo "Refusing to overwrite existing release tag $target ($digest)" >&2
              exit 1
            fi
            grep -Eiq 'MANIFEST_UNKNOWN|manifest unknown|not found|404' "$error_file" || {
              cat "$error_file" >&2
              exit 1
            }
          done

          for entry in "${targets[@]}"; do
            source_ref=${entry%%|*}
            target=${entry#*|}
            repo=${source_ref%@*}
            tag=${target#"$repo:"}
            created+=("$entry")
            crane tag "$source_ref" "$tag"
          done

          for entry in "${targets[@]}"; do
            source_ref=${entry%%|*}
            target=${entry#*|}
            expected=${source_ref##*@}
            actual=$(crane digest "$target")
            [[ $actual == "$expected" ]] || {
              echo "Digest mismatch: $target resolved to $actual, expected $expected" >&2
              exit 1
            }
            echo "$target -> $actual"
            cosign verify \
              --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/ci.yml@refs/tags/v" \
              --certificate-oidc-issuer https://token.actions.githubusercontent.com \
              "$target"
            docker buildx imagetools inspect "$target" --format '{{ json .SBOM }}' | grep -q '"SPDXID"'
            provenance=$(docker buildx imagetools inspect "$target" --format '{{ json .Provenance }}')
            grep -q '"SLSA"' <<< "$provenance"
            grep -q "$GITHUB_SHA" <<< "$provenance"
          done

          trap - EXIT INT TERM
          for repo in "${!cleanup_refs[@]}"; do
            delete_cleanup_version "$repo"
            wait_for_tag_absent "$repo:$cleanup_tag_name"
          done

          echo "GoChat image: \`${{ steps.ref.outputs.image_ref }}\`" >> "$GITHUB_STEP_SUMMARY"
          echo "Shangwutong image: \`${{ steps.ref.outputs.shangwutong_image_ref }}\`" >> "$GITHUB_STEP_SUMMARY"