HH-443: establish production CI and supply-chain evidence (#93)

* ci: enforce production release gates * ci: close release gate review blockers * ci: preserve verified digests during promotion * ci: serialize and verify release cleanup * ci: build govulncheck with Shangwutong toolchain --------- Co-authored-by: Rogee <rogee@ipao.vip>
2026-08-22 03:26:38 +08:00
parent 798ea43c2f
commit 02a188dc29
17 changed files with 483 additions and 89 deletions
@@ -13,21 +13,27 @@ permissions:
 env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
+  SHANGWUTONG_IMAGE_NAME: ${{ github.repository_owner }}/shangwutong
+  GOSEC_VERSION: v2.28.0
+  GOVULNCHECK_VERSION: v1.1.4
+  TRIVY_VERSION: v0.74.0
+  COSIGN_VERSION: v3.1.3
+  CRANE_VERSION: v0.21.9

 jobs:
  backend:
    name: Backend (SQLite)
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
-          go-version-file: backend/go.mod
+          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.12"
-      - uses: browser-actions/setup-chrome@v2
+      - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2
        id: chrome
      - name: Install browser harness
        run: python -m pip install browser-harness==0.1.9
@@ -60,7 +66,7 @@ jobs:
    runs-on: ubuntu-latest
    services:
      postgres:
-        image: pgvector/pgvector:pg16
+        image: pgvector/pgvector:pg16@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
        env:
          POSTGRES_DB: gochat_test
          POSTGRES_USER: postgres
@@ -73,10 +79,10 @@ jobs:
          --health-timeout 5s
          --health-retries 5
    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
-          go-version-file: backend/go.mod
+          go-version: 1.25.13
          cache-dependency-path: backend/go.sum
      - name: Test auto-assignment concurrency
        working-directory: backend
@@ -91,6 +97,12 @@ jobs:
            echo "PostgreSQL auto-assignment concurrency test did not run" >&2
            exit 1
          }
+      - name: Test backend against PostgreSQL
+        working-directory: backend
+        env:
+          GOCHAT_TEST_DB: postgres
+          GOCHAT_TEST_DB_URL: postgres://postgres:postgres@localhost:5432/gochat_test?sslmode=disable
+        run: go test -count=1 -timeout 10m ./internal/... ./pkg/... ./cmd/...
      - name: Test production upload migrations
        working-directory: backend
        env:
@@ -155,21 +167,58 @@ jobs:
    name: Frontend Build
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 20
-      - run: corepack enable
+      - run: corepack enable && corepack prepare pnpm@10.2.0 --activate
      - run: pnpm install --frozen-lockfile
+      - name: Test
+        run: pnpm --dir frontend test
      - name: ESLint representative JS/Vue files
        run: pnpm --dir frontend exec eslint app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.spec.js app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.vue
      - name: Prettier representative JS/TS/Vue files
        run: pnpm --dir frontend exec prettier --check app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.spec.js app/javascript/dashboard/components-next/captain/assistant/AssistantPlayground.vue vite.config.ts app/javascript/histoire.setup.ts
-      - run: pnpm --dir frontend build
+      - name: Build and verify artifacts
+        run: pnpm --dir frontend test:build
+
+  security:
+    name: Security (SAST/SCA)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - name: Test release promotion safeguards
+        run: .github/scripts/test_release_promotion.sh
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 1.25.13
+          cache-dependency-path: |
+            backend/go.sum
+            channels/shangwutong/go.sum
+      - name: Install pinned Go security tools
+        env:
+          GOBIN: ${{ runner.temp }}/security-tools
+        run: |
+          go install github.com/securego/gosec/v2/cmd/gosec@"${GOSEC_VERSION}"
+          GOTOOLCHAIN=go1.26.6+auto go install golang.org/x/vuln/cmd/govulncheck@"${GOVULNCHECK_VERSION}"
+          echo "$GOBIN" >> "$GITHUB_PATH"
+      - name: Scan Go code and dependencies
+        run: |
+          (cd backend && gosec -quiet -severity high -confidence high ./... && govulncheck ./...)
+          (cd channels/shangwutong && GOTOOLCHAIN=go1.26.6+auto gosec -quiet -severity high -confidence high ./... && GOTOOLCHAIN=go1.26.6+auto govulncheck ./...)
+      - name: Scan repository dependencies
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          scan-type: fs
+          scan-ref: .
+          scanners: vuln
+          version: ${{ env.TRIVY_VERSION }}
+          severity: HIGH,CRITICAL
+          exit-code: 1

  production-smoke:
    name: Production image smoke
-    if: github.event_name == 'pull_request'
+    if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    env:
      GOCHAT_IMAGE_REF: gochat:production-smoke
@@ -180,11 +229,12 @@ jobs:
      MEILI_MASTER_KEY: ci-meili-secret-16
      GOCHAT_JWT_SECRET: ci-smoke-jwt-secret-at-least-32-characters
      GOCHAT_JWT_PREVIOUS_SECRETS: ci-previous-jwt-secret-at-least-32-characters
+      GOCHAT_CONNECTOR_SERVICE_TOKEN: ci-connector-service-token
      GOCHAT_BACKUP_OFFSITE_DIR: /mnt/gochat-offsite
      GOCHAT_BACKUP_OFFSITE_SOURCE: backup.example.test:/gochat
      GOCHAT_BACKUP_OFFSITE_FSTYPE: nfs4
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      - name: Build production images from repository root
        run: |
          docker build -t "$GOCHAT_IMAGE_REF" -f deploy/docker/Dockerfile .
@@ -197,20 +247,38 @@ jobs:
          docker push localhost:5000/shangwutong:production-smoke
          echo "GOCHAT_IMAGE_REF=$(docker inspect --format '{{index .RepoDigests 0}}' localhost:5000/gochat:production-smoke)" >> "$GITHUB_ENV"
          echo "SHANGWUTONG_IMAGE_REF=$(docker inspect --format '{{index .RepoDigests 0}}' localhost:5000/shangwutong:production-smoke)" >> "$GITHUB_ENV"
+      - name: Scan GoChat image
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          image-ref: gochat:production-smoke
+          scanners: vuln
+          version: ${{ env.TRIVY_VERSION }}
+          severity: HIGH,CRITICAL
+          exit-code: 1
+      - name: Scan Shangwutong image
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          image-ref: shangwutong:production-smoke
+          scanners: vuln
+          version: ${{ env.TRIVY_VERSION }}
+          severity: HIGH,CRITICAL
+          exit-code: 1
      - name: Verify artifact contents
-        run: >-
-          docker run --rm --entrypoint sh "$GOCHAT_IMAGE_REF" -c
-          'test -s /app/configs/config.production.yaml &&
-          test -d /app/migrations &&
-          test -s /app/frontend/dist/index.html'
+        run: |
+          docker run --rm --entrypoint sh "$GOCHAT_IMAGE_REF" -c \
+            'test -s /app/configs/config.production.yaml && test -d /app/migrations && test -s /app/frontend/dist/index.html'
+          docker run --rm --entrypoint sh "$SHANGWUTONG_IMAGE_REF" -c \
+            'test -x /usr/local/bin/shangwutong && test -d /data && test -d /backup && test "$(id -u)" -eq 10001'
      - name: Start production Compose and smoke core pages
        run: |
          deploy/docker/preflight_test.sh
          docker compose -f deploy/docker/docker-compose.prod.yml config --format json | python3 -c 'import json, os, sys; config = json.load(sys.stdin); assert all(config["services"][service]["environment"]["GOCHAT_JWT_PREVIOUS_SECRETS"] == os.environ["GOCHAT_JWT_PREVIOUS_SECRETS"] for service in ("gochat", "worker"))'
          docker compose -f deploy/docker/docker-compose.prod.yml --profile ops run --rm migrate
-          docker compose -f deploy/docker/docker-compose.prod.yml up -d --wait gochat
+          docker compose -f deploy/docker/docker-compose.prod.yml up -d --wait gochat shangwutong
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/health" | grep -q '"status":"healthy"'
          curl -fsS "http://127.0.0.1:$GOCHAT_PORT/app" | grep -q '/assets/'
+          docker compose -f deploy/docker/docker-compose.prod.yml exec -T shangwutong \
+            wget -q -T 3 -O - http://127.0.0.1:9100/readyz | grep -q '"status":"ready"'
      - name: Stop production Compose
        if: always()
        run: |
@@ -218,54 +286,261 @@ jobs:
          docker rm -f gochat-ci-registry || true

  release:
-    name: Publish immutable image
+    name: Publish signed immutable images
    if: startsWith(github.ref, 'refs/tags/v')
-    needs: [backend, backend-postgres, frontend]
+    needs: [backend, backend-postgres, frontend, security, production-smoke]
    runs-on: ubuntu-latest
+    concurrency:
+      group: release-promotion
+      cancel-in-progress: false
    permissions:
      contents: read
      packages: write
+      id-token: write
    outputs:
      image_ref: ${{ steps.ref.outputs.image_ref }}
      image_digest: ${{ steps.build.outputs.digest }}
+      shangwutong_image_ref: ${{ steps.ref.outputs.shangwutong_image_ref }}
+      shangwutong_image_digest: ${{ steps.shangwutong-build.outputs.digest }}
    steps:
-      - uses: actions/checkout@v4
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/login-action@v3
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Extract immutable tags
        id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: latest=false
          tags: |
            type=semver,pattern={{version}}
            type=sha
-      - name: Capture build date
+      - name: Extract Shangwutong immutable tags
+        id: shangwutong-meta
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.SHANGWUTONG_IMAGE_NAME }}
+          flavor: latest=false
+          tags: |
+            type=semver,pattern={{version}}
+            type=sha
+      - name: Capture reproducible build metadata
        id: date
-        run: echo "value=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT"
-      - name: Build and push
+        run: |
+          epoch=$(git show -s --format=%ct "$GITHUB_SHA")
+          echo "epoch=$epoch" >> "$GITHUB_OUTPUT"
+          echo "value=$(date -u -d "@$epoch" +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT"
+      - name: Define isolated staging references
+        id: staging
+        run: |
+          suffix="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          echo "gochat=${REGISTRY}/${IMAGE_NAME}:$suffix" >> "$GITHUB_OUTPUT"
+          echo "shangwutong=${REGISTRY}/${SHANGWUTONG_IMAGE_NAME}:$suffix" >> "$GITHUB_OUTPUT"
+      - name: Build and push GoChat staging image
        id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: .
          file: deploy/docker/Dockerfile
          push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          sbom: true
+          provenance: mode=max
+          tags: ${{ steps.staging.outputs.gochat }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            VERSION=${{ github.ref_name }}
            COMMIT_SHA=${{ github.sha }}
            BUILD_DATE=${{ steps.date.outputs.value }}
+            SOURCE_DATE_EPOCH=${{ steps.date.outputs.epoch }}
          cache-from: type=gha,scope=gochat
          cache-to: type=gha,mode=max,scope=gochat
-      - name: Record digest reference
+      - name: Build and push Shangwutong staging image
+        id: shangwutong-build
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        with:
+          context: .
+          file: channels/shangwutong/Dockerfile
+          push: true
+          sbom: true
+          provenance: mode=max
+          tags: ${{ steps.staging.outputs.shangwutong }}
+          labels: ${{ steps.shangwutong-meta.outputs.labels }}
+          build-args: |
+            VERSION=${{ github.ref_name }}
+            COMMIT_SHA=${{ github.sha }}
+            BUILD_DATE=${{ steps.date.outputs.value }}
+            SOURCE_DATE_EPOCH=${{ steps.date.outputs.epoch }}
+          cache-from: type=gha,scope=shangwutong
+          cache-to: type=gha,mode=max,scope=shangwutong
+      - name: Record staging digest references
        id: ref
        run: |
          image_ref="${REGISTRY}/${IMAGE_NAME}@${{ steps.build.outputs.digest }}"
+          shangwutong_image_ref="${REGISTRY}/${SHANGWUTONG_IMAGE_NAME}@${{ steps.shangwutong-build.outputs.digest }}"
          echo "image_ref=$image_ref" >> "$GITHUB_OUTPUT"
-          echo "GoChat image: \`$image_ref\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "shangwutong_image_ref=$shangwutong_image_ref" >> "$GITHUB_OUTPUT"
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+        with:
+          cosign-release: ${{ env.COSIGN_VERSION }}
+      - name: Install crane
+        uses: imjasonh/setup-crane@feee3b6bb0d4c68370f256a4502498c9227e5c6b # v0.7
+        with:
+          version: ${{ env.CRANE_VERSION }}
+      - name: Sign image digests
+        run: |
+          cosign sign --yes "${{ steps.ref.outputs.image_ref }}"
+          cosign sign --yes "${{ steps.ref.outputs.shangwutong_image_ref }}"
+      - name: Verify signatures and attestations
+        run: |
+          for image_ref in "${{ steps.ref.outputs.image_ref }}" "${{ steps.ref.outputs.shangwutong_image_ref }}"; do
+            cosign verify \
+              --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/ci.yml@refs/tags/v" \
+              --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+              "$image_ref"
+            docker buildx imagetools inspect "$image_ref" --format '{{ json .SBOM }}' | grep -q '"SPDXID"'
+            provenance=$(docker buildx imagetools inspect "$image_ref" --format '{{ json .Provenance }}')
+            grep -q '"SLSA"' <<< "$provenance"
+            grep -q "$GITHUB_SHA" <<< "$provenance"
+          done
+      - name: Promote and reverify release tags
+        env:
+          GOCHAT_RELEASE_TAGS: ${{ steps.meta.outputs.tags }}
+          SHANGWUTONG_RELEASE_TAGS: ${{ steps.shangwutong-meta.outputs.tags }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PACKAGE_OWNER: ${{ github.repository_owner }}
+          PACKAGE_OWNER_TYPE: ${{ github.event.repository.owner.type }}
+        run: |
+          set -euo pipefail
+          source .github/scripts/release_helpers.sh
+
+          declare -a targets=() created=()
+          declare -A cleanup_refs=()
+          cleanup_tag_name="rollback-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+
+          add_targets() {
+            local source_ref=$1 tags=$2 repo=${1%@*} target count=0
+            while IFS= read -r target; do
+              [[ -n $target ]] || continue
+              [[ $target == "$repo:"* ]] || {
+                echo "Release tag is outside source repository: $target" >&2
+                return 1
+              }
+              targets+=("$source_ref|$target")
+              ((++count))
+            done <<< "$tags"
+            ((count > 0))
+          }
+
+          delete_cleanup_version() {
+            local repo=$1 cleanup_ref=${cleanup_refs[$1]} package endpoint
+            package=${repo#"ghcr.io/$PACKAGE_OWNER/"}
+            [[ $package != "$repo" ]] || return 1
+            package=$(jq -rn --arg value "$package" '$value | @uri')
+            case $PACKAGE_OWNER_TYPE in
+              User) endpoint="/users/$PACKAGE_OWNER/packages/container/$package/versions" ;;
+              Organization) endpoint="/orgs/$PACKAGE_OWNER/packages/container/$package/versions" ;;
+              *) return 1 ;;
+            esac
+            delete_package_version "$endpoint" "$cleanup_tag_name" "${cleanup_ref##*@}"
+          }
+
+          rollback() {
+            local status=$? entry source_ref target repo tag cleanup_ref rollback_failed=0
+            ((status != 0)) || return
+            trap - EXIT INT TERM
+            set +e
+            echo "Promotion failed; removing created release tags" >&2
+            for entry in "${created[@]}"; do
+              source_ref=${entry%%|*}
+              target=${entry#*|}
+              repo=${source_ref%@*}
+              tag=${target#"$repo:"}
+              cleanup_ref=${cleanup_refs[$repo]}
+              crane tag "$cleanup_ref" "$tag" || rollback_failed=1
+            done
+            for repo in "${!cleanup_refs[@]}"; do
+              delete_cleanup_version "$repo" || rollback_failed=1
+              wait_for_tag_absent "$repo:$cleanup_tag_name" || rollback_failed=1
+            done
+            for entry in "${created[@]}"; do
+              target=${entry#*|}
+              wait_for_tag_absent "$target" || rollback_failed=1
+            done
+            ((rollback_failed == 0)) || echo "Release-tag rollback was incomplete" >&2
+            exit "$status"
+          }
+
+          add_targets "${{ steps.ref.outputs.image_ref }}" "$GOCHAT_RELEASE_TAGS"
+          add_targets "${{ steps.ref.outputs.shangwutong_image_ref }}" "$SHANGWUTONG_RELEASE_TAGS"
+
+          trap rollback EXIT
+          trap 'exit 1' INT TERM
+          cleanup_dir=$(mktemp -d)
+          printf '%s\n' "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" > "$cleanup_dir/rollback-marker"
+          tar -C "$cleanup_dir" -cf "$cleanup_dir/rollback.tar" rollback-marker
+          for source_ref in \
+            "${{ steps.ref.outputs.image_ref }}" \
+            "${{ steps.ref.outputs.shangwutong_image_ref }}"; do
+            repo=${source_ref%@*}
+            cleanup_tag="$repo:$cleanup_tag_name"
+            cleanup_refs["$repo"]=$(crane append \
+              --oci-empty-base \
+              --new_layer "$cleanup_dir/rollback.tar" \
+              --new_tag "$cleanup_tag")
+          done
+
+          error_file="$RUNNER_TEMP/crane-digest.err"
+          for entry in "${targets[@]}"; do
+            target=${entry#*|}
+            if digest=$(crane digest "$target" 2> "$error_file"); then
+              echo "Refusing to overwrite existing release tag $target ($digest)" >&2
+              exit 1
+            fi
+            grep -Eiq 'MANIFEST_UNKNOWN|manifest unknown|not found|404' "$error_file" || {
+              cat "$error_file" >&2
+              exit 1
+            }
+          done
+
+          for entry in "${targets[@]}"; do
+            source_ref=${entry%%|*}
+            target=${entry#*|}
+            repo=${source_ref%@*}
+            tag=${target#"$repo:"}
+            created+=("$entry")
+            crane tag "$source_ref" "$tag"
+          done
+
+          for entry in "${targets[@]}"; do
+            source_ref=${entry%%|*}
+            target=${entry#*|}
+            expected=${source_ref##*@}
+            actual=$(crane digest "$target")
+            [[ $actual == "$expected" ]] || {
+              echo "Digest mismatch: $target resolved to $actual, expected $expected" >&2
+              exit 1
+            }
+            echo "$target -> $actual"
+            cosign verify \
+              --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/ci.yml@refs/tags/v" \
+              --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+              "$target"
+            docker buildx imagetools inspect "$target" --format '{{ json .SBOM }}' | grep -q '"SPDXID"'
+            provenance=$(docker buildx imagetools inspect "$target" --format '{{ json .Provenance }}')
+            grep -q '"SLSA"' <<< "$provenance"
+            grep -q "$GITHUB_SHA" <<< "$provenance"
+          done
+
+          trap - EXIT INT TERM
+          for repo in "${!cleanup_refs[@]}"; do
+            delete_cleanup_version "$repo"
+            wait_for_tag_absent "$repo:$cleanup_tag_name"
+          done
+
+          echo "GoChat image: \`${{ steps.ref.outputs.image_ref }}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "Shangwutong image: \`${{ steps.ref.outputs.shangwutong_image_ref }}\`" >> "$GITHUB_STEP_SUMMARY"