HH-500: add reproducible production dist bundle (#111)

* HH-500: add reproducible production dist bundle * HH-500: make production bundle builds reproducible * HH-500: lock complete runtime APK closure --------- Co-authored-by: Rogee <rogee@ipao.vip>
2026-08-23 00:22:20 +08:00
parent ef8931099c
commit a5d244d293
10 changed files with 279 additions and 2 deletions
@@ -5,5 +5,6 @@
 frontend/dist
 backend/bin
 backend/coverage.out
+dist
 .env
 .env.*
@@ -46,3 +46,10 @@ PROMETHEUS_RETENTION=30d
 # Optional connector. Supply the digest published by its release pipeline.
 SHANGWUTONG_IMAGE_REF=ghcr.io/rogeecn/shangwutong@sha256:CHANGE_ME
 GOCHAT_CONNECTOR_SERVICE_TOKEN=
+
+# Backup paths. The off-site directory must be an existing external mount.
+GOCHAT_BACKUP_DIR=./backups/local
+GOCHAT_BACKUP_OFFSITE_DIR=/mnt/gochat-offsite
+GOCHAT_BACKUP_OFFSITE_SOURCE=CHANGE_ME_TO_APPROVED_MOUNT_SOURCE
+GOCHAT_BACKUP_OFFSITE_FSTYPE=CHANGE_ME_TO_APPROVED_MOUNT_TYPE
+GOCHAT_BACKUP_PASSPHRASE_FILE=../../.secrets/backup-passphrase
@@ -7,6 +7,7 @@ tmp/
 backend/gochat
 backend/route_parity
 /output/
+/dist/

 # IDE
 .idea/
@@ -16,6 +16,7 @@ FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc
 ARG VERSION=dev
 ARG COMMIT_SHA=unknown
 ARG BUILD_DATE=unknown
+ARG SOURCE_DATE_EPOCH=0
 LABEL org.opencontainers.image.version="${VERSION}" \
      org.opencontainers.image.revision="${COMMIT_SHA}" \
      org.opencontainers.image.created="${BUILD_DATE}"
@@ -61,12 +61,24 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
 ARG VERSION=dev
 ARG COMMIT_SHA=unknown
 ARG BUILD_DATE=unknown
+ARG SOURCE_DATE_EPOCH=0
 LABEL org.opencontainers.image.version="${VERSION}" \
      org.opencontainers.image.revision="${COMMIT_SHA}" \
      org.opencontainers.image.created="${BUILD_DATE}"

-# Install runtime and recovery dependencies.
-RUN apk --no-cache add bash ca-certificates tzdata curl openssl postgresql16-client && \
+# Install and verify the complete pinned runtime dependency closure.
+COPY deploy/docker/gochat-runtime-apks.txt /tmp/gochat-runtime-apks.txt
+RUN apk --no-cache add bash=5.2.37-r0 brotli-libs=1.1.0-r2 busybox=1.37.0-r14 \
+    busybox-binsh=1.37.0-r14 c-ares=1.34.8-r0 ca-certificates=20260413-r0 \
+    ca-certificates-bundle=20260413-r0 curl=8.14.1-r2 libcrypto3=3.3.7-r0 \
+    libcurl=8.14.1-r2 libidn2=2.3.7-r0 libssl3=3.3.7-r0 \
+    libncursesw=6.5_p20241006-r3 libpq=17.11-r0 libpsl=0.21.5-r3 libunistring=1.2-r0 \
+    lz4-libs=1.10.0-r0 musl=1.2.5-r11 ncurses-terminfo-base=6.5_p20241006-r3 \
+    nghttp2-libs=1.69.0-r0 \
+    openssl=3.3.7-r0 postgresql-common=1.2-r1 postgresql16-client=16.15-r0 \
+    readline=8.2.13-r0 tzdata=2026c-r0 zlib=1.3.2-r0 zstd-libs=1.5.6-r2 && \
+    apk info -vv | sed 's/ .*//' | sort | diff -u /tmp/gochat-runtime-apks.txt - && \
+    rm /tmp/gochat-runtime-apks.txt && \
    addgroup -S gochat && adduser -S gochat -G gochat

 WORKDIR /app
@@ -0,0 +1,27 @@
+# GoChat production bundle
+
+This directory is a source-free delivery package. It contains the GoChat and
+Shangwutong images plus every repository file referenced by the production
+Compose configuration.
+
+```bash
+sha256sum -c SHA256SUMS
+docker load --input images/gochat-*-images.tgz
+cp .env.example .env
+# Replace every CHANGE_ME value and create the external secret/mount paths.
+docker compose --env-file .env -f deploy/docker/docker-compose.prod.yml config --quiet
+docker compose --env-file .env -f deploy/docker/docker-compose.prod.yml --profile ops run --rm migrate
+docker compose --env-file .env -f deploy/docker/docker-compose.prod.yml up -d --wait
+```
+
+The bundle uses source-versioned local image tags so it can start immediately
+after `docker load`; `SHA256SUMS` protects the handoff. Before registry-based
+production promotion, push both images, replace the two image references in
+`.env` with immutable `@sha256:` references, then run:
+
+```bash
+deploy/docker/preflight.sh .env
+```
+
+See `docs/ops/02-production-operations.md` for TLS, backup, rollback, and drill
+requirements.
@@ -0,0 +1,35 @@
+alpine-baselayout-3.6.8-r1
+alpine-baselayout-data-3.6.8-r1
+alpine-keys-2.5-r0
+alpine-release-3.21.7-r0
+apk-tools-2.14.6-r3
+bash-5.2.37-r0
+brotli-libs-1.1.0-r2
+busybox-1.37.0-r14
+busybox-binsh-1.37.0-r14
+c-ares-1.34.8-r0
+ca-certificates-20260413-r0
+ca-certificates-bundle-20260413-r0
+curl-8.14.1-r2
+libcrypto3-3.3.7-r0
+libcurl-8.14.1-r2
+libidn2-2.3.7-r0
+libncursesw-6.5_p20241006-r3
+libpq-17.11-r0
+libpsl-0.21.5-r3
+libssl3-3.3.7-r0
+libunistring-1.2-r0
+lz4-libs-1.10.0-r0
+musl-1.2.5-r11
+musl-utils-1.2.5-r11
+ncurses-terminfo-base-6.5_p20241006-r3
+nghttp2-libs-1.69.0-r0
+openssl-3.3.7-r0
+postgresql-common-1.2-r1
+postgresql16-client-16.15-r0
+readline-8.2.13-r0
+scanelf-1.3.8-r1
+ssl_client-1.37.0-r14
+tzdata-2026c-r0
+zlib-1.3.2-r0
+zstd-libs-1.5.6-r2
@@ -11,6 +11,8 @@
    "build:frontend": "cd frontend && pnpm build",
    "verify:frontend:delivery": "pnpm install --frozen-lockfile && pnpm --dir frontend test:build && pnpm audit --prod --audit-level high",
    "build:sdk": "cd frontend && pnpm build:sdk",
+    "build:production": "bash scripts/build-production.sh",
+    "test:production-build": "bash scripts/build-production.test.sh",
    "lint:frontend": "cd frontend && pnpm eslint",
    "test:frontend": "cd frontend && pnpm test",
    "test:shangwutong": "cd channels/shangwutong && go test ./...",
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+set -euo pipefail
+export LC_ALL=C
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+cd "$root"
+
+if [[ -n $(git status --porcelain --untracked-files=normal) ]]; then
+  echo "refusing production build from a dirty worktree" >&2
+  exit 1
+fi
+
+version=${GOCHAT_VERSION:-$(node -p "require('./package.json').version")}
+[[ $version =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || {
+  echo "invalid GOCHAT_VERSION: $version" >&2
+  exit 1
+}
+
+commit=$(git rev-parse --short=12 HEAD)
+build_date=$(git show -s --format=%cI HEAD)
+source_date_epoch=$(git show -s --format=%ct HEAD)
+tag="$version-$commit"
+gochat_image="gochat/gochat:$tag"
+connector_image="gochat/shangwutong:$tag"
+output_dir=$(realpath -m "${GOCHAT_DIST_DIR:-$root/dist}")
+[[ $output_dir != / && $output_dir != "$root" ]] || {
+  echo "invalid GOCHAT_DIST_DIR: $output_dir" >&2
+  exit 1
+}
+mkdir -p "$(dirname "$output_dir")"
+
+stage=$(mktemp -d "$(dirname "$output_dir")/.gochat-dist.XXXXXX")
+builder="gochat-dist-$$-$RANDOM"
+buildkit_image='moby/buildkit:v0.13.2@sha256:9194b5ec1be368f41c516df7f93f7f540630ea06136056b2ffebb62226ed4ad6'
+cleanup() {
+  [[ -z $builder ]] || docker buildx rm "$builder" >/dev/null 2>&1 || true
+  rm -rf "$stage"
+}
+trap cleanup EXIT HUP INT TERM
+
+docker buildx create --name "$builder" --driver docker-container \
+  --driver-opt "image=$buildkit_image" >/dev/null
+build_args=(
+  --build-arg "VERSION=$version"
+  --build-arg "COMMIT_SHA=$commit"
+  --build-arg "BUILD_DATE=$build_date"
+  --build-arg "SOURCE_DATE_EPOCH=$source_date_epoch"
+)
+build_image() {
+  local dockerfile=$1 image=$2 output=$3
+  docker buildx build --builder "$builder" --platform linux/amd64 \
+    "${build_args[@]}" --tag "$image" --file "$dockerfile" \
+    --output "type=docker,rewrite-timestamp=true,dest=$output" .
+  docker load --input "$output"
+}
+
+build_image deploy/docker/Dockerfile "$gochat_image" "$stage/gochat.image.tar"
+build_image channels/shangwutong/Dockerfile "$connector_image" "$stage/shangwutong.image.tar"
+docker image inspect "$gochat_image" "$connector_image" >/dev/null
+docker buildx rm "$builder" >/dev/null
+builder=
+rm "$stage/gochat.image.tar" "$stage/shangwutong.image.tar"
+
+archive="images/gochat-$tag-images.tgz"
+mkdir -p "$stage/images"
+docker save --output "$stage/images.tar" "$gochat_image" "$connector_image"
+mkdir "$stage/.image-archive"
+tar -xf "$stage/images.tar" -C "$stage/.image-archive"
+mapfile -t image_archive_entries < <(find "$stage/.image-archive" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort)
+tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner \
+  -cf "$stage/images.normalized.tar" -C "$stage/.image-archive" "${image_archive_entries[@]}"
+rm -rf "$stage/.image-archive" "$stage/images.tar"
+mv "$stage/images.normalized.tar" "$stage/images.tar"
+gzip -n "$stage/images.tar"
+mv "$stage/images.tar.gz" "$stage/$archive"
+
+files=(
+  deploy/docker/docker-compose.prod.yml
+  deploy/docker/docker-compose.prod-smoke.yml
+  deploy/docker/database_client_entrypoint.sh
+  deploy/docker/database_host_rejection_cases.txt
+  deploy/docker/preflight.sh
+  deploy/fluentd/fluent.conf
+  deploy/prometheus/alertmanager.yml
+  deploy/prometheus/blackbox.yml
+  deploy/prometheus/postgres_queries.yml
+  deploy/prometheus/prometheus.yml
+  backend/configs/prometheus_alerts.yml
+  docs/ops/01-rolling-upgrade.md
+  docs/ops/02-production-operations.md
+)
+for file in "${files[@]}"; do
+  mkdir -p "$stage/$(dirname "$file")"
+  cp "$file" "$stage/$file"
+done
+
+sed \
+  -e "s|^GOCHAT_IMAGE_REF=.*|GOCHAT_IMAGE_REF=$gochat_image|" \
+  -e "s|^SHANGWUTONG_IMAGE_REF=.*|SHANGWUTONG_IMAGE_REF=$connector_image|" \
+  .env.example > "$stage/.env.example"
+cp deploy/docker/README.dist.md "$stage/README.md"
+printf 'SOURCE_COMMIT=%s\nGOCHAT_IMAGE_REF=%s\nSHANGWUTONG_IMAGE_REF=%s\n' \
+  "$(git rev-parse HEAD)" "$gochat_image" "$connector_image" > "$stage/MANIFEST"
+(cd "$stage" && find . -type f ! -name SHA256SUMS -print | sort | xargs sha256sum > SHA256SUMS)
+
+rm -rf "$output_dir"
+mv "$stage" "$output_dir"
+trap - EXIT HUP INT TERM
+echo "Production bundle: $output_dir"
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+test_tmp=$(mktemp -d)
+dirty_file=
+trap 'rm -rf "$test_tmp"; [[ -z $dirty_file ]] || rm -f "$dirty_file"' EXIT HUP INT TERM
+real_docker=$(command -v docker)
+output="$test_tmp/dist"
+
+mkdir -p "$test_tmp/bin"
+cat > "$test_tmp/bin/docker" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "$*" >> "$DOCKER_LOG"
+[[ $1 == buildx && $2 == create ]] && exit 0
+[[ $1 == buildx && $2 == rm ]] && exit 0
+if [[ $1 == buildx && $2 == build ]]; then
+  for arg in "$@"; do
+    [[ $arg == type=docker,*dest=* ]] && output=${arg##*dest=}
+  done
+  fake_archive=$(mktemp -d)
+  printf '[]\n' > "$fake_archive/manifest.json"
+  tar -cf "$output" -C "$fake_archive" manifest.json
+  rm -rf "$fake_archive"
+  exit 0
+fi
+[[ $1 == load && $2 == --input && -s $3 ]] && exit 0
+[[ $1 == image && $2 == inspect ]] && exit 0
+if [[ $1 == save && $2 == --output ]]; then
+  fake_archive=$(mktemp -d)
+  printf '[]\n' > "$fake_archive/manifest.json"
+  tar -cf "$3" -C "$fake_archive" manifest.json
+  rm -rf "$fake_archive"
+  exit 0
+fi
+exit 1
+EOF
+chmod +x "$test_tmp/bin/docker"
+
+cd "$root"
+dirty_file=$(mktemp "$root/.build-production-dirty.XXXXXX")
+if PATH="$test_tmp/bin:$PATH" DOCKER_LOG="$test_tmp/docker.log" GOCHAT_DIST_DIR="$output" \
+  GOCHAT_VERSION=1.2.3 bash scripts/build-production.sh >"$test_tmp/dirty.out" 2>&1; then
+  echo 'dirty worktree build unexpectedly succeeded' >&2
+  exit 1
+fi
+grep -q 'refusing production build from a dirty worktree' "$test_tmp/dirty.out"
+rm "$dirty_file"
+dirty_file=
+
+: > "$test_tmp/docker.log"
+PATH="$test_tmp/bin:$PATH" DOCKER_LOG="$test_tmp/docker.log" GOCHAT_DIST_DIR="$output" \
+  GOCHAT_VERSION=1.2.3 bash scripts/build-production.sh
+
+tag=1.2.3-$(git rev-parse --short=12 HEAD)
+archive="$output/images/gochat-$tag-images.tgz"
+test -s "$archive"
+gzip -t "$archive"
+(cd "$output" && sha256sum -c SHA256SUMS)
+grep -q '^GOCHAT_IMAGE_REF=gochat/gochat:1.2.3-' "$output/.env.example"
+grep -q '^SHANGWUTONG_IMAGE_REF=gochat/shangwutong:1.2.3-' "$output/.env.example"
+test -f "$output/deploy/fluentd/fluent.conf"
+test -f "$output/deploy/prometheus/prometheus.yml"
+test -f "$output/backend/configs/prometheus_alerts.yml"
+test "$(grep -c '^buildx build ' "$test_tmp/docker.log")" -eq 2
+test "$(grep -c '^load --input ' "$test_tmp/docker.log")" -eq 2
+test "$(grep -Fc -- "--build-arg VERSION=1.2.3 --build-arg COMMIT_SHA=$(git rev-parse --short=12 HEAD) --build-arg BUILD_DATE=$(git show -s --format=%cI HEAD) --build-arg SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)" "$test_tmp/docker.log")" -eq 2
+grep -Fq "image inspect gochat/gochat:$tag gochat/shangwutong:$tag" "$test_tmp/docker.log"
+test "$(grep -c '^save --output ' "$test_tmp/docker.log")" -eq 1
+grep '^save --output ' "$test_tmp/docker.log" | \
+  grep -Fq " gochat/gochat:$tag gochat/shangwutong:$tag"
+
+cp "$output/.env.example" "$output/.env"
+(cd "$output" && "$real_docker" compose --env-file .env -f deploy/docker/docker-compose.prod.yml config --quiet)
+rm "$output/.env"
+first_sum=$(sha256sum "$archive")
+PATH="$test_tmp/bin:$PATH" DOCKER_LOG="$test_tmp/docker.log" GOCHAT_DIST_DIR="$output" \
+  GOCHAT_VERSION=1.2.3 bash scripts/build-production.sh
+test "$first_sum" = "$(sha256sum "$archive")"
+
+echo 'production build script test passed'