From a5d244d2933f59fcb087e58c99009242a2e459f8 Mon Sep 17 00:00:00 2001 From: Rogee Date: Sun, 23 Aug 2026 00:22:20 +0800 Subject: [PATCH] HH-500: add reproducible production dist bundle (#111) * HH-500: add reproducible production dist bundle * HH-500: make production bundle builds reproducible * HH-500: lock complete runtime APK closure --------- Co-authored-by: Rogee --- .dockerignore | 1 + .env.example | 7 ++ .gitignore | 1 + channels/shangwutong/Dockerfile | 1 + deploy/docker/Dockerfile | 16 +++- deploy/docker/README.dist.md | 27 +++++++ deploy/docker/gochat-runtime-apks.txt | 35 +++++++++ package.json | 2 + scripts/build-production.sh | 109 ++++++++++++++++++++++++++ scripts/build-production.test.sh | 82 +++++++++++++++++++ 10 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 deploy/docker/README.dist.md create mode 100644 deploy/docker/gochat-runtime-apks.txt create mode 100644 scripts/build-production.sh create mode 100644 scripts/build-production.test.sh diff --git a/.dockerignore b/.dockerignore index dbb1eac6..12cbc83c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,6 @@ frontend/dist backend/bin backend/coverage.out +dist .env .env.* diff --git a/.env.example b/.env.example index 12865908..cf804f8d 100644 --- a/.env.example +++ b/.env.example @@ -46,3 +46,10 @@ PROMETHEUS_RETENTION=30d # Optional connector. Supply the digest published by its release pipeline. SHANGWUTONG_IMAGE_REF=ghcr.io/rogeecn/shangwutong@sha256:CHANGE_ME GOCHAT_CONNECTOR_SERVICE_TOKEN= + +# Backup paths. The off-site directory must be an existing external mount. +GOCHAT_BACKUP_DIR=./backups/local +GOCHAT_BACKUP_OFFSITE_DIR=/mnt/gochat-offsite +GOCHAT_BACKUP_OFFSITE_SOURCE=CHANGE_ME_TO_APPROVED_MOUNT_SOURCE +GOCHAT_BACKUP_OFFSITE_FSTYPE=CHANGE_ME_TO_APPROVED_MOUNT_TYPE +GOCHAT_BACKUP_PASSPHRASE_FILE=../../.secrets/backup-passphrase diff --git a/.gitignore b/.gitignore index 1cee7ea8..3ce5d445 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ tmp/ backend/gochat backend/route_parity /output/ +/dist/ # IDE .idea/ diff --git a/channels/shangwutong/Dockerfile b/channels/shangwutong/Dockerfile index 9255e5fc..25b68dd5 100644 --- a/channels/shangwutong/Dockerfile +++ b/channels/shangwutong/Dockerfile @@ -16,6 +16,7 @@ FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc ARG VERSION=dev ARG COMMIT_SHA=unknown ARG BUILD_DATE=unknown +ARG SOURCE_DATE_EPOCH=0 LABEL org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.created="${BUILD_DATE}" diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 6fe22a9f..075884f0 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -61,12 +61,24 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650 ARG VERSION=dev ARG COMMIT_SHA=unknown ARG BUILD_DATE=unknown +ARG SOURCE_DATE_EPOCH=0 LABEL org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.created="${BUILD_DATE}" -# Install runtime and recovery dependencies. -RUN apk --no-cache add bash ca-certificates tzdata curl openssl postgresql16-client && \ +# Install and verify the complete pinned runtime dependency closure. +COPY deploy/docker/gochat-runtime-apks.txt /tmp/gochat-runtime-apks.txt +RUN apk --no-cache add bash=5.2.37-r0 brotli-libs=1.1.0-r2 busybox=1.37.0-r14 \ + busybox-binsh=1.37.0-r14 c-ares=1.34.8-r0 ca-certificates=20260413-r0 \ + ca-certificates-bundle=20260413-r0 curl=8.14.1-r2 libcrypto3=3.3.7-r0 \ + libcurl=8.14.1-r2 libidn2=2.3.7-r0 libssl3=3.3.7-r0 \ + libncursesw=6.5_p20241006-r3 libpq=17.11-r0 libpsl=0.21.5-r3 libunistring=1.2-r0 \ + lz4-libs=1.10.0-r0 musl=1.2.5-r11 ncurses-terminfo-base=6.5_p20241006-r3 \ + nghttp2-libs=1.69.0-r0 \ + openssl=3.3.7-r0 postgresql-common=1.2-r1 postgresql16-client=16.15-r0 \ + readline=8.2.13-r0 tzdata=2026c-r0 zlib=1.3.2-r0 zstd-libs=1.5.6-r2 && \ + apk info -vv | sed 's/ .*//' | sort | diff -u /tmp/gochat-runtime-apks.txt - && \ + rm /tmp/gochat-runtime-apks.txt && \ addgroup -S gochat && adduser -S gochat -G gochat WORKDIR /app diff --git a/deploy/docker/README.dist.md b/deploy/docker/README.dist.md new file mode 100644 index 00000000..cef123fc --- /dev/null +++ b/deploy/docker/README.dist.md @@ -0,0 +1,27 @@ +# GoChat production bundle + +This directory is a source-free delivery package. It contains the GoChat and +Shangwutong images plus every repository file referenced by the production +Compose configuration. + +```bash +sha256sum -c SHA256SUMS +docker load --input images/gochat-*-images.tgz +cp .env.example .env +# Replace every CHANGE_ME value and create the external secret/mount paths. +docker compose --env-file .env -f deploy/docker/docker-compose.prod.yml config --quiet +docker compose --env-file .env -f deploy/docker/docker-compose.prod.yml --profile ops run --rm migrate +docker compose --env-file .env -f deploy/docker/docker-compose.prod.yml up -d --wait +``` + +The bundle uses source-versioned local image tags so it can start immediately +after `docker load`; `SHA256SUMS` protects the handoff. Before registry-based +production promotion, push both images, replace the two image references in +`.env` with immutable `@sha256:` references, then run: + +```bash +deploy/docker/preflight.sh .env +``` + +See `docs/ops/02-production-operations.md` for TLS, backup, rollback, and drill +requirements. diff --git a/deploy/docker/gochat-runtime-apks.txt b/deploy/docker/gochat-runtime-apks.txt new file mode 100644 index 00000000..d2427500 --- /dev/null +++ b/deploy/docker/gochat-runtime-apks.txt @@ -0,0 +1,35 @@ +alpine-baselayout-3.6.8-r1 +alpine-baselayout-data-3.6.8-r1 +alpine-keys-2.5-r0 +alpine-release-3.21.7-r0 +apk-tools-2.14.6-r3 +bash-5.2.37-r0 +brotli-libs-1.1.0-r2 +busybox-1.37.0-r14 +busybox-binsh-1.37.0-r14 +c-ares-1.34.8-r0 +ca-certificates-20260413-r0 +ca-certificates-bundle-20260413-r0 +curl-8.14.1-r2 +libcrypto3-3.3.7-r0 +libcurl-8.14.1-r2 +libidn2-2.3.7-r0 +libncursesw-6.5_p20241006-r3 +libpq-17.11-r0 +libpsl-0.21.5-r3 +libssl3-3.3.7-r0 +libunistring-1.2-r0 +lz4-libs-1.10.0-r0 +musl-1.2.5-r11 +musl-utils-1.2.5-r11 +ncurses-terminfo-base-6.5_p20241006-r3 +nghttp2-libs-1.69.0-r0 +openssl-3.3.7-r0 +postgresql-common-1.2-r1 +postgresql16-client-16.15-r0 +readline-8.2.13-r0 +scanelf-1.3.8-r1 +ssl_client-1.37.0-r14 +tzdata-2026c-r0 +zlib-1.3.2-r0 +zstd-libs-1.5.6-r2 diff --git a/package.json b/package.json index ebde2ee4..15cb5ce6 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,8 @@ "build:frontend": "cd frontend && pnpm build", "verify:frontend:delivery": "pnpm install --frozen-lockfile && pnpm --dir frontend test:build && pnpm audit --prod --audit-level high", "build:sdk": "cd frontend && pnpm build:sdk", + "build:production": "bash scripts/build-production.sh", + "test:production-build": "bash scripts/build-production.test.sh", "lint:frontend": "cd frontend && pnpm eslint", "test:frontend": "cd frontend && pnpm test", "test:shangwutong": "cd channels/shangwutong && go test ./...", diff --git a/scripts/build-production.sh b/scripts/build-production.sh new file mode 100644 index 00000000..f10971dd --- /dev/null +++ b/scripts/build-production.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" + +if [[ -n $(git status --porcelain --untracked-files=normal) ]]; then + echo "refusing production build from a dirty worktree" >&2 + exit 1 +fi + +version=${GOCHAT_VERSION:-$(node -p "require('./package.json').version")} +[[ $version =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || { + echo "invalid GOCHAT_VERSION: $version" >&2 + exit 1 +} + +commit=$(git rev-parse --short=12 HEAD) +build_date=$(git show -s --format=%cI HEAD) +source_date_epoch=$(git show -s --format=%ct HEAD) +tag="$version-$commit" +gochat_image="gochat/gochat:$tag" +connector_image="gochat/shangwutong:$tag" +output_dir=$(realpath -m "${GOCHAT_DIST_DIR:-$root/dist}") +[[ $output_dir != / && $output_dir != "$root" ]] || { + echo "invalid GOCHAT_DIST_DIR: $output_dir" >&2 + exit 1 +} +mkdir -p "$(dirname "$output_dir")" + +stage=$(mktemp -d "$(dirname "$output_dir")/.gochat-dist.XXXXXX") +builder="gochat-dist-$$-$RANDOM" +buildkit_image='moby/buildkit:v0.13.2@sha256:9194b5ec1be368f41c516df7f93f7f540630ea06136056b2ffebb62226ed4ad6' +cleanup() { + [[ -z $builder ]] || docker buildx rm "$builder" >/dev/null 2>&1 || true + rm -rf "$stage" +} +trap cleanup EXIT HUP INT TERM + +docker buildx create --name "$builder" --driver docker-container \ + --driver-opt "image=$buildkit_image" >/dev/null +build_args=( + --build-arg "VERSION=$version" + --build-arg "COMMIT_SHA=$commit" + --build-arg "BUILD_DATE=$build_date" + --build-arg "SOURCE_DATE_EPOCH=$source_date_epoch" +) +build_image() { + local dockerfile=$1 image=$2 output=$3 + docker buildx build --builder "$builder" --platform linux/amd64 \ + "${build_args[@]}" --tag "$image" --file "$dockerfile" \ + --output "type=docker,rewrite-timestamp=true,dest=$output" . + docker load --input "$output" +} + +build_image deploy/docker/Dockerfile "$gochat_image" "$stage/gochat.image.tar" +build_image channels/shangwutong/Dockerfile "$connector_image" "$stage/shangwutong.image.tar" +docker image inspect "$gochat_image" "$connector_image" >/dev/null +docker buildx rm "$builder" >/dev/null +builder= +rm "$stage/gochat.image.tar" "$stage/shangwutong.image.tar" + +archive="images/gochat-$tag-images.tgz" +mkdir -p "$stage/images" +docker save --output "$stage/images.tar" "$gochat_image" "$connector_image" +mkdir "$stage/.image-archive" +tar -xf "$stage/images.tar" -C "$stage/.image-archive" +mapfile -t image_archive_entries < <(find "$stage/.image-archive" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort) +tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner \ + -cf "$stage/images.normalized.tar" -C "$stage/.image-archive" "${image_archive_entries[@]}" +rm -rf "$stage/.image-archive" "$stage/images.tar" +mv "$stage/images.normalized.tar" "$stage/images.tar" +gzip -n "$stage/images.tar" +mv "$stage/images.tar.gz" "$stage/$archive" + +files=( + deploy/docker/docker-compose.prod.yml + deploy/docker/docker-compose.prod-smoke.yml + deploy/docker/database_client_entrypoint.sh + deploy/docker/database_host_rejection_cases.txt + deploy/docker/preflight.sh + deploy/fluentd/fluent.conf + deploy/prometheus/alertmanager.yml + deploy/prometheus/blackbox.yml + deploy/prometheus/postgres_queries.yml + deploy/prometheus/prometheus.yml + backend/configs/prometheus_alerts.yml + docs/ops/01-rolling-upgrade.md + docs/ops/02-production-operations.md +) +for file in "${files[@]}"; do + mkdir -p "$stage/$(dirname "$file")" + cp "$file" "$stage/$file" +done + +sed \ + -e "s|^GOCHAT_IMAGE_REF=.*|GOCHAT_IMAGE_REF=$gochat_image|" \ + -e "s|^SHANGWUTONG_IMAGE_REF=.*|SHANGWUTONG_IMAGE_REF=$connector_image|" \ + .env.example > "$stage/.env.example" +cp deploy/docker/README.dist.md "$stage/README.md" +printf 'SOURCE_COMMIT=%s\nGOCHAT_IMAGE_REF=%s\nSHANGWUTONG_IMAGE_REF=%s\n' \ + "$(git rev-parse HEAD)" "$gochat_image" "$connector_image" > "$stage/MANIFEST" +(cd "$stage" && find . -type f ! -name SHA256SUMS -print | sort | xargs sha256sum > SHA256SUMS) + +rm -rf "$output_dir" +mv "$stage" "$output_dir" +trap - EXIT HUP INT TERM +echo "Production bundle: $output_dir" diff --git a/scripts/build-production.test.sh b/scripts/build-production.test.sh new file mode 100644 index 00000000..ccc374c8 --- /dev/null +++ b/scripts/build-production.test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +test_tmp=$(mktemp -d) +dirty_file= +trap 'rm -rf "$test_tmp"; [[ -z $dirty_file ]] || rm -f "$dirty_file"' EXIT HUP INT TERM +real_docker=$(command -v docker) +output="$test_tmp/dist" + +mkdir -p "$test_tmp/bin" +cat > "$test_tmp/bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "$DOCKER_LOG" +[[ $1 == buildx && $2 == create ]] && exit 0 +[[ $1 == buildx && $2 == rm ]] && exit 0 +if [[ $1 == buildx && $2 == build ]]; then + for arg in "$@"; do + [[ $arg == type=docker,*dest=* ]] && output=${arg##*dest=} + done + fake_archive=$(mktemp -d) + printf '[]\n' > "$fake_archive/manifest.json" + tar -cf "$output" -C "$fake_archive" manifest.json + rm -rf "$fake_archive" + exit 0 +fi +[[ $1 == load && $2 == --input && -s $3 ]] && exit 0 +[[ $1 == image && $2 == inspect ]] && exit 0 +if [[ $1 == save && $2 == --output ]]; then + fake_archive=$(mktemp -d) + printf '[]\n' > "$fake_archive/manifest.json" + tar -cf "$3" -C "$fake_archive" manifest.json + rm -rf "$fake_archive" + exit 0 +fi +exit 1 +EOF +chmod +x "$test_tmp/bin/docker" + +cd "$root" +dirty_file=$(mktemp "$root/.build-production-dirty.XXXXXX") +if PATH="$test_tmp/bin:$PATH" DOCKER_LOG="$test_tmp/docker.log" GOCHAT_DIST_DIR="$output" \ + GOCHAT_VERSION=1.2.3 bash scripts/build-production.sh >"$test_tmp/dirty.out" 2>&1; then + echo 'dirty worktree build unexpectedly succeeded' >&2 + exit 1 +fi +grep -q 'refusing production build from a dirty worktree' "$test_tmp/dirty.out" +rm "$dirty_file" +dirty_file= + +: > "$test_tmp/docker.log" +PATH="$test_tmp/bin:$PATH" DOCKER_LOG="$test_tmp/docker.log" GOCHAT_DIST_DIR="$output" \ + GOCHAT_VERSION=1.2.3 bash scripts/build-production.sh + +tag=1.2.3-$(git rev-parse --short=12 HEAD) +archive="$output/images/gochat-$tag-images.tgz" +test -s "$archive" +gzip -t "$archive" +(cd "$output" && sha256sum -c SHA256SUMS) +grep -q '^GOCHAT_IMAGE_REF=gochat/gochat:1.2.3-' "$output/.env.example" +grep -q '^SHANGWUTONG_IMAGE_REF=gochat/shangwutong:1.2.3-' "$output/.env.example" +test -f "$output/deploy/fluentd/fluent.conf" +test -f "$output/deploy/prometheus/prometheus.yml" +test -f "$output/backend/configs/prometheus_alerts.yml" +test "$(grep -c '^buildx build ' "$test_tmp/docker.log")" -eq 2 +test "$(grep -c '^load --input ' "$test_tmp/docker.log")" -eq 2 +test "$(grep -Fc -- "--build-arg VERSION=1.2.3 --build-arg COMMIT_SHA=$(git rev-parse --short=12 HEAD) --build-arg BUILD_DATE=$(git show -s --format=%cI HEAD) --build-arg SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)" "$test_tmp/docker.log")" -eq 2 +grep -Fq "image inspect gochat/gochat:$tag gochat/shangwutong:$tag" "$test_tmp/docker.log" +test "$(grep -c '^save --output ' "$test_tmp/docker.log")" -eq 1 +grep '^save --output ' "$test_tmp/docker.log" | \ + grep -Fq " gochat/gochat:$tag gochat/shangwutong:$tag" + +cp "$output/.env.example" "$output/.env" +(cd "$output" && "$real_docker" compose --env-file .env -f deploy/docker/docker-compose.prod.yml config --quiet) +rm "$output/.env" +first_sum=$(sha256sum "$archive") +PATH="$test_tmp/bin:$PATH" DOCKER_LOG="$test_tmp/docker.log" GOCHAT_DIST_DIR="$output" \ + GOCHAT_VERSION=1.2.3 bash scripts/build-production.sh +test "$first_sum" = "$(sha256sum "$archive")" + +echo 'production build script test passed'