From e8f2c192bac18aeb863c6df39a0e785192aaf6ca Mon Sep 17 00:00:00 2001 From: Rogee Date: Fri, 18 Sep 2026 16:50:18 +0800 Subject: [PATCH] refactor: migrate browser gateway to native xvfb --- .env.example | 7 +- .gitea/workflows/douyin-release-gate.yaml | 21 +- AGENTS.md | 10 +- Dockerfile | 6 +- README.md | 87 +- .../__init__.py | 0 .../douyin.py | 10 +- .../gateway.py | 1445 ++++------------ .../proxy.py | 2 +- cmd/browser_gateway/runtime.py | 1515 +++++++++++++++++ .../test_gateway.py | 1253 +++----------- cmd/browser_gateway/test_runtime.py | 436 +++++ .../test_xiaohongshu.py | 2 +- .../xiaohongshu.py | 0 cmd/control-plane/account_deletion.go | 18 +- cmd/control-plane/creator.go | 127 +- cmd/control-plane/creator_events.go | 14 +- cmd/control-plane/creator_events_test.go | 10 +- cmd/control-plane/creator_events_unit_test.go | 101 ++ cmd/control-plane/creator_helper_test.go | 6 +- cmd/control-plane/creator_login_test.go | 12 +- cmd/control-plane/creator_material.go | 107 +- cmd/control-plane/creator_material_test.go | 155 +- cmd/control-plane/creator_pure_unit_test.go | 177 ++ .../creator_route_validation_test.go | 60 + cmd/control-plane/douyin_test.go | 4 +- .../gateway_browser_unit_test.go | 266 +++ cmd/control-plane/hub.go | 494 +++--- cmd/control-plane/hub_native_unit_test.go | 152 ++ cmd/control-plane/hub_test.go | 947 ++++++----- cmd/control-plane/main.go | 4 +- cmd/control-plane/main_test.go | 14 +- cmd/control-plane/runtime_use.go | 102 ++ cmd/control-plane/runtime_use_test.go | 131 ++ cmd/control-plane/xiaohongshu_test.go | 2 +- cmd/docker_gateway/docker_client.py | 975 ----------- compose.dev.yaml | 18 +- compose.yaml | 30 +- deploy/browser-gateway.env.example | 16 + deploy/creatorhub-browser-gateway.service.in | 18 + docker/browser-wrapper/Dockerfile | 5 - docker/browser-wrapper/README.md | 14 - docker/browser-wrapper/docker-entrypoint.sh | 32 - docs/architecture/container-control.md | 109 +- docs/deployment.md | 584 ++----- docs/deployment_stop_test.sh | 119 +- docs/e2e-test-plan.md | 148 +- docs/evidence/native-browser-p0-2026-09-17.md | 66 + ...tive-browser-reference-audit-2026-09-18.md | 20 + .../native-browser-verification-2026-09-18.md | 126 ++ ...ve-control-plane-real-douyin-2026-09-18.md | 29 + ...ative-douyin-service-browser-2026-09-18.md | 20 + docs/native-browser-change-review.md | 36 +- docs/native-browser-implementation-plan.md | 49 +- docs/native-browser-verification.md | 71 +- docs/plan01.md | 4 +- docs/python-gateway-branch-review.md | 2 +- docs/research/xhs-all-in-one.md | 2 +- internal/creator/actions.go | 2 +- internal/creator/store.go | 12 +- internal/douyin/connector.go | 24 +- internal/douyin/creator_collector.go | 12 +- internal/douyin/creator_collector_test.go | 8 + internal/hub/environment.go | 197 ++- internal/hub/migration_test.go | 18 +- .../hub/migrations/003_unified_accounts.sql | 2 +- .../017_native_browser_versions.sql | 13 + internal/hub/store.go | 145 +- internal/hub/store_test.go | 159 +- .../migrations/036_runtime_use_leases.sql | 36 + internal/phasea/store.go | 35 +- internal/phasea/store_test.go | 19 +- package.json | 2 +- scripts/dev-backend.mjs | 61 +- scripts/install-native-browser-gateway.sh | 26 + web/src/AccountEditPage.jsx | 2 +- ...ImagesPage.jsx => BrowserVersionsPage.jsx} | 203 +-- ....test.jsx => BrowserVersionsPage.test.jsx} | 62 +- web/src/BrowsersPage.jsx | 125 +- web/src/BrowsersPage.test.jsx | 16 +- web/src/GatewaysPage.jsx | 4 +- web/src/GatewaysPage.test.jsx | 4 +- web/src/Layout.jsx | 14 +- web/src/dataProvider.js | 6 +- web/src/main.jsx | 18 +- web/tests/responsive.e2e.js | 12 +- 86 files changed, 6104 insertions(+), 5323 deletions(-) rename cmd/{docker_gateway => browser_gateway}/__init__.py (100%) rename cmd/{docker_gateway => browser_gateway}/douyin.py (99%) rename cmd/{docker_gateway => browser_gateway}/gateway.py (51%) rename cmd/{docker_gateway => browser_gateway}/proxy.py (99%) create mode 100644 cmd/browser_gateway/runtime.py rename cmd/{docker_gateway => browser_gateway}/test_gateway.py (59%) create mode 100644 cmd/browser_gateway/test_runtime.py rename cmd/{docker_gateway => browser_gateway}/test_xiaohongshu.py (98%) rename cmd/{docker_gateway => browser_gateway}/xiaohongshu.py (100%) create mode 100644 cmd/control-plane/creator_events_unit_test.go create mode 100644 cmd/control-plane/creator_pure_unit_test.go create mode 100644 cmd/control-plane/creator_route_validation_test.go create mode 100644 cmd/control-plane/gateway_browser_unit_test.go create mode 100644 cmd/control-plane/hub_native_unit_test.go create mode 100644 cmd/control-plane/runtime_use.go create mode 100644 cmd/control-plane/runtime_use_test.go delete mode 100644 cmd/docker_gateway/docker_client.py create mode 100644 deploy/browser-gateway.env.example create mode 100644 deploy/creatorhub-browser-gateway.service.in delete mode 100644 docker/browser-wrapper/Dockerfile delete mode 100644 docker/browser-wrapper/README.md delete mode 100644 docker/browser-wrapper/docker-entrypoint.sh create mode 100644 docs/evidence/native-browser-p0-2026-09-17.md create mode 100644 docs/evidence/native-browser-reference-audit-2026-09-18.md create mode 100644 docs/evidence/native-browser-verification-2026-09-18.md create mode 100644 docs/evidence/native-control-plane-real-douyin-2026-09-18.md create mode 100644 docs/evidence/native-douyin-service-browser-2026-09-18.md create mode 100644 internal/hub/migrations/017_native_browser_versions.sql create mode 100644 internal/phasea/migrations/036_runtime_use_leases.sql mode change 100644 => 100755 scripts/dev-backend.mjs create mode 100755 scripts/install-native-browser-gateway.sh rename web/src/{BrowserImagesPage.jsx => BrowserVersionsPage.jsx} (60%) rename web/src/{BrowserImagesPage.test.jsx => BrowserVersionsPage.test.jsx} (70%) diff --git a/.env.example b/.env.example index ff4e60a..d4ea5db 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,11 @@ -DOCKER_GID=989 CREATORHUB_PORT=8082 +NATIVE_GATEWAY_ENDPOINT=http://127.0.0.1:8081 GATEWAY_TOKEN=dev-creatorhub-gateway-token +BROWSER_STATE_DIR=~/.local/state/creatorhub/browser-gateway +BROWSER_PROFILE_ROOT=~/.local/share/creatorhub/browser-profiles +BROWSER_VERSION=148.0.7778.215 +BROWSER_PATH=~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome +NODE_NAME= CONTROL_PLANE_USERNAME= CONTROL_PLANE_PASSWORD= CREATORHUB_CREDENTIAL_MASTER_KEY= diff --git a/.gitea/workflows/douyin-release-gate.yaml b/.gitea/workflows/douyin-release-gate.yaml index 05c60c5..6143bea 100644 --- a/.gitea/workflows/douyin-release-gate.yaml +++ b/.gitea/workflows/douyin-release-gate.yaml @@ -27,7 +27,6 @@ jobs: CONTROL_PLANE_PASSWORD: ci-password CREATORHUB_CREDENTIAL_MASTER_KEY: >- AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= - DOCKER_GID: 999 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 @@ -88,9 +87,9 @@ jobs: python3 -m pip install -r requirements-gateway.lock coverage==7.16.0 python3 -m coverage erase python3 -m coverage run \ - --source=cmd/docker_gateway \ - --omit='cmd/docker_gateway/test_*.py' \ - -m unittest discover -s cmd/docker_gateway -t cmd -p 'test_*.py' + --source=cmd/browser_gateway \ + --omit='cmd/browser_gateway/test_*.py' \ + -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' python3 -m coverage report --precision=2 --fail-under=65 python3 -m coverage json -o evidence/python-coverage.json python3 - <<'PY' @@ -118,23 +117,11 @@ jobs: cp "$file" ../evidence/ done npm run build - - name: Compose configuration and image gate + - name: Compose configuration gate run: | set -Eeuo pipefail - export CREATORHUB_PORT=18080 - trap 'docker compose down' EXIT docker compose config --quiet docker compose -f compose.yaml -f compose.dev.yaml config --quiet - docker compose build creator-hub docker-gateway - docker compose up -d --wait creator-hub docker-gateway - docker compose exec -T creator-hub ffmpeg -version - docker compose exec -T creator-hub ffprobe -version - docker compose images --quiet creator-hub docker-gateway \ - | sort -u | tee evidence/compose-image-ids.txt - docker compose exec -T creator-hub python3 - <<'PY' - import urllib.request - urllib.request.urlopen("http://127.0.0.1:8080/readyz", timeout=5) - PY - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 if: always() with: diff --git a/AGENTS.md b/AGENTS.md index 923267e..769a1ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,8 +65,8 @@ - 当前业务范围与验收以 [docs/plan01.md](docs/plan01.md) 为准：竞品分析、账号与大小号响应、环境与代理、评论线索和私信；先完成抖音完整流程，再完成小红书。 - 与旧探索规划或现有功能冲突时，以上述需求及使用者最新确认为准。自动响应按策略和 UID 冷却执行；人工发送逐次确认，两者不得混淆。自有账号互动与私信采用事件监听，不用轮询或 Mock 冒充实际能力。 -- 上述内容是目标范围，不代表已经实现。平台能力缺口、新的业务歧义必须向使用者确认；禁止自行删减需求、隐藏失败或以未要求的通用框架扩大实现范围。控制面继续使用 Go，浏览器 gateway 使用 Python。 -- 浏览器环境目标改为各机器 gateway 管理宿主机浏览器进程与 Xvfb，不再通过 Docker 创建浏览器环境；复用现有多机控制。采集结束（含失败、取消、超时）回收任务创建的临时资源，保留账号 Profile、正式结果和合法长期监听。评审、实施与验收见 [变更评审](docs/native-browser-change-review.md)、[实施计划](docs/native-browser-implementation-plan.md)、[验证文档](docs/native-browser-verification.md)。本轮仅交付文档，现有 Docker 实现尚未替换，具体契约与部署方案须批准后实施。 +- 上述内容是目标范围，不代表所有平台能力都已通过真实账号验收。平台能力缺口、新的业务歧义必须向使用者确认；禁止自行删减需求、隐藏失败或以未要求的通用框架扩大实现范围。控制面继续使用 Go，浏览器 gateway 使用 Python。 +- 浏览器环境由各机器 gateway 管理宿主机浏览器进程与 Xvfb，不再通过 Docker 创建浏览器环境；复用现有多机控制契约。采集结束（含失败、取消、超时）回收任务创建的临时资源，保留账号 Profile、正式结果和合法长期监听。评审、实施与验收见 [变更评审](docs/native-browser-change-review.md)、[实施计划](docs/native-browser-implementation-plan.md)、[验证文档](docs/native-browser-verification.md)。当前 worktree 实施单节点 native browser；多节点、A/B、跨机故障恢复和性能对比另行验收。 ## 已批准的技术栈 @@ -81,8 +81,8 @@ ### Python 浏览器 gateway -- 浏览器 gateway 使用 Python 3.12+；优先使用标准库 HTTP、进程管理、socket/ssl/asyncio 与显式输入校验，复用已有 CDP/WebSocket 能力。仅在实际需求明确且批准后新增锁定的浏览器依赖。 -- 目标为每台 gateway 仅管理本机浏览器/Xvfb 生命周期、运行代次、Profile、临时资源、代理及页面动作，复用现有多机路由；账号身份必须在每次写操作前核对。当前代码仍依赖 Docker socket；实施批准后删除浏览器容器、卷、网络及镜像代码路径，不保留 Docker 回退。 +- 浏览器 gateway 使用 Python 3.12+；优先使用标准库 HTTP、进程管理、socket/ssl/asyncio 与显式输入校验，复用已有 CDP/WebSocket 能力。当前 native gateway 由宿主机非 root systemd user service 管理 Xvfb、浏览器、Profile、代理和运行代次；仅在实际需求明确且批准后新增锁定的浏览器依赖。 +- 每台 gateway 仅管理本机浏览器/Xvfb 生命周期、运行代次、Profile、临时资源、代理及页面动作，复用现有多机路由；账号身份必须在每次写操作前核对。浏览器链路不依赖 Docker socket，不创建浏览器容器、卷、网络或镜像，也不保留 Docker 回退。 - 任务清理必须核对节点、运行代次和资源归属，不能误删账号登录资料、正式素材或他人会话；控制面与 gateway 各自处理本机创建的临时文件，清理失败必须可见。 - Python 依赖必须写入锁定文件；不允许自动登录、任意 CDP、Cookie/验证码/密码回显或把不确定写结果转换为成功。 @@ -106,4 +106,4 @@ - 每个非平凡行为变更附带最小的回归测试，且该测试在无此变更时会失败。在信任与集成边界覆盖成功、校验、失败和兼容路径；单元测试覆盖率保证 65% 以上。 - 控制面变更必须通过 `go test ./...`、`go vet ./...`，并构建 `./cmd/control-plane`；涉及并发、生命周期或共享状态的变更须运行 `go test -race ./...`。Python gateway 必须通过其非交互式单元测试与覆盖率检查；只有明确涉及 gateway/Docker/Compose 变更且获得使用者同意时，才执行 Compose 构建/健康检查。Docker 或 Compose 变更还须通过 `docker compose config --quiet`。 - 前端变更必须从 lockfile 安装、通过仓库的非交互式测试命令，并通过 `npm --prefix web run build`。主题、Layout、导航、资源动作或 data provider 的变更需要聚焦的交互覆盖，包括适用的错误与禁用状态。 -- 除非 issue 明确批准契约变更，保持既有 API 与 Docker 生命周期行为不变。在 PR 中文档化任何状态码、载荷、配置、迁移、安全或重试方面的影响。 +- 除非 issue 明确批准契约变更，保持既有 API 行为不变；浏览器生命周期已按批准的 native runtime 契约替换旧 Docker 生命周期。在 PR 中文档化任何状态码、载荷、配置、迁移、安全或重试方面的影响。 diff --git a/Dockerfile b/Dockerfile index c07cb31..568b16c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,17 +15,13 @@ COPY internal/ ./internal/ RUN CGO_ENABLED=0 go build -buildvcs=false -trimpath -ldflags='-s -w' -o /out/control-plane ./cmd/control-plane FROM python:3.13-alpine@sha256:7415fbc3c9e4979cc717d92377ab2bc7b2b4a2af1ac03cc52b5f3f88efedaf3a -COPY requirements-gateway.lock /tmp/requirements-gateway.lock RUN apk add --no-cache ffmpeg \ - && python -m pip install --no-cache-dir -r /tmp/requirements-gateway.lock \ && addgroup -g 65532 app && adduser -D -u 65532 -G app app \ && install -d -o app -g app -m 0700 /var/lib/creatorhub/credentials /var/lib/creatorhub/materials WORKDIR /app COPY --from=go /out/control-plane /app/ -COPY cmd/__init__.py /app/cmd/__init__.py -COPY cmd/docker_gateway /app/cmd/docker_gateway COPY --from=web /src/web/dist /app/web USER 65532:65532 ENV WEB_DIR=/app/web -EXPOSE 8080 8081 +EXPOSE 8080 CMD ["/app/control-plane"] diff --git a/README.md b/README.md index 6dd7023..5efde7b 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,35 @@ # CreatorHub -面向团队的新媒体多账号运营管理平台，统一管理浏览器环境、代理资源、账号池、负责人和自动化运营任务。 +面向团队的新媒体多账号运营管理平台，统一管理浏览器环境、代理资源、账号池、负责人和运营任务。 ## 当前阶段 -业务范围与验收以 [docs/plan01.md](docs/plan01.md) 为准；规划范围不代表已经实现，离线检查不能替代真实平台验收。 +业务范围与验收以 [docs/plan01.md](docs/plan01.md) 为准；离线检查不能替代真实平台验收。 -浏览器环境目标改为 **各机器 gateway 管理原生浏览器 + Xvfb**，并在采集结束后清理任务临时资源、保留登录状态和正式结果。本轮仅完成方案文档，当前代码及下方运行命令仍使用 Docker： +浏览器环境由每台机器上的 **native browser gateway + Xvfb** 管理。gateway 不访问 Docker socket，不创建容器、镜像、卷或网络；登录 Profile 和正式素材持久保留，任务临时资源按执行归属清理。 -- [变更评审](docs/native-browser-change-review.md)：现状、风险、资源保留与清理边界。 -- [实施计划](docs/native-browser-implementation-plan.md)：改动范围、阶段顺序与批准条件。 -- [验证文档](docs/native-browser-verification.md)：多机、异常清理、磁盘与性能的手工验收。 +- [变更评审](docs/native-browser-change-review.md) +- [实施计划](docs/native-browser-implementation-plan.md) +- [验证文档](docs/native-browser-verification.md) + +当前目标是单机单节点闭环；多节点调度、A/B 环境和跨机恢复另立目标。 ## 本地运行 -以下是完整 Compose 启动方式，适合人工部署验证。命令应在仓库根目录、同一个 shell 中执行；不要把 `compose.dev.yaml` 混入本次标准 gateway 验证。 +native gateway 必须先在宿主机以非 root 用户运行。准备已安装的 fingerprint Chromium、Xvfb、systemd user session 和 gateway 配置： -首次部署可生成一组新的本地凭据： +```bash +mkdir -p ~/.config/creatorhub +cp deploy/browser-gateway.env.example ~/.config/creatorhub/browser-gateway.env +$EDITOR ~/.config/creatorhub/browser-gateway.env +scripts/install-native-browser-gateway.sh +curl --fail --silent --show-error http://127.0.0.1:8081/healthz +``` + +首次部署可生成控制面凭据： ```bash -git pull --ff-only -export DOCKER_GID="$(stat -c '%g' /var/run/docker.sock)" export CREATORHUB_PORT=8080 -export GATEWAY_TOKEN="$(openssl rand -hex 24)" export CONTROL_PLANE_USERNAME=creatorhub export CONTROL_PLANE_PASSWORD="$(openssl rand -hex 24)" export CREATORHUB_CREDENTIAL_MASTER_KEY="$(openssl rand -base64 32)" @@ -31,65 +38,63 @@ docker compose config --quiet docker compose up --detach --build ``` -已有数据的部署重启必须复用原来的 `GATEWAY_TOKEN`、控制面账号密码和 `CREATORHUB_CREDENTIAL_MASTER_KEY`，不要重新生成主密钥。三个控制面凭据变量均为必填，主密钥变化会导致已有账号凭据无法解密。 - -启动后检查服务： +Compose 只运行 control-plane 和 PostgreSQL，浏览器 gateway 仍是宿主机 systemd user service。启动后检查： ```bash docker compose ps -curl --fail --silent --show-error "http://127.0.0.1:${CREATORHUB_PORT}/healthz" -curl --fail --silent --show-error "http://127.0.0.1:${CREATORHUB_PORT}/readyz" -docker compose logs --tail=200 creator-hub docker-gateway postgres +curl --fail --silent --show-error "http://127.0.0.1:${CREATORHUB_PORT:-8080}/healthz" +curl --fail --silent --show-error "http://127.0.0.1:${CREATORHUB_PORT:-8080}/readyz" +docker compose logs --tail=200 creator-hub postgres ``` -打开 `http://127.0.0.1:${CREATORHUB_PORT}`（默认端口为 8080），使用控制面账号登录。首次配置和人工抖音验证： +打开 `http://127.0.0.1:${CREATORHUB_PORT:-8080}` 登录。首次配置： -1. 在「网关管理」注册 `http://docker-gateway:8081`，令牌填写当前 shell 中的 `GATEWAY_TOKEN`。 -2. 在「镜像版本」添加已登记的 immutable 浏览器镜像引用。 -3. 创建抖音账号和运行环境，恢复账号后显式启动运行环境。 -4. 在账号页点击「显示登录二维码」；完成扫码或验证码后，点击「核验浏览器身份」。 -5. 所有真实平台操作必须由 CreatorHub 发起，不要直接操作抖音页面；二维码或验证码画面不能作为身份核验成功的证明。 +1. 在「网关管理」注册 gateway。control-plane 在 Compose 中运行时使用 `http://host.docker.internal:8081`；裸机运行 control-plane 时使用 `http://127.0.0.1:8081`。令牌必须与 gateway 配置一致。 +2. 在「浏览器版本」登记宿主机上实际存在的浏览器版本和绝对路径。 +3. 创建抖音账号和运行环境，显式启动环境。 +4. 在账号页显示登录二维码，完成登录后核验浏览器身份。 +5. 按验证文档手工完成抖音采集、结果保存、停止、再启动和资源释放检查。 -停止服务但保留数据库、账号凭据和素材： +已有数据的重启必须复用原控制面主密钥、gateway 令牌、Profile 根目录和素材目录。不要使用 `docker compose down -v`，也不要删除 gateway 的 Profile 根目录。 + +停止控制面和数据库： ```bash docker compose down +systemctl --user stop creatorhub-browser-gateway.service ``` -不要使用 `docker compose down -v`。架构、API 契约、失败语义和 `docker.sock` 风险边界见 -[《浏览器容器控制面》](docs/architecture/container-control.md)。 - ## 本地开发（热加载） -日常开发不再整仓重建镜像，改用源码热加载： - ```bash npm --prefix web ci pnpm dev ``` -- `pnpm dev`：同时起后端（air 热重载）与前端（vite HMR）； -- `pnpm dev:backend` / `pnpm dev:frontend`：单独启动其中一端； -- `pnpm dev:deps`：仅启动 postgres 与 docker-gateway 两个容器（`compose.dev.yaml` 会把 5432/8081 映射到宿主机）。 +- `pnpm dev`：同时运行 Go control-plane 和 Vite HMR； +- `pnpm dev:backend`：启动 PostgreSQL 依赖，检查宿主机 native gateway 后运行 control-plane； +- `pnpm dev:frontend`：单独运行前端。 服务地址： | 端 | 地址 | 说明 | -| --- | --- | -| 前端 | | `/api` 由 vite 代理到本地 control-plane | -| 后端 control-plane | | Go 源码改动即自动重启 | -| docker-gateway | | 容器内常驻，重启不频繁 | -| postgres | `127.0.0.1:5432` | 容器内常驻，数据库重建直接销毁 volume | +| --- | --- | --- | +| 前端 | | `/api` 代理到本地 control-plane | +| control-plane | | Go 热加载 | +| native browser gateway | | 宿主机 Xvfb/浏览器生命周期 | +| PostgreSQL | `127.0.0.1:5432` | 仅数据库依赖使用容器 | -开发默认值（`.env` 缺失或留空时）：登录 `admin` / `admin123`；控制面监听 `:8082`；凭据主密钥使用本地开发专用密钥；本地网关注册地址用 `http://127.0.0.1:8081`（容器名 `docker-gateway` 仅存在于 Compose 网络内）。 +开发默认登录信息和目录可在 `.env.example`、`deploy/browser-gateway.env.example` 中查看。不要把真实密码、Cookie、验证码或令牌写入仓库。 -注意：本地开发后端占用 `:8082`，与整仓 `docker compose up` 的容器端口互斥，两者二选一运行。 - -最小验证： +最小检查： ```bash go test ./... +go vet ./... +go build ./cmd/control-plane +go test -race ./... npm --prefix web ci +npm --prefix web test -- --run npm --prefix web run build docker compose config --quiet ``` diff --git a/cmd/docker_gateway/__init__.py b/cmd/browser_gateway/__init__.py similarity index 100% rename from cmd/docker_gateway/__init__.py rename to cmd/browser_gateway/__init__.py diff --git a/cmd/docker_gateway/douyin.py b/cmd/browser_gateway/douyin.py similarity index 99% rename from cmd/docker_gateway/douyin.py rename to cmd/browser_gateway/douyin.py index 5f9ac00..1641d60 100644 --- a/cmd/docker_gateway/douyin.py +++ b/cmd/browser_gateway/douyin.py @@ -22,7 +22,7 @@ from urllib.parse import parse_qs, urlsplit import websocket -from .docker_client import RUNTIME_ID_RE +from .runtime import ALIAS_RE LOG = logging.getLogger("creatorhub.douyin") ORIGIN = "https://www.douyin.com" @@ -247,9 +247,7 @@ class DouyinBrowser: media_selector: str = "video", target_id: str = "", ) -> None: - self.endpoint = endpoint or ( - lambda alias: f"http://creatorhub-browser-{alias}:9222" - ) + self.endpoint = endpoint self.origin = origin self.url_validator = url_validator or is_douyin_url self.media_validator = media_validator or is_douyin_media_url @@ -265,8 +263,10 @@ class DouyinBrowser: connection.close() def _connect(self, alias: str) -> CDPConnection: - if not RUNTIME_ID_RE.fullmatch(alias): + if not ALIAS_RE.fullmatch(alias): raise DouyinError("browser alias is invalid") + if self.endpoint is None: + raise DouyinError("browser endpoint is not configured") base = self.endpoint(alias).rstrip("/") try: parsed = urlsplit(base) diff --git a/cmd/docker_gateway/gateway.py b/cmd/browser_gateway/gateway.py similarity index 51% rename from cmd/docker_gateway/gateway.py rename to cmd/browser_gateway/gateway.py index 6a288c7..409edb4 100644 --- a/cmd/docker_gateway/gateway.py +++ b/cmd/browser_gateway/gateway.py @@ -1,4 +1,4 @@ -"""CreatorHub Python Docker/browser gateway.""" +"""CreatorHub native browser gateway.""" from __future__ import annotations @@ -8,34 +8,18 @@ import logging import math import os import re +import secrets import signal import socket import threading import time from collections.abc import Mapping -from contextlib import nullcontext, suppress +from contextlib import suppress from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import cast -from urllib.parse import parse_qs, quote, urlsplit +from urllib.parse import parse_qs, urlsplit -from .docker_client import ( - BINDING_VERSION_LABEL, - DISPLAY_NAME_LABEL, - MANAGED_LABEL, - NAME_PREFIX, - NETWORK_EXIT_LABEL, - NETWORK_ID_LABEL, - PROXY_PORT_LABEL, - RUNTIME_ID_LABEL, - RUNTIME_ID_RE, - AliasReservationManager, - DockerClient, - DockerError, - GenerationConflict, - NetworkSetupError, - TenantNetworkGeneration, - UnmanagedContainer, -) from .douyin import ( ACCOUNT_KEY_RE, ACTIONS, @@ -52,15 +36,26 @@ from .douyin import ( is_xiaohongshu_share_url, ) from .proxy import ProxyExit, ProxyRegistry +from .runtime import ( + ALIAS_RE, + BROWSER_VERSION_RE, + BrowserRuntimeError, + GenerationConflict, + NativeRuntimeManager, + NETWORK_ID_RE, + PROFILE_ID_RE, + RUNTIME_CLEANUP_SENTINEL, + RUNTIME_ID_RE, + has_control, + parse_proxy_exit as _parse_proxy_exit, + validate_proxy_exit as _validate_proxy_exit, + validate_runtime_input, +) LOG = logging.getLogger("creatorhub.gateway") -CONTROL_NETWORK = "creatorhub_control" -BROWSER_ENTRYPOINT = "/usr/local/bin/docker-entrypoint.sh" -BROWSER_USER = "1000:1000" -IMAGE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$") -VOLUME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") -CONTAINER_ID_RE = re.compile(r"^[a-f0-9]{64}$") -RUNTIME_CLEANUP_SENTINEL = "runtime-not-found" +RUNTIME_ID_RE = RUNTIME_ID_RE +NETWORK_ID_RE = NETWORK_ID_RE +RUNTIME_CLEANUP_SENTINEL = RUNTIME_CLEANUP_SENTINEL EXIT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$") DOUYIN_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$") DOUYIN_ORIGIN = "https://www.douyin.com" @@ -82,395 +77,58 @@ def _noop() -> None: return None -class RequestError(RuntimeError): - def __init__(self, message: str, status: int = 502, network_id: str = "") -> None: - super().__init__(message) - self.status = status - self.network_id = network_id +class RequestError(BrowserRuntimeError): + pass class Gateway: def __init__( self, - docker: DockerClient, - network: str, + runtimes: NativeRuntimeManager, token: str, - self_name: str, + node_id: str, browser: DouyinBrowser | None = None, xiaohongshu_browser: XiaohongshuBrowser | None = None, - external_cdp: Mapping[str, object] | None = None, ) -> None: - self.docker = docker - self.network = network + self.runtimes = runtimes self.token = token - self.self_name = self_name - self.external_cdp = dict(external_cdp or {}) + self.node_id = node_id self.browser = browser or DouyinBrowser(self._browser_endpoint) self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser( self._browser_endpoint ) - self.proxies = ProxyRegistry() - self.reservations = AliasReservationManager(docker, self_name) + self.proxies = runtimes.proxies self.subscriptions = SubscriptionManager(self.browser) self._action_ownership_lock = threading.Lock() self._uncertain_actions: dict[str, float] = {} def _browser_endpoint(self, alias: str) -> str: - if self.external_cdp: - if alias != self.external_cdp["alias"]: - raise GenerationConflict("external browser alias does not match") - return cast(str, self.external_cdp["url"]) - container_id, labels = self.docker.managed_container(alias) - network_id = labels.get(NETWORK_ID_LABEL, "") - if not isinstance(network_id, str) or not network_id: - raise GenerationConflict("browser container has no isolated network") - address = self.docker.container_network_address(container_id, network_id) - return f"http://{address}:9222" + return self.runtimes.endpoint(alias) def list_browsers(self) -> list[dict]: - if self.external_cdp: - return [ - { - "id": self.external_cdp["runtime_id"], - "alias": self.external_cdp["alias"], - "name": self.external_cdp["alias"], - "state": "external", - "status": "external CDP", - "endpoint": self.external_cdp["url"], - "binding_version": self.external_cdp["binding_version"], - "network_exit_id": "", - "network_id": self.external_cdp["network_id"], - "proxy_ready": True, - } - ] - filters = quote( - json.dumps({"label": [f"{MANAGED_LABEL}=true"]}, separators=(",", ":")), - safe="", - ) - response = self.docker.request( - "GET", f"/containers/json?all=1&filters={filters}" - ) - if response.status != 200: - raise RequestError( - f"Docker returned HTTP {response.status}", response.status - ) - try: - containers = json.loads(response.body) - except json.JSONDecodeError as exc: - raise RequestError("Docker container list is invalid") from exc - if not isinstance(containers, list): - raise RequestError("Docker container list is invalid") - result = [] - for container in containers: - if not isinstance(container, dict): - raise RequestError("Docker container list is invalid") - labels = container.get("Labels") or {} - if not isinstance(labels, dict): - raise RequestError("Docker container labels are invalid") - alias = labels.get(RUNTIME_ID_LABEL, "") - if not isinstance(alias, str) or not RUNTIME_ID_RE.fullmatch(alias): - continue - try: - binding = int(labels.get(BINDING_VERSION_LABEL, "0")) - proxy_port = int(labels.get(PROXY_PORT_LABEL, "0")) - except (TypeError, ValueError): - binding = proxy_port = 0 - network_exit_id = labels.get(NETWORK_EXIT_LABEL, "") - network_id = labels.get(NETWORK_ID_LABEL, "") - container_id = container.get("Id", "") - if ( - not isinstance(network_exit_id, str) - or not isinstance(network_id, str) - or not isinstance(container_id, str) - ): - raise RequestError("Docker container metadata is invalid") - direct = not network_exit_id - endpoint = f"http://{NAME_PREFIX}{alias}:9222" - network_error = "" - if network_id and container.get("State") == "running": - try: - address = self.docker.container_network_address( - container_id, network_id - ) - except (DockerError, FileNotFoundError, GenerationConflict) as exc: - network_error = str(exc) - endpoint = "" - LOG.warning( - "browser network address unavailable", - extra={ - "alias": alias, - "network_id": network_id, - "error": network_error, - }, - ) - else: - endpoint = f"http://{address}:9222" - result.append( - { - "id": container.get("Id", ""), - "alias": alias, - "name": labels.get(DISPLAY_NAME_LABEL) or alias, - "state": container.get("State", ""), - "status": container.get("Status", ""), - "endpoint": endpoint, - "binding_version": binding, - "network_exit_id": network_exit_id, - "network_id": network_id, - "proxy_ready": (not network_error) - and ( - direct - or self.proxies.ready( - alias, proxy_port, container.get("Id", ""), network_id - ) - ), - **({"error": network_error} if network_error else {}), - } - ) - return result + # Released generations remain in the runtime journal for cleanup/audit, + # but are not live browser bindings and must not collide with a reused alias. + return [ + browser + for browser in self.runtimes.list_public() + if browser.get("state") != "released" + ] + + def info(self) -> dict: + return { + "service": "browser-gateway", + "node_id": self.node_id, + "browser_versions": sorted(self.runtimes.browser_versions), + } def create(self, input: dict) -> dict: - validate_create(input) - self.docker.pull_if_missing(input["image"]) - alias = input["alias"] - release = self.reservations.acquire(alias) - try: - # The reservation is the cross-process alias lock; checking before it - # was acquired leaves a create/create race window. - try: - self.docker.managed_container(alias) - except FileNotFoundError: - pass - else: - raise RequestError("browser alias is already in use", 409) - except Exception: - release() - raise - network_generation = TenantNetworkGeneration() - undo_proxy = _noop - keep_network = bool(input.get("stopped")) - keep_proxy = False - keep_container = False - container_attempted = False - created_id = "" - try: - direct = not input["network_exit_id"] - network = "none" - proxy_url = "" - if not input.get("stopped"): - network_generation, bind_host = self.docker.ensure_tenant_network( - self.network, alias, self.self_name, input["binding_version"] - ) - network = network_generation.id - if not direct: - proxy_url, undo_proxy = self.proxies.configure( - alias, - input["binding_version"], - bind_host, - 0, - input["network_exit"], - network_generation.id, - ) - command = list(input["cmd"]) - if not input.get("stopped") and not direct: - command = command[:-1] + [ - f"--proxy-server={proxy_url}", - "--disable-non-proxied-udp", - command[-1], - ] - pids_limit = 512 - payload = { - "Image": input["image"], - "User": BROWSER_USER, - "Entrypoint": [BROWSER_ENTRYPOINT], - "Cmd": command, - "Env": ["REMOTE_DEBUGGING_PORT=9222"], - "Labels": { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: alias, - DISPLAY_NAME_LABEL: input["name"], - BINDING_VERSION_LABEL: str(input["binding_version"]), - NETWORK_EXIT_LABEL: input["network_exit_id"], - NETWORK_ID_LABEL: network_generation.id, - PROXY_PORT_LABEL: str(proxy_port(proxy_url)), - }, - "ExposedPorts": {"9222/tcp": {}}, - "HostConfig": { - "NetworkMode": network, - "ReadonlyRootfs": True, - "CapDrop": ["ALL"], - "SecurityOpt": ["no-new-privileges"], - "PidsLimit": pids_limit, - "Memory": 1 << 30, - "NanoCpus": 2_000_000_000, - "Tmpfs": browser_tmpfs(), - "Mounts": [ - {"Type": "volume", "Source": input["volume"], "Target": "/data"} - ], - }, - } - container_attempted = True - response = self.docker.request( - "POST", - "/containers/create?" + "name=" + quote(NAME_PREFIX + alias, safe=""), - payload, - ) - if response.status == 409: - raise RequestError( - "browser alias is already in use", 409, network_generation.id - ) - if response.status != 201: - raise RequestError( - f"Docker container creation failed with HTTP {response.status}", - response.status, - network_generation.id, - ) - try: - created_id = json.loads(response.body)["Id"] - except (KeyError, TypeError, json.JSONDecodeError) as exc: - raise RequestError( - "Docker returned an invalid container id", - 502, - network_generation.id, - ) from exc - if not isinstance(created_id, str) or not created_id: - raise RequestError( - "Docker returned an invalid container id", - 502, - network_generation.id, - ) - if not input.get("stopped"): - if not direct and not self.proxies.bind( - alias, - input["binding_version"], - proxy_url, - created_id, - network_generation.id, - ): - self.docker.expect( - "DELETE", - f"/containers/{quote(created_id, safe='')}?force=1&v=0", - ) - raise RequestError( - "proxy generation changed", 409, network_generation.id - ) - try: - self.docker.expect( - "POST", - f"/containers/{quote(created_id, safe='')}/start", - allowed=(204, 304), - ) - except Exception as exc: - try: - self.docker.expect( - "DELETE", - f"/containers/{quote(created_id, safe='')}?force=1&v=0", - ) - except Exception: - LOG.exception( - "failed to remove container after start failure", - extra={"container_id": created_id}, - ) - raise RequestError( - "container did not start and was removed", - 502, - network_generation.id, - ) from exc - keep_proxy = not direct - keep_network = True - keep_container = True - self._release_action_ownership(alias) - return { - "id": created_id, - "alias": alias, - "network_id": network_generation.id, - } - except NetworkSetupError as exc: - network_generation = exc.generation - raise RequestError(str(exc), 502, network_generation.id) from exc - finally: - if not keep_proxy: - undo_proxy() - if container_attempted and not keep_container: - self._reconcile_created_container( - alias, - created_id, - input["binding_version"], - network_generation.id, - ) - if not keep_network and network_generation.id: - self._cleanup_network( - alias, input["binding_version"], "", network_generation - ) - try: - release() - except Exception: - LOG.exception( - "failed to release browser alias reservation", - extra={"alias": alias}, - ) - - def _reconcile_created_container( - self, alias: str, created_id: str, binding_version: int, network_id: str - ) -> None: - # A response without a container ID is not attributable to this - # request. Never delete an alias-matching container created by another - # request; an unknown outcome is logged and reconciled by the control - # plane's generation-aware cleanup instead. - if not created_id: - LOG.error( - "container creation outcome has no attributable container id", - extra={"alias": alias, "binding_version": binding_version}, - ) - return - try: - observed_id, labels = self.docker.managed_container(alias) - if created_id and observed_id != created_id: - LOG.error( - "container creation outcome has a replacement generation", - extra={ - "alias": alias, - "created_id": created_id, - "observed_id": observed_id, - }, - ) - return - if ( - labels.get(RUNTIME_ID_LABEL) != alias - or labels.get(BINDING_VERSION_LABEL) != str(binding_version) - or labels.get(NETWORK_ID_LABEL, "") != network_id - ): - LOG.error( - "container creation outcome is not safely attributable", - extra={"alias": alias, "observed_id": observed_id}, - ) - return - self.docker.expect( - "DELETE", - f"/containers/{quote(observed_id, safe='')}?force=1&v=0", - ) - except FileNotFoundError: - return - except (DockerError, OSError, TypeError, ValueError, KeyError): - LOG.exception( - "failed to reconcile container creation outcome", - extra={"alias": alias, "created_id": created_id}, - ) + return self.runtimes.create(input) def change_state(self, alias: str, action: str, input: dict) -> None: generation = decode_generation( - input, require_runtime=True, require_network=action == "start" + input, require_runtime=True, require_network=True ) - with self._alias_lock(alias): - container_id, exists = self._require_generation(alias, generation) - if not exists: - raise RequestError("browser not found", 404) - path = f"/containers/{quote(container_id, safe='')}/{'start' if action == 'start' else 'stop?t=10'}" - try: - self.docker.expect("POST", path, allowed=(204, 304)) - except FileNotFoundError as exc: - raise RequestError("browser not found", 404) from exc - except Exception as exc: - raise RequestError("Docker state change failed") from exc + self.runtimes.change_state(alias, action, generation) def remove(self, alias: str, input: dict) -> None: generation = decode_generation( @@ -480,171 +138,33 @@ class Gateway: allow_profile_purge=True, ) purge_profile = input.get("purge_profile", False) - profile_volume = input.get("profile_volume", "") - if type(purge_profile) is not bool or not isinstance(profile_volume, str): - raise RequestError("profile purge fields are invalid", 400) - if purge_profile and profile_volume != f"creatorhub-profile-{alias}": - raise RequestError("profile volume does not belong to browser alias", 400) - with self._alias_lock(alias): - try: - container_id, labels = self.docker.managed_container(alias) - exists = True - except FileNotFoundError: - container_id, labels, exists = "", {}, False - if exists and ( - not generation["runtime_id"] - or container_id != generation["runtime_id"] - or labels.get(BINDING_VERSION_LABEL) - != str(generation["binding_version"]) - or labels.get(RUNTIME_ID_LABEL) != alias - or labels.get(NETWORK_ID_LABEL) != generation["network_id"] - ): - raise RequestError("container generation does not match request", 409) - try: - network_generation, _, network_exists = ( - self.docker.inspect_tenant_network( - self.network, - alias, - generation["binding_version"], - generation["runtime_id"], - self.self_name, - generation["network_id"], - ) - ) - if ( - network_exists - and generation["runtime_id"] == RUNTIME_CLEANUP_SENTINEL - and network_generation.runtime_attached - ): - raise RequestError( - "container generation is required while the network is attached", - 409, - ) - if network_exists: - self._remove_network( - alias, - generation["binding_version"], - generation["runtime_id"], - network_generation, - ) - elif exists and generation["network_id"]: - raise RequestError("container network generation is missing", 409) - except (GenerationConflict, RequestError): - raise - except (DockerError, OSError, TypeError, ValueError, KeyError) as exc: - raise RequestError( - "runtime_cleanup_pending", 202, generation["network_id"] - ) from exc - if not self.proxies.remove( - alias, - generation["binding_version"], - generation["runtime_id"], - generation["network_id"], - ): - raise RequestError("proxy generation does not match request", 409) - if exists: - try: - self.docker.expect( - "DELETE", - f"/containers/{quote(container_id, safe='')}?force=1&v=0", - ) - except FileNotFoundError: - pass - except Exception as exc: - raise RequestError("Docker container removal failed") from exc - if purge_profile: - try: - self.docker.expect( - "DELETE", - f"/volumes/{quote(profile_volume, safe='')}", - ) - except FileNotFoundError: - pass - except Exception as exc: - raise RequestError("Docker profile volume removal failed") from exc - self._release_action_ownership(alias) + if type(purge_profile) is not bool: + raise RequestError("purge_profile must be boolean", 400) + self.runtimes.remove(alias, generation, purge_profile) def restore_proxy(self, alias: str, input: dict) -> None: validate_proxy_restore(input, alias) - with self._alias_lock(alias): - container_id, labels = self.docker.managed_container(alias) - expected = { - "binding_version": input["binding_version"], - "runtime_id": input["runtime_id"], - "network_id": input["network_id"], - } - if ( - labels.get(RUNTIME_ID_LABEL) != alias - or labels.get(BINDING_VERSION_LABEL) != str(expected["binding_version"]) - or labels.get(NETWORK_ID_LABEL) != expected["network_id"] - or labels.get(NETWORK_EXIT_LABEL) != input["network_exit_id"] - ): - raise RequestError( - "container binding does not match recovery request", 409 - ) - try: - port = int(labels.get(PROXY_PORT_LABEL, "0")) - except (TypeError, ValueError) as exc: - raise RequestError( - "container binding has an invalid proxy port", 409 - ) from exc - if port < 1 and input["network_exit_id"]: - raise RequestError("container binding has no proxy port", 409) - generation = TenantNetworkGeneration(id=input["network_id"]) - configured = False - undo = _noop - try: - generation, bind_host = self.docker.ensure_tenant_network( - self.network, - alias, - self.self_name, - input["binding_version"], - input["runtime_id"], - input["network_id"], - True, - ) - self._require_proxy_network_generation( - alias, input, generation, bind_host - ) - if not input["network_exit_id"]: - return - proxy_url, undo = self.proxies.configure( - alias, - input["binding_version"], - bind_host, - port, - input["network_exit"], - input["network_id"], - ) - configured = True - self._require_proxy_network_generation( - alias, input, generation, bind_host - ) - if not self.proxies.bind( - alias, - input["binding_version"], - proxy_url, - container_id, - input["network_id"], - ): - raise RequestError("proxy generation changed", 409) - self._require_proxy_network_generation( - alias, input, generation, bind_host - ) - except Exception: - if configured: - undo() - else: - self.proxies.remove( - alias, - input["binding_version"], - input["runtime_id"], - input["network_id"], - ) - self._cleanup_network( - alias, input["binding_version"], input["runtime_id"], generation - ) - raise + generation = decode_generation( + {key: input.get(key) for key in ("binding_version", "runtime_id", "network_id")}, + True, + True, + ) + self.runtimes.restore_proxy(alias, generation, input["network_exit"]) + + def _require_generation(self, alias: str, generation: dict) -> tuple[str, bool]: + try: + record = self.runtimes.require_generation(alias, generation) + except FileNotFoundError: + return "", False + return record.runtime_id, True + + def _require_douyin_generation(self, alias: str, input: dict) -> None: + record = self.runtimes.require_generation(alias, input) + if input.get("network_exit_id", "") != record.network_exit_id: + raise RequestError("runtime proxy generation does not match request", 409) + + def _alias_lock(self, alias: str): + return self.runtimes.alias_lock(alias) def get_douyin(self, alias: str, input: dict) -> dict: if not valid_douyin_generation(input) or not valid_douyin_url( @@ -1093,216 +613,6 @@ class Gateway: self._require_douyin_generation(alias, input) self.subscriptions.stop(alias) - def _require_generation(self, alias: str, generation: dict) -> tuple[str, bool]: - try: - container_id, labels = self.docker.managed_container(alias) - except FileNotFoundError: - return "", False - if ( - container_id != generation["runtime_id"] - or labels.get(RUNTIME_ID_LABEL) != alias - or labels.get(BINDING_VERSION_LABEL) != str(generation["binding_version"]) - or labels.get(NETWORK_ID_LABEL) != generation["network_id"] - ): - raise RequestError("container generation does not match request", 409) - return container_id, True - - def _require_douyin_generation(self, alias: str, input: dict) -> None: - if self.external_cdp: - if ( - alias != self.external_cdp["alias"] - or any( - input.get(key) != self.external_cdp[key] - for key in ("binding_version", "runtime_id", "network_id") - ) - or input.get("network_exit_id", "") - ): - raise RequestError( - "external browser generation does not match request", 409 - ) - return - container_id, labels = self.docker.managed_container(alias) - if ( - container_id != input["runtime_id"] - or labels.get(RUNTIME_ID_LABEL) != alias - or labels.get(BINDING_VERSION_LABEL) != str(input["binding_version"]) - or labels.get(NETWORK_ID_LABEL) != input["network_id"] - or labels.get(NETWORK_EXIT_LABEL, "") != input.get("network_exit_id", "") - ): - raise RequestError("container generation does not match request", 409) - generation, _, exists = self.docker.inspect_tenant_network( - self.network, - alias, - input["binding_version"], - input["runtime_id"], - self.self_name, - input["network_id"], - ) - if ( - not exists - or not generation.runtime_attached - or not generation.self_member - or not generation.gateway_members - ): - raise RequestError( - "container network generation does not match request", 409 - ) - _, _, networks = self.docker.managed_container_state(alias) - if networks != {generation.name: generation.id}: - raise RequestError( - "container network generation does not match request", 409 - ) - - def _require_proxy_network_generation( - self, alias: str, input: dict, expected: TenantNetworkGeneration, bind_host: str - ) -> None: - container_id, labels = self.docker.managed_container(alias) - if ( - container_id != input["runtime_id"] - or labels.get(NETWORK_ID_LABEL) != expected.id - ): - raise RequestError("network generation changed", 409) - current, addresses, exists = self.docker.inspect_tenant_network( - self.network, - alias, - input["binding_version"], - input["runtime_id"], - self.self_name, - expected.id, - ) - if ( - not exists - or not same_network_members(current, expected) - or addresses.get(current.self_member, "").split("/", 1)[0] != bind_host - ): - raise RequestError("network generation changed", 409) - - def _cleanup_network( - self, - alias: str, - binding_version: int, - runtime_id: str, - generation: TenantNetworkGeneration, - ) -> None: - try: - current, _, exists = self.docker.inspect_tenant_network( - self.network, - alias, - binding_version, - runtime_id, - self.self_name, - generation.id, - ) - if not exists: - return - if generation.created: - self._remove_network(alias, binding_version, runtime_id, current, True) - else: - if generation.connected_runtime and runtime_id: - self.docker.disconnect_member( - self.network, - alias, - binding_version, - runtime_id, - current, - runtime_id, - self.self_name, - missing_ok=True, - ) - if generation.connected_self: - member = current.self_member or generation.self_member - if member: - self.docker.disconnect_member( - self.network, - alias, - binding_version, - runtime_id, - current, - member, - self.self_name, - missing_ok=True, - ) - except (DockerError, OSError, TypeError, ValueError, KeyError): - LOG.exception( - "failed to clean up isolated browser network", - extra={"alias": alias, "network_id": generation.id}, - ) - - def _remove_network( - self, - alias: str, - binding_version: int, - runtime_id: str, - generation: TenantNetworkGeneration, - missing_ok: bool = False, - ) -> None: - current = generation - if current.runtime_attached or current.connected_runtime: - current = self.docker.disconnect_member( - self.network, - alias, - binding_version, - runtime_id, - current, - runtime_id, - self.self_name, - missing_ok=missing_ok, - ) - for member in list(current.gateway_members): - current = self.docker.disconnect_member( - self.network, - alias, - binding_version, - runtime_id, - current, - member, - self.self_name, - missing_ok=missing_ok, - ) - if current.self_member: - current = self.docker.disconnect_member( - self.network, - alias, - binding_version, - runtime_id, - current, - current.self_member, - self.self_name, - missing_ok=missing_ok, - ) - self.docker.delete_tenant_network( - self.network, - alias, - binding_version, - runtime_id, - current, - self.self_name, - missing_ok=missing_ok, - ) - - def _alias_lock(self, alias: str): - if self.external_cdp: - if alias != self.external_cdp["alias"]: - raise GenerationConflict("external browser alias does not match") - return nullcontext() - return _AliasLock(self.reservations, alias) - - -class _AliasLock: - def __init__(self, reservations: AliasReservationManager, alias: str) -> None: - self.reservations = reservations - self.alias = alias - self.release = None - - def __enter__(self): - self.release = self.reservations.acquire(self.alias) - return self - - def __exit__(self, exc_type, exc_value, traceback) -> None: - if self.release: - self.release() - - class GatewayHTTPServer(ThreadingHTTPServer): daemon_threads = True allow_reuse_address = True @@ -1385,7 +695,7 @@ class GatewayHandler(BaseHTTPRequestHandler): return try: needs_body = method in {"POST", "DELETE"} or ( - method == "GET" and parsed.path != "/v1/browsers" + method == "GET" and parsed.path.endswith("/douyin/events") ) body = self._body() if needs_body else {} result = self._route(method, parsed.path, parse_qs(parsed.query), body) @@ -1409,15 +719,11 @@ class GatewayHandler(BaseHTTPRequestHandler): self._respond(exc.status, json_bytes(payload)) elif isinstance(exc, FileNotFoundError): self._respond(404, json_bytes({"error": str(exc)})) - elif isinstance(exc, (GenerationConflict, UnmanagedContainer)): - self._respond(409, json_bytes({"error": str(exc)})) - elif isinstance(exc, DockerError): - status = ( - exc.status - if exc.status is not None and 400 <= exc.status < 500 - else 502 - ) - self._respond(status, json_bytes({"error": str(exc)})) + elif isinstance(exc, BrowserRuntimeError): + payload = {"error": str(exc)} + if exc.network_id: + payload["network_id"] = exc.network_id + self._respond(exc.status, json_bytes(payload)) elif isinstance(exc, ValueError): self._respond(400, json_bytes({"error": str(exc)})) else: @@ -1426,23 +732,22 @@ class GatewayHandler(BaseHTTPRequestHandler): def _route(self, method: str, path: str, query: dict, body: dict): gateway = self.server_as_gateway().gateway + if method == "GET" and path == "/v1/info": + return gateway.info() if method == "GET" and path == "/v1/browsers": return gateway.list_browsers() if method == "POST" and path == "/v1/browsers": return 201, gateway.create(body) match = re.fullmatch(r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})", path) if match and method == "DELETE": - gateway.remove( - match.group(1), - decode_generation(body, require_runtime=False, require_network=False), - ) + gateway.remove(match.group(1), body) return None match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/(start|stop|proxy)", path + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/(start|stop|cancel|proxy)", path ) if match: alias, action = match.groups() - if method == "POST" and action in {"start", "stop"}: + if method == "POST" and action in {"start", "stop", "cancel"}: gateway.change_state(alias, action, body) return None if method == "POST" and action == "proxy": @@ -1537,237 +842,6 @@ def json_bytes(value: object) -> bytes: return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode() -def validate_create(input: dict) -> None: - allowed = { - "alias", - "name", - "image", - "cmd", - "volume", - "binding_version", - "network_exit_id", - "network_exit", - "stopped", - } - if set(input) - allowed: - raise RequestError("body contains unknown fields", 400) - alias = input.get("alias", "") - name = input.get("name", "") - image = input.get("image", "") - volume = input.get("volume", "") - command = input.get("cmd") - binding = input.get("binding_version") - exit_id = input.get("network_exit_id", "") - stopped = input.get("stopped", False) - if not isinstance(alias, str) or not RUNTIME_ID_RE.fullmatch(alias): - raise RequestError("alias must match [a-z0-9][a-z0-9-]{0,31}", 400) - if not isinstance(name, str) or not 1 <= len(name) <= 64 or has_control(name): - raise RequestError("name must be 1..64 visible characters", 400) - if not isinstance(image, str) or not IMAGE_RE.fullmatch(image): - raise RequestError("image must be a valid image reference", 400) - if not isinstance(volume, str) or not VOLUME_RE.fullmatch(volume): - raise RequestError("volume must be a valid volume name", 400) - if type(binding) is not int or binding < 1: - raise RequestError( - "binding_version and network_exit_id must identify the current binding", 400 - ) - if type(stopped) is not bool: - raise RequestError("stopped must be boolean", 400) - if not isinstance(exit_id, str): - raise RequestError("network_exit_id must be a string", 400) - input["network_exit_id"] = exit_id - input.setdefault("network_exit", {}) - exit_value = parse_proxy_exit(input.get("network_exit", {})) - direct = not exit_id and exit_value == ProxyExit("", "", 0) - if stopped and not direct: - raise RequestError("stopped browsers must use direct networking", 400) - if bool(exit_id) != (exit_value != ProxyExit("", "", 0)): - raise RequestError( - "binding_version and network_exit_id must identify the current binding", 400 - ) - if exit_id and not EXIT_ID_RE.fullmatch(exit_id): - raise RequestError("network_exit_id is invalid", 400) - if ( - not isinstance(command, list) - or not 1 <= len(command) <= 64 - or command[-1] != "about:blank" - ): - raise RequestError("cmd must contain 1..64 arguments", 400) - total = 0 - for item in command: - if ( - not isinstance(item, str) - or not item - or has_control(item) - or item.startswith("--proxy-server") - or item == "--disable-non-proxied-udp" - ): - raise RequestError("cmd arguments are invalid", 400) - total += len(item) - if total > 4096: - raise RequestError("cmd arguments exceed 4096 characters", 400) - if not stopped and not direct: - validate_proxy_exit(exit_value) - input["network_exit"] = exit_value - - -def validate_proxy_exit(exit: ProxyExit) -> None: - if ( - exit.protocol not in {"http", "https", "socks4", "socks5"} - or not exit.host - or len(exit.host) > 253 - or any(char in exit.host for char in "@/[]?# \t\r\n") - or not 1 <= exit.port <= 65535 - or (not exit.username and exit.credential) - or len(exit.username) > 255 - or len(exit.credential) > 255 - or has_control(exit.username) - or has_control(exit.credential) - ): - raise RequestError("network_exit must contain a valid proxy endpoint", 400) - - -def parse_proxy_exit(value: object) -> ProxyExit: - if not isinstance(value, dict): - raise RequestError("network_exit must be an object", 400) - allowed = {"protocol", "host", "port", "username", "password"} - if set(value) - allowed: - raise RequestError("network_exit contains unknown fields", 400) - try: - exit = ProxyExit( - value.get("protocol", ""), - value.get("host", ""), - value.get("port", 0), - value.get("username", ""), - value.get("password", ""), - ) - except (TypeError, ValueError) as exc: - raise RequestError("network_exit is invalid", 400) from exc - if ( - not all( - isinstance(item, str) - for item in (exit.protocol, exit.host, exit.username, exit.credential) - ) - or type(exit.port) is not int - ): - raise RequestError("network_exit is invalid", 400) - return exit - - -def browser_tmpfs() -> dict[str, str]: - # These are in-container tmpfs mounts; no host path or bind mount is exposed. - tmp = os.path.join(os.sep, "tmp") - return { - tmp: "rw,nosuid,nodev,noexec,mode=1777,size=256m", - os.path.join(tmp, ".X11-unix"): "rw,nosuid,nodev,noexec,mode=1777,size=1m", - os.path.join(os.sep, "dev", "shm"): "rw,nosuid,nodev,noexec,size=256m", - os.path.join( - os.sep, "home", "ubuntu" - ): "rw,nosuid,nodev,noexec,uid=1000,gid=1000,mode=700,size=64m", - } - - -def proxy_port(proxy_url: str) -> int: - return urlsplit(proxy_url).port or 0 - - -def has_control(value: str) -> bool: - return any(ord(char) < 0x20 or ord(char) == 0x7F for char in value) - - -def decode_generation( - value: dict, - require_runtime: bool, - require_network: bool, - allow_profile_purge: bool = False, -) -> dict: - allowed = {"binding_version", "runtime_id", "network_id"} - if allow_profile_purge: - allowed.update({"purge_profile", "profile_volume"}) - if not isinstance(value, dict) or set(value) - allowed: - raise RequestError( - "binding_version, runtime_id and network_id must identify the expected generation", - 400, - ) - binding = value.get("binding_version") - runtime = value.get("runtime_id", "") - network = value.get("network_id", "") - if ( - type(binding) is not int - or binding < 1 - or not isinstance(runtime, str) - or not isinstance(network, str) - or ( - runtime - and runtime != RUNTIME_CLEANUP_SENTINEL - and not CONTAINER_ID_RE.fullmatch(runtime) - ) - or (network and not EXIT_ID_RE.fullmatch(network)) - or (require_runtime and not runtime) - or (require_network and not network) - ): - raise RequestError( - "binding_version, runtime_id and network_id must identify the expected generation", - 400, - ) - return {"binding_version": binding, "runtime_id": runtime, "network_id": network} - - -def validate_proxy_restore(value: dict, alias: str) -> None: - allowed = { - "binding_version", - "runtime_id", - "network_id", - "network_exit_id", - "network_exit", - } - if set(value) - allowed: - raise RequestError("invalid proxy recovery request", 400) - generation = decode_generation( - { - key: value.get(key) - for key in ("binding_version", "runtime_id", "network_id") - }, - True, - True, - ) - exit_id = value.get("network_exit_id", "") - if not isinstance(exit_id, str) or (exit_id and not EXIT_ID_RE.fullmatch(exit_id)): - raise RequestError("invalid proxy recovery request", 400) - exit = parse_proxy_exit(value.get("network_exit", {})) - direct = not exit_id and exit == ProxyExit("", "", 0) - if bool(exit_id) != (not direct): - raise RequestError("invalid proxy recovery request", 400) - if not direct: - validate_proxy_exit(exit) - value["network_exit"] = exit - value.update(generation) - value["network_exit_id"] = exit_id - - -def valid_douyin_generation(value: dict) -> bool: - if not isinstance(value, dict): - return False - binding = value.get("binding_version") - runtime = value.get("runtime_id", "") - network = value.get("network_id", "") - exit_id = value.get("network_exit_id", "") - return ( - type(binding) is int - and binding > 0 - and isinstance(runtime, str) - and isinstance(network, str) - and isinstance(exit_id, str) - and bool(CONTAINER_ID_RE.fullmatch(runtime)) - and bool(EXIT_ID_RE.fullmatch(network)) - and (not exit_id or bool(EXIT_ID_RE.fullmatch(exit_id))) - ) - - -def valid_xiaohongshu_generation(value: dict) -> bool: - return valid_douyin_generation(value) - - def valid_xhs_query( query: object, allowed: set[str], required: set[str] | None = None ) -> bool: @@ -2003,96 +1077,234 @@ def numeric_cursor(values: list[str] | None) -> bool: return False -def same_network_members( - current: TenantNetworkGeneration, expected: TenantNetworkGeneration -) -> bool: - return ( - current.id == expected.id - and current.name == expected.name - and current.runtime_attached == expected.runtime_attached - and current.self_member == expected.self_member - and set(current.gateway_members) == set(expected.gateway_members) +def parse_proxy_exit(value: object) -> ProxyExit: + try: + return _parse_proxy_exit(value) + except BrowserRuntimeError as exc: + raise RequestError(str(exc), exc.status, exc.network_id) from exc + + +def validate_proxy_exit(exit: ProxyExit) -> None: + try: + _validate_proxy_exit(exit) + except BrowserRuntimeError as exc: + raise RequestError(str(exc), exc.status, exc.network_id) from exc + + +def validate_create(input: dict) -> None: + try: + validate_runtime_input(input) + except BrowserRuntimeError as exc: + raise RequestError(str(exc), exc.status, exc.network_id) from exc + + +def proxy_port(proxy_url: str) -> int: + return urlsplit(proxy_url).port or 0 + + +def decode_generation( + value: dict, + require_runtime: bool, + require_network: bool, + allow_profile_purge: bool = False, +) -> dict: + allowed = {"binding_version", "runtime_id", "network_id"} + if allow_profile_purge: + allowed.add("purge_profile") + if not isinstance(value, dict) or set(value) - allowed: + raise RequestError( + "binding_version, runtime_id and network_id must identify the expected generation", + 400, + ) + binding = value.get("binding_version") + runtime = value.get("runtime_id", "") + network = value.get("network_id", "") + if ( + type(binding) is not int + or binding < 1 + or not isinstance(runtime, str) + or not isinstance(network, str) + or ( + runtime + and runtime != RUNTIME_CLEANUP_SENTINEL + and not RUNTIME_ID_RE.fullmatch(runtime) + ) + or (network and not NETWORK_ID_RE.fullmatch(network)) + or (require_runtime and not runtime) + or (require_network and not network) + ): + raise RequestError( + "binding_version, runtime_id and network_id must identify the expected generation", + 400, + ) + return {"binding_version": binding, "runtime_id": runtime, "network_id": network} + + +def validate_proxy_restore(value: dict, alias: str) -> None: + del alias + allowed = { + "binding_version", + "runtime_id", + "network_id", + "network_exit_id", + "network_exit", + } + if not isinstance(value, dict) or set(value) - allowed: + raise RequestError("invalid proxy recovery request", 400) + generation = decode_generation( + { + key: value.get(key) + for key in ("binding_version", "runtime_id", "network_id") + }, + True, + True, ) + exit_id = value.get("network_exit_id", "") + if not isinstance(exit_id, str) or (exit_id and not EXIT_ID_RE.fullmatch(exit_id)): + raise RequestError("invalid proxy recovery request", 400) + exit = parse_proxy_exit(value.get("network_exit", {})) + direct = not exit_id and exit == ProxyExit("", "", 0) + if bool(exit_id) != (not direct): + raise RequestError("invalid proxy recovery request", 400) + if not direct: + validate_proxy_exit(exit) + value["network_exit"] = exit + value.update(generation) + value["network_exit_id"] = exit_id + + +def valid_douyin_generation(value: dict) -> bool: + if not isinstance(value, dict): + return False + binding = value.get("binding_version") + runtime = value.get("runtime_id", "") + network = value.get("network_id", "") + exit_id = value.get("network_exit_id", "") + return ( + type(binding) is int + and binding > 0 + and isinstance(runtime, str) + and isinstance(network, str) + and isinstance(exit_id, str) + and bool(RUNTIME_ID_RE.fullmatch(runtime)) + and bool(NETWORK_ID_RE.fullmatch(network)) + and (not exit_id or bool(EXIT_ID_RE.fullmatch(exit_id))) + ) + + +def valid_xiaohongshu_generation(value: dict) -> bool: + return valid_douyin_generation(value) + + +def _positive_float(env: Mapping[str, str], key: str, default: float, maximum: float) -> float: + raw = env.get(key, str(default)).strip() + try: + value = float(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be a number") from exc + if not 0 < value <= maximum: + raise ValueError(f"{key} is out of range") + return value + + +def _optional_positive_int(env: Mapping[str, str], key: str) -> int | None: + raw = env.get(key, "").strip() + if not raw: + return None + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be an integer") from exc + if value < 1: + raise ValueError(f"{key} is out of range") + return value + + +def _browser_versions(env: Mapping[str, str], default_version: str, default_path: str) -> dict[str, str]: + result = {default_version: default_path} + raw = env.get("BROWSER_VERSION_PATHS", "").strip() + if raw: + try: + decoded = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("BROWSER_VERSION_PATHS must be a JSON object") from exc + if not isinstance(decoded, dict) or not decoded: + raise ValueError("BROWSER_VERSION_PATHS must be a non-empty JSON object") + result = {} + for version, path in decoded.items(): + if not isinstance(version, str) or not BROWSER_VERSION_RE.fullmatch(version): + raise ValueError("BROWSER_VERSION_PATHS contains an invalid version") + if not isinstance(path, str) or not os.path.isabs(os.path.expanduser(path)): + raise ValueError("BROWSER_VERSION_PATHS contains a non-absolute path") + result[version] = os.path.abspath(os.path.expanduser(path)) + if default_version not in result: + result[default_version] = default_path + return result def load_config(env: Mapping[str, str] | None = None) -> dict: env = os.environ if env is None else env - listen = env.get("LISTEN_ADDR", ":8081").strip() - socket_path = env.get("DOCKER_SOCKET", "/var/run/docker.sock").strip() - network = env.get("BROWSER_NETWORK", "creatorhub_browser").strip() + listen = env.get("LISTEN_ADDR", "0.0.0.0:8081").strip() token = env.get("GATEWAY_TOKEN", "").strip() - cdp_url = env.get("BROWSER_CDP_URL", "").strip() - cdp_target_id = env.get("BROWSER_CDP_TARGET_ID", "").strip() + state_dir = os.path.abspath(os.path.expanduser(env.get( + "BROWSER_STATE_DIR", "~/.local/state/creatorhub/browser-gateway" + ).strip())) + profile_root = os.path.abspath(os.path.expanduser(env.get( + "BROWSER_PROFILE_ROOT", "~/.local/share/creatorhub/browser-profiles" + ).strip())) + browser_version = env.get("BROWSER_VERSION", "148.0.7778.215").strip() + browser_path = os.path.abspath(os.path.expanduser(env.get( + "BROWSER_PATH", + "~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome", + ).strip())) + node_id = env.get("NODE_ID", "").strip() + node_name = env.get("NODE_NAME", socket.gethostname()).strip() host, port = split_listen_address(listen) - if ( - not socket_path - or len(token) < 16 - or not re.fullmatch(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", network) - or network == CONTROL_NETWORK - ): - raise ValueError("invalid gateway configuration") + if len(token) < 16: + raise ValueError("GATEWAY_TOKEN must be at least 16 characters") + if not state_dir or not profile_root or not os.path.isabs(state_dir) or not os.path.isabs(profile_root): + raise ValueError("BROWSER_STATE_DIR and BROWSER_PROFILE_ROOT must be absolute") + if not BROWSER_VERSION_RE.fullmatch(browser_version): + raise ValueError("BROWSER_VERSION is invalid") + if not node_id: + node_id = "" + elif not re.fullmatch(r"^[a-z0-9][a-z0-9._-]{0,63}$", node_id): + raise ValueError("NODE_ID is invalid") + if not node_name or has_control(node_name) or len(node_name) > 128: + raise ValueError("NODE_NAME is invalid") if not 1 <= port <= 65535: raise ValueError("LISTEN_ADDR port must be 1..65535") - external_cdp = None - if cdp_url: - if not valid_cdp_url(cdp_url) or ( - cdp_target_id and not re.fullmatch(r"^[A-Za-z0-9_-]{1,128}$", cdp_target_id) - ): - raise ValueError("BROWSER_CDP_URL or BROWSER_CDP_TARGET_ID is invalid") - try: - binding_version = int(env.get("BROWSER_CDP_BINDING_VERSION", "1")) - except (TypeError, ValueError) as exc: - raise ValueError("BROWSER_CDP_BINDING_VERSION must be an integer") from exc - external_cdp = { - "url": cdp_url, - "target_id": cdp_target_id, - "alias": env.get("BROWSER_CDP_ALIAS", "local-cdp").strip(), - "runtime_id": env.get("BROWSER_CDP_RUNTIME_ID", "0" * 64).strip(), - "network_id": env.get("BROWSER_CDP_NETWORK_ID", "local-cdp").strip(), - "binding_version": binding_version, - } - if ( - not RUNTIME_ID_RE.fullmatch(external_cdp["alias"]) - or not CONTAINER_ID_RE.fullmatch(external_cdp["runtime_id"]) - or not EXIT_ID_RE.fullmatch(external_cdp["network_id"]) - or external_cdp["binding_version"] < 1 - ): - raise ValueError("BROWSER_CDP generation is invalid") + versions = _browser_versions(env, browser_version, browser_path) + external_display = _optional_positive_int(env, "RUNTIME_EXTERNAL_DISPLAY") + for version, path in versions.items(): + if not os.path.isfile(path) or not os.access(path, os.X_OK): + raise ValueError(f"browser version {version} is unavailable") return { "listen": (host, port), - "docker_socket": socket_path, - "network": network, + "state_dir": state_dir, + "profile_root": profile_root, + "browser_versions": versions, + "browser_version": browser_version, + "node_id": node_id, + "node_name": node_name, "token": token, - "external_cdp": external_cdp, + "cleanup_timeout": _positive_float(env, "RUNTIME_CLEANUP_TIMEOUT", 30.0, 300.0), + "ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 15.0, 300.0), + "min_free_bytes": int(env.get("RUNTIME_MIN_FREE_BYTES", str(20 * 1024**3))), + "log_max_bytes": int(env.get("RUNTIME_LOG_MAX_BYTES", str(1 * 1024**3))), + "profile_cache_max_bytes": int(env.get("PROFILE_CACHE_MAX_BYTES", str(20 * 1024**3))), + "external_display": external_display, } -def valid_cdp_url(raw: str) -> bool: - try: - parsed = urlsplit(raw) - port = parsed.port - except ValueError: - return False - return ( - parsed.scheme == "http" - and bool(parsed.hostname) - and port is not None - and parsed.path in ("", "/") - and not parsed.username - and not parsed.password - and not parsed.query - and not parsed.fragment - ) - - def split_listen_address(value: str) -> tuple[str, int]: if value.startswith(":"): host, port_text = "", value[1:] elif value.startswith("["): - closing = value.find("]:") + closing = value.find("]:" ) if closing <= 1: raise ValueError("LISTEN_ADDR must be host:port") - host, port_text = value[1:closing], value[closing + 2 :] + host, port_text = value[1:closing], value[closing + 2:] else: if ":" not in value: raise ValueError("LISTEN_ADDR must be host:port") @@ -2104,41 +1316,62 @@ def split_listen_address(value: str) -> tuple[str, int]: return host, port +def load_stable_node_id(state_dir: str, configured: str) -> str: + if configured: + return configured + path = Path(state_dir) / "node-id" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if path.exists(): + value = path.read_text(encoding="utf-8").strip() + if not re.fullmatch(r"^[a-z0-9][a-z0-9._-]{0,63}$", value): + raise ValueError("persisted NODE_ID is invalid") + return value + value = "node-" + secrets.token_hex(16) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as output: + output.write(value + "\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + return value + + def run() -> None: config = load_config() + if os.geteuid() == 0: + raise RuntimeError("browser gateway must run as a non-root user") logging.basicConfig(level=logging.INFO, format="%(message)s") - docker = DockerClient(config["docker_socket"]) - external_cdp = config["external_cdp"] - browser = None - xiaohongshu_browser = None - if external_cdp: - - def endpoint(_alias: str) -> str: - return cast(str, external_cdp["url"]) - - browser = DouyinBrowser( - endpoint, target_id=cast(str, external_cdp["target_id"]) - ) - xiaohongshu_browser = XiaohongshuBrowser( - endpoint, target_id=cast(str, external_cdp["target_id"]) - ) - gateway = Gateway( - docker, - config["network"], - config["token"], - socket.gethostname(), - browser=browser, - xiaohongshu_browser=xiaohongshu_browser, - external_cdp=external_cdp, + node_id = load_stable_node_id(config["state_dir"], config["node_id"]) + runtimes = NativeRuntimeManager( + state_dir=config["state_dir"], + profile_root=config["profile_root"], + node_id=node_id, + browser_versions=config["browser_versions"], + cleanup_timeout=config["cleanup_timeout"], + ready_timeout=config["ready_timeout"], + min_free_bytes=config["min_free_bytes"], + log_max_bytes=config["log_max_bytes"], + profile_cache_max_bytes=config["profile_cache_max_bytes"], + external_display=config["external_display"], ) + gateway = Gateway(runtimes, config["token"], node_id) server = GatewayHTTPServer(config["listen"], gateway) LOG.info( json.dumps( { - "service": "docker-gateway", + "service": "browser-gateway", + "node_id": node_id, + "node_name": config["node_name"], "listen_addr": f"{config['listen'][0]}:{config['listen'][1]}", - "network": config["network"], - } + "browser_version": config["browser_version"], + "external_display": config["external_display"], + }, + ensure_ascii=False, ) ) shutdown_requested = threading.Event() @@ -2158,11 +1391,9 @@ def run() -> None: try: server.serve_forever() finally: - # Stop accepting first, then let in-flight work finish before closing - # the browser and proxy dependencies it may still own. server.wait_for_requests(30.0) gateway.subscriptions.close() - gateway.proxies.close() + runtimes.close() server.server_close() diff --git a/cmd/docker_gateway/proxy.py b/cmd/browser_gateway/proxy.py similarity index 99% rename from cmd/docker_gateway/proxy.py rename to cmd/browser_gateway/proxy.py index 12ea92c..6d70b12 100644 --- a/cmd/docker_gateway/proxy.py +++ b/cmd/browser_gateway/proxy.py @@ -133,7 +133,7 @@ class MemoryProxy: self.server = _ThreadingTCPServer((bind_host, port), handler_type) self.listener = self.server.socket actual_port = self.listener.getsockname()[1] - self.url = f"http://docker-gateway:{actual_port}" + self.url = f"http://{bind_host}:{actual_port}" self._thread = threading.Thread( target=self.server.serve_forever, name=f"creatorhub-proxy-{alias}", diff --git a/cmd/browser_gateway/runtime.py b/cmd/browser_gateway/runtime.py new file mode 100644 index 0000000..8f14dd3 --- /dev/null +++ b/cmd/browser_gateway/runtime.py @@ -0,0 +1,1515 @@ +"""Native browser runtime ownership and lifecycle management.""" + +from __future__ import annotations + +import fcntl +import hashlib +import http.client +import json +import logging +import os +import re +import shutil +import signal +import socket +import subprocess +import threading +import time +from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress +from dataclasses import dataclass, fields +from pathlib import Path +from typing import Any, Protocol +from urllib.parse import urlsplit + +from .proxy import ProxyExit, ProxyRegistry + +LOG = logging.getLogger("creatorhub.runtime") + +RUNTIME_ID_RE = re.compile(r"^[a-f0-9]{64}$") +ALIAS_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,31}$") +NETWORK_ID_RE = re.compile(r"^native-[a-f0-9]{32}$") +PROFILE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$") +BROWSER_VERSION_RE = re.compile(r"^[0-9]+(?:\.[0-9]+){2,3}$") +RUNTIME_CLEANUP_SENTINEL = "runtime-not-found" + +DEFAULT_CLEANUP_TIMEOUT = 30.0 +DEFAULT_READY_TIMEOUT = 15.0 +DEFAULT_DISPLAY_START = 100 +DEFAULT_DISPLAY_END = 199 +DEFAULT_CDP_PORT_START = 19000 +DEFAULT_CDP_PORT_END = 19999 +DEFAULT_PROXY_PORT_START = 20000 +DEFAULT_PROXY_PORT_END = 20999 + + +class BrowserRuntimeError(RuntimeError): + """An error with a stable HTTP-facing status and optional generation ID.""" + + def __init__(self, message: str, status: int = 502, network_id: str = "") -> None: + super().__init__(message) + self.status = status + self.network_id = network_id + + +class GenerationConflict(BrowserRuntimeError): + def __init__(self, message: str = "runtime generation does not match request") -> None: + super().__init__(message, 409) + + +class UnmanagedRuntime(BrowserRuntimeError): + def __init__(self, message: str = "runtime is not owned by this gateway") -> None: + super().__init__(message, 409) + + +class RuntimeCleanupPending(BrowserRuntimeError): + def __init__(self, message: str, network_id: str = "") -> None: + super().__init__(message, 202, network_id) + + +class RuntimeCancelled(BrowserRuntimeError): + def __init__(self, message: str = "runtime creation was cancelled", network_id: str = "") -> None: + super().__init__(message, 409, network_id) + + +@dataclass(frozen=True) +class UnitStatus: + active: bool + state: str + pid: int + start_time: str + exit_code: str + + +class UnitManager(Protocol): + def start( + self, + unit: str, + command: list[str], + *, + environment: Mapping[str, str], + working_directory: Path, + stdout_path: Path, + limits: Mapping[str, str], + ) -> UnitStatus: ... + + def status(self, unit: str) -> UnitStatus: ... + + def stop(self, unit: str, timeout: float) -> None: ... + + def reset(self, unit: str) -> None: ... + + +class FileLock: + """A process-safe exclusive lock which is released with the file descriptor.""" + + def __init__(self, path: Path) -> None: + self.path = path + self._fd: int | None = None + + def acquire(self, blocking: bool = False) -> bool: + if self._fd is not None: + return True + self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600) + operation = fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB + try: + fcntl.flock(fd, operation) + except (BlockingIOError, OSError) as exc: + os.close(fd) + if isinstance(exc, BlockingIOError) or getattr(exc, "errno", None) in ( + 11, + 13, + ): + return False + raise + self._fd = fd + return True + + def release(self) -> None: + fd, self._fd = self._fd, None + if fd is None: + return + try: + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + def __enter__(self) -> FileLock: + if not self.acquire(blocking=True): + raise BrowserRuntimeError("resource lock could not be acquired", 409) + return self + + def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None: + self.release() + + +@dataclass +class _Lease: + value: int + lock: FileLock + + +class PortAllocator: + def __init__(self, lock_dir: Path) -> None: + self.lock_dir = lock_dir + + def reserve( + self, + prefix: str, + start: int, + end: int, + unavailable: Callable[[int], bool], + ) -> _Lease: + for value in range(start, end + 1): + if unavailable(value): + continue + lock = FileLock(self.lock_dir / f"{prefix}-{value}.lock") + if not lock.acquire(): + continue + try: + probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + probe.bind(("127.0.0.1", value)) + finally: + probe.close() + except OSError: + lock.release() + continue + return _Lease(value, lock) + raise BrowserRuntimeError(f"no free {prefix} port is available", 503) + + def reserve_existing(self, prefix: str, value: int) -> _Lease: + if not 1 <= value <= 65535: + raise BrowserRuntimeError(f"invalid {prefix} port", 409) + lock = FileLock(self.lock_dir / f"{prefix}-{value}.lock") + if not lock.acquire(): + raise BrowserRuntimeError(f"{prefix} port is already reserved", 409) + return _Lease(value, lock) + + +class DisplayAllocator: + def __init__(self, lock_dir: Path) -> None: + self.lock_dir = lock_dir + + def reserve( + self, + start: int, + end: int, + unavailable: Callable[[int], bool] | None = None, + ) -> _Lease: + unavailable = unavailable or (lambda _value: False) + for value in range(start, end + 1): + if unavailable(value) or Path(f"/tmp/.X11-unix/X{value}").exists(): + continue + lock = FileLock(self.lock_dir / f"display-{value}.lock") + if lock.acquire(): + if not Path(f"/tmp/.X11-unix/X{value}").exists(): + return _Lease(value, lock) + lock.release() + raise BrowserRuntimeError("no free Xvfb display is available", 503) + + def reserve_existing(self, value: int) -> _Lease: + if value < 1: + raise BrowserRuntimeError("runtime display is invalid", 409) + lock = FileLock(self.lock_dir / f"display-{value}.lock") + if not lock.acquire(): + raise BrowserRuntimeError("runtime display is already reserved", 409) + return _Lease(value, lock) + + +class SystemdUnitManager: + """Starts only explicitly supplied commands in per-runtime user units.""" + + def __init__( + self, + systemd_run: str | None = None, + systemctl: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + ) -> None: + self.systemd_run = systemd_run or shutil.which("systemd-run") or "" + self.systemctl = systemctl or shutil.which("systemctl") or "" + self.runner = runner or subprocess.run + + def _execute(self, args: list[str], timeout: float) -> subprocess.CompletedProcess[str]: + if not self.systemd_run or not self.systemctl: + raise BrowserRuntimeError("systemd user units are unavailable", 503) + try: + return self.runner( + args, + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise BrowserRuntimeError("systemd operation failed", 503) from exc + + def start( + self, + unit: str, + command: list[str], + *, + environment: Mapping[str, str], + working_directory: Path, + stdout_path: Path, + limits: Mapping[str, str], + ) -> UnitStatus: + stdout_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + args = [ + self.systemd_run, + "--user", + "--unit", + unit, + "--collect", + "--no-block", + "--property=Type=exec", + "--property=KillMode=mixed", + "--property=NoNewPrivileges=yes", + f"--property=StandardOutput=append:{stdout_path}", + f"--property=StandardError=append:{stdout_path}", + "--working-directory", + str(working_directory), + ] + for key, value in sorted(environment.items()): + args.extend(["--setenv", f"{key}={value}"]) + for key, value in sorted(limits.items()): + args.append(f"--property={key}={value}") + args.extend(["--", *command]) + result = self._execute(args, 30.0) + if result.returncode != 0: + LOG.error( + "native runtime unit start rejected", + extra={"unit": unit, "returncode": result.returncode}, + ) + raise BrowserRuntimeError("native runtime unit could not start", 503) + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline: + status = self.status(unit) + if status.pid or status.active: + return status + time.sleep(0.05) + raise BrowserRuntimeError("native runtime unit did not become active", 503) + + def status(self, unit: str) -> UnitStatus: + if not self.systemctl: + raise BrowserRuntimeError("systemd user units are unavailable", 503) + result = self._execute( + [ + self.systemctl, + "--user", + "show", + unit, + "--property=ActiveState", + "--property=SubState", + "--property=MainPID", + "--property=ExecMainStartTimestampMonotonic", + "--property=ExecMainStatus", + ], + 10.0, + ) + if result.returncode != 0: + return UnitStatus(False, "not-found", 0, "", "") + values: dict[str, str] = {} + for line in result.stdout.splitlines(): + if "=" in line: + key, value = line.split("=", 1) + values[key] = value + try: + pid = int(values.get("MainPID", "0")) + except ValueError: + pid = 0 + active_state = values.get("ActiveState", "inactive") + return UnitStatus( + active_state in {"active", "activating", "deactivating"}, + values.get("SubState", active_state), + pid, + values.get("ExecMainStartTimestampMonotonic", ""), + values.get("ExecMainStatus", ""), + ) + + def stop(self, unit: str, timeout: float) -> None: + result = self._execute( + [self.systemctl, "--user", "stop", unit], max(timeout, 1.0) + ) + if result.returncode == 0: + return + kill = self._execute( + [ + self.systemctl, + "--user", + "kill", + "--kill-who=all", + "--signal=TERM", + unit, + ], + 10.0, + ) + if kill.returncode != 0: + raise BrowserRuntimeError("native runtime unit could not stop", 503) + result = self._execute( + [self.systemctl, "--user", "stop", unit], max(timeout, 1.0) + ) + if result.returncode != 0: + raise BrowserRuntimeError("native runtime unit could not stop", 503) + + def reset(self, unit: str) -> None: + if not self.systemctl: + return + result = self._execute([self.systemctl, "--user", "reset-failed", unit], 10.0) + # `--collect` removes a transient unit before reset-failed runs; systemctl + # returns 1 for that already-cleaned unit. This is the idempotent success + # case, not an ignored cleanup error. + if result.returncode not in (0, 1, 5): + raise BrowserRuntimeError("native runtime unit cleanup failed", 503) + + +@dataclass +class RuntimeRecord: + alias: str + name: str + runtime_id: str + network_id: str + binding_version: int + profile_id: str + profile_dir: str + browser_version: str + browser_path: str + command: list[str] + network_exit_id: str = "" + proxy_port: int = 0 + display: int = 0 + cdp_port: int = 0 + node_id: str = "" + owner: str = "" + state: str = "created" + cleanup_state: str = "none" + cleanup_error: str = "" + browser_unit: str = "" + xvfb_unit: str = "" + browser_pid: int = 0 + xvfb_pid: int = 0 + browser_start_time: str = "" + xvfb_start_time: str = "" + created_at: float = 0.0 + updated_at: float = 0.0 + + def to_dict(self) -> dict[str, Any]: + return {field.name: getattr(self, field.name) for field in fields(self)} + + @classmethod + def from_dict(cls, value: object) -> RuntimeRecord: + if not isinstance(value, dict): + raise BrowserRuntimeError("runtime metadata is invalid", 500) + required = { + "alias", + "name", + "runtime_id", + "network_id", + "binding_version", + "profile_id", + "profile_dir", + "browser_version", + "browser_path", + "command", + "state", + "cleanup_state", + } + if not required.issubset(value): + raise BrowserRuntimeError("runtime metadata is incomplete", 500) + command = value["command"] + if not isinstance(command, list) or not all(isinstance(item, str) for item in command): + raise BrowserRuntimeError("runtime command metadata is invalid", 500) + try: + record = cls( + alias=value["alias"], + name=value["name"], + runtime_id=value["runtime_id"], + network_id=value["network_id"], + binding_version=value["binding_version"], + profile_id=value["profile_id"], + profile_dir=value["profile_dir"], + browser_version=value["browser_version"], + browser_path=value["browser_path"], + command=command, + network_exit_id=value.get("network_exit_id", ""), + proxy_port=value.get("proxy_port", 0), + display=value.get("display", 0), + cdp_port=value.get("cdp_port", 0), + node_id=value.get("node_id", ""), + owner=value.get("owner", ""), + state=value["state"], + cleanup_state=value["cleanup_state"], + cleanup_error=value.get("cleanup_error", ""), + browser_unit=value.get("browser_unit", ""), + xvfb_unit=value.get("xvfb_unit", ""), + browser_pid=value.get("browser_pid", 0), + xvfb_pid=value.get("xvfb_pid", 0), + browser_start_time=value.get("browser_start_time", ""), + xvfb_start_time=value.get("xvfb_start_time", ""), + created_at=value.get("created_at", 0.0), + updated_at=value.get("updated_at", 0.0), + ) + except (KeyError, TypeError, ValueError) as exc: + raise BrowserRuntimeError("runtime metadata is invalid", 500) from exc + if ( + not isinstance(record.alias, str) + or not ALIAS_RE.fullmatch(record.alias) + or not isinstance(record.name, str) + or not isinstance(record.runtime_id, str) + or not RUNTIME_ID_RE.fullmatch(record.runtime_id) + or not isinstance(record.network_id, str) + or not NETWORK_ID_RE.fullmatch(record.network_id) + or type(record.binding_version) is not int + or record.binding_version < 1 + or not isinstance(record.profile_id, str) + or not PROFILE_ID_RE.fullmatch(record.profile_id) + or not isinstance(record.browser_version, str) + or not BROWSER_VERSION_RE.fullmatch(record.browser_version) + or not isinstance(record.state, str) + or not isinstance(record.cleanup_state, str) + ): + raise BrowserRuntimeError("runtime metadata is invalid", 500) + return record + + +class NativeRuntimeManager: + def __init__( + self, + *, + state_dir: str | Path, + profile_root: str | Path, + node_id: str, + browser_versions: Mapping[str, str], + unit_manager: UnitManager | None = None, + proxy_registry: ProxyRegistry | None = None, + cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT, + ready_timeout: float = DEFAULT_READY_TIMEOUT, + display_start: int = DEFAULT_DISPLAY_START, + display_end: int = DEFAULT_DISPLAY_END, + external_display: int | None = None, + cdp_port_start: int = DEFAULT_CDP_PORT_START, + cdp_port_end: int = DEFAULT_CDP_PORT_END, + proxy_port_start: int = DEFAULT_PROXY_PORT_START, + proxy_port_end: int = DEFAULT_PROXY_PORT_END, + min_free_bytes: int = 20 * 1024**3, + log_max_bytes: int = 1 * 1024**3, + profile_cache_max_bytes: int = 20 * 1024**3, + clock: Callable[[], float] = time.time, + recover: bool = True, + ) -> None: + self.state_dir = Path(state_dir).expanduser().resolve() + self.profile_root = Path(profile_root).expanduser().resolve() + self.node_id = node_id + self.browser_versions = { + version: str(Path(path).expanduser().resolve()) + for version, path in browser_versions.items() + } + self.cleanup_timeout = cleanup_timeout + self.ready_timeout = ready_timeout + self.display_start = display_start + self.display_end = display_end + self.external_display = external_display + if external_display is not None and external_display < 1: + raise BrowserRuntimeError("external Xvfb display is invalid", 500) + self.cdp_port_start = cdp_port_start + self.cdp_port_end = cdp_port_end + self.proxy_port_start = proxy_port_start + self.proxy_port_end = proxy_port_end + self.min_free_bytes = min_free_bytes + self.log_max_bytes = log_max_bytes + self.profile_cache_max_bytes = profile_cache_max_bytes + if min_free_bytes < 0 or log_max_bytes < 0 or profile_cache_max_bytes < 0: + raise BrowserRuntimeError("runtime resource limits are invalid", 500) + self.clock = clock + self.unit_manager = unit_manager or SystemdUnitManager() + self.xvfb_path = shutil.which("Xvfb") or "" + self.proxies = proxy_registry or ProxyRegistry() + self.port_allocator = PortAllocator(self.state_dir / "locks") + self.display_allocator = DisplayAllocator(self.state_dir / "locks") + self._lock = threading.RLock() + self._alias_locks: dict[str, FileLock] = {} + self._cancel_events: dict[str, threading.Event] = {} + self._profile_locks: dict[str, FileLock] = {} + self._display_leases: dict[str, _Lease] = {} + self._port_leases: dict[tuple[str, str], _Lease] = {} + self._proxy_undo: dict[str, Callable[[], None]] = {} + self._mkdirs() + if recover: + self._recover() + + def _check_capacity(self) -> None: + for path in (self.state_dir, self.profile_root): + usage = shutil.disk_usage(path) + if usage.free < self.min_free_bytes: + raise BrowserRuntimeError("insufficient free disk space for browser runtime", 507) + + def _mkdirs(self) -> None: + for path in ( + self.state_dir, + self.state_dir / "runtimes", + self.state_dir / "locks", + self.state_dir / "logs", + self.profile_root, + ): + path.mkdir(parents=True, exist_ok=True, mode=0o700) + if path.stat().st_uid != os.getuid() or path.stat().st_mode & 0o077: + raise BrowserRuntimeError(f"runtime directory ownership is invalid: {path}", 500) + + def close(self) -> None: + with self._lock: + for lock in list(self._alias_locks.values()): + lock.release() + self._alias_locks.clear() + for lock in list(self._profile_locks.values()): + lock.release() + self._profile_locks.clear() + for lease in list(self._display_leases.values()): + lease.lock.release() + self._display_leases.clear() + for lease in list(self._port_leases.values()): + lease.lock.release() + self._port_leases.clear() + self._proxy_undo.clear() + self.proxies.close() + + def alias_lock(self, alias: str): + return _ManagedAliasLock(self, alias) + + def create(self, value: dict[str, Any]) -> dict[str, Any]: + validate_runtime_input(value) + alias = value["alias"] + with self.alias_lock(alias): + existing = self._find_alias(alias, include_released=False) + if existing is not None: + if existing.cleanup_state == "pending": + raise RuntimeCleanupPending( + "runtime cleanup is pending", existing.network_id + ) + raise BrowserRuntimeError("browser alias is already in use", 409) + runtime_id = __import__("secrets").token_hex(32) + network_id = "native-" + __import__("secrets").token_hex(16) + profile_id = value.get("profile_id", alias) + profile_dir = self._profile_dir(profile_id) + browser_path = self.browser_versions.get(value["browser_version"]) + if not browser_path or not os.access(browser_path, os.X_OK): + raise BrowserRuntimeError("browser version is unavailable", 422) + self._check_capacity() + now = self.clock() + record = RuntimeRecord( + alias=alias, + name=value["name"], + runtime_id=runtime_id, + network_id=network_id, + binding_version=value["binding_version"], + profile_id=profile_id, + profile_dir=str(profile_dir), + browser_version=value["browser_version"], + browser_path=browser_path, + command=list(value["cmd"]), + network_exit_id=value.get("network_exit_id", ""), + node_id=self.node_id, + owner=self.node_id, + state="stopped" if value.get("stopped", False) else "starting", + cleanup_state="none", + created_at=now, + updated_at=now, + ) + self._prepare_profile(record) + self._write(record) + if record.state == "stopped": + return self._public(record, False) + self._cancel_events[record.runtime_id] = threading.Event() + try: + self._start_locked(record, value["network_exit"]) + return self._public(record, True) + except BrowserRuntimeError: + raise + except Exception as exc: + self._mark_cleanup_pending(record, str(exc)) + raise BrowserRuntimeError("native browser runtime creation failed", 502, record.network_id) from exc + finally: + self._cancel_events.pop(record.runtime_id, None) + + def change_state(self, alias: str, action: str, generation: Mapping[str, Any]) -> None: + if action == "cancel": + self.cancel(alias, generation) + return + if action not in {"start", "stop"}: + raise BrowserRuntimeError("invalid runtime action", 400) + with self.alias_lock(alias): + record = self._require_record(alias, generation) + if action == "start": + if record.state == "running": + if not self._ready(record): + raise BrowserRuntimeError("browser runtime is not ready", 503, record.network_id) + return + if record.state != "stopped": + raise BrowserRuntimeError("runtime cannot be started in its current state", 409, record.network_id) + self._start_locked(record, ProxyExit("", "", 0)) + return + if record.state == "stopped": + return + if record.state not in {"running", "degraded", "starting"}: + if record.cleanup_state == "pending": + raise RuntimeCleanupPending("runtime cleanup is pending", record.network_id) + raise BrowserRuntimeError("runtime cannot be stopped in its current state", 409, record.network_id) + self._stop_locked(record, released=False) + + def remove(self, alias: str, generation: Mapping[str, Any], purge_profile: bool = False) -> None: + with self.alias_lock(alias): + record = self._find_record_for_generation(alias, generation) + if record is None: + if generation.get("runtime_id") == RUNTIME_CLEANUP_SENTINEL: + return + raise FileNotFoundError(alias) + self._fence(record, generation) + if record.state == "released" and record.cleanup_state == "cleaned": + if purge_profile: + self._purge_profile(record) + return + self._stop_locked(record, released=True) + if purge_profile: + self._purge_profile(record) + + def restore_proxy(self, alias: str, generation: Mapping[str, Any], exit: ProxyExit) -> None: + with self.alias_lock(alias): + record = self._require_record(alias, generation, require_active=False) + if record.network_exit_id and record.network_exit_id != generation.get("network_exit_id", record.network_exit_id): + raise GenerationConflict("proxy generation does not match runtime") + if exit == ProxyExit("", "", 0): + self.proxies.remove( + alias, record.binding_version, record.runtime_id, record.network_id + ) + record.proxy_port = 0 + self._write(record) + return + if record.state not in {"running", "degraded"}: + raise BrowserRuntimeError("proxy can only be restored for a running runtime", 409, record.network_id) + port = record.proxy_port + if port < 1: + lease = self._reserve_port(record, "proxy", self.proxy_port_start, self.proxy_port_end) + port = lease.value + self._port_leases[(record.runtime_id, "proxy")] = lease + if (record.runtime_id, "proxy") not in self._port_leases: + self._port_leases[(record.runtime_id, "proxy")] = self.port_allocator.reserve_existing("proxy", port) + url, undo = self.proxies.configure( + alias, + record.binding_version, + "127.0.0.1", + port, + exit, + record.network_id, + ) + if not self.proxies.bind( + alias, record.binding_version, url, record.runtime_id, record.network_id + ): + undo() + raise GenerationConflict("proxy generation changed") + self._proxy_undo[record.runtime_id] = undo + record.proxy_port = urlsplit(url).port or 0 + record.network_exit_id = generation.get("network_exit_id", record.network_exit_id) + self._write(record) + + def cancel(self, alias: str, generation: Mapping[str, Any]) -> None: + with self._lock: + record = self._require_record(alias, generation, require_active=False) + if record.state != "starting": + raise BrowserRuntimeError("runtime is not starting", 409, record.network_id) + event = self._cancel_events.get(record.runtime_id) + if event is not None: + event.set() + return + with self.alias_lock(alias): + record = self._require_record(alias, generation, require_active=False) + self._stop_locked(record, released=False) + + def endpoint(self, alias: str) -> str: + with self._lock: + record = self._find_alias(alias, include_released=False) + if record is None or record.state not in {"running", "degraded"}: + raise FileNotFoundError(alias) + self._refresh(record) + if record.state not in {"running", "degraded"} or not record.cdp_port: + raise BrowserRuntimeError("browser runtime is unavailable", 503, record.network_id) + return f"http://127.0.0.1:{record.cdp_port}" + + def require_generation(self, alias: str, generation: Mapping[str, Any]) -> RuntimeRecord: + with self._lock: + record = self._require_record(alias, generation) + self._refresh(record) + if record.state != "running" or not self._ready(record): + raise BrowserRuntimeError("browser runtime is not ready", 503, record.network_id) + return record + + def list_public(self) -> list[dict[str, Any]]: + with self._lock: + records = self._records() + result = [] + for record in records: + if record.state not in {"released"}: + self._refresh(record) + result.append(self._public(record, self._ready(record) if record.state in {"running", "degraded"} else False)) + return result + + def retry_cleanup(self, alias: str, generation: Mapping[str, Any]) -> None: + with self.alias_lock(alias): + record = self._require_record(alias, generation, require_active=False) + if record.cleanup_state != "pending": + return + self._stop_locked(record, released=record.state != "stopped") + + def _start_locked(self, record: RuntimeRecord, proxy_exit: ProxyExit) -> None: + self._check_cancel(record) + self._prepare_profile(record) + self._check_profile_size(record) + if self._profile_in_use(record): + raise BrowserRuntimeError("Profile is already in use", 409, record.network_id) + profile_lock = FileLock(Path(record.profile_dir) / ".creatorhub-profile.lock") + if not profile_lock.acquire(): + raise BrowserRuntimeError("Profile is already in use", 409, record.network_id) + self._profile_locks[record.runtime_id] = profile_lock + display_lease: _Lease | None = None + cdp_lease: _Lease | None = None + proxy_lease: _Lease | None = None + try: + display_lease = self._reserve_display(record) + self._display_leases[record.runtime_id] = display_lease + self._check_cancel(record) + cdp_lease = self._reserve_port(record, "cdp", self.cdp_port_start, self.cdp_port_end) + self._port_leases[(record.runtime_id, "cdp")] = cdp_lease + record.display = display_lease.value + record.cdp_port = cdp_lease.value + record.state = "starting" + record.cleanup_state = "none" + record.cleanup_error = "" + record.browser_unit = self._unit_name("browser", record) + record.xvfb_unit = ( + self._unit_name("xvfb", record) + if self.external_display is None + else "" + ) + runtime_dir = self._runtime_dir(record) + runtime_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + (runtime_dir / "tmp").mkdir(parents=True, exist_ok=True, mode=0o700) + self._write(record) + + proxy_url = "" + if proxy_exit != ProxyExit("", "", 0): + proxy_lease = self._reserve_port(record, "proxy", self.proxy_port_start, self.proxy_port_end) + self._port_leases[(record.runtime_id, "proxy")] = proxy_lease + proxy_url, undo = self.proxies.configure( + record.alias, + record.binding_version, + "127.0.0.1", + proxy_lease.value, + proxy_exit, + record.network_id, + ) + self._proxy_undo[record.runtime_id] = undo + record.proxy_port = urlsplit(proxy_url).port or 0 + self._write(record) + + if self.external_display is None: + xvfb_log = self._log_path(record, "xvfb") + xvfb_command = self._locked_command( + self._display_lock_path(record.display), + [ + self.xvfb_path or "Xvfb", + f":{record.display}", + "-screen", + "0", + "1280x720x24", + "-nolisten", + "tcp", + ], + ) + xvfb_status = self.unit_manager.start( + record.xvfb_unit, + xvfb_command, + environment={"DISPLAY": f":{record.display}"}, + working_directory=runtime_dir, + stdout_path=xvfb_log, + limits={"MemoryMax": "256M", "TasksMax": "128", "TimeoutStopSec": "10s"}, + ) + record.xvfb_pid = xvfb_status.pid + record.xvfb_start_time = xvfb_status.start_time + else: + record.xvfb_pid = 0 + record.xvfb_start_time = "external" + self._release_display_lease(record) + self._write(record) + self._wait_for_display(record) + self._check_cancel(record) + + browser_command = self._browser_command(record, proxy_url) + browser_log = self._log_path(record, "browser") + browser_status = self.unit_manager.start( + record.browser_unit, + self._locked_command( + Path(record.profile_dir) / ".creatorhub-profile.lock", browser_command + ), + environment={"DISPLAY": f":{record.display}"}, + working_directory=runtime_dir, + stdout_path=browser_log, + limits={"MemoryMax": "1G", "TasksMax": "512", "CPUQuota": "200%", "TimeoutStopSec": "10s"}, + ) + self._release_startup_locks(record) + record.browser_pid = browser_status.pid + record.browser_start_time = browser_status.start_time + self._write(record) + self._wait_for_cdp(record) + self._check_cancel(record) + if proxy_url and not self.proxies.bind( + record.alias, + record.binding_version, + proxy_url, + record.runtime_id, + record.network_id, + ): + raise GenerationConflict("proxy generation changed") + record.state = "running" + record.cleanup_state = "none" + record.updated_at = self.clock() + self._write(record) + # Keep the Profile lock while the unit owns the browser. The unit + # also takes this lock through `flock`, so a gateway restart cannot + # create a second browser for the same Profile. + except Exception as exc: + self._cleanup_after_failure(record, exc) + if isinstance(exc, BrowserRuntimeError): + raise + raise BrowserRuntimeError("native browser runtime start failed", 502, record.network_id) from exc + + def _stop_units(self, record: RuntimeRecord) -> list[str]: + errors: list[str] = [] + for unit in (record.browser_unit, record.xvfb_unit): + if not unit: + continue + try: + status = self.unit_manager.status(unit) + if status.active or status.pid: + self.unit_manager.stop(unit, self.cleanup_timeout) + after = self.unit_manager.status(unit) + if after.active or after.pid: + errors.append(f"{unit}: unit remains active after stop") + self.unit_manager.reset(unit) + except Exception as exc: + errors.append(f"{unit}: {exc}") + return errors + + def _stop_locked(self, record: RuntimeRecord, released: bool) -> None: + record.state = "stopping" + self._write(record) + errors = self._stop_units(record) + try: + if not self.proxies.remove( + record.alias, + record.binding_version, + record.runtime_id, + record.network_id, + ): + errors.append("proxy generation does not match runtime") + except Exception as exc: + errors.append(f"proxy: {exc}") + if errors: + self._mark_cleanup_pending(record, "; ".join(errors)) + raise RuntimeCleanupPending("runtime cleanup is pending", record.network_id) + self._release_runtime_leases(record) + self._release_startup_locks(record) + self._remove_runtime_tmp(record) + record.state = "released" if released else "stopped" + record.cleanup_state = "cleaned" + record.cleanup_error = "" + record.browser_pid = record.xvfb_pid = 0 + record.display = record.cdp_port = record.proxy_port = 0 + record.updated_at = self.clock() + self._write(record) + + def _cleanup_after_failure(self, record: RuntimeRecord, error: Exception) -> None: + errors = [str(error), *self._stop_units(record)] + if len(errors) == 1: + try: + if not self.proxies.remove( + record.alias, + record.binding_version, + record.runtime_id, + record.network_id, + ): + errors.append("proxy generation does not match runtime") + except Exception as exc: + errors.append(f"proxy: {exc}") + if len(errors) == 1: + self._release_runtime_leases(record) + self._remove_runtime_tmp(record) + self._release_startup_locks(record) + self._mark_cleanup_pending(record, "; ".join(item for item in errors if item)) + + def _mark_cleanup_pending(self, record: RuntimeRecord, error: str) -> None: + record.state = "failed" + record.cleanup_state = "pending" + record.cleanup_error = error[:4096] + record.updated_at = self.clock() + self._write(record) + LOG.error( + "native runtime cleanup pending", + extra={ + "alias": record.alias, + "runtime_id": record.runtime_id, + "binding_version": record.binding_version, + "network_id": record.network_id, + "error": record.cleanup_error, + }, + ) + + def _recover(self) -> None: + with self._lock: + for record in self._records(): + if record.state not in {"running", "starting", "stopping", "degraded"}: + continue + browser = self.unit_manager.status(record.browser_unit) if record.browser_unit else UnitStatus(False, "missing", 0, "", "") + xvfb = self._display_status(record) + if browser.active and xvfb.active: + record.browser_pid = browser.pid + record.xvfb_pid = xvfb.pid + record.browser_start_time = browser.start_time + record.xvfb_start_time = xvfb.start_time + record.state = "running" + record.cleanup_state = "none" + record.cleanup_error = "" + self._write(record) + else: + self._mark_cleanup_pending( + record, + "runtime unit disappeared during gateway restart", + ) + + def _refresh(self, record: RuntimeRecord) -> None: + if record.state not in {"running", "degraded", "starting"}: + return + browser = self.unit_manager.status(record.browser_unit) if record.browser_unit else UnitStatus(False, "missing", 0, "", "") + xvfb = self._display_status(record) + if not browser.active or not xvfb.active: + record.state = "degraded" + record.cleanup_state = "pending" + record.cleanup_error = "runtime unit is not active" + record.updated_at = self.clock() + self._write(record) + return + record.browser_pid = browser.pid + record.xvfb_pid = xvfb.pid + if record.state == "starting": + record.state = "degraded" + self._write(record) + + def _ready(self, record: RuntimeRecord) -> bool: + if record.cdp_port < 1: + return False + try: + version = self._get_json(record.cdp_port, "/json/version") + targets = self._get_json(record.cdp_port, "/json/list") + except (OSError, ValueError, BrowserRuntimeError): + return False + return isinstance(version, dict) and isinstance(targets, list) and any( + isinstance(target, dict) and target.get("type") == "page" for target in targets + ) + + def _display_status(self, record: RuntimeRecord) -> UnitStatus: + if record.xvfb_unit: + return self.unit_manager.status(record.xvfb_unit) + available = self._display_available(record.display) + return UnitStatus(available, "external" if available else "missing", 0, "", "") + + def _display_available(self, display: int) -> bool: + return display > 0 and Path(f"/tmp/.X11-unix/X{display}").exists() + + def _wait_for_display(self, record: RuntimeRecord) -> None: + deadline = time.monotonic() + self.ready_timeout + while time.monotonic() < deadline: + self._check_cancel(record) + if self._display_available(record.display): + return + status = self._display_status(record) + if not status.active and status.pid == 0: + raise BrowserRuntimeError("Xvfb unit exited before readiness", 503, record.network_id) + time.sleep(0.05) + raise BrowserRuntimeError("Xvfb did not become ready", 503, record.network_id) + + def _wait_for_cdp(self, record: RuntimeRecord) -> None: + deadline = time.monotonic() + self.ready_timeout + while time.monotonic() < deadline: + self._check_cancel(record) + if self._ready(record): + return + status = self.unit_manager.status(record.browser_unit) + if not status.active and status.pid == 0: + raise BrowserRuntimeError("browser unit exited before CDP readiness", 503, record.network_id) + time.sleep(0.1) + raise BrowserRuntimeError("browser CDP did not become ready", 503, record.network_id) + + def _get_json(self, port: int, path: str) -> object: + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=1.0) + try: + connection.request("GET", path, headers={"Accept": "application/json"}) + response = connection.getresponse() + body = response.read(64 * 1024 + 1) + except (OSError, http.client.HTTPException) as exc: + raise BrowserRuntimeError("browser CDP probe failed", 503) from exc + finally: + connection.close() + if response.status != 200 or len(body) > 64 * 1024: + raise BrowserRuntimeError("browser CDP probe failed", 503) + try: + return json.loads(body) + except json.JSONDecodeError as exc: + raise BrowserRuntimeError("browser CDP response is invalid", 503) from exc + + def _check_cancel(self, record: RuntimeRecord) -> None: + event = self._cancel_events.get(record.runtime_id) + if event is not None and event.is_set(): + raise RuntimeCancelled(network_id=record.network_id) + + def _check_profile_size(self, record: RuntimeRecord) -> None: + if self.profile_cache_max_bytes == 0: + return + total = 0 + root = Path(record.profile_dir) + for path in root.rglob("*"): + if path.is_symlink() or not path.is_file(): + continue + try: + total += path.stat().st_size + except OSError as exc: + raise BrowserRuntimeError("Profile size could not be checked", 503, record.network_id) from exc + if total > self.profile_cache_max_bytes: + raise BrowserRuntimeError("Profile cache exceeds configured limit", 507, record.network_id) + + def _browser_command(self, record: RuntimeRecord, proxy_url: str) -> list[str]: + command = list(record.command) + if not command or command[-1] != "about:blank": + raise BrowserRuntimeError("browser command must end with about:blank", 400, record.network_id) + arguments = command[:-1] + arguments.extend( + [ + "--remote-debugging-address=127.0.0.1", + f"--remote-debugging-port={record.cdp_port}", + "--remote-allow-origins=http://127.0.0.1", + "--disable-gpu", + "--disable-gpu-compositing", + f"--user-data-dir={record.profile_dir}", + "--no-first-run", + "--no-default-browser-check", + ] + ) + if proxy_url: + arguments.extend([f"--proxy-server={proxy_url}", "--disable-non-proxied-udp"]) + arguments.append("about:blank") + return [record.browser_path, *arguments] + + def _locked_command(self, lock_path: Path, command: list[str]) -> list[str]: + flock = shutil.which("flock") + if not flock: + raise BrowserRuntimeError("flock is unavailable for runtime ownership", 503) + return [flock, str(lock_path), *command] + + def _profile_dir(self, profile_id: str) -> Path: + if not isinstance(profile_id, str) or not PROFILE_ID_RE.fullmatch(profile_id): + raise BrowserRuntimeError("profile_id is invalid", 400) + digest = hashlib.sha256(profile_id.encode()).hexdigest()[:32] + path = (self.profile_root / digest).resolve() + try: + path.relative_to(self.profile_root) + except ValueError as exc: + raise BrowserRuntimeError("profile path escaped configured root", 500) from exc + return path + + def _prepare_profile(self, record: RuntimeRecord) -> None: + path = Path(record.profile_dir) + path.mkdir(parents=True, exist_ok=True, mode=0o700) + stat = path.stat() + if stat.st_uid != os.getuid() or stat.st_mode & 0o077: + raise BrowserRuntimeError("Profile ownership or permissions are invalid", 409, record.network_id) + + def _runtime_dir(self, record: RuntimeRecord) -> Path: + return self.state_dir / "runtimes" / record.runtime_id + + def _log_path(self, record: RuntimeRecord, component: str) -> Path: + path = self.state_dir / "logs" / record.runtime_id / f"{component}.log" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if self.log_max_bytes and path.exists() and path.stat().st_size > self.log_max_bytes: + raise BrowserRuntimeError("runtime log exceeds configured limit", 507, record.network_id) + return path + + def _display_lock_path(self, display: int) -> Path: + return self.state_dir / "locks" / f"display-{display}.lock" + + def _record_path(self, record: RuntimeRecord) -> Path: + return self._runtime_dir(record) / "runtime.json" + + def _write(self, record: RuntimeRecord) -> None: + path = self._record_path(record) + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + encoded = json.dumps(record.to_dict(), ensure_ascii=False, separators=(",", ":")) + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as output: + output.write(encoded) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + + def _records(self) -> list[RuntimeRecord]: + result = [] + for path in sorted((self.state_dir / "runtimes").glob("*/runtime.json")): + try: + result.append(RuntimeRecord.from_dict(json.loads(path.read_text(encoding="utf-8")))) + except OSError as exc: + raise BrowserRuntimeError("runtime metadata cannot be read", 500) from exc + except json.JSONDecodeError as exc: + raise BrowserRuntimeError("runtime metadata is invalid", 500) from exc + return sorted(result, key=lambda item: (item.created_at, item.alias)) + + def _find_alias(self, alias: str, include_released: bool) -> RuntimeRecord | None: + if not ALIAS_RE.fullmatch(alias): + raise BrowserRuntimeError("runtime alias is invalid", 400) + matches = [record for record in self._records() if record.alias == alias] + active = [record for record in matches if include_released or record.state != "released"] + if len(active) > 1: + if include_released: + return max(active, key=lambda item: (item.created_at, item.runtime_id)) + raise UnmanagedRuntime("multiple active runtime generations use the same alias") + return active[0] if active else None + + def _find_record_for_generation( + self, alias: str, generation: Mapping[str, Any] + ) -> RuntimeRecord | None: + if not ALIAS_RE.fullmatch(alias): + raise BrowserRuntimeError("runtime alias is invalid", 400) + matches = [record for record in self._records() if record.alias == alias] + runtime_id = generation.get("runtime_id", "") + if isinstance(runtime_id, str) and runtime_id not in {"", RUNTIME_CLEANUP_SENTINEL}: + exact = [record for record in matches if record.runtime_id == runtime_id] + if exact: + return max(exact, key=lambda item: (item.created_at, item.runtime_id)) + nonreleased = [record for record in matches if record.state != "released"] + if nonreleased: + return max(nonreleased, key=lambda item: (item.created_at, item.runtime_id)) + return max(matches, key=lambda item: (item.created_at, item.runtime_id)) if matches else None + + def _require_record( + self, + alias: str, + generation: Mapping[str, Any], + require_active: bool = True, + ) -> RuntimeRecord: + record = self._find_alias(alias, include_released=True) + if record is None: + raise FileNotFoundError(alias) + self._fence(record, generation) + if require_active and record.state == "released": + raise FileNotFoundError(alias) + return record + + def _fence(self, record: RuntimeRecord, generation: Mapping[str, Any]) -> None: + if ( + generation.get("runtime_id") != record.runtime_id + or generation.get("binding_version") != record.binding_version + or generation.get("network_id") != record.network_id + ): + raise GenerationConflict() + + def _public(self, record: RuntimeRecord, ready: bool) -> dict[str, Any]: + endpoint = f"http://127.0.0.1:{record.cdp_port}" if ready and record.cdp_port else "" + proxy_ready = bool( + ready + and ( + not record.network_exit_id + or self.proxies.ready( + record.alias, + record.proxy_port, + record.runtime_id, + record.network_id, + ) + ) + ) + return { + "id": record.runtime_id, + "runtime_id": record.runtime_id, + "alias": record.alias, + "name": record.name, + "state": record.state, + "status": "ready" if ready else record.cleanup_error or record.state, + "ready": ready, + "endpoint": endpoint, + "binding_version": record.binding_version, + "network_exit_id": record.network_exit_id, + "network_id": record.network_id, + "node_id": record.node_id, + "owner": record.owner, + "browser_version": record.browser_version, + "proxy_ready": proxy_ready, + "cleanup_state": record.cleanup_state, + "cleanup_error": record.cleanup_error, + "display": record.display, + "cdp_port": record.cdp_port, + "proxy_port": record.proxy_port, + "display_mode": "external" if not record.xvfb_unit and record.display else "managed", + } + + def _unit_name(self, component: str, record: RuntimeRecord) -> str: + return f"creatorhub-{record.alias}-{record.runtime_id[:16]}-{component}.service" + + def _reserve_display(self, record: RuntimeRecord) -> _Lease: + if self.external_display is not None: + if any( + item.runtime_id != record.runtime_id + and item.state != "released" + and item.display == self.external_display + for item in self._records() + ): + raise BrowserRuntimeError("external Xvfb display is already in use", 409, record.network_id) + if not self._display_available(self.external_display): + raise BrowserRuntimeError("external Xvfb display is unavailable", 503, record.network_id) + return self.display_allocator.reserve_existing(self.external_display) + + def unavailable(value: int) -> bool: + return any( + item.runtime_id != record.runtime_id + and item.state != "released" + and item.display == value + for item in self._records() + ) + + return self.display_allocator.reserve( + self.display_start, + self.display_end, + unavailable, + ) + + def _reserve_port(self, record: RuntimeRecord, prefix: str, start: int, end: int) -> _Lease: + def unavailable(value: int) -> bool: + for item in self._records(): + if item.runtime_id == record.runtime_id: + continue + if item.state in {"released"}: + continue + if value in {item.cdp_port, item.proxy_port}: + return True + return False + + return self.port_allocator.reserve(prefix, start, end, unavailable) + + def _release_startup_locks(self, record: RuntimeRecord) -> None: + lock = self._profile_locks.pop(record.runtime_id, None) + if lock: + lock.release() + + def _release_display_lease(self, record: RuntimeRecord) -> None: + lease = self._display_leases.pop(record.runtime_id, None) + if lease: + lease.lock.release() + + def _profile_in_use(self, record: RuntimeRecord) -> bool: + return any( + item.runtime_id != record.runtime_id + and item.state != "released" + and item.profile_dir == record.profile_dir + for item in self._records() + ) + + def _release_runtime_leases(self, record: RuntimeRecord) -> None: + display = self._display_leases.pop(record.runtime_id, None) + if display: + display.lock.release() + for key in ((record.runtime_id, "cdp"), (record.runtime_id, "proxy")): + lease = self._port_leases.pop(key, None) + if lease: + lease.lock.release() + undo = self._proxy_undo.pop(record.runtime_id, None) + if undo: + with suppress(Exception): + undo() + + def _remove_runtime_tmp(self, record: RuntimeRecord) -> None: + temporary = self._runtime_dir(record) / "tmp" + if not temporary.exists(): + return + for path in sorted(temporary.rglob("*"), reverse=True): + if path.is_file() or path.is_symlink(): + path.unlink() + elif path.is_dir(): + path.rmdir() + with suppress(OSError): + temporary.rmdir() + + def _purge_profile(self, record: RuntimeRecord) -> None: + path = Path(record.profile_dir).resolve() + try: + path.relative_to(self.profile_root) + except ValueError as exc: + raise BrowserRuntimeError("Profile path does not belong to gateway", 409) from exc + if path == self.profile_root or not path.exists(): + return + lock = FileLock(path / ".creatorhub-profile.lock") + if not lock.acquire(): + raise RuntimeCleanupPending("Profile is still in use", record.network_id) + try: + for child in sorted(path.rglob("*"), reverse=True): + if child.is_file() or child.is_symlink(): + child.unlink() + elif child.is_dir(): + child.rmdir() + path.rmdir() + finally: + lock.release() + + +def validate_runtime_input(value: Mapping[str, Any]) -> None: + if not isinstance(value, Mapping): + raise BrowserRuntimeError("request body must be one JSON object", 400) + allowed = { + "alias", + "name", + "browser_version", + "profile_id", + "cmd", + "binding_version", + "network_exit_id", + "network_exit", + "stopped", + } + if set(value) - allowed: + raise BrowserRuntimeError("body contains unknown fields", 400) + alias = value.get("alias", "") + name = value.get("name", "") + version = value.get("browser_version", "") + profile_id = value.get("profile_id", alias) + command = value.get("cmd") + binding = value.get("binding_version") + exit_id = value.get("network_exit_id", "") + stopped = value.get("stopped", False) + if not isinstance(alias, str) or not ALIAS_RE.fullmatch(alias): + raise BrowserRuntimeError("alias must match [a-z0-9][a-z0-9-]{0,31}", 400) + if not isinstance(name, str) or not 1 <= len(name) <= 64 or has_control(name): + raise BrowserRuntimeError("name must be 1..64 visible characters", 400) + if not isinstance(version, str) or not BROWSER_VERSION_RE.fullmatch(version): + raise BrowserRuntimeError("browser_version is invalid", 400) + if not isinstance(profile_id, str) or not PROFILE_ID_RE.fullmatch(profile_id): + raise BrowserRuntimeError("profile_id is invalid", 400) + if type(binding) is not int or binding < 1: + raise BrowserRuntimeError("binding_version is invalid", 400) + if not isinstance(command, list) or not 1 <= len(command) <= 64 or command[-1] != "about:blank": + raise BrowserRuntimeError("cmd must contain 1..64 arguments and end with about:blank", 400) + total = 0 + reserved = ( + "--user-data-dir", + "--remote-debugging-address", + "--remote-debugging-port", + "--remote-allow-origins", + "--remote-debugging-pipe", + "--proxy-server", + "--display", + "--headless", + ) + for item in command: + if not isinstance(item, str) or not item or has_control(item) or item == "--no-sandbox": + raise BrowserRuntimeError("cmd arguments are invalid", 400) + if any(item == flag or item.startswith(flag + "=") for flag in reserved): + raise BrowserRuntimeError("cmd contains a gateway-managed flag", 400) + if item == "--disable-non-proxied-udp": + raise BrowserRuntimeError("cmd contains a gateway-managed flag", 400) + total += len(item) + if total > 4096: + raise BrowserRuntimeError("cmd arguments exceed 4096 characters", 400) + if not isinstance(exit_id, str): + raise BrowserRuntimeError("network_exit_id must be a string", 400) + if exit_id and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}", exit_id): + raise BrowserRuntimeError("network_exit_id is invalid", 400) + exit_value = parse_proxy_exit(value.get("network_exit", {})) + direct = not exit_id and exit_value == ProxyExit("", "", 0) + if bool(exit_id) != (exit_value != ProxyExit("", "", 0)): + raise BrowserRuntimeError("network_exit_id and network_exit must identify one binding", 400) + if type(stopped) is not bool: + raise BrowserRuntimeError("stopped must be boolean", 400) + if stopped and not direct: + raise BrowserRuntimeError("stopped browsers must use direct networking", 400) + if not direct: + validate_proxy_exit(exit_value) + if isinstance(value, dict): + value["profile_id"] = profile_id + value["network_exit"] = exit_value + value["network_exit_id"] = exit_id + + +def parse_proxy_exit(value: object) -> ProxyExit: + if not isinstance(value, dict): + raise BrowserRuntimeError("network_exit must be an object", 400) + allowed = {"protocol", "host", "port", "username", "password"} + if set(value) - allowed: + raise BrowserRuntimeError("network_exit contains unknown fields", 400) + try: + result = ProxyExit( + value.get("protocol", ""), + value.get("host", ""), + value.get("port", 0), + value.get("username", ""), + value.get("password", ""), + ) + except (TypeError, ValueError) as exc: + raise BrowserRuntimeError("network_exit is invalid", 400) from exc + if ( + not all(isinstance(item, str) for item in (result.protocol, result.host, result.username, result.credential)) + or type(result.port) is not int + ): + raise BrowserRuntimeError("network_exit is invalid", 400) + return result + + +def validate_proxy_exit(exit: ProxyExit) -> None: + if ( + exit.protocol not in {"http", "https", "socks4", "socks5"} + or not exit.host + or len(exit.host) > 253 + or any(char in exit.host for char in "@/[]?# \t\r\n") + or not 1 <= exit.port <= 65535 + or (not exit.username and exit.credential) + or len(exit.username) > 255 + or len(exit.credential) > 255 + or has_control(exit.username) + or has_control(exit.credential) + ): + raise BrowserRuntimeError("network_exit must contain a valid proxy endpoint", 400) + + +def has_control(value: str) -> bool: + return any(ord(char) < 0x20 or ord(char) == 0x7F for char in value) + + +class _ManagedAliasLock: + def __init__(self, manager: NativeRuntimeManager, alias: str) -> None: + self.manager = manager + self.alias = alias + self.lock: FileLock | None = None + + def __enter__(self) -> _ManagedAliasLock: + if not ALIAS_RE.fullmatch(self.alias): + raise BrowserRuntimeError("runtime alias is invalid", 400) + with self.manager._lock: + lock = FileLock(self.manager.state_dir / "locks" / f"alias-{self.alias}.lock") + if not lock.acquire(): + raise BrowserRuntimeError("browser alias is busy", 409) + self.manager._alias_locks[self.alias] = lock + self.lock = lock + return self + + def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None: + with self.manager._lock: + lock = self.manager._alias_locks.pop(self.alias, None) + if lock: + lock.release() + self.lock = None diff --git a/cmd/docker_gateway/test_gateway.py b/cmd/browser_gateway/test_gateway.py similarity index 59% rename from cmd/docker_gateway/test_gateway.py rename to cmd/browser_gateway/test_gateway.py index 419edeb..7237d05 100644 --- a/cmd/docker_gateway/test_gateway.py +++ b/cmd/browser_gateway/test_gateway.py @@ -7,35 +7,15 @@ import socket import threading import unittest from collections import deque -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from importlib import import_module +from types import SimpleNamespace from typing import Any, cast from unittest.mock import Mock, patch import websocket -from .docker_client import ( - BINDING_VERSION_LABEL, - BROWSER_NETWORK_ROLE, - GATEWAY_MEMBER_LABEL, - MANAGED_LABEL, - NETWORK_EXIT_LABEL, - NETWORK_ID_LABEL, - NETWORK_ROLE_LABEL, - RESERVATION_LABEL, - RESERVATION_OWNER_LABEL, - RUNTIME_ID_LABEL, - AliasReservationManager, - DockerClient, - DockerError, - DockerResponse, - GenerationConflict, - NetworkSetupError, - TenantNetworkGeneration, - UnmanagedContainer, - split_image_ref, - tenant_network_name, -) +from .runtime import BrowserRuntimeError, GenerationConflict, NativeRuntimeManager, UnitStatus from .douyin import ( BrowserResponse, CDPConnection, @@ -73,10 +53,8 @@ from .proxy import ( gateway_module = import_module(f"{__package__}.gateway") douyin_module = import_module(f"{__package__}.douyin") proxy_module = import_module(f"{__package__}.proxy") -docker_client_module = import_module(f"{__package__}.docker_client") Gateway = gateway_module.Gateway RequestError = gateway_module.RequestError -browser_tmpfs = gateway_module.browser_tmpfs decode_generation = gateway_module.decode_generation json_bytes = gateway_module.json_bytes load_config = gateway_module.load_config @@ -126,21 +104,6 @@ class FakeConnection: return self.values.pop(0) -class FakeDocker: - def __init__(self, containers: list[dict] | None = None) -> None: - self.containers = containers or [] - - def container_network_address(self, container_id: str, network_id: str) -> str: - del container_id, network_id - return "192.0.2.10" - - def request( - self, method: str, path: str, body: object | None = None - ) -> DockerResponse: - del body - if method == "GET" and path.startswith("/containers/json"): - return DockerResponse(200, "OK", json.dumps(self.containers).encode()) - return DockerResponse(404, "Not Found", b"") class GatewayValidationTests(unittest.TestCase): @@ -148,10 +111,12 @@ class GatewayValidationTests(unittest.TestCase): value = { "alias": "safe-account", "name": "Safe account", - "image": "creatorhub/browser:latest", + "browser_version": "148.0.7778.215", + "profile_id": "safe-account", "cmd": ["about:blank"], - "volume": "creatorhub-safe-account", "binding_version": 1, + "network_exit_id": "", + "network_exit": {}, "stopped": True, } validate_create(value) @@ -162,7 +127,7 @@ class GatewayValidationTests(unittest.TestCase): { "binding_version": 1, "runtime_id": "runtime-not-found", - "network_id": "network-id", + "network_id": "native-" + "a" * 32, }, False, False, @@ -186,33 +151,31 @@ class GatewayValidationTests(unittest.TestCase): config = load_config( { "LISTEN_ADDR": ":8081", - "DOCKER_SOCKET": "/var/run/docker.sock", - "BROWSER_NETWORK": "creatorhub_browser", + "BROWSER_PATH": "/bin/true", + "BROWSER_VERSION": "1.2.3", "GATEWAY_TOKEN": "0123456789abcdef", } ) self.assertEqual(config["listen"], ("", 8081)) + self.assertEqual(config["browser_versions"]["1.2.3"], "/bin/true") + self.assertIsNone(config["external_display"]) external = load_config( { - "LISTEN_ADDR": ":8081", - "DOCKER_SOCKET": "/var/run/docker.sock", - "BROWSER_NETWORK": "creatorhub_browser", + "BROWSER_PATH": "/bin/true", + "BROWSER_VERSION": "1.2.3", "GATEWAY_TOKEN": "0123456789abcdef", - "BROWSER_CDP_URL": "http://127.0.0.1:9222", - "BROWSER_CDP_TARGET_ID": "target_1", + "RUNTIME_EXTERNAL_DISPLAY": "99", } ) - self.assertEqual(external["external_cdp"]["url"], "http://127.0.0.1:9222") - self.assertEqual(external["external_cdp"]["target_id"], "target_1") + self.assertEqual(external["external_display"], 99) with self.assertRaises(ValueError): load_config( { "GATEWAY_TOKEN": "0123456789abcdef", - "BROWSER_CDP_URL": "http://user:pass@127.0.0.1:9222", + "BROWSER_PATH": "/not/a/browser", + "BROWSER_VERSION": "1.2.3", } ) - tmp_mount = next(path for path in browser_tmpfs() if path.endswith("tmp")) - self.assertTrue(browser_tmpfs()[tmp_mount].startswith("rw,")) self.assertTrue( valid_douyin_url( "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp" @@ -247,33 +210,6 @@ class GatewayValidationTests(unittest.TestCase): ) ) - def test_external_cdp_is_virtual_and_generation_bound(self) -> None: - external = { - "url": "http://127.0.0.1:9222", - "target_id": "target_1", - "alias": "local-cdp", - "runtime_id": "0" * 64, - "network_id": "local-cdp", - "binding_version": 1, - } - gateway = Gateway( - cast(DockerClient, FakeDocker([])), - "creatorhub_browser", - "0123456789abcdef", - "gateway", - external_cdp=external, - ) - self.assertEqual(gateway.list_browsers()[0]["state"], "external") - generation = { - "binding_version": 1, - "runtime_id": "0" * 64, - "network_id": "local-cdp", - } - gateway._require_douyin_generation("local-cdp", generation) - with self.assertRaises(RequestError): - gateway._require_douyin_generation( - "local-cdp", {**generation, "binding_version": 2} - ) def test_http_routes_and_body_validation(self) -> None: handler = gateway_module.GatewayHandler.__new__(gateway_module.GatewayHandler) @@ -365,7 +301,7 @@ class GatewayValidationTests(unittest.TestCase): ) def test_validation_boundaries(self) -> None: - self.assertEqual(proxy_port("http://docker-gateway:1234"), 1234) + self.assertEqual(proxy_port("http://127.0.0.1:1234"), 1234) self.assertTrue(has_control("bad\nvalue")) exit_value = parse_proxy_exit( {"protocol": "http", "host": "proxy", "port": 8080} @@ -380,7 +316,7 @@ class GatewayValidationTests(unittest.TestCase): generation = { "binding_version": 1, "runtime_id": "a" * 64, - "network_id": "b" * 64, + "network_id": "native-" + "b" * 32, "network_exit_id": "exit", } self.assertTrue(valid_douyin_generation(generation)) @@ -394,7 +330,7 @@ class GatewayValidationTests(unittest.TestCase): restore = { "binding_version": 1, "runtime_id": "a" * 64, - "network_id": "b" * 64, + "network_id": "native-" + "b" * 32, "network_exit_id": "exit", "network_exit": {"protocol": "http", "host": "proxy", "port": 8080}, } @@ -410,14 +346,23 @@ class GatewayValidationTests(unittest.TestCase): { "alias": "safe", "name": "Safe", - "image": "bad image", - "volume": "safe", + "browser_version": "bad", + "profile_id": "safe", "binding_version": 1, "cmd": ["about:blank"], } ) with self.assertRaises(ValueError): load_config({"GATEWAY_TOKEN": "short"}) + with self.assertRaises(ValueError): + load_config( + { + "BROWSER_PATH": "/bin/true", + "BROWSER_VERSION": "1.2.3", + "GATEWAY_TOKEN": "0123456789abcdef", + "RUNTIME_EXTERNAL_DISPLAY": "0", + } + ) self.assertFalse(valid_douyin_url("http://www.douyin.com/video/1")) self.assertFalse(valid_douyin_url("https://www.douyin.com/unknown")) with self.assertRaises(RequestError): @@ -425,111 +370,196 @@ class GatewayValidationTests(unittest.TestCase): {"binding_version": 1, "runtime_id": "a" * 64}, True, True ) - def test_list_keeps_other_environments_when_one_network_is_missing(self) -> None: - class PartialDocker(FakeDocker): - def container_network_address( - self, container_id: str, network_id: str - ) -> str: - if container_id == "broken": - raise GenerationConflict("network attachment disappeared") - return "192.0.2.10" - containers = [ - { - "Id": "broken", - "Labels": { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: "broken-account", - BINDING_VERSION_LABEL: "1", - NETWORK_EXIT_LABEL: "", - NETWORK_ID_LABEL: "b" * 64, - }, - "State": "running", - "Status": "Up", - }, - { - "Id": "healthy", - "Labels": { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: "healthy-account", - BINDING_VERSION_LABEL: "1", - NETWORK_EXIT_LABEL: "", - NETWORK_ID_LABEL: "c" * 64, - }, - "State": "running", - "Status": "Up", - }, + + + +class GatewayBusinessMethodTests(unittest.TestCase): + def setUp(self) -> None: + self.runtime_id = "a" * 64 + self.network_id = "native-" + "b" * 32 + self.generation = { + "binding_version": 1, + "runtime_id": self.runtime_id, + "network_id": self.network_id, + "network_exit_id": "exit-1", + } + self.runtimes = Mock() + self.runtimes.proxies = Mock() + self.runtimes.browser_versions = {"148.0.7778.215": "/bin/true"} + self.runtimes.alias_lock.return_value = nullcontext() + self.runtimes.require_generation.return_value = SimpleNamespace( + runtime_id=self.runtime_id, network_exit_id="exit-1" + ) + self.runtimes.endpoint.return_value = "http://127.0.0.1:19001" + self.runtimes.list_public.return_value = [] + self.browser = Mock() + self.browser.get.return_value = Mock(status=200, body="body", challenge="") + self.browser.resolve.return_value = "https://www.douyin.com/video/123" + self.browser.get_media.return_value = Mock( + status=200, content_type="video/mp4", body_base64="dm"); + self.browser.identity.return_value = { + "uid": "12345678901234567890", + "sec_uid": "sec", + "unique_id": "name", + } + self.browser.login_qr.return_value = Mock( + content_type="image/png", body_base64="cG5n", qr_detected=True + ) + self.browser.message_history.return_value = {"status": "succeeded"} + self.browser.action.return_value = {"status": "succeeded"} + self.browser.action_ownership.return_value = None + self.xhs_browser = Mock() + self.xhs_browser.get.return_value = Mock(status=200, body="body", challenge="") + self.xhs_browser.post.return_value = Mock(status=200, body="body", challenge="") + self.xhs_browser.resolve.return_value = "https://www.xiaohongshu.com/explore/abc" + self.xhs_browser.get_media.return_value = Mock( + status=200, content_type="video/mp4", body_base64="dm" + ) + self.xhs_browser.identity.return_value = {"uid": "xhs-user"} + self.gateway = Gateway( + self.runtimes, + "gateway-token-123456", + "node-a", + browser=self.browser, + xiaohongshu_browser=self.xhs_browser, + ) + self.gateway.subscriptions = Mock() + self.gateway.subscriptions.start.return_value = {"status": "started"} + self.gateway.subscriptions.poll.return_value = [] + + def test_list_browsers_excludes_released_journal_rows(self) -> None: + self.runtimes.list_public.return_value = [ + {"alias": "same", "state": "released"}, + {"alias": "same", "state": "stopped"}, ] - result = Gateway( - cast(DockerClient, PartialDocker(containers)), - "creatorhub_browser", - "0123456789abcdef", - "gateway", - ).list_browsers() - self.assertEqual( - [item["alias"] for item in result], ["broken-account", "healthy-account"] - ) - self.assertEqual(result[0]["endpoint"], "") - self.assertIn("error", result[0]) - self.assertEqual(result[1]["endpoint"], "http://192.0.2.10:9222") + self.assertEqual(self.gateway.list_browsers(), [{"alias": "same", "state": "stopped"}]) - def test_container_network_address(self) -> None: - docker = DockerClient("/var/run/docker.sock") - cast(Any, docker).request = lambda method, path: DockerResponse( + def test_lifecycle_info_and_douyin_operations(self) -> None: + self.runtimes.create.return_value = {"state": "running"} + self.assertEqual(self.gateway.info()["node_id"], "node-a") + self.assertEqual(self.gateway.list_browsers(), []) + self.assertEqual(self.gateway.create({"alias": "safe"}), {"state": "running"}) + runtime_generation = {key: self.generation[key] for key in ("binding_version", "runtime_id", "network_id")} + self.gateway.change_state("safe", "stop", runtime_generation) + self.gateway.remove("safe", {**runtime_generation, "purge_profile": True}) + self.gateway.restore_proxy( + "safe", + {**self.generation, "network_exit": {"protocol": "http", "host": "proxy", "port": 8080}}, + ) + self.assertEqual( + self.gateway.get_douyin( + "safe", + {**self.generation, "url": "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp"}, + )["status"], 200, - "OK", - json.dumps( - { - "NetworkSettings": { - "Networks": { - "tenant": { - "NetworkID": "network", - "IPAddress": "192.0.2.20", - } - } - } - } - ).encode(), ) self.assertEqual( - docker.container_network_address("container", "network"), "192.0.2.20" + self.gateway.resolve_douyin( + "safe", {**self.generation, "url": "https://v.douyin.com/abc123/"} + )["url"], + "https://www.douyin.com/video/123", ) - cast(Any, docker).request = lambda method, path: DockerResponse( - 200, "OK", b'{"NetworkSettings":{"Networks":{}}}' + self.assertEqual( + self.gateway.get_douyin_media( + "safe", {**self.generation, "url": "https://cdn.example/video.mp4"} + )["content_type"], + "video/mp4", + ) + self.assertEqual( + self.gateway.douyin_identity( + "safe", {**self.generation, "expected_account_key": "12345678901234567890"} + )["uid"], + "12345678901234567890", + ) + self.assertTrue( + self.gateway.douyin_login_qr("safe", self.generation)["qr_detected"] + ) + expected = "12345678901234567890" + target = "22345678901234567890" + self.assertEqual( + self.gateway.douyin_message_history( + "safe", {**self.generation, "expected_uid": expected, "target_uid": target} + )["status"], + "succeeded", + ) + self.assertEqual( + self.gateway.douyin_action( + "safe", + { + **self.generation, + "expected_uid": expected, + "target_uid": target, + "action": "follow", + "operation_id": "operation-1", + "confirm": True, + }, + )["status"], + "succeeded", ) - with self.assertRaises(GenerationConflict): - docker.container_network_address("container", "network") - def test_list_browsers_and_json(self) -> None: - docker = FakeDocker( - [ - { - "Id": "container-id", - "Labels": { - "io.creatorhub.managed": "true", - "io.creatorhub.runtime-id": "safe-account", - "io.creatorhub.binding-version": "2", - "io.creatorhub.network-exit-id": "", - "io.creatorhub.network-id": "b" * 64, - }, - "State": "running", - "Status": "Up 1 second", - } - ] - ) - gateway = Gateway( - cast(DockerClient, docker), - "creatorhub_browser", - "0123456789abcdef", - "gateway", - ) - self.assertEqual(gateway.list_browsers()[0]["id"], "container-id") + def test_xhs_and_event_operations(self) -> None: + page = "https://www.xiaohongshu.com/explore/abc" + generation = {**self.generation, "network_exit_id": "exit-1"} self.assertEqual( - gateway.list_browsers()[0]["endpoint"], "http://192.0.2.10:9222" + self.gateway.get_xiaohongshu( + "safe", {**generation, "url": "https://edith.xiaohongshu.com/api/sns/web/v2/user/me"} + )["status"], + 200, ) self.assertEqual( - json_bytes({"text": "中文"}), b'{"text":"\xe4\xb8\xad\xe6\x96\x87"}' + self.gateway.post_xiaohongshu( + "safe", + {**generation, "url": "https://edith.xiaohongshu.com/api/sns/web/v1/feed", "body": {"ok": True}}, + )["status"], + 200, ) + self.assertEqual( + self.gateway.resolve_xiaohongshu("safe", {**generation, "url": page})["url"], + page, + ) + self.assertEqual( + self.gateway.get_xiaohongshu_media("safe", {**generation, "url": page})["status"], + 200, + ) + self.assertEqual( + self.gateway.xiaohongshu_identity( + "safe", {**generation, "expected_account_key": "xhs-user"} + )["uid"], + "xhs-user", + ) + self.assertEqual( + self.gateway.start_douyin_events( + "safe", {**self.generation, "expected_uid": "12345678901234567890"} + ), + {"status": "started"}, + ) + self.assertEqual( + self.gateway.poll_douyin_events("safe", self.generation, {"limit": ["1"], "wait": ["0"]}), + [], + ) + self.gateway.stop_douyin_events("safe", self.generation) + + def test_http_info_route_and_action_error_ownership(self) -> None: + handler = gateway_module.GatewayHandler.__new__(gateway_module.GatewayHandler) + server = Mock() + server.gateway = self.gateway + cast(Any, handler).server = server + cast(Any, handler).server_as_gateway = lambda: server + self.assertEqual(handler._route("GET", "/v1/info", {}, {}), self.gateway.info()) + cast(Any, handler).path = "/v1/info" + cast(Any, handler).headers = {"Authorization": "Bearer gateway-token-123456"} + cast(Any, handler)._respond = Mock() + handler._dispatch("GET") + self.assertEqual(handler._respond.call_args.args[0], 200) + self.gateway._handle_douyin_action_error("safe", "follow", DouyinError("timed out"), "operation-1") + self.assertEqual(self.gateway._uncertain_actions["safe"], float("inf")) + self.gateway._release_action_ownership("safe", "operation-1") + self.assertNotIn("safe", self.gateway._uncertain_actions) + with self.assertRaises(RequestError): + self.gateway.post_xiaohongshu( + "safe", {**self.generation, "url": "https://edith.xiaohongshu.com/api/sns/web/v1/feed", "body": []} + ) class CDPTests(unittest.TestCase): @@ -1076,10 +1106,10 @@ class ProxyTests(unittest.TestCase): "safe", 1, "127.0.0.1", 0, ProxyExit("http", "127.0.0.1", 8080), "network-1" ) port = int(url.rsplit(":", 1)[1]) - self.assertFalse(registry.ready("safe", port, "container-1", "network-1")) - self.assertTrue(registry.bind("safe", 1, url, "container-1", "network-1")) - self.assertTrue(registry.ready("safe", port, "container-1", "network-1")) - self.assertFalse(registry.remove("safe", 1, "container-2", "network-1")) + self.assertFalse(registry.ready("safe", port, "runtime-1", "network-1")) + self.assertTrue(registry.bind("safe", 1, url, "runtime-1", "network-1")) + self.assertTrue(registry.ready("safe", port, "runtime-1", "network-1")) + self.assertFalse(registry.remove("safe", 1, "runtime-2", "network-1")) undo() registry.close() @@ -1213,337 +1243,14 @@ class ChunkSocket: return None -class DockerClientTests(unittest.TestCase): - def test_digest_pull_preserves_digest_in_docker_query(self) -> None: - client = ScriptedDocker( - [ - DockerResponse(404, "Not Found", b""), - DockerResponse(200, "OK", b""), - ] - ) - client.pull_if_missing("registry.example/repo@sha256:" + "a" * 64) - self.assertIn( - "fromImage=registry.example%2Frepo%40sha256%3A" + "a" * 64, - client.calls[1][1], - ) - - def test_image_ref_and_expected_statuses(self) -> None: - self.assertEqual( - split_image_ref("registry.example/repo:tag"), - ("registry.example/repo", "tag"), - ) - self.assertEqual(split_image_ref("repo@sha256:abc"), ("repo", "sha256:abc")) - client = ScriptedDocker( - [ - DockerResponse(204, "No Content", b""), - DockerResponse(404, "Not Found", b""), - DockerResponse(500, "Error", b"failure"), - ] - ) - client.expect("POST", "/ok") - with self.assertRaises(FileNotFoundError): - client.expect("POST", "/missing") - with self.assertRaises(DockerError): - client.expect("POST", "/error") - self.assertEqual(tenant_network_name("creatorhub", "safe"), "creatorhub-safe") - - def test_pull_and_managed_container_response_validation(self) -> None: - client = ScriptedDocker( - [ - DockerResponse(404, "Not Found", b""), - DockerResponse(200, "OK", b"{}"), - ] - ) - client.pull_if_missing("repo:tag") - self.assertIn("/images/repo%3Atag/json", client.calls[0][1]) - managed = { - "Id": "container-id", - "Config": {"Labels": {MANAGED_LABEL: "true", RUNTIME_ID_LABEL: "safe"}}, - "NetworkSettings": {"Networks": {"tenant": {"NetworkID": "network-id"}}}, - } - client = ScriptedDocker( - [DockerResponse(200, "OK", json.dumps(managed).encode())] - ) - self.assertEqual(client.managed_container_state("safe")[0], "container-id") - unmanaged = { - **managed, - "Config": {"Labels": {MANAGED_LABEL: "false", RUNTIME_ID_LABEL: "safe"}}, - } - client = ScriptedDocker( - [DockerResponse(200, "OK", json.dumps(unmanaged).encode())] - ) - with self.assertRaises(UnmanagedContainer): - client.managed_container("safe") - - def test_existing_network_generation_and_disconnect(self) -> None: - network = { - "Id": "network-id", - "Name": "creatorhub-safe", - "Driver": "bridge", - "Internal": False, - "Attachable": False, - "Ingress": False, - "Labels": { - MANAGED_LABEL: "true", - NETWORK_ROLE_LABEL: BROWSER_NETWORK_ROLE, - RUNTIME_ID_LABEL: "safe", - BINDING_VERSION_LABEL: "1", - }, - "Containers": { - "gateway-id": {"Name": "gateway", "IPv4Address": "10.0.0.2/24"}, - "runtime-id": {"Name": "runtime", "IPv4Address": "10.0.0.3/24"}, - }, - } - gateway = { - "Id": "gateway-id", - "Config": {"Labels": {GATEWAY_MEMBER_LABEL: "true"}}, - } - client = ScriptedDocker( - [ - DockerResponse(200, "OK", json.dumps(network).encode()), - DockerResponse(200, "OK", json.dumps(gateway).encode()), - ] - ) - generation, addresses, exists = client.inspect_tenant_network( - "creatorhub", "safe", 1, "runtime-id", "gateway", "network-id" - ) - self.assertTrue(exists) - self.assertTrue(generation.runtime_attached) - self.assertEqual(addresses["gateway-id"], "10.0.0.2/24") - self.assertEqual(generation.self_member, "gateway-id") - - after = { - **network, - "Containers": {"gateway-id": network["Containers"]["gateway-id"]}, - } - client = ScriptedDocker( - [ - DockerResponse(200, "OK", json.dumps(network).encode()), - DockerResponse(200, "OK", json.dumps(gateway).encode()), - DockerResponse(200, "OK", json.dumps(network).encode()), - DockerResponse(200, "OK", json.dumps(gateway).encode()), - DockerResponse(200, "OK", b""), - DockerResponse(200, "OK", json.dumps(after).encode()), - DockerResponse(200, "OK", json.dumps(gateway).encode()), - ] - ) - current, _, _ = client.inspect_tenant_network( - "creatorhub", "safe", 1, "runtime-id", "gateway", "network-id" - ) - updated = client.disconnect_member( - "creatorhub", "safe", 1, "runtime-id", current, "runtime-id", "gateway" - ) - self.assertFalse(updated.runtime_attached) - - def test_new_network_is_generation_fenced(self) -> None: - empty = { - "Id": "network-id", - "Name": "creatorhub-safe", - "Driver": "bridge", - "Internal": False, - "Attachable": False, - "Ingress": False, - "Labels": { - MANAGED_LABEL: "true", - NETWORK_ROLE_LABEL: BROWSER_NETWORK_ROLE, - RUNTIME_ID_LABEL: "safe", - BINDING_VERSION_LABEL: "1", - }, - "Containers": {}, - } - connected = { - **empty, - "Containers": { - "runtime-id": {"Name": "runtime", "IPv4Address": "10.0.0.3/24"}, - "gateway-id": {"Name": "gateway", "IPv4Address": "10.0.0.2/24"}, - }, - } - gateway = { - "Id": "gateway-id", - "Config": {"Labels": {GATEWAY_MEMBER_LABEL: "true"}}, - } - client = ScriptedDocker( - [ - DockerResponse(404, "Not Found", b""), - DockerResponse(201, "Created", b'{"Id":"network-id"}'), - DockerResponse(200, "OK", json.dumps(empty).encode()), - DockerResponse(200, "OK", b""), - DockerResponse(200, "OK", b""), - DockerResponse(200, "OK", json.dumps(connected).encode()), - DockerResponse(200, "OK", json.dumps(gateway).encode()), - ] - ) - generation, bind_host = client.ensure_tenant_network( - "creatorhub", "safe", "gateway", 1, "runtime-id" - ) - self.assertTrue(generation.created) - self.assertEqual(bind_host, "10.0.0.2") - - def test_alias_reservation_releases_only_its_generation(self) -> None: - gateway = { - "Id": "gateway-id", - "Image": "creatorhub/gateway:latest", - "Config": {"Labels": {MANAGED_LABEL: "true", GATEWAY_MEMBER_LABEL: "true"}}, - } - reservation = { - "Id": "reservation-id", - "Config": { - "Labels": { - "io.creatorhub.alias-reservation": "true", - "io.creatorhub.reservation-generation": "reservation-generation", - RUNTIME_ID_LABEL: "safe", - } - }, - } - client = ScriptedDocker( - [ - DockerResponse(200, "OK", json.dumps(gateway).encode()), - DockerResponse(201, "Created", b'{"Id":"reservation-id"}'), - DockerResponse(200, "OK", json.dumps(reservation).encode()), - DockerResponse(200, "OK", json.dumps(reservation).encode()), - DockerResponse(204, "No Content", b""), - DockerResponse(404, "Not Found", b""), - ] - ) - with patch.object( - docker_client_module, - "random_reservation_generation", - return_value="reservation-generation", - ): - release = AliasReservationManager(client, "gateway").acquire("safe") - release() - self.assertTrue(any(call[0] == "DELETE" for call in client.calls)) - - def test_stale_alias_reservation_is_reclaimed(self) -> None: - gateway = { - "Id": "gateway-id", - "Image": "creatorhub/gateway:latest", - "Config": {"Labels": {MANAGED_LABEL: "true", GATEWAY_MEMBER_LABEL: "true"}}, - } - stale = { - "Id": "stale-reservation-id", - "Config": { - "Labels": { - RESERVATION_LABEL: "true", - RUNTIME_ID_LABEL: "safe", - RESERVATION_OWNER_LABEL: "old-gateway", - } - }, - } - current = { - "Id": "current-reservation-id", - "Config": { - "Labels": { - RESERVATION_LABEL: "true", - RUNTIME_ID_LABEL: "safe", - RESERVATION_OWNER_LABEL: "gateway", - "io.creatorhub.reservation-generation": "current-generation", - } - }, - } - client = ScriptedDocker( - [ - DockerResponse(200, "OK", json.dumps(gateway).encode()), - DockerResponse(409, "Conflict", b""), - DockerResponse(200, "OK", json.dumps(stale).encode()), - DockerResponse(404, "Not Found", b""), - DockerResponse(204, "No Content", b""), - DockerResponse(404, "Not Found", b""), - DockerResponse(201, "Created", b'{"Id":"current-reservation-id"}'), - DockerResponse(200, "OK", json.dumps(current).encode()), - DockerResponse(200, "OK", json.dumps(current).encode()), - DockerResponse(204, "No Content", b""), - DockerResponse(404, "Not Found", b""), - ] - ) - with patch.object( - docker_client_module, - "random_reservation_generation", - return_value="current-generation", - ): - release = AliasReservationManager(client, "gateway").acquire("safe") - release() - self.assertEqual( - client.calls[4][1], - "/containers/stale-reservation-id?force=1&v=0", - ) - self.assertEqual(client.calls[6][0], "POST") -class ScriptedDocker(DockerClient): - def __init__(self, responses: list[DockerResponse]) -> None: - super().__init__("/dev/null") - self.responses = list(responses) - self.calls: list[tuple[str, str, object | None]] = [] - - def request( - self, - method: str, - path: str, - payload: object | None = None, - timeout: float = 30.0, - body_limit: int = 16 * 1024 * 1024, - ) -> DockerResponse: - del timeout, body_limit - self.calls.append((method, path, payload)) - if not self.responses: - raise AssertionError(f"unexpected Docker call: {method} {path}") - return self.responses.pop(0) -class ReservationStub: - def __init__(self) -> None: - self.released = 0 - - def acquire(self, alias: str): - del alias - - def release() -> None: - self.released += 1 - - return release -class LockingReservationStub(ReservationStub): - def __init__(self) -> None: - super().__init__() - self.lock = threading.Lock() - - def acquire(self, alias: str): - del alias - self.lock.acquire() - - def release() -> None: - self.released += 1 - self.lock.release() - - return release -class LifecycleDocker(ScriptedDocker): - def ensure_tenant_network( - self, *args: object, **kwargs: object - ) -> tuple[TenantNetworkGeneration, str]: - del args, kwargs - return TenantNetworkGeneration( - id="network-id", self_member="gateway-id" - ), "10.0.0.2" - - def inspect_tenant_network( - self, *args: object, **kwargs: object - ) -> tuple[TenantNetworkGeneration, dict[str, str], bool]: - del args, kwargs - return ( - TenantNetworkGeneration( - id="network-id", - name="creatorhub-safe", - self_member="gateway-id", - gateway_members=["gateway-id"], - runtime_attached=True, - ), - {"gateway-id": "10.0.0.2/24"}, - True, - ) class GatewayLifecycleTests(unittest.TestCase): @@ -1555,288 +1262,28 @@ class GatewayLifecycleTests(unittest.TestCase): finally: server.server_close() - def test_network_setup_error_keeps_created_generation(self) -> None: - client = ScriptedDocker( - [ - DockerResponse(404, "Not Found", b""), - DockerResponse(201, "Created", b'{"Id":"network-id"}'), - DockerResponse(500, "Error", b"inspect failed"), - ] - ) - with self.assertRaises(docker_client_module.NetworkSetupError) as caught: - client.ensure_tenant_network("creatorhub", "safe", "gateway", 1, "runtime") - self.assertEqual(caught.exception.generation.id, "network-id") - self.assertTrue(caught.exception.generation.created) - def test_network_cleanup_failure_returns_pending_contract(self) -> None: - docker = ScriptedDocker( - [ - DockerResponse(404, "Not Found", b""), - DockerResponse(500, "Error", b"temporary"), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - with self.assertRaises(RequestError) as caught: - gateway.remove( - "safe", - { - "binding_version": 1, - "runtime_id": "runtime-not-found", - "network_id": "network-id", - }, - ) - self.assertEqual(caught.exception.status, 202) - self.assertEqual(caught.exception.network_id, "network-id") - self.assertEqual(str(caught.exception), "runtime_cleanup_pending") - def test_timed_out_action_retains_alias_ownership(self) -> None: - gateway = Gateway( - cast(DockerClient, Mock()), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway._claim_action("safe") - gateway._retain_action_ownership("safe") - with self.assertRaises(RequestError): - gateway._claim_action("safe") def _input(self, stopped: bool = True) -> dict: return { "alias": "safe", "name": "Safe", - "image": "creatorhub/browser:latest", + "browser_version": "1.2.3", + "profile_id": "safe", "cmd": ["about:blank"], - "volume": "creatorhub-safe", "binding_version": 1, "network_exit_id": "", + "network_exit": {}, "stopped": stopped, } - def test_concurrent_create_cannot_delete_the_winner(self) -> None: - class ConcurrentDocker(DockerClient): - def __init__(self) -> None: - super().__init__("/dev/null") - self.created = False - self.calls: list[tuple[str, str]] = [] - def managed_container(self, alias: str) -> tuple[str, dict[str, str]]: - if not self.created: - raise FileNotFoundError(alias) - return "a" * 64, { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: alias, - BINDING_VERSION_LABEL: "1", - NETWORK_ID_LABEL: "", - } - def request( - self, - method: str, - path: str, - payload: object | None = None, - timeout: float = 30.0, - body_limit: int = 16 * 1024 * 1024, - ) -> DockerResponse: - del payload, timeout, body_limit - self.calls.append((method, path)) - if method == "GET" and path.startswith("/images/"): - return DockerResponse(200, "OK", b"{}") - if method == "POST" and path.startswith("/containers/create"): - if self.created: - return DockerResponse(409, "Conflict", b"") - self.created = True - return DockerResponse( - 201, "Created", b'{"Id":"' + b"b" * 64 + b'"}' - ) - raise AssertionError(f"unexpected Docker call: {method} {path}") - docker = ConcurrentDocker() - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, LockingReservationStub()) - results: list[object] = [] - def create() -> None: - try: - results.append(gateway.create(self._input())) - except ( - AssertionError, - DockerError, - OSError, - RequestError, - ValueError, - ) as exc: - results.append(exc) - threads = [threading.Thread(target=create) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - self.assertEqual(sum(isinstance(item, dict) for item in results), 1) - self.assertEqual(sum(isinstance(item, RequestError) for item in results), 1) - self.assertFalse(any(method == "DELETE" for method, _ in docker.calls)) - def test_create_stopped_pulls_image_and_keeps_container(self) -> None: - docker = ScriptedDocker( - [ - DockerResponse(200, "OK", b"{}"), - DockerResponse(404, "Not Found", b""), - DockerResponse(201, "Created", b'{"Id":"container-id"}'), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - reservations = ReservationStub() - gateway.reservations = cast(AliasReservationManager, reservations) - result = gateway.create(self._input()) - self.assertEqual(result["id"], "container-id") - self.assertEqual(reservations.released, 1) - self.assertEqual(docker.calls[2][0], "POST") - - def test_running_create_starts_container_and_network(self) -> None: - docker = LifecycleDocker( - [ - DockerResponse(200, "OK", b"{}"), - DockerResponse(404, "Not Found", b""), - DockerResponse(201, "Created", b'{"Id":"container-id"}'), - DockerResponse(204, "No Content", b""), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - result = gateway.create(self._input(False)) - self.assertEqual(result["network_id"], "network-id") - self.assertEqual(docker.calls[-1][0], "POST") - - def test_change_state_and_remove_are_generation_fenced(self) -> None: - labels = { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: "safe", - BINDING_VERSION_LABEL: "1", - NETWORK_ID_LABEL: "network-id", - } - inspected = { - "Id": "a" * 64, - "Config": {"Labels": labels}, - "NetworkSettings": {"Networks": {}}, - } - docker = ScriptedDocker( - [ - DockerResponse(200, "OK", json.dumps(inspected).encode()), - DockerResponse(204, "No Content", b""), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - gateway.change_state( - "safe", - "stop", - { - "binding_version": 1, - "runtime_id": "a" * 64, - "network_id": "network-id", - }, - ) - self.assertEqual(docker.calls[-1][0], "POST") - - docker = LifecycleDocker( - [ - DockerResponse(200, "OK", json.dumps(inspected).encode()), - DockerResponse(204, "No Content", b""), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - gateway._remove_network = lambda *args, **kwargs: None - gateway.remove( - "safe", - { - "binding_version": 1, - "runtime_id": "a" * 64, - "network_id": "network-id", - }, - ) - self.assertEqual(docker.calls[-1][0], "DELETE") - - def test_remove_stopped_direct_container_without_network(self) -> None: - labels = { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: "safe", - BINDING_VERSION_LABEL: "1", - NETWORK_ID_LABEL: "", - } - inspected = { - "Id": "a" * 64, - "Config": {"Labels": labels}, - "NetworkSettings": {"Networks": {}}, - } - docker = ScriptedDocker( - [ - DockerResponse(200, "OK", json.dumps(inspected).encode()), - DockerResponse(404, "Not Found", b""), - DockerResponse(204, "No Content", b""), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - gateway.remove( - "safe", - {"binding_version": 1, "runtime_id": "a" * 64, "network_id": ""}, - ) - self.assertEqual(docker.calls[-1][0], "DELETE") - - def test_remove_rejects_runtime_replacement(self) -> None: - labels = { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: "safe", - BINDING_VERSION_LABEL: "1", - NETWORK_ID_LABEL: "", - } - inspected = { - "Id": "a" * 64, - "Config": {"Labels": labels}, - "NetworkSettings": {"Networks": {}}, - } - docker = ScriptedDocker( - [DockerResponse(200, "OK", json.dumps(inspected).encode())] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - with self.assertRaises(RequestError): - gateway.remove( - "safe", - {"binding_version": 1, "runtime_id": "b" * 64, "network_id": ""}, - ) - - def test_create_failure_reconciles_unknown_container(self) -> None: - docker = ScriptedDocker( - [ - DockerResponse(200, "OK", b"{}"), - DockerResponse(404, "Not Found", b""), - DockerResponse(500, "Error", b"failed"), - DockerResponse(404, "Not Found", b""), - ] - ) - gateway = Gateway( - cast(DockerClient, docker), "creatorhub", "0123456789abcdef", "gateway" - ) - gateway.reservations = cast(AliasReservationManager, ReservationStub()) - with self.assertRaises(RequestError): - gateway.create(self._input()) class AdditionalGatewayCoverageTests(unittest.TestCase): @@ -1982,148 +1429,8 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): ) self.assertFalse(valid_account_key_query({"key": ["bad key"]}, "key")) - def test_proxy_rejected_handshakes_and_docker_errors(self) -> None: - bad_http = ChunkSocket([b"HTTP/1.1 407 Proxy Authentication Required\r\n\r\n"]) - with ( - patch.object(proxy_module, "_open_host", return_value=bad_http), - self.assertRaises(OSError), - ): - _dial_http_proxy(ProxyExit("http", "proxy", 8080), "target:443", 1.0) - bad_socks5 = ChunkSocket([b"\x05\x02"]) - with ( - patch.object(proxy_module, "_open_host", return_value=bad_socks5), - self.assertRaises(OSError), - ): - _dial_socks5( - ProxyExit("socks5", "proxy", 1080, "u", "p"), "127.0.0.1:80", 1.0 - ) - client = DockerClient("/not/a/socket") - with ( - patch.object( - docker_client_module, "UnixHTTPConnection", side_effect=OSError("down") - ), - self.assertRaises(DockerError), - ): - client.request("GET", "/version", timeout=0.01) - with ( - patch.object( - client, "request", return_value=DockerResponse(500, "bad", b"x") - ), - self.assertRaises(DockerError), - ): - client.expect("POST", "/x") - with ( - patch.object( - client, "request", return_value=DockerResponse(404, "missing", b"") - ), - self.assertRaises(FileNotFoundError), - ): - client.expect("DELETE", "/gone") - def test_docker_client_transport_and_metadata_edges(self) -> None: - client = DockerClient("/unused") - response = FakeHTTPResponse(200, b"{}") - cast(Any, response).reason = "OK" - connection = FakeHTTPConnection(response) - with patch.object( - docker_client_module, "UnixHTTPConnection", return_value=connection - ): - result = client.request("POST", "/version", {"ok": True}) - self.assertEqual(result.status, 200) - self.assertEqual(connection.requested, [("POST", "/v1.43/version")]) - huge = FakeHTTPResponse(200, b"12345") - cast(Any, huge).reason = "OK" - with ( - patch.object( - docker_client_module, - "UnixHTTPConnection", - return_value=FakeHTTPConnection(huge), - ), - self.assertRaises(DockerError), - ): - client.request("GET", "/version", body_limit=4) - labels = { - MANAGED_LABEL: "true", - RUNTIME_ID_LABEL: "safe", - GATEWAY_MEMBER_LABEL: "true", - } - inspected = { - "Id": "a" * 64, - "Config": {"Labels": labels}, - "NetworkSettings": { - "Networks": {"n": {"NetworkID": "network", "IPAddress": "198.51.100.5"}} - }, - } - with patch.object( - client, - "request", - return_value=DockerResponse(200, "OK", json.dumps(inspected).encode()), - ): - self.assertTrue(client.trusted_gateway_member("gateway")) - self.assertEqual( - client.container_network_address("a" * 64, "network"), "198.51.100.5" - ) - bad = DockerResponse(200, "OK", b"[]") - with patch.object(client, "request", return_value=bad): - self.assertFalse(client.trusted_gateway_member("gateway")) - with self.assertRaises(DockerError): - client.managed_container_state("safe") - def test_client_connection_pull_and_inspect_edges(self) -> None: - class ConnectedSocket: - def __init__(self) -> None: - self.timeout = None - self.path = "" - - def settimeout(self, value: float) -> None: - self.timeout = value - - def connect(self, path: str) -> None: - self.path = path - - connected = ConnectedSocket() - with patch.object( - docker_client_module.socket, "socket", return_value=connected - ): - connection = docker_client_module.UnixHTTPConnection( - "/run/docker.sock", 1.0 - ) - connection.connect() - self.assertEqual(connected.path, "/run/docker.sock") - client = DockerClient("/unused") - with patch.object( - client, - "request", - side_effect=[ - DockerResponse(404, "missing", b""), - DockerResponse(200, "OK", b"{}"), - ], - ) as request: - client.pull_if_missing("registry.example/repo:tag") - self.assertIn("tag=tag", request.call_args_list[1].args[1]) - with ( - patch.object( - client, "request", return_value=DockerResponse(500, "bad", b"no") - ), - self.assertRaises(DockerError), - ): - client.pull_if_missing("registry.example/repo:tag") - invalid = DockerResponse( - 200, - "OK", - json.dumps( - { - "Id": "a" * 64, - "Config": {"Labels": []}, - "NetworkSettings": {"Networks": {}}, - } - ).encode(), - ) - with ( - patch.object(client, "request", return_value=invalid), - self.assertRaises(DockerError), - ): - client.managed_container_state("safe") def test_cdp_error_and_proxy_auth_paths(self) -> None: socket_ = FakeSocket([{"id": 1, "result": {"result": {"value": {"ok": True}}}}]) @@ -2234,40 +1541,6 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): sent = [json.loads(item) for item in socket_.sent] self.assertEqual(sent[1]["method"], "Runtime.terminateExecution") - def test_network_create_reconciliation_preserves_observed_generation(self) -> None: - client = DockerClient("/unused") - observed = TenantNetworkGeneration(name="creatorhub-safe", id="observed") - generation = TenantNetworkGeneration(name="creatorhub-safe") - with ( - patch.object( - client, "inspect_tenant_network", return_value=(observed, [], True) - ), - self.assertRaises(NetworkSetupError) as caught, - ): - client._finish_network_create( - "creatorhub", - "safe", - "gateway", - 1, - "runtime", - DockerResponse(201, "Created", b"{}"), - generation, - ) - self.assertEqual(caught.exception.generation.id, "observed") - - with patch.object( - client, "inspect_tenant_network", return_value=(observed, [], True) - ): - result = client._finish_network_create( - "creatorhub", - "safe", - "gateway", - 1, - "runtime", - DockerResponse(201, "Created", b'{"Id":"created"}'), - TenantNetworkGeneration(name="creatorhub-safe"), - ) - self.assertEqual(result.id, "observed") def test_configured_target_and_media_wait_edge_cases(self) -> None: targets = [ @@ -2405,101 +1678,7 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): ["reconnected", "baseline"], ) - def test_external_message_history_and_generation_fences(self) -> None: - external = { - "url": "http://127.0.0.1:9222", - "target_id": "target_1", - "alias": "local-cdp", - "runtime_id": "0" * 64, - "network_id": "local-cdp", - "binding_version": 1, - } - gateway = Gateway( - cast(DockerClient, FakeDocker([])), - "creatorhub_browser", - "0123456789abcdef", - "gateway", - external_cdp=external, - ) - self.assertEqual(gateway._browser_endpoint("local-cdp"), external["url"]) - with self.assertRaises(GenerationConflict): - gateway._browser_endpoint("other") - with gateway._alias_lock("local-cdp"): - pass - browser = Mock() - browser.message_history.return_value = { - "status": "succeeded", - "history_source": "im_sdk_pull", - "messages": [], - } - gateway.browser = browser - generation = { - "binding_version": 1, - "runtime_id": "0" * 64, - "network_id": "local-cdp", - "expected_uid": "123", - "target_uid": "456", - "limit": 20, - } - self.assertEqual( - gateway.douyin_message_history("local-cdp", generation)["status"], - "succeeded", - ) - browser.message_history.assert_called_once_with("local-cdp", "123", "456", 20) - with self.assertRaises(RequestError): - gateway.douyin_message_history( - "local-cdp", {**generation, "target_uid": "123"} - ) - browser.message_history.side_effect = DouyinError("offline") - with self.assertRaises(RequestError): - gateway.douyin_message_history("local-cdp", generation) - - def test_douyin_api_urls_and_external_config_edges(self) -> None: - self.assertTrue( - valid_douyin_url( - "https://www.douyin.com" - f"{gateway_module.DOUYIN_WORKS_PATH}" - "?aid=6383&device_platform=webapp&sec_user_id=sec&count=20&max_cursor=0" - ) - ) - self.assertTrue( - valid_douyin_url( - "https://www.douyin.com" - f"{gateway_module.DOUYIN_COMMENTS_PATH}" - "?aid=6383&device_platform=webapp&aweme_id=123&count=20&cursor=0" - ) - ) - self.assertFalse( - valid_douyin_url( - "https://www.douyin.com" - f"{gateway_module.DOUYIN_WORKS_PATH}" - "?aid=6383&device_platform=webapp&sec_user_id=sec&count=10&max_cursor=0" - ) - ) - self.assertTrue(gateway_module.valid_cdp_url("http://127.0.0.1:9222/")) - for raw in ( - "https://127.0.0.1:9222", - "http://127.0.0.1:9222/devtools", - "http://127.0.0.1:9222?x=1", - "http://user:pass@127.0.0.1:9222", - "http://127.0.0.1:not-a-port", - ): - with self.subTest(raw=raw): - self.assertFalse(gateway_module.valid_cdp_url(raw)) - base = { - "LISTEN_ADDR": ":8081", - "DOCKER_SOCKET": "/var/run/docker.sock", - "BROWSER_NETWORK": "creatorhub_browser", - "GATEWAY_TOKEN": "0123456789abcdef", - "BROWSER_CDP_URL": "http://127.0.0.1:9222", - } - with self.assertRaises(ValueError): - load_config({**base, "BROWSER_CDP_BINDING_VERSION": "not-an-int"}) - with self.assertRaises(ValueError): - load_config({**base, "BROWSER_CDP_BINDING_VERSION": "0"}) - with self.assertRaises(ValueError): - load_config({**base, "BROWSER_CDP_ALIAS": "bad alias"}) class DouyinReleaseRemediationTests(unittest.TestCase): diff --git a/cmd/browser_gateway/test_runtime.py b/cmd/browser_gateway/test_runtime.py new file mode 100644 index 0000000..f86f4af --- /dev/null +++ b/cmd/browser_gateway/test_runtime.py @@ -0,0 +1,436 @@ +from __future__ import annotations + +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +from .runtime import ( + BrowserRuntimeError, + GenerationConflict, + NativeRuntimeManager, + RuntimeCleanupPending, + UnitStatus, +) + + +class FakeUnits: + def __init__(self) -> None: + self.units: dict[str, UnitStatus] = {} + self.commands: list[tuple[str, list[str]]] = [] + self.next_pid = 1000 + self.fail_stop: set[str] = set() + + def start(self, unit: str, command: list[str], **kwargs: Any) -> UnitStatus: + del kwargs + self.commands.append((unit, list(command))) + self.next_pid += 1 + status = UnitStatus(True, "active", self.next_pid, str(self.next_pid), "") + self.units[unit] = status + return status + + def status(self, unit: str) -> UnitStatus: + return self.units.get(unit, UnitStatus(False, "inactive", 0, "", "")) + + def stop(self, unit: str, timeout: float) -> None: + del timeout + if unit in self.fail_stop: + raise OSError("stop failed") + self.units[unit] = UnitStatus(False, "inactive", 0, "", "") + + def reset(self, unit: str) -> None: + del unit + + +class SystemdUnitManagerTests(unittest.TestCase): + def test_start_status_stop_and_reset(self) -> None: + calls: list[list[str]] = [] + stop_calls = 0 + + def runner(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + nonlocal stop_calls + del kwargs + calls.append(args) + if args[0] == "systemctl" and args[2] == "show": + return subprocess.CompletedProcess( + args, + 0, + "ActiveState=active\nSubState=running\nMainPID=42\n" + "ExecMainStartTimestampMonotonic=1\nExecMainStatus=0\n", + "", + ) + if args[0] == "systemctl" and args[2] == "stop": + stop_calls += 1 + return subprocess.CompletedProcess(args, 1 if stop_calls == 1 else 0, "", "") + return subprocess.CompletedProcess(args, 0, "", "") + + from .runtime import SystemdUnitManager + + with tempfile.TemporaryDirectory() as directory: + manager = SystemdUnitManager("systemd-run", "systemctl", runner) + status = manager.start( + "creatorhub-test.service", + ["/bin/true"], + environment={"B": "2", "A": "1"}, + working_directory=Path(directory), + stdout_path=Path(directory) / "runtime.log", + limits={"MemoryMax": "1G"}, + ) + self.assertEqual(status.pid, 42) + self.assertTrue(any("--setenv" in item for item in calls[0])) + self.assertEqual(manager.status("creatorhub-test.service").state, "running") + manager.stop("creatorhub-test.service", 1) + manager.reset("creatorhub-test.service") + self.assertTrue(any(args[2] == "kill" for args in calls)) + self.assertTrue(any(args[2] == "reset-failed" for args in calls)) + + def test_status_and_start_failures_are_explicit(self) -> None: + from .runtime import BrowserRuntimeError, SystemdUnitManager + + with self.assertRaises(BrowserRuntimeError): + SystemdUnitManager(runner=lambda *args, **kwargs: (_ for _ in ()).throw(OSError("down"))).status("x") + + def failed_start(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + del kwargs + return subprocess.CompletedProcess(args, 1, "", "rejected") + + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(BrowserRuntimeError): + SystemdUnitManager("systemd-run", "systemctl", failed_start).start( + "x", ["/bin/true"], environment={}, working_directory=Path(directory), + stdout_path=Path(directory) / "x.log", limits={}, + ) + + def missing_status(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + del kwargs + if args[2] == "show": + return subprocess.CompletedProcess(args, 1, "", "missing") + return subprocess.CompletedProcess(args, 0, "", "") + + manager = SystemdUnitManager("systemd-run", "systemctl", missing_status) + self.assertFalse(manager.status("missing").active) + self.assertEqual(manager.status("missing").state, "not-found") + manager.reset("missing") + + def test_stop_and_reset_failures_are_not_hidden(self) -> None: + from .runtime import BrowserRuntimeError, SystemdUnitManager + + def kill_fails(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + del kwargs + if args[2] == "stop": + return subprocess.CompletedProcess(args, 1, "", "failed") + if args[2] == "kill": + return subprocess.CompletedProcess(args, 1, "", "failed") + return subprocess.CompletedProcess(args, 0, "", "") + + with self.assertRaises(BrowserRuntimeError): + SystemdUnitManager("systemd-run", "systemctl", kill_fails).stop("x", 1) + + def reset_fails(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + del kwargs + return subprocess.CompletedProcess(args, 2, "", "failed") + + with self.assertRaises(BrowserRuntimeError): + SystemdUnitManager("systemd-run", "systemctl", reset_fails).reset("x") + + def test_allocator_existing_and_file_lock_edges(self) -> None: + from .runtime import BrowserRuntimeError, DisplayAllocator, FileLock, PortAllocator + + with tempfile.TemporaryDirectory() as directory: + lock_dir = Path(directory) + lock = FileLock(lock_dir / "resource.lock") + self.assertTrue(lock.acquire()) + self.assertTrue(lock.acquire()) + lock.release() + lock.release() + ports = PortAllocator(lock_dir) + lease = ports.reserve_existing("cdp", 19999) + with self.assertRaises(BrowserRuntimeError): + ports.reserve_existing("cdp", 19999) + lease.lock.release() + with self.assertRaises(BrowserRuntimeError): + ports.reserve_existing("cdp", 0) + displays = DisplayAllocator(lock_dir) + display = displays.reserve_existing(100) + with self.assertRaises(BrowserRuntimeError): + displays.reserve_existing(100) + display.lock.release() + with self.assertRaises(BrowserRuntimeError): + displays.reserve_existing(0) + + +class NativeRuntimeManagerTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + root = Path(self.temp.name) + self.state = root / "state" + self.profiles = root / "profiles" + self.units = FakeUnits() + self.manager = NativeRuntimeManager( + state_dir=self.state, + profile_root=self.profiles, + node_id="node-a", + browser_versions={"148.0.7778.215": "/bin/true"}, + unit_manager=self.units, + min_free_bytes=0, + ) + self.manager._wait_for_display = lambda record: None + self.manager._wait_for_cdp = lambda record: None + + def tearDown(self) -> None: + self.manager.close() + self.temp.cleanup() + + def payload(self, **extra: Any) -> dict[str, Any]: + value: dict[str, Any] = { + "alias": "account-a", + "name": "Account A", + "browser_version": "148.0.7778.215", + "profile_id": "account-a-id", + "cmd": ["--fingerprint=1000", "about:blank"], + "binding_version": 1, + "network_exit_id": "", + "network_exit": {}, + "stopped": False, + } + value.update(extra) + return value + + def test_create_allocates_native_runtime_and_persists_generation(self) -> None: + result = self.manager.create(self.payload()) + self.assertEqual(result["state"], "running") + self.assertEqual(result["node_id"], "node-a") + self.assertRegex(result["runtime_id"], r"^[a-f0-9]{64}$") + self.assertTrue(result["endpoint"].startswith("http://127.0.0.1:")) + runtime_file = next(self.state.glob("runtimes/*/runtime.json")) + stored = json.loads(runtime_file.read_text()) + self.assertEqual(stored["runtime_id"], result["runtime_id"]) + browser_command = self.units.commands[-1][1] + self.assertNotIn("--no-sandbox", browser_command) + self.assertNotIn("--", browser_command) + self.assertIn("--disable-gpu", browser_command) + self.assertIn("--disable-gpu-compositing", browser_command) + self.assertIn("--remote-debugging-address=127.0.0.1", browser_command) + self.assertIn("--user-data-dir=" + stored["profile_dir"], browser_command) + + def test_insufficient_disk_is_rejected_before_process_side_effects(self) -> None: + limited = NativeRuntimeManager( + state_dir=self.state / "disk-state", + profile_root=self.profiles / "disk-profiles", + node_id="node-a", + browser_versions={"148.0.7778.215": "/bin/true"}, + unit_manager=self.units, + min_free_bytes=10**20, + ) + with self.assertRaises(BrowserRuntimeError) as caught: + limited.create(self.payload(alias="disk-account")) + self.assertIn("free disk space", str(caught.exception)) + self.assertEqual(self.units.commands, []) + limited.close() + + def test_external_display_reuses_existing_xvfb_without_starting_or_stopping_it(self) -> None: + external = NativeRuntimeManager( + state_dir=self.state / "external-state", + profile_root=self.profiles / "external-profiles", + node_id="node-a", + browser_versions={"148.0.7778.215": "/bin/true"}, + unit_manager=self.units, + external_display=99, + min_free_bytes=0, + ) + external._display_available = lambda display: display == 99 + external._wait_for_cdp = lambda record: None + result = external.create(self.payload(alias="external-account")) + self.assertEqual(result["display"], 99) + self.assertEqual(result["display_mode"], "external") + self.assertEqual(len(self.units.commands), 1) + self.assertIn("browser", self.units.commands[0][0]) + generation = { + "binding_version": 1, + "runtime_id": result["runtime_id"], + "network_id": result["network_id"], + } + external.change_state("external-account", "stop", generation) + self.assertEqual(external.list_public()[0]["state"], "stopped") + external.close() + + def test_external_display_rejects_a_second_active_runtime(self) -> None: + external = NativeRuntimeManager( + state_dir=self.state / "external-state-2", + profile_root=self.profiles / "external-profiles-2", + node_id="node-a", + browser_versions={"148.0.7778.215": "/bin/true"}, + unit_manager=self.units, + external_display=99, + min_free_bytes=0, + ) + external._display_available = lambda display: display == 99 + external._wait_for_cdp = lambda record: None + external.create(self.payload(alias="external-account")) + with self.assertRaises(BrowserRuntimeError) as caught: + external.create(self.payload(alias="external-account-2", profile_id="other")) + self.assertIn("already in use", str(caught.exception)) + external.close() + + def test_generation_mismatch_cannot_stop_or_remove_new_runtime(self) -> None: + result = self.manager.create(self.payload()) + wrong = { + "binding_version": 1, + "runtime_id": "0" * 64, + "network_id": result["network_id"], + } + with self.assertRaises(GenerationConflict): + self.manager.change_state("account-a", "stop", wrong) + with self.assertRaises(GenerationConflict): + self.manager.remove("account-a", wrong) + self.assertEqual(self.manager.list_public()[0]["state"], "running") + + def test_stop_and_repeated_remove_keep_profile(self) -> None: + result = self.manager.create(self.payload()) + generation = { + "binding_version": 1, + "runtime_id": result["runtime_id"], + "network_id": result["network_id"], + } + self.manager.change_state("account-a", "stop", generation) + self.assertEqual(self.manager.list_public()[0]["state"], "stopped") + self.manager.remove("account-a", generation) + self.manager.remove("account-a", generation) + self.assertEqual(self.manager.list_public()[0]["state"], "released") + profile_dirs = list(self.profiles.iterdir()) + self.assertEqual(len(profile_dirs), 1) + self.assertTrue(profile_dirs[0].is_dir()) + + def test_same_profile_is_exclusive_across_manager_instances(self) -> None: + first = self.manager.create(self.payload()) + del first + other = NativeRuntimeManager( + state_dir=self.state, + profile_root=self.profiles, + node_id="node-a", + browser_versions={"148.0.7778.215": "/bin/true"}, + unit_manager=self.units, + min_free_bytes=0, + ) + other._wait_for_display = lambda record: None + other._wait_for_cdp = lambda record: None + with self.assertRaises(BrowserRuntimeError): + other.create(self.payload(alias="account-b")) + other.close() + + def test_cleanup_failure_is_visible_and_retryable(self) -> None: + result = self.manager.create(self.payload()) + generation = { + "binding_version": 1, + "runtime_id": result["runtime_id"], + "network_id": result["network_id"], + } + self.units.fail_stop.update( + { + f"creatorhub-{result['alias']}-{result['runtime_id'][:16]}-browser.service", + } + ) + with self.assertRaises(RuntimeCleanupPending): + self.manager.remove("account-a", generation) + public = self.manager.list_public()[0] + self.assertEqual(public["cleanup_state"], "pending") + self.assertTrue(public["cleanup_error"]) + self.units.fail_stop.clear() + self.manager.retry_cleanup("account-a", generation) + self.assertEqual(self.manager.list_public()[0]["state"], "released") + + def test_reserved_browser_flags_are_rejected_before_side_effects(self) -> None: + with self.assertRaises(BrowserRuntimeError): + self.manager.create( + self.payload(cmd=["--no-sandbox", "about:blank"]) + ) + self.assertEqual(list(self.state.glob("runtimes/*/runtime.json")), []) + + def test_stopped_runtime_can_be_created_without_processes(self) -> None: + result = self.manager.create(self.payload(stopped=True)) + self.assertEqual(result["state"], "stopped") + self.assertEqual(self.units.commands, []) + + def test_timeout_marks_runtime_cleanup_pending(self) -> None: + self.manager.ready_timeout = 0.001 + self.manager._wait_for_display = NativeRuntimeManager._wait_for_display.__get__(self.manager) + self.manager._wait_for_cdp = NativeRuntimeManager._wait_for_cdp.__get__(self.manager) + with self.assertRaises(BrowserRuntimeError) as caught: + self.manager.create(self.payload()) + self.assertIn("Xvfb", str(caught.exception)) + public = self.manager.list_public()[0] + self.assertEqual(public["cleanup_state"], "pending") + self.assertEqual(public["state"], "failed") + + def test_cancel_during_start_is_visible_and_does_not_start_browser(self) -> None: + started = __import__("threading").Event() + original_wait = self.manager._wait_for_display + + def wait(record: object) -> None: + started.set() + while True: + self.manager._check_cancel(record) # type: ignore[arg-type] + __import__("time").sleep(0.001) + + self.manager._wait_for_display = wait + result: list[object] = [] + + def create() -> None: + try: + result.append(self.manager.create(self.payload())) + except Exception as exc: # the assertion below checks the typed outcome + result.append(exc) + + thread = __import__("threading").Thread(target=create) + thread.start() + self.assertTrue(started.wait(1)) + runtime_file = next(self.state.glob("runtimes/*/runtime.json")) + stored = json.loads(runtime_file.read_text()) + self.manager.cancel( + "account-a", + { + "binding_version": stored["binding_version"], + "runtime_id": stored["runtime_id"], + "network_id": stored["network_id"], + }, + ) + thread.join(1) + self.assertFalse(thread.is_alive()) + self.assertTrue(result and isinstance(result[0], BrowserRuntimeError)) + self.assertEqual(self.manager.list_public()[0]["cleanup_state"], "pending") + self.manager._wait_for_display = original_wait + + def test_restart_recovers_only_matching_active_units(self) -> None: + result = self.manager.create(self.payload()) + self.manager.close() + recovered = NativeRuntimeManager( + state_dir=self.state, + profile_root=self.profiles, + node_id="node-a", + browser_versions={"148.0.7778.215": "/bin/true"}, + unit_manager=self.units, + min_free_bytes=0, + ) + self.assertEqual(recovered.list_public()[0]["runtime_id"], result["runtime_id"]) + self.assertEqual(recovered.list_public()[0]["node_id"], "node-a") + recovered.close() + + def test_old_released_generation_cannot_touch_new_runtime(self) -> None: + old = self.manager.create(self.payload(stopped=True)) + old_generation = { + "binding_version": 1, + "runtime_id": old["runtime_id"], + "network_id": old["network_id"], + } + self.manager.remove("account-a", old_generation) + new = self.manager.create(self.payload(stopped=True)) + self.assertNotEqual(old["runtime_id"], new["runtime_id"]) + self.manager.remove("account-a", old_generation) + self.assertEqual(self.manager.list_public()[-1]["runtime_id"], new["runtime_id"]) + self.assertEqual(self.manager.list_public()[-1]["state"], "stopped") + + +if __name__ == "__main__": + unittest.main() diff --git a/cmd/docker_gateway/test_xiaohongshu.py b/cmd/browser_gateway/test_xiaohongshu.py similarity index 98% rename from cmd/docker_gateway/test_xiaohongshu.py rename to cmd/browser_gateway/test_xiaohongshu.py index 0822560..fa97b61 100644 --- a/cmd/docker_gateway/test_xiaohongshu.py +++ b/cmd/browser_gateway/test_xiaohongshu.py @@ -68,7 +68,7 @@ class XiaohongshuValidationTests(unittest.TestCase): { "binding_version": 1, "runtime_id": "a" * 64, - "network_id": "network", + "network_id": "native-" + "b" * 32, "network_exit_id": "", } ) diff --git a/cmd/docker_gateway/xiaohongshu.py b/cmd/browser_gateway/xiaohongshu.py similarity index 100% rename from cmd/docker_gateway/xiaohongshu.py rename to cmd/browser_gateway/xiaohongshu.py diff --git a/cmd/control-plane/account_deletion.go b/cmd/control-plane/account_deletion.go index bce2ff1..1db6088 100644 --- a/cmd/control-plane/account_deletion.go +++ b/cmd/control-plane/account_deletion.go @@ -34,11 +34,11 @@ func registerAccountDeletion(app *fiber.App, phaseAStore *phasea.Store, hubStore if environment.Exit.ID != "" { exitIDs = append(exitIDs, environment.Exit.ID) } - imageVersions := []string{} - if environment.ImageVersion != "" { - imageVersions = append(imageVersions, environment.ImageVersion) + browserVersions := []string{} + if environment.BrowserVersion != "" { + browserVersions = append(browserVersions, environment.BrowserVersion) } - unlock, err = hubStore.LockResources(c.Context(), aliases, exitIDs, imageVersions) + unlock, err = hubStore.LockResources(c.Context(), aliases, exitIDs, browserVersions) if err != nil { return hubError(c, err) } @@ -83,7 +83,14 @@ func registerAccountDeletion(app *fiber.App, phaseAStore *phasea.Store, hubStore }) } -func accountEnvironment(ctx context.Context, store *hub.Store, accountID string) (hub.EnvironmentContext, bool, error) { +type environmentContextStore interface { + GetEnvironmentContextForAccount(context.Context, string) (hub.EnvironmentContext, error) +} + +func accountEnvironment(ctx context.Context, store environmentContextStore, accountID string) (hub.EnvironmentContext, bool, error) { + if store == nil { + return hub.EnvironmentContext{}, false, creator.ErrUnavailable + } environment, err := store.GetEnvironmentContextForAccount(ctx, accountID) if errors.Is(err, hub.ErrNotFound) { return hub.EnvironmentContext{}, false, nil @@ -97,7 +104,6 @@ func accountEnvironment(ctx context.Context, store *hub.Store, accountID string) func purgeAccountProfile(ctx context.Context, gateway hub.Gateway, environment hub.EnvironmentContext) error { payload := gatewayCleanupGenerationPayload(environment) payload["purge_profile"] = true - payload["profile_volume"] = "creatorhub-profile-" + environment.Alias status, body, err := gatewayCall(ctx, gateway, http.MethodDelete, "/v1/browsers/"+environment.Alias, payload, 30*time.Second) if err != nil { return gatewayUnreachable(err) diff --git a/cmd/control-plane/creator.go b/cmd/control-plane/creator.go index e2c3f66..9e8f4ca 100644 --- a/cmd/control-plane/creator.go +++ b/cmd/control-plane/creator.go @@ -361,13 +361,21 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto if input.Page == 0 { input.Page = 1 } - collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID) + environment, err := hubStore.GetEnvironmentContextForAccount(c.Context(), input.AccountID) if err != nil { return creatorError(c, err) } - page, err := collector.SearchNotes(c.Context(), input.Query, input.Page) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(c.Context(), hubStore, environment, "task", "creator-xhs-read-"+input.AccountID) if err != nil { - return creatorError(c, err) + return creatorError(c, fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err)) + } + collector, err := newXiaohongshuReadCollector(useCtx, store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID) + if err != nil { + return creatorError(c, errors.Join(err, runtimeUse.Close())) + } + page, callErr := collector.SearchNotes(useCtx, input.Query, input.Page) + if closeErr := runtimeUse.Close(); callErr != nil || closeErr != nil { + return creatorError(c, errors.Join(callErr, closeErr)) } return c.JSON(page) }) @@ -379,13 +387,21 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto if err := decodeCreator(c, &input); err != nil { return creatorError(c, err) } - collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID) + environment, err := hubStore.GetEnvironmentContextForAccount(c.Context(), input.AccountID) if err != nil { return creatorError(c, err) } - item, err := collector.GetNoteDetail(c.Context(), input.URL) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(c.Context(), hubStore, environment, "task", "creator-xhs-detail-"+input.AccountID) if err != nil { - return creatorError(c, err) + return creatorError(c, fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err)) + } + collector, err := newXiaohongshuReadCollector(useCtx, store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID) + if err != nil { + return creatorError(c, errors.Join(err, runtimeUse.Close())) + } + item, callErr := collector.GetNoteDetail(useCtx, input.URL) + if closeErr := runtimeUse.Close(); callErr != nil || closeErr != nil { + return creatorError(c, errors.Join(callErr, closeErr)) } return c.JSON(item) }) @@ -857,18 +873,28 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto if err != nil { return creatorError(c, err) } - browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - accountUID, err := browser.Identity(c.Context(), profile.PlatformAccountKey) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(c.Context(), hubStore, environment, "task", "creator-conversation-"+conversation.ID) if err != nil { + return creatorError(c, fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err)) + } + browser := creatorGatewayBrowser{gateway: gateway, environment: environment} + accountUID, err := browser.Identity(useCtx, profile.PlatformAccountKey) + if err != nil { + _ = runtimeUse.Close() return creatorError(c, fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)) } - history, err := browser.MessageHistory(c.Context(), accountUID, conversation.PeerUID, conversation.HistoryCursor, limit) + history, err := browser.MessageHistory(useCtx, accountUID, conversation.PeerUID, conversation.HistoryCursor, limit) if err != nil { + _ = runtimeUse.Close() return creatorError(c, err) } if history.AccountUID != accountUID { + _ = runtimeUse.Close() return creatorError(c, creator.ErrConflict) } + if err := runtimeUse.Close(); err != nil { + return creatorError(c, err) + } inserted, err := persistDouyinMessageHistory(c.Context(), store, conversation, accountUID, history.Messages) if err != nil { return creatorError(c, err) @@ -1089,7 +1115,7 @@ type creatorGatewayActionExecutor struct { hubStore *hub.Store } -func (executor creatorGatewayActionExecutor) Execute(ctx context.Context, request creator.ActionRequest) (creator.ActionResult, error) { +func (executor creatorGatewayActionExecutor) Execute(ctx context.Context, request creator.ActionRequest) (result creator.ActionResult, resultErr error) { if request.Platform != creator.PlatformDouyin || executor.store == nil || executor.phaseAStore == nil || executor.hubStore == nil { return creator.ActionResult{}, creator.ErrUnavailable } @@ -1108,16 +1134,24 @@ func (executor creatorGatewayActionExecutor) Execute(ctx context.Context, reques if err != nil { return creator.ActionResult{}, err } + if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { + return creator.ActionResult{}, fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable) + } gateway, err := executor.hubStore.GetGateway(ctx, environment.Gateway) if err != nil { return creator.ActionResult{}, err } + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, executor.hubStore, environment, "task", request.OperationID) + if err != nil { + return creator.ActionResult{}, fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err) + } + defer func() { resultErr = errors.Join(resultErr, runtimeUse.Close()) }() browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - uid, identityErr := browser.Identity(ctx, profile.PlatformAccountKey) + uid, identityErr := browser.Identity(useCtx, profile.PlatformAccountKey) if identityErr != nil { return creator.ActionResult{}, fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr) } - if _, verifyErr := executor.store.RecordVerifiedLoginResult(ctx, request.AccountID, uid); verifyErr != nil { + if _, verifyErr := executor.store.RecordVerifiedLoginResult(useCtx, request.AccountID, uid); verifyErr != nil { return creator.ActionResult{}, fmt.Errorf("persist verified account identity: %w", verifyErr) } payload := gatewayGenerationPayload(environment) @@ -1160,7 +1194,7 @@ func (executor creatorGatewayActionExecutor) Execute(ctx context.Context, reques } payload["text"] = request.Text payload["confirm"] = true - status, body, err := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(environment.Alias)+"/douyin/action", payload, 30*time.Second) + status, body, err := gatewayCall(useCtx, gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(environment.Alias)+"/douyin/action", payload, 30*time.Second) if err != nil { return creator.ActionResult{}, err } @@ -1314,9 +1348,17 @@ func creatorLoginQRCode(ctx context.Context, store *creator.Store, phaseAStore * if err != nil { return nil, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - response, err := (douyinGatewayBrowser{gateway: gateway, environment: environment}).LoginQR(ctx) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, hubStore, environment, "task", "creator-login-qr-"+accountID) if err != nil { - return nil, fmt.Errorf("%w: capture the Douyin login screen: %v", creator.ErrUnavailable, err) + return nil, fmt.Errorf("%w: login runtime use unavailable: %v", creator.ErrUnavailable, err) + } + response, callErr := (douyinGatewayBrowser{gateway: gateway, environment: environment}).LoginQR(useCtx) + closeErr := runtimeUse.Close() + if callErr != nil { + return nil, fmt.Errorf("%w: capture the Douyin login screen: %v", creator.ErrUnavailable, callErr) + } + if closeErr != nil { + return nil, fmt.Errorf("%w: release login runtime use: %v", creator.ErrUnavailable, closeErr) } return map[string]any{ "status": "manual_login", @@ -1353,9 +1395,17 @@ func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore if err != nil { return creator.LoginResult{}, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - uid, err := verifyCreatorPlatformIdentity(ctx, account.Platform, gateway, environment, profile.PlatformAccountKey) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, hubStore, environment, "task", "creator-verify-"+accountID) if err != nil { - return creator.LoginResult{}, fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, err) + return creator.LoginResult{}, fmt.Errorf("%w: verification runtime use unavailable: %v", creator.ErrUnavailable, err) + } + uid, identityErr := verifyCreatorPlatformIdentity(useCtx, account.Platform, gateway, environment, profile.PlatformAccountKey) + closeErr := runtimeUse.Close() + if identityErr != nil { + return creator.LoginResult{}, fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr) + } + if closeErr != nil { + return creator.LoginResult{}, fmt.Errorf("%w: release verification runtime use: %v", creator.ErrUnavailable, closeErr) } result, err := store.RecordVerifiedLoginResult(ctx, accountID, uid) if err != nil { @@ -1714,15 +1764,22 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p if err != nil { return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)) } - collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, competitor.PlatformAccountKey, competitor.HomepageURL, creator.SourceCompetitor, competitor.ID) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, hubStore, environment, "task", "creator-sync-"+competitor.ID) if err != nil { - return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)) + return blocked(fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err)) + } + collector, _, err := newCreatorCollector(useCtx, competitor.Platform, gateway, environment, account.PlatformAccountKey, competitor.PlatformAccountKey, competitor.HomepageURL, creator.SourceCompetitor, competitor.ID) + if err != nil { + return blocked(errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), runtimeUse.Close())) } collectionNow := now if competitor.NextSyncAt != nil && !competitor.NextSyncAt.After(now) { collectionNow = competitor.NextSyncAt.UTC() } - report, collectErr := store.CollectSource(ctx, competitor.Platform, creator.SourceCompetitor, competitor.ID, collector, collectionNow) + report, collectErr := store.CollectSource(useCtx, competitor.Platform, creator.SourceCompetitor, competitor.ID, collector, collectionNow) + if releaseErr := runtimeUse.Close(); releaseErr != nil { + collectErr = errors.Join(collectErr, releaseErr) + } if collectErr != nil { nextBase := now if competitor.NextSyncAt != nil { @@ -1824,7 +1881,7 @@ func runCreatorMetricScheduleOnce(ctx context.Context, store *creator.Store, pha return nil } -func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, work creator.Work, accountID string, settings creator.Settings, now time.Time) error { +func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, work creator.Work, accountID string, settings creator.Settings, now time.Time) (resultErr error) { if work.SourceType == creator.SourceCompetitor { competitor, err := store.GetCompetitor(ctx, work.SourceID) if err != nil { @@ -1852,10 +1909,18 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if err != nil { return fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) } + if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { + return fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable) + } gateway, err := hubStore.GetGateway(ctx, environment.Gateway) if err != nil { return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, hubStore, environment, "task", "creator-metric-"+work.ID) + if err != nil { + return fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err) + } + defer func() { resultErr = errors.Join(resultErr, runtimeUse.Close()) }() targetAccountKey, homepageURL := account.PlatformAccountKey, "" if work.SourceType == creator.SourceCompetitor { competitor, competitorErr := store.GetCompetitor(ctx, work.SourceID) @@ -1867,13 +1932,13 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS } targetAccountKey, homepageURL = competitor.PlatformAccountKey, competitor.HomepageURL } - collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, targetAccountKey, homepageURL, work.SourceType, work.SourceID) + collector, collectionKey, err := newCreatorCollector(useCtx, work.Platform, gateway, environment, account.PlatformAccountKey, targetAccountKey, homepageURL, work.SourceType, work.SourceID) if err != nil { return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err) } cursor := "" for page := 0; page < 100; page++ { - result, pageErr := collector.ListWorks(ctx, collectionKey, cursor) + result, pageErr := collector.ListWorks(useCtx, collectionKey, cursor) if pageErr != nil { return pageErr } @@ -1884,7 +1949,7 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if item.Likes == nil && item.CommentsCount == nil && item.Shares == nil { return creator.ErrUnavailable } - _, metricErr := store.RecordMetric(ctx, creator.MetricInput{WorkID: work.ID, CollectedAt: now, Likes: item.Likes, CommentsCount: item.CommentsCount, Shares: item.Shares}, settings, now) + _, metricErr := store.RecordMetric(useCtx, creator.MetricInput{WorkID: work.ID, CollectedAt: now, Likes: item.Likes, CommentsCount: item.CommentsCount, Shares: item.Shares}, settings, now) return metricErr } if !result.HasMore { @@ -1940,17 +2005,21 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed") } }() - collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, account.PlatformAccountKey, "", creator.SourceOwned, account.ID) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, hubStore, environment, "task", "creator-owned-"+account.ID) + if err != nil { + return fmt.Errorf("%w: runtime use unavailable: %v", creator.ErrUnavailable, err) + } + collector, _, err := newCreatorCollector(useCtx, account.Platform, gateway, environment, account.PlatformAccountKey, account.PlatformAccountKey, "", creator.SourceOwned, account.ID) if err != nil { blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays) - return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr) + return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr, runtimeUse.Close()) } _, collectionNow, windowErr := store.NextCollectionWindow(ctx, creator.SourceOwned, account.ID, now, time.Duration(settings.NewWorkIntervalSeconds)*time.Second, settings.LookbackDays) if windowErr != nil { - return windowErr + return errors.Join(windowErr, runtimeUse.Close()) } - _, err = store.CollectSource(ctx, account.Platform, creator.SourceOwned, account.ID, collector, collectionNow) - return err + _, err = store.CollectSource(useCtx, account.Platform, creator.SourceOwned, account.ID, collector, collectionNow) + return errors.Join(err, runtimeUse.Close()) } func creatorCollectionAccount(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, platform string) (string, error) { diff --git a/cmd/control-plane/creator_events.go b/cmd/control-plane/creator_events.go index f281c82..72922b0 100644 --- a/cmd/control-plane/creator_events.go +++ b/cmd/control-plane/creator_events.go @@ -48,6 +48,7 @@ type creatorEventBinding struct { uid string env hub.EnvironmentContext gateway hub.Gateway + hubStore *hub.Store sessionToken string } @@ -195,7 +196,7 @@ func (manager *creatorEventListenerManager) reconcile(ctx context.Context, store logrus.WithError(gatewayErr).WithField("account_id", account.ID).Warn("creator event listener gateway unavailable") continue } - desired[account.ID] = creatorEventBinding{accountID: account.ID, uid: profile.PlatformAccountKey, env: environment, gateway: gateway} + desired[account.ID] = creatorEventBinding{accountID: account.ID, uid: profile.PlatformAccountKey, env: environment, gateway: gateway, hubStore: hubStore} } var stopping []creatorEventListenerHandle @@ -253,6 +254,17 @@ func runCreatorEventListener(ctx context.Context, store *creator.Store, binding binding.sessionToken = creatorListenerSessionToken(binding.env) path := "/v1/browsers/" + url.PathEscape(binding.env.Alias) + "/douyin/events" generation := gatewayGenerationPayload(binding.env) + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, binding.hubStore, binding.env, "listener", "creator-listener-"+binding.accountID) + if err != nil { + persistCreatorListenerState(context.WithoutCancel(ctx), store, binding, "error", "runtime use unavailable: "+err.Error(), nil, "") + return + } + ctx = useCtx + defer func() { + if releaseErr := runtimeUse.Close(); releaseErr != nil { + logrus.WithError(releaseErr).WithField("account_id", binding.accountID).Error("creator listener runtime use release failed") + } + }() startPayload := make(map[string]any, len(generation)+1) for key, value := range generation { startPayload[key] = value diff --git a/cmd/control-plane/creator_events_test.go b/cmd/control-plane/creator_events_test.go index 348429a..142c8fb 100644 --- a/cmd/control-plane/creator_events_test.go +++ b/cmd/control-plane/creator_events_test.go @@ -10,9 +10,9 @@ import ( ) func TestCreatorListenerPrimitives(t *testing.T) { - env := hub.EnvironmentContext{Env: hub.Env{Alias: "browser/a"}, BindingVersion: 3, RuntimeID: "runtime", RuntimeNetworkID: "network", Exit: hub.NetworkExit{ID: "exit"}} + env := hub.EnvironmentContext{Env: hub.Env{Alias: "browser/a"}, BindingVersion: 3, RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-dddddddddddddddddddddddddddddddd", Exit: hub.NetworkExit{ID: "exit"}} generation := creatorListenerGeneration(env) - if generation != "runtime:network:3" { + if generation != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd:native-dddddddddddddddddddddddddddddddd:3" { t.Fatalf("unexpected generation: %q", generation) } tokenA, tokenB := creatorListenerSessionToken(env), creatorListenerSessionToken(env) @@ -76,11 +76,11 @@ func TestCreatorUpdateHubPublishesAndUnsubscribes(t *testing.T) { func TestGatewayGenerationPayloadIncludesCurrentProxyExit(t *testing.T) { payload := gatewayGenerationPayload(hub.EnvironmentContext{ BindingVersion: 3, - RuntimeID: "runtime", - RuntimeNetworkID: "network", + RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + RuntimeNetworkID: "native-dddddddddddddddddddddddddddddddd", Exit: hub.NetworkExit{ID: "exit-current"}, }) - if payload["binding_version"] != int64(3) || payload["runtime_id"] != "runtime" || payload["network_id"] != "network" || payload["network_exit_id"] != "exit-current" { + if payload["binding_version"] != int64(3) || payload["runtime_id"] != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || payload["network_id"] != "native-dddddddddddddddddddddddddddddddd" || payload["network_exit_id"] != "exit-current" { t.Fatalf("unexpected generation payload: %#v", payload) } } diff --git a/cmd/control-plane/creator_events_unit_test.go b/cmd/control-plane/creator_events_unit_test.go new file mode 100644 index 0000000..7364d21 --- /dev/null +++ b/cmd/control-plane/creator_events_unit_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +func TestCreatorEventFromGatewayNoticeValidatesAndNormalizesInput(t *testing.T) { + input, err := creatorEventFromGatewayNotice("account-1", creatorGatewayEventNotice{ + EventKey: "1", + EventType: "dm", + InteractorUID: "123", + MessageText: " hello ", + PlatformEventAt: "2026-09-17T08:00:00+08:00", + GatewayReceivedAt: "2026-09-17T00:00:01Z", + }) + if err != nil { + t.Fatal(err) + } + if input.MessageType != creator.MessageTypeText || input.MessageText != "hello" || input.PlatformEventAt == nil || input.GatewayReceivedAt == nil { + t.Fatalf("normalized event = %#v", input) + } + if !input.PlatformEventAt.Equal(time.Date(2026, 9, 17, 0, 0, 0, 0, time.UTC)) { + t.Fatalf("platform event time = %v", input.PlatformEventAt) + } + + invalid := []creatorGatewayEventNotice{ + {EventKey: "", EventType: "comment"}, + {EventKey: "1", EventType: "unknown"}, + {EventKey: "1", EventType: "comment", InteractorUID: "0"}, + {EventKey: "1", EventType: "comment", CommentID: "bad"}, + {EventKey: "1", EventType: "comment", MessageType: "bad"}, + {EventKey: "1", EventType: "comment", PlatformEventAt: "bad"}, + {EventKey: "1", EventType: "comment", GatewayReceivedAt: "bad"}, + } + for index, notice := range invalid { + if _, err := creatorEventFromGatewayNotice("account-1", notice); err == nil { + t.Errorf("invalid notice %d returned no error", index) + } + } + if _, err := creatorEventFromGatewayNotice("", creatorGatewayEventNotice{EventKey: "1", EventType: "comment"}); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("empty account error = %v", err) + } +} + +func TestCreatorEventListenerControlPathsRemainVisible(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if waitCreatorEventBackoff(ctx, time.Millisecond) { + t.Fatal("canceled listener backoff was reported as ready") + } + if !waitCreatorEventBackoff(context.Background(), 0) { + t.Fatal("zero listener backoff was not ready") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusBadGateway) + })) + stopCreatorEventListener("account-1", hub.Gateway{Endpoint: server.URL}, "/v1/events", map[string]any{}) + server.Close() + server = httptest.NewServer(http.NotFoundHandler()) + stopCreatorEventListener("account-1", hub.Gateway{Endpoint: server.URL}, "/v1/events", map[string]any{}) + server.Close() + + binding := creatorEventBinding{accountID: "account-1", env: testRunnableEnvironment()} + runCreatorEventListener(context.Background(), nil, binding, nil, nil) + manager := &creatorEventListenerManager{items: map[string]creatorEventListenerHandle{}} + if err := manager.reconcile(context.Background(), nil, nil, nil, nil, nil); !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("nil listener dependencies error = %v", err) + } + listenerCtx, listenerCancel := context.WithCancel(context.Background()) + listenerCancel() + runCreatorEventListeners(listenerCtx, nil, nil, nil, nil, nil) +} + +func TestHandleCreatorGatewayEventClassifiesNonNoticeEventsWithoutStore(t *testing.T) { + binding := creatorEventBinding{accountID: "account-1"} + for _, kind := range []string{"error", "reconnected", "open", "baseline", "close", "unknown"} { + handleCreatorGatewayEvent(context.Background(), nil, binding, creatorGatewayEvent{Kind: kind}, nil, nil) + } + handleCreatorGatewayEvent(context.Background(), nil, binding, creatorGatewayEvent{Kind: "notice"}, nil, nil) + handleCreatorGatewayEvent(context.Background(), nil, binding, creatorGatewayEvent{ + Kind: "notice", + Notice: &creatorGatewayEventNotice{EventKey: "1", EventType: "comment", InteractorUID: "123"}, + }, nil, nil) + + binding.env = hub.EnvironmentContext{Env: hub.Env{Alias: "account-1"}, RuntimeID: "runtime-1", RuntimeNetworkID: "network-1", BindingVersion: 1} + if got := creatorListenerGeneration(binding.env); got != "runtime-1:network-1:1" { + t.Fatalf("listener generation = %q", got) + } +} diff --git a/cmd/control-plane/creator_helper_test.go b/cmd/control-plane/creator_helper_test.go index 852e36d..14b9e3e 100644 --- a/cmd/control-plane/creator_helper_test.go +++ b/cmd/control-plane/creator_helper_test.go @@ -181,7 +181,7 @@ func TestCreatorGatewayBrowserHistoryAndMedia(t *testing.T) { } })) defer server.Close() - browser := creatorGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 1}} + browser := creatorGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-dddddddddddddddddddddddddddddddd", BindingVersion: 1}} history, err := browser.MessageHistory(context.Background(), "account", "peer", "", 10) if err != nil || history.Status != "succeeded" || history.HistorySource != "douyin" { t.Fatalf("history = %+v, %v", history, err) @@ -208,7 +208,7 @@ func TestXiaohongshuGatewayMedia(t *testing.T) { _, _ = w.Write([]byte(`{"status":200,"content_type":"image/jpeg","body_base64":"` + base64.StdEncoding.EncodeToString([]byte("image")) + `"}`)) })) defer server.Close() - browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, AccountID: "account", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "binding", RuntimeInstanceID: "instance", RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit", HealthStatus: "healthy"}}} + browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, AccountID: "account", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "binding", RuntimeInstanceID: "instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-dddddddddddddddddddddddddddddddd", BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit", HealthStatus: "healthy"}}} data, contentType, err := browser.Media(context.Background(), "https://www.xiaohongshu.com/explore/abc") if err != nil || string(data) != "image" || contentType != "image/jpeg" { t.Fatalf("media = %q, %q, %v", data, contentType, err) @@ -226,7 +226,7 @@ func TestCreatorGatewayBrowserIdentity(t *testing.T) { defer server.Close() browser := creatorGatewayBrowser{ gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, - environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 1}, + environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-dddddddddddddddddddddddddddddddd", BindingVersion: 1}, } uid, err := browser.Identity(context.Background(), "expected-key") if err != nil || uid != "verified-uid" { diff --git a/cmd/control-plane/creator_login_test.go b/cmd/control-plane/creator_login_test.go index 44b16df..57ee025 100644 --- a/cmd/control-plane/creator_login_test.go +++ b/cmd/control-plane/creator_login_test.go @@ -14,15 +14,15 @@ func TestStartCreatorEnvironmentStartsBoundRuntime(t *testing.T) { server := httptest.NewServer(gateway.handler(t)) defer server.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} - _ = store.CreateImage(context.Background(), hub.Image{ - Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true, + _ = store.CreateBrowserVersion(context.Background(), hub.BrowserVersion{ + Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true, }) environment := hub.EnvironmentContext{ Env: hub.Env{ - Alias: "account-a", - Name: "账号一", - Gateway: "gw-1", - ImageVersion: "148.0.7778.215", + Alias: "account-a", + Name: "账号一", + Gateway: "gw-1", + BrowserVersion: "148.0.7778.215", }, AccountID: "account-a", AccountStatus: "active", diff --git a/cmd/control-plane/creator_material.go b/cmd/control-plane/creator_material.go index 68313e0..06bdd77 100644 --- a/cmd/control-plane/creator_material.go +++ b/cmd/control-plane/creator_material.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "errors" "fmt" "os" "os/exec" @@ -13,6 +14,7 @@ import ( "git.ipao.vip/rogee/creator-hub/internal/creator" "git.ipao.vip/rogee/creator-hub/internal/hub" "git.ipao.vip/rogee/creator-hub/internal/phasea" + "github.com/sirupsen/logrus" ) type creatorMaterialDownloader struct { @@ -21,7 +23,7 @@ type creatorMaterialDownloader struct { hubStore *hub.Store } -func (downloader creatorMaterialDownloader) Download(ctx context.Context, work creator.Work, destination string) error { +func (downloader creatorMaterialDownloader) Download(ctx context.Context, work creator.Work, destination string) (resultErr error) { if (work.Platform != creator.PlatformDouyin && work.Platform != creator.PlatformXiaohongshu) || downloader.store == nil || downloader.phaseAStore == nil || downloader.hubStore == nil { return fmt.Errorf("%w: creator media gateway is unavailable", creator.ErrUnavailable) } @@ -48,17 +50,25 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c if err != nil { return fmt.Errorf("%w: media browser environment unavailable: %v", creator.ErrUnavailable, err) } + if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { + return fmt.Errorf("%w: media browser runtime is not running", creator.ErrUnavailable) + } gateway, err := downloader.hubStore.GetGateway(ctx, environment.Gateway) if err != nil { return fmt.Errorf("%w: media gateway unavailable: %v", creator.ErrUnavailable, err) } - if _, err := verifyCreatorPlatformIdentity(ctx, work.Platform, gateway, environment, profile.PlatformAccountKey); err != nil { + useCtx, runtimeUse, err := beginRuntimeUseForEnvironment(ctx, downloader.hubStore, environment, "task", "creator-material-"+work.ID) + if err != nil { + return fmt.Errorf("%w: media runtime use unavailable: %v", creator.ErrUnavailable, err) + } + defer func() { resultErr = errors.Join(resultErr, runtimeUse.Close()) }() + if _, err := verifyCreatorPlatformIdentity(useCtx, work.Platform, gateway, environment, profile.PlatformAccountKey); err != nil { return fmt.Errorf("%w: media browser identity verification failed: %v", creator.ErrConflict, err) } if work.Platform == creator.PlatformDouyin { - return (creatorGatewayBrowser{gateway: gateway, environment: environment}).Media(ctx, work.OriginalURL, destination) + return (creatorGatewayBrowser{gateway: gateway, environment: environment}).Media(useCtx, work.OriginalURL, destination) } - data, contentType, err := (xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}).Media(ctx, work.OriginalURL) + data, contentType, err := (xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}).Media(useCtx, work.OriginalURL) if err != nil { return err } @@ -88,19 +98,42 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto if root == "" { root = "/var/lib/creatorhub/materials" } - root = filepath.Clean(root) + root, err = filepath.Abs(filepath.Clean(root)) + if err != nil { + return creator.MaterialJob{}, fmt.Errorf("resolve material directory: %w", err) + } dir := filepath.Join(root, workID) if err := os.MkdirAll(dir, 0o700); err != nil { return creator.MaterialJob{}, fmt.Errorf("create material directory: %w", err) } - videoPath := filepath.Join(dir, "source") - videoReference := filepath.Join(workID, "source") - if job.DownloadStatus != "succeeded" || !fileExists(videoPath) { - if job.DownloadStatus == "succeeded" { - if _, err := store.SetMaterialStep(ctx, workID, "download", "failed", "", "下载产物不存在"); err != nil { - return creator.MaterialJob{}, err + executionID, err := materialClaimToken() + if err != nil { + return creator.MaterialJob{}, err + } + executionDir := filepath.Join(dir, ".runs", executionID) + if err := os.MkdirAll(executionDir, 0o700); err != nil { + return creator.MaterialJob{}, fmt.Errorf("create material execution directory: %w", err) + } + keepExecutionDir := false + defer func() { + if !keepExecutionDir { + if cleanupErr := os.RemoveAll(executionDir); cleanupErr != nil { + logrus.WithError(cleanupErr).WithField("execution_dir", executionDir).Error("creator material execution cleanup failed") } } + }() + + videoPath := "" + if job.DownloadStatus == "succeeded" { + videoPath, err = materialArtifactPath(root, workID, job.VideoReference) + if err != nil || !fileExists(videoPath) { + if _, setErr := store.SetMaterialStep(ctx, workID, "download", "failed", "", "下载产物不存在"); setErr != nil { + return creator.MaterialJob{}, setErr + } + job.DownloadStatus = "failed" + } + } + if job.DownloadStatus != "succeeded" { token, tokenErr := materialClaimToken() if tokenErr != nil { return creator.MaterialJob{}, tokenErr @@ -112,23 +145,29 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto if !claimed { return job, fmt.Errorf("%w: download step is already in progress", creator.ErrConflict) } + videoPath = filepath.Join(executionDir, "source") if err := (creatorMaterialDownloader{store: store, phaseAStore: phaseAStore, hubStore: hubStore}).Download(ctx, work, videoPath); err != nil { return setMaterialFailure(ctx, store, workID, "download", token, err) } + keepExecutionDir = true + videoReference := materialArtifactReference(workID, executionID, "source") job, err = store.CompleteMaterialStep(ctx, workID, "download", token, "succeeded", videoReference, "") if err != nil { return creator.MaterialJob{}, err } } - audioPath := filepath.Join(dir, "audio.wav") - audioReference := filepath.Join(workID, "audio.wav") - if job.AudioStatus != "succeeded" && job.AudioStatus != "no_audio" || job.AudioStatus == "succeeded" && !fileExists(audioPath) { - if job.AudioStatus == "succeeded" { - if _, err := store.SetMaterialStep(ctx, workID, "audio", "failed", "", "音频产物不存在"); err != nil { - return creator.MaterialJob{}, err + audioPath := "" + if job.AudioStatus == "succeeded" { + audioPath, err = materialArtifactPath(root, workID, job.AudioReference) + if err != nil || !fileExists(audioPath) { + if _, setErr := store.SetMaterialStep(ctx, workID, "audio", "failed", "", "音频产物不存在"); setErr != nil { + return creator.MaterialJob{}, setErr } + job.AudioStatus = "failed" } + } + if job.AudioStatus != "succeeded" && job.AudioStatus != "no_audio" { token, tokenErr := materialClaimToken() if tokenErr != nil { return creator.MaterialJob{}, tokenErr @@ -140,6 +179,7 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto if !claimed { return job, fmt.Errorf("%w: audio step is already in progress", creator.ErrConflict) } + audioPath = filepath.Join(executionDir, "audio.wav") hasAudio, err := creatorMaterialHasAudio(ctx, videoPath) if err != nil { return setMaterialFailure(ctx, store, workID, "audio", token, err) @@ -149,7 +189,8 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto } else if err := extractCreatorAudio(ctx, videoPath, audioPath); err != nil { return setMaterialFailure(ctx, store, workID, "audio", token, err) } else { - job, err = store.CompleteMaterialStep(ctx, workID, "audio", token, "succeeded", audioReference, "") + keepExecutionDir = true + job, err = store.CompleteMaterialStep(ctx, workID, "audio", token, "succeeded", materialArtifactReference(workID, executionID, "audio.wav"), "") } if err != nil { return creator.MaterialJob{}, err @@ -184,11 +225,12 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto } else if strings.TrimSpace(transcript) == "" { job, err = store.CompleteMaterialStep(ctx, workID, "transcription", token, "no_speech", "", "转写未检测到语音") } else { - transcriptPath := filepath.Join(dir, "transcript.txt") + transcriptPath := filepath.Join(executionDir, "transcript.txt") if writeErr := os.WriteFile(transcriptPath, []byte(transcript), 0o600); writeErr != nil { job, err = setMaterialFailure(ctx, store, workID, "transcription", token, writeErr) } else { - job, err = store.CompleteMaterialStep(ctx, workID, "transcription", token, "succeeded", transcript, "") + keepExecutionDir = true + job, err = store.CompleteMaterialStep(ctx, workID, "transcription", token, "succeeded", materialArtifactReference(workID, executionID, "transcript.txt"), "") } } } @@ -199,6 +241,31 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto return job, nil } +func materialArtifactReference(workID, executionID, name string) string { + return filepath.ToSlash(filepath.Join(workID, ".runs", executionID, name)) +} + +func materialArtifactPath(root, workID, reference string) (string, error) { + if reference == "" || filepath.IsAbs(reference) { + return "", creator.ErrInvalid + } + clean := filepath.Clean(filepath.FromSlash(reference)) + prefix := workID + string(filepath.Separator) + if clean == "." || !strings.HasPrefix(clean, prefix) || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", creator.ErrInvalid + } + root, err := filepath.Abs(root) + if err != nil { + return "", err + } + path := filepath.Join(root, clean) + relative, err := filepath.Rel(root, path) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", creator.ErrInvalid + } + return path, nil +} + func materialClaimToken() (string, error) { var data [16]byte if _, err := rand.Read(data[:]); err != nil { diff --git a/cmd/control-plane/creator_material_test.go b/cmd/control-plane/creator_material_test.go index 2292a87..ad10a01 100644 --- a/cmd/control-plane/creator_material_test.go +++ b/cmd/control-plane/creator_material_test.go @@ -2,91 +2,88 @@ package main import ( "context" + "errors" "os" "path/filepath" - "strings" "testing" + + "git.ipao.vip/rogee/creator-hub/internal/creator" ) -func TestWriteCreatorMediaPublishesOnlyCompleteFiles(t *testing.T) { - destination := filepath.Join(t.TempDir(), "source") - if err := writeCreatorMedia(destination, []byte("media")); err != nil { - t.Fatalf("write media: %v", err) - } - content, err := os.ReadFile(destination) - if err != nil { - t.Fatalf("read media: %v", err) - } - if string(content) != "media" { - t.Fatalf("unexpected media: %q", content) - } - matches, err := filepath.Glob(filepath.Join(filepath.Dir(destination), ".creator-media-*")) - if err != nil { - t.Fatalf("find temporary media: %v", err) - } - if len(matches) != 0 { - t.Fatalf("temporary files remain: %v", matches) - } -} - -func TestWriteCreatorMediaRejectsEmptyAndOversizedFiles(t *testing.T) { - destination := filepath.Join(t.TempDir(), "source") - if err := writeCreatorMedia(destination, nil); err == nil { - t.Fatal("expected empty media to be rejected") - } - if err := writeCreatorMedia(destination, make([]byte, maxCreatorMediaBytes+1)); err == nil { - t.Fatal("expected oversized media to be rejected") - } -} - -func TestExtractCreatorAudioPublishesWavAtomically(t *testing.T) { - dir := t.TempDir() - fakeFFmpeg := filepath.Join(dir, "ffmpeg") - argsFile := filepath.Join(dir, "args") - script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$ARGS_FILE\"\nfor arg in \"$@\"; do output=\"$arg\"; done\nprintf 'RIFFfake' > \"$output\"\n" - if err := os.WriteFile(fakeFFmpeg, []byte(script), 0o700); err != nil { - t.Fatalf("write fake ffmpeg: %v", err) - } - t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) - t.Setenv("ARGS_FILE", argsFile) - videoPath := filepath.Join(dir, "source") - audioPath := filepath.Join(dir, "audio.wav") - if err := os.WriteFile(videoPath, []byte("video"), 0o600); err != nil { - t.Fatalf("write source: %v", err) - } - if err := extractCreatorAudio(context.Background(), videoPath, audioPath); err != nil { - t.Fatalf("extract audio: %v", err) - } - if !fileExists(audioPath) { - t.Fatal("expected published audio") - } - if _, err := os.Stat(audioPath + ".tmp"); !os.IsNotExist(err) { - t.Fatalf("temporary audio remains: %v", err) - } - args, err := os.ReadFile(argsFile) - if err != nil { - t.Fatalf("read ffmpeg args: %v", err) - } - if !strings.Contains(string(args), "-f\n") || !strings.Contains(string(args), "\nwav\n") { - t.Fatalf("ffmpeg did not request WAV output: %s", args) - } -} - -func TestTranscribeCreatorAudioRequiresExplicitProvider(t *testing.T) { - t.Setenv("CREATOR_TRANSCRIPTION_BIN", "") - if _, err := transcribeCreatorAudio(context.Background(), "/tmp/audio.wav", "whisper", "base"); err == nil { - t.Fatal("expected missing provider error") - } -} - -func TestTranscribeCreatorAudioUsesConfiguredBinaryPath(t *testing.T) { - path := filepath.Join(t.TempDir(), "whisper") - if err := os.WriteFile(path, []byte("#!/bin/sh\nprintf configured-transcript\n"), 0o700); err != nil { +func TestCreatorMaterialAudioAndTranscriptionFailuresAreVisible(t *testing.T) { + binDir := t.TempDir() + fakeFFmpeg := filepath.Join(binDir, "ffmpeg") + if err := os.WriteFile(fakeFFmpeg, []byte("#!/bin/sh\nfor arg in \"$@\"; do last=\"$arg\"; done\nprintf x > \"$last\"\n"), 0o700); err != nil { t.Fatal(err) } - t.Setenv("CREATOR_TRANSCRIPTION_BIN", path) - transcript, err := transcribeCreatorAudio(context.Background(), "/tmp/audio.wav", "whisper", "base") - if err != nil || transcript != "configured-transcript" { - t.Fatalf("configured transcription: transcript=%q err=%v", transcript, err) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + audioPath := filepath.Join(t.TempDir(), "audio.wav") + if err := extractCreatorAudio(context.Background(), "/missing/video.mp4", audioPath); err != nil { + t.Fatalf("extractCreatorAudio returned %v", err) + } + if _, err := os.Stat(audioPath); err != nil { + t.Fatalf("extracted audio was not published: %v", err) + } + + if err := os.WriteFile(fakeFFmpeg, []byte("#!/bin/sh\necho ffmpeg failed >&2\nexit 1\n"), 0o700); err != nil { + t.Fatal(err) + } + if err := extractCreatorAudio(context.Background(), "/missing/video.mp4", filepath.Join(t.TempDir(), "failed.wav")); err == nil { + t.Fatal("failed ffmpeg command was reported as successful") + } + if err := validateTranscriptionBinary("missing-transcription-binary"); err == nil { + t.Fatal("missing transcription binary was reported as available") + } + if _, err := transcribeCreatorAudio(context.Background(), "audio.wav", "", "model"); err == nil { + t.Fatal("missing transcription provider was reported as successful") + } + if _, err := transcribeCreatorAudio(context.Background(), "audio.wav", "unknown", "model"); err == nil { + t.Fatal("unsupported transcription provider was reported as successful") + } + t.Setenv("CREATOR_TRANSCRIPTION_BIN", filepath.Join(binDir, "not-whisper")) + if _, err := transcribeCreatorAudio(context.Background(), "audio.wav", "whisper", "model"); err == nil { + t.Fatal("mismatched transcription binary was reported as successful") + } + + whisper := filepath.Join(binDir, "whisper") + if err := os.WriteFile(whisper, []byte("#!/bin/sh\nprintf 'transcript'\n"), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("CREATOR_TRANSCRIPTION_BIN", whisper) + text, err := transcribeCreatorAudio(context.Background(), audioPath, "whisper", "small") + if err != nil || text != "transcript" { + t.Fatalf("transcription = %q, err = %v", text, err) + } + if err := os.WriteFile(whisper, []byte("#!/bin/sh\nexit 1\n"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := transcribeCreatorAudio(context.Background(), audioPath, "whisper", "small"); err == nil { + t.Fatal("failed transcription command was reported as successful") + } +} + +func TestMaterialArtifactReferenceAndPathStayWithinWorkRoot(t *testing.T) { + if got := materialArtifactReference("work-1", "execution-1", "cover.jpg"); got != "work-1/.runs/execution-1/cover.jpg" { + t.Fatalf("artifact reference = %q", got) + } + + root := t.TempDir() + want := filepath.Join(root, "work-1", ".runs", "execution-1", "cover.jpg") + got, err := materialArtifactPath(root, "work-1", "work-1/.runs/execution-1/cover.jpg") + if err != nil || got != want { + t.Fatalf("artifact path = %q, err = %v, want %q", got, err, want) + } + + for _, reference := range []string{ + "", + "/tmp/cover.jpg", + "other/.runs/execution-1/cover.jpg", + "work-1/../other/cover.jpg", + "work-1/../../outside.jpg", + "work-1", + } { + if _, err := materialArtifactPath(root, "work-1", reference); !errors.Is(err, creator.ErrInvalid) { + t.Errorf("materialArtifactPath(%q) error = %v, want creator.ErrInvalid", reference, err) + } } } diff --git a/cmd/control-plane/creator_pure_unit_test.go b/cmd/control-plane/creator_pure_unit_test.go new file mode 100644 index 0000000..037b968 --- /dev/null +++ b/cmd/control-plane/creator_pure_unit_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gofiber/fiber/v3" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +func TestDecodeCreatorRejectsUnknownAndTrailingJSON(t *testing.T) { + app := fiber.New() + app.Post("/", func(c fiber.Ctx) error { + var value struct { + Name string `json:"name"` + } + if err := decodeCreator(c, &value); err != nil { + return fiber.ErrBadRequest + } + return c.SendStatus(http.StatusNoContent) + }) + valid := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"name":"ok"}`)) + valid.Header.Set("Content-Type", "application/json") + response, err := app.Test(valid) + if err != nil || response.StatusCode != http.StatusNoContent { + t.Fatalf("valid creator JSON status = %d, err = %v", response.StatusCode, err) + } + for _, body := range []string{`{"unknown":true}`, `{"name":"ok"}{"name":"extra"}`} { + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil || response.StatusCode != http.StatusBadRequest { + t.Errorf("invalid creator JSON %q status = %d, err = %v", body, response.StatusCode, err) + } + } +} + +func TestCreatorErrorMapsDomainErrorsAndReasons(t *testing.T) { + for _, testCase := range []struct { + err error + status int + }{ + {err: creator.ErrInvalid, status: http.StatusBadRequest}, + {err: creator.ErrConflict, status: http.StatusConflict}, + {err: creator.ErrNotFound, status: http.StatusNotFound}, + {err: creator.ErrUnavailable, status: http.StatusServiceUnavailable}, + {err: creator.ErrUncertain, status: http.StatusConflict}, + {err: errors.New("unexpected"), status: http.StatusInternalServerError}, + } { + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { return creatorError(c, testCase.err) }) + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil)) + if err != nil || response.StatusCode != testCase.status { + t.Errorf("creator error %v status = %d, err = %v", testCase.err, response.StatusCode, err) + } + } + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { + return creatorError(c, fmt.Errorf("%w: gateway stopped", creator.ErrUnavailable)) + }) + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil)) + if err != nil || response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("unavailable reason status = %d, err = %v", response.StatusCode, err) + } +} + +func TestCreatorPageQueryRejectsNonNumericValues(t *testing.T) { + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { + _, _, _, err := creatorPageQuery(c) + if err != nil { + return fiber.ErrBadRequest + } + return c.SendStatus(http.StatusNoContent) + }) + for _, query := range []string{"page=bad", "page_size=bad"} { + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/?"+query, nil)) + if err != nil || response.StatusCode != http.StatusBadRequest { + t.Errorf("invalid page query %q status = %d, err = %v", query, response.StatusCode, err) + } + } +} + +func TestWorkFilterParsesNumericAndTimeFilters(t *testing.T) { + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { + filter, err := workFilter(c) + if err != nil { + return fiber.ErrBadRequest + } + return c.JSON(filter) + }) + request := httptest.NewRequest(http.MethodGet, "/?platform=douyin&source_id=source-1&source_type=owned&published_at_status=published&min_likes=1&min_comments=2&min_shares=3&published_after=2026-09-17T00:00:00Z&published_before=2026-09-18T00:00:00Z", nil) + response, err := app.Test(request) + if err != nil || response.StatusCode != http.StatusOK { + t.Fatalf("valid work filter status = %d, err = %v", response.StatusCode, err) + } + for _, query := range []string{"min_likes=bad", "min_comments=bad", "min_shares=bad", "published_after=bad", "published_before=bad"} { + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/?"+query, nil)) + if err != nil || response.StatusCode != http.StatusBadRequest { + t.Errorf("invalid work filter %q status = %d, err = %v", query, response.StatusCode, err) + } + } +} + +type accountEnvironmentTestStore struct { + environment hub.EnvironmentContext + err error +} + +func (s accountEnvironmentTestStore) GetEnvironmentContextForAccount(context.Context, string) (hub.EnvironmentContext, error) { + return s.environment, s.err +} + +func TestCreatorSchedulerAndAccountEnvironmentFailWithoutStores(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + runCreatorScheduler(ctx, nil, nil, nil) + if _, found, err := accountEnvironment(context.Background(), nil, "account-1"); !errors.Is(err, creator.ErrUnavailable) || found { + t.Fatalf("nil account environment store = found=%v err=%v", found, err) + } + if _, found, err := accountEnvironment(context.Background(), accountEnvironmentTestStore{err: hub.ErrNotFound}, "account-1"); err != nil || found { + t.Fatalf("missing account environment = found=%v err=%v", found, err) + } + sentinel := errors.New("database failed") + if _, found, err := accountEnvironment(context.Background(), accountEnvironmentTestStore{err: sentinel}, "account-1"); !errors.Is(err, sentinel) || found { + t.Fatalf("account environment error = found=%v err=%v", found, err) + } + environment := testRunnableEnvironment() + got, found, err := accountEnvironment(context.Background(), accountEnvironmentTestStore{environment: environment}, "account-1") + if err != nil || !found || got.RuntimeID != environment.RuntimeID { + t.Fatalf("account environment = %#v, found=%v, err=%v", got, found, err) + } +} + +func TestCreatorPreviewAndLifecycleHelpersRejectUnavailableDependencies(t *testing.T) { + if _, err := previewCompetitorShare(context.Background(), nil, nil, nil, "account-1", "", "not-a-url"); err == nil { + t.Fatal("invalid competitor share URL was accepted") + } + if _, err := previewCompetitorShare(context.Background(), nil, nil, nil, "account-1", creator.PlatformXiaohongshu, "https://www.douyin.com/video/123"); err == nil { + t.Fatal("platform mismatch was accepted") + } + if _, err := previewCompetitorShare(context.Background(), nil, nil, nil, "account-1", "", "https://www.douyin.com/video/123"); err == nil { + t.Fatal("unavailable Douyin preview was reported as successful") + } + if _, err := previewCompetitorShare(context.Background(), nil, nil, nil, "account-1", "", "https://www.xiaohongshu.com/explore/123"); err == nil { + t.Fatal("unavailable Xiaohongshu preview was reported as successful") + } + if got := (competitorSharePreview{Platform: creator.PlatformDouyin, PlatformAccountKey: "uid", Nickname: "name", AvatarURL: "avatar", HomepageURL: "home"}).input(); got.PlatformAccountKey != "uid" || got.Nickname != "name" { + t.Fatalf("preview input = %#v", got) + } + if _, err := newDouyinAccountBrowser(context.Background(), nil, nil, nil, "account-1"); err == nil { + t.Fatal("missing Douyin dependencies were accepted") + } + if _, err := verifyCreatorAccount(context.Background(), nil, nil, nil, "account-1"); err == nil { + t.Fatal("missing account verification dependencies were accepted") + } + if _, err := creatorLoginQRCode(context.Background(), nil, nil, nil, "account-1"); err == nil { + t.Fatal("missing login QR dependencies were accepted") + } + if _, err := processCreatorMaterial(context.Background(), nil, nil, nil, "work-1"); err == nil { + t.Fatal("missing material dependencies were accepted") + } + if err := startCreatorEnvironment(context.Background(), nil, hub.EnvironmentContext{}); err == nil { + t.Fatal("missing runtime store was accepted") + } + if _, err := creatorCollectionAccount(context.Background(), nil, nil, nil, creator.PlatformDouyin); err == nil { + t.Fatal("missing collection dependencies were accepted") + } +} diff --git a/cmd/control-plane/creator_route_validation_test.go b/cmd/control-plane/creator_route_validation_test.go new file mode 100644 index 0000000..ad0d3d4 --- /dev/null +++ b/cmd/control-plane/creator_route_validation_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gofiber/fiber/v3" +) + +func TestCreatorWriteRoutesRejectMalformedInputBeforeStoreAccess(t *testing.T) { + app := fiber.New() + registerCreatorWithServices(app, nil, nil, nil, nil, nil, nil) + routes := []struct { + method string + path string + }{ + {method: http.MethodPut, path: "/api/creator/settings"}, + {method: http.MethodPut, path: "/api/creator/accounts/account-1/profile"}, + {method: http.MethodPut, path: "/api/creator/accounts/account-1/tags"}, + {method: http.MethodPost, path: "/api/creator/accounts/account-1/login-result"}, + {method: http.MethodPost, path: "/api/creator/accounts/account-1/big-account"}, + {method: http.MethodPost, path: "/api/creator/relations"}, + {method: http.MethodPost, path: "/api/creator/accounts/account-1/strategies"}, + {method: http.MethodPut, path: "/api/creator/strategies/strategy-1"}, + {method: http.MethodPost, path: "/api/creator/competitors/preview"}, + {method: http.MethodPost, path: "/api/creator/competitors"}, + {method: http.MethodPut, path: "/api/creator/competitors/competitor-1"}, + {method: http.MethodPost, path: "/api/creator/competitors/competitor-1/sync"}, + {method: http.MethodPost, path: "/api/creator/xiaohongshu/search"}, + {method: http.MethodPost, path: "/api/creator/xiaohongshu/detail"}, + {method: http.MethodPost, path: "/api/creator/test/works"}, + {method: http.MethodPost, path: "/api/creator/works/work-1/metrics"}, + {method: http.MethodPost, path: "/api/creator/works/work-1/material/rewrite/confirm"}, + {method: http.MethodPut, path: "/api/creator/works/work-1/material/rewrite"}, + {method: http.MethodPost, path: "/api/creator/test/comments"}, + {method: http.MethodPost, path: "/api/creator/rules"}, + {method: http.MethodPut, path: "/api/creator/rules/rule-1"}, + {method: http.MethodPost, path: "/api/creator/comments/analyze"}, + {method: http.MethodPost, path: "/api/creator/comments/comment-1/analyze"}, + {method: http.MethodPost, path: "/api/creator/test/events"}, + {method: http.MethodPost, path: "/api/creator/events/process"}, + {method: http.MethodPost, path: "/api/creator/messages"}, + } + for _, route := range routes { + route := route + t.Run(route.method+" "+route.path, func(t *testing.T) { + request := httptest.NewRequest(route.method, route.path, strings.NewReader("{")) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusBadRequest && response.StatusCode != http.StatusConflict { + t.Fatalf("malformed request status = %d", response.StatusCode) + } + }) + } +} diff --git a/cmd/control-plane/douyin_test.go b/cmd/control-plane/douyin_test.go index 58c7868..91bef60 100644 --- a/cmd/control-plane/douyin_test.go +++ b/cmd/control-plane/douyin_test.go @@ -23,7 +23,7 @@ func TestDouyinGatewayBrowserFencesAccountGeneration(t *testing.T) { } var body map[string]any if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) || - body["runtime_id"] != "runtime-a" || body["network_id"] != "network-a" || body["network_exit_id"] != "exit-a" { + body["runtime_id"] != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || body["network_id"] != "native-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || body["network_exit_id"] != "exit-a" { t.Fatalf("generation fence missing: %#v", body) } switch request.URL.Path { @@ -110,5 +110,5 @@ func readyDouyinEnvironment() hub.EnvironmentContext { return hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Gateway: "gateway-a"}, AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "binding-a", BindingVersion: 2, Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "healthy"}, RuntimeInstanceID: "instance-a", - RuntimeID: "runtime-a", RuntimeNetworkID: "network-a"} + RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} } diff --git a/cmd/control-plane/gateway_browser_unit_test.go b/cmd/control-plane/gateway_browser_unit_test.go new file mode 100644 index 0000000..ab57793 --- /dev/null +++ b/cmd/control-plane/gateway_browser_unit_test.go @@ -0,0 +1,266 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +func testRunnableEnvironment() hub.EnvironmentContext { + return hub.EnvironmentContext{ + Env: hub.Env{Alias: "account-1", BrowserVersion: "148.0.7778.215"}, + AccountID: "account-1", + AccountStatus: "active", + AuthorizationStatus: "authorized", + BindingID: "binding-1", + RuntimeID: "runtime-1", + RuntimeInstanceID: "runtime-instance-1", + RuntimeNetworkID: "network-1", + Exit: hub.NetworkExit{ID: "exit-1", HealthStatus: "healthy"}, + BindingVersion: 1, + } +} + +func TestPurgeAccountProfileTreatsMissingRuntimeAsIdempotent(t *testing.T) { + environment := testRunnableEnvironment() + for _, status := range []int{http.StatusNoContent, http.StatusNotFound} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Path != "/v1/browsers/account-1" { + t.Fatalf("purge request = %s %s", r.Method, r.URL.Path) + } + w.WriteHeader(status) + })) + if err := purgeAccountProfile(context.Background(), hub.Gateway{Endpoint: server.URL}, environment); err != nil { + t.Errorf("status %d: %v", status, err) + } + server.Close() + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte("gateway failure")) + })) + if err := purgeAccountProfile(context.Background(), hub.Gateway{Endpoint: server.URL}, environment); err == nil { + t.Fatal("gateway rejection was reported as successful") + } + server.Close() + if err := purgeAccountProfile(context.Background(), hub.Gateway{Endpoint: "http://127.0.0.1:1"}, environment); err == nil { + t.Fatal("unreachable gateway was reported as successful") + } +} + +func TestCreatorGatewayBrowserOperationsValidateResponses(t *testing.T) { + response := map[string]string{ + "identity": `{"uid":"123"}`, + "messages": `{"status":"succeeded","history_source":"live","history_cursor":"next","account_uid":"123","messages":[]}`, + "get": `{"status":200,"body":"ok"}`, + "resolve": `{"url":"https://www.douyin.com/video/1"}`, + "media": `{"status":200,"content_type":"video/mp4","body_base64":"aGk="}`, + "login-qr": `{"content_type":"image/png","body_base64":"aGk=","qr_detected":true}`, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := "" + switch r.URL.Path { + case "/v1/browsers/account-1/douyin/identity": + key = "identity" + case "/v1/browsers/account-1/douyin/messages": + key = "messages" + case "/v1/browsers/account-1/douyin/get": + key = "get" + case "/v1/browsers/account-1/douyin/resolve": + key = "resolve" + case "/v1/browsers/account-1/douyin/media": + key = "media" + case "/v1/browsers/account-1/douyin/login-qr": + key = "login-qr" + default: + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(response[key])) + })) + defer server.Close() + + environment := testRunnableEnvironment() + browser := creatorGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL}, environment: environment} + if uid, err := browser.Identity(context.Background(), "douyin-uid"); err != nil || uid != "123" { + t.Fatalf("identity = %q, %v", uid, err) + } + history, err := browser.MessageHistory(context.Background(), "123", "456", "", 20) + if err != nil || history.HistoryCursor != "next" { + t.Fatalf("history = %#v, %v", history, err) + } + if got, err := browser.Get(context.Background(), "https://www.douyin.com/video/1"); err != nil || got.Status != http.StatusOK || string(got.Body) != "ok" || got.Challenge != douyin.ChallengeNone { + t.Fatalf("get = %#v, %v", got, err) + } + if got, err := browser.Resolve(context.Background(), "https://www.douyin.com/video/1"); err != nil || got == "" { + t.Fatalf("resolve = %q, %v", got, err) + } + destination := filepath.Join(t.TempDir(), "video.mp4") + if err := browser.Media(context.Background(), "https://www.douyin.com/video/1", destination); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(destination); err != nil || string(data) != "hi" { + t.Fatalf("media = %q, %v", data, err) + } + + douyinBrowser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "test-token"}, environment: environment} + qr, err := douyinBrowser.LoginQR(context.Background()) + if err != nil || qr.ContentType != "image/png" || !qr.QRDetected { + t.Fatalf("login QR = %#v, %v", qr, err) + } + if got, err := douyinBrowser.Get(context.Background(), "https://www.douyin.com/video/1"); err != nil || got.Status != http.StatusOK { + t.Fatalf("douyin get = %#v, %v", got, err) + } + if _, err := douyinBrowser.request(); err != nil { + t.Fatal(err) + } +} + +func TestWriteCreatorMediaRejectsInvalidAndPublishesAtomically(t *testing.T) { + if err := writeCreatorMedia("", []byte("data")); err == nil { + t.Fatal("empty destination was accepted") + } + if err := writeCreatorMedia(filepath.Join(t.TempDir(), "video"), nil); err == nil { + t.Fatal("empty media was accepted") + } + if err := writeCreatorMedia(filepath.Join(t.TempDir(), "video"), make([]byte, maxCreatorMediaBytes+1)); err == nil { + t.Fatal("oversized media was accepted") + } + if err := writeCreatorMedia(filepath.Join(t.TempDir(), "missing", "video"), []byte("data")); err == nil { + t.Fatal("unwritable media destination was accepted") + } + destination := filepath.Join(t.TempDir(), "video") + if err := writeCreatorMedia(destination, []byte("data")); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(destination) + if err != nil || string(data) != "data" { + t.Fatalf("published media = %q, err = %v", data, err) + } +} + +func TestDouyinWorkKeyFromURLValidatesCanonicalVideoURLs(t *testing.T) { + for _, testCase := range []struct { + value string + want string + valid bool + }{ + {value: "https://www.douyin.com/video/123", want: "123", valid: true}, + {value: "https://www.douyin.com/video/", valid: false}, + {value: "https://www.douyin.com/video/0", valid: false}, + {value: "https://v.douyin.com/video/123", valid: false}, + {value: "https://www.douyin.com/user/123", valid: false}, + {value: "https://www.douyin.com/video/abc", valid: false}, + } { + got, err := douyinWorkKeyFromURL(testCase.value) + if testCase.valid && (err != nil || got != testCase.want) { + t.Errorf("douyinWorkKeyFromURL(%q) = %q, %v", testCase.value, got, err) + } + if !testCase.valid && err == nil { + t.Errorf("douyinWorkKeyFromURL(%q) was accepted", testCase.value) + } + } +} + +func TestCreatorGatewayBrowserRejectsMalformedGatewayData(t *testing.T) { + response := `{"uid":""}` + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(response)) + })) + defer server.Close() + environment := testRunnableEnvironment() + browser := creatorGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL}, environment: environment} + if _, err := browser.Identity(context.Background(), "expected"); err == nil { + t.Fatal("empty identity was accepted") + } + status = http.StatusBadGateway + if _, err := browser.Identity(context.Background(), "expected"); err == nil { + t.Fatal("rejected identity was accepted") + } + status = http.StatusBadGateway + if _, err := browser.MessageHistory(context.Background(), "expected", "target", "", 1); err == nil { + t.Fatal("rejected message history was accepted") + } + if _, err := browser.Get(context.Background(), "target"); err == nil { + t.Fatal("rejected browser request was accepted") + } + if _, err := browser.Resolve(context.Background(), "target"); err == nil { + t.Fatal("rejected share resolution was accepted") + } + if err := browser.Media(context.Background(), "target", filepath.Join(t.TempDir(), "video")); err == nil { + t.Fatal("rejected media request was accepted") + } + douyinBrowser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "test-token"}, environment: environment} + if _, err := douyinBrowser.LoginQR(context.Background()); err == nil { + t.Fatal("rejected login QR request was accepted") + } + if _, err := douyinBrowser.Get(context.Background(), "target"); err == nil { + t.Fatal("rejected native browser request was accepted") + } + status = http.StatusOK + for _, args := range [][4]any{ + {"", "target", "", 1}, + {"expected", "", "", 1}, + {"expected", "target", "cursor", 0}, + {"expected", "target", "cursor", 201}, + } { + if _, err := browser.MessageHistory(context.Background(), args[0].(string), args[1].(string), args[2].(string), args[3].(int)); err == nil { + t.Fatalf("invalid history args %#v were accepted", args) + } + } + + response = `{` + if _, err := browser.Get(context.Background(), "target"); err == nil { + t.Fatal("malformed get response was accepted") + } + if _, err := browser.Resolve(context.Background(), "target"); err == nil { + t.Fatal("malformed resolve response was accepted") + } + if err := browser.Media(context.Background(), "target", filepath.Join(t.TempDir(), "video")); err == nil { + t.Fatal("malformed media response was accepted") + } + + if _, err := douyinBrowser.LoginQR(context.Background()); err == nil { + t.Fatal("malformed login QR response was accepted") + } + invalid := environment + invalid.RuntimeID = "" + if _, err := (douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL}, environment: invalid}).request(); err == nil { + t.Fatal("non-running environment was accepted") + } + + response = `{"status":200,"body":"ok","challenge":"unexpected"}` + if _, err := douyinBrowser.Get(context.Background(), "target"); err == nil { + t.Fatal("invalid challenge was accepted") + } + response = `{"content_type":"text/plain","body_base64":"aGk="}` + if _, err := douyinBrowser.LoginQR(context.Background()); err == nil { + t.Fatal("non-image login QR was accepted") + } + response = `{"content_type":"image/png","body_base64":"!!!"}` + if _, err := douyinBrowser.LoginQR(context.Background()); err == nil { + t.Fatal("invalid login QR base64 was accepted") + } + response = `{"content_type":"image/png","body_base64":""}` + if _, err := douyinBrowser.LoginQR(context.Background()); err == nil { + t.Fatal("empty login QR was accepted") + } + response = `{"status":200,"content_type":"text/plain","body_base64":"aGk="}` + if err := browser.Media(context.Background(), "target", filepath.Join(t.TempDir(), "video")); err == nil { + t.Fatal("non-video media was accepted") + } + response = `{"status":200,"content_type":"video/mp4","body_base64":"!!!"}` + if err := browser.Media(context.Background(), "target", filepath.Join(t.TempDir(), "video")); err == nil { + t.Fatal("invalid media base64 was accepted") + } + +} diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index 5055e18..89dce28 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -19,17 +19,17 @@ import ( // hubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。 type hubStore interface { - LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error) + LockResources(ctx context.Context, aliases, exitIDs, browserVersions []string) (func(), error) CreateGateway(ctx context.Context, name, endpoint, token string) (hub.Gateway, error) UpdateGateway(ctx context.Context, currentName, name, endpoint, token string) (hub.Gateway, error) ListGateways(ctx context.Context) ([]hub.Gateway, error) GetGateway(ctx context.Context, name string) (hub.Gateway, error) DeleteGateway(ctx context.Context, name string) error - CreateImage(ctx context.Context, image hub.Image) error - UpdateImage(ctx context.Context, image hub.Image) error - ListImages(ctx context.Context, enabledOnly bool) ([]hub.Image, error) - DeleteImage(ctx context.Context, version string) error - ImageRef(ctx context.Context, version string) (string, error) + CreateBrowserVersion(ctx context.Context, version hub.BrowserVersion) error + UpdateBrowserVersion(ctx context.Context, version hub.BrowserVersion) error + ListBrowserVersions(ctx context.Context, enabledOnly bool) ([]hub.BrowserVersion, error) + DeleteBrowserVersion(ctx context.Context, version string) error + BrowserPath(ctx context.Context, version string) (string, error) CreateEnv(ctx context.Context, env hub.Env) error ListEnvs(ctx context.Context) ([]hub.Env, error) GetEnv(ctx context.Context, alias string) (hub.Env, error) @@ -58,7 +58,7 @@ type networkExitAdminStore interface { } type runtimeStopStore interface { - LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error) + LockResources(ctx context.Context, aliases, exitIDs, browserVersions []string) (func(), error) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) GetGateway(ctx context.Context, name string) (hub.Gateway, error) ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error @@ -77,7 +77,14 @@ const ( missingRuntimeID = "runtime-not-found" ) -var gatewayGenerationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`) +var ( + // Runtime and network identifiers are opaque, gateway-issued generations. + // Their exact value is fenced and persisted; only control characters and + // oversized values are rejected here. + gatewayGenerationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`) + gatewayRuntimeIDPattern = gatewayGenerationIDPattern + gatewayNetworkIDPattern = gatewayGenerationIDPattern +) const ( defaultGatewayResponseLimit = 1 << 20 @@ -117,25 +124,28 @@ func gatewayCallWithLimit(ctx context.Context, target hub.Gateway, method, path return response.StatusCode, responseBody, err } -func gatewayCreatePayload(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any { +func gatewayCreatePayload(environment hub.EnvironmentContext, _ string, networkExit gatewayNetworkExit) map[string]any { fingerprint := environment.Fingerprint fingerprint.ProxyServer = "" fingerprint.DisableNonProxiedUDP = false - cmd := append(fingerprint.Args(), "--remote-allow-origins=*", "about:blank") + profileID := environment.AccountID + if profileID == "" { + profileID = environment.Alias + } return map[string]any{ "alias": environment.Alias, "name": environment.Name, - "image": imageRef, - "cmd": cmd, - "volume": "creatorhub-profile-" + environment.Alias, + "browser_version": environment.BrowserVersion, + "profile_id": profileID, + "cmd": append(fingerprint.Args(), "about:blank"), "binding_version": environment.BindingVersion, "network_exit_id": environment.Exit.ID, "network_exit": networkExit, } } -func gatewayCreatePayloadForAccount(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any { - payload := gatewayCreatePayload(environment, imageRef, networkExit) +func gatewayCreatePayloadForAccount(environment hub.EnvironmentContext, browserPath string, networkExit gatewayNetworkExit) map[string]any { + payload := gatewayCreatePayload(environment, browserPath, networkExit) if !accountRunnable(environment) { payload["network_exit_id"], payload["network_exit"], payload["stopped"] = "", gatewayNetworkExit{}, true } @@ -179,7 +189,7 @@ func gatewayNetworkID(body []byte) string { var envelope struct { NetworkID string `json:"network_id"` } - if json.Unmarshal(body, &envelope) == nil && gatewayGenerationIDPattern.MatchString(envelope.NetworkID) { + if json.Unmarshal(body, &envelope) == nil && gatewayNetworkIDPattern.MatchString(envelope.NetworkID) { return envelope.NetworkID } return "" @@ -187,7 +197,7 @@ func gatewayNetworkID(body []byte) string { func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext, status int, callErr error, body []byte) error { - // 网关 4xx 拒绝且未产生 network 代：创建在网关侧任何 Docker 变更之前就被确定性拒绝， + // 网关 4xx 拒绝且未产生 network 代：创建在 gateway 侧任何 runtime 变更之前就被确定性拒绝， // 无需清理 fence；其余情况（5xx、断连、无效 201）结果未知，仍走 fail-closed。 deterministicRejection := callErr == nil && status >= 400 && status < 500 logEntry := logrus.WithFields(logrus.Fields{ @@ -204,7 +214,7 @@ func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, targ logEntry = logEntry.WithError(callErr) } if deterministicRejection { - logEntry.Info("gateway rejected runtime create before Docker side effects") + logEntry.Info("gateway rejected runtime create before native runtime side effects") } else { logEntry.Warn("gateway runtime create result requires reconciliation") } @@ -223,7 +233,7 @@ func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupS pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "") return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true)) } - container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias) + runtime, found, err := reconcileGatewayRuntime(ctx, target, environment.Alias) if err != nil { runtimeID := missingRuntimeID if bindingVersion == environment.BindingVersion && environment.RuntimeID != "" { @@ -232,12 +242,12 @@ func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupS pending := runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID) return errors.Join(err, store.SetRuntimeCleanupPending(ctx, pending, true)) } - if found && (container.BindingVersion != bindingVersion || container.NetworkID != networkID) { + if found && (runtime.BindingVersion != bindingVersion || runtime.NetworkID != networkID) { pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID) return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true)) } if found { - environment = runtimeCleanupGeneration(environment, bindingVersion, container.ID, networkID) + environment = runtimeCleanupGeneration(environment, bindingVersion, runtime.ID, networkID) } else { environment = runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID) } @@ -264,7 +274,7 @@ func gatewayUnreachable(err error) error { return gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf("gateway unreachable: %v", err)} } -func reconcileGatewayContainer(ctx context.Context, target hub.Gateway, alias string) (containerStatus, bool, error) { +func reconcileGatewayRuntime(ctx context.Context, target hub.Gateway, alias string) (runtimeStatus, bool, error) { var lastErr error for attempt := 0; attempt < gatewayReconcileAttempts; attempt++ { if attempt > 0 { @@ -272,7 +282,7 @@ func reconcileGatewayContainer(ctx context.Context, target hub.Gateway, alias st select { case <-ctx.Done(): timer.Stop() - return containerStatus{}, false, ctx.Err() + return runtimeStatus{}, false, ctx.Err() case <-timer.C: } } @@ -297,18 +307,18 @@ func reconcileGatewayContainer(ctx context.Context, target hub.Gateway, alias st } } } - return containerStatus{}, false, lastErr + return runtimeStatus{}, false, lastErr } -func parseGatewayBrowserList(body []byte) ([]containerStatus, error) { +func parseGatewayBrowserList(body []byte) ([]runtimeStatus, error) { var entries []json.RawMessage if err := json.Unmarshal(body, &entries); err != nil || entries == nil { return nil, errors.New("gateway returned an invalid browser list") } - browsers := make([]containerStatus, 0, len(entries)) + browsers := make([]runtimeStatus, 0, len(entries)) aliases := make(map[string]bool, len(entries)) for _, entry := range entries { - var browser *containerStatus + var browser *runtimeStatus if err := json.Unmarshal(entry, &browser); err != nil || browser == nil || browser.ID == "" || browser.Alias == "" || browser.State == "" || browser.BindingVersion < 0 || aliases[browser.Alias] { return nil, errors.New("gateway returned an invalid browser list") @@ -329,7 +339,7 @@ func errorFromBody(body []byte, status int) string { return http.StatusText(status) } -type containerStatus struct { +type runtimeStatus struct { ID string `json:"id"` Alias string `json:"alias"` Name string `json:"name"` @@ -339,6 +349,9 @@ type containerStatus struct { BindingVersion int64 `json:"binding_version"` NetworkExitID string `json:"network_exit_id"` NetworkID string `json:"network_id"` + NodeID string `json:"node_id"` + CleanupState string `json:"cleanup_state"` + CleanupError string `json:"cleanup_error"` ProxyReady bool `json:"proxy_ready"` } @@ -346,7 +359,7 @@ type envView struct { hub.Env State string `json:"state"` Status string `json:"status"` - ContainerID string `json:"container_id"` + RuntimeID string `json:"runtime_id"` Endpoint string `json:"endpoint"` AccountID string `json:"account_id"` AccountStatus string `json:"account_status"` @@ -355,6 +368,11 @@ type envView struct { NetworkExitHealth string `json:"network_exit_health"` BindingVersion int64 `json:"binding_version"` RuntimeInstanceID string `json:"runtime_instance_id"` + RuntimeNodeID string `json:"runtime_node_id,omitempty"` + RuntimeStatus string `json:"runtime_status,omitempty"` + RuntimeCleanupState string `json:"runtime_cleanup_state,omitempty"` + RuntimeCleanupError string `json:"runtime_cleanup_error,omitempty"` + GatewayReachable bool `json:"gateway_reachable"` ScheduleStatus string `json:"schedule_status"` ScheduleBlockReason string `json:"schedule_block_reason,omitempty"` RecoveryRequired bool `json:"recovery_required"` @@ -378,17 +396,17 @@ func environmentScheduleReadiness(environment hub.EnvironmentContext) (string, s } } -func containerMatchesBinding(container containerStatus, environment hub.EnvironmentContext) bool { - if !validCreatedRuntime(container, environment, container.State == "running") || container.BindingVersion != environment.BindingVersion || - (container.NetworkExitID != environment.Exit.ID && (container.State == "running" || container.NetworkExitID != "")) { +func runtimeMatchesBinding(runtime runtimeStatus, environment hub.EnvironmentContext) bool { + if !validCreatedRuntime(runtime, environment, runtime.State == "running") || runtime.BindingVersion != environment.BindingVersion || + (runtime.NetworkExitID != environment.Exit.ID && (runtime.State == "running" || runtime.NetworkExitID != "")) { return false } return true } -func validCreatedRuntime(created containerStatus, environment hub.EnvironmentContext, running bool) bool { - if !gatewayGenerationIDPattern.MatchString(created.ID) || - (created.NetworkID != "" && !gatewayGenerationIDPattern.MatchString(created.NetworkID)) { +func validCreatedRuntime(created runtimeStatus, environment hub.EnvironmentContext, running bool) bool { + if !gatewayRuntimeIDPattern.MatchString(created.ID) || + (created.NetworkID != "" && !gatewayNetworkIDPattern.MatchString(created.NetworkID)) { return false } if !running { @@ -484,19 +502,19 @@ func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitPro return c.SendStatus(fiber.StatusNoContent) }) - app.Get("/api/browser-images", func(c fiber.Ctx) error { - images, err := store.ListImages(c.Context(), false) + app.Get("/api/browser-versions", func(c fiber.Ctx) error { + versions, err := store.ListBrowserVersions(c.Context(), false) if err != nil { return hubError(c, err) } - return c.JSON(images) + return c.JSON(versions) }) - app.Post("/api/browser-images", func(c fiber.Ctx) error { + app.Post("/api/browser-versions", func(c fiber.Ctx) error { input := struct { - Version string `json:"version"` - ImageRef string `json:"image_ref"` - Note string `json:"note"` - Enabled *bool `json:"enabled"` + Version string `json:"version"` + BrowserPath string `json:"browser_path"` + Note string `json:"note"` + Enabled *bool `json:"enabled"` }{} if err := decodeHubJSON(c, &input); err != nil { return hubError(c, err) @@ -505,18 +523,18 @@ func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitPro if input.Enabled != nil { enabled = *input.Enabled } - if err := store.CreateImage(c.Context(), hub.Image{Version: input.Version, ImageRef: input.ImageRef, Note: input.Note, Enabled: enabled}); err != nil { + if err := store.CreateBrowserVersion(c.Context(), hub.BrowserVersion{Version: input.Version, BrowserPath: input.BrowserPath, Note: input.Note, Enabled: enabled}); err != nil { return hubError(c, err) } return c.Status(fiber.StatusCreated).JSON(map[string]any{ - "version": input.Version, "image_ref": input.ImageRef, "note": input.Note, "enabled": enabled, + "version": input.Version, "browser_path": input.BrowserPath, "note": input.Note, "enabled": enabled, }) }) - app.Put("/api/browser-images/:version", lockBrowserImage(store, func(c fiber.Ctx) error { + app.Put("/api/browser-versions/:version", lockBrowserVersion(store, func(c fiber.Ctx) error { input := struct { - ImageRef string `json:"image_ref"` - Note string `json:"note"` - Enabled *bool `json:"enabled"` + BrowserPath string `json:"browser_path"` + Note string `json:"note"` + Enabled *bool `json:"enabled"` }{} if err := decodeHubJSON(c, &input); err != nil { return hubError(c, err) @@ -525,13 +543,13 @@ func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitPro if input.Enabled != nil { enabled = *input.Enabled } - if err := store.UpdateImage(c.Context(), hub.Image{Version: c.Params("version"), ImageRef: input.ImageRef, Note: input.Note, Enabled: enabled}); err != nil { + if err := store.UpdateBrowserVersion(c.Context(), hub.BrowserVersion{Version: c.Params("version"), BrowserPath: input.BrowserPath, Note: input.Note, Enabled: enabled}); err != nil { return hubError(c, err) } return c.SendStatus(fiber.StatusNoContent) })) - app.Delete("/api/browser-images/:version", lockBrowserImage(store, func(c fiber.Ctx) error { - if err := store.DeleteImage(c.Context(), c.Params("version")); err != nil { + app.Delete("/api/browser-versions/:version", lockBrowserVersion(store, func(c fiber.Ctx) error { + if err := store.DeleteBrowserVersion(c.Context(), c.Params("version")); err != nil { return hubError(c, err) } return c.SendStatus(fiber.StatusNoContent) @@ -588,13 +606,44 @@ func getBrowser(store hubStore) fiber.Handler { if err != nil { return hubError(c, err) } - return c.JSON(environment) + view := envView{ + Env: environment.Env, State: "recorded", Status: "已记录运行实例", + AccountID: environment.AccountID, AccountStatus: environment.AccountStatus, + AuthorizationStatus: environment.AuthorizationStatus, NetworkExitID: environment.Exit.ID, + NetworkExitHealth: environment.Exit.HealthStatus, BindingVersion: environment.BindingVersion, + RuntimeInstanceID: environment.RuntimeInstanceID, RuntimeID: environment.RuntimeID, RuntimeNodeID: environment.RuntimeNodeID, + ScheduleStatus: "blocked", ScheduleBlockReason: "binding_missing", + RecoveryRequired: true, CleanupPending: environment.RuntimeCleanupPending, + } + view.ScheduleStatus, view.ScheduleBlockReason = environmentScheduleReadiness(environment) + if environment.RuntimeInstanceID == "" { + view.State, view.Status = "missing", "未记录运行实例" + } + runtimes, gatewayRead, gatewayErrors := gatewayRuntimeSnapshot(c.Context(), store, []hub.Env{environment.Env}) + if gatewayErrors[environment.Gateway] != nil { + view.State, view.Status, view.GatewayReachable = "gateway_unreachable", "网关不可达", false + } else if gatewayRead[environment.Gateway] { + view.GatewayReachable = true + if runtime, found := runtimes[environment.Gateway][environment.Alias]; found { + view.RuntimeID, view.RuntimeStatus, view.RuntimeNodeID = runtime.ID, runtime.Status, runtime.NodeID + view.State, view.Status = runtime.State, runtime.Status + view.RuntimeCleanupState, view.RuntimeCleanupError = runtime.CleanupState, runtime.CleanupError + view.CleanupPending = view.CleanupPending || runtime.CleanupState == "pending" + view.RecoveryRequired = view.RecoveryRequired || runtime.State != "running" || runtime.CleanupState == "pending" + if runtime.State == "running" && runtime.Status == "ready" { + view.Endpoint = runtime.Endpoint + } + } else if view.RuntimeInstanceID != "" { + view.State, view.Status = "runtime_missing", "网关中未找到运行实例" + } + } + return c.JSON(view) } } func lockBrowserAlias(store hubStore, handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { - requestedExitID, imageVersion := "", "" + requestedExitID, browserVersion := "", "" if c.Params("action") == "rebind" { var input struct { NetworkExitID string `json:"network_exit_id"` @@ -610,13 +659,13 @@ func lockBrowserAlias(store hubStore, handler fiber.Handler) fiber.Handler { Version string `json:"version"` } if json.Unmarshal(c.Body(), &input) == nil { - if hub.ValidImageVersion(input.Version) { - imageVersion = input.Version + if hub.ValidBrowserVersion(input.Version) { + browserVersion = input.Version } } } alias := c.Params("alias") - unlock, err := lockAliasResources(c.Context(), store, alias, requestedExitID, imageVersion) + unlock, err := lockAliasResources(c.Context(), store, alias, requestedExitID, browserVersion) if err != nil { return hubError(c, err) } @@ -625,7 +674,7 @@ func lockBrowserAlias(store hubStore, handler fiber.Handler) fiber.Handler { } } -func lockBrowserImage(store hubStore, handler fiber.Handler) fiber.Handler { +func lockBrowserVersion(store hubStore, handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { unlock, err := store.LockResources(c.Context(), nil, nil, []string{c.Params("version")}) if err != nil { @@ -639,22 +688,22 @@ func lockBrowserImage(store hubStore, handler fiber.Handler) fiber.Handler { func lockBrowserCreate(store hubStore, handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { var input struct { - Alias string `json:"alias"` - AccountID string `json:"account_id"` - NetworkExitID string `json:"network_exit_id"` - ImageVersion string `json:"image_version"` + Alias string `json:"alias"` + AccountID string `json:"account_id"` + NetworkExitID string `json:"network_exit_id"` + BrowserVersion string `json:"browser_version"` } if json.Unmarshal(c.Body(), &input) != nil || input.Alias == "" { return handler(c) } - exitIDs, imageVersions := []string(nil), []string(nil) + exitIDs, browserVersions := []string(nil), []string(nil) if hub.ValidNetworkExitID(input.NetworkExitID) { exitIDs = []string{input.NetworkExitID} } - if hub.ValidImageVersion(input.ImageVersion) { - imageVersions = []string{input.ImageVersion} + if hub.ValidBrowserVersion(input.BrowserVersion) { + browserVersions = []string{input.BrowserVersion} } - unlock, err := store.LockResources(c.Context(), nonEmpty(input.Alias, input.AccountID), exitIDs, imageVersions) + unlock, err := store.LockResources(c.Context(), nonEmpty(input.Alias, input.AccountID), exitIDs, browserVersions) if err != nil { return hubError(c, err) } @@ -677,8 +726,8 @@ func lockEnvironmentResources(ctx context.Context, store hubStore, envs []hub.En if err != nil { return nil, err } - exitIDs, imageVersions := environmentResourceValues(before) - unlock, err := store.LockResources(ctx, aliases, exitIDs, imageVersions) + exitIDs, browserVersions := environmentResourceValues(before) + unlock, err := store.LockResources(ctx, aliases, exitIDs, browserVersions) if err != nil { return nil, err } @@ -695,23 +744,23 @@ func lockEnvironmentResources(ctx context.Context, store hubStore, envs []hub.En } type environmentResource struct { - alias string - exitID string - imageVersion string + alias string + exitID string + browserVersion string } type resourceLockStore interface { - LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error) + LockResources(ctx context.Context, aliases, exitIDs, browserVersions []string) (func(), error) GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) } -func lockAliasResources(ctx context.Context, store resourceLockStore, alias, requestedExitID, requestedImageVersion string) (func(), error) { +func lockAliasResources(ctx context.Context, store resourceLockStore, alias, requestedExitID, requestedBrowserVersion string) (func(), error) { for { before, beforeFound, err := environmentResources(ctx, store, alias) if err != nil { return nil, err } - unlock, err := store.LockResources(ctx, []string{alias}, nonEmpty(before.exitID, requestedExitID), nonEmpty(before.imageVersion, requestedImageVersion)) + unlock, err := store.LockResources(ctx, []string{alias}, nonEmpty(before.exitID, requestedExitID), nonEmpty(before.browserVersion, requestedBrowserVersion)) if err != nil { return nil, err } @@ -736,7 +785,7 @@ func lockAccountResources(ctx context.Context, store runtimeStopStore, accountID if err != nil { return nil, err } - unlock, err := store.LockResources(ctx, nonEmpty(accountID, before.alias), nonEmpty(before.exitID), nonEmpty(before.imageVersion)) + unlock, err := store.LockResources(ctx, nonEmpty(accountID, before.alias), nonEmpty(before.exitID), nonEmpty(before.browserVersion)) if err != nil { return nil, err } @@ -760,7 +809,7 @@ func accountEnvironmentResources(ctx context.Context, store runtimeStopStore, ac if err != nil { return environmentResource{}, false, err } - return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, imageVersion: environment.ImageVersion}, true, nil + return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, browserVersion: environment.BrowserVersion}, true, nil } func environmentResources(ctx context.Context, store resourceLockStore, alias string) (environmentResource, bool, error) { @@ -771,7 +820,7 @@ func environmentResources(ctx context.Context, store resourceLockStore, alias st if err != nil { return environmentResource{}, false, err } - return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, imageVersion: environment.ImageVersion}, true, nil + return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, browserVersion: environment.BrowserVersion}, true, nil } func environmentResourceMap(ctx context.Context, store hubStore, aliases []string) (map[string]environmentResource, error) { @@ -801,12 +850,12 @@ func nonEmpty(values ...string) []string { } func environmentResourceValues(values map[string]environmentResource) ([]string, []string) { - exitIDs, imageVersions := make([]string, 0, len(values)), make([]string, 0, len(values)) + exitIDs, browserVersions := make([]string, 0, len(values)), make([]string, 0, len(values)) for _, value := range values { exitIDs = append(exitIDs, value.exitID) - imageVersions = append(imageVersions, value.imageVersion) + browserVersions = append(browserVersions, value.browserVersion) } - return nonEmpty(exitIDs...), nonEmpty(imageVersions...) + return nonEmpty(exitIDs...), nonEmpty(browserVersions...) } func equalEnvironmentResourceMaps(left, right map[string]environmentResource) bool { @@ -938,23 +987,50 @@ func listBrowsers(store hubStore) fiber.Handler { if err != nil { return hubError(c, err) } + runtimes, gatewayRead, gatewayErrors := gatewayRuntimeSnapshot(c.Context(), store, envs) views := make([]envView, 0, len(envs)) for _, env := range envs { - view := envView{Env: env, State: "missing", Status: "未记录运行实例", ScheduleStatus: "blocked", ScheduleBlockReason: "binding_missing", RecoveryRequired: true} + view := envView{ + Env: env, State: "missing", Status: "未记录运行实例", + ScheduleStatus: "blocked", ScheduleBlockReason: "binding_missing", + RecoveryRequired: true, GatewayReachable: false, + } if environment, contextErr := store.GetEnvironmentContext(c.Context(), env.Alias); contextErr == nil { view.AccountID, view.AccountStatus, view.AuthorizationStatus = environment.AccountID, environment.AccountStatus, environment.AuthorizationStatus view.NetworkExitID, view.NetworkExitHealth, view.BindingVersion = environment.Exit.ID, environment.Exit.HealthStatus, environment.BindingVersion view.RuntimeInstanceID = environment.RuntimeInstanceID - view.ContainerID = environment.RuntimeID + view.RuntimeID = environment.RuntimeID + view.RuntimeNodeID = environment.RuntimeNodeID view.ScheduleStatus, view.ScheduleBlockReason = environmentScheduleReadiness(environment) view.CleanupPending = environment.RuntimeCleanupPending view.RecoveryRequired = (environment.Exit.ID != "" && environment.Exit.HealthStatus != "healthy") || environment.RuntimeCleanupPending - if environment.RuntimeInstanceID != "" { - view.State, view.Status = "running", "已记录运行实例" + if environment.RuntimeCleanupPending { + view.State, view.Status = "cleanup_pending", "运行资源待清理" + } else if environment.RuntimeInstanceID != "" { + view.State, view.Status = "recorded", "已记录运行实例" } } else if !errors.Is(contextErr, hub.ErrNotFound) { return hubError(c, contextErr) } + if gatewayErrors[env.Gateway] != nil { + view.State, view.Status = "gateway_unreachable", "网关不可达" + view.RecoveryRequired = true + } else if gatewayRead[env.Gateway] { + view.GatewayReachable = true + if runtime, found := runtimes[env.Gateway][env.Alias]; found { + view.RuntimeID, view.RuntimeStatus, view.RuntimeNodeID = runtime.ID, runtime.Status, runtime.NodeID + view.State, view.Status = runtime.State, runtime.Status + view.RuntimeCleanupState, view.RuntimeCleanupError = runtime.CleanupState, runtime.CleanupError + view.CleanupPending = view.CleanupPending || runtime.CleanupState == "pending" + view.RecoveryRequired = view.RecoveryRequired || runtime.State != "running" || runtime.CleanupState == "pending" + if runtime.State == "running" && runtime.Status == "ready" { + view.Endpoint = runtime.Endpoint + } + } else if view.RuntimeInstanceID != "" { + view.State, view.Status = "runtime_missing", "网关中未找到运行实例" + view.RecoveryRequired = true + } + } views = append(views, view) } return c.JSON(views) @@ -975,13 +1051,13 @@ func reconcileRuntimeLeasesUnlocked(ctx context.Context, store hubStore, probe n return err } defer unlock() - containers, gatewayRead, gatewayErrors := gatewayContainerSnapshot(ctx, store, envs) - return reconcileRuntimeSnapshot(ctx, store, probe, resolve, envs, containers, gatewayRead, gatewayErrors) + runtimes, gatewayRead, gatewayErrors := gatewayRuntimeSnapshot(ctx, store, envs) + return reconcileRuntimeSnapshot(ctx, store, probe, resolve, envs, runtimes, gatewayRead, gatewayErrors) } -func gatewayContainerSnapshot(ctx context.Context, store hubStore, envs []hub.Env) (map[string]map[string]containerStatus, map[string]bool, map[string]error) { +func gatewayRuntimeSnapshot(ctx context.Context, store hubStore, envs []hub.Env) (map[string]map[string]runtimeStatus, map[string]bool, map[string]error) { gateways := map[string]bool{} - containers := map[string]map[string]containerStatus{} + runtimes := map[string]map[string]runtimeStatus{} gatewayRead := map[string]bool{} gatewayErrors := map[string]error{} for _, env := range envs { @@ -1008,18 +1084,18 @@ func gatewayContainerSnapshot(ctx context.Context, store hubStore, envs []hub.En gatewayErrors[env.Gateway] = gatewayFailure{status: http.StatusBadGateway, message: "gateway returned an invalid browser list"} continue } - byAlias := map[string]containerStatus{} - for _, container := range list { - byAlias[container.Alias] = container + byAlias := map[string]runtimeStatus{} + for _, runtime := range list { + byAlias[runtime.Alias] = runtime } - containers[env.Gateway] = byAlias + runtimes[env.Gateway] = byAlias gatewayRead[env.Gateway] = true } - return containers, gatewayRead, gatewayErrors + return runtimes, gatewayRead, gatewayErrors } func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), envs []hub.Env, - containers map[string]map[string]containerStatus, gatewayRead map[string]bool, gatewayErrors map[string]error) error { + runtimes map[string]map[string]runtimeStatus, gatewayRead map[string]bool, gatewayErrors map[string]error) error { var firstErr error for _, env := range envs { if readErr := gatewayErrors[env.Gateway]; readErr != nil { @@ -1045,9 +1121,9 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network } continue } - container, found := containers[env.Gateway][env.Alias] + runtime, found := runtimes[env.Gateway][env.Alias] if !accountRunnable(environment) { - if found && container.State == "running" { + if found && runtime.State == "running" { if err := stopEnvironmentRuntime(ctx, store, environment); err != nil { return err } @@ -1058,15 +1134,15 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network } continue } - if found && container.State == "running" { - auditRecovery := !containerMatchesBinding(container, environment) || !container.ProxyReady + if found && runtime.State == "running" { + auditRecovery := !runtimeMatchesBinding(runtime, environment) || !runtime.ProxyReady action := actionForEnvironment("reconcile", environment) if auditRecovery { if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { return err } } - ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, container) + ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, runtime) if auditRecovery { current, contextErr := store.GetEnvironmentContext(ctx, environment.Alias) if contextErr == nil { @@ -1113,7 +1189,7 @@ func runtimeRecoveryFailure(ctx context.Context, store hubStore, alias string, e } func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkExitProbe, - _ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, container containerStatus) (bool, error) { + _ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, runtime runtimeStatus) (bool, error) { if environment.RuntimeCleanupPending { target, err := store.GetGateway(ctx, environment.Gateway) if err != nil { @@ -1134,26 +1210,26 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE if err != nil { return false, err } - if containerMatchesBinding(container, environment) && container.ProxyReady { - _, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID) + if runtimeMatchesBinding(runtime, environment) && runtime.ProxyReady { + _, err := activateGatewayRuntime(ctx, store, target, environment, runtime.ID, runtime.NetworkID, runtime.NodeID) return err == nil, err } networkExit := gatewayNetworkExit{} if environment.Exit.ID != "" { networkExit = gatewayNetworkExitFor(access) } - if containerMatchesBinding(container, environment) { + if runtimeMatchesBinding(runtime, environment) { if environment.Exit.ID == "" { - container.ProxyReady = true + runtime.ProxyReady = true } else { status, _, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/proxy", - gatewayProxyPayload(environment, container.ID, container.NetworkID, networkExit), 30*time.Second) + gatewayProxyPayload(environment, runtime.ID, runtime.NetworkID, networkExit), 30*time.Second) if callErr == nil && status == http.StatusNoContent { - container.ProxyReady = true + runtime.ProxyReady = true } } - if container.ProxyReady { - _, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID) + if runtime.ProxyReady { + _, err := activateGatewayRuntime(ctx, store, target, environment, runtime.ID, runtime.NetworkID, runtime.NodeID) return err == nil, err } } @@ -1165,35 +1241,61 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE if err != nil { return false, err } - imageRef, err := store.ImageRef(ctx, environment.ImageVersion) + browserPath, err := store.BrowserPath(ctx, environment.BrowserVersion) if err != nil { return false, err } status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", - gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) + gatewayCreatePayloadForAccount(environment, browserPath, networkExit), gatewayLongTimeout) if callErr != nil { return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body)) } if status != http.StatusCreated { return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body)) } - var created containerStatus + var created runtimeStatus if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) { return false, errors.Join(errors.New("gateway returned an invalid runtime generation"), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body)) } - _, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID) + _, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID, created.NodeID) return err == nil, err } -func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string) (hub.EnvironmentContext, error) { - if !validCreatedRuntime(containerStatus{ID: runtimeID, NetworkID: networkID}, environment, true) { +type runtimeNodeStore interface { + SetRuntimeNode(context.Context, string, string, string) error +} + +func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string, nodeIDs ...string) (hub.EnvironmentContext, error) { + if !validCreatedRuntime(runtimeStatus{ID: runtimeID, NetworkID: networkID}, environment, true) { return hub.EnvironmentContext{}, hub.ErrConflict } current, err := store.ActivateRuntime(ctx, environment.Alias, runtimeID, environment.BindingVersion, environment.Exit.ID, networkID) + if err == nil && len(nodeIDs) > 0 && nodeIDs[0] != "" { + if recorder, ok := store.(runtimeNodeStore); ok { + if nodeErr := recorder.SetRuntimeNode(ctx, environment.Alias, runtimeID, nodeIDs[0]); nodeErr != nil { + err = nodeErr + } else { + current.RuntimeNodeID = nodeIDs[0] + } + } + } if err == nil { return current, nil } + // A scheduler heartbeat can win the activation race after the action has + // created the runtime. If it persisted this exact generation, keep the + // runtime and treat the activation as successful instead of deleting the + // winner from the gateway. + if errors.Is(err, hub.ErrConflict) { + if reconciled, readErr := store.GetEnvironmentContext(ctx, environment.Alias); readErr == nil && + reconciled.BindingVersion == environment.BindingVersion && + reconciled.RuntimeID == runtimeID && + reconciled.RuntimeNetworkID == networkID && + reconciled.RuntimeInstanceID != "" { + return reconciled, nil + } + } cleanup := environment cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtimeID, networkID _, cleanupErr := removeGatewayRuntime(ctx, store, target, cleanup) @@ -1212,13 +1314,13 @@ func discardRuntime(ctx context.Context, store hubStore, environment hub.Environ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { input := struct { - Alias string `json:"alias"` - Name string `json:"name"` - Gateway string `json:"gateway"` - ImageVersion string `json:"image_version"` - Fingerprint hub.Fingerprint `json:"fingerprint"` - AccountID string `json:"account_id"` - NetworkExitID string `json:"network_exit_id"` + Alias string `json:"alias"` + Name string `json:"name"` + Gateway string `json:"gateway"` + BrowserVersion string `json:"browser_version"` + Fingerprint hub.Fingerprint `json:"fingerprint"` + AccountID string `json:"account_id"` + NetworkExitID string `json:"network_exit_id"` }{} if err := decodeHubJSON(c, &input); err != nil { return hubError(c, err) @@ -1227,7 +1329,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return hubError(c, hub.ErrInvalid) } input.Fingerprint.DisableNonProxiedUDP = false - env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, ImageVersion: input.ImageVersion, Fingerprint: input.Fingerprint} + env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, BrowserVersion: input.BrowserVersion, Fingerprint: input.Fingerprint} if err := env.Fingerprint.Validate(); err != nil { return c.Status(fiber.StatusBadRequest).JSON(map[string]string{"error": err.Error()}) } @@ -1269,9 +1371,9 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw } } running := accountRunnable(environment) - imageRef, err := store.ImageRef(c.Context(), env.ImageVersion) + browserPath, err := store.BrowserPath(c.Context(), env.BrowserVersion) if err != nil { - _ = finish("failed", "image_unavailable", environment) + _ = finish("failed", "browser_version_unavailable", environment) return hubError(c, err) } networkExit := gatewayNetworkExit{} @@ -1279,19 +1381,19 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw networkExit = gatewayNetworkExitFor(access) } if !created { - container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, env.Alias) + runtime, found, reconcileErr := reconcileGatewayRuntime(c.Context(), gateway, env.Alias) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, gatewayUnreachable(reconcileErr)) } if found && !running { - if !containerMatchesBinding(container, environment) { + if !runtimeMatchesBinding(runtime, environment) { _ = finish("failed", "runtime_unavailable", environment) return hubError(c, hub.ErrConflict) } - if container.State == "running" { + if runtime.State == "running" { stopped := environment - stopped.RuntimeID, stopped.RuntimeNetworkID = container.ID, container.NetworkID + stopped.RuntimeID, stopped.RuntimeNetworkID = runtime.ID, runtime.NetworkID if err := stopEnvironmentRuntime(c.Context(), store, stopped); err != nil { _ = finish("unknown", "cleanup_result_unknown", environment) return hubError(c, err) @@ -1302,8 +1404,8 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw } return c.JSON(map[string]string{"alias": env.Alias}) } - if found && container.State == "running" { - ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, container) + if found && runtime.State == "running" { + ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, runtime) if restoreErr != nil { outcome, reason := runtimeRecoveryFailure(c.Context(), store, environment.Alias, restoreErr) _ = finish(outcome, reason, environment) @@ -1324,7 +1426,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw } } status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", - gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) + gatewayCreatePayloadForAccount(environment, browserPath, networkExit), gatewayLongTimeout) if callErr != nil || status != http.StatusCreated { createErr := gatewayRejected(status, body) if callErr != nil { @@ -1338,7 +1440,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw _ = finish("failed", "gateway_create_failed", environment) return hubError(c, createErr) } - var createdRuntime containerStatus + var createdRuntime runtimeStatus if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) { reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) _ = finish("unknown", "gateway_result_unknown", environment) @@ -1438,7 +1540,7 @@ func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environ return finish("succeeded", "environment_stopped") } if environment.RuntimeID == "" { - container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias) + runtime, found, reconcileErr := reconcileGatewayRuntime(ctx, gateway, environment.Alias) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown") return gatewayUnreachable(reconcileErr) @@ -1446,11 +1548,11 @@ func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environ if !found { return finish("succeeded", "environment_stopped") } - if container.BindingVersion != environment.BindingVersion { + if runtime.BindingVersion != environment.BindingVersion { _ = finish("failed", "runtime_generation_conflict") return hub.ErrConflict } - environment = runtimeCleanupGeneration(environment, environment.BindingVersion, container.ID, container.NetworkID) + environment = runtimeCleanupGeneration(environment, environment.BindingVersion, runtime.ID, runtime.NetworkID) } else { environment = runtimeCleanupGeneration(environment, environment.BindingVersion, environment.RuntimeID) } @@ -1461,7 +1563,7 @@ func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environ status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop", gatewayCleanupGenerationPayload(environment), 30*time.Second) if callErr != nil || status >= http.StatusInternalServerError { - container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias) + runtime, found, reconcileErr := reconcileGatewayRuntime(ctx, gateway, environment.Alias) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown") if callErr != nil { @@ -1469,7 +1571,7 @@ func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environ } return gatewayRejected(status, body) } - if !found || container.State != "running" { + if !found || runtime.State != "running" { if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil { _ = finish("failed", "runtime_release_failed") return err @@ -1538,26 +1640,26 @@ func startBrowserRuntime(ctx context.Context, store hubStore, probe networkExitP return hub.ErrConflict } } - imageRef, err := store.ImageRef(ctx, environment.ImageVersion) + browserPath, err := store.BrowserPath(ctx, environment.BrowserVersion) if err != nil { if cleanupErr := discardRuntime(ctx, store, environment); cleanupErr != nil { _ = finish("unknown", "cleanup_result_unknown", environment) return cleanupErr } - _ = finish("failed", "image_unavailable", environment) + _ = finish("failed", "browser_version_unavailable", environment) return err } networkExit := gatewayNetworkExit{} if environment.Exit.ID != "" { networkExit = gatewayNetworkExitFor(access) } - container, found, err := reconcileGatewayContainer(ctx, gateway, environment.Alias) + runtime, found, err := reconcileGatewayRuntime(ctx, gateway, environment.Alias) if err != nil { _ = finish("unknown", "gateway_result_unknown", environment) return gatewayUnreachable(err) } - if found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { - environment, err = activateGatewayRuntime(ctx, store, gateway, environment, container.ID, container.NetworkID) + if found && runtime.State == "running" && runtime.ProxyReady && runtimeMatchesBinding(runtime, environment) { + environment, err = activateGatewayRuntime(ctx, store, gateway, environment, runtime.ID, runtime.NetworkID, runtime.NodeID) if err != nil { _ = finish("failed", "runtime_persistence_failed", environment) return err @@ -1568,7 +1670,9 @@ func startBrowserRuntime(ctx context.Context, store hubStore, probe networkExitP return nil } if found { - if _, removeErr := removeGatewayRuntime(ctx, store, gateway, environment); removeErr != nil { + cleanup := environment + cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtime.ID, runtime.NetworkID + if _, removeErr := removeGatewayRuntime(ctx, store, gateway, cleanup); removeErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) return removeErr } @@ -1577,7 +1681,7 @@ func startBrowserRuntime(ctx context.Context, store hubStore, probe networkExitP return err } status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers", - gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) + gatewayCreatePayload(environment, browserPath, networkExit), gatewayLongTimeout) if callErr != nil || status != http.StatusCreated { reconcileErr := reconcileGatewayCreate(ctx, store, gateway, environment, status, callErr, body) if reconcileErr != nil { @@ -1590,13 +1694,13 @@ func startBrowserRuntime(ctx context.Context, store hubStore, probe networkExitP } return gatewayRejected(status, body) } - var created containerStatus + var created runtimeStatus if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) { reconcileErr := reconcileGatewayCreate(ctx, store, gateway, environment, status, callErr, body) _ = finish("unknown", "gateway_result_unknown", environment) return errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr) } - environment, err = activateGatewayRuntime(ctx, store, gateway, environment, created.ID, created.NetworkID) + environment, err = activateGatewayRuntime(ctx, store, gateway, environment, created.ID, created.NetworkID, created.NodeID) if err != nil { _ = finish("failed", "runtime_persistence_failed", environment) return err @@ -1618,17 +1722,17 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx if err != nil { return hubError(c, err) } - var imageRef string - var imageErr error - if !hub.ValidImageVersion(input.Version) { - imageErr = hub.ErrInvalid + var browserPath string + var browserVersionErr error + if !hub.ValidBrowserVersion(input.Version) { + browserVersionErr = hub.ErrInvalid } else { - imageRef, imageErr = store.ImageRef(c.Context(), input.Version) + browserPath, browserVersionErr = store.BrowserPath(c.Context(), input.Version) } action := actionForEnvironment("upgrade", environment) - action.OldImageVersion = environment.ImageVersion - if imageErr != nil { - action.NewImageVersion, action.ReasonCode = "", "upgrade_input_rejected" + action.OldBrowserVersion = environment.BrowserVersion + if browserVersionErr != nil { + action.NewBrowserVersion, action.ReasonCode = "", "upgrade_input_rejected" if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { return hubError(c, err) } @@ -1636,9 +1740,9 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx if err := store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action); err != nil { return hubError(c, err) } - return hubError(c, imageErr) + return hubError(c, browserVersionErr) } - action.NewImageVersion = input.Version + action.NewBrowserVersion = input.Version if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { return hubError(c, err) } @@ -1685,8 +1789,8 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx } running = accountRunnable(environment) status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", - gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) - var createdRuntime containerStatus + gatewayCreatePayloadForAccount(environment, browserPath, networkExit), gatewayLongTimeout) + var createdRuntime runtimeStatus if callErr != nil || status != http.StatusCreated { reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) if reconcileErr != nil { @@ -1722,13 +1826,13 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx } type runtimeCreateSpec struct { - imageRef string + browserPath string networkExit gatewayNetworkExit } func prepareRuntimeCreate(ctx context.Context, store hubStore, _ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, access hub.NetworkExitAccess) (runtimeCreateSpec, error) { - imageRef, err := store.ImageRef(ctx, environment.ImageVersion) + browserPath, err := store.BrowserPath(ctx, environment.BrowserVersion) if err != nil { return runtimeCreateSpec{}, err } @@ -1736,15 +1840,15 @@ func prepareRuntimeCreate(ctx context.Context, store hubStore, _ func(hub.Networ if environment.Exit.ID != "" { networkExit = gatewayNetworkExitFor(access) } - return runtimeCreateSpec{imageRef: imageRef, networkExit: networkExit}, nil + return runtimeCreateSpec{browserPath: browserPath, networkExit: networkExit}, nil } -func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (containerStatus, error) { +func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (runtimeStatus, error) { status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", - gatewayCreatePayloadForAccount(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout) - unknown := containerStatus{NetworkID: gatewayNetworkID(body)} + gatewayCreatePayloadForAccount(environment, spec.browserPath, spec.networkExit), gatewayLongTimeout) + unknown := runtimeStatus{NetworkID: gatewayNetworkID(body)} if callErr == nil && status == http.StatusCreated { - var created containerStatus + var created runtimeStatus if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, accountRunnable(environment)) { created.Alias, created.BindingVersion = environment.Alias, environment.BindingVersion if accountRunnable(environment) { @@ -1764,12 +1868,12 @@ func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment h return unknown, errors.New("gateway returned an invalid runtime generation") } -func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, imageRef string) error { - payload := gatewayCreatePayload(environment, imageRef, gatewayNetworkExit{}) +func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, browserPath string) error { + payload := gatewayCreatePayload(environment, browserPath, gatewayNetworkExit{}) payload["network_exit_id"], payload["stopped"] = "", true status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", payload, gatewayLongTimeout) if callErr == nil && status == http.StatusCreated { - var created containerStatus + var created runtimeStatus if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, false) { return nil } @@ -1794,7 +1898,7 @@ func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target if environment.RuntimeCleanupBindingVersion < 1 { return false, hub.ErrReconcileRequired } - discovered, found, discoverErr := reconcileGatewayContainer(ctx, target, environment.Alias) + discovered, found, discoverErr := reconcileGatewayRuntime(ctx, target, environment.Alias) if discoverErr != nil { return false, discoverErr } @@ -1816,7 +1920,7 @@ func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target } } if !environment.RuntimeCleanupPending { - container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias) + runtime, found, err := reconcileGatewayRuntime(ctx, target, environment.Alias) if err != nil { return false, err } @@ -1826,17 +1930,17 @@ func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID } if found { - if container.BindingVersion != bindingVersion || runtimeID == missingRuntimeID || - (runtimeID != "" && runtimeID != container.ID) || (runtimeID == "" && networkID == "" && container.State == "running") { + if runtime.BindingVersion != bindingVersion || runtimeID == missingRuntimeID || + (runtimeID != "" && runtimeID != runtime.ID) || (runtimeID == "" && networkID == "" && runtime.State == "running") { return false, hub.ErrConflict } - runtimeID = container.ID + runtimeID = runtime.ID if networkID != "" { - if container.NetworkID != "" && container.NetworkID != networkID { + if runtime.NetworkID != "" && runtime.NetworkID != networkID { return false, hub.ErrConflict } } else { - networkID = container.NetworkID + networkID = runtime.NetworkID } } else if environment.RuntimeID == "" { runtimeID = missingRuntimeID @@ -1858,7 +1962,7 @@ func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target removed = true continue } - _, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias) + _, found, reconcileErr := reconcileGatewayRuntime(ctx, target, environment.Alias) if reconcileErr != nil { if callErr != nil { return removed, errors.Join(gatewayUnreachable(callErr), gatewayUnreachable(reconcileErr)) @@ -1878,13 +1982,13 @@ func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target return true, gatewayRejected(status, body) } if removed { - return true, gatewayFailure{status: http.StatusBadGateway, message: "gateway removed the container but network cleanup is pending"} + return true, gatewayFailure{status: http.StatusBadGateway, message: "gateway removed the runtime but network cleanup is pending"} } return false, gatewayFailure{status: http.StatusBadGateway, message: "gateway runtime cleanup is pending"} } func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error), - target hub.Gateway, environment hub.EnvironmentContext, previous containerStatus, found bool, prepared *runtimeCreateSpec) (bool, error) { + target hub.Gateway, environment hub.EnvironmentContext, previous runtimeStatus, found bool, prepared *runtimeCreateSpec) (bool, error) { if !found { return true, releaseRuntime(ctx, store, environment) } @@ -1895,17 +1999,17 @@ func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub. if previous.State == "running" { return false, nil } - imageRef := "" + browserPath := "" if prepared != nil { - imageRef = prepared.imageRef + browserPath = prepared.browserPath } else { var err error - imageRef, err = store.ImageRef(ctx, environment.ImageVersion) + browserPath, err = store.BrowserPath(ctx, environment.BrowserVersion) if err != nil { return false, err } } - if err := createStoppedGatewayRuntime(ctx, target, environment, imageRef); err != nil { + if err := createStoppedGatewayRuntime(ctx, target, environment, browserPath); err != nil { _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment) return false, errors.Join(err, cleanupErr) } @@ -1967,11 +2071,11 @@ func rebindRecoveryError(ready bool, err error) error { } func runtimeCreateSpecMatches(ctx context.Context, store hubStore, current, previous hub.EnvironmentContext, prepared *runtimeCreateSpec) bool { - if prepared == nil || current.BindingVersion != previous.BindingVersion || current.ImageVersion != previous.ImageVersion || current.Exit.ID != previous.Exit.ID { + if prepared == nil || current.BindingVersion != previous.BindingVersion || current.BrowserVersion != previous.BrowserVersion || current.Exit.ID != previous.Exit.ID { return false } - imageRef, err := store.ImageRef(ctx, current.ImageVersion) - return err == nil && imageRef == prepared.imageRef + browserPath, err := store.BrowserPath(ctx, current.BrowserVersion) + return err == nil && browserPath == prepared.browserPath } func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { @@ -2020,24 +2124,24 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return hubError(c, err) } } - container, found, err := reconcileGatewayContainer(c.Context(), target, before.Alias) + runtime, found, err := reconcileGatewayRuntime(c.Context(), target, before.Alias) if err != nil { _ = finish("unknown", "gateway_result_unknown", before) return hubError(c, gatewayUnreachable(err)) } - wasRunning := found && container.State == "running" + wasRunning := found && runtime.State == "running" if err := store.ValidateEnvironmentRebind(c.Context(), before.Alias, input.NetworkExitID, before.BindingVersion); err != nil { _ = finish("failed", "rebind_not_allowed", before) return hubError(c, err) } var previousSpec *runtimeCreateSpec if found && before.Exit.ID == "" { - imageRef, imageErr := store.ImageRef(c.Context(), before.ImageVersion) - if imageErr != nil { + browserPath, browserVersionErr := store.BrowserPath(c.Context(), before.BrowserVersion) + if browserVersionErr != nil { _ = finish("failed", "runtime_prepare_failed", before) - return hubError(c, imageErr) + return hubError(c, browserVersionErr) } - previousSpec = &runtimeCreateSpec{imageRef: imageRef} + previousSpec = &runtimeCreateSpec{browserPath: browserPath} } else if found { previousAccess, accessErr := store.GetNetworkExitAccess(c.Context(), before.Exit.ID) if accessErr != nil { @@ -2069,7 +2173,7 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return hubError(c, removeErr) } } - var candidateRuntime containerStatus + var candidateRuntime runtimeStatus if wasRunning { candidateRuntime, err = createGatewayRuntime(c.Context(), target, candidate, nextSpec) if err != nil { @@ -2086,7 +2190,7 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias) ready := false if contextErr == nil { - ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, container, found, previousSpec) + ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, runtime, found, previousSpec) } contextErr = rebindRecoveryError(ready, contextErr) if contextErr != nil { @@ -2124,7 +2228,7 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw if !runtimeCreateSpecMatches(c.Context(), store, current, before, previousSpec) { prepared = nil } - ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, container, true, prepared) + ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, runtime, true, prepared) } contextErr = rebindRecoveryError(ready, contextErr) if contextErr != nil { @@ -2175,7 +2279,7 @@ func actionForEnvironment(actionName string, environment hub.EnvironmentContext) OperationID: hub.NewOperationID(), Action: actionName, ReasonCode: "action_requested", AccountID: environment.AccountID, BrowserEnvAlias: environment.Alias, NetworkExitID: environment.Exit.ID, RuntimeInstanceID: environment.RuntimeInstanceID, BindingVersion: environment.BindingVersion, - OldImageVersion: environment.ImageVersion, NewImageVersion: environment.ImageVersion, + OldBrowserVersion: environment.BrowserVersion, NewBrowserVersion: environment.BrowserVersion, } } diff --git a/cmd/control-plane/hub_native_unit_test.go b/cmd/control-plane/hub_native_unit_test.go new file mode 100644 index 0000000..21b304c --- /dev/null +++ b/cmd/control-plane/hub_native_unit_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +func TestCreateStoppedGatewayRuntimeValidatesGatewayResponses(t *testing.T) { + environment := hub.EnvironmentContext{Env: hub.Env{Alias: "account-1", Name: "账号一", BrowserVersion: "148.0.7778.215"}, BindingVersion: 1} + cases := []struct { + name string + status int + body string + wantErr bool + }{ + {name: "success", status: http.StatusCreated, body: `{"id":"runtime-1","state":"stopped"}`}, + {name: "invalid body", status: http.StatusCreated, body: `{`, wantErr: true}, + {name: "rejected", status: http.StatusBadRequest, body: `{"error":"invalid"}`, wantErr: true}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/browsers" { + t.Fatalf("gateway request = %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(testCase.status) + _, _ = w.Write([]byte(testCase.body)) + })) + defer server.Close() + err := createStoppedGatewayRuntime(context.Background(), hub.Gateway{Endpoint: server.URL}, environment, "/opt/chrome") + if (err != nil) != testCase.wantErr { + t.Fatalf("create stopped runtime error = %v, wantErr=%v", err, testCase.wantErr) + } + }) + } + + server := httptest.NewServer(http.NotFoundHandler()) + endpoint := server.URL + server.Close() + if err := createStoppedGatewayRuntime(context.Background(), hub.Gateway{Endpoint: endpoint}, environment, "/opt/chrome"); err == nil { + t.Fatal("unreachable gateway was reported as successful") + } +} + +func TestRuntimeCreateSpecMatchesCurrentEnvironment(t *testing.T) { + store := newMemoryStore() + store.images["148.0.7778.215"] = hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/chrome", Enabled: true} + current := hub.EnvironmentContext{Env: hub.Env{BrowserVersion: "148.0.7778.215"}, Exit: hub.NetworkExit{ID: "exit-1"}, BindingVersion: 2} + previous := current + prepared := &runtimeCreateSpec{browserPath: "/opt/chrome"} + if !runtimeCreateSpecMatches(context.Background(), store, current, previous, prepared) { + t.Fatal("matching runtime specification was rejected") + } + for name, mutate := range map[string]func(*hub.EnvironmentContext){ + "binding": func(value *hub.EnvironmentContext) { value.BindingVersion++ }, + "browser version": func(value *hub.EnvironmentContext) { value.BrowserVersion = "149.0.0.0" }, + "network exit": func(value *hub.EnvironmentContext) { value.Exit.ID = "exit-2" }, + } { + t.Run(name, func(t *testing.T) { + candidate := current + mutate(&candidate) + if runtimeCreateSpecMatches(context.Background(), store, candidate, previous, prepared) { + t.Fatal("mismatched runtime specification was accepted") + } + }) + } + if runtimeCreateSpecMatches(context.Background(), store, current, previous, &runtimeCreateSpec{browserPath: "/other/chrome"}) { + t.Fatal("mismatched browser path was accepted") + } + if runtimeCreateSpecMatches(context.Background(), store, current, previous, nil) { + t.Fatal("nil runtime specification was accepted") + } +} + +func TestCreatorGatewayBrowserResolveAndSharePlatformValidation(t *testing.T) { + environment := hub.EnvironmentContext{Env: hub.Env{Alias: "account-1"}, RuntimeID: "runtime-1", RuntimeNetworkID: "network-1", BindingVersion: 3} + cases := []struct { + name string + status int + body string + wantURL string + wantErr bool + }{ + {name: "success", status: http.StatusOK, body: `{"url":"https://www.douyin.com/video/1"}`, wantURL: "https://www.douyin.com/video/1"}, + {name: "invalid json", status: http.StatusOK, body: `{`, wantErr: true}, + {name: "empty url", status: http.StatusOK, body: `{"url":""}`, wantErr: true}, + {name: "rejected", status: http.StatusBadGateway, body: `gateway down`, wantErr: true}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/browsers/account-1/douyin/resolve" { + t.Fatalf("resolve path = %q", r.URL.Path) + } + w.WriteHeader(testCase.status) + _, _ = w.Write([]byte(testCase.body)) + })) + defer server.Close() + got, err := (creatorGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL}, environment: environment}).Resolve(context.Background(), "https://www.douyin.com/video/1") + if (err != nil) != testCase.wantErr || got != testCase.wantURL { + t.Fatalf("resolve = %q, err = %v", got, err) + } + }) + } + + for _, testCase := range []struct { + value string + platform string + wantErr bool + }{ + {value: "https://www.douyin.com/video/1", platform: creator.PlatformDouyin}, + {value: "https://v.douyin.com/abc", platform: creator.PlatformDouyin}, + {value: "https://www.xiaohongshu.com/explore/1", platform: creator.PlatformXiaohongshu}, + {value: "https://xhslink.com/abc", platform: creator.PlatformXiaohongshu}, + {value: "http://www.douyin.com/video/1", wantErr: true}, + {value: "https://user@www.douyin.com/video/1", wantErr: true}, + {value: "https://www.douyin.com:443/video/1", wantErr: true}, + {value: "https://www.douyin.com/video/1#part", wantErr: true}, + {value: "https://example.com/video/1", wantErr: true}, + } { + platform, err := competitorSharePlatform(testCase.value) + if testCase.wantErr { + if !errors.Is(err, creator.ErrInvalid) { + t.Errorf("competitorSharePlatform(%q) error = %v", testCase.value, err) + } + continue + } + if err != nil || platform != testCase.platform { + t.Errorf("competitorSharePlatform(%q) = %q, %v", testCase.value, platform, err) + } + } +} + +func TestRebindRecoveryErrorPreservesFailureStates(t *testing.T) { + sentinel := errors.New("rebind failed") + if got := rebindRecoveryError(true, sentinel); !errors.Is(got, sentinel) { + t.Fatalf("existing error = %v", got) + } + if got := rebindRecoveryError(false, nil); got == nil { + t.Fatal("incomplete recovery was reported as successful") + } + if got := rebindRecoveryError(true, nil); got != nil { + t.Fatalf("complete recovery error = %v", got) + } +} diff --git a/cmd/control-plane/hub_test.go b/cmd/control-plane/hub_test.go index ed918dd..96b43bc 100644 --- a/cmd/control-plane/hub_test.go +++ b/cmd/control-plane/hub_test.go @@ -29,7 +29,7 @@ type memoryStore struct { locksMu sync.Mutex locks map[string]*sync.Mutex gateways map[string]hub.Gateway - images map[string]hub.Image + images map[string]hub.BrowserVersion envs map[string]hub.Env exits map[string]hub.NetworkExit bindings map[string]hub.EnvironmentContext @@ -58,7 +58,7 @@ func newMemoryStore() *memoryStore { return &memoryStore{ locks: map[string]*sync.Mutex{}, gateways: map[string]hub.Gateway{}, - images: map[string]hub.Image{}, + images: map[string]hub.BrowserVersion{}, envs: map[string]hub.Env{}, exits: map[string]hub.NetworkExit{ "exit-1": {ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1}, @@ -82,7 +82,7 @@ func TestResumeBlockReasonIsStable(t *testing.T) { "direct exit": {account, hub.EnvironmentContext{}, true, "account_conflict"}, "unhealthy exit": {account, hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "unhealthy"}}, true, "network_exit_unhealthy"}, "cleanup pending": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeCleanupPending: true}, true, "runtime_stop_pending"}, - "runtime active": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeInstanceID: "runtime-a"}, true, "runtime_active"}, + "runtime active": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeInstanceID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, true, "runtime_active"}, } { t.Run(name, func(t *testing.T) { if got := resumeBlockReason(test.account, test.env, test.found); got != test.want { @@ -204,36 +204,36 @@ func (s *memoryStore) GetGateway(_ context.Context, name string) (hub.Gateway, e return gateway, nil } func (s *memoryStore) DeleteGateway(context.Context, string) error { return nil } -func (s *memoryStore) CreateImage(_ context.Context, image hub.Image) error { +func (s *memoryStore) CreateBrowserVersion(_ context.Context, image hub.BrowserVersion) error { s.mu.Lock() defer s.mu.Unlock() s.images[image.Version] = image return nil } -func (s *memoryStore) UpdateImage(_ context.Context, image hub.Image) error { +func (s *memoryStore) UpdateBrowserVersion(_ context.Context, image hub.BrowserVersion) error { s.mu.Lock() defer s.mu.Unlock() s.images[image.Version] = image return nil } -func (s *memoryStore) ListImages(context.Context, bool) ([]hub.Image, error) { +func (s *memoryStore) ListBrowserVersions(context.Context, bool) ([]hub.BrowserVersion, error) { s.mu.Lock() defer s.mu.Unlock() - images := make([]hub.Image, 0, len(s.images)) + images := make([]hub.BrowserVersion, 0, len(s.images)) for _, image := range s.images { images = append(images, image) } return images, nil } -func (s *memoryStore) DeleteImage(context.Context, string) error { return nil } -func (s *memoryStore) ImageRef(_ context.Context, version string) (string, error) { +func (s *memoryStore) DeleteBrowserVersion(context.Context, string) error { return nil } +func (s *memoryStore) BrowserPath(_ context.Context, version string) (string, error) { s.mu.Lock() defer s.mu.Unlock() image, ok := s.images[version] if !ok || !image.Enabled { return "", hub.ErrNotFound } - return image.ImageRef, nil + return image.BrowserPath, nil } func (s *memoryStore) CreateEnv(_ context.Context, env hub.Env) error { s.mu.Lock() @@ -241,7 +241,7 @@ func (s *memoryStore) CreateEnv(_ context.Context, env hub.Env) error { if _, exists := s.envs[env.Alias]; exists { return hub.ErrConflict } - if image, exists := s.images[env.ImageVersion]; !exists || !image.Enabled { + if image, exists := s.images[env.BrowserVersion]; !exists || !image.Enabled { return hub.ErrNotFound } s.envs[env.Alias] = env @@ -282,7 +282,7 @@ func (s *memoryStore) UpgradeEnv(_ context.Context, alias, version string) error } s.upgraded[alias] = version env := s.envs[alias] - env.ImageVersion = version + env.BrowserVersion = version s.envs[alias] = env bound, ok := s.bindings[alias] if !ok { @@ -563,7 +563,7 @@ func TestPauseSerializesResumeAndRetainsClaimGateOnUnknownStop(t *testing.T) { if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil { t.Fatal(err) } - if _, err := fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old"); err != nil { + if _, err := fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666"); err != nil { t.Fatal(err) } blocking := &blockingRuntimeStopStore{Store: fixture.store, entered: make(chan struct{}, 1), release: make(chan struct{})} @@ -602,14 +602,14 @@ type recordedRequest struct { body map[string]any } -// fakeGateway 模拟 docker-gateway：按路由表应答并记录请求。 +// fakeGateway 模拟 native browser gateway：按路由表应答并记录请求。 type fakeGateway struct { mu sync.Mutex createOnce sync.Once deleteOnce sync.Once token string requests []recordedRequest - containers []containerStatus + runtimes []runtimeStatus failCreate int // 前 N 次 create 返回失败 failCreateStatus int failDelete int // 前 N 次 delete 返回 500 且保留容器 @@ -711,19 +711,19 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { <-g.releaseCreate } state, proxyReady := "running", true - networkID := "network-id" + networkID := "native-cccccccccccccccccccccccccccccccc" if stopped, _ := body["stopped"].(bool); stopped { state, proxyReady, networkID = "exited", false, "" } g.mu.Lock() - g.containers = []containerStatus{{ - ID: "container-id", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady, + g.runtimes = []runtimeStatus{{ + ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady, BindingVersion: int64(body["binding_version"].(float64)), NetworkExitID: body["network_exit_id"].(string), NetworkID: networkID, }} g.mu.Unlock() response.WriteHeader(http.StatusCreated) - _, _ = response.Write([]byte(`{"id":"container-id","alias":"account-a","network_id":"` + networkID + `"}`)) + _, _ = response.Write([]byte(`{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a","network_id":"` + networkID + `"}`)) case request.Method == http.MethodGet && request.URL.Path == "/v1/browsers": g.mu.Lock() if g.failList > 0 { @@ -758,13 +758,13 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { _ = connection.Close() return } - containers := append([]containerStatus{}, g.containers...) + runtimes := append([]runtimeStatus{}, g.runtimes...) g.mu.Unlock() - _ = json.NewEncoder(response).Encode(containers) + _ = json.NewEncoder(response).Encode(runtimes) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/v1/browsers/"): g.mu.Lock() - if len(g.containers) > 0 && (body["runtime_id"] != g.containers[0].ID || - int64(body["binding_version"].(float64)) != g.containers[0].BindingVersion) { + if len(g.runtimes) > 0 && (body["runtime_id"] != g.runtimes[0].ID || + int64(body["binding_version"].(float64)) != g.runtimes[0].BindingVersion) { g.mu.Unlock() response.WriteHeader(http.StatusConflict) return @@ -776,7 +776,7 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { _, _ = response.Write([]byte(`{"error":"docker delete failed"}`)) return } - g.containers = nil + g.runtimes = nil if g.deleteNotFound > 0 { g.deleteNotFound-- g.mu.Unlock() @@ -811,27 +811,27 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"): g.mu.Lock() - if len(g.containers) > 0 { - g.containers[0].State, g.containers[0].Status = "running", "Up" + if len(g.runtimes) > 0 { + g.runtimes[0].State, g.runtimes[0].Status = "running", "Up" } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"): g.mu.Lock() - if len(g.containers) > 0 && (body["runtime_id"] != g.containers[0].ID || - int64(body["binding_version"].(float64)) != g.containers[0].BindingVersion) { + if len(g.runtimes) > 0 && (body["runtime_id"] != g.runtimes[0].ID || + int64(body["binding_version"].(float64)) != g.runtimes[0].BindingVersion) { g.mu.Unlock() response.WriteHeader(http.StatusConflict) return } - if len(g.containers) > 0 { - g.containers[0].State, g.containers[0].Status = "exited", "Exited" + if len(g.runtimes) > 0 { + g.runtimes[0].State, g.runtimes[0].Status = "exited", "Exited" } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/proxy"): g.mu.Lock() - if len(g.containers) > 0 && body["runtime_id"] != g.containers[0].ID { + if len(g.runtimes) > 0 && body["runtime_id"] != g.runtimes[0].ID { g.mu.Unlock() response.WriteHeader(http.StatusConflict) return @@ -841,8 +841,8 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { response.WriteHeader(http.StatusBadGateway) return } - if len(g.containers) > 0 { - g.containers[0].ProxyReady = true + if len(g.runtimes) > 0 { + g.runtimes[0].ProxyReady = true } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) @@ -890,7 +890,7 @@ func do(app *fiber.App, method, path, body string, credentials ...string) *httpt return response } -const createEnvBody = `{"alias":"account-a","name":"店铺一号","gateway":"gw-1","image_version":"148.0.7778.215",` + +const createEnvBody = `{"alias":"account-a","name":"店铺一号","gateway":"gw-1","browser_version":"148.0.7778.215",` + `"fingerprint":{"seed":2024,"platform":"windows","timezone":"Asia/Shanghai"},"account_id":"account-a","network_exit_id":"exit-1"}` func TestUpdateGatewayRenamesAndPreservesReferences(t *testing.T) { @@ -959,12 +959,12 @@ func TestParseGatewayBrowserListStrict(t *testing.T) { ok bool }{ {name: "empty", body: `[]`, ok: true}, - {name: "minimal browser", body: `[{"id":"container-id","alias":"account-a","state":"running"}]`, ok: true}, + {name: "minimal browser", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a","state":"running"}]`, ok: true}, {name: "top-level null", body: `null`}, {name: "null element", body: `[null]`}, {name: "missing id", body: `[{"alias":"account-a","state":"running"}]`}, - {name: "missing alias", body: `[{"id":"container-id","state":"running"}]`}, - {name: "missing state", body: `[{"id":"container-id","alias":"account-a"}]`}, + {name: "missing alias", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"running"}]`}, + {name: "missing state", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a"}]`}, {name: "duplicate alias", body: `[{"id":"one","alias":"account-a","state":"running"},{"id":"two","alias":"account-a","state":"exited"}]`}, } { t.Run(test.name, func(t *testing.T) { @@ -991,7 +991,7 @@ func TestGatewayCallPropagatesBodyReadError(t *testing.T) { func TestGatewayCreatePayloadStripsLegacyProxyFingerprint(t *testing.T) { payload := gatewayCreatePayload(hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Name: "甲", Fingerprint: hub.Fingerprint{ Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true, - }}, BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1"}}, "registry.example/browser:1", gatewayNetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}) + }}, BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1"}}, "/opt/creatorhub/browsers/1", gatewayNetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}) encoded, _ := json.Marshal(payload) if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") { t.Fatalf("legacy proxy URI entered the gateway contract: %s", encoded) @@ -1000,7 +1000,7 @@ func TestGatewayCreatePayloadStripsLegacyProxyFingerprint(t *testing.T) { func TestCreateBrowserOrchestratesGateway(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser@sha256:abc", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/1", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -1017,17 +1017,17 @@ func TestCreateBrowserOrchestratesGateway(t *testing.T) { } payload := requests[0].body if payload["alias"] != "account-a" || payload["name"] != "店铺一号" || - payload["image"] != "registry.example/browser@sha256:abc" || - payload["volume"] != "creatorhub-profile-account-a" { - t.Fatalf("platform must fully specify the gateway payload: %#v", payload) + payload["browser_version"] != "148.0.7778.215" || + payload["profile_id"] != "account-a" || payload["image"] != nil || payload["volume"] != nil { + t.Fatalf("platform must send the native browser contract: %#v", payload) } exit := payload["network_exit"].(map[string]any) if exit["protocol"] != "socks5" || exit["host"] != "127.0.0.1" || exit["port"] != float64(1080) { t.Fatalf("platform must force the bound exit: %#v", payload) } cmd := payload["cmd"].([]any) - if len(cmd) != 5 || cmd[0] != "--fingerprint=2024" || cmd[1] != "--fingerprint-platform=windows" || - cmd[2] != "--timezone=Asia/Shanghai" || cmd[3] != "--remote-allow-origins=*" || cmd[4] != "about:blank" { + if len(cmd) != 4 || cmd[0] != "--fingerprint=2024" || cmd[1] != "--fingerprint-platform=windows" || + cmd[2] != "--timezone=Asia/Shanghai" || cmd[3] != "about:blank" { t.Fatalf("cmd must carry fingerprint args plus start url: %#v", cmd) } if stored := store.envs["account-a"].Fingerprint; stored.ProxyServer != "" || stored.DisableNonProxiedUDP { @@ -1041,7 +1041,7 @@ func TestCreateBrowserOrchestratesGateway(t *testing.T) { func TestCreateBrowserSupportsDirectMachineExit(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} server := httptest.NewServer(gateway.handler(t)) defer server.Close() @@ -1050,7 +1050,7 @@ func TestCreateBrowserSupportsDirectMachineExit(t *testing.T) { registerHubWithNetwork(app, store, fakeExitProbe{failure: "must_not_probe"}, func(hub.NetworkExitAccess) (string, error) { return "", errors.New("must not resolve direct exit credentials") }) - body := `{"alias":"direct-env","name":"直连环境","gateway":"gw-1","image_version":"148.0.7778.215",` + + body := `{"alias":"direct-env","name":"直连环境","gateway":"gw-1","browser_version":"148.0.7778.215",` + `"fingerprint":{"seed":2024},"account_id":"account-a","network_exit_id":""}` response := do(app, http.MethodPost, "/api/browsers", body) @@ -1087,7 +1087,7 @@ func TestExitFailuresStopCreateBeforeGateway(t *testing.T) { ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, ExpectedPublicIP: "203.0.113.10", HealthStatus: "healthy", Version: 1, } - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} server := httptest.NewServer(gateway.handler(t)) defer server.Close() @@ -1115,7 +1115,7 @@ func TestExitFailuresStopCreateBeforeGateway(t *testing.T) { func TestRepeatedCreateAndStartReuseStableEnvironment(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -1129,15 +1129,15 @@ func TestRepeatedCreateAndStartReuseStableEnvironment(t *testing.T) { t.Fatalf("idempotent start failed: %d %s", response.Code, response.Body.String()) } bound := store.bindings["account-a"] - if bound.Alias != "account-a" || bound.Exit.ID != "exit-1" || bound.RuntimeID != "container-id" { + if bound.Alias != "account-a" || bound.Exit.ID != "exit-1" || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("repeated actions changed stable environment identity: %#v", bound) } createCalls := 0 for _, request := range gateway.recorded() { if request.method == http.MethodPost && request.path == "/v1/browsers" { createCalls++ - if request.body["volume"] != "creatorhub-profile-account-a" || request.body["network_exit"].(map[string]any)["host"] != "127.0.0.1" { - t.Fatalf("create changed Profile volume or exit: %#v", request.body) + if request.body["profile_id"] != "account-a" || request.body["browser_version"] != "148.0.7778.215" || request.body["network_exit"].(map[string]any)["host"] != "127.0.0.1" { + t.Fatalf("create changed Profile identity or exit: %#v", request.body) } } } @@ -1148,7 +1148,7 @@ func TestRepeatedCreateAndStartReuseStableEnvironment(t *testing.T) { func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1} app := newTestApp(t, store, gateway) @@ -1163,7 +1163,7 @@ func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) { func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadGateway} app := newTestApp(t, store, gateway) @@ -1183,7 +1183,7 @@ func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testi func TestCreateBrowserDeterministicRejectionDoesNotWedgeBinding(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadRequest} app := newTestApp(t, store, gateway) @@ -1205,10 +1205,10 @@ func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], - RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"} + RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"} store.bindings[environment.Alias] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-c1", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-n1", ProxyReady: true, + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-99999999999999999999999999999999", ProxyReady: true, }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() @@ -1216,36 +1216,36 @@ func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) { err := reconcileGatewayCreate(context.Background(), store, target, environment, http.StatusBadGateway, nil, []byte(`{"error":"result unknown"}`)) after := store.bindings[environment.Alias] - if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.containers) != 1 || + if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.runtimes) != 1 || !after.RuntimeCleanupPending || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" { - t.Fatalf("unknown create reused the old generation: err=%v requests=%#v containers=%#v environment=%#v", - err, gateway.recorded(), gateway.containers, after) + t.Fatalf("unknown create reused the old generation: err=%v requests=%#v runtimes=%#v environment=%#v", + err, gateway.recorded(), gateway.runtimes, after) } } func TestValidCreatedRuntimeGeneration(t *testing.T) { - environment := hub.EnvironmentContext{RuntimeNetworkID: "network-n1"} + environment := hub.EnvironmentContext{RuntimeNetworkID: "native-99999999999999999999999999999999"} for _, test := range []struct { name string - created containerStatus + created runtimeStatus environment hub.EnvironmentContext running bool want bool }{ - {name: "active matching generation", created: containerStatus{ID: "container-c1", NetworkID: "network-n1"}, running: true, want: true}, - {name: "active empty runtime", created: containerStatus{NetworkID: "network-n1"}, running: true}, - {name: "active invalid runtime", created: containerStatus{ID: "container c1", NetworkID: "network-n1"}, running: true}, - {name: "active empty network", created: containerStatus{ID: "container-c1"}, running: true}, - {name: "active invalid network", created: containerStatus{ID: "container-c1", NetworkID: "network n1"}, running: true}, - {name: "active replacement network", created: containerStatus{ID: "container-c1", NetworkID: "network-n2"}, running: true}, - {name: "active existing matching generation", created: containerStatus{ID: "container-c1", NetworkID: "network-n1"}, running: true, want: true, - environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"}}, - {name: "active existing successor container", created: containerStatus{ID: "container-c2", NetworkID: "network-n1"}, running: true, - environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"}}, - {name: "active legacy lease without network", created: containerStatus{ID: "container-c2", NetworkID: "network-n2"}, running: true, - environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1"}}, - {name: "stopped empty network", created: containerStatus{ID: "container-c1"}, want: true}, - {name: "stopped invalid network", created: containerStatus{ID: "container-c1", NetworkID: "network n1"}}, + {name: "active matching generation", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-99999999999999999999999999999999"}, running: true, want: true}, + {name: "active empty runtime", created: runtimeStatus{NetworkID: "native-99999999999999999999999999999999"}, running: true}, + {name: "active invalid runtime", created: runtimeStatus{ID: "container c1", NetworkID: "native-99999999999999999999999999999999"}, running: true}, + {name: "active empty network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, running: true}, + {name: "active invalid network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "network n1"}, running: true}, + {name: "active replacement network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-88888888888888888888888888888888"}, running: true}, + {name: "active existing matching generation", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-99999999999999999999999999999999"}, running: true, want: true, + environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}}, + {name: "active existing successor container", created: runtimeStatus{ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", NetworkID: "native-99999999999999999999999999999999"}, running: true, + environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}}, + {name: "active legacy lease without network", created: runtimeStatus{ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", NetworkID: "native-88888888888888888888888888888888"}, running: true, + environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}}, + {name: "stopped empty network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, want: true}, + {name: "stopped invalid network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "network n1"}}, } { t.Run(test.name, func(t *testing.T) { current := environment @@ -1259,24 +1259,66 @@ func TestValidCreatedRuntimeGeneration(t *testing.T) { } } +type activationConflictStore struct { + *memoryStore + current hub.EnvironmentContext +} + +func (s *activationConflictStore) GetEnvironmentContext(_ context.Context, _ string) (hub.EnvironmentContext, error) { + return s.current, nil +} + +func (s *activationConflictStore) ActivateRuntime(context.Context, string, string, int64, string, ...string) (hub.EnvironmentContext, error) { + return hub.EnvironmentContext{}, hub.ErrConflict +} + +func TestActivationConflictKeepsTheGenerationWonByTheHeartbeat(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} + candidateID := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + candidateNetwork := "native-88888888888888888888888888888888" + current := hub.EnvironmentContext{ + Env: store.envs["account-a"], + AccountID: "account-a", + AccountStatus: "active", + AuthorizationStatus: "authorized", + BindingID: "account-a", + BindingVersion: 1, + RuntimeInstanceID: "runtime-instance-c1", + RuntimeID: candidateID, + RuntimeNetworkID: candidateNetwork, + Exit: store.exits["exit-1"], + } + store.bindings["account-a"] = current + conflict := &activationConflictStore{memoryStore: store, current: current} + + got, err := activateGatewayRuntime(context.Background(), conflict, hub.Gateway{}, current, candidateID, candidateNetwork) + if err != nil { + t.Fatalf("activation race was reported as failure: %v", err) + } + if got.RuntimeID != candidateID || got.RuntimeNetworkID != candidateNetwork || got.RuntimeInstanceID == "" { + t.Fatalf("heartbeat winner was not returned: %#v", got) + } +} + func TestLegacyActiveRuntimeWithoutNetworkGenerationDoesNotTouchSuccessor(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], - RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1"} + RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} store.bindings[environment.Alias] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-c2", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-n2", ProxyReady: true, + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-88888888888888888888888888888888", ProxyReady: true, }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} - if containerMatchesBinding(gateway.containers[0], environment) { + if runtimeMatchesBinding(gateway.runtimes[0], environment) { t.Fatal("legacy active lease accepted a successor network generation") } - if _, err := activateGatewayRuntime(context.Background(), store, target, environment, "container-c2", "network-n2"); !errors.Is(err, hub.ErrConflict) { + if _, err := activateGatewayRuntime(context.Background(), store, target, environment, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "native-88888888888888888888888888888888"); !errors.Is(err, hub.ErrConflict) { t.Fatalf("successor activation did not fail closed: %v", err) } if _, err := removeGatewayRuntime(context.Background(), store, target, environment); !errors.Is(err, hub.ErrConflict) { @@ -1284,14 +1326,14 @@ func TestLegacyActiveRuntimeWithoutNetworkGenerationDoesNotTouchSuccessor(t *tes } after := store.bindings[environment.Alias] requests := gateway.recorded() - if len(requests) != 1 || requests[0].method != http.MethodGet || len(gateway.containers) != 1 || - after.RuntimeID != "container-c1" || after.RuntimeNetworkID != "" || after.RuntimeCleanupPending { - t.Fatalf("legacy C1 cleanup touched C2/N2: requests=%#v containers=%#v environment=%#v", requests, gateway.containers, after) + if len(requests) != 1 || requests[0].method != http.MethodGet || len(gateway.runtimes) != 1 || + after.RuntimeID != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || after.RuntimeNetworkID != "" || after.RuntimeCleanupPending { + t.Fatalf("legacy C1 cleanup touched C2/N2: requests=%#v runtimes=%#v environment=%#v", requests, gateway.runtimes, after) } } func TestCreateBrowserInvalid201TracksNetworkCleanupGeneration(t *testing.T) { - for _, networkID := range []string{"network-n1", ""} { + for _, networkID := range []string{"native-99999999999999999999999999999999", ""} { name := "known network" if networkID == "" { name = "unknown network" @@ -1322,7 +1364,7 @@ func TestCreateBrowserInvalid201TracksNetworkCleanupGeneration(t *testing.T) { })) defer gatewayServer.Close() store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) @@ -1350,11 +1392,11 @@ func TestCreateBrowserRejectsActiveRuntimeWithoutNetworkGeneration(t *testing.T) t.Fatalf("active NULL network response triggered alias reconciliation: %s %s", request.Method, request.URL.Path) } response.WriteHeader(http.StatusCreated) - _, _ = response.Write([]byte(`{"id":"container-c1","network_id":""}`)) + _, _ = response.Write([]byte(`{"id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","network_id":""}`)) })) defer gatewayServer.Close() store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) @@ -1389,10 +1431,10 @@ func TestCreateBrowserDoesNotDiscoverDisconnectedCreateByAlias(t *testing.T) { exists := created mu.Unlock() if exists { - _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) + _ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) return } - _ = json.NewEncoder(response).Encode([]containerStatus{}) + _ = json.NewEncoder(response).Encode([]runtimeStatus{}) return } if request.Method == http.MethodPost { @@ -1415,7 +1457,7 @@ func TestCreateBrowserDoesNotDiscoverDisconnectedCreateByAlias(t *testing.T) { defer gatewayServer.Close() store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) @@ -1466,15 +1508,15 @@ func TestCreateBrowserDoesNotDiscoverBadGatewayCreateByAlias(t *testing.T) { missing := listCalls <= test.missingReads mu.Unlock() if missing { - _ = json.NewEncoder(response).Encode([]containerStatus{}) + _ = json.NewEncoder(response).Encode([]runtimeStatus{}) return } - _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) + _ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) })) defer gatewayServer.Close() store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) @@ -1498,12 +1540,12 @@ func TestCreateBrowserDoesNotDiscoverBadGatewayCreateByAlias(t *testing.T) { func TestCreateBrowserRejectsInvalidFingerprintBeforeSideEffects(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers", - `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148.0.7778.215","fingerprint":{"seed":0}}`) + `{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148.0.7778.215","fingerprint":{"seed":0}}`) if response.Code != http.StatusBadRequest { t.Fatalf("expected 400 for invalid fingerprint, got %d: %s", response.Code, response.Body.String()) } @@ -1512,18 +1554,18 @@ func TestCreateBrowserRejectsInvalidFingerprintBeforeSideEffects(t *testing.T) { } } -func TestListBrowsersUsesPersistedStateWithoutProbing(t *testing.T) { +func TestListBrowsersReflectsGatewayState(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148"} - store.envs["account-b"] = hub.Env{Alias: "account-b", Name: "店铺二号", Gateway: "gw-1", ImageVersion: "148"} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148"} + store.envs["account-b"] = hub.Env{Alias: "account-b", Name: "店铺二号", Gateway: "gw-1", BrowserVersion: "148"} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-1", RuntimeID: "persisted-container", } gateway := &fakeGateway{ token: "unit-test-gateway-token", - containers: []containerStatus{ - {ID: "live-container", Alias: "account-a", State: "running", Status: "Up", Endpoint: "http://creatorhub-browser-account-a:9222", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-id", ProxyReady: true}, + runtimes: []runtimeStatus{ + {ID: "live-container", Alias: "account-a", State: "running", Status: "ready", Endpoint: "http://127.0.0.1:19001", NodeID: "node-a", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-cccccccccccccccccccccccccccccccc", ProxyReady: true}, }, } app := newTestApp(t, store, gateway) @@ -1540,11 +1582,11 @@ func TestListBrowsersUsesPersistedStateWithoutProbing(t *testing.T) { for _, view := range views { byAlias[view.Alias] = view } - if view := byAlias["account-a"]; view.State != "running" || view.Status != "已记录运行实例" || view.ContainerID != "persisted-container" || view.Endpoint != "" { - t.Fatalf("list must expose persisted runtime data only: %#v", view) + if view := byAlias["account-a"]; view.State != "running" || view.Status != "ready" || view.RuntimeID != "live-container" || view.Endpoint != "http://127.0.0.1:19001" || view.RuntimeNodeID != "node-a" || !view.GatewayReachable { + t.Fatalf("list must expose live native runtime data: %#v", view) } if view := byAlias["account-a"]; view.AccountID != "account-a" || view.NetworkExitHealth != "healthy" || view.ScheduleStatus != "ready" || view.ScheduleBlockReason != "" { - t.Fatalf("binding readiness must be exposed without gateway access: %#v", view) + t.Fatalf("binding readiness must remain visible with gateway data: %#v", view) } if byAlias["account-b"].State != "missing" { t.Fatalf("env without a persisted runtime must report missing: %#v", byAlias["account-b"]) @@ -1552,33 +1594,33 @@ func TestListBrowsersUsesPersistedStateWithoutProbing(t *testing.T) { if response = do(app, http.MethodGet, "/api/browsers/account-a", ""); response.Code != http.StatusOK { t.Fatalf("browser detail returned %d: %s", response.Code, response.Body.String()) } - if requests := gateway.recorded(); len(requests) != 0 { - t.Fatalf("browser list/detail performed live probes: %#v", requests) + if requests := gateway.recorded(); len(requests) != 2 || requests[0].method != http.MethodGet || requests[0].path != "/v1/browsers" || requests[1].method != http.MethodGet || requests[1].path != "/v1/browsers" { + t.Fatalf("browser list and detail must each use the native gateway snapshot: %#v", requests) } } func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 3, Exit: store.exits["exit-1"], } - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", NetworkID: "network-id", ProxyReady: false, + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", NetworkID: "native-cccccccccccccccccccccccccccccccc", ProxyReady: false, }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("gateway restart recovery failed: %v", err) } - if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" { + if runtime := store.bindings["account-a"].RuntimeID; runtime != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("runtime was not activated after proxy recovery: %q", runtime) } requests := gateway.recorded() if len(requests) != 2 || requests[0].path != "/v1/browsers" || requests[1].path != "/v1/browsers/account-a/proxy" { t.Fatalf("expected list then proxy recovery without rebuild: %#v", requests) } - if requests[1].body["binding_version"] != float64(3) || requests[1].body["runtime_id"] != "container-id" || + if requests[1].body["binding_version"] != float64(3) || requests[1].body["runtime_id"] != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || requests[1].body["network_exit_id"] != "exit-1" { t.Fatalf("proxy recovery did not use the current binding: %#v", requests[1].body) } @@ -1586,11 +1628,11 @@ func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) { func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"]} - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "network-old", + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666", }}} _ = newTestApp(t, store, gateway) @@ -1609,10 +1651,10 @@ func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing. func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{ - Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", + Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024, Timezone: "Asia/Shanghai"}, } - _ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -1626,8 +1668,8 @@ func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { t.Fatalf("upgrade must delete then recreate: %#v", requests) } payload := requests[len(requests)-1].body - if payload["image"] != "registry.example/browser:144" || payload["volume"] != "creatorhub-profile-account-a" { - t.Fatalf("upgrade must reuse the profile volume and switch image: %#v", payload) + if payload["browser_version"] != "144.0.7559.132" || payload["profile_id"] != "account-a" || payload["image"] != nil || payload["volume"] != nil { + t.Fatalf("upgrade must reuse the profile identity and switch browser version: %#v", payload) } if payload["binding_version"] != float64(2) || payload["network_exit_id"] != "exit-1" { t.Fatalf("upgrade must create from the committed binding generation: %#v", payload) @@ -1636,12 +1678,12 @@ func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { if cmd[0] != "--fingerprint=2024" || cmd[len(cmd)-1] != "about:blank" { t.Fatalf("upgrade must reuse stored fingerprint params: %#v", cmd) } - if store.upgraded["account-a"] != "144.0.7559.132" || store.envs["account-a"].ImageVersion != "144.0.7559.132" { - t.Fatal("image version must be persisted after successful upgrade") + if store.upgraded["account-a"] != "144.0.7559.132" || store.envs["account-a"].BrowserVersion != "144.0.7559.132" { + t.Fatal("browser version must be persisted after successful upgrade") } - if len(store.actions) != 2 || store.actions[0].OldImageVersion != "148" || - store.actions[1].NewImageVersion != "144.0.7559.132" || store.actions[1].Outcome != "succeeded" { - t.Fatalf("upgrade must emit image-aware audit evidence: %#v", store.actions) + if len(store.actions) != 2 || store.actions[0].OldBrowserVersion != "148" || + store.actions[1].NewBrowserVersion != "144.0.7559.132" || store.actions[1].Outcome != "succeeded" { + t.Fatalf("upgrade must emit browser-version audit evidence: %#v", store.actions) } } @@ -1685,11 +1727,11 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil { t.Fatal(err) } - for _, image := range []hub.Image{ - {Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}, - {Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}, + for _, image := range []hub.BrowserVersion{ + {Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}, + {Version: "149", BrowserPath: "/opt/creatorhub/browsers/149", Enabled: true}, } { - if err := store.CreateImage(ctx, image); err != nil { + if err := store.CreateBrowserVersion(ctx, image); err != nil { t.Fatal(err) } } @@ -1702,7 +1744,7 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { t.Fatal(err) } before, created, err := store.CreateBoundEnv(ctx, hub.Env{ - Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}, + Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}, }, "account-a", exit.ID) if err != nil || !created { t.Fatalf("create bound environment: created=%v err=%v", created, err) @@ -1718,7 +1760,7 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { if err != nil { t.Fatal(err) } - if after.ImageVersion != "149" || after.BindingVersion != before.BindingVersion+1 || after.RuntimeID != "" { + if after.BrowserVersion != "149" || after.BindingVersion != before.BindingVersion+1 || after.RuntimeID != "" { t.Fatalf("paused upgrade did not preserve the stopped-runtime contract: before=%#v after=%#v", before, after) } requests := gateway.recorded() @@ -1728,15 +1770,15 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { } setFixtureAccountStatus(t, databaseURL, "active") - after, err = store.ActivateRuntime(ctx, "account-a", "stale-container", after.BindingVersion, after.Exit.ID, "network-old") + after, err = store.ActivateRuntime(ctx, "account-a", "stale-container", after.BindingVersion, after.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } gateway.mu.Lock() gateway.failDelete = 1 - gateway.containers = []containerStatus{{ + gateway.runtimes = []runtimeStatus{{ ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: after.BindingVersion, - NetworkExitID: "stale-exit", NetworkID: "network-old", ProxyReady: true, + NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", ProxyReady: true, }} gateway.mu.Unlock() if err := reconcileRuntimeLeases(ctx, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil { @@ -1753,7 +1795,7 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { if err != nil || released.RuntimeCleanupPending { t.Fatalf("confirmed cleanup remained pending: %#v err=%v", released, err) } - active, err := store.ActivateRuntime(ctx, "account-a", "coherent-container", released.BindingVersion, released.Exit.ID, "network-coherent") + active, err := store.ActivateRuntime(ctx, "account-a", "coherent-container", released.BindingVersion, released.Exit.ID, "native-22222222222222222222222222222222") if err != nil { t.Fatal(err) } @@ -1764,9 +1806,9 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { setFixtureAccountStatus(t, databaseURL, "paused") gateway.mu.Lock() gateway.failDelete = 1 - gateway.containers = []containerStatus{{ + gateway.runtimes = []runtimeStatus{{ ID: "coherent-container", Alias: "account-a", State: "running", BindingVersion: released.BindingVersion, - NetworkExitID: released.Exit.ID, NetworkID: "network-coherent", ProxyReady: true, + NetworkExitID: released.Exit.ID, NetworkID: "native-22222222222222222222222222222222", ProxyReady: true, }} gateway.mu.Unlock() response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+released.Exit.ID+`"}`) @@ -1870,7 +1912,7 @@ func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFi if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil { t.Fatal(err) } - if err := store.CreateImage(ctx, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}); err != nil { + if err := store.CreateBrowserVersion(ctx, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}); err != nil { t.Fatal(err) } exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080}) @@ -1882,7 +1924,7 @@ func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFi t.Fatal(err) } bound, created, err := store.CreateBoundEnv(ctx, hub.Env{ - Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}, + Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}, }, "account-a", exit.ID) if err != nil || !created { t.Fatalf("create bound environment: created=%v err=%v", created, err) @@ -1925,11 +1967,11 @@ func TestPauseClosesClaimGateBeforeStoppingRuntime(t *testing.T) { if err != nil { t.Fatal(err) } - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "active-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-active") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "active-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-33333333333333333333333333333333") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ + fixture.gateway.runtimes = []runtimeStatus{{ ID: "active-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} @@ -1999,7 +2041,7 @@ func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = accountStore.Close() }) - fixture.gateway.containers = []containerStatus{{ + fixture.gateway.runtimes = []runtimeStatus{{ ID: "stopped-container", Alias: fixture.bound.Alias, State: "exited", BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} @@ -2016,7 +2058,7 @@ func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { t.Fatal(err) } fixture.gateway.mu.Lock() - fixture.gateway.containers[0].State = "running" + fixture.gateway.runtimes[0].State = "running" fixture.gateway.mu.Unlock() if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/revoke", ""); response.Code != http.StatusNoContent { t.Fatalf("revoke running orphan without lease: %d %s", response.Code, response.Body.String()) @@ -2031,7 +2073,7 @@ func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { t.Fatalf("revoke gate did not remain closed: %#v err=%v", account, err) } fixture.gateway.mu.Lock() - container := fixture.gateway.containers[0] + container := fixture.gateway.runtimes[0] fixture.gateway.mu.Unlock() if container.State != "exited" { t.Fatalf("running orphan was not stopped: %#v", container) @@ -2047,7 +2089,7 @@ func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { } } if stopCalls != 2 { - t.Fatalf("pause/revoke did not reconcile both lease-free containers: %#v", requests) + t.Fatalf("pause/revoke did not reconcile both lease-free runtimes: %#v", requests) } } @@ -2100,7 +2142,7 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) var err error if test.race == "upgrade" { - if err := fixture.store.CreateImage(ctx, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}); err != nil { + if err := fixture.store.CreateBrowserVersion(ctx, hub.BrowserVersion{Version: "149", BrowserPath: "/opt/creatorhub/browsers/149", Enabled: true}); err != nil { t.Fatal(err) } } @@ -2115,12 +2157,12 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { } if test.state == "running" { setFixtureAccountStatus(t, fixture.databaseURL, "active") - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, - fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, + fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - pending := runtimeCleanupGeneration(fixture.bound, fixture.bound.BindingVersion, "old-container", "network-old") + pending := runtimeCleanupGeneration(fixture.bound, fixture.bound.BindingVersion, "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "native-66666666666666666666666666666666") if err := fixture.store.SetRuntimeCleanupPending(ctx, pending, true); err != nil { t.Fatal(err) } @@ -2131,12 +2173,12 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { } deleteDone, releaseDelete := make(chan struct{}), make(chan struct{}) fixture.gateway.deleteDone, fixture.gateway.releaseDelete = deleteDone, releaseDelete - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: test.state, ProxyReady: true, + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: test.state, ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} if test.state == "running" { - fixture.gateway.containers[0].NetworkID = "network-old" + fixture.gateway.runtimes[0].NetworkID = "native-66666666666666666666666666666666" } app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) @@ -2213,19 +2255,19 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { if test.race == "version" || test.race == "upgrade" { expectedVersion++ } - expectedImage := fixture.bound.ImageVersion - if err != nil || after.BindingVersion != expectedVersion || after.Exit.ID != fixture.bound.Exit.ID || after.ImageVersion != expectedImage { + expectedBrowserVersion := fixture.bound.BrowserVersion + if err != nil || after.BindingVersion != expectedVersion || after.Exit.ID != fixture.bound.Exit.ID || after.BrowserVersion != expectedBrowserVersion { t.Fatalf("failed rebind changed PostgreSQL binding: before=%#v after=%#v err=%v", fixture.bound, after, err) } fixture.gateway.mu.Lock() - containers := append([]containerStatus{}, fixture.gateway.containers...) + runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() if test.race == "version" { - if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(containers) != 0 { - t.Fatalf("stale cleanup clear crossed the new binding generation: after=%#v containers=%#v", after, containers) + if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(runtimes) != 0 { + t.Fatalf("stale cleanup clear crossed the new binding generation: after=%#v runtimes=%#v", after, runtimes) } - } else if after.RuntimeID != "" || len(containers) != 1 || containers[0].State == "running" || !containerMatchesBinding(containers[0], after) { - t.Fatalf("paused recovery did not preserve a stopped container without a lease: after=%#v containers=%#v", after, containers) + } else if after.RuntimeID != "" || len(runtimes) != 1 || runtimes[0].State == "running" || !runtimeMatchesBinding(runtimes[0], after) { + t.Fatalf("paused recovery did not preserve a stopped container without a lease: after=%#v runtimes=%#v", after, runtimes) } if test.race == "upgrade" { requests := fixture.gateway.recorded() @@ -2235,7 +2277,7 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { lastCreate = request.body } } - if lastCreate["image"] != "registry.example/browser:148" || int64(lastCreate["binding_version"].(float64)) != after.BindingVersion { + if lastCreate["image"] != "/opt/creatorhub/browsers/148" || int64(lastCreate["binding_version"].(float64)) != after.BindingVersion { t.Fatalf("blocked upgrade changed the rebind generation: %#v", lastCreate) } } @@ -2278,13 +2320,13 @@ func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} fixture.gateway.cleanupPending = test.cleanupPending fixture.gateway.disconnectDelete = test.disconnectDelete @@ -2332,14 +2374,14 @@ func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) { func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { newRuntime := func() (*memoryStore, *fakeGateway, hub.EnvironmentContext, hub.Gateway) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", ImageVersion: "148"} + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", BrowserVersion: "148"} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } store.bindings[environment.Alias] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: environment.Alias, State: "running", ProxyReady: true, + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: environment.Alias, State: "running", ProxyReady: true, BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, }}} server := httptest.NewServer(gateway.handler(t)) @@ -2356,8 +2398,8 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { t.Fatalf("failed pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) } after := store.bindings[environment.Alias] - if after.RuntimeCleanupPending || after.RuntimeID != environment.RuntimeID || len(gateway.containers) != 1 { - t.Fatalf("rollback did not preserve the active generation: after=%#v containers=%#v", after, gateway.containers) + if after.RuntimeCleanupPending || after.RuntimeID != environment.RuntimeID || len(gateway.runtimes) != 1 { + t.Fatalf("rollback did not preserve the active generation: after=%#v runtimes=%#v", after, gateway.runtimes) } }) @@ -2371,16 +2413,16 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { t.Fatalf("unknown pre-mark result touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) } after := store.bindings[environment.Alias] - if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 1 { - t.Fatalf("committed pre-mark was not retryable: after=%#v containers=%#v", after, gateway.containers) + if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.runtimes) != 1 { + t.Fatalf("committed pre-mark was not retryable: after=%#v runtimes=%#v", after, gateway.runtimes) } store.cleanupPendingErr = nil store.cleanupPendingErrAfterMutation = false removed, err = removeGatewayRuntime(context.Background(), store, target, after) - if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending || len(gateway.containers) != 0 { - t.Fatalf("retry did not reconcile the committed pre-mark: removed=%v err=%v after=%#v containers=%#v", - removed, err, store.bindings[environment.Alias], gateway.containers) + if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending || len(gateway.runtimes) != 0 { + t.Fatalf("retry did not reconcile the committed pre-mark: removed=%v err=%v after=%#v runtimes=%#v", + removed, err, store.bindings[environment.Alias], gateway.runtimes) } }) @@ -2402,8 +2444,8 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { t.Fatalf("clear rollback lost the delete fact: removed=%v err=%v", removed, err) } after := store.bindings[environment.Alias] - if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 0 { - t.Fatalf("clear rollback was not retryable: after=%#v containers=%#v", after, gateway.containers) + if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.runtimes) != 0 { + t.Fatalf("clear rollback was not retryable: after=%#v runtimes=%#v", after, gateway.runtimes) } store.cleanupPendingErr = nil @@ -2418,7 +2460,7 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { store, gateway, environment, target := newRuntime() environment.RuntimeInstanceID, environment.RuntimeID = "", "" store.bindings[environment.Alias] = environment - gateway.containers[0].BindingVersion = environment.BindingVersion + 1 + gateway.runtimes[0].BindingVersion = environment.BindingVersion + 1 removed, err := removeGatewayRuntime(context.Background(), store, target, environment) if !errors.Is(err, hub.ErrConflict) || removed || len(gateway.recorded()) != 1 || @@ -2431,7 +2473,7 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { store, gateway, environment, target := newRuntime() environment.RuntimeInstanceID, environment.RuntimeID = "", "" store.bindings[environment.Alias] = environment - gateway.containers = nil + gateway.runtimes = nil removed, err := removeGatewayRuntime(context.Background(), store, target, environment) if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending { @@ -2446,7 +2488,7 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { t.Run("known database runtime survives missing gateway container", func(t *testing.T) { store, gateway, environment, target := newRuntime() - gateway.containers = nil + gateway.runtimes = nil removed, err := removeGatewayRuntime(context.Background(), store, target, environment) if err != nil || !removed { @@ -2478,7 +2520,7 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) { newPoisoned := func() (*memoryStore, *fakeGateway, hub.EnvironmentContext, hub.Gateway) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", ImageVersion: "148"} + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", BrowserVersion: "148"} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], @@ -2486,9 +2528,9 @@ func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) { RuntimeCleanupRuntimeID: missingRuntimeID, } store.bindings[environment.Alias] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "orphan-container", Alias: environment.Alias, State: "running", ProxyReady: true, - BindingVersion: 1, NetworkExitID: environment.Exit.ID, NetworkID: "network-orphan", + BindingVersion: 1, NetworkExitID: environment.Exit.ID, NetworkID: "native-55555555555555555555555555555555", }}} server := httptest.NewServer(gateway.handler(t)) t.Cleanup(server.Close) @@ -2503,8 +2545,8 @@ func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) { t.Fatalf("unknown generation did not converge: removed=%v err=%v", removed, err) } after := store.bindings[environment.Alias] - if after.RuntimeCleanupPending || len(gateway.containers) != 0 { - t.Fatalf("cleanup did not finish: after=%#v containers=%#v", after, gateway.containers) + if after.RuntimeCleanupPending || len(gateway.runtimes) != 0 { + t.Fatalf("cleanup did not finish: after=%#v runtimes=%#v", after, gateway.runtimes) } deleteBody := map[string]any{} for _, request := range gateway.recorded() { @@ -2512,14 +2554,14 @@ func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) { deleteBody = request.body } } - if deleteBody["runtime_id"] != "orphan-container" || deleteBody["network_id"] != "network-orphan" { + if deleteBody["runtime_id"] != "orphan-container" || deleteBody["network_id"] != "native-55555555555555555555555555555555" { t.Fatalf("cleanup did not adopt the discovered generation: %#v", deleteBody) } }) t.Run("absent runtime clears the unknown generation", func(t *testing.T) { store, gateway, environment, target := newPoisoned() - gateway.containers = nil + gateway.runtimes = nil removed, err := removeGatewayRuntime(context.Background(), store, target, environment) if err != nil || !removed { @@ -2538,15 +2580,15 @@ func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) { t.Run("replacement generation keeps manual reconcile", func(t *testing.T) { store, gateway, environment, target := newPoisoned() - gateway.containers[0].BindingVersion = 2 + gateway.runtimes[0].BindingVersion = 2 removed, err := removeGatewayRuntime(context.Background(), store, target, environment) if !errors.Is(err, hub.ErrReconcileRequired) || removed { t.Fatalf("replacement generation was not fail-closed: removed=%v err=%v", removed, err) } after := store.bindings[environment.Alias] - if !after.RuntimeCleanupPending || len(gateway.containers) != 1 { - t.Fatalf("fail-closed path mutated state: after=%#v containers=%#v", after, gateway.containers) + if !after.RuntimeCleanupPending || len(gateway.runtimes) != 1 { + t.Fatalf("fail-closed path mutated state: after=%#v runtimes=%#v", after, gateway.runtimes) } }) } @@ -2563,16 +2605,16 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { } { t.Run("restore "+test.name, func(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } store.cleanupPendingErr = errors.New("cleanup state unavailable") store.cleanupPendingErrAfterMutation = !test.commitKnown - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "network-old", + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }}} app := newTestApp(t, store, gateway) @@ -2581,14 +2623,14 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { } after := store.bindings["account-a"] if test.commitKnown { - if after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + if after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("rollback changed the old generation: %#v", after) } } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("commit-unknown lost its retry marker: %#v", after) } - if len(gateway.containers) != 1 { - t.Fatalf("failed cleanup mark touched the old container: containers=%#v requests=%#v", gateway.containers, gateway.recorded()) + if len(gateway.runtimes) != 1 { + t.Fatalf("failed cleanup mark touched the old container: runtimes=%#v requests=%#v", gateway.runtimes, gateway.recorded()) } for _, request := range gateway.recorded() { if request.method != http.MethodGet { @@ -2612,22 +2654,22 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { } after = store.bindings["account-a"] } - if after.RuntimeCleanupPending || after.RuntimeID == "" || len(gateway.containers) != 1 || !containerMatchesBinding(gateway.containers[0], after) { - t.Fatalf("retry ended inconsistently: after=%#v containers=%#v", after, gateway.containers) + if after.RuntimeCleanupPending || after.RuntimeID == "" || len(gateway.runtimes) != 1 || !runtimeMatchesBinding(gateway.runtimes[0], after) { + t.Fatalf("retry ended inconsistently: after=%#v runtimes=%#v", after, gateway.runtimes) } }) t.Run("discard "+test.name, func(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } store.cleanupPendingErr = errors.New("cleanup state unavailable") store.cleanupPendingErrAfterMutation = !test.commitKnown - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", }}} app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) @@ -2638,14 +2680,14 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { } after := store.bindings["account-a"] if test.commitKnown { - if after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + if after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("discard rollback changed the old generation: %#v", after) } } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("discard commit-unknown lost its retry marker: %#v", after) } - if len(gateway.containers) != 1 { - t.Fatalf("failed discard mark touched the gateway: containers=%#v requests=%#v", gateway.containers, gateway.recorded()) + if len(gateway.runtimes) != 1 { + t.Fatalf("failed discard mark touched the gateway: runtimes=%#v requests=%#v", gateway.runtimes, gateway.recorded()) } for _, request := range gateway.recorded() { if request.method != http.MethodGet { @@ -2663,8 +2705,8 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String()) } after = store.bindings["account-a"] - if after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 0 { - t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v", after, gateway.containers) + if after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.runtimes) != 0 { + t.Fatalf("discard retry ended inconsistently: after=%#v runtimes=%#v", after, gateway.runtimes) } if len(store.actions) != 4 || store.actions[3].Outcome != "failed" || store.actions[3].ReasonCode != "proxy_auth_failed" { t.Fatalf("discard retry audit mismatch: %#v", store.actions) @@ -2675,23 +2717,23 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } - gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-old", + gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666", }}} app := newTestApp(t, store, gateway) - body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}` + body := `{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}` response := do(app, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusBadGateway { t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String()) } - if after := store.bindings["account-a"]; after.RuntimeID != "old-container" || after.RuntimeCleanupPending { + if after := store.bindings["account-a"]; after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || after.RuntimeCleanupPending { t.Fatalf("gateway unknown changed the existing lease: %#v", after) } for _, request := range gateway.recorded() { @@ -2707,11 +2749,11 @@ func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) { if response.Code != http.StatusOK { t.Fatalf("create retry did not reuse the runtime: %d: %s", response.Code, response.Body.String()) } - if after := store.bindings["account-a"]; after.RuntimeID != "old-container" || after.RuntimeCleanupPending { + if after := store.bindings["account-a"]; after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || after.RuntimeCleanupPending { t.Fatalf("create retry changed the matching generation: %#v", after) } - if len(gateway.containers) != 1 || len(store.actions) != 4 || store.actions[3].Outcome != "succeeded" || store.actions[3].ReasonCode != "environment_reused" { - t.Fatalf("create retry ended inconsistently: containers=%#v actions=%#v", gateway.containers, store.actions) + if len(gateway.runtimes) != 1 || len(store.actions) != 4 || store.actions[3].Outcome != "succeeded" || store.actions[3].ReasonCode != "environment_reused" { + t.Fatalf("create retry ended inconsistently: runtimes=%#v actions=%#v", gateway.runtimes, store.actions) } } @@ -2722,29 +2764,29 @@ func TestRemoveGatewayRuntimePreservesKnownNetworkGeneration(t *testing.T) { wantConflict bool }{ {name: "legacy gateway omits network", observedNetwork: ""}, - {name: "replacement network conflicts", observedNetwork: "network-n2", wantConflict: true}, + {name: "replacement network conflicts", observedNetwork: "native-88888888888888888888888888888888", wantConflict: true}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], - RuntimeInstanceID: "runtime-instance", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"} + RuntimeInstanceID: "runtime-instance", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"} store.bindings["account-a"] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "container-c1", Alias: "account-a", + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: test.observedNetwork, ProxyReady: true}}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} _, err := removeGatewayRuntime(context.Background(), store, target, environment) if test.wantConflict { - if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 1 || store.bindings["account-a"].RuntimeID != "container-c1" { + if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 1 || store.bindings["account-a"].RuntimeID != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { t.Fatalf("replacement network was not fenced: err=%v requests=%#v environment=%#v", err, gateway.recorded(), store.bindings["account-a"]) } return } requests := gateway.recorded() - if err != nil || len(requests) != 2 || requests[1].body["network_id"] != "network-n1" || store.bindings["account-a"].RuntimeCleanupPending { + if err != nil || len(requests) != 2 || requests[1].body["network_id"] != "native-99999999999999999999999999999999" || store.bindings["account-a"].RuntimeCleanupPending { t.Fatalf("known N1 was not preserved through cleanup: err=%v requests=%#v environment=%#v", err, requests, store.bindings["account-a"]) } }) @@ -2753,22 +2795,22 @@ func TestRemoveGatewayRuntimePreservesKnownNetworkGeneration(t *testing.T) { func TestGatewayLookupFailurePreservesLease(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } store.bindings[environment.Alias] = environment store.gatewayFn = func(string) (hub.Gateway, error) { return hub.Gateway{}, errors.New("gateway lookup unavailable") } if _, err := restoreOrRebuildRuntime(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }, - environment, containerStatus{ID: "old-container", Alias: environment.Alias, State: "running"}); err == nil { + environment, runtimeStatus{ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: environment.Alias, State: "running"}); err == nil { t.Fatal("restore accepted an unknown gateway") } if err := discardRuntime(context.Background(), store, environment); err == nil { t.Fatal("discard accepted an unknown gateway") } - if after := store.bindings[environment.Alias]; after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + if after := store.bindings[environment.Alias]; after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("gateway lookup failure changed the old generation: %#v", after) } } @@ -2784,12 +2826,12 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) @@ -2803,8 +2845,8 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { t.Fatalf("rolled-back pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, cleanupErr, fixture.gateway.recorded()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { - t.Fatalf("PostgreSQL rollback did not preserve the active generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { + t.Fatalf("PostgreSQL rollback did not preserve the active generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) @@ -2815,8 +2857,8 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { t.Fatalf("202 retry did not retain pending cleanup: removed=%v err=%v", removed, cleanupErr) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 { - t.Fatalf("202 retry was not durable: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.runtimes) != 0 { + t.Fatalf("202 retry was not durable: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) }) @@ -2825,12 +2867,12 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} fixture.gateway.deleteNotFound = 1 @@ -2845,8 +2887,8 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { t.Fatalf("404 clear rollback lost the delete fact: removed=%v err=%v", removed, cleanupErr) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 { - t.Fatalf("clear rollback was not durably pending: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.runtimes) != 0 { + t.Fatalf("clear rollback was not durably pending: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) @@ -2862,10 +2904,10 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() - containers := append([]containerStatus{}, fixture.gateway.containers...) + runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() - if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { - t.Fatalf("lifecycle retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) { + t.Fatalf("lifecycle retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } }) } @@ -2905,7 +2947,7 @@ func TestPostgresCleanupMissingContainerReleasesKnownLease(t *testing.T) { ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") - environment, err := fixture.store.ActivateRuntime(ctx, fixture.bound.Alias, "known-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-known") + environment, err := fixture.store.ActivateRuntime(ctx, fixture.bound.Alias, "known-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-44444444444444444444444444444444") if err != nil { t.Fatal(err) } @@ -2949,12 +2991,12 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} var store hubStore = fixture.store @@ -2973,13 +3015,13 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) } else { - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("restore rollback changed the old generation: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } - if len(fixture.gateway.containers) != 1 { - t.Fatalf("restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) + if len(fixture.gateway.runtimes) != 1 { + t.Fatalf("restore cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded()) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { @@ -3011,10 +3053,10 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() - containers := append([]containerStatus{}, fixture.gateway.containers...) + runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() - if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { - t.Fatalf("restore retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) { + t.Fatalf("restore retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) }) @@ -3023,12 +3065,12 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} var store hubStore = fixture.store @@ -3039,7 +3081,7 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) - body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` + body := `{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` response := do(app, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusInternalServerError { @@ -3052,13 +3094,13 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) } else { - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("create restore rollback changed the old generation: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } - if len(fixture.gateway.containers) != 1 { - t.Fatalf("create restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) + if len(fixture.gateway.runtimes) != 1 { + t.Fatalf("create restore cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded()) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { @@ -3086,10 +3128,10 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() - containers := append([]containerStatus{}, fixture.gateway.containers...) + runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() - if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { - t.Fatalf("create restore retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) { + t.Fatalf("create restore retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) }) @@ -3098,13 +3140,13 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} var store hubStore = fixture.store if test.commitUnknown { @@ -3126,13 +3168,13 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) } else { - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("discard rollback changed the old generation: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } - if len(fixture.gateway.containers) != 1 { - t.Fatalf("discard cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) + if len(fixture.gateway.runtimes) != 1 { + t.Fatalf("discard cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded()) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { @@ -3157,11 +3199,11 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" { - t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + t.Fatalf("discard retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } - for _, container := range fixture.gateway.containers { + for _, container := range fixture.gateway.runtimes { if container.State == "running" { - t.Fatalf("discard retry retained a running container: %#v", fixture.gateway.containers) + t.Fatalf("discard retry retained a running container: %#v", fixture.gateway.runtimes) } } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) @@ -3187,13 +3229,13 @@ func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate(t *testing. fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} test.set(fixture.gateway, 1) app := fiber.New() @@ -3203,12 +3245,13 @@ func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate(t *testing. t.Fatalf("gateway failure affected browser list: %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { - t.Fatalf("browser list changed the generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { + t.Fatalf("browser list changed the generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) - if requests := fixture.gateway.recorded(); len(requests) != 0 { - t.Fatalf("browser list performed live gateway detection: %#v", requests) + requests := fixture.gateway.recorded() + if len(requests) != 1 || requests[0].method != http.MethodGet || requests[0].path != "/v1/browsers" { + t.Fatalf("browser list did not perform the expected live gateway status read: %#v", requests) } }) @@ -3216,25 +3259,25 @@ func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate(t *testing. fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} test.set(fixture.gateway, gatewayReconcileAttempts) app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) - body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` + body := `{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusBadGateway { t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { - t.Fatalf("gateway unknown create changed the generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { + t.Fatalf("gateway unknown create changed the generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } for _, request := range fixture.gateway.recorded() { if request.method == http.MethodPost { @@ -3255,8 +3298,8 @@ func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate(t *testing. t.Fatalf("gateway create retry failed: %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { - t.Fatalf("gateway create retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + if err != nil || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { + t.Fatalf("gateway create retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'create' @@ -3302,17 +3345,17 @@ func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry(t *testing.T) if test.active { setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } } - state, networkExitID, networkID := "running", fixture.bound.Exit.ID, "network-old" + state, networkExitID, networkID := "running", fixture.bound.Exit.ID, "native-66666666666666666666666666666666" if test.action == "rebind" { state, networkExitID, networkID = "exited", "", "" } - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: state, ProxyReady: state == "running", + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: state, ProxyReady: state == "running", BindingVersion: fixture.bound.BindingVersion, NetworkExitID: networkExitID, NetworkID: networkID, }} attempts := gatewayReconcileAttempts @@ -3324,7 +3367,7 @@ func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry(t *testing.T) registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) body := "" if test.action == "create" { - body = `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` + body = `{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` } else if test.action == "rebind" { body = `{"network_exit_id":"` + fixture.exit.ID + `"}` } @@ -3335,9 +3378,9 @@ func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry(t *testing.T) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.BindingVersion != fixture.bound.BindingVersion || after.Exit.ID != fixture.bound.Exit.ID || - after.RuntimeCleanupPending || after.RuntimeID != fixture.bound.RuntimeID || len(fixture.gateway.containers) != 1 { - t.Fatalf("strict gateway validation changed the generation: before=%#v after=%#v containers=%#v err=%v", - fixture.bound, after, fixture.gateway.containers, err) + after.RuntimeCleanupPending || after.RuntimeID != fixture.bound.RuntimeID || len(fixture.gateway.runtimes) != 1 { + t.Fatalf("strict gateway validation changed the generation: before=%#v after=%#v runtimes=%#v err=%v", + fixture.bound, after, fixture.gateway.runtimes, err) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { @@ -3367,18 +3410,18 @@ func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry(t *testing.T) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() - containers := append([]containerStatus{}, fixture.gateway.containers...) + runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() wantVersion := fixture.bound.BindingVersion if test.action == "rebind" { wantVersion++ } - validRuntime := after.RuntimeID != "" && len(containers) == 1 && containerMatchesBinding(containers[0], after) + validRuntime := after.RuntimeID != "" && len(runtimes) == 1 && runtimeMatchesBinding(runtimes[0], after) if test.action == "rebind" { - validRuntime = after.RuntimeID == "" && len(containers) == 0 + validRuntime = after.RuntimeID == "" && len(runtimes) == 0 } if err != nil || after.BindingVersion != wantVersion || !validRuntime { - t.Fatalf("strict gateway retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + t.Fatalf("strict gateway retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } activeAfter := 1 if test.action == "rebind" { @@ -3439,9 +3482,9 @@ func assertControlPlaneDatabaseCount(t *testing.T, db *sql.DB, query string, wan func TestUpgradeBrowserStopsBeforeCreateWhenPersistenceFails(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} store.upgradeErr = hub.ErrNotFound - _ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -3454,16 +3497,16 @@ func TestUpgradeBrowserStopsBeforeCreateWhenPersistenceFails(t *testing.T) { t.Fatalf("failed persistence must abort before creating the upgraded container: %#v", requests) } env, err := store.GetEnv(context.Background(), "account-a") - if err != nil || env.ImageVersion != "148" { + if err != nil || env.BrowserVersion != "148" { t.Fatalf("failed upgrade must preserve the stored version: %#v %v", env, err) } } func TestUpgradeStopsBeforeCreateWhenRuntimeReleaseFails(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} store.releaseErr = errors.New("database unavailable") - _ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -3484,7 +3527,7 @@ func TestUpgradeStopsBeforeCreateWhenRuntimeReleaseFails(t *testing.T) { func TestUpgradeRejectsInvalidVersionWithoutAuditingRawInput(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -3496,7 +3539,7 @@ func TestUpgradeRejectsInvalidVersionWithoutAuditingRawInput(t *testing.T) { t.Fatalf("invalid version must be rejected before gateway side effects: requests=%#v actions=%#v", gateway.recorded(), store.actions) } for _, action := range store.actions { - if action.NewImageVersion != "" || action.ReasonCode != "upgrade_input_rejected" { + if action.NewBrowserVersion != "" || action.ReasonCode != "upgrade_input_rejected" { t.Fatalf("raw invalid version reached audit: %#v", store.actions) } } @@ -3555,19 +3598,19 @@ func TestPostgresReconcileFinishedAuditUsesActivatedRuntime(t *testing.T) { setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(context.Background(), "account-a", "stale-container", - fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ + fixture.gateway.runtimes = []runtimeStatus{{ ID: "stale-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", NetworkID: "network-old", + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }} if err := reconcileRuntimeLeases(context.Background(), fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile failed: %v", err) } after, err := fixture.store.GetEnvironmentContext(context.Background(), "account-a") - if err != nil || after.RuntimeInstanceID == "" || after.RuntimeID != "container-id" { + if err != nil || after.RuntimeInstanceID == "" || after.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("reconcile did not activate replacement runtime: %#v err=%v", after, err) } var auditedRuntime string @@ -3610,7 +3653,7 @@ func TestPostgresNonRunnableReconcileAuditsRuntimeRelease(t *testing.T) { t.Fatal(err) } if observation == "stopped" { - fixture.gateway.containers = []containerStatus{{ + fixture.gateway.runtimes = []runtimeStatus{{ ID: "stopped-container", Alias: "account-a", State: "exited", BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} @@ -3682,18 +3725,18 @@ func TestPostgresImageDisableWaitsForEveryImageConsumer(t *testing.T) { t.Fatal(err) } path, expected = "/api/browsers", http.StatusCreated - body = fmt.Sprintf(`{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"%s"}`, fixture.exit.ID) + body = fmt.Sprintf(`{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"%s"}`, fixture.exit.ID) case "reconcile": method, path, expected = http.MethodGet, "/api/browsers", http.StatusOK var err error fixture.bound, err = fixture.store.ActivateRuntime(context.Background(), "account-a", "stale-container", - fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") + fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } - fixture.gateway.containers = []containerStatus{{ + fixture.gateway.runtimes = []runtimeStatus{{ ID: "stale-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", NetworkID: "network-old", + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }} case "rebind": expected = http.StatusConflict @@ -3701,9 +3744,9 @@ func TestPostgresImageDisableWaitsForEveryImageConsumer(t *testing.T) { fixture.gateway.createStarted = nil fixture.gateway.releaseCreate = nil probe = &blockingExitProbe{started: lifecycleStarted, release: releaseLifecycle} - fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", + fixture.gateway.runtimes = []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} } @@ -3764,8 +3807,8 @@ func TestPostgresImageDisableWaitsForEveryImageConsumer(t *testing.T) { disableDone := make(chan result, 1) go func() { - request, err := http.NewRequest(http.MethodPut, server.URL+"/api/browser-images/148", strings.NewReader( - `{"image_ref":"registry.example/browser:148","enabled":false}`)) + request, err := http.NewRequest(http.MethodPut, server.URL+"/api/browser-versions/148", strings.NewReader( + `{"browser_path":"/opt/creatorhub/browsers/148","enabled":false}`)) if err == nil { request.Header.Set("Content-Type", "application/json") var response *http.Response @@ -3804,8 +3847,8 @@ func TestImageDisableWaitsForUpgradeCommit(t *testing.T) { releaseCreate: releaseCreate, } store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} - _ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: true}) gatewayServer := httptest.NewServer(gateway.handler(t)) defer gatewayServer.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token} @@ -3844,8 +3887,8 @@ func TestImageDisableWaitsForUpgradeCommit(t *testing.T) { disableDone := make(chan result, 1) go func() { - request, err := http.NewRequest(http.MethodPut, server.URL+"/api/browser-images/144.0.7559.132", strings.NewReader( - `{"image_ref":"registry.example/browser:144","enabled":false}`)) + request, err := http.NewRequest(http.MethodPut, server.URL+"/api/browser-versions/144.0.7559.132", strings.NewReader( + `{"browser_path":"/opt/creatorhub/browsers/144","enabled":false}`)) if err != nil { disableDone <- result{err: err} return @@ -3874,10 +3917,10 @@ func TestImageDisableWaitsForUpgradeCommit(t *testing.T) { t.Fatalf("disable failed: %#v", result) } env, err := store.GetEnv(context.Background(), "account-a") - if err != nil || env.ImageVersion != "144.0.7559.132" { + if err != nil || env.BrowserVersion != "144.0.7559.132" { t.Fatalf("upgrade must commit before disable: %#v %v", env, err) } - if _, err := store.ImageRef(context.Background(), "144.0.7559.132"); !errors.Is(err, hub.ErrNotFound) { + if _, err := store.BrowserPath(context.Background(), "144.0.7559.132"); !errors.Is(err, hub.ErrNotFound) { t.Fatalf("disable must apply after upgrade: %v", err) } } @@ -3890,8 +3933,8 @@ func TestListDoesNotWaitForUpgradeWhileHeartbeatDoes(t *testing.T) { releaseCreate: releaseCreate, } store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} - _ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "149", BrowserPath: "/opt/creatorhub/browsers/149", Enabled: true}) gatewayServer := httptest.NewServer(gateway.handler(t)) defer gatewayServer.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token} @@ -3973,8 +4016,8 @@ func TestListDoesNotWaitForUpgradeWhileHeartbeatDoes(t *testing.T) { func TestBrowserActionRoutesStartStopAndRejectsUnknown(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -4002,7 +4045,7 @@ func TestStartRejectsPausedOrRevokedAccountBeforeGatewayCall(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: test.status, AuthorizationStatus: test.authorization, BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], @@ -4025,14 +4068,14 @@ func TestReconcileStopsRuntimeForPausedOrRevokedAccount(t *testing.T) { for _, authorization := range []string{"authorized", "revoked"} { t.Run(authorization, func(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: authorization, BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], - RuntimeInstanceID: "old-instance", RuntimeID: "old-container", + RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", }}} _ = newTestApp(t, store, gateway) @@ -4062,25 +4105,25 @@ func TestActivationConflictCleanupIsGenerationSafe(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], } store.bindings[environment.Alias] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: test.containerID, Alias: environment.Alias, State: "running", ProxyReady: true, - BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, NetworkID: "network-candidate", + BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, NetworkID: "native-11111111111111111111111111111111", }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} - _, err := activateGatewayRuntime(context.Background(), store, target, environment, "candidate-container", "network-candidate") + _, err := activateGatewayRuntime(context.Background(), store, target, environment, "candidate-container", "native-11111111111111111111111111111111") if err == nil { t.Fatal("paused account unexpectedly activated the candidate") } - deleted := len(gateway.containers) == 0 + deleted := len(gateway.runtimes) == 0 if deleted != test.wantDeleted { t.Fatalf("generation-safe cleanup mismatch: deleted=%v requests=%#v", deleted, gateway.recorded()) } @@ -4097,37 +4140,37 @@ func TestActivationConflictCleanupIsGenerationSafe(t *testing.T) { func TestExplicitStopRejectsStaleBindingBeforeGatewaySideEffect(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "new-instance", RuntimeID: "new-container", } - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "new-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} stale := store.bindings["account-a"] - stale.BindingVersion, stale.RuntimeInstanceID, stale.RuntimeID = 1, "old-instance", "old-container" + stale.BindingVersion, stale.RuntimeInstanceID, stale.RuntimeID = 1, "old-instance", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" if err := stopEnvironmentRuntime(context.Background(), store, stale); !errors.Is(err, hub.ErrConflict) { t.Fatalf("stale explicit stop was not rejected: %v", err) } - if len(gateway.recorded()) != 0 || len(gateway.containers) != 1 || gateway.containers[0].ID != "new-container" { - t.Fatalf("stale explicit stop reached the gateway: requests=%#v containers=%#v", gateway.recorded(), gateway.containers) + if len(gateway.recorded()) != 0 || len(gateway.runtimes) != 1 || gateway.runtimes[0].ID != "new-container" { + t.Fatalf("stale explicit stop reached the gateway: requests=%#v runtimes=%#v", gateway.recorded(), gateway.runtimes) } } func TestStopEnvironmentRuntimeReconcilesContainerWithoutLease(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148"} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148"} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], } store.bindings[environment.Alias] = environment - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "orphan-container", Alias: environment.Alias, State: "running", BindingVersion: environment.BindingVersion, }}} server := httptest.NewServer(gateway.handler(t)) @@ -4138,8 +4181,8 @@ func TestStopEnvironmentRuntimeReconcilesContainerWithoutLease(t *testing.T) { t.Fatalf("stop orphan without lease: %v", err) } after := store.bindings[environment.Alias] - if after.RuntimeCleanupPending || after.RuntimeID != "" || gateway.containers[0].State != "exited" { - t.Fatalf("lease-free orphan stop did not converge: after=%#v containers=%#v", after, gateway.containers) + if after.RuntimeCleanupPending || after.RuntimeID != "" || gateway.runtimes[0].State != "exited" { + t.Fatalf("lease-free orphan stop did not converge: after=%#v runtimes=%#v", after, gateway.runtimes) } requests := gateway.recorded() if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost || @@ -4151,14 +4194,14 @@ func TestStopEnvironmentRuntimeReconcilesContainerWithoutLease(t *testing.T) { func TestReconcileFinishedAuditUsesRebuiltRuntime(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", - BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-2"], RuntimeInstanceID: "old-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-2"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "network-old", + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666", }}} _ = newTestApp(t, store, gateway) @@ -4173,14 +4216,14 @@ func TestReconcileFinishedAuditUsesRebuiltRuntime(t *testing.T) { func TestReconcileContextRefreshFailureClearsAllAuditCorrelation(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", - BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "old-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "network-old", + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() @@ -4199,7 +4242,7 @@ func TestReconcileContextRefreshFailureClearsAllAuditCorrelation(t *testing.T) { func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) { gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet { - _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "exited"}}) + _ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "exited"}}) return } connection, _, err := response.(http.Hijacker).Hijack() @@ -4209,8 +4252,8 @@ func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) { })) defer gatewayServer.Close() store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} - store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id"} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} store.releaseErr = errors.New("database unavailable") store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() @@ -4227,14 +4270,14 @@ func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) { func TestReconcileAuditsRuntimeReleaseFailure(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } store.releaseErr = errors.New("database unavailable") - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-id", Alias: "account-a", State: "exited", BindingVersion: 1, NetworkExitID: "exit-1", + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "exited", BindingVersion: 1, NetworkExitID: "exit-1", }}} _ = newTestApp(t, store, gateway) @@ -4250,14 +4293,14 @@ func TestReconcileAuditsRuntimeReleaseFailure(t *testing.T) { func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4266,7 +4309,7 @@ func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { t.Fatalf("running runtime rebind failed: %d: %s", response.Code, response.Body.String()) } bound := store.bindings["account-a"] - if bound.Exit.ID != "exit-2" || bound.BindingVersion != 2 || bound.RuntimeID != "container-id" { + if bound.Exit.ID != "exit-2" || bound.BindingVersion != 2 || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("running runtime was not rebuilt on the latest binding: %#v", bound) } requests := gateway.recorded() @@ -4280,14 +4323,14 @@ func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "expired-runtime", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "expired-runtime", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4296,7 +4339,7 @@ func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) { t.Fatalf("same-exit rebind failed: %d: %s", response.Code, response.Body.String()) } bound := store.bindings["account-a"] - if bound.BindingVersion != 2 || bound.RuntimeID != "container-id" { + if bound.BindingVersion != 2 || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("same-exit rebind did not rotate the binding generation and lease: %#v", bound) } requests := gateway.recorded() @@ -4310,13 +4353,13 @@ func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) { func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} probe := fakeExitProbe{failure: "exit_auth_failed"} _ = newTestAppWithNetwork(t, store, gateway, probe, @@ -4336,13 +4379,13 @@ func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) { func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", RuntimeNetworkID: "network-old", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", RuntimeNetworkID: "native-66666666666666666666666666666666", } - gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{ - ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "network-old", ProxyReady: true, + gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, runtimes: []runtimeStatus{{ + ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", ProxyReady: true, }}} _ = newTestApp(t, store, gateway) @@ -4356,20 +4399,20 @@ func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) { store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("delete failure was not audited as retryable unknown: %#v", store.actions) } - if len(gateway.containers) != 1 { + if len(gateway.runtimes) != 1 { t.Fatal("failed delete unexpectedly removed the gateway container") } } func TestDisableExitImmediatelyDiscardsRuntime(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4394,11 +4437,11 @@ func TestDisableExitWaitsForInFlightBrowserLifecycle(t *testing.T) { releaseCreate: releaseCreate, } store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) gatewayServer := httptest.NewServer(gateway.handler(t)) defer gatewayServer.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token} @@ -4472,10 +4515,10 @@ func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) gatewayServer := httptest.NewServer(test.handler) defer gatewayServer.Close() store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() @@ -4485,7 +4528,7 @@ func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) if response.Code != http.StatusBadGateway { t.Fatalf("gateway unknown must not return 200: %d: %s", response.Code, response.Body.String()) } - if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" { + if runtime := store.bindings["account-a"].RuntimeID; runtime != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("gateway unknown changed the unconfirmed runtime lease: %q", runtime) } if store.exits["exit-1"].HealthStatus != "disabled" { @@ -4497,15 +4540,15 @@ func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) func TestDirectBindingIsListable(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{ + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{ Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true, }} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - RuntimeInstanceID: "direct-runtime", RuntimeID: "direct-container", RuntimeNetworkID: "direct-network", + RuntimeInstanceID: "4444444444444444444444444444444444444444444444444444444444444444", RuntimeID: "direct-container", RuntimeNetworkID: "direct-network", } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "direct-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkID: "direct-network", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4529,22 +4572,22 @@ func TestRebindDeleteFailureKeepsOriginalBinding(t *testing.T) { {name: "running binding", binding: hub.EnvironmentContext{ AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy"}, - RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", }, targetID: "exit-2"}, {name: "legacy NULL binding", binding: hub.EnvironmentContext{ AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - RuntimeInstanceID: "legacy-runtime", RuntimeID: "legacy-container", + RuntimeInstanceID: "5555555555555555555555555555555555555555555555555555555555555555", RuntimeID: "legacy-container", }, targetID: "exit-1"}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} before := test.binding before.Env = store.envs["account-a"] store.bindings["account-a"] = before - gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{ + gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, runtimes: []runtimeStatus{{ ID: before.RuntimeID, Alias: "account-a", State: "running", BindingVersion: before.BindingVersion, NetworkExitID: before.Exit.ID, ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4557,8 +4600,8 @@ func TestRebindDeleteFailureKeepsOriginalBinding(t *testing.T) { if after.BindingVersion != before.BindingVersion || after.Exit.ID != before.Exit.ID || !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("binding committed before old container deletion: before=%#v after=%#v", before, after) } - if len(gateway.containers) != 1 { - t.Fatalf("non-final delete result lost the existing container: %#v", gateway.containers) + if len(gateway.runtimes) != 1 { + t.Fatalf("non-final delete result lost the existing container: %#v", gateway.runtimes) } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("delete failure was not audited as unknown: %#v", store.actions) @@ -4579,17 +4622,17 @@ func TestRebindNetworkCleanupPendingBlocksUntilConfirmed(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: test.cleanupPending, disconnectDelete: test.disconnectDelete, disconnectListAfterDelete: test.disconnectListAfterDelete, - containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4604,8 +4647,8 @@ func TestRebindNetworkCleanupPendingBlocksUntilConfirmed(t *testing.T) { if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("network cleanup uncertainty audit mismatch: %#v", store.actions) } - if len(gateway.containers) != 0 { - t.Fatalf("container-removed fact was lost: %#v", gateway.containers) + if len(gateway.runtimes) != 0 { + t.Fatalf("container-removed fact was lost: %#v", gateway.runtimes) } response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) @@ -4624,7 +4667,7 @@ func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) { for _, test := range []struct { name, method, path, body string }{ - {name: "create reuse", method: http.MethodPost, path: "/api/browsers", body: `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`}, + {name: "create reuse", method: http.MethodPost, path: "/api/browsers", body: `{"alias":"account-a","name":"甲","gateway":"gw-1","browser_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`}, {name: "start", method: http.MethodPost, path: "/api/browsers/account-a/start"}, {name: "upgrade", method: http.MethodPost, path: "/api/browsers/account-a/upgrade", body: `{"version":"149"}`}, {name: "rebind", method: http.MethodPost, path: "/api/browsers/account-a/rebind", body: `{"network_exit_id":"exit-1"}`}, @@ -4632,14 +4675,14 @@ func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, RuntimeCleanupPending: true, RuntimeCleanupBindingVersion: 1, - RuntimeCleanupRuntimeID: missingRuntimeID, RuntimeCleanupNetworkID: "network-old", Exit: store.exits["exit-1"], + RuntimeCleanupRuntimeID: missingRuntimeID, RuntimeCleanupNetworkID: "native-66666666666666666666666666666666", Exit: store.exits["exit-1"], } - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - _ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "149", BrowserPath: "/opt/creatorhub/browsers/149", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: 2} app := newTestApp(t, store, gateway) @@ -4663,11 +4706,11 @@ func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) { func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", - BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container"} - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"} + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, nil) @@ -4675,20 +4718,20 @@ func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) { if response.Code < 400 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { t.Fatalf("runtime preparation failure touched the old container: status=%d requests=%#v", response.Code, gateway.recorded()) } - if after := store.bindings["account-a"]; after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "old-container" { + if after := store.bindings["account-a"]; after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("runtime preparation failure changed state: %#v", after) } } func TestRebindCandidateUnknownCreateLeavesGenerationPending(t *testing.T) { store := newMemoryStore() - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", - BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container"} - gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"} + gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, runtimes: []runtimeStatus{{ + ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -4699,8 +4742,8 @@ func TestRebindCandidateUnknownCreateLeavesGenerationPending(t *testing.T) { after := store.bindings["account-a"] if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" || !after.RuntimeCleanupPending || after.RuntimeCleanupBindingVersion != 2 || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" || - len(gateway.containers) != 0 { - t.Fatalf("candidate create unknown result was not retained for manual reconcile: after=%#v containers=%#v", after, gateway.containers) + len(gateway.runtimes) != 0 { + t.Fatalf("candidate create unknown result was not retained for manual reconcile: after=%#v runtimes=%#v", after, gateway.runtimes) } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("candidate create failure audit mismatch: %#v", store.actions) @@ -4722,14 +4765,14 @@ func TestExistingEnvironmentCleanupNeverReturnsReusedSuccess(t *testing.T) { ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1, Username: "username", Password: "password", } - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148.0.7778.215", Fingerprint: hub.Fingerprint{Seed: 2024, Platform: "windows", Timezone: "Asia/Shanghai"}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", BrowserVersion: "148.0.7778.215", Fingerprint: hub.Fingerprint{Seed: 2024, Platform: "windows", Timezone: "Asia/Shanghai"}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } - _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ + ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", }}} app := newTestAppWithNetwork(t, store, gateway, test.probe, test.resolve) @@ -4753,10 +4796,10 @@ func TestExistingEnvironmentCleanupNeverReturnsReusedSuccess(t *testing.T) { func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1} - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"]} - _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "old-container", Alias: "account-a", State: "exited", BindingVersion: 1}}} + _ = store.CreateBrowserVersion(nil, hub.BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "exited", BindingVersion: 1}}} app := newTestApp(t, store, gateway) if response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`); response.Code != http.StatusOK { @@ -4783,7 +4826,7 @@ func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) { func TestRecycleBrowserKeepsStableEnvironment(t *testing.T) { store := newMemoryStore() - store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) @@ -4804,15 +4847,15 @@ func TestGatewayAndImageCRUDRoutes(t *testing.T) { gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) - if response := do(app, http.MethodPost, "/api/browser-images", - `{"version":"148.0.7778.215","image_ref":"registry.example/browser:148","note":"main","enabled":true}`); response.Code != http.StatusCreated { + if response := do(app, http.MethodPost, "/api/browser-versions", + `{"version":"148.0.7778.215","browser_path":"/opt/creatorhub/browsers/148","note":"main","enabled":true}`); response.Code != http.StatusCreated { t.Fatalf("expected 201 for image create, got %d: %s", response.Code, response.Body.String()) } - if image, ok := store.images["148.0.7778.215"]; !ok || image.ImageRef != "registry.example/browser:148" || !image.Enabled { + if image, ok := store.images["148.0.7778.215"]; !ok || image.BrowserPath != "/opt/creatorhub/browsers/148" || !image.Enabled { t.Fatalf("image must be stored: %#v", store.images) } - if response := do(app, http.MethodPost, "/api/browser-images", - `{"version":"148.0.7778.215","image_ref":"registry.example/browser:148"}`); response.Code != http.StatusCreated { + if response := do(app, http.MethodPost, "/api/browser-versions", + `{"version":"148.0.7778.215","browser_path":"/opt/creatorhub/browsers/148"}`); response.Code != http.StatusCreated { t.Fatalf("enabled must default to true, got %d", response.Code) } } @@ -4848,25 +4891,25 @@ func TestNetworkExitRoutesStoreAndExposePlainCredentials(t *testing.T) { } } -func TestCreateImageReturnsJSONOverHTTP(t *testing.T) { +func TestCreateBrowserVersionReturnsJSONOverHTTP(t *testing.T) { store := newMemoryStore() app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) server := httptest.NewServer(adaptor.FiberApp(app)) defer server.Close() - response, err := server.Client().Post(server.URL+"/api/browser-images", "application/json", strings.NewReader( - `{"version":"148.0.7778.215","image_ref":"registry.example/browser:148","note":"main","enabled":true}`)) + response, err := server.Client().Post(server.URL+"/api/browser-versions", "application/json", strings.NewReader( + `{"version":"148.0.7778.215","browser_path":"/opt/creatorhub/browsers/148","note":"main","enabled":true}`)) if err != nil { t.Fatal(err) } defer response.Body.Close() - var image hub.Image + var image hub.BrowserVersion if err := json.NewDecoder(response.Body).Decode(&image); err != nil { t.Fatalf("201 response must be JSON: %v", err) } if response.StatusCode != http.StatusCreated || !strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") || - image.Version != "148.0.7778.215" || image.ImageRef != "registry.example/browser:148" || !image.Enabled { + image.Version != "148.0.7778.215" || image.BrowserPath != "/opt/creatorhub/browsers/148" || !image.Enabled { t.Fatalf("unexpected create response: status=%d content-type=%q image=%#v", response.StatusCode, response.Header.Get("Content-Type"), image) } } diff --git a/cmd/control-plane/main.go b/cmd/control-plane/main.go index 58c2cc0..9554e54 100644 --- a/cmd/control-plane/main.go +++ b/cmd/control-plane/main.go @@ -278,7 +278,7 @@ func newHandlerWithCreatorAndAI(webDirectory, username, password string, phaseAS app := fiber.New(fiber.Config{ AppName: "CreatorHub control plane", BodyLimit: 1 << 20, - // 读超时只约束请求读取；创建/升级环境的处理器可等待网关拉取镜像（最长 11 分钟）。 + // 读超时只约束请求读取；创建/升级环境的处理器可等待 native gateway 就绪（最长 11 分钟）。 ReadTimeout: 5 * time.Second, IdleTimeout: 60 * time.Second, }) @@ -342,7 +342,7 @@ func newHandlerWithCreatorAndAI(webDirectory, username, password string, phaseAS } func isControlPlaneAPIPath(path string) bool { - for _, prefix := range []string{"/api", "/phase-a", "/gateways", "/browser-images", "/browsers", "/network-exits"} { + for _, prefix := range []string{"/api", "/phase-a", "/gateways", "/browser-versions", "/browsers", "/network-exits"} { if path == prefix || strings.HasPrefix(path, prefix+"/") { return true } diff --git a/cmd/control-plane/main_test.go b/cmd/control-plane/main_test.go index b4bebd9..253b985 100644 --- a/cmd/control-plane/main_test.go +++ b/cmd/control-plane/main_test.go @@ -744,10 +744,10 @@ func controlPlaneRouteMatrix() []controlPlaneRouteCase { {http.MethodPost, "/api/network-exits/:id/enable", "/api/network-exits/missing/enable", "", http.StatusNotFound}, {http.MethodDelete, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusNotFound}, - {http.MethodGet, "/api/browser-images", "/api/browser-images", "", http.StatusOK}, - {http.MethodPost, "/api/browser-images", "/api/browser-images", "", http.StatusBadRequest}, - {http.MethodPut, "/api/browser-images/:version", "/api/browser-images/999.0.0", `{"image_ref":"registry.example/browser:missing"}`, http.StatusNotFound}, - {http.MethodDelete, "/api/browser-images/:version", "/api/browser-images/999.0.0", "", http.StatusNotFound}, + {http.MethodGet, "/api/browser-versions", "/api/browser-versions", "", http.StatusOK}, + {http.MethodPost, "/api/browser-versions", "/api/browser-versions", "", http.StatusBadRequest}, + {http.MethodPut, "/api/browser-versions/:version", "/api/browser-versions/999.0.0", `{"browser_path":"/opt/creatorhub/browsers/missing"}`, http.StatusNotFound}, + {http.MethodDelete, "/api/browser-versions/:version", "/api/browser-versions/999.0.0", "", http.StatusNotFound}, {http.MethodGet, "/api/gateways", "/api/gateways", "", http.StatusOK}, {http.MethodPost, "/api/gateways", "/api/gateways", "", http.StatusBadRequest}, @@ -873,7 +873,7 @@ func TestPhaseAAccountHTTPWorkflowRedactsSecrets(t *testing.T) { {http.MethodPost, "/api/phase-a/accounts/account-http/pause"}, {http.MethodGet, "/api/phase-a/tasks"}, {http.MethodPost, "/api/phase-a/mock/execute"}, {http.MethodGet, "/api/phase-a/audit"}, {http.MethodGet, "/api/browsers"}, {http.MethodPost, "/api/browsers/account-http/start"}, - {http.MethodDelete, "/api/browsers/account-http"}, {http.MethodPut, "/api/browser-images/1"}, + {http.MethodDelete, "/api/browsers/account-http"}, {http.MethodPut, "/api/browser-versions/1"}, {http.MethodPost, "/api/network-exits/exit-http/check"}, {http.MethodPost, "/api/network-exits/exit-http/disable"}, } { if response := do(protectedApp, route.method, route.path, ""); response.Code != http.StatusUnauthorized { @@ -972,8 +972,8 @@ func TestPhaseAAccountHTTPWorkflowRedactsSecrets(t *testing.T) { if _, err := db.ExecContext(ctx, ` INSERT INTO gateway (name, endpoint, token) VALUES ('phase-http', 'http://127.0.0.1:8081', 'phase-http-gateway-token'); - INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1'); - INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + INSERT INTO browser_version (version, browser_path) VALUES ('1', '/opt/creatorhub/browsers/1'); + INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES ('account-http', 'Phase HTTP', 'phase-http', '1', '{"seed":1}'); INSERT INTO network_exit (id, protocol, host, port, health_status) VALUES ('exit-http', 'socks5', '127.0.0.1', 1080, 'healthy')`); err != nil { diff --git a/cmd/control-plane/runtime_use.go b/cmd/control-plane/runtime_use.go new file mode 100644 index 0000000..b417635 --- /dev/null +++ b/cmd/control-plane/runtime_use.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "errors" + "reflect" + "sync" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/hub" + "github.com/sirupsen/logrus" +) + +var runtimeUseRenewInterval = 20 * time.Second + +type runtimeUseStore interface { + AcquireRuntimeUse(context.Context, string, string, string, string) (hub.RuntimeUseLease, error) + RenewRuntimeUse(context.Context, string) (hub.RuntimeUseLease, error) + ReleaseRuntimeUse(context.Context, string) error +} + +type runtimeUseHandle struct { + store runtimeUseStore + lease hub.RuntimeUseLease + ctx context.Context + cancel context.CancelFunc + done chan struct{} + mu sync.Mutex + renewErr error + closeOnce sync.Once + closeErr error +} + +func beginRuntimeUse(ctx context.Context, store any, alias, purpose, ownerID string) (context.Context, *runtimeUseHandle, error) { + useStore, ok := store.(runtimeUseStore) + if !ok || useStore == nil || (reflect.ValueOf(useStore).Kind() == reflect.Ptr && reflect.ValueOf(useStore).IsNil()) { + return ctx, nil, errors.New("runtime use lease store is unavailable") + } + lease, err := useStore.AcquireRuntimeUse(ctx, alias, purpose, ownerID, "") + if err != nil { + return ctx, nil, err + } + useCtx, cancel := context.WithCancel(ctx) + handle := &runtimeUseHandle{store: useStore, lease: lease, ctx: useCtx, cancel: cancel, done: make(chan struct{})} + go handle.renew() + return useCtx, handle, nil +} + +func beginRuntimeUseForEnvironment(ctx context.Context, store any, environment hub.EnvironmentContext, purpose, ownerID string) (context.Context, *runtimeUseHandle, error) { + if environment.Alias == "" || environment.RuntimeInstanceID == "" { + return ctx, nil, hub.ErrConflict + } + useCtx, handle, err := beginRuntimeUse(ctx, store, environment.Alias, purpose, ownerID) + if err != nil { + return ctx, nil, err + } + if handle.lease.RuntimeInstanceID != environment.RuntimeInstanceID { + return ctx, nil, errors.Join(hub.ErrConflict, handle.Close()) + } + return useCtx, handle, nil +} + +func (h *runtimeUseHandle) renew() { + ticker := time.NewTicker(runtimeUseRenewInterval) + defer ticker.Stop() + defer close(h.done) + for { + select { + case <-h.ctx.Done(): + return + case <-ticker.C: + if _, err := h.store.RenewRuntimeUse(h.ctx, h.lease.Token); err != nil { + h.mu.Lock() + h.renewErr = err + h.mu.Unlock() + logrus.WithError(err).WithFields(logrus.Fields{ + "runtime_use_token": h.lease.Token, + "runtime_use_purpose": h.lease.Purpose, + }).Error("runtime use lease renewal failed") + h.cancel() + return + } + } + } +} + +func (h *runtimeUseHandle) Close() error { + if h == nil { + return nil + } + h.closeOnce.Do(func() { + h.cancel() + <-h.done + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(h.ctx), 5*time.Second) + defer cancel() + releaseErr := h.store.ReleaseRuntimeUse(releaseCtx, h.lease.Token) + h.mu.Lock() + defer h.mu.Unlock() + h.closeErr = errors.Join(h.renewErr, releaseErr) + }) + return h.closeErr +} diff --git a/cmd/control-plane/runtime_use_test.go b/cmd/control-plane/runtime_use_test.go new file mode 100644 index 0000000..d1f5e31 --- /dev/null +++ b/cmd/control-plane/runtime_use_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +type runtimeUseTestStore struct { + lease hub.RuntimeUseLease + acquireErr error + renewErr error + releaseErr error + renewCalls int + releaseCount int +} + +func (s *runtimeUseTestStore) AcquireRuntimeUse(context.Context, string, string, string, string) (hub.RuntimeUseLease, error) { + return s.lease, s.acquireErr +} + +func (s *runtimeUseTestStore) RenewRuntimeUse(context.Context, string) (hub.RuntimeUseLease, error) { + s.renewCalls++ + return s.lease, s.renewErr +} + +func (s *runtimeUseTestStore) ReleaseRuntimeUse(context.Context, string) error { + s.releaseCount++ + return s.releaseErr +} + +func TestBeginRuntimeUseForEnvironmentFencesGenerationAndClosesIdempotently(t *testing.T) { + store := &runtimeUseTestStore{lease: hub.RuntimeUseLease{ + Token: "runtime-use-token", + RuntimeInstanceID: "runtime-instance", + OwnerID: "owner", + Purpose: "task", + LeaseUntil: time.Now().Add(time.Minute), + }} + environment := hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeInstanceID: "runtime-instance"} + useCtx, handle, err := beginRuntimeUseForEnvironment(context.Background(), store, environment, "task", "owner") + if err != nil || handle == nil || useCtx == nil { + t.Fatalf("begin runtime use: handle=%#v err=%v", handle, err) + } + if err := handle.Close(); err != nil { + t.Fatal(err) + } + if err := handle.Close(); err != nil { + t.Fatal(err) + } + if store.releaseCount != 1 { + t.Fatalf("runtime-use close released %d times", store.releaseCount) + } + + store.lease.RuntimeInstanceID = "successor-runtime" + if _, handle, err := beginRuntimeUseForEnvironment(context.Background(), store, environment, "task", "owner"); !errors.Is(err, hub.ErrConflict) || handle != nil { + t.Fatalf("stale runtime lease was accepted: handle=%#v err=%v", handle, err) + } + if store.releaseCount != 2 { + t.Fatalf("mismatched lease was not released: %d", store.releaseCount) + } + + if _, handle, err := beginRuntimeUseForEnvironment(context.Background(), store, hub.EnvironmentContext{}, "task", "owner"); !errors.Is(err, hub.ErrConflict) || handle != nil { + t.Fatalf("missing runtime generation was accepted: handle=%#v err=%v", handle, err) + } + + var nilHandle *runtimeUseHandle + if err := nilHandle.Close(); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeUseRenewalCancelsWorkWhenRenewalFails(t *testing.T) { + previousInterval := runtimeUseRenewInterval + runtimeUseRenewInterval = time.Millisecond + defer func() { runtimeUseRenewInterval = previousInterval }() + + renewErr := errors.New("renewal failed") + store := &runtimeUseTestStore{ + lease: hub.RuntimeUseLease{ + Token: "runtime-use-token", + RuntimeInstanceID: "runtime-instance", + Purpose: "task", + }, + renewErr: renewErr, + } + useCtx, handle, err := beginRuntimeUse(context.Background(), store, "browser", "task", "owner") + if err != nil { + t.Fatal(err) + } + select { + case <-useCtx.Done(): + case <-time.After(time.Second): + t.Fatal("runtime-use context was not canceled after renewal failure") + } + if err := handle.Close(); !errors.Is(err, renewErr) { + t.Fatalf("close error = %v, want %v", err, renewErr) + } + if store.renewCalls == 0 { + t.Fatal("runtime-use renewal was not attempted") + } +} + +func TestRuntimeUseCloseReportsReleaseFailure(t *testing.T) { + releaseErr := errors.New("release failed") + store := &runtimeUseTestStore{ + lease: hub.RuntimeUseLease{Token: "token", RuntimeInstanceID: "runtime"}, + releaseErr: releaseErr, + } + _, handle, err := beginRuntimeUse(context.Background(), store, "browser", "task", "owner") + if err != nil { + t.Fatal(err) + } + if err := handle.Close(); !errors.Is(err, releaseErr) { + t.Fatalf("close error = %v, want %v", err, releaseErr) + } +} + +func TestBeginRuntimeUseReportsUnavailableStoreAndAcquireFailure(t *testing.T) { + if _, handle, err := beginRuntimeUse(context.Background(), struct{}{}, "browser", "task", "owner"); err == nil || handle != nil { + t.Fatalf("unavailable lease store: handle=%#v err=%v", handle, err) + } + acquireErr := errors.New("acquire failed") + store := &runtimeUseTestStore{acquireErr: acquireErr} + if _, handle, err := beginRuntimeUse(context.Background(), store, "browser", "task", "owner"); !errors.Is(err, acquireErr) || handle != nil { + t.Fatalf("acquire error: handle=%#v err=%v", handle, err) + } +} diff --git a/cmd/control-plane/xiaohongshu_test.go b/cmd/control-plane/xiaohongshu_test.go index 27d20d7..67ea2af 100644 --- a/cmd/control-plane/xiaohongshu_test.go +++ b/cmd/control-plane/xiaohongshu_test.go @@ -20,7 +20,7 @@ func TestXiaohongshuGatewayBrowserFencesAccountGeneration(t *testing.T) { t.Fatal("missing gateway authorization") } var body map[string]any - if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) || body["runtime_id"] != "runtime-a" || body["network_id"] != "network-a" || body["network_exit_id"] != "exit-a" { + if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) || body["runtime_id"] != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || body["network_id"] != "native-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || body["network_exit_id"] != "exit-a" { t.Fatalf("generation fence missing: %#v", body) } if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/get" || body["url"] != testXiaohongshuIdentityURL { diff --git a/cmd/docker_gateway/docker_client.py b/cmd/docker_gateway/docker_client.py deleted file mode 100644 index 1f82893..0000000 --- a/cmd/docker_gateway/docker_client.py +++ /dev/null @@ -1,975 +0,0 @@ -"""Minimal Docker Engine client and generation-fenced network helpers.""" - -from __future__ import annotations - -import http.client -import json -import logging -import os -import re -import socket -import threading -from collections.abc import Callable -from dataclasses import dataclass, field -from ipaddress import ip_interface -from urllib.parse import quote, urlencode - -MANAGED_LABEL = "io.creatorhub.managed" -RUNTIME_ID_LABEL = "io.creatorhub.runtime-id" -DISPLAY_NAME_LABEL = "io.creatorhub.display-name" -BINDING_VERSION_LABEL = "io.creatorhub.binding-version" -NETWORK_EXIT_LABEL = "io.creatorhub.network-exit-id" -PROXY_PORT_LABEL = "io.creatorhub.proxy-port" -NETWORK_ID_LABEL = "io.creatorhub.network-id" -NETWORK_ROLE_LABEL = "io.creatorhub.network-role" -GATEWAY_MEMBER_LABEL = "io.creatorhub.gateway-member" -BROWSER_NETWORK_ROLE = "browser" -BROWSER_PROXY_HOST = "docker-gateway" -NAME_PREFIX = "creatorhub-browser-" -RESERVATION_PREFIX = "creatorhub-reservation-" -RESERVATION_LABEL = "io.creatorhub.alias-reservation" -RESERVATION_GENERATION_LABEL = "io.creatorhub.reservation-generation" -RESERVATION_OWNER_LABEL = "io.creatorhub.reservation-owner" - -RUNTIME_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,31}$") -NETWORK_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") -LOG = logging.getLogger("creatorhub.docker") - - -class DockerError(RuntimeError): - def __init__(self, message: str, status: int | None = None) -> None: - super().__init__(message) - self.status = status - - -class GenerationConflict(DockerError): - pass - - -class NetworkSetupError(DockerError): - def __init__(self, message: str, generation: TenantNetworkGeneration) -> None: - super().__init__(message) - self.generation = generation - - -class UnmanagedContainer(DockerError): - pass - - -@dataclass(frozen=True) -class DockerResponse: - status: int - reason: str - body: bytes - - -class UnixHTTPConnection(http.client.HTTPConnection): - def __init__(self, socket_path: str, timeout: float) -> None: - super().__init__("docker", timeout=timeout) - self.socket_path = socket_path - - def connect(self) -> None: - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.sock.settimeout(self.timeout) - self.sock.connect(self.socket_path) - - -@dataclass -class TenantNetworkGeneration: - id: str = "" - name: str = "" - created: bool = False - connected_self: bool = False - connected_runtime: bool = False - gateway_members: list[str] = field(default_factory=list) - self_member: str = "" - runtime_attached: bool = False - - -class DockerClient: - def __init__(self, socket_path: str, api_version: str = "v1.43") -> None: - self.socket_path = socket_path - self.base_path = "/" + api_version.strip("/") - - def request( - self, - method: str, - path: str, - payload: object | None = None, - timeout: float = 30.0, - body_limit: int = 16 * 1024 * 1024, - ) -> DockerResponse: - encoded = ( - None - if payload is None - else json.dumps(payload, separators=(",", ":")).encode() - ) - connection: UnixHTTPConnection | None = None - try: - connection = UnixHTTPConnection(self.socket_path, timeout) - headers = {"Accept": "application/json"} - if encoded is not None: - headers["Content-Type"] = "application/json" - connection.request(method, self.base_path + path, encoded, headers) - response = connection.getresponse() - body = response.read(body_limit + 1) - if len(body) > body_limit: - raise DockerError( - "Docker response exceeded the configured limit", response.status - ) - return DockerResponse(response.status, response.reason, body) - except (OSError, http.client.HTTPException) as exc: - raise DockerError("Docker API request failed") from exc - finally: - if connection is not None: - connection.close() - - def expect( - self, - method: str, - path: str, - payload: object | None = None, - allowed: tuple[int, ...] = (204,), - ) -> None: - response = self.request(method, path, payload) - if response.status in allowed: - return - if response.status == 404: - raise FileNotFoundError(path) - raise DockerError( - f"Docker returned HTTP {response.status}: {response.body[:4096].decode('utf-8', 'replace').strip()}", - response.status, - ) - - def pull_if_missing(self, image: str) -> None: - encoded = quote(image, safe="") - response = self.request("GET", f"/images/{encoded}/json") - if response.status == 200: - return - if response.status != 404: - raise DockerError( - f"inspect image returned HTTP {response.status}", response.status - ) - repository, tag = split_image_ref(image) - query = {"fromImage": image if "@" in image else repository} - if "@" not in image and tag: - query["tag"] = tag - response = self.request( - "POST", - "/images/create?" + urlencode(query), - timeout=600.0, - body_limit=64 * 1024 * 1024, - ) - if response.status != 200: - raise DockerError( - f"pull image returned HTTP {response.status}: {response.body[:4096].decode('utf-8', 'replace').strip()}", - response.status, - ) - - def managed_container_state( - self, alias: str - ) -> tuple[str, dict[str, str], dict[str, str]]: - if not RUNTIME_ID_RE.fullmatch(alias): - raise ValueError("invalid runtime id") - response = self.request( - "GET", f"/containers/{quote(NAME_PREFIX + alias, safe='')}/json" - ) - if response.status == 404: - raise FileNotFoundError(alias) - if response.status != 200: - raise DockerError( - f"Docker inspect returned HTTP {response.status}", response.status - ) - try: - inspected = json.loads(response.body) - labels = inspected["Config"]["Labels"] or {} - container_id = inspected["Id"] - networks = inspected.get("NetworkSettings", {}).get("Networks", {}) or {} - if ( - not isinstance(labels, dict) - or not isinstance(networks, dict) - or not isinstance(container_id, str) - ): - raise TypeError("Docker inspect container metadata is invalid") - if not all( - isinstance(key, str) and isinstance(value, str) - for key, value in labels.items() - ): - raise TypeError("Docker inspect labels are invalid") - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: - raise DockerError( - "Docker inspect response is invalid", response.status - ) from exc - if labels.get(MANAGED_LABEL) != "true" or labels.get(RUNTIME_ID_LABEL) != alias: - raise UnmanagedContainer("refusing to operate on an unowned container") - network_ids: dict[str, str] = {} - for name, value in networks.items(): - if ( - not isinstance(name, str) - or not isinstance(value, dict) - or not isinstance(value.get("NetworkID", ""), str) - ): - raise DockerError("Docker inspect network metadata is invalid") - network_ids[name] = value.get("NetworkID", "") - return container_id, labels, network_ids - - def managed_container(self, alias: str) -> tuple[str, dict[str, str]]: - container_id, labels, _ = self.managed_container_state(alias) - return container_id, labels - - def trusted_gateway_member(self, container_id: str) -> bool: - response = self.request( - "GET", f"/containers/{quote(container_id, safe='')}/json" - ) - if response.status != 200: - return False - try: - inspected = json.loads(response.body) - if not isinstance(inspected, dict): - return False - config = inspected.get("Config", {}) - labels = config.get("Labels", {}) if isinstance(config, dict) else {} - return bool( - isinstance(inspected.get("Id"), str) - and isinstance(labels, dict) - and labels.get(GATEWAY_MEMBER_LABEL) == "true" - ) - except (TypeError, ValueError, json.JSONDecodeError): - return False - - def container_network_address(self, container_id: str, network_id: str) -> str: - response = self.request( - "GET", f"/containers/{quote(container_id, safe='')}/json" - ) - if response.status == 404: - raise FileNotFoundError("Docker container is missing") - if response.status != 200: - raise DockerError("inspect browser container failed", response.status) - try: - inspected = json.loads(response.body) - settings = inspected["NetworkSettings"] - networks = settings["Networks"] - if not isinstance(networks, dict): - raise TypeError("Docker container networks are invalid") - for value in networks.values(): - if isinstance(value, dict) and value.get("NetworkID") == network_id: - address = value.get("IPAddress") - if isinstance(address, str) and address: - return address - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: - raise DockerError("Docker container network metadata is invalid") from exc - raise GenerationConflict("browser container is not attached to its network") - - def inspect_tenant_network( - self, - base: str, - alias: str, - binding_version: int, - runtime_id: str, - self_name: str, - expected_id: str = "", - allow_unversioned: bool = False, - ) -> tuple[TenantNetworkGeneration, dict[str, str], bool]: - name = tenant_network_name(base, alias) - generation = TenantNetworkGeneration(id=expected_id, name=name) - reference = expected_id or name - response = self.request("GET", f"/networks/{quote(reference, safe='')}") - if response.status == 404: - return generation, {}, False - if response.status != 200: - raise DockerError( - f"Docker network inspect returned HTTP {response.status}", - response.status, - ) - try: - network = json.loads(response.body) - labels = network["Labels"] or {} - containers = network.get("Containers") or {} - network_id = network["Id"] - network_name = network["Name"] - if ( - not isinstance(labels, dict) - or not isinstance(containers, dict) - or not isinstance(network_id, str) - or not isinstance(network_name, str) - ): - raise TypeError("Docker network metadata is invalid") - if not all( - isinstance(key, str) and isinstance(value, str) - for key, value in labels.items() - ): - raise TypeError("Docker network labels are invalid") - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: - raise DockerError("Docker network inspect response is invalid") from exc - if ( - not network_id - or network_name != name - or network.get("Driver") != "bridge" - or network.get("Internal") - or network.get("Attachable") - or network.get("Ingress") - or labels.get(MANAGED_LABEL) != "true" - or labels.get(NETWORK_ROLE_LABEL) != BROWSER_NETWORK_ROLE - or labels.get(RUNTIME_ID_LABEL) != alias - ): - raise UnmanagedContainer( - "refusing to operate on an unowned browser network" - ) - if expected_id and network_id != expected_id: - raise GenerationConflict("network generation does not match request") - network_version = labels.get(BINDING_VERSION_LABEL, "") - if network_version != str(binding_version) and not ( - allow_unversioned and not network_version - ): - raise GenerationConflict("network generation does not match request") - generation.id = network_id - addresses: dict[str, str] = {} - for member_id, member in containers.items(): - if not isinstance(member, dict): - raise DockerError("Docker network member is invalid") - addresses[member_id] = str(member.get("IPv4Address", "")) - if member_id == runtime_id: - generation.runtime_attached = True - continue - if not self.trusted_gateway_member(member_id): - raise GenerationConflict( - "isolated network contains an untrusted member" - ) - generation.gateway_members.append(member_id) - if same_container_reference( - member_id, str(member.get("Name", "")), self_name - ): - generation.self_member = member_id - return generation, addresses, True - - def _finish_network_create( - self, - base: str, - alias: str, - self_name: str, - binding_version: int, - runtime_id: str, - response: DockerResponse, - generation: TenantNetworkGeneration, - ) -> TenantNetworkGeneration: - try: - created_id = json.loads(response.body)["Id"] - except (KeyError, TypeError, json.JSONDecodeError) as exc: - try: - observed, _, observed_exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ): - observed_exists = False - observed = TenantNetworkGeneration(name=generation.name) - if observed_exists: - generation = preserve_generation(generation, observed) - raise NetworkSetupError( - "create isolated browser network returned no id", generation - ) from exc - if not isinstance(created_id, str) or not created_id: - try: - observed, _, observed_exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ) as exc: - raise NetworkSetupError( - "create isolated browser network returned no id", generation - ) from exc - if observed_exists: - generation = preserve_generation(generation, observed) - raise NetworkSetupError( - "create isolated browser network returned no id", generation - ) - generation.id = created_id - generation.created = True - try: - observed, _, observed_exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name, created_id - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ) as exc: - raise NetworkSetupError( - "created browser network could not be verified", generation - ) from exc - generation = preserve_generation(generation, observed) - if not observed_exists: - raise NetworkSetupError("created browser network disappeared", generation) - return generation - - def ensure_tenant_network( - self, - base: str, - alias: str, - self_name: str, - binding_version: int, - runtime_id: str = "", - expected_id: str = "", - allow_unversioned: bool = False, - ) -> tuple[TenantNetworkGeneration, str]: - if not self_name: - raise DockerError("gateway identity is invalid") - generation, addresses, exists = self.inspect_tenant_network( - base, - alias, - binding_version, - runtime_id, - self_name, - expected_id, - allow_unversioned, - ) - if not exists: - if expected_id: - raise GenerationConflict("network generation does not match request") - response = self.request( - "POST", - "/networks/create", - { - "Name": generation.name, - "CheckDuplicate": True, - "Driver": "bridge", - "Labels": { - MANAGED_LABEL: "true", - NETWORK_ROLE_LABEL: BROWSER_NETWORK_ROLE, - RUNTIME_ID_LABEL: alias, - BINDING_VERSION_LABEL: str(binding_version), - }, - }, - ) - if response.status != 201: - try: - observed, _, observed_exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ): - observed_exists = False - observed = TenantNetworkGeneration(name=generation.name) - if not observed_exists: - raise DockerError( - "create isolated browser network failed", response.status - ) - generation = preserve_generation(generation, observed) - else: - generation = self._finish_network_create( - base, - alias, - self_name, - binding_version, - runtime_id, - response, - generation, - ) - if runtime_id and not generation.runtime_attached: - generation.connected_runtime = True - try: - self.expect( - "POST", - f"/networks/{quote(generation.id, safe='')}/connect", - {"Container": runtime_id}, - (200,), - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ) as exc: - raise self._network_setup_error( - base, - alias, - binding_version, - runtime_id, - self_name, - generation, - "connecting the browser to its isolated network failed", - ) from exc - generation.runtime_attached = True - if not generation.self_member: - generation.connected_self = True - try: - self.expect( - "POST", - f"/networks/{quote(generation.id, safe='')}/connect", - { - "Container": self_name, - "EndpointConfig": {"Aliases": [BROWSER_PROXY_HOST]}, - }, - (200,), - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ) as exc: - raise self._network_setup_error( - base, - alias, - binding_version, - runtime_id, - self_name, - generation, - "connecting the gateway to its isolated network failed", - ) from exc - try: - observed, addresses, observed_exists = self.inspect_tenant_network( - base, - alias, - binding_version, - runtime_id, - self_name, - generation.id, - allow_unversioned, - ) - generation = preserve_generation(generation, observed) - if not observed_exists or not generation.self_member: - raise NetworkSetupError( - "Docker did not connect gateway to the isolated network", generation - ) - bind_host = str(ip_interface(addresses[generation.self_member]).ip) - except NetworkSetupError: - raise - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ) as exc: - raise NetworkSetupError( - "Docker could not verify the isolated network", generation - ) from exc - return generation, bind_host - - def _network_setup_error( - self, - base: str, - alias: str, - binding_version: int, - runtime_id: str, - self_name: str, - generation: TenantNetworkGeneration, - message: str, - ) -> NetworkSetupError: - try: - observed, _, exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name, generation.id - ) - generation = ( - preserve_generation(generation, observed) if exists else generation - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ) as verification_error: - return NetworkSetupError( - f"{message}; network state verification failed: {verification_error}", - generation, - ) - return NetworkSetupError(message, generation) - - def disconnect_member( - self, - base: str, - alias: str, - binding_version: int, - runtime_id: str, - generation: TenantNetworkGeneration, - member: str, - self_name: str, - missing_ok: bool = False, - ) -> TenantNetworkGeneration: - current, _, exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name, generation.id - ) - if not exists: - if missing_ok: - return current - raise GenerationConflict("isolated network generation is missing") - if not same_network_members(current, generation): - raise GenerationConflict("isolated network generation changed") - if not member_present(current, member, runtime_id): - return current - self.expect( - "POST", - f"/networks/{quote(generation.id, safe='')}/disconnect", - {"Container": member, "Force": True}, - (200,), - ) - observed, _, observed_exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name, generation.id - ) - if not observed_exists or not same_network_members( - observed, generation_without_member(generation, member, runtime_id) - ): - raise GenerationConflict( - "Docker retained an isolated network member after disconnect" - ) - return observed - - def delete_tenant_network( - self, - base: str, - alias: str, - binding_version: int, - runtime_id: str, - generation: TenantNetworkGeneration, - self_name: str, - missing_ok: bool = False, - ) -> None: - current, _, exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name, generation.id - ) - if not exists: - if missing_ok: - return - raise GenerationConflict("isolated network generation is missing") - if current.runtime_attached or current.gateway_members: - raise GenerationConflict("isolated network still has members") - self.expect( - "DELETE", f"/networks/{quote(generation.id, safe='')}", allowed=(204,) - ) - _, _, exists = self.inspect_tenant_network( - base, alias, binding_version, runtime_id, self_name, generation.id - ) - if exists: - raise DockerError("Docker retained isolated browser network") - - -def split_image_ref(ref: str) -> tuple[str, str]: - if "@" in ref: - return ref.split("@", 1)[0], ref.split("@", 1)[1] - colon = ref.rfind(":") - slash = ref.rfind("/") - return (ref[:colon], ref[colon + 1 :]) if colon > slash else (ref, "") - - -def tenant_network_name(base: str, alias: str) -> str: - name = f"{base}-{alias}" - if not NETWORK_NAME_RE.fullmatch(name): - raise ValueError("isolated browser network name is invalid") - return name - - -def same_container_reference(container_id: str, name: str, reference: str) -> bool: - return bool( - reference - and ( - container_id == reference - or name == reference - or container_id.startswith(reference) - or reference.startswith(container_id) - ) - ) - - -def preserve_generation( - known: TenantNetworkGeneration, observed: TenantNetworkGeneration -) -> TenantNetworkGeneration: - if not observed.id: - observed.id = known.id - if not observed.name: - observed.name = known.name - observed.created |= known.created - observed.runtime_attached |= known.runtime_attached - observed.connected_runtime |= known.connected_runtime - observed.connected_self |= known.connected_self - if not observed.self_member: - observed.self_member = known.self_member - for member in known.gateway_members: - if not member_present(observed, member, ""): - observed.gateway_members.append(member) - return observed - - -def same_network_members( - current: TenantNetworkGeneration, expected: TenantNetworkGeneration -) -> bool: - return ( - current.id == expected.id - and current.name == expected.name - and current.runtime_attached == expected.runtime_attached - and current.self_member == expected.self_member - and set(current.gateway_members) == set(expected.gateway_members) - ) - - -def member_present( - generation: TenantNetworkGeneration, member: str, runtime_id: str -) -> bool: - if ( - generation.runtime_attached - and runtime_id - and same_container_reference(member, "", runtime_id) - ): - return True - return any( - same_container_reference(existing, "", member) - for existing in generation.gateway_members - ) or same_container_reference(member, "", generation.self_member) - - -def generation_without_member( - generation: TenantNetworkGeneration, member: str, runtime_id: str -) -> TenantNetworkGeneration: - result = TenantNetworkGeneration( - id=generation.id, - name=generation.name, - created=generation.created, - connected_self=generation.connected_self, - connected_runtime=generation.connected_runtime, - gateway_members=list(generation.gateway_members), - self_member=generation.self_member, - runtime_attached=generation.runtime_attached, - ) - if runtime_id and same_container_reference(member, "", runtime_id): - result.runtime_attached = False - if same_container_reference(member, "", result.self_member): - result.self_member = "" - result.gateway_members = [ - x for x in result.gateway_members if not same_container_reference(x, "", member) - ] - return result - - -def random_reservation_generation() -> str: - return os.urandom(16).hex() - - -class AliasReservationManager: - def __init__(self, docker: DockerClient, self_name: str) -> None: - self.docker = docker - self.self_name = self_name - self._locks: dict[str, threading.Lock] = {} - self._locks_guard = threading.Lock() - - def acquire(self, alias: str) -> Callable[[], None]: - with self._locks_guard: - lock = self._locks.setdefault(alias, threading.Lock()) - lock.acquire() - generation = random_reservation_generation() - created_id = "" - try: - response = self.docker.request( - "GET", f"/containers/{quote(self.self_name, safe='')}/json" - ) - if response.status != 200: - raise DockerError( - "inspect trusted gateway for alias reservation", response.status - ) - inspected = json.loads(response.body) - config = inspected.get("Config", {}) - labels = config.get("Labels", {}) if isinstance(config, dict) else {} - image = inspected.get("Image", "") - if ( - not isinstance(labels, dict) - or not isinstance(image, str) - or not image - or labels.get(GATEWAY_MEMBER_LABEL) != "true" - ): - raise DockerError("inspect trusted gateway for alias reservation") - reservation_payload = { - "Image": image, - "Labels": { - RESERVATION_LABEL: "true", - RUNTIME_ID_LABEL: alias, - RESERVATION_GENERATION_LABEL: generation, - RESERVATION_OWNER_LABEL: self.self_name, - }, - "HostConfig": {"NetworkMode": "none"}, - } - response = self.docker.request( - "POST", - "/containers/create?" + urlencode({"name": RESERVATION_PREFIX + alias}), - reservation_payload, - ) - if response.status == 409 and self._reclaim_stale(alias): - response = self.docker.request( - "POST", - "/containers/create?" - + urlencode({"name": RESERVATION_PREFIX + alias}), - reservation_payload, - ) - if response.status == 409: - raise GenerationConflict("browser alias is already in use") - if response.status != 201: - raise DockerError("create alias reservation failed", response.status) - created_id = json.loads(response.body).get("Id", "") - if not isinstance(created_id, str) or not created_id: - raise DockerError("Docker returned an invalid alias reservation id") - check = self.docker.request( - "GET", f"/containers/{quote(RESERVATION_PREFIX + alias, safe='')}/json" - ) - if check.status != 200: - raise DockerError( - "alias reservation could not be verified", check.status - ) - observed = json.loads(check.body) - observed_config = observed.get("Config", {}) - observed_labels = ( - observed_config.get("Labels", {}) - if isinstance(observed_config, dict) - else {} - ) - if ( - not isinstance(observed_labels, dict) - or observed.get("Id") != created_id - or observed_labels.get(RESERVATION_LABEL) != "true" - or observed_labels.get(RESERVATION_GENERATION_LABEL) != generation - ): - raise GenerationConflict( - "alias reservation generation is not immutable" - ) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ): - try: - self._reconcile(alias, generation, created_id) - except ( - DockerError, - FileNotFoundError, - OSError, - TypeError, - ValueError, - KeyError, - ): - LOG.exception( - "failed to reconcile alias reservation", - extra={"alias": alias, "generation": generation}, - ) - lock.release() - raise - - def release() -> None: - try: - self._reconcile(alias, generation, created_id) - finally: - lock.release() - - return release - - def _reclaim_stale(self, alias: str) -> bool: - response = self.docker.request( - "GET", f"/containers/{quote(RESERVATION_PREFIX + alias, safe='')}/json" - ) - if response.status == 404: - return True - if response.status != 200: - raise DockerError("inspect alias reservation failed", response.status) - try: - observed = json.loads(response.body) - config = observed.get("Config", {}) - labels = config.get("Labels", {}) if isinstance(config, dict) else {} - observed_id = observed.get("Id") - except (TypeError, ValueError, json.JSONDecodeError) as exc: - raise DockerError("alias reservation inspect response is invalid") from exc - if ( - not isinstance(labels, dict) - or labels.get(RESERVATION_LABEL) != "true" - or labels.get(RUNTIME_ID_LABEL) != alias - or not isinstance(observed_id, str) - or not observed_id - ): - raise GenerationConflict("alias reservation generation changed") - owner = labels.get(RESERVATION_OWNER_LABEL) - if not isinstance(owner, str) or not owner: - return False - if owner != self.self_name: - owner_response = self.docker.request( - "GET", f"/containers/{quote(owner, safe='')}/json" - ) - if owner_response.status == 200: - return False - if owner_response.status != 404: - raise DockerError( - "inspect alias reservation owner failed", owner_response.status - ) - self.docker.expect( - "DELETE", - f"/containers/{quote(observed_id, safe='')}?force=1&v=0", - allowed=(204, 404), - ) - return ( - self.docker.request( - "GET", f"/containers/{quote(RESERVATION_PREFIX + alias, safe='')}/json" - ).status - == 404 - ) - - def _reconcile(self, alias: str, generation: str, created_id: str) -> None: - response = self.docker.request( - "GET", f"/containers/{quote(RESERVATION_PREFIX + alias, safe='')}/json" - ) - if response.status == 404: - return - if response.status != 200: - raise DockerError("inspect alias reservation failed", response.status) - try: - observed = json.loads(response.body) - config = observed.get("Config", {}) - labels = config.get("Labels", {}) if isinstance(config, dict) else {} - observed_id = observed.get("Id") - except (TypeError, ValueError, json.JSONDecodeError) as exc: - raise DockerError("alias reservation inspect response is invalid") from exc - if ( - not isinstance(labels, dict) - or not isinstance(observed_id, str) - or labels.get(RESERVATION_LABEL) != "true" - or labels.get(RUNTIME_ID_LABEL) != alias - or labels.get(RESERVATION_GENERATION_LABEL) != generation - or (created_id and observed_id != created_id) - ): - raise GenerationConflict("alias reservation generation changed") - self.docker.expect( - "DELETE", - f"/containers/{quote(observed_id, safe='')}?force=1&v=0", - allowed=(204,), - ) - if ( - self.docker.request( - "GET", f"/containers/{quote(RESERVATION_PREFIX + alias, safe='')}/json" - ).status - != 404 - ): - raise DockerError("Docker retained alias reservation") diff --git a/compose.dev.yaml b/compose.dev.yaml index f8f0e36..e598e11 100644 --- a/compose.dev.yaml +++ b/compose.dev.yaml @@ -1,23 +1,11 @@ --- -# 本地热加载开发用 override：只跑 postgres + docker-gateway，宿主机直接跑 control-plane 与前端。 -# 用法：docker compose -f compose.yaml -f compose.dev.yaml up -d \ -# postgres docker-gateway +# 本地开发 override：只运行 PostgreSQL；control-plane、native browser gateway 和前端在宿主机运行。 +# 用法：docker compose -f compose.yaml -f compose.dev.yaml up -d postgres services: creator-hub: - # 本地开发不跑容器版 control-plane；profile 化后不启动。 + # 本地开发直接运行 control-plane，不启动 compose 中的控制面。 profiles: [compose-only] postgres: ports: - "5432:5432" - - docker-gateway: - ports: - - "8081:8081" - extra_hosts: - - "host.docker.internal:host-gateway" - environment: - BROWSER_CDP_URL: ${BROWSER_CDP_URL:-} - BROWSER_CDP_TARGET_ID: ${BROWSER_CDP_TARGET_ID:-} - BROWSER_CDP_ALIAS: ${BROWSER_CDP_ALIAS:-local-cdp} - BROWSER_CDP_NETWORK_ID: ${BROWSER_CDP_NETWORK_ID:-local-cdp} diff --git a/compose.yaml b/compose.yaml index b4ddcb4..ea28d1a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,9 +21,9 @@ services: - /tmp:size=16m,noexec,nosuid,nodev cap_drop: [ALL] security_opt: [no-new-privileges:true] + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: - docker-gateway: - condition: service_healthy postgres: condition: service_healthy networks: [control] @@ -65,32 +65,6 @@ services: networks: [control] restart: unless-stopped - docker-gateway: - build: . - stop_grace_period: 45s - command: ["python3", "-m", "cmd.docker_gateway.gateway"] - labels: - io.creatorhub.gateway-member: "true" - environment: - BROWSER_NETWORK: creatorhub_browser - GATEWAY_TOKEN: ${GATEWAY_TOKEN:-dev-creatorhub-gateway-token} - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - group_add: - - "${DOCKER_GID:?required}" - healthcheck: - test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8081/healthz] - interval: 2s - timeout: 2s - retries: 15 - read_only: true - tmpfs: - - /tmp:size=16m,noexec,nosuid,nodev - cap_drop: [ALL] - security_opt: [no-new-privileges:true] - networks: [control] - restart: unless-stopped - networks: control: name: creatorhub_control diff --git a/deploy/browser-gateway.env.example b/deploy/browser-gateway.env.example new file mode 100644 index 0000000..69753d0 --- /dev/null +++ b/deploy/browser-gateway.env.example @@ -0,0 +1,16 @@ +# Copy to ~/.config/creatorhub/browser-gateway.env and replace the token. +LISTEN_ADDR=0.0.0.0:8081 +GATEWAY_TOKEN=replace-with-at-least-16-random-characters +BROWSER_STATE_DIR=~/.local/state/creatorhub/browser-gateway +BROWSER_PROFILE_ROOT=~/.local/share/creatorhub/browser-profiles +BROWSER_VERSION=148.0.7778.215 +BROWSER_PATH=~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome +NODE_NAME= +RUNTIME_CLEANUP_TIMEOUT=30 +RUNTIME_READY_TIMEOUT=15 +RUNTIME_MIN_FREE_BYTES=21474836480 +RUNTIME_LOG_MAX_BYTES=1073741824 +PROFILE_CACHE_MAX_BYTES=21474836480 +# Optional single-node manual observation only: reuse an already-running Xvfb. +# Keep this unset for the normal per-runtime isolated Xvfb mode. +# RUNTIME_EXTERNAL_DISPLAY=99 diff --git a/deploy/creatorhub-browser-gateway.service.in b/deploy/creatorhub-browser-gateway.service.in new file mode 100644 index 0000000..5c3f0d5 --- /dev/null +++ b/deploy/creatorhub-browser-gateway.service.in @@ -0,0 +1,18 @@ +[Unit] +Description=CreatorHub native browser gateway +After=graphical-session.target + +[Service] +Type=simple +WorkingDirectory=@PROJECT_DIR@ +EnvironmentFile=%h/.config/creatorhub/browser-gateway.env +ExecStart=/usr/bin/python3 -m cmd.browser_gateway.gateway +Restart=on-failure +RestartSec=5 +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=no +LimitNOFILE=8192 + +[Install] +WantedBy=default.target diff --git a/docker/browser-wrapper/Dockerfile b/docker/browser-wrapper/Dockerfile deleted file mode 100644 index 37c46c3..0000000 --- a/docker/browser-wrapper/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG BROWSER_BASE_IMAGE -FROM ${BROWSER_BASE_IMAGE} - -COPY --chmod=0755 docker-entrypoint.sh /usr/local/bin/creatorhub-browser-entrypoint.sh -ENTRYPOINT ["/usr/local/bin/creatorhub-browser-entrypoint.sh"] diff --git a/docker/browser-wrapper/README.md b/docker/browser-wrapper/README.md deleted file mode 100644 index a293710..0000000 --- a/docker/browser-wrapper/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# CreatorHub browser wrapper - -这是浏览器运行时的受管理入口，不包含 Chromium 本体。构建时必须传入已登记的 immutable base image digest，禁止使用 tag 或 `latest`： - -```bash -docker build \ - --build-arg BROWSER_BASE_IMAGE=git.ipao.vip/rogee/fingerprint-chromium@sha256: \ - -t creatorhub-browser-wrapper: \ - docker/browser-wrapper -``` - -将构建结果推送到登记的镜像仓库后，用 `docker image inspect` 或仓库 manifest 查询发布 digest；Compose 和验收只使用 `repo/image@sha256:`，不把 digest 当作本地 tag。 - -发布记录必须同时保存 base image 仓库、base digest、构建提交、此目录 Dockerfile 和发布 digest。入口会等待 Xvfb、检查 x11vnc、绑定容器地址上的 CDP 端口，再启动 `/opt/chromium/chrome`。 diff --git a/docker/browser-wrapper/docker-entrypoint.sh b/docker/browser-wrapper/docker-entrypoint.sh deleted file mode 100644 index 0aea049..0000000 --- a/docker/browser-wrapper/docker-entrypoint.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -set -eu - -Xvfb :99 -screen 0 "${SCREEN_SIZE:-1920x1080x24}" -nolisten tcp -ac & -xvfb_pid=$! -i=0 -while [ "$i" -lt 300 ]; do - [ -S /tmp/.X11-unix/X99 ] && break - kill -0 "$xvfb_pid" 2>/dev/null || exit 1 - i=$((i + 1)) - sleep 0.1 -done -[ -S /tmp/.X11-unix/X99 ] || { - echo "Xvfb did not become ready" >&2 - exit 1 -} -export DISPLAY=:99 -x11vnc -display :99 -listen "$(hostname -i)" -rfbport 5900 -forever -nopw -nevershared -dontdisconnect -noclipboard -nosetclipboard -noprimary -nosetprimary -quiet & -rfb_pid=$! -kill -0 "$rfb_pid" 2>/dev/null || { - echo "RFB service did not start" >&2 - exit 1 -} -remote_debugging_port=${REMOTE_DEBUGGING_PORT:-9222} -case "$remote_debugging_port" in -'' | *[!0-9]*) - echo "REMOTE_DEBUGGING_PORT must be numeric" >&2 - exit 2 - ;; -esac -socat "TCP-LISTEN:${remote_debugging_port},fork,reuseaddr,bind=$(hostname -i)" "TCP:127.0.0.1:${remote_debugging_port}" & -exec /opt/chromium/chrome --disable-dev-shm-usage --no-sandbox --no-default-browser-check --no-first-run --remote-debugging-port="$remote_debugging_port" --user-data-dir=/data "$@" diff --git a/docs/architecture/container-control.md b/docs/architecture/container-control.md index d19bea9..7cf2431 100644 --- a/docs/architecture/container-control.md +++ b/docs/architecture/container-control.md @@ -1,86 +1,83 @@ -# 浏览器容器控制面 +# 宿主机浏览器控制面 -> 当前实现说明，核对基线 `main@1fbf126`；不是新业务完成证明。目标范围与验收以 [plan01](../plan01.md) 为准，运行步骤见[部署说明](../deployment.md)。现状限制不自动成为新产品约束。 +> 当前实现说明，基于已同步的 `main@7e3808cf4ce453b3583079680a3b59ca2ed64ad4`。单节点 native browser + Xvfb 已实现；真实平台、代理、LAN 和多节点能力仍以[验证文档](../native-browser-verification.md)为准。业务范围以 [plan01](../plan01.md) 为准，部署步骤见[部署说明](../deployment.md)。 ## 技术选型 -- 前端：React 19 + Vite 8 + Refine + shadcn/ui + Tailwind CSS，图标 RemixIcon；项目自有 Layout 与 HashRouter，当前页面见 [main.jsx](../../web/src/main.jsx)，依赖见 [package.json](../../web/package.json)。 -- 后端：Go 模块化单体，Fiber v3 提供 HTTP 路由，Viper 读取并校验启动配置，Logrus 输出 JSON 结构化日志，Cobra 保持当前两个服务入口。控制面提供同源 API 和静态文件，并编排网关；受限网关单独封装 Docker Engine API，是纯执行器。 -- 数据：环境配置（别名、中文名、网关、镜像版本、指纹参数）持久化在 Postgres，列表/详情读取持久运行记录，不触发网关实时探测；后台独立维护运行租约；Profile 使用命名卷持久化；阶段 A 账号、凭据引用、确认、任务、尝试和审计实体同样由控制面持久化到 Postgres。 -- 部署：Docker Compose 启动控制面和受限网关；浏览器容器由网关按平台下发的镜像引用动态创建，缺失时自动拉取。 +- 前端：React 19 + Vite 8 + Refine + shadcn/ui + Tailwind CSS，图标 RemixIcon；项目自有 Layout 与 HashRouter。 +- 控制面：Go、Fiber v3、Viper、Logrus、Cobra。控制面编排浏览器 gateway、环境、账号、任务和审计，不直接启动浏览器。 +- 浏览器 gateway：Python 3.12+ 标准库 HTTP/进程/文件/socket 能力；以非 root `systemctl --user` 服务运行，在宿主机管理 Xvfb、native/fingerprint browser、Profile、CDP、代理和运行代次。 +- 数据：PostgreSQL 保存网关登记、浏览器版本、环境绑定、运行实例、清理状态、租约、账号、任务、结果和审计。`browser_version.browser_path` 是宿主机已安装的可执行文件路径，不是镜像引用。 +- 部署：Compose 只提供控制面和 PostgreSQL；gateway 与 Xvfb 在宿主机运行。控制面在 Compose 中通过 `host.docker.internal:8081` 访问 gateway，裸机运行使用 `127.0.0.1:8081`。 -## 调用链与契约 +## 调用链与职责 ```text -React ── /api/* (Basic Auth) ──> control-plane - ├─ /v1/browsers (Bearer token) ──> docker-gateway ──> docker.sock - │ └─> browser container - ├─ /v1/browsers/.../douyin/events (Bearer token) ──> docker-gateway ──> browser event stream - └─ 账号/环境/任务/互动事件持久记录 ──> PostgreSQL +React ── /api/* (Basic Auth) ──> control-plane ──> PostgreSQL + │ + └─ /v1/browsers (Bearer token) + └─ host-native browser gateway + ├─ Xvfb display + ├─ fingerprint browser + Profile + ├─ CDP endpoint + └─ loopback proxy / event stream ``` -控制面是唯一事实源：网关不持有镜像清单和业务规则，镜像引用、启动命令和卷名均随请求下发。 +控制面是业务事实源，gateway 是本机浏览器资源事实源。gateway 不接受任意 Profile 路径、不暴露通用 CDP、不创建 Docker 资源；Profile 由 gateway 根据 `profile_id` 映射到 gateway 自己管理的目录。 -当前生命周期契约集中如下（代码：[控制面](../../cmd/control-plane/hub.go)、[环境存储](../../internal/hub/environment.go)）： +## 生命周期契约 + +生命周期操作都必须带当前 `binding_version`，并在需要时带 `runtime_id`、`network_id` 和 `runtime_instance_id`。服务端以 alias 串行化操作，拿锁后重新读取账号、环境、版本和出口。 | 接口/动作 | 当前行为与失败语义 | | --- | --- | -| `POST /api/browsers` / create | 严格接收 `{alias, name, gateway, image_version, fingerprint, account_id, network_exit_id}`，未知字段拒绝；`account_id` 必填，`network_exit_id` 可空表示明确选择直连。新绑定要求账号 authorized/paused、镜像启用，指定出口须健康并再次核验。先保存环境与稳定 binding，创建停止态容器；不是创建即启动。相同绑定及配置可复用，冲突拒绝；已存在且账号可运行的环境可调和/恢复运行。网关失败不删除已保存环境/binding，保留以供核验/恢复 | -| `GET /api/browsers`、`GET /api/browsers/{alias}` | 只读数据库环境/运行记录；列表的 running 表示已记录运行实例，不保证实时存活。页面可直接展示返回 status，不主动探测或轮询。后台租约调和与列表读取独立 | -| `POST /api/browsers/{alias}/start` | 要求账号可运行；有出口时重新核验，不健康则失败而非直连。当前匹配且就绪的运行容器可复用；停止/缺失容器按当前 binding 重建并激活租约；支持预先选定的直连 | -| `POST /api/browsers/{alias}/stop` | 停止容器并处理租约/清理状态；失败或结果不明可见，不据请求发出即声称已停止 | -| `POST /api/browsers/{alias}/upgrade` | 接收 `{version}`，要求启用镜像；删除旧容器并保留 Profile，更新版本后按原指纹/binding 重建，账号可运行才启动，否则为停止态。无自动回滚；失败保留结果，人工重试进入现有调和流程。当前实现无条件核验出口，空出口的直连环境不能据 create/start 成功推断 upgrade 可用 | -| `POST /api/browsers/{alias}/rebind` | 接收 `{network_exit_id}`；要求 paused、无 executing task、无活动 runtime，验证目标出口后变更。当前要求有效出口 ID，不支持以空值切回直连 | -| `DELETE /api/browsers/{alias}` / recycle | 回收容器、处理运行记录，但保留环境、稳定 binding 与命名 Profile 卷；后续 create/start 复用。不是永久删除账号/环境或素材；没有新增永久删除 API | +| `POST /api/browsers` | 接收 `alias`、`name`、`gateway`、`browser_version`、`fingerprint`、`account_id`、`network_exit_id`。版本必须启用且路径为绝对可执行路径；账号必须满足授权/暂停规则，指定出口必须健康。先保存环境和稳定 binding，再向 gateway 创建停止态 runtime；失败保留环境、operation 和清理状态，不伪造成功。 | +| `GET /api/browsers`、`GET /api/browsers/{alias}` | 返回数据库状态与 gateway 可达时的 runtime 快照，包括 `node_id`、generation、ready、运行状态、cleanup 状态和失败原因。不可达、版本缺失、Profile 占用和待清理均保持可见；读取不把未知状态改成 stopped。 | +| `POST /api/browsers/{alias}/start` | 重新核对账号、版本、Profile、出口和 binding。匹配的 ready runtime 可复用；缺失或不匹配时按当前 binding 创建新代次并激活。必需代理失败时不得直连。 | +| `POST /api/browsers/{alias}/stop` | 以当前 generation 停止浏览器、Xvfb、代理和订阅，并记录独立 cleanup 状态。结果未知、强杀或清理失败返回可见失败/待清理状态，而不是请求发出即成功。 | +| `POST /api/browsers/{alias}/upgrade` | 接收 `{version}`，要求目标 browser version 启用；停止旧代次、保留 Profile、按新路径创建并按需启动。无隐式回退和自动回滚，失败保留证据供重试。 | +| `POST /api/browsers/{alias}/rebind` | 接收有效 `network_exit_id`，要求没有执行中的 runtime-use 或业务任务；成功后递增 binding version。空值切直连不通过此接口隐式完成。 | +| `DELETE /api/browsers/{alias}` | 回收当前 runtime 和本次临时资源，保留环境、稳定 binding、账号 Profile、正式结果和长期监听。旧 generation 的迟到清理不会触碰 successor。 | -`name` 是环境展示名，`alias` 限 `^[a-z0-9][a-z0-9-]{0,31}$`；容器名 `creatorhub-browser-`，Profile 卷 `creatorhub-profile-`。指纹中的代理字段拒绝，使用所选出口；已绑定代理失败不能静默直连。create/start/stop/upgrade/recycle 写同一 operation ID 的 requested/finished 审计对，网关断连且无法调和时 outcome 为 `unknown`；幂等/重试是生命周期核验，不等于 plan01 中人工/自动业务发送可重发。 +每个 runtime 独立拥有：`runtime_id`、`runtime_instance_id`、`generation`、`node_id`、owner、Profile、display、CDP 端口、loopback proxy 端口、browser/Xvfb unit、日志目录和 cleanup 状态。runtime 结束后，任务目录按结果语义清理；正式素材和结果目录不由通用 runtime 清理逻辑删除。 -- `GET/POST /api/browser-images` 维护可用镜像版本（版本号不可改，`PUT /{version}` 仅接受 `image_ref/note/enabled`）；仅启用版本可用于创建与升级；被环境引用时拒绝删除。 -- `GET/POST /api/gateways` 注册网关（`POST` 可携带令牌，否则平台生成 48 位十六进制令牌并明文存储），`DELETE /api/gateways/{name}` 删除；仍被环境引用时拒绝删除。 -- 别名唯一约束由数据库保证；控制面不自动重试结果不明的用户生命周期请求，后台租约恢复的现状另见下文。 +## 浏览器版本、网关与 Profile -## docker.sock 安全边界 +- `GET/POST /api/browser-versions` 管理 `{version, browser_path, note, enabled}`；版本号不可改，只有启用版本可创建或升级环境。旧迁移 SQL 中出现的 `browser_image`/`image_ref` 仅是 PostgreSQL 前向迁移的历史表名，当前 API、模型、页面和运行时均使用 browser version/path。 +- `GET/POST /api/gateways` 注册稳定 gateway endpoint、node name 和令牌；gateway `/v1/info` 返回稳定 `node_id`、版本和能力。当前目标只调度单 gateway，不执行跨机迁移。 +- `profile_id` 由控制面绑定账号和环境，gateway 只接受已登记的标识并生成目录。Profile 目录权限为 owner-only；同一 Profile 的第二个 runtime 在资源锁阶段拒绝。 +- 每个执行使用独立的 `.runs/` 目录和不可变 token。业务结果引用相对 work root 的正式产物；路径穿越、绝对路径、跨 work 目录和临时文件均拒绝。 -将 socket 以只读文件挂载**不会**限制 Docker API 的写操作；拥有 socket 等价于拥有宿主机 root 权限。因此： +## generation 与 runtime-use lease -- 只有 `docker-gateway` 挂载 socket，控制面和浏览器容器均不可见；网关加入 control 并按需接入浏览器隔离网络，浏览器在代理模式只拿到无凭据的内存转发代理地址，`/v1` 仍必须通过容器内不可见的网关令牌； -- 网关只暴露面向领域的路由，不提供通用 Docker 代理；`/v1` 全部接口校验 `Authorization: Bearer `（常数时间比较），令牌由部署者在网关环境变量与平台注册表中保持一致； -- 网关直连的 `POST /v1/browsers/{alias}/start|stop` 仅供内部维护使用，必须提交并精确匹配容器标签中的 `{binding_version,runtime_id,network_id}`；直连 start 拒绝空 `network_id`，generation 不匹配返回 `409`，控制面生命周期编排不依赖无 fence 的直连 start； -- 网关恢复内存代理时会同时 fence 隔离网络成员及网关成员 IPv4，地址变化返回 `409` 并关闭刚恢复的监听；代理移除或代际替换会立即关闭所有已 hijack 的 CONNECT 双向连接，任一端先关闭也会关闭隧道两端，不等待优雅 drain； -- 网关固定命令、网络、挂载和资源限制；外部输入是受校验的别名，以及平台下发的镜像引用、启动参数和卷名——镜像引用来自平台维护的版本表，新增/变更由人工在页面审核启用，不再写死在代码中； -- 启停和删除前必须同时匹配固定名称前缀及 `io.creatorhub.managed`、`io.creatorhub.runtime-id` 标签； -- 动态容器使用只读根文件系统、非 root `1000:1000` 与固定镜像入口、全部 capability drop、`no-new-privileges`、CPU/内存/PID 限制，且无宿主机端口和目录挂载； -- 控制面发布到宿主机所有网卡；控制网络为固定名称的 Compose 网络；浏览器 bridge 按 ownership、role、driver、Internal 失败关闭校验，且拒绝复用 control 网络； -- Compose 基础镜像锁定 digest；浏览器镜像推荐使用 `@sha256:` 摘要引用以获得不可变性，tag 引用由部署者自行把控。 +- 激活 runtime 时保存 owner、node、binding version、runtime/network ID 和 runtime instance ID。所有 gateway 写操作核对这些值；旧代次收到迟到请求时返回冲突，不删除 successor。 +- 业务任务和长期监听不共享“环境存在”这一隐含占用。`runtime_use_lease` 默认 60 秒，20 秒续租；任务、素材处理、平台读取、会话历史和监听均显式 acquire/renew/release。 +- lease 续租失败会取消使用上下文并保留失败原因；释放操作幂等。gateway 重启后先按 runtime instance 恢复或标记 cleanup pending，再允许新 lease。 +- 采集 task lease 与合法长期监听 lease 分开；监听重连会核对 node/generation/binding，事件去重和 UID 冷却仍由 creator 数据层负责。 -网关自身一旦被攻破，socket 仍允许接管宿主机；应用内校验不能消除这个平台级风险。开发目标不新增认证或访问限制，安全由部署者自行把控；但当前代码仍强制 HTTP Basic Auth（除 `/healthz` 外的 API 与静态页面），Compose 仍要求用户名/密码。见 [main.go](../../cmd/control-plane/main.go) 与 [compose.yaml](../../compose.yaml)；本轮未移除现有策略，也不新增 RBAC 或认证 profile。 +## 代理边界 -## 运行 +控制面只把经过校验的出口配置交给 gateway。gateway 代理绑定 `127.0.0.1` 动态端口，并以 alias、binding version、runtime/network generation 绑定；代理恢复或移除失败必须可见。必需代理启动失败时不切换直连，不把不确定的写结果转换成成功。 -使用[部署说明](../deployment.md)中的完整变量及启动命令（含 Basic Auth、凭据主密钥与 socket GID），不维护另一套省略必填配置的命令。首次注册网关和镜像、创建账号、再创建停止态环境；恢复账号后显式启动。网关拉取镜像上限约 10 分钟；失败查看审计和保留的环境，不能按“数据库已回滚”直接假定没有资源。 +出口凭据只存在于控制面单次请求和 gateway 内存代理，不能进入 URL、日志、数据库、浏览器命令行或 API 响应。代理检查、重启恢复和清理均按当前 generation fence。 -## 现有账号与阶段 A 离线闭环 +## 非 root 与系统边界 -本节描述已运行的基础及历史 schema，不规定 plan01 的新业务范围。旧阶段 A 的 Mock 任务验证不等于 G0 平台能力或 G1 抖音完整验收；当前抖音受限读取连接器也未接成完整竞品/监听/发送流程。 +- gateway 由宿主机非 root 用户的 `systemctl --user` 服务运行；Xvfb、browser 和 Profile 的 unit 都属于该用户。 +- native gateway 不读取 Docker socket，不创建/删除浏览器 container、image、volume 或 network，也不保留 Docker fallback。 +- 启动命令拒绝 `--no-sandbox` 等越权参数；无法满足 sandbox、磁盘、路径、端口、权限或版本条件时直接失败并记录原因。 +- `/healthz` 用于存活；其它 gateway API 使用 Bearer token。控制面继续按现有部署策略提供 Basic Auth；本目标不新增 RBAC。 -`POST /api/phase-a/accounts` 只接受 `{name, platform, platform_account_key, tags, cookies}`；`platform` 限定为 `douyin`、`xiaohongshu`、`wechat-official`、`kuaishou`，`cookies` 可空，非空时须为浏览器 Cookie Header 格式。控制面通过持久 provider bridge 安全写入凭据：部署侧 Secret Manager/OS Keyring 注入 32 字节主密钥，独立凭据卷只保存 AES-GCM 密文，数据库只记录凭据引用；外部 API 不返回 Cookies、provider 或 `reference_key`。数据库明确回滚时清理凭据，提交结果未知时保留凭据并返回 `account_creation_result_unknown`，不自动破坏可能已提交的账号。内部账号 ID 由服务端生成，新账号默认 `paused`，`(platform, platform_account_key)` 全局唯一。pause/revoke 会递增账号版本并将 queued 任务置为 `policy_hold`，resume 要求稳定 binding，若绑定出口则需 healthy，并满足无活动 runtime/待清理等条件；未绑定出口的显式直连不要求出口记录。账号与浏览器环境通过一对一 `environment_binding` 关联，出口可复用；运行实例保留历史，并以 binding 和外部 runtime id 的部分唯一索引限制活动实例。 +## 运行与恢复 -草稿经 `POST /api/phase-a/confirmations` 显式确认后才可投递到 `/api/phase-a/tasks`。任务由幂等键去重；`POST /api/phase-a/mock/execute` 使用 `FOR UPDATE SKIP LOCKED` 领取一分钟租约，执行前统一核对账号、草稿和确认版本。缺少确认或版本不一致会进入 `needs_confirmation`，暂停账号或 Mock 策略结果会进入 `policy_hold`，不确定结果与过期租约进入 `needs_confirmation`；这些状态都不会自动重试。`GET /api/phase-a/audit` 只导出账号、确认版本、尝试和结果等非秘密证据。 +使用[部署说明](../deployment.md)中的变量和命令。首次部署时先安装已批准的浏览器版本和 Xvfb，再以 `scripts/install-native-browser-gateway.sh` 安装 gateway user service，随后在控制面注册 gateway、browser version、账号和出口。 -启动时控制面先应用 Phase A v1，再由 Hub runner 顺序应用 v2 至 v14；CreatorHub 业务迁移继续顺序应用至 v26（竞品同步 lease token、事件消息正文）；每一步都在事务和 advisory lock 下前向执行。v3 保留旧表、列和历史记录，旧账号回填为 `platform=mock` 并暂停，仅账号 ID 与环境 alias 相同的记录自动建立 binding；v4 追加环境动作审计字段与索引，v5 清理持久 fingerprint 中的旧代理字段，v6 增加可重试的 runtime cleanup 状态，v7 为 runtime lease 增加 binding version 并回填可确定的既有记录，v8 至 v10 补齐 cleanup/runtime 的不可变 generation 与兼容约束，v11、v12 增加任务恢复状态并修复兼容约束，v13 增加账号名称和 TAGS；v14 仅前向修复旧 PR v13 的空数据 schema。若旧 v13 已产生空引用或明文 Cookies，v14 会在删除前阻断启动，必须先将凭据迁入 provider。其余记录等待显式绑定。本阶段不提供破坏性自动回滚。 +启动、停止、回收和升级分别写 requested/finished operation；gateway 不可达、返回未知、cleanup pending、Profile 被占用和版本缺失都在页面和 API 显示。gateway 重启只恢复仍匹配当前代次的 runtime；机器重启不自动恢复已过期短任务。控制面后台 heartbeat 续租 running runtime 并调和 cleanup，读取列表不会用轮询伪装业务事件监听。 -`POST /api/network-exits` 只接受协议、主机、端口、已有 `credential_reference: {id}` 和预期出口身份；新出口为 `unchecked`，由 `POST /api/network-exits/:id/check` 经实际代理链路变为 `healthy` 或 `unhealthy`，`disable` 不可被检查重新启用。credential reference 的 `reference_key` 不出现在 API、日志或审计中；OS Keyring/Secret Manager bridge 在控制面进程启动前注入 `CREATORHUB_CREDENTIAL_`（大写十六进制），值为请求期解析的 `username:password`，控制面不持久化解析值。 +## 阶段 A 与 Creator 业务边界 -环境创建/启动/回收及直连边界以上方生命周期表为唯一说明。已配置代理时由控制面下发代理信息与 `disable_non_proxied_udp`；直连并非代理失败后的替代路径。 +阶段 A 账号、凭据引用、确认、任务和审计继续由 PostgreSQL 持久化。账号身份在每次写操作前由 gateway 返回的实际 UID 与已授权 `platform_account_key` 核对；Cookie、密码、token 和 provider reference 不回显。 -解析后的出口凭据只存在于控制面单次请求和网关内存转发器中；Docker inspect、容器环境、标签、挂载、`Config.Cmd` 与进程参数只包含 `docker-gateway` 的无凭据本地代理地址。网关内存代理以 alias、binding version 和 exit ID 共同标识 generation；生命周期操作按 alias 串行，重启恢复或重建必须重新核对该 generation，旧出口代理不能被新容器复用。 +Creator 采集流程使用独立 runtime-use lease：登录二维码、身份核对、抖音读取、评论/私信历史、素材下载、音频提取和指标处理完成后才释放浏览器使用权。结果保存与资源清理分开记录；结果未知时保留执行目录和证据，不能重复写入或删除可能仍被引用的素材。 -控制面后台每 20 秒调和网关（[runtimeLeaseHeartbeat](../../cmd/control-plane/main.go)），续租 running runtime、释放 stopped/missing runtime；列表/详情读取不触发调和，过期 lease 也会在绑定事务中回收。控制面还按账号调和抖音事件监听（[creator_events.go](../../cmd/control-plane/creator_events.go)）：每个已授权且有有效运行代际的账号只有一个监听，网关断连或代际改变时停止旧监听并退避重连；事件通知在控制面转换为互动事件后进入 `ProcessAutomaticEvent`，数据库的事件唯一键负责去重，写入结果不明不会由监听器重复发送。当前通知边界、平台事件游标/基线连续性和真实写操作仍需真机证据，不能把网关轮询队列视为平台监听验收。控制面用 PostgreSQL advisory transaction lock 按 alias 协调多副本；每个 Store 最多允许 5 个锁会话占用 10 连接池的一半，为锁内数据库调用保留连接。create 同时锁定账号 ID、alias、请求出口和请求镜像；start、reconcile/rebuild、rebind 和 upgrade 锁定 alias、当前出口及当前镜像（upgrade 还锁目标镜像），拿锁后重新读取出口与镜像版本。账号 pause/resume/revoke 使用账号 ID 与当前 binding alias 加入同一协调域；镜像禁用、引用更新或账号状态变更不能穿透在途生命周期。 - -非法 upgrade/rebind 目标在进入 advisory lock key 前按公开格式校验；审计仅保留环境原有的非秘密资源关联，并以 `upgrade_input_rejected` / `rebind_input_rejected` 写同一 operation ID 的 requested/finished 对。reconcile 恢复或重建后会重新读取 context，finished 事件关联实际激活的 runtime instance、binding version 与出口；后台释放 runtime 的成功或失败也写独立的 `reconcile` 审计对。 - -## 与新业务计划的边界 - -- 当前受限抖音读取仅允许自身身份与 `count=20/max_cursor=0` 首批作品，JSON 响应限制为 1 MiB，见 [douyin.py](../../cmd/docker_gateway/douyin.py)。竞品同步仍需外部平台证据；媒体通过 `POST /api/creator/works/:id/material/process` 下载到持久卷、用 FFmpeg 提取音轨，再调用显式配置的 `CREATOR_TRANSCRIPTION_BIN`。未配置转写入口时明确记为 failed，不接受手写 succeeded 或二进制塞入通用 JSON。 -- 当前账号凭据入口处理 Cookie；登录密码可按现有凭据保存流程保存，读取不回显、不进入日志，但绝不由系统自动注入或用于绕过人工登录。 -- 新业务平台监听、前端业务推送和现有后台运行租约是三件事；前两者要求见 plan01 A6，列表不主动探测的约定不禁止业务事件推送。现有生命周期/租约可能核验出口，不应误写成已完成 plan01 的手动代理管理目标。 +当前仍需真实账号、真实代理和人工 LAN 验收的项目见[验证记录](../evidence/native-browser-verification-2026-09-18.md)。多节点、A/B、跨机故障恢复和性能对比明确移出本目标。 diff --git a/docs/deployment.md b/docs/deployment.md index 62ff92b..4625f4a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,525 +1,155 @@ -# CreatorHub 部署 +# CreatorHub 单节点部署 -本文档按当前发布提交说明单台 Linux 主机 Docker Compose 部署及离线检查，不证明新产品功能已实现。[plan01](plan01.md) 是业务范围与验收依据；离线结果不能替代抖音/小红书最终真机验收。 +本文档描述当前单节点方案：control-plane 管理业务和数据库，宿主机上的 native browser gateway 管理 Xvfb、预安装浏览器、Profile、CDP 与代理。Docker 只可作为 PostgreSQL 的独立部署方式，不参与浏览器生命周期。 -当前控制面仍使用单用户 HTTP Basic Auth（除 `/healthz` 外，包括静态页面），不提供 RBAC 或多租户隔离。开发目标不新增认证/访问限制，但本轮未删除现有代码或配置；以下变量仍须填写，不新增认证 profile。 +## 服务与数据边界 -## 部署内容 +| 服务 | 位置 | 责任 | +| --- | --- | --- | +| `creator-hub` | 宿主机或 Compose | Go control-plane 与前端静态文件 | +| native browser gateway | 宿主机 systemd user service | Xvfb、浏览器、Profile、runtime、CDP、代理 | +| PostgreSQL | 宿主机或 Compose | 业务、绑定、runtime generation、cleanup 和 lease 状态 | -`compose.yaml` 会启动以下服务： +gateway 以非 root 用户运行。每个 runtime 有独立的 Xvfb display、CDP/代理端口、systemd transient unit、临时目录、owner、generation 和日志；账号 Profile、正式素材和长期监听不属于任务临时清理对象。 -- `creator-hub`：Go 1.26 控制面，同时提供 React 19/Vite 8 构建的静态页面； -- `docker-gateway`：受限 Docker API 网关，是唯一挂载 `/var/run/docker.sock` 的服务； -- `postgres`：PostgreSQL 17，数据保存在 `creatorhub_postgres` 命名卷； -- 浏览器容器：由网关按需创建，镜像引用由平台「镜像版本」页配置（缺失时网关自动拉取），Profile 保存在 `creatorhub-profile-<别名>` 命名卷。 - -控制面发布到宿主机所有网卡，局域网内可直接访问；浏览器容器可访问外网。 -`creator-hub` 会等待 `docker-gateway` 健康检查通过后再启动。 +当前目标是单机单节点。多节点调度、A/B 环境和跨机恢复不属于本轮部署步骤。 ## 前置条件 -- Linux 主机； -- Docker Engine 26 或兼容版本； -- Docker Compose v2； -- 当前用户可访问 Docker daemon； -- 可访问镜像仓库（如 `git.ipao.vip`），并已完成登录（如仓库要求认证）；镜像也可不在宿主机预拉取，网关会在缺失时按引用自动拉取； -- `curl`、`jq`、`openssl` 和 GNU `stat`，用于启动与业务验证命令； -- 若验证真实浏览器运行环境：Linux x86_64、一个可访问的固定 HTTP/HTTPS/SOCKS 出口，以及可被 Docker daemon 拉取的、以 immutable digest 固定的 `fingerprint-chromium` 镜像。镜像来源必须登记仓库地址、构建提交和 digest，禁止使用 `latest` 或未登记 tag；Xvfb/CDP 包装入口的可复现源码见 [`docker/browser-wrapper`](../docker/browser-wrapper/)。 +宿主机必须具备： -在仓库根目录执行预检： +- Linux、systemd user session 和可用的 `systemd-run --user`； +- Xvfb、`flock`、`curl`、`jq`、`openssl`； +- 合法的 fingerprint Chromium 及其绝对路径；浏览器以普通用户运行并保留 sandbox，禁止 `--no-sandbox`； +- 至少 20 GB 可用磁盘，Profile 缓存上限 20 GB，runtime 日志上限 1 GB； +- 如使用代理，准备真实可访问的 HTTP/HTTPS/SOCKS4/SOCKS5 出口及认证失败测试条件； +- PostgreSQL 17。Docker Compose 仅用于单独运行 PostgreSQL 时才需要 Docker。 + +预检： ```bash -test -S /var/run/docker.sock -docker info >/dev/null -docker compose version -: "${DOCKER_GID:?export DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) in this shell}" -: "${CONTROL_PLANE_USERNAME:?export CONTROL_PLANE_USERNAME in this shell}" -: "${CONTROL_PLANE_PASSWORD:?export CONTROL_PLANE_PASSWORD in this shell}" -: "${CREATORHUB_CREDENTIAL_MASTER_KEY:?export CREATORHUB_CREDENTIAL_MASTER_KEY in this shell}" -docker compose config --quiet +id -u +systemctl --user is-system-running +command -v systemd-run systemctl Xvfb flock +systemd-run --user --wait --pipe --unit=creatorhub-preflight-$$ -- /usr/bin/true ``` -## 首次部署 +## 安装 native gateway -以下命令应在同一个 shell、仓库根目录执行。本说明采用显式导出模式：Compose 启动和手工验证必须读取当前 shell 中导出的同一组变量；验证脚本不会自行读取 `.env`，也不会使用 Compose 默认值。若变量只写在 `.env` 中，请先将相同值 export 到当前 shell。`DOCKER_GID` 必须与宿主机 Docker socket 的组一致；`CREATORHUB_PORT` 控制控制面的宿主机端口。 +复制配置并填写真实浏览器路径。配置文件只允许 gateway 用户读取： + +```bash +install -d -m 0700 ~/.config/creatorhub +cp deploy/browser-gateway.env.example ~/.config/creatorhub/browser-gateway.env +chmod 600 ~/.config/creatorhub/browser-gateway.env +$EDITOR ~/.config/creatorhub/browser-gateway.env +``` + +关键配置： + +- `LISTEN_ADDR=0.0.0.0:8081`：允许局域网 control-plane 联调；防火墙只放行受信网络； +- `GATEWAY_TOKEN`：至少 16 个字符，和控制面登记值完全一致； +- `BROWSER_PATH`、`BROWSER_VERSION`：预安装浏览器的绝对路径和版本； +- `BROWSER_STATE_DIR`：runtime 清单、锁和日志目录； +- `BROWSER_PROFILE_ROOT`：持久账号 Profile 根目录；不得指向临时目录； +- `RUNTIME_MIN_FREE_BYTES`、`RUNTIME_LOG_MAX_BYTES`、`PROFILE_CACHE_MAX_BYTES`：资源硬限制。 + +安装并启动 user service： + +```bash +scripts/install-native-browser-gateway.sh +curl --fail --silent --show-error http://127.0.0.1:8081/healthz +curl --fail --silent --show-error -H "Authorization: Bearer ${GATEWAY_TOKEN}" http://127.0.0.1:8081/v1/info +systemctl --user --no-pager status creatorhub-browser-gateway.service +``` + +若 user service 不随登录启动，按主机运行规范启用 lingering；不要把 gateway 改成 root service： + +```bash +loginctl enable-linger "$(id -un)" +``` + +## 运行 control-plane + +### Compose 运行 control-plane 和 PostgreSQL + +Compose 不再创建 gateway 服务，也不挂载 Docker socket： ```bash -export DOCKER_GID="$(stat -c '%g' /var/run/docker.sock)" export CREATORHUB_PORT=8080 -export GATEWAY_TOKEN="$(openssl rand -hex 24)" export CONTROL_PLANE_USERNAME=creatorhub export CONTROL_PLANE_PASSWORD="$(openssl rand -hex 24)" export CREATORHUB_CREDENTIAL_MASTER_KEY="$(openssl rand -base64 32)" docker compose config --quiet docker compose up --detach --build -``` - -网关配置了 45 秒停止宽限期，并在收到 SIGTERM/SIGINT 时停止接收请求、排空有限期限内的在途请求，再关闭事件订阅和代理；本地测试覆盖该顺序，但部署验证仍须记录正常退出码、实际耗时和没有 SIGKILL，不能仅以“等待了 45 秒”证明优雅退出。 - -控制面启动时会连接 PostgreSQL，并在事务和 advisory lock 保护下自动执行前向迁移。迁移失败时控制面会退出，由 Compose 按 `restart: unless-stopped` 重启；先检查日志，不要删除数据卷。 - -### 首次配置 - -服务起来后打开 ： - -1. 「网关管理」页注册网关：名称如 `gw-main`，Endpoint `http://docker-gateway:8081`，令牌填 `GATEWAY_TOKEN` 的值（即 `openssl rand -hex 24` 生成的值）。 -2. 按 [`docker/browser-wrapper/README.md`](../docker/browser-wrapper/README.md) 用已登记 base digest 构建包装镜像，再在「镜像版本」页添加发布 digest。生产与验收必须填写 `git.ipao.vip/rogee/creatorhub-browser-wrapper@sha256:<已登记摘要>`；`148.0.7778.215` 只作为浏览器版本元数据，不能单独作为不可变引用。登记内容同时包含 base 镜像源仓库、base digest、构建提交、Dockerfile 路径和发布摘要。 -3. 「社媒账号」页先创建账号（默认暂停），再在「运行环境」选该账号、网关、镜像，填写中文名、小写别名及指纹参数。代理可选；指定代理须先手动检测为健康，留空是明确直连，不是失败回退。 -4. 创建得到停止态容器；在账号页恢复账号后回环境页显式启动。容器名 `creatorhub-browser-<别名>`，Profile 卷 `creatorhub-profile-<别名>`。回收只删除容器、保留环境/binding/Profile；完整契约见[架构说明](architecture/container-control.md)。当前直连 create/start 不代表 upgrade/rebind 已支持空出口。 - -### 人工登录二维码 - -抖音账号页的「显示登录二维码」由 CreatorHub 通过已绑定 gateway 请求登录画面，并直接展示在后台；它不会注入密码、Cookie 或自动完成登录。登录画面只在内存中返回，前端显示两分钟有效期；完成扫码或验证码后，点击「核验浏览器身份」确认 UID 与账号绑定一致。gateway 无法取得二维码时，后台展示实际登录/验证码画面并要求人工处理，不把不确定结果标记为成功。 - -对应接口为 `POST /api/creator/accounts//login-qr`，仅接受已授权、已运行的抖音账号；响应中的 `image_base64` 只用于当前页面展示，不应写入日志、数据库或备份。 - -## 部署验证 - -```bash -curl --fail --silent --show-error \ - --retry 30 --retry-delay 2 --retry-connrefused \ - --output /dev/null "http://127.0.0.1:${CREATORHUB_PORT}/healthz" - -curl --fail --silent --show-error \ - --retry 30 --retry-delay 2 --retry-connrefused \ - --output /dev/null "http://127.0.0.1:${CREATORHUB_PORT}/readyz" - -curl --fail --silent --show-error \ - --retry 30 --retry-delay 2 --retry-connrefused \ - --user "${CONTROL_PLANE_USERNAME}:${CONTROL_PLANE_PASSWORD}" \ - "http://127.0.0.1:${CREATORHUB_PORT}/api/browsers" >/dev/null - -docker compose exec -T postgres \ - psql -U creatorhub -d creatorhub -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 32;' \ - | grep -qx 1 -docker compose exec -T postgres \ - psql -U creatorhub -d creatorhub -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 35;' \ - | grep -qx 1 - +curl --fail --silent --show-error "http://127.0.0.1:${CREATORHUB_PORT}/healthz" +curl --fail --silent --show-error "http://127.0.0.1:${CREATORHUB_PORT}/readyz" docker compose ps +docker compose logs --tail=200 creator-hub postgres ``` -健康检查应成功，浏览器列表接口应返回 JSON，迁移查询当前应输出 `1`，三个 Compose 服务应为运行状态。`CREATORHUB_CREDENTIAL_MASTER_KEY` 必须由部署侧 Secret Manager/OS Keyring 持久保存并在每次启动时注入同一值；账号凭据以 AES-GCM 密文写入独立 `creatorhub_credentials` 卷，轮换主密钥前必须先迁移已有凭据。然后访问 ；修改过 `CREATORHUB_PORT` 时使用对应端口。 +Compose 中的 control-plane 访问宿主机 gateway 时，网关 Endpoint 登记为 `http://host.docker.internal:8081`。裸机运行 control-plane 时登记为 `http://127.0.0.1:8081`。两种情况下都必须使用同一个 `GATEWAY_TOKEN`。 -旧实验版本的数据库异常应先记录版本、备份并核实，不把历史文档中的建议迁移当成本期开发要求；本期不新增兼容迁移、回填或双写。下文更新/恢复命令仅描述现有部署的数据操作，不改变 plan01 验收范围。 - -排障时读取结构化服务日志： +### 裸机开发 ```bash -docker compose logs --tail=200 creator-hub docker-gateway postgres +npm --prefix web ci +pnpm dev ``` -## 手工业务验证（阶段 A Mock） +`pnpm dev:backend` 只启动 PostgreSQL 依赖，并在启动 control-plane 前检查宿主机 gateway；它不会创建 browser runtime 或替代 gateway。 -本节的旧阶段 A 业务执行器是 Mock，不向真实社交平台发送；容器、数据库、网关和可选代理则是实际运行资源。最小业务路径是：账号 → 草稿 → 显式确认 → 任务入队 → Mock 执行 → 审计回溯。 +默认联调地址： -页面路径如下： +| 服务 | 地址 | +| --- | --- | +| Vite | `http://127.0.0.1:5173` | +| control-plane | `http://0.0.0.0:8082` | +| native gateway | `http://0.0.0.0:8081` | +| PostgreSQL | `127.0.0.1:5432` | -1. 登录后在「网关管理」注册 ，令牌必须等于 GATEWAY_TOKEN。 -2. 在「镜像版本」添加并启用一个可拉取的 fingerprint-chromium 镜像。 -3. 可选：在「网络出口」创建出口并点击「检测」，健康状态必须为「健康」；这里只填写凭据引用 ID，不填写密码、Cookie 或 token。留空则使用网关所在机器的网络出口直连。 -4. 在「社媒账号」创建平台为抖音的账号；创建后默认暂停，后续仍可使用 Mock 执行器验证离线闭环。 -5. 在「运行环境」选择该账号、镜像和可选的健康出口，使用正整数 Fingerprint Seed 创建环境；随后在账号详情点击「恢复账号」，再在「运行环境」点击「启动」。 -6. 在账号详情创建文本草稿，点击「核对草稿」，勾选“我已核对当前账号、草稿内容、运行环境和固定出口”，依次执行「确认当前快照」→「保存确认」→「加入队列」。 -7. Mock 执行器没有独立页面，使用下面的 POST /api/phase-a/mock/execute，再到「任务中心」和「审计」核对结果。 +## 首次配置与生命周期 -以下命令可在已经启动的仓库根目录直接执行，并且必须在启动 Compose 的同一个 shell 中执行。开始前确认 `CREATORHUB_PORT`、`GATEWAY_TOKEN`、`CONTROL_PLANE_USERNAME` 和 `CONTROL_PLANE_PASSWORD` 都已 export；脚本不会读取未 export 的 `.env` 值或 Compose 默认值。出口参数必须替换成已批准且可从 creator-hub 容器访问的无认证代理；若代理需要认证，先按部署规范准备 credential_reference，不要把秘密值写进命令或仓库。镜像引用也可改成已发布的 registry 引用；本地 smoke test 可以使用宿主机上已有的镜像标签。 +1. 在「网关管理」登记 native gateway；读取 `/v1/info` 核对稳定 `node_id` 和浏览器版本能力。 +2. 在「浏览器版本」登记 gateway 上存在的版本和绝对路径；版本缺失时创建动作必须失败，不得自动拉取或改用其它版本。 +3. 创建账号和环境，选择账号、gateway、浏览器版本及可选网络出口。 +4. 显式启动环境；gateway 返回 runtime ID、network ID、node ID、generation、display、CDP/代理端口和 readiness。 +5. 登录二维码只在内存响应中短暂展示；完成人工登录后执行账号身份核验。 +6. 采集和人工写操作必须通过账号身份、binding version、runtime generation 和 runtime use lease 检查。 +7. 停止或失败时，业务结果和 cleanup 状态分别记录。cleanup 失败显示 `pending` 并可重试；不得把 pending 伪装为已停止。 +8. 重新启动使用相同 Profile 根目录，核对浏览器版本和账号身份，不删除登录资料。 -~~~bash -set -Eeuo pipefail +gateway 的 runtime 清单、Profile 和日志目录不得由 `docker compose down` 删除。只有明确授权时才执行账号 Profile 或正式素材删除。 -: "${CREATORHUB_PORT:?export CREATORHUB_PORT (same value used by Compose)}" -: "${CONTROL_PLANE_USERNAME:?set CONTROL_PLANE_USERNAME}" -: "${CONTROL_PLANE_PASSWORD:?set CONTROL_PLANE_PASSWORD}" -: "${GATEWAY_TOKEN:?export GATEWAY_TOKEN (same value registered in the gateway)}" -: "${PROXY_HOST:?set PROXY_HOST to an approved fixed proxy host}" -: "${PROXY_PORT:?set PROXY_PORT to an approved fixed proxy port}" -BASE_URL="http://127.0.0.1:${CREATORHUB_PORT}" +## 验证与故障处理 -api() { - curl --fail-with-body --silent --show-error \ - --user "${CONTROL_PLANE_USERNAME}:${CONTROL_PLANE_PASSWORD}" \ - -H 'Content-Type: application/json' "$@" -} - -[[ "$PROXY_PORT" =~ ^[0-9]+$ ]] && (( PROXY_PORT >= 1 && PROXY_PORT <= 65535 )) - -run_id="manual-$(date +%s)" -gateway_name="gw-${run_id}" -image_version="${IMAGE_VERSION:-148.0.7778.215}" -: "${IMAGE_REF:?IMAGE_REF must be an immutable browser-wrapper @sha256 reference}" -case "$IMAGE_REF" in - *@sha256:*) image_ref="$IMAGE_REF" ;; - *) echo "IMAGE_REF must contain @sha256:" >&2; exit 2 ;; -esac -exit_protocol="${PROXY_PROTOCOL:-http}" -account_key="${run_id}-platform" -env_alias="${run_id}-env" - -gateway_json="$(api -X POST "$BASE_URL/api/gateways" --data "$(jq -n \ - --arg name "$gateway_name" --arg token "$GATEWAY_TOKEN" \ - '{name:$name,endpoint:"http://docker-gateway:8081",token:$token}')")" -jq -e --arg name "$gateway_name" '.name == $name' <<<"$gateway_json" >/dev/null - -image_json="$(api -X POST "$BASE_URL/api/browser-images" --data "$(jq -n \ - --arg version "$image_version" --arg ref "$image_ref" \ - '{version:$version,image_ref:$ref,note:"manual validation",enabled:true}')")" -jq -e --arg version "$image_version" '.version == $version and .enabled == true' <<<"$image_json" >/dev/null - -exit_json="$(api -X POST "$BASE_URL/api/network-exits" --data "$(jq -n \ - --arg protocol "$exit_protocol" --arg host "$PROXY_HOST" --argjson port "$PROXY_PORT" \ - '{protocol:$protocol,host:$host,port:$port}')")" -exit_id="$(jq -er '.id' <<<"$exit_json")" -checked_exit="$(api -X POST "$BASE_URL/api/network-exits/$exit_id/check")" -jq -e '.health_status == "healthy"' <<<"$checked_exit" >/dev/null - -account_json="$(api -X POST "$BASE_URL/api/phase-a/accounts" --data "$(jq -n \ - --arg name "手工验证账号" --arg key "$account_key" --arg cookies "${ACCOUNT_COOKIES:-sessionid=manual-test}" \ - '{name:$name,platform:"douyin",platform_account_key:$key,tags:["manual"],cookies:$cookies}')")" -account_id="$(jq -er '.id' <<<"$account_json")" -jq -e '.authorization_status == "authorized" and .runtime_status == "paused"' <<<"$account_json" >/dev/null - -created_env="$(api -X POST "$BASE_URL/api/browsers" --data "$(jq -n \ - --arg alias "$env_alias" --arg name "手工验证环境" --arg gateway "$gateway_name" \ - --arg version "$image_version" --arg account "$account_id" --arg exit "$exit_id" \ - '{alias:$alias,name:$name,gateway:$gateway,image_version:$version,account_id:$account,network_exit_id:$exit,fingerprint:{seed:1000}}')")" -jq -e --arg alias "$env_alias" '.alias == $alias' <<<"$created_env" >/dev/null - -api -X POST "$BASE_URL/api/phase-a/accounts/$account_id/resume" >/dev/null -api -X POST "$BASE_URL/api/browsers/$env_alias/start" >/dev/null - -for _ in $(seq 1 30); do - runtime_json="$(api "$BASE_URL/api/browsers/$env_alias")" - if jq -e '.runtime_id != "" and .runtime_instance_id != ""' <<<"$runtime_json" >/dev/null; then - break - fi - sleep 1 -done -jq -e '.runtime_id != "" and .runtime_instance_id != ""' <<<"$runtime_json" >/dev/null - -account_json="$(api "$BASE_URL/api/phase-a/accounts/$account_id")" -account_version="$(jq -er '.version' <<<"$account_json")" -draft_json="$(api -X POST "$BASE_URL/api/phase-a/drafts" --data "$(jq -n \ - --arg account "$account_id" '{account_id:$account,content:"阶段 A 手工验证内容"}')")" -draft_id="$(jq -er '.id' <<<"$draft_json")" -draft_version="$(jq -er '.version' <<<"$draft_json")" - -confirmation_json="$(api -X POST "$BASE_URL/api/phase-a/confirmations" --data "$(jq -n \ - --arg draft "$draft_id" --argjson account_version "$account_version" --argjson draft_version "$draft_version" \ - '{draft_id:$draft,account_version:$account_version,draft_version:$draft_version}')")" -confirmation_id="$(jq -er '.id' <<<"$confirmation_json")" - -task_json="$(api -X POST "$BASE_URL/api/phase-a/tasks" --data "$(jq -n --arg confirmation "$confirmation_id" '{confirmation_id:$confirmation}')")" -task_id="$(jq -er '.id' <<<"$task_json")" -jq -e '.state == "queued"' <<<"$task_json" >/dev/null -duplicate_task_json="$(api -X POST "$BASE_URL/api/phase-a/tasks" --data "$(jq -n --arg confirmation "$confirmation_id" '{confirmation_id:$confirmation}')")" -jq -e --arg task_id "$task_id" '.id == $task_id' <<<"$duplicate_task_json" >/dev/null - -execution_json="$(api -X POST "$BASE_URL/api/phase-a/mock/execute" --data \ - '{"worker_id":"manual-worker","outcome":"succeeded"}')" -jq -e '.was_claimed == true and .state == "succeeded"' <<<"$execution_json" >/dev/null - -task_detail="$(api "$BASE_URL/api/phase-a/tasks/$task_id")" -jq -e '(.state == "succeeded") and ((.attempts | length) >= 1) and (.attempts[0].evidence.mock_outcome == "succeeded")' <<<"$task_detail" >/dev/null - -audit_json="$(api "$BASE_URL/api/phase-a/audit?task_id=$task_id&page_size=100")" -for event_type in task_queued task_claimed task_finished; do - jq -e --arg event_type "$event_type" '.data | any(.[]; .event_type == $event_type)' <<<"$audit_json" >/dev/null -done - -printf 'PASS account=%s environment=%s task=%s state=succeeded audit=task_queued,task_claimed,task_finished\n' \ - "$account_id" "$env_alias" "$task_id" -~~~ - -通过标准：/healthz 返回 204；受保护的 /api/browsers 返回 JSON；网关、控制面、PostgreSQL 均为运行状态；运行环境有 runtime_id 和 runtime_instance_id；任务最终为 succeeded，且执行尝试的脱敏证据为 mock_outcome=succeeded；审计至少包含 task_queued、task_claimed、task_finished。重复提交同一确认时应返回同一任务 ID，不应产生第二条任务。 - -验证完成后的容器回收（保留账号、环境/binding、PostgreSQL 数据与 Profile 卷，不是永久删除环境）： - -~~~bash -api() { - curl --fail-with-body --silent --show-error \ - --user "${CONTROL_PLANE_USERNAME}:${CONTROL_PLANE_PASSWORD}" \ - -H 'Content-Type: application/json' "$@" -} -api -X POST "$BASE_URL/api/browsers/$env_alias/stop" >/dev/null -api -X DELETE "$BASE_URL/api/browsers/$env_alias" >/dev/null -docker compose stop -~~~ - -## 常见故障排查 - -| 现象 | 先检查 | 处理 | -| --- | --- | --- | -| docker compose config 报 required | CONTROL_PLANE_USERNAME、CONTROL_PLANE_PASSWORD、CREATORHUB_CREDENTIAL_MASTER_KEY 是否在当前 shell 非空 | 重新 export 三个变量；用户名不能含冒号，密码至少 6 字节，主密钥必须为 32 字节的 base64 | -| 手工验证脚本在 `:?` 处退出 | CREATORHUB_PORT、GATEWAY_TOKEN、CONTROL_PLANE_USERNAME、CONTROL_PLANE_PASSWORD、CREATORHUB_CREDENTIAL_MASTER_KEY 是否都已 export | 在启动 Compose 的同一个 shell 中 export 完整变量集；不要只依赖 `.env` 或 Compose 默认值 | -| creator-hub 未启动 | docker compose ps、docker compose logs --tail=200 postgres docker-gateway creator-hub | 先确认 PostgreSQL 与网关 health 为 healthy；网关需能访问 /var/run/docker.sock，DOCKER_GID 使用 stat -c '%g' /var/run/docker.sock 的实际值 | -| API 返回 401 | curl 是否带 --user CONTROL_PLANE_USERNAME:CONTROL_PLANE_PASSWORD | /healthz 不需要认证，其余 /api/* 需要控制面 Basic Auth | -| 生命周期动作报网关不可用 | 网关注册的 Endpoint、令牌与 Compose 的 GATEWAY_TOKEN | Endpoint 应为 ；令牌必须完全一致。列表/详情只读持久记录，其成功不能证明网关在线 | -| 出口一直是 unchecked/unhealthy | 出口协议、主机、端口；控制面容器到代理的连通性；last_check_reason | 先用无认证代理完成最小验证；有认证时只提供已配置的凭据引用，不把认证值放到请求、日志或文档 | -| 创建环境时报 image_unavailable 或拉取超时 | image_ref 格式、镜像架构、Docker daemon 的 registry 登录和网络 | 版本表中的镜像必须可被 Docker daemon 拉取，最长约 10 分钟；已保存环境/binding 不会因网关失败自动删掉，核对记录后按原配置恢复 | -| 恢复/入队返回 503 | readiness、GET /api/browsers/、GET /api/network-exits/ | binding_missing、network_exit_unhealthy、runtime_missing 表示固定资源未就绪；先修复出口并启动原环境，不要换出口重试 | -| Mock 执行没有领取任务 | GET /api/phase-a/tasks/ 的 state、hold_reason | POST /api/phase-a/mock/execute 只领取满足账号、确认、已绑定出口健康（显式直连无此项）、活动 runtime 和租约条件的 queued 任务；policy_hold/needs_confirmation 不会自动重试 | -| 停止 Compose 后浏览器仍在运行 | docker ps --filter 'name=^creatorhub-browser-' | 动态浏览器不由 Compose 管理；先通过「运行环境」停止/回收，再 docker compose stop，不要误删 Profile 卷 | - -不要执行 docker compose down --volumes 作为普通排障手段；它会删除 PostgreSQL 数据卷。不要把控制面密码、网关令牌、代理密码、Cookie 或 token 写入仓库、截图、日志或审计查询。 - -## 配置 - -Compose 部署时通常只需设置以下宿主机变量： - -| 变量 | 默认值 | 说明 | -| --- | --- | --- | -| `CREATORHUB_PORT` | `8080` | 控制面宿主机端口，局域网可访问 | -| `DOCKER_GID` | 无（必填） | Docker socket 的宿主机组 ID；必须按实际值设置 | -| `GATEWAY_TOKEN` | `dev-creatorhub-gateway-token` | Compose 未提供变量时的默认值；本说明要求显式 export 随机值，并同步填入网关注册表单 | -| `CONTROL_PLANE_USERNAME` | 无（必填） | 控制面唯一用户；不能包含冒号 | -| `CONTROL_PLANE_PASSWORD` | 无（必填） | 控制面密码，至少 6 字节；使用随机值 | -| `CREATORHUB_CREDENTIAL_MASTER_KEY` | 无（必填） | 32 字节 base64；由部署 Secret Manager/OS Keyring 持久注入，重启后必须保持一致 | -| `BAILIAN_API_KEY` | 空（按需） | 已批准的生产 AI API key；为空时文本 AI 动作明确返回 unavailable，不使用 Mock | -| `BAILIAN_BASE_URL` | 客户端默认值 | 无用户信息的 HTTP(S) 地址；供应商变更前需完成审批和脱敏样本验证 | -| `CREATOR_MEDIA_DIR` | `/var/lib/creatorhub/materials` | 持久媒体目录；必须挂载持久卷，不能使用临时目录替代 | -| `CREATOR_TRANSCRIPTION_BIN` | 空（按需） | 可执行的本地转写入口；为空时转写明确失败，不伪造 succeeded | - -服务本身支持并校验以下环境变量；`compose.yaml` 会在控制面凭据缺失或为空时拒绝渲染。下表中的 Compose 默认值不由手工验证脚本隐式读取；手工验证沿用上文的显式 export 要求： - -| 服务 | 变量 | 当前 Compose 值 | -| --- | --- | --- | -| `creator-hub` | `LISTEN_ADDR` | 默认 `:8080` | -| `creator-hub` | `WEB_DIR` | 镜像内固定为 `/app/web` | -| `creator-hub` | `DATABASE_URL` | `postgres://creatorhub@postgres/creatorhub?sslmode=disable` | -| `creator-hub` | `LOG_LEVEL` | 默认 `info` | -| `creator-hub` | `CONTROL_PLANE_USERNAME` | 必填；HTTP Basic Auth 用户名 | -| `creator-hub` | `CONTROL_PLANE_PASSWORD` | 必填且至少 6 字节；不会写入日志或响应 | -| `creator-hub` | `CREATORHUB_CREDENTIAL_MASTER_KEY` | 必填；解密独立凭据卷，不写入数据库、日志或响应 | -| `creator-hub` | `CREATORHUB_CREDENTIAL_STORE_DIR` | 默认 `/var/lib/creatorhub/credentials`；必须为绝对路径且持久可写 | -| `creator-hub` | `BAILIAN_API_KEY` | 可选；仅用于已批准的文本 AI，缺失时不回退 Mock | -| `creator-hub` | `BAILIAN_BASE_URL` | 可选；HTTP(S) 供应商地址，禁止携带用户信息 | -| `docker-gateway` | `LISTEN_ADDR` | 默认 `:8081` | -| `docker-gateway` | `DOCKER_SOCKET` | 默认值和 Compose 挂载均固定为 `/var/run/docker.sock`；不能只覆盖环境变量 | -| `docker-gateway` | `BROWSER_NETWORK` | `creatorhub_browser` | -| `docker-gateway` | `GATEWAY_TOKEN` | 与平台注册值一致，长度 ≥16；`/v1` 全部接口校验 Bearer 令牌 | -| `docker-gateway` | `BROWSER_CDP_URL` | 空；仅本地联调时连接外部 CDP，例如 `http://host.docker.internal:9222` | -| `docker-gateway` | `BROWSER_CDP_TARGET_ID` | 空；外部 CDP 多页面时必须指定抖音页面 ID | -| `docker-gateway` | `LOG_LEVEL` | 默认 `info` | - -不要把凭据写入仓库或 Compose 文件。 - -### 本地 9222 CDP 联调 - -网关支持显式连接已经登录的本地 CDP，不创建或回收 Docker 浏览器。仅用于开发和真实平台链路测试，不应在生产 Compose 中设置。先从 `http://127.0.0.1:9222/json/list` 选择唯一的抖音页面 `id`，再在宿主机启动网关： +代码检查： ```bash -export LISTEN_ADDR=127.0.0.1:18081 -export GATEWAY_TOKEN=0123456789abcdef -export BROWSER_CDP_URL=http://127.0.0.1:9222 -export BROWSER_CDP_TARGET_ID=<已登录抖音页面的 target id> -export BROWSER_CDP_ALIAS=local-cdp -export BROWSER_CDP_NETWORK_ID=local-cdp -python3 -m cmd.docker_gateway.gateway -``` - -该模式的 generation 固定为 `binding_version=1`、`runtime_id=64 个 0`、`network_id=local-cdp`。生命周期接口明确不可用于此浏览器；身份、受限读取、事件订阅和动作接口仍要求完整 generation，并继续执行 UID 核对。`BROWSER_CDP_TARGET_ID` 必须固定到抖音页面，不能把包含多个网站的 CDP 页面列表交给自动猜测。 - -### P0-lite 停机通知 - -当前只使用 `creator-hub` 的专用 Logrus JSON logger 作为通知渠道；它固定输出警告，不继承业务 `LOG_LEVEL`。选择它是因为 Compose 已可靠收集服务日志，不需要新增外部账号、凭据、网络重试或通知依赖。仅 `policy_hold` 和 `needs_confirmation` 会产生 `operator attention required`，字段限定为 `event_type`、`reason_code`、账号/任务 ID；不会包含请求头、Secret 引用或凭据值。运维可用下列命令接入现有日志采集或人工查看： - -```bash -docker compose logs creator-hub | grep 'operator attention required' -``` - -该渠道是单实例 P0-lite 能力，不保证外部送达、升级或确认回执；只有出现明确的多渠道/送达需求时才增加 webhook 或消息平台。 - -## 更新与回滚 - -更新前记录当前版本并备份数据库： - -```bash -set -Eeuo pipefail -git rev-parse HEAD -umask 077 -export BACKUP_FILE="$(pwd)/creatorhub-$(date +%Y%m%d-%H%M%S).dump" -docker compose exec -T postgres \ - pg_dump -U creatorhub -d creatorhub --format=custom \ - > "$BACKUP_FILE" -test -s "$BACKUP_FILE" -docker compose exec -T postgres pg_restore --list \ - < "$BACKUP_FILE" >/dev/null -``` - -PostgreSQL dump 之外，发布备份必须同时包含 credentials、materials 和动态 Profile 卷；这些归档存放在部署侧受限目录，不放进仓库或共享聊天。示例（`BACKUP_DIR` 使用独立磁盘或 Secret Manager 的受控挂载目录）： - -```bash -set -Eeuo pipefail -umask 077 -BACKUP_DIR=/srv/creatorhub-backups/$(date +%Y%m%d-%H%M%S) -mkdir -p "$BACKUP_DIR/profiles" -# PostgreSQL dump -pg_dump_file="$BACKUP_DIR/creatorhub.dump" -docker compose exec -T postgres pg_dump -U creatorhub -d creatorhub --format=custom > "$pg_dump_file" -test -s "$pg_dump_file" -# Credential and material volumes -for volume in creatorhub_credentials creatorhub_materials; do - docker run --rm -v "$volume:/data:ro" -v "$BACKUP_DIR:/backup" alpine:3.22 \ - tar czf "/backup/${volume}.tar.gz" -C /data . -done -# Dynamic browser Profile volumes; absence is an explicit empty set. -for volume in $(docker volume ls -q --filter name='^creatorhub-profile-'); do - docker run --rm -v "$volume:/data:ro" -v "$BACKUP_DIR/profiles:/backup" alpine:3.22 \ - tar czf "/backup/${volume}.tar.gz" -C /data . -done -find "$BACKUP_DIR" -type f -exec sha256sum {} + > "$BACKUP_DIR/SHA256SUMS" -``` - -`CREATORHUB_CREDENTIAL_MASTER_KEY` 不写入上述归档；部署管理员必须在独立的 Secret Manager/OS Keyring 保留一份受访问控制的密钥托管记录，并确认恢复主机能注入同一值。恢复前核对 `SHA256SUMS`、主密钥记录和备份目录权限；恢复后在隔离 Compose 项目中还原四类卷，检查账号凭据可解密、材料文件可读、Profile 可挂载，再切换服务。任何一类缺失都判定为恢复失败，不把应用健康检查当作数据恢复证据。 - -拉取已审核版本后，重新执行部署和验证： - -```bash -set -Eeuo pipefail -git pull --ff-only -export DOCKER_GID="$(stat -c '%g' /var/run/docker.sock)" -export CREATORHUB_PORT=8080 +python3 -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' -q +python3 -m coverage run --source=cmd/browser_gateway --branch -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' -q +python3 -m coverage report --omit='cmd/browser_gateway/test_*.py' --fail-under=65 +go test ./... +go vet ./... +go build ./cmd/control-plane +go test -race ./... +npm --prefix web ci +npm --prefix web test -- --run +npm --prefix web run build docker compose config --quiet -docker compose up --detach --build ``` -镜像版本升级在页面「运行环境 → 升级」完成：删除旧容器并保留 Profile，用新镜像与原指纹/binding 重建，账号可运行才启动，否则保持停止态；无自动回滚。失败先查看记录，人工重试由现有流程调和，不承诺所有失败都可直接重试消除。当前 upgrade 无条件要求有效代理出口，直连环境会失败；rebind 也不支持空出口切回直连。此为现状限制，不是新产品范围裁决。 +资源和失败场景必须由人工按 [native-browser-verification.md](native-browser-verification.md) 留证：Profile 占用、版本缺失、代理认证/网络失败、Xvfb 或浏览器启动失败、取消、超时、gateway 重启、重复清理、磁盘不足、旧 generation、监听与采集并行，以及真实抖音登录和采集。单元测试、Fake systemd 和 Mock gateway 不能替代真实平台证据。 -数据库迁移只支持安全前进，不提供自动破坏性回滚。需要同时恢复旧代码和更新前数据库时，修改下面两个变量后**整块执行一次**；不要逐行或拆块执行。预检、恢复演练、动态容器停止、停服、主库恢复、提交切换和启动都位于同一个 fail-fast subshell 中。 +常用诊断： ```bash -( - set -Eeuo pipefail - - BACKUP_FILE=/absolute/path/to/creatorhub-YYYYmmdd-HHMMSS.dump - RESTORE_REV=PREVIOUS_REVIEWED_COMMIT_SHA - : "${BACKUP_FILE:?set BACKUP_FILE to the absolute archive path}" - : "${RESTORE_REV:?set RESTORE_REV to the previous reviewed commit SHA}" - test -r "$BACKUP_FILE" - test -s "$BACKUP_FILE" - git cat-file -e "${RESTORE_REV}^{commit}" - docker compose exec -T postgres pg_restore --list \ - < "$BACKUP_FILE" >/dev/null - - docker compose exec -T postgres \ - dropdb --if-exists --force -U creatorhub creatorhub_restore_check - docker compose exec -T postgres \ - createdb -U creatorhub creatorhub_restore_check - docker compose exec -T postgres \ - pg_restore -U creatorhub -d creatorhub_restore_check \ - --exit-on-error --no-owner --no-privileges \ - < "$BACKUP_FILE" - docker compose exec -T postgres \ - psql -U creatorhub -d creatorhub_restore_check -v ON_ERROR_STOP=1 -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 32;' \ - | grep -qx 1 - docker compose exec -T postgres \ - psql -U creatorhub -d creatorhub_restore_check -v ON_ERROR_STOP=1 -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 35;' \ - | grep -qx 1 - docker compose exec -T postgres \ - dropdb --force -U creatorhub creatorhub_restore_check - - list_running_browsers() { - docker ps --quiet \ - --filter 'name=^creatorhub-browser-' \ - --filter 'label=io.creatorhub.managed=true' \ - --filter 'label=io.creatorhub.runtime-id' - } - - browser_ids="$(list_running_browsers)" || exit 1 - for browser_id in $browser_ids; do - if ! docker stop "$browser_id"; then - docker rm --force "$browser_id" - fi - done - - browser_ids="$(list_running_browsers)" || exit 1 - test -z "$browser_ids" - docker compose stop creator-hub docker-gateway - - umask 077 - PRE_ROLLBACK_BACKUP="$(pwd)/creatorhub-pre-rollback-$(date +%Y%m%d-%H%M%S).dump" - docker compose exec -T postgres \ - pg_dump -U creatorhub -d creatorhub --format=custom \ - > "$PRE_ROLLBACK_BACKUP" - test -s "$PRE_ROLLBACK_BACKUP" - docker compose exec -T postgres pg_restore --list \ - < "$PRE_ROLLBACK_BACKUP" >/dev/null - - test -r "$BACKUP_FILE" - test -s "$BACKUP_FILE" - git cat-file -e "${RESTORE_REV}^{commit}" - docker compose exec -T postgres pg_restore --list \ - < "$BACKUP_FILE" >/dev/null - - docker compose exec -T postgres \ - dropdb --if-exists --force -U creatorhub creatorhub - docker compose exec -T postgres createdb -U creatorhub creatorhub - docker compose exec -T postgres \ - pg_restore -U creatorhub -d creatorhub \ - --exit-on-error --no-owner --no-privileges \ - < "$BACKUP_FILE" - docker compose exec -T postgres \ - psql -U creatorhub -d creatorhub -v ON_ERROR_STOP=1 -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 32;' \ - | grep -qx 1 - docker compose exec -T postgres \ - psql -U creatorhub -d creatorhub -v ON_ERROR_STOP=1 -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 35;' \ - | grep -qx 1 - - git switch --detach "$RESTORE_REV" - if ! docker compose up --detach --build; then - docker compose stop creator-hub docker-gateway || true - exit 1 - fi -) +systemctl --user --no-pager --full status creatorhub-browser-gateway.service +journalctl --user -u creatorhub-browser-gateway.service --since=-30m +find ~/.local/state/creatorhub/browser-gateway -maxdepth 3 -type f -name runtime.json -print +curl --fail --silent --show-error http://127.0.0.1:8081/v1/browsers \ + -H "Authorization: Bearer ${GATEWAY_TOKEN}" ``` -任何命令失败时 subshell 立即退出；若失败发生在主库 `dropdb` 之后，应用保持停止。`docker compose up` 自身失败时也会显式停回应用服务。成功后重新执行“部署验证”。不要直接删除数据卷或手工改写迁移记录。 - -## 停止与数据保留 - -`docker compose stop/down` 不管理网关动态创建的浏览器容器；直接执行会留下运行中的浏览器和活动会话。先严格按固定容器名前缀及两个管理标签筛选并停止；只有单个容器停止失败或超时时才强制删除。命令不带 `--volumes`，因此 Profile 卷仍会保留： - -```bash -set -Eeuo pipefail - -list_running_browsers() { - docker ps --quiet \ - --filter 'name=^creatorhub-browser-' \ - --filter 'label=io.creatorhub.managed=true' \ - --filter 'label=io.creatorhub.runtime-id' -} - -browser_ids="$(list_running_browsers)" || exit 1 -for browser_id in $browser_ids; do - if ! docker stop "$browser_id"; then - docker rm --force "$browser_id" - fi -done - -browser_ids="$(list_running_browsers)" || exit 1 -test -z "$browser_ids" - -docker compose stop -``` - -恢复服务使用 `docker compose up --detach`。需要删除服务容器和网络但保留数据时使用： - -```bash -docker compose down -``` - -不要执行 `docker compose down --volumes`：它会删除 PostgreSQL 数据卷。浏览器 Profile 卷不属于 Compose 声明卷，删除浏览器容器或执行 `docker compose down` 时仍会保留；可用以下命令核对： - -```bash -docker volume ls --filter name=creatorhub-profile- -``` - -`docker.sock` 即使以只读文件方式挂载，也仍允许 Docker API 写操作，等价于宿主机 root 权限。完整安全边界、API 契约和失败语义见[《浏览器容器控制面》](architecture/container-control.md)。 +强杀或网络中断后，不要重复发送真实平台写操作。先查询 control-plane 的运行、结果未知和 cleanup 状态，再按 generation 执行显式恢复或清理。 diff --git a/docs/deployment_stop_test.sh b/docs/deployment_stop_test.sh index 0f6de2a..8190733 100755 --- a/docs/deployment_stop_test.sh +++ b/docs/deployment_stop_test.sh @@ -3,111 +3,18 @@ set -Eeuo pipefail cd "$(dirname "$0")/.." -extract_block() { - awk -v wanted="$1" ' - /^```bash$/ { - in_code = 1 - strict = "" - wrapper = "" - next - } - in_code && /^\($/ { wrapper = $0 } - in_code && /^[[:space:]]*set -Eeuo pipefail$/ { strict = $0 } - /^[[:space:]]*list_running_browsers\(\) \{/ { - count++ - capture = count == wanted - if (capture) { - if (wrapper) print wrapper - print strict - } - } - capture { print } - capture && /^[[:space:]]*docker compose stop/ { - capture = 0 - if (!wrapper) exit - closing = 1 - } - closing && /^\)$/ { print; exit } - /^```$/ { in_code = 0 } - ' docs/deployment.md -} +# Deployment documentation must describe only the host-native browser lifecycle. +grep -Fq 'systemctl --user' docs/deployment.md +grep -Fq 'creatorhub-browser-gateway.service' docs/deployment.md +grep -Fq 'BROWSER_PROFILE_ROOT' docs/deployment.md +grep -Fq 'Compose 不再创建 gateway 服务' docs/deployment.md +! grep -Fq 'docker-gateway' docs/deployment.md +! grep -Fq 'DOCKER_SOCKET' docs/deployment.md +! grep -Fq 'docker.sock' docs/deployment.md -docker() { - printf '%s\n' "$*" >>"$calls" - case "$1" in - ps) - [[ "$*" == 'ps --quiet --filter name=^creatorhub-browser- --filter label=io.creatorhub.managed=true --filter label=io.creatorhub.runtime-id' ]] || return 64 - query_count=0 - [[ ! -e "$state/query-count" ]] || read -r query_count <"$state/query-count" - ((query_count += 1)) - printf '%s\n' "$query_count" >"$state/query-count" - [[ "$mode:$query_count" != query_failure:1 && "$mode:$query_count" != recheck_failure:2 ]] || return 42 - [[ -e "$state/stopped" || -e "$state/removed" ]] || printf '%s\n' browser-1 - ;; - stop) - case "$mode" in - stop_success) : >"$state/stopped" ;; - stop_failure) return 1 ;; - stop_timeout) return 124 ;; - esac - ;; - rm) - [[ "$*" == 'rm --force browser-1' ]] || return 64 - : >"$state/removed" - ;; - compose) - [[ "$*" == 'compose stop' || "$*" == 'compose stop creator-hub docker-gateway' ]] || return 64 - ;; - esac -} -export -f docker +# The install unit must remain a user service and must not run the gateway as root. +grep -Fq 'EnvironmentFile=%h/.config/creatorhub/browser-gateway.env' deploy/creatorhub-browser-gateway.service.in +grep -Fq 'cmd.browser_gateway.gateway' deploy/creatorhub-browser-gateway.service.in +! grep -Fq 'User=root' deploy/creatorhub-browser-gateway.service.in -work="$(mktemp -d)" -trap 'rm -rf "$work"' EXIT -query='ps --quiet --filter name=^creatorhub-browser- --filter label=io.creatorhub.managed=true --filter label=io.creatorhub.runtime-id' -scenario_count=0 - -for block_number in 1 2; do - block="$(extract_block "$block_number")" - bash -n <<<"$block" - [[ "$block" == *'set -Eeuo pipefail'* ]] - if [[ $block_number -eq 1 ]]; then - [[ "${block%%$'\n'*}" == '(' && "${block##*$'\n'}" == ')' ]] - compose='compose stop creator-hub docker-gateway' - else - [[ "${block%%$'\n'*}" == 'set -Eeuo pipefail' ]] - compose='compose stop' - fi - - for mode in stop_success stop_failure stop_timeout query_failure recheck_failure; do - state="$work/$block_number-$mode" - calls="$state/calls" - mkdir "$state" - export mode state calls - - status=0 - bash -c "$block" || status=$? - - case "$mode" in - stop_success) - [[ $status -eq 0 ]] - diff -u <(printf '%s\n' "$query" 'stop browser-1' "$query" "$compose") "$calls" - ;; - stop_failure | stop_timeout) - [[ $status -eq 0 ]] - diff -u <(printf '%s\n' "$query" 'stop browser-1' 'rm --force browser-1' "$query" "$compose") "$calls" - ;; - query_failure) - [[ $status -eq 1 ]] - diff -u <(printf '%s\n' "$query") "$calls" - ;; - recheck_failure) - [[ $status -eq 1 ]] - diff -u <(printf '%s\n' "$query" 'stop browser-1' "$query") "$calls" - ;; - esac - scenario_count=$((scenario_count + 1)) - done -done - -[[ $scenario_count -eq 10 ]] +echo 'native browser deployment documentation checks passed' diff --git a/docs/e2e-test-plan.md b/docs/e2e-test-plan.md index f877457..537d298 100644 --- a/docs/e2e-test-plan.md +++ b/docs/e2e-test-plan.md @@ -1,10 +1,10 @@ -# 当前系统全量功能测试计划（不含小红书） +# 当前系统全量功能测试计划（单节点 native browser；抖音主流程与小红书读取） ## 1. 基线、目标与执行边界 -- 源码基线：`main`，HEAD `4d1dd37438b7c7001f572695feb5e1b56b9fc78d`。编写前父会话已同步远程且工作树干净。本文件是测试计划，**所有用例初始状态均为“未执行”**，不是测试报告。 -- 目标：验证当前页面、公开 API、Docker/浏览器 gateway、持久数据及实际抖音链路；以当前源码为准，不将 `docs/plan01.md` 的未实现目标列为既有功能。 -- 本次只编写并静态核验文档；不运行测试、不启动或停止服务、不改配置、不访问外部平台、不提交或推送。下文命令和故障操作均供**后续获准执行**，本次未执行。 +- 源码基线：已同步的 `main` / `origin/main`，HEAD `7e3808cf4ce453b3583079680a3b59ca2ed64ad4`。本文件是测试计划；已执行项目和真实平台阻塞项目以 [验证记录](evidence/native-browser-verification-2026-09-18.md) 为准。 +- 目标：验证当前页面、公开 API、host-native browser gateway、持久数据及实际抖音链路；以当前源码为准，不将 `docs/plan01.md` 的未实现目标列为既有功能。 +- 自动检查和本地 native gateway smoke 已执行并记录；真实平台、代理、LAN、Profile 占用和破坏性资源用例仍须后续由授权操作者逐项执行。下文未执行项目不得以测试替身或内部函数补齐。 - 排除：小红书全部业务测试；未实现的自动密码登录、真实文章/视频发布、自动聊天、附件发送、运营工作台新建私信会话、永久删除账号/环境/出口、RBAC/多租户及性能压测。公众号/快手仅覆盖现有通用账号建档，不测试其采集或平台动作。 - 不排除：已有代码但尚无真实验收证据的抖音读取、通知监听、点赞、回复、关注、私信、素材处理。外部条件不具备时标“阻塞”，不得改为“不适用”以提高通过率。 @@ -22,7 +22,7 @@ ### 1.2 核对来源与纠偏 -主要来源：`web/src/main.jsx`、`web/src/dataProvider.js`、`web/src/*Page.jsx`、`web/src/Layout.jsx`、`web/src/lib/ui.jsx`；`cmd/control-plane/{main,hub,phasea,creator,creator_events,creator_material}.go`；`internal/creator/{models,accounts,actions,collection,content,metrics,rules,settings,bailian}.go`；`internal/hub/{environment,fingerprint}.go`；`internal/douyin/creator_collector.go`；`cmd/docker_gateway/{gateway,douyin,proxy,docker_client}.py`；`compose.yaml`、`compose.dev.yaml`、`Dockerfile`、`docker/browser-wrapper/`。 +主要来源：`web/src/main.jsx`、`web/src/dataProvider.js`、`web/src/*Page.jsx`、`web/src/Layout.jsx`、`web/src/lib/ui.jsx`；`cmd/control-plane/{main,hub,phasea,creator,creator_events,creator_material}.go`；`internal/creator/{models,accounts,actions,collection,content,metrics,rules,settings,bailian}.go`；`internal/hub/{environment,fingerprint}.go`；`internal/douyin/creator_collector.go`；`cmd/browser_gateway/{gateway,douyin,proxy,runtime}.py`；`compose.yaml`、`compose.dev.yaml`、`Dockerfile`、`deploy/`。 其他运行文档只作操作参考：其中“代理仅凭据引用”“仿写自动生成”“抖音仅首批读取”等旧表述不能覆盖当前路由和源码。当前出口表单/API 接收用户名与密码，列表/详情会明文展示；证据必须脱敏。仿写确认只保存要求与确认时间，**没有 AI 生成调用或按钮**。`decodeCreator` 直接读 JSON body，不依据 Content-Type 选择解码，故保存仿写缺 header 不等于已经证实失败。 @@ -30,14 +30,14 @@ ### 2.1 环境 -1. 使用专用、可销毁的 Linux Docker 测试主机；禁止与现有业务共用固定 `creatorhub_control` 网络、浏览器 alias 或数据卷。只修改 Compose project 名不足以隔离固定网络名。 -2. 由执行负责人按 `docs/deployment.md` 准备一次完整 Compose 部署，记录控制面/gateway 构建提交、镜像 digest、Docker/Compose/浏览器版本、操作系统、数据库版本、时区、测试起止 UTC；另用独立时段验证 `compose.dev.yaml` 的开发入口。不得一边测试一边自动升级源码。 -3. 准备 `curl`、`jq`、Docker CLI、浏览器 DevTools；媒体核验使用当前镜像内的 `ffprobe`/`ffmpeg`。真实浏览器镜像使用 `docker/browser-wrapper/README.md` 中已登记来源与 digest 的包装镜像，准备两个可用版本和一个不存在引用。 -4. 配置必须来自测试凭据保管渠道：`CONTROL_PLANE_USERNAME`、`CONTROL_PLANE_PASSWORD`、32 字节 Base64 `CREATORHUB_CREDENTIAL_MASTER_KEY`、`DOCKER_GID`、`GATEWAY_TOKEN`。不得将实际密码、Cookie、token、证件号贴入文档/缺陷/日志截图。 -5. 记录控制面 `LISTEN_ADDR`、`WEB_DIR`、`DATABASE_URL`、`CREATORHUB_CREDENTIAL_STORE_DIR`、`LOG_LEVEL`；gateway 的 `LISTEN_ADDR`、`DOCKER_SOCKET`、`BROWSER_NETWORK`。媒体目录 `CREATOR_MEDIA_DIR` 可写；如测转写，`CREATOR_TRANSCRIPTION_BIN` 必须是已安装的真实供应商适配程序，接受音轨路径参数、stdout 输出正文；不能用 `echo` 冒充真实转写。 +1. 使用专用、可销毁的 Linux 主机；gateway 以非 root `systemctl --user` 服务运行。禁止与现有业务共用固定 browser alias、Profile 目录或数据库 schema；不得删除现有 Docker PostgreSQL 之外的业务服务。 +2. 由执行负责人按 `docs/deployment.md` 准备控制面/PostgreSQL Compose 与 host-native gateway，记录构建提交、浏览器版本/path、systemd/Xvfb/操作系统、数据库版本、时区、测试起止 UTC；另用独立时段验证 `compose.dev.yaml` 的开发入口。不得一边测试一边自动升级源码。 +3. 准备 `curl`、`jq`、必要时 Docker CLI（仅用于 PostgreSQL Compose）、`systemctl --user`、浏览器 DevTools；媒体核验使用宿主机 `ffprobe`/`ffmpeg`。准备两个已安装且可执行的 browser version/path 和一个不存在或不可执行的 path。 +4. 配置必须来自测试凭据保管渠道：`CONTROL_PLANE_USERNAME`、`CONTROL_PLANE_PASSWORD`、32 字节 Base64 `CREATORHUB_CREDENTIAL_MASTER_KEY`、`GATEWAY_TOKEN`。gateway 还需合法 `BROWSER_STATE_DIR`、`BROWSER_PROFILE_ROOT`、`NODE_ID` 和 browser version/path。不得将实际密码、Cookie、token、证件号贴入文档/缺陷/日志截图。 +5. 记录控制面 `LISTEN_ADDR`、`WEB_DIR`、`DATABASE_URL`、`CREATORHUB_CREDENTIAL_STORE_DIR`、`LOG_LEVEL`；gateway 的 `LISTEN_ADDR`、`BROWSER_STATE_DIR`、`BROWSER_PROFILE_ROOT`、`NODE_ID`、browser version/path。媒体目录 `CREATOR_MEDIA_DIR` 可写；如测转写，`CREATOR_TRANSCRIPTION_BIN` 必须是已安装的真实供应商适配程序，接受音轨路径参数、stdout 输出正文；不能用 `echo` 冒充真实转写。 6. 如测 AI：准备已审批的 Bailian 账号、配额、`BAILIAN_API_KEY`，可选 `BAILIAN_BASE_URL`；页面 provider 填 `bailian`，model 填实际批准模型。勾选“已完成审批”并不验证密钥。转写页面元数据也不替代可执行程序配置。 7. 对所有代理协议 http/https/socks4/socks5 分别准备真实可控测试端点；至少一个无认证端点、一个带认证端点、两个不同公网 IP、一个不可达端点，保存实际地区与供应商定义。没有某协议服务时该行阻塞，不能用另一协议代替。 -8. 记录观察窗口：普通 API 30 秒；创建/升级含拉镜像最多按控制面 11 分钟预算；运行租约调和每 20 秒、采集调度每 30 秒、监听绑定调和每 10 秒；事件长等待至多 25 秒，重连退避 1–30 秒。超时记录请求/日志，不自动重复写。 +8. 记录观察窗口：普通 API 30 秒；创建/升级含拉browser version/path最多按控制面 11 分钟预算；运行租约调和每 20 秒、采集调度每 30 秒、监听绑定调和每 10 秒；事件长等待至多 25 秒，重连退避 1–30 秒。超时记录请求/日志，不自动重复写。 ### 2.2 账号、目标与数据 @@ -50,7 +50,7 @@ | C | 已同意的竞品测试账号，必须与采集执行账号不同；有可人工核对的近期/历史作品、分页及一级评论 | | W / K | 从真实采集得到的内部作品 ID / 评论 ID；另存平台 work_key/comment_key、作者 UID、原页面链接和时间证据 | | M1 / M2 / M3 | 有明确授权的带语音视频、无音轨视频、音轨无语音视频；保存原片时长与参考文稿，不把网页 HTML 当媒体 | -| G / V1 / V2 | 测试网关名、两个镜像版本；升级使用第二个已启用版本 | +| G / V1 / V2 | 测试网关名、两个已安装 browser version/path；升级使用第二个已启用版本 | | E0 / E1 / P1 / P2 | 直连环境、代理环境、两个已健康出口；每个环境固定不同账号 | | T / D / Q / AT | 本轮生成的任务、草稿、确认、Attempt ID；不得拿旧轮次同名结果充数 | @@ -90,13 +90,13 @@ curl --silent --show-error --config "$API_AUTH_FILE" \ | --- | --- | --- | | F1 浏览器请求失败 | DevTools → Network request blocking 添加准确 URL（如 `*/api/creator/rules*`），启用后刷新对应页；网络断开用 Network Offline | 记录报错、写请求数；取消规则/恢复 Online，只刷新读取；不能把页面断网当后端平台失败 | | F2 慢响应/页面竞争 | DevTools Network 选 Slow 3G、勾 Preserve log，快速切账号/素材/会话并观察完成顺序 | 恢复 No throttling；若没有形成倒序响应，记该竞争子项未触发，不宣称已验证 | -| F3 gateway 故障 | 专用主机执行 `docker compose stop docker-gateway`；持久化错误场景可在生命周期操作已发出时停止，记时刻 | 读取控制面错误/审计/cleanup 状态；`docker compose start docker-gateway`，先查询环境与 Docker 实物，再按提示显式恢复。不得重发真实平台动作 | +| F3 gateway 故障 | 专用主机执行 `systemctl --user stop creatorhub-browser-gateway.service`；持久化错误场景可在生命周期操作已发出时停止，记时刻 | 读取控制面错误/审计/cleanup 状态；`systemctl --user start creatorhub-browser-gateway.service`，先查询 runtime 实物和 generation，再按提示显式恢复。不得重发真实平台动作 | | F4 控制面中断 | 在目标 operation 为 processing 时执行 `docker compose stop creator-hub`；崩溃专项才使用 `docker compose kill -s SIGKILL creator-hub` | `docker compose start creator-hub`，记录自动重启竞争；只读核对旧操作、租约及平台结果；不重复 execute。优雅与强杀分别记录 | | F5 PostgreSQL 故障 | `docker compose stop postgres`，发一次指定读取/保存；记录是否已发出保存请求 | `docker compose start postgres`，待 `pg_isready` 后重读；不盲目补写。数据库不可用使多个后台报错是预期故障影响 | | F6 实际代理断连 | 对专用测试代理容器 `docker stop "$TEST_PROXY_CONTAINER"`（先核验其归属）；或使用供应商已批准的暂停端点动作 | 显式检测出口/访问测试目标，确认未回退直连；`docker start "$TEST_PROXY_CONTAINER"` 或恢复供应商端点后重新检测健康 | -| F7 实际浏览器丢失/断线 | 先核验 `docker inspect creatorhub-browser-$ALIAS` 的 RUN 对应别名/managed 标签，再 `docker stop`；仅丢失容器专项用 `docker rm -f`，不带 `-v` | 观察后台调和及动作审计；控制面重新启动环境，不直接复用旧 generation；确认 Profile 仍在 | -| F8 媒体本地故障 | 专用测试部署中，读取已选素材 `video_reference` 并核验属于 RUN；在 control-plane 停止后由卷维护人员将该文件重命名为 `.e2e-backup`，再启动并 process；或对下一条全新素材临时撤掉真实转写程序配置 | 记录原文件/配置；停止控制面，按原路径恢复备份/原 `CREATOR_TRANSCRIPTION_BIN` 并重建该服务；清除临时文件。禁止修改 step 状态冒充成功；不存在可安全操作的独立卷则阻塞 | -| F9 服务启动配置 | 不动运行中服务，用 `docker compose run --rm --no-deps -e NAME=VALUE creator-hub` 启动一次性命令；仅对预校验会失败的变量使用。gateway 以 `docker-gateway` 服务同法 | 校验失败应无新监听、无新文件/数据库/Docker 副作用；一次性容器退出即恢复。不将有效未知配置指向真实环境 | +| F7 实际浏览器丢失/断线 | 先核验 `systemctl --user` 中 alias 对应的 browser/Xvfb unit 和 runtime metadata，再只停止或强杀该 runtime 的 unit；不得删除 Profile 目录 | 观察后台调和及动作审计；控制面重新启动环境，不直接复用旧 generation；确认 Profile 仍在 | +| F8 媒体本地故障 | 专用测试部署中，读取已选素材 `video_reference` 并核验属于 RUN；停止 control-plane 后由负责人将该文件重命名为 `.e2e-backup`，再启动并 process；或对下一条全新素材临时撤掉真实转写程序配置 | 记录原文件/配置；停止控制面，按原路径恢复备份/原 `CREATOR_TRANSCRIPTION_BIN` 并重建该服务；清除临时文件。禁止修改 step 状态冒充成功；不存在可安全操作的独立目录则阻塞 | +| F9 服务启动配置 | 不动运行中服务，用 control-plane 的一次性裸启动或 `systemd-run --user --wait` gateway 进程验证预校验会失败的变量；仅对预校验会失败的变量使用 | 校验失败应无新监听、无新 Profile/runtime/数据库副作用；进程退出即恢复。不将有效未知配置指向真实环境 | | F10 UI 异常响应 | 浏览器 Local Overrides 对指定 GET 保存脱敏副本，仅改待测字段，例如任务 `hold_reason` 与 `allowed_action`；记录原响应与改动 | 仅作 UI 注入，不算服务端 E2E；禁止用该响应点击实际执行/恢复写按钮，观察禁用状态即可；关闭 Overrides 并重新读取 | F8 的可执行操作补充（只供后续获准的专用部署；`SOURCE_PATH`必须由本轮material.video_reference定位到媒体目录下的普通文件，人工核对不是目录/其他账号文件；操作时无其他素材处理）： @@ -128,7 +128,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random | --- | --- | --- | | 1 基础部署与配置 | `/healthz`，Compose、Cobra、配置、持久化、日志、静态资源 | SYS-01～SYS-06 | | 2 登录/导航/公共组件 | `/#/login`、`/#/`、Layout、全部页面共用控件 | UI-01～UI-07 | -| 3 网关与镜像 | `/#/gateways`、`/#/browser-images` | GW-01～GW-05、IMG-01～IMG-04 | +| 3 网关与浏览器版本 | `/#/gateways`、`/#/browser-versions` | GW-01～GW-05、VER-01～VER-04 | | 4 出口 | `/#/network-exits`、`/#/network-exits/:id` | NET-01～NET-06 | | 5 账号/环境 | `/#/accounts`、`/#/accounts/:id`、`/#/browsers`、`/#/browsers/new`、`/#/browsers/:id` | AC-01～AC-07、ENV-01～ENV-10 | | 6 gateway 领域能力 | `/v1/browsers`、代际/代理/CDP/抖音受限路由 | API 附录 G、GATE-01～GATE-06 | @@ -146,8 +146,8 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 依赖执行顺序（从干净环境开始）： -1. 完成 SYS-01～03、SYS-06 及 UI 登录/导航，创建 GW-01 网关、IMG-01 镜像、NET-01 出口、AC-01 账号与 ENV 创建/启动所需记录；公共组件的有数据/错误状态随对应模块补测。 -2. 取得环境引用后执行 GW-02 的引用冲突分支及其他资源关联检查；删除/停用/令牌失效等分支使用独立资源，或延后至这些资源的所有依赖用例完成。不得提前删掉后续真实测试使用的网关、镜像或出口。 +1. 完成 SYS-01～03、SYS-06 及 UI 登录/导航，创建 GW-01 网关、VER-01 browser version/path、NET-01 出口、AC-01 账号与 ENV 创建/启动所需记录；公共组件的有数据/错误状态随对应模块补测。 +2. 取得环境引用后执行 GW-02 的引用冲突分支及其他资源关联检查；删除/停用/令牌失效等分支使用独立资源，或延后至这些资源的所有依赖用例完成。不得提前删掉后续真实测试使用的网关、browser version/path或出口。 3. 完成 AC-06 登录检查及非破坏性环境/gateway 检查；创建 Phase A 草稿并完成任务流程；配置采集设置与账号资料/关系。策略可保存，但自动写入保持关闭。 4. 完成竞品采集，再以真实作品/评论执行素材、规则/线索和人工操作。完成前述数据创建后，才执行 SYS-04 持久化检查：账号、环境、草稿、规则、素材均实际存在；保存各项证据后重启。不能为满足前置而改库造数据。 5. 人工动作留证后执行 EVT 自动响应，再立即停用策略；审计用例在所需动作/任务记录产生后执行。API-only 导入可在合法账号/来源就绪后独立执行，但修改全局指标设置的 API-02 应在 REAL 指标观察完成后执行并恢复设置。 @@ -165,35 +165,35 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：专用干净主机，2.1 配置齐备；允许后续部署。 - 操作： - 1. 按部署说明启动本轮三个服务并记录 ps、镜像 digest + 1. 按部署说明启动 control-plane、native gateway 和前端，并记录进程、browser version/path digest 2. GET 控制面和 gateway 的 /healthz；GET 带认证的 /api/gateways 3. 打开 BASE/，再打开实际 JS/CSS 资源；刷新 /#/accounts 及 /#/browsers/new - 预期：两个健康接口 204；数据库健康、服务持续运行；API 返回 JSON，页面/资源可加载。healthz 不等于数据库、平台或全部依赖已通过。 #### SYS-02 控制面配置与命令校验 [API/FAULT] -- 前置：SYS-01 镜像；采用 F9，禁止改运行配置。 +- 前置：SYS-01 browser version/path；采用 F9，禁止改运行配置。 - 操作： 1. 分别单项设置 LISTEN_ADDR=:0、LOG_LEVEL=bogus、WEB_DIR仅三个空格、DATABASE_URL=`http://example.invalid/db` 2. 分别设置用户名仅三个空格或含冒号、密码不足6字节、主密钥非Base64或解码长度不为32、凭据目录为相对路径、BAILIAN_BASE_URL 含 userinfo 3. 一次性容器运行 control-plane 的 --help；另传不支持的位置参数 -- 预期：非法输入非零退出、日志清楚指出字段且不泄露值；配置校验先于数据库/凭据/Docker副作用。只有当前最小根命令，无编造子命令。help 不启动服务；未支持参数拒绝。Viper可能将空环境变量视为未设置并采用默认值，因此空白字符串校验与“未设置/空变量默认值”另行记录，不能混为必然启动失败。 +- 预期：非法输入非零退出、日志清楚指出字段且不泄露值；配置校验先于数据库/凭据/runtime/systemd 副作用。只有当前最小根命令，无编造子命令。help 不启动服务；未支持参数拒绝。Viper可能将空环境变量视为未设置并采用默认值，因此空白字符串校验与“未设置/空变量默认值”另行记录，不能混为必然启动失败。 #### SYS-03 gateway 配置与依赖错误 [API/FAULT] -- 前置：SYS-01 镜像；F9 独立命令。 +- 前置：SYS-01 browser version/path；F9 独立命令。 - 操作： - 1. 单项试 LISTEN_ADDR=:0、GATEWAY_TOKEN=short、DOCKER_SOCKET=空、BROWSER_NETWORK=creatorhub_control 或非法名称 - 2. 恢复有效配置；在专用测试实例令 DOCKER_SOCKET 指向不存在路径，GET /v1/browsers + 1. 单项试 LISTEN_ADDR=:0、GATEWAY_TOKEN=short、BROWSER_STATE_DIR=空、BROWSER_PROFILE_ROOT=非 owner-only 或 browser path 不存在 + 2. 恢复有效配置；在专用测试实例令 browser path 指向不存在路径，GET /v1/browsers 3. 结束一次性实例并回读原服务列表 -- 预期：非法配置明确失败；缺 Docker socket 的真实领域请求报错、不返回假空成功；原环境不受影响。健康接口能返回不证明 Docker socket 可用。 +- 预期：非法配置明确失败；缺 browser path、Profile 权限或 systemd/Xvfb 条件的真实领域请求报错、不返回假空成功；原环境不受影响。健康接口能返回不证明 runtime 可用。 #### SYS-04 持久化与重启 [UI/FAULT] - 前置：已有账号/资料密码、环境、草稿、规则、素材至少各一条；主密钥备份。 - 操作： 1. 记录各 ID、版本、非秘密字段和 Profile 内自建 RUN 书签 - 2. 正常重启 creator-hub、docker-gateway，保留 PostgreSQL/凭据/素材/Profile卷 + 2. 正常重启 control-plane、native gateway，保留 PostgreSQL/凭据/素材/Profile目录 3. 重新登录并逐页读取；浏览器启动后核对书签；用原真实会话执行一次已授权只读身份检查 - 预期：记录、版本链及文件不丢；原主密钥可解密既有凭据。启动不得无限复制记录；事件恢复另按 EVT-03。无主密钥不做破坏性换密钥实验。 @@ -211,7 +211,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：可切换独立测试时段；本轮资源已登记。 - 操作： 1. 正常停止 gateway，记录耗时/退出码/日志并检查无 SIGKILL，随后恢复 - 2. 核对只有 gateway 挂载 Docker socket，浏览器由其创建且命名与标签可追踪 + 2. 核对只有 host-native gateway 管理 browser/Xvfb unit；runtime metadata、owner、node 和 generation 可追踪 3. 完整部署结束后按 compose.dev.yaml 单独启动依赖与开发进程；打开5173入口并观察 /api 到8082、网关8081映射 - 预期：正常关停在45秒宽限内完成或明确失败；代理/事件订阅不悬挂。生产8080与开发5173/8082入口均可用，错误网关地址不能假成功；两种模式不共用活动测试实例。 @@ -255,7 +255,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random #### UI-05 弹窗取消、忙碌和重复点击 [UI/FAULT] -- 前置：镜像或账号普通创建弹窗、出口停用确认框；F2。 +- 前置：browser version/path或账号普通创建弹窗、出口停用确认框；F2。 - 操作： 1. 普通创建输入后分别用取消、关闭图标、Escape、遮罩退出再打开 2. 确认框先取消一次，检查无写请求 @@ -266,7 +266,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：干净数据时留存各页空态；F1可用。 - 操作： - 1. 分别阻断 accounts、browsers、gateways、browser-images、network-exits、creator各列表、drafts、audit、attempt详情 GET + 1. 分别阻断 accounts、browsers、gateways、browser-versions、network-exits、creator各列表、drafts、audit、attempt详情 GET 2. 逐页检查加载结束后错误区、操作禁用及是否显示假空态 3. 取消阻断，只有存在重试按钮的页面点击重试，其余重新进入页面 - 预期：失败与空数据可区分，不能隐藏依赖错误；没有重试按钮不写“重试成功”。特别记录账号草稿、环境创建依赖、规则列表异常呈现；不得自动重做先前写请求。 @@ -280,7 +280,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 3. 复制可显示的CDP和网关token到本地私密临时区，随后撤销剪贴板权限再试 - 预期：核心内容可读、禁用可辨；复制内容与来源一致但不要求当前不存在的成功提示。焦点圈定/自动聚焦、aria-describedby缺口如实报，不把缺失能力当支持；截图无秘密。 -### 5.3 网关和镜像 +### 5.3 网关和browser version/path #### GW-01 注册、令牌与列表 [UI] @@ -316,7 +316,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 1. 点击删除先取消，确认列表仍在且无DELETE 2. 再打开确认删除并确认一次，观察Network、Console与页面 3. 刷新列表读取实际记录 -- 预期：预期删除应发送领域DELETE并移除记录；当前 provider.deleteOne 仅支持browser-images，预计阻断/报错，证实则记失败。不得使用后端删除成功掩盖UI失败；后续清理另用GW-05。 +- 预期：预期删除应发送领域DELETE并移除记录；browser version 被环境引用时必须拒绝删除。不得使用后端删除成功掩盖UI失败；runtime 回收另用 GW-05。 #### GW-05 网关删除 API 与引用冲突 [API] @@ -327,16 +327,16 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 3. DELETE仍被环境引用的G，回读G和环境 - 预期：未引用项成功删除；不存在返回明确结果、不影响其他项；引用中的网关409拒绝且环境引用不丢。此用例独立于UI删除缺陷。 -#### IMG-01 创建镜像版本与字段边界 [UI/API] +#### VER-01 创建browser version/path与字段边界 [UI/API] -- 前置：两个可拉取的已登记镜像摘要。 +- 前置：两个可拉取的已登记browser version/path。 - 操作： 1. 创建启用V1，检查版本/引用/备注；再创建V2 2. 重复V1；试非法版本首字符、65字符版本、空引用、非法引用字符、超长备注 3. F1阻断一次创建，恢复后只读查重 -- 预期：成功持久化，重复/非法值不覆盖已有版本；失败不清表单或误报成功。仅登记版本不证明镜像可运行。 +- 预期：成功持久化，重复/非法值不覆盖已有版本；失败不清表单或误报成功。仅登记版本不证明browser version/path可运行。 -#### IMG-02 编辑、禁用和跨页候选 [UI] +#### VER-02 编辑、禁用和跨页候选 [UI] - 前置：V1/V2已登记，V1被E1引用。 - 操作： @@ -345,7 +345,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 3. 再启用V2；编辑时清空引用点击保存 - 预期：停用版本不出现在新建/升级候选，已有环境不静默换版；空引用应阻止请求或被后端拒绝；错误在编辑框可见。 -#### IMG-03 删除引用与取消 [UI] +#### VER-03 删除引用与取消 [UI] - 前置：V1被引用，另有未引用临时版本。 - 操作： @@ -354,12 +354,12 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 3. 恢复临时登记项以供下一用例或登记已删除 - 预期：未引用删除成功；取消无DELETE；引用中拒绝且保留信息，不移除V1或破坏E1。 -#### IMG-04 镜像不可拉取与动作忙碌 [UI/FAULT] +#### VER-04 browser version/path不可拉取与动作忙碌 [UI/FAULT] - 前置：临时版本指向不存在的仓库引用，独立暂停账号。 - 操作： 1. 新建环境选该版本并提交，记录拉取过程、按钮禁用和超时 - 2. 查看环境列表、审计和Docker实际容器，不假定登记已回滚 + 2. 查看环境列表、审计和gateway runtime 实物，不假定登记已回滚 3. 修改该临时版本为正确测试digest，核验残留后显式恢复环境一次 - 预期：失败可见，有requested/finished证据；没有假running；已保存环境/binding按实物核验恢复，不能重复生成绑定。 @@ -491,16 +491,16 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：已授权暂停账号、G、启用V1。 - 操作： 1. 打开创建环境填写名称、alias、seed1000、账号/G/V1，出口选直连，提交 - 2. 读取列表/详情及Docker实例状态 + 2. 读取列表/详情及gateway runtime状态 3. 去账号详情恢复账号，回环境列表显式启动，复制CDP并按已批准连接方式打开浏览器 -- 预期：创建先停止态，固定绑定正确；恢复账号和启动分离；运行有实际容器/租约，直连无出口也可用。CDP展示/复制不代表公网端口已发布。 +- 预期：创建先停止态，固定绑定正确；恢复账号和启动分离；运行有实际 browser/Xvfb unit 与 lease，直连无出口也可用。CDP展示/复制不代表公网端口已发布。 #### ENV-02 代理环境和高级指纹 [UI/REAL] - 前置：另一暂停账号、healthy P1、G/V1。 - 操作： 1. 创建E1选择P1；高级字段依次填windows/linux/macos中的一个、Chrome/Edge/Opera/Vivaldi中的一个、版本、CPU、语言、时区与伪装禁用项 - 2. 恢复账号并启动，Docker inspect核对argv、Profile卷和无凭据代理地址 + 2. 恢复账号并启动，Docker inspect核对argv、Profile目录和无凭据代理地址 3. 浏览器查看navigator语言/CPU等及真实公网IP；停启后重新核对 - 预期：选择参数保留且argv独立传递；实际IP匹配P1，凭据不出现在命令/标签。OS/brand其余合法选项分轮独立记录，不要求同一环境改指纹入口。 @@ -508,7 +508,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：空依赖与齐备依赖分时段；附录D。 - 操作： - 1. 缺网关/镜像/可绑账号/健康出口时打开表单查看准备提示；F1阻断依赖GET后再开 + 1. 缺网关/browser version/path/可绑账号/健康出口时打开表单查看准备提示；F1阻断依赖GET后再开 2. 分别试alias非法/超32、seed0/2147483648/小数、CPU负数/129、非法语言/时区、指纹内proxy_server 3. 修正后创建；并用API重复同alias不同账号或已绑定账号的新alias - 预期：默认只选合法依赖；错误读取不伪装业务缺项；字段越界后端拒绝。相同资源配置重复按幂等核验，冲突配置409；一账号不出现两个活动绑定。 @@ -554,9 +554,9 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：仅E1测试账号；两个API客户端，不发平台业务动作。 - 操作： 1. 两端同时POST start，保存每个响应与operation_id - 2. 在一次启动/升级进行时另端pause账号或disable目标镜像 - 3. 读取账号、镜像、environment及Docker实物，按最终有效配置恢复 -- 预期：不会双runtime或使用已失效配置绕过锁；完成审计与实际generation匹配；冲突/失败明确，未批准状态不能偷偷启动。取消禁用镜像或恢复账号只在核验后显式执行。 + 2. 在一次启动/升级进行时另端pause账号或disable目标browser version/path + 3. 读取账号、browser version/path、environment及runtime/systemd 实物，按最终有效配置恢复 +- 预期：不会双 runtime 或使用已失效配置绕过锁；完成审计与实际generation匹配；冲突/失败明确，未批准状态不能偷偷启动。取消禁用browser version/path或恢复账号只在核验后显式执行。 #### ENV-09 gateway失联、清理待定与租约 [UI/FAULT] @@ -564,7 +564,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 操作： 1. F3停gateway后停止或回收环境一次，记录cleanup_pending/unknown/错误 2. 恢复gateway，只读观察至少两个20秒调和周期及审计 - 3. 另轮F7移除容器，不删除Profile，观察missing/runtime释放，再显式启动 + 3. 另轮 F7 停止/强杀 browser unit，不删除 Profile，观察missing/runtime释放，再显式启动 - 预期：失败不能假已停止/无绑定；后台恢复有requested/finished证据，租约不长期显示可调度假running；新旧generation不混用，Profile不丢。 #### ENV-10 非法动作与资源失效 [API] @@ -572,9 +572,9 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：附录D；测试alias及不存在alias。 - 操作： 1. POST现有alias/unknown-action；给upgrade增加未知字段或非法version - 2. 给create未知字段、已停用镜像或未授权账号；GET不存在alias + 2. 给create未知字段、已停用browser version/path或未授权账号；GET不存在alias 3. 回读原环境和动作审计 -- 预期：非法操作400/冲突404等明确，不修改旧环境；升级/重绑拒绝输入的审计不泄露输入秘密；没有通用PUT更新或任意Docker操作入口。 +- 预期：非法操作400/冲突404等明确，不修改旧环境；升级/重绑拒绝输入的审计不泄露输入秘密；没有通用 PUT 更新或任意 browser/CDP 操作入口。 ### 5.7 gateway 领域 API @@ -585,16 +585,16 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 1. 分别无token/错误token GET /v1/browsers，再用正确Bearer读取 2. POST create提交未知字段、非对象JSON、缺body及超过1MiB body 3. 访问非领域路径与不存在alias，最后回读正常列表 -- 预期：除healthz外领域请求校验token；非法body400，未知路径404，不提供通用Docker或任意CDP代理；列表未夹带其他非managed容器。 +- 预期：除healthz外领域请求校验token；非法body400，未知路径404，不提供通用 CDP 代理；列表未夹带其他非本 gateway runtime。 #### GATE-02 创建/启停/删除与代际 [API] -- 前置：单独的gateway实验alias，不对应活动控制面环境；附录G创建样例及已批准V1镜像。 +- 前置：单独的gateway实验alias，不对应活动控制面环境；附录G创建样例及已批准V1browser version/path。 - 操作： - 1. POST /v1/browsers 创建停止态实验容器，GET列表提取实际runtime/network/binding + 1. POST /v1/browsers 创建停止态实验 runtime，GET列表提取实际runtime/network/binding 2. 按附录G正确generation进行stop/remove；重复remove核对幂等或明确冲突 - 3. 另轮G-create改stopped=false创建运行态实验容器，GET取得真实network与runtime后用正确generation依次stop/start，核对实际状态；再用旧runtime_id或错误binding执行start/stop/remove并读取Docker实物 -- 预期：合法操作可观察，创建返回201；过期代际409且不操作新容器；不匹配managed标签的对象不可删除。实验资源按END单独回收，不交给控制面冒充业务环境。 + 3. 另轮G-create改stopped=false创建运行态实验 runtime，GET取得真实network与runtime后用正确generation依次stop/start，核对实际状态；再用旧runtime_id或错误binding执行start/stop/remove并读取runtime/systemd 实物 +- 预期：合法操作可观察，创建返回201；过期代际409且不操作新 runtime；不匹配 owner/node/generation 的对象不可删除。实验资源按END单独回收，不交给控制面冒充业务环境。 #### GATE-03 代理恢复与隧道生命周期 [API/REAL/FAULT] @@ -1273,7 +1273,7 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random 1. 在各类POST/PUT复制合法body分别加入unknown字段、尾随第二对象、错误类型；试空body与超过1MiB 2. GET各详情不存在ID，试POST路径拼错/不支持方法，记录实际HTTP和body 3. 正确body再次只读核验原记录，检查服务日志无Cookie/token/账号密码 -- 预期：字段错误400、资源不存在404、冲突409、依赖不可用503等按对应handler语义可辨；不把HTML SPA兜底当API成功。gateway Docker依赖可能502，不能强求所有模块统一503。 +- 预期：字段错误400、资源不存在404、冲突409、依赖不可用503等按对应handler语义可辨；不把HTML SPA兜底当API成功。gateway native gateway 依赖可能502，不能强求所有模块统一503。 ### 5.18 清理 @@ -1290,27 +1290,27 @@ OP-08的UI注入在浏览器Console执行 `Object.defineProperty(crypto, 'random - 前置：END-01；所有备份和RUN资源清单。 - 操作： - 1. 按3.2恢复全部网络/配置/文件/Overrides；回收测试浏览器容器，核对Profile保留 - 2. 删除未引用镜像/网关（网关用API）；停用测试出口；恢复原settings/策略/业务状态并只读核对 - 3. 需彻底销毁时，先停止专用Compose部署，由负责人逐一核对RUN容器/网络/卷后删除本轮数据卷和Profile；导出脱敏报告 -- 预期：只清本轮资源，禁止全局docker prune或不核对就down -v；回收后仍保留环境binding会阻止删除被引用网关/镜像，应登记保留或在获批整套测试库销毁时一起清理，不伪造永久删除API。原始敏感证据受限，最终报告脱敏。 + 1. 按3.2恢复全部网络/配置/文件/Overrides；回收测试browser runtime，核对Profile保留 + 2. 删除未引用browser version/path/网关（网关用API）；停用测试出口；恢复原settings/策略/业务状态并只读核对 + 3. 需彻底销毁时，先停止专用Compose部署，由负责人逐一核对RUN runtime/Profile/数据库临时资源后删除本轮数据卷和Profile；导出脱敏报告 +- 预期：只清本轮资源，禁止全局docker prune或不核对就down -v；回收后仍保留环境binding会阻止删除被引用网关/browser version/path，应登记保留或在获批整套测试库销毁时一起清理，不伪造永久删除API。原始敏感证据受限，最终报告脱敏。 ## 6. API 覆盖及请求样例 以下目录是第5节用例的执行附录，不额外计算用例数。每个写接口必须使用指定用例的前置与授权，不能把目录当成可直接批量运行的脚本。`无`表示无请求 body。所有对象样例均为合法 JSON 结构，里面的 A/S1/C/W/K/R 等必须替换为账本真实内部 ID；平台数字 UID、作品 key 和 comment key另取真实值。返回字段不要全量复制进严格输入接口。除明确 API 补充数据外，优先使用真实采集/监听输出；标记 `RUN-api-only` 的样例绝不能进入 REAL 通过统计。 -### 附录 A：网关登记与镜像 +### 附录 A：网关登记与browser version/path | 方法 / 路径 | body或查询样例 | 入口 / 用例 | | --- | --- | --- | | GET `/api/gateways` | 无 | UI、GW-01 | -| POST `/api/gateways` | `{"name":"e2e-gw","endpoint":"http://docker-gateway:8081","token":""}`；显式token从私密文件替换 | UI、GW-01/03 | -| PUT `/api/gateways/{G}` | `{"name":"e2e-gw-renamed","endpoint":"http://docker-gateway:8081","token":""}` | UI、GW-02 | +| POST `/api/gateways` | `{"name":"e2e-gw","endpoint":"http://host.docker.internal:8081","token":""}`（裸机改为 `http://127.0.0.1:8081`）；显式 token 从私密文件替换 | UI、GW-01/03 | +| PUT `/api/gateways/{G}` | `{"name":"e2e-gw-renamed","endpoint":"http://host.docker.internal:8081","token":""}`（裸机改为 `http://127.0.0.1:8081`） | UI、GW-02 | | DELETE `/api/gateways/{G}` | 无；只删除未引用项 | **API可用，UI接线失败待复现**，GW-04/05 | -| GET `/api/browser-images` | 无 | UI、IMG-01 | -| POST `/api/browser-images` | `{"version":"1-e2e","image_ref":"registry.example.invalid/test/browser:approved","note":"RUN","enabled":true}`；image_ref必须换已登记真实digest | UI、IMG-01 | -| PUT `/api/browser-images/{V2}` | `{"image_ref":"registry.example.invalid/test/browser:approved","note":"RUN编辑","enabled":false}`；不传version | UI、IMG-02 | -| DELETE `/api/browser-images/{V2}` | 无 | UI、IMG-03 | +| GET `/api/browser-versions` | 无 | UI、VER-01 | +| POST `/api/browser-versions` | `{"version":"148.0.7778.215","browser_path":"/absolute/path/to/chrome","note":"RUN","enabled":true}`；path必须换本机已核验的可执行文件 | UI、VER-01 | +| PUT `/api/browser-versions/{V2}` | `{"browser_path":"/absolute/path/to/chrome","note":"RUN编辑","enabled":false}`；不传 version | UI、VER-02 | +| DELETE `/api/browser-versions/{V2}` | 无 | UI、VER-03 | ### 附录 B：网络出口 @@ -1530,18 +1530,18 @@ F-message（仅隔离API存储，platform_message_key带API标记；message_at 这些接口不经 `/api`；独立实验用例才直接调用，正常业务优先从控制面进入。`GATEWAY_BASE` 的可达方式见3.1。除 `GET /v1/browsers` 外，领域GET/DELETE也要提交 JSON body及Content-Length，不能照普通GET省略。 -**代际获取**：控制面 `GET /api/browsers/{E1}` 返回对象内运行记录的 binding_version（先按实际JSON层级读取），与 gateway `GET /v1/browsers` 中同alias的 `id`、`network_id`、`network_exit_id` 共同核验；`id`是Docker runtime ID，不是控制面 runtime_instance_id。实际字段回读与Docker标签一致后再填请求。不存在时不得随意填 `1`/假ID来做成功测试。 +**代际获取**：控制面 `GET /api/browsers/{E1}` 返回对象内运行记录的 binding_version（先按实际JSON层级读取），与 gateway `GET /v1/browsers` 中同alias的 `id`、`network_id`、`network_exit_id` 共同核验；`id`是gateway runtime ID，不是控制面 runtime_instance_id。实际字段回读与 runtime metadata 一致后再填请求。不存在时不得随意填 `1`/假ID来做成功测试。 G-generation（start/stop/remove仅这三字段，不能混入network_exit_id）： ```json -{"binding_version":1,"runtime_id":"实际Docker容器ID","network_id":"实际Docker网络ID"} +{"binding_version":1,"runtime_id":"实际 runtime ID","network_id":"实际 network ID"} ``` G-douyin-base（cookies/get/identity/action/events均在此对象上加各自字段；直连出口为空，但仍须真实network_id）： ```json -{"binding_version":1,"runtime_id":"实际Docker容器ID","network_id":"实际Docker网络ID","network_exit_id":"P1"} +{"binding_version":1,"runtime_id":"实际 runtime ID","network_id":"实际 network ID","network_exit_id":"P1"} ``` | 方法 / 路径 | 请求样例 / 在base上追加字段 | 用例 | @@ -1569,7 +1569,7 @@ G-douyin-base（cookies/get/identity/action/events均在此对象上加各自字 - 一级评论：`https://www.douyin.com/aweme/v1/web/comment/list/?aweme_id=实际平台work_key&count=20&cursor=0`；下一页同理。 - 不允许其他host、额外参数、任意CDP或脚本。源码当前允许非负数字分页，不采用旧文档“只能max_cursor=0”的描述。 -G-create（独立实验alias，不与ENV共用；镜像换真实摘要。`stopped=true`仅允许直连；运行态实验需要受控网络并明确授权）： +G-create（独立实验alias，不与ENV共用；browser version/path换真实摘要。`stopped=true`仅允许直连；运行态实验需要受控网络并明确授权）： ```json {"alias":"e2e-gate-only","name":"RUN gateway实验","image":"registry.example.invalid/test/browser:approved","cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-e2e-gate-only","binding_version":1,"network_exit_id":"","network_exit":{},"stopped":true} @@ -1578,13 +1578,13 @@ G-create（独立实验alias，不与ENV共用；镜像换真实摘要。`stoppe G-proxy： ```json -{"binding_version":1,"runtime_id":"实际Docker容器ID","network_id":"实际Docker网络ID","network_exit_id":"P1","network_exit":{"protocol":"socks5","host":"proxy.example.invalid","port":1080,"username":"","password":""}} +{"binding_version":1,"runtime_id":"实际 runtime ID","network_id":"实际 network ID","network_exit_id":"P1","network_exit":{"protocol":"socks5","host":"proxy.example.invalid","port":1080,"username":"","password":""}} ``` G-action（gateway字段使用**平台**标识，不是内部K/W；仅预览）： ```json -{"binding_version":1,"runtime_id":"实际Docker容器ID","network_id":"实际Docker网络ID","network_exit_id":"P1","expected_uid":"真实发送账号UID","action":"reply_comment","target_uid":"真实作者UID","target_comment_id":"真实平台comment_key","target_work_id":"真实平台work_key","text":"RUN已批准文本","confirm":false} +{"binding_version":1,"runtime_id":"实际 runtime ID","network_id":"实际 network ID","network_exit_id":"P1","expected_uid":"真实发送账号UID","action":"reply_comment","target_uid":"真实作者UID","target_comment_id":"真实平台comment_key","target_work_id":"真实平台work_key","text":"RUN已批准文本","confirm":false} ``` GET带body调用示例（仅在后续执行时，`BODY_FILE`为替换后的G-douyin-base，认证配置含Bearer）： @@ -1603,7 +1603,7 @@ curl --silent --show-error --config "$GATEWAY_AUTH_FILE" \ | 项目 | 基线事实 / 风险 | 对应判定 | | --- | --- | --- | | 首次已验证登录闭环 | 新账号unknown；公开login-result拒绝logged_in；采集/写/监听又要求logged_in。人工浏览器登录到控制面资料的可达路径需验证 | AC-06；不能改库解决测试前置 | -| 网关UI删除 | provider仅允许删除browser-images；页面网关删除未正确接通 | GW-04失败与GW-05 API成功分开 | +| 网关 UI 删除 | gateway 删除必须只回收 runtime，保留 Profile 与正式结果；页面状态和 API 结果分开核对 | GW-04 失败与 GW-05 API 成功分开 | | 线索人工联系 | 线索页未渲染回复编辑区、没有自动切评论页 | LEAD-06，不能把手动绕路当通过 | | 规则错误区、依赖失败 | 多处辅助列表失败可能退化空数据/准备提示 | UI-06、LEAD-02、ENV-03 | | 仿写 | 确认只保存要求；PUT缺Content-Type，但服务端直接JSON解码；切换/确认后表单一致性需实测 | MAT-05/06，不期待自动生成 | @@ -1621,7 +1621,7 @@ curl --silent --show-error --config "$GATEWAY_AUTH_FILE" \ 复制下表，每例及每个参数化子项各一行。不要把多协议、四种Mock结果、六类动作合并成一个“通过”。 -| RUN / 用例 / 子项 | 层级 | HEAD / 镜像digest | 执行人 / UTC起止 | 前置账号与资源ID | 授权单 | 实际有序操作 | HTTP/operation/event/Attempt ID | 实际UI与平台结果 | 证据路径 | 状态 | 缺陷/阻塞及恢复清理 | +| RUN / 用例 / 子项 | 层级 | HEAD / browser version/path | 执行人 / UTC起止 | 前置账号与资源ID | 授权单 | 实际有序操作 | HTTP/operation/event/Attempt ID | 实际UI与平台结果 | 证据路径 | 状态 | 缺陷/阻塞及恢复清理 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 待填写 | UI/API/REAL/MOCK/FAULT | 待填写 | 待填写 | 脱敏ID | 无写填不适用 | 记录实际步骤，不只复制计划 | 无则说明原因 | 不以“正常”代替事实 | 脱敏截图/响应/日志索引 | 未执行 | 待填写 | diff --git a/docs/evidence/native-browser-p0-2026-09-17.md b/docs/evidence/native-browser-p0-2026-09-17.md new file mode 100644 index 0000000..cb1b5ef --- /dev/null +++ b/docs/evidence/native-browser-p0-2026-09-17.md @@ -0,0 +1,66 @@ +# Native Browser P0 本机前置核验记录 + +- 核验时间：2026-09-17T07:51:43Z 起，按单节点范围复核至 2026-09-17 +- 状态：**P0 已通过，放行本机单节点实现；真实平台验收证据待 P6 补齐** +- 依据：`docs/native-browser-change-review.md`、`docs/native-browser-implementation-plan.md`、`docs/native-browser-verification.md` +- 本目标范围：本机单节点；多节点控制、A/B 环境、跨机恢复与多节点性能对比移至后续目标。 + +## Git 基线与隔离 + +- 同步时间：2026-09-17T08:23:28Z +- 同步源：`main` 与 `origin/main` 均为 `7e3808cf4ce453b3583079680a3b59ca2ed64ad4` +- worktree 同步前提交：`1bf905599cb843959d6b1d4d64918771a0f87aa7` +- worktree 同步后提交：`7e3808cf4ce453b3583079680a3b59ca2ed64ad4` +- 实施分支：`refactor/native-browser-xvfb-impl` +- 实施 worktree：`/home/rogee/Workspace/creator-hub-native-browser` +- 采用 fast-forward 同步，未使用强制 reset。原工作目录由其它线程使用，本次未对其 stash、reset、删除或覆盖；其既有修改和未跟踪文件继续保留。 +- 本 worktree 尚未修改业务代码；未执行 Docker 删除、远程服务修改、数据库重建或浏览器资源回收。 + +## 本机能力 + +| 项目 | 证据 | 结果 | +| --- | --- | --- | +| 操作系统/用户 | Linux `home-dev`，uid 1000 `rogee` | 通过：gateway 可按非 root 用户运行 | +| Python/Go/Node | Python 3.13.5、Go 1.26.4、Node v22.21.0 | 通过：满足项目版本要求 | +| systemd | systemd 257.9；system/user manager 可访问，但总体状态 `degraded`，失败项为既有桌面/Compose 服务 | 部分通过：可继续核验，不能把既有服务当作 CreatorHub native gateway | +| Xvfb | 已安装；隔离 `-displayfd` 启动/退出探针成功 | 通过：可按 runtime 分配 display；不得复用固定 `:99` | +| 磁盘 | `/` 使用率 66%，`/tmp` 使用率 16%；用户已批准最低可用磁盘 20 GB | 通过：实现和验收必须在低于下限前拒绝新 runtime 并保留清理证据 | +| 网络 | `ens18` 为 `10.1.1.104/24`，默认路由为 `10.1.1.10` | 通过：局域网监听/联调具备基础条件；实际代理和平台访问仍需单独授权核验 | +| sandbox / fingerprint 浏览器探针 | uid 1000 使用独立临时 Profile、未传 `--no-sandbox` 启动 fingerprint Chromium 148.0.7778.215；`unshare -Ur` 与 `unshare -Urnm` 成功，指纹参数使 UA/platform/cores/lang/timezone 生效 | 通过：可使用 Linux user-namespace sandbox；不继承现有普通 Chrome 或 Docker 进程 | + +## 浏览器与 Profile 边界 + +- 已按用户指定的上游仓库安装 fingerprint Chromium：。 +- 上游版本：tag `148.0.7778.215`，源码仓库提交 `3f61b0dfa665e883da8824b1450601fc529dd006`；发布资产为 `ungoogled-chromium-148.0.7778.215-1-x86_64_linux.tar.xz`，GitHub API 提供的 SHA-256 为 `70d239830332e5820aa34dfcb284161cac0429eee25da642830afe04bda717f4`。 +- 本机安装路径：`/home/rogee/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome`；安装二进制 SHA-256 为 `abd700e6015e259a00f1a31e99ad16f99f63365222328937454e8f603f575284`，权限为 `rogee:rogee 0755`。 +- 上游 `LICENSE` 为 BSD-3-Clause；仓库 README 说明该构建基于 Ungoogled Chromium，并公开上述指纹参数。许可证和来源已记录，未把其代码复制进本项目。 +- 当前看到的 `/opt/chromium/chrome` 只存在于另一进程的 mount namespace，主机路径不存在；该进程命令行带 `--no-sandbox`，不能作为 native gateway 的浏览器或 sandbox 证据。 +- `~/.chrome/profile` 当前为活动浏览器 Profile（权限 775），不能被新 runtime 抢占或改写；`~/.config/google-chrome` 与 `~/.config/chromium` 为现有用户目录，也不能直接作为 CreatorHub 账号 Profile 根目录。 +- 已创建独立的 CreatorHub Profile 根目录：`/home/rogee/.local/share/creatorhub/browser-profiles`，权限 `rogee:rogee 0700`。每个账号/运行代次必须由 gateway 依据内部 ID 生成子目录并独占锁定；禁止请求方传入任意删除路径，禁止触碰现有用户 Profile。 +- 当前已有 `:99`/5900/9222 浏览器会话，证明不能假定固定 display、VNC 或 CDP 端口；runtime 必须做冲突安全分配。 + +## 已批准的单节点验证参数 + +用户已批准以下初始参数，复杂业务采集最大时长仍按具体场景单独配置，不因清理预算强行截断： + +| 参数 | 初始值 | +| --- | --- | +| 短任务清理预算 `T_cleanup` | 30 秒 | +| 服务恢复预算 `T_recover` | 60 秒 | +| 验证并发 | 1（基线）/2（压力） | +| runtime lease | 60 秒过期，20 秒续租 | +| 最低可用磁盘 | 20 GB | +| 单 runtime 日志上限 | 1 GB | +| Profile 缓存上限 | 20 GB | +| 业务任务最大时长 | 按场景配置；不设全局强制截断 | + +## 仍需在后续真实验收时形成的证据（不阻塞本机代码实现） + +1. gateway 使用固定 fingerprint 版本和指纹参数的持久化清单，以及浏览器升级对照记录。 +2. 真实代理端点/认证授权（若执行代理用例）；直连只能作为明确的无代理配置，不得作为代理失败回退。 +3. 抖音测试账号、登录二维码、UID 身份核对及真实采集/写操作的逐项人工授权；不得把密码、Cookie、验证码或 token 写入记录。 +4. 旧 Docker 对照是否实际执行；没有可信旧基线时性能改善必须标记未验证。 + +## 结论 + +本机 Linux、非 root 用户、Python、systemd、Xvfb、磁盘、基础网络、已安装的 fingerprint Chromium sandbox 探针和独立 Profile 根目录已具备；用户已批准单节点验证参数，P0 放行本机单节点实现。真实平台写操作、代理用例和性能对照仍须在验收阶段按授权执行。不得切回 Docker、使用普通 Chrome 冒充 fingerprint browser、传入 `--no-sandbox`、抢占现有 Profile，或删除未确认的旧 Docker 资源。 diff --git a/docs/evidence/native-browser-reference-audit-2026-09-18.md b/docs/evidence/native-browser-reference-audit-2026-09-18.md new file mode 100644 index 0000000..4de8749 --- /dev/null +++ b/docs/evidence/native-browser-reference-audit-2026-09-18.md @@ -0,0 +1,20 @@ +# Native browser 旧 Docker 引用审计 + +- 日期：2026-09-18 +- 范围：仓库文本引用；排除 `.git`、`node_modules`、构建产物和 source map。 + +## 生产路径结果 + +`cmd/`、`internal/`、`web/src/` 未发现 `docker.sock`、`DockerClient`、浏览器容器/镜像/卷/网络生命周期、browser wrapper 或 Docker fallback 引用。 + +## 保留引用分类 + +| 文件 | 分类 | 原因 | +| --- | --- | --- | +| `docs/plans/2026-09-15-douyin-release-remediation.md` | 历史计划 | 记录旧发布方案中的 wrapper/schema 风险，不是当前部署入口。 | +| `docs/native-browser-change-review.md` | 迁移审计 | 对照记录旧 `cmd/docker_gateway` 和旧 wrapper，并明确当前已删除。 | +| `docs/deployment_stop_test.sh` | 检查规则 | 以 `docker.sock` 作为禁止旧浏览器链路的负向断言。 | +| `docs/native-browser-verification.md` | 验收规则 | 定义 production static audit 必须排除 Docker 浏览器生命周期。 | +| `docs/python-gateway-branch-review.md` | 历史评审 | 记录迁移前分支的缺陷和旧设计，不是可执行代码。 | + +审计结论：剩余引用均属于历史、审计或负向检查文档；未发现当前浏览器生产路径的隐式 Docker 回退。PostgreSQL 独立 Compose 资产不在本审计的删除范围内。 diff --git a/docs/evidence/native-browser-verification-2026-09-18.md b/docs/evidence/native-browser-verification-2026-09-18.md new file mode 100644 index 0000000..7f1d8cb --- /dev/null +++ b/docs/evidence/native-browser-verification-2026-09-18.md @@ -0,0 +1,126 @@ +# 单节点 native browser 验证记录 + +- 执行时间：2026-09-18（Asia/Shanghai） +- 执行者：本 worktree 的开发检查 +- worktree：`/home/rogee/Workspace/creator-hub-native-browser` +- 分支：`refactor/native-browser-xvfb-impl` +- 基线：`7e3808cf4ce453b3583079680a3b59ca2ed64ad4` +- 远程同步：执行时 `main` 与 `origin/main` 均为 `7e3808cf4ce453b3583079680a3b59ca2ed64ad4`；当前 worktree 未改写原工作目录。 + +## 前置条件 + +| 项目 | 实际记录 | 状态 | +| --- | --- | --- | +| Linux 用户 | `rogee`，UID 1000，非 root | 通过 | +| Xvfb | `/usr/bin/Xvfb` | 通过 | +| systemd | `systemd-run` 可用，`systemctl --user` 可用 | 通过 | +| 浏览器 | fingerprint Chromium `148.0.7778.215`，可执行文件存在且可执行 | 通过 | +| Profile/runtime | 使用专用临时目录；目录权限为 0700 | 通过 | +| 数据库 | PostgreSQL `127.0.0.1:15432`；每次集成检查使用临时 schema | 通过 | +| 真实平台账号/真实代理 | 用户已提供抖音账号并完成只读身份/作品读取；合法代理仍未提供 | 部分通过 | + +## 自动检查 + +| 检查 | 实际结果 | 状态 | +| --- | --- | --- | +| Go PostgreSQL 集成测试 | `go test -p 1 -count=1 ./...`，全部通过 | 通过 | +| Go 覆盖率 | 总覆盖率 `65.01%`；creator `65.9%`、hub `65.8%`、phasea `68.1%` | 通过 | +| `go vet ./...` | 通过 | 通过 | +| control-plane 构建 | `go build ./cmd/control-plane`，通过 | 通过 | +| Go race | `go test -race ./...`，全部通过 | 通过 | +| Python gateway | 69 tests 通过；生产文件覆盖率 `67%–76%` | 通过 | +| Frontend | 101 tests 通过；覆盖率 `66.31%`；Vite build 通过 | 通过 | +| Compose 配置 | PostgreSQL-only Compose 配置检查通过 | 通过 | +| 静态旧路径检查 | production gateway/control-plane 无 Docker socket、container/image/volume/network 生命周期或 Docker fallback | 通过 | + +## Real native gateway smoke + +### 正常生命周期 + +- 临时 gateway 以 UID 1000 启动，监听 `127.0.0.1:28183`，稳定 `node_id=native-smoke-node`。 +- `/healthz` 返回 `204`。 +- `/v1/info` 返回 `200`，列出浏览器版本 `148.0.7778.215`。 +- 创建 `smoke-runtime` 返回 `201`，`state=running`、`status=ready`、`ready=true`，返回独立 display、CDP 端口、runtime/network ID 和 `proxy_ready=true`。 +- stop 返回 `204`，delete 返回 `204`；查询结果为 `state=released`、`cleanup_state=cleaned`。 +- 清理后 `systemctl --user list-units 'creatorhub-browser-*'` 为 0；临时 smoke 目录已删除。 + +### gateway 强制重启恢复 + +- 临时 gateway 监听 `127.0.0.1:28184`，创建一个 ready runtime 后对 gateway 进程执行 `SIGKILL`。 +- 使用相同 state/profile 根目录和稳定 node ID 重启 gateway；重新查询得到同一 runtime，`state=running`、`ready=true`、`node_id=native-restart-node`。 +- stop/delete 均返回 `204`，清理后受管 systemd unit 数量为 0。 + +上述 smoke 使用专用临时目录；为避免测试盘空间限制，仅该 smoke 将 `RUNTIME_MIN_FREE_BYTES` 覆盖为 0，不能替代 20 GB 磁盘阈值用例。 + +## 局域网裸启动 smoke + +- 使用临时 PostgreSQL schema 启动 control-plane `LISTEN_ADDR=0.0.0.0:28082`，`/healthz` 返回 `204`；使用 Vite `--host 0.0.0.0 --port 5175`，首页返回 `200`。 +- `ss` 确认两个监听分别为 `0.0.0.0:28082` 和 `0.0.0.0:5175`；本机 LAN 地址为 `10.1.1.104`，Vite 输出该地址。 +- smoke 结束后停止临时进程并删除临时 schema/目录；原工作目录中已存在的服务和端口未触碰。 + +## 33 项单节点记录 + +| ID | 记录结果 | 证据/阻塞原因 | +| --- | --- | --- | +| A01 | 通过 | 独立 native runtime 新 Profile 通过 VNC 手工登录；control plane 身份核对返回 `logged_in`，Profile 未接管用户的 `chrome.service` | +| A02 | 阻塞 | 真实二维码、Profile 登录保持和版本升级未执行 | +| A03 | 阻塞 | 缺少真实竞品账号与采集授权 | +| A04 | 通过 | 独立 native runtime 通过 control plane scheduler 完成真实 Douyin works/comments checkpoint；当前账号无公开作品，保存 0 条，不伪造作品数据；证据见 `native-control-plane-real-douyin-2026-09-18.md` | +| A05 | 阻塞 | 同节点双账号真实采集未执行 | +| A06 | 阻塞 | 同 Profile 并发/人工登录占用未执行 | +| A07 | 阻塞 | 长期监听与采集并行缺少真实账号 | +| A08 | 阻塞 | 长期 holder 借用场景未执行 | +| A09 | 阻塞 | 缺少合法 HTTP/HTTPS/SOCKS4/SOCKS5 代理 | +| A10 | 阻塞 | 真实人工发送授权未提供 | +| B01 | 阻塞 | 真实采集取消未执行；代码取消路径有回归测试 | +| B02 | 阻塞 | 真实超时任务未执行 | +| B03 | 阻塞 | 真实代理失败与无效 browser path 未逐项手工执行 | +| B04 | 阻塞 | 真实浏览器/Xvfb 故障注入未执行 | +| B05 | 阻塞 | 真实目录清理失败未执行 | +| B06 | 阻塞 | 20 GB 阈值/磁盘写满未执行；smoke 覆盖了非阈值生命周期 | +| B07 | 通过 | native stop/delete 重复清理回归测试和 smoke 通过 | +| B08 | 通过 | gateway 强杀后 generation 恢复 smoke 通过 | +| B09 | 阻塞 | 真实长任务续租及旧 lease 迟到未执行；确定性 lease 测试通过 | +| B10 | 阻塞 | 真实结果保存失败/响应丢失未执行 | +| C01 | 通过 | native gateway SIGKILL 后同 runtime/node/ready 恢复 smoke 通过 | +| C02 | 阻塞 | 真实事件、冷却和订阅恢复未执行；事件去重/lease 自动测试通过 | +| C03 | 阻塞 | 真实短任务最长寿命观察未执行 | +| C04 | 阻塞 | control-plane 强杀期间素材处理未执行 | +| C05 | 阻塞 | 真实旧执行与新执行竞争未执行；资源归属测试通过 | +| C06 | 阻塞 | 真实机器重启未执行 | +| C07 | 通过 | 旧 generation/无关资源保护有确定性回归测试；真实目录手工核对未执行 | +| D01 | 未执行 | 多节点后续目标 | +| D02 | 未执行 | 多节点后续目标 | +| D03 | 未执行 | 多节点后续目标 | +| D04 | 未执行 | 多节点后续目标 | +| D05 | 未执行 | 多节点后续目标 | +| D06 | 未执行 | 多节点后续目标 | + +## 尚未执行/阻塞的真实验收 + +- Douyin 独立 native runtime 登录、身份核对、只读采集和空结果 checkpoint 已通过；Xiaohongshu、长期监听、自动响应、写操作仍未执行，分别因范围/授权/真实平台条件标记未执行或阻塞。 +- HTTP/HTTPS/SOCKS4/SOCKS5 真实出口及认证失败：缺少合法代理端点，阻塞。 +- 另一台设备的 LAN 手工访问与前端完整点击流程：未执行，需联调操作者完成；本机 `0.0.0.0` 裸启动 smoke 已通过。 +- 磁盘不足、真实 Profile 占用、清理失败和真实平台写操作：真实破坏性/写入场景未执行；确定性测试、真实 stop/repeat-stop/delete 和 runtime cleanup 已通过，不能把它们替代为全部真机故障证据。 +- 多节点、A/B、跨机恢复、性能对比：按目标边界延期，不属于本目标。 + +本记录没有保存密码、Cookie、token、真实账号标识或平台数据。 + +## 补充：用户授权的 service-managed Chrome 真实读取 + +初始记录完成后，用户授权使用当前由 `chrome.service` 管理的 Chrome，仅通过本机 CDP `127.0.0.1:9222` 连接测试交互；没有让 native gateway 接管该 Chrome，也没有改变原有 `:99` Xvfb/VNC 启动方式。详细记录见 [`native-douyin-service-browser-2026-09-18.md`](native-douyin-service-browser-2026-09-18.md)。 + +- 抖音身份核对：通过，用户提供的账号标识与平台返回 UID 匹配。 +- 抖音资料读取：通过，UID 与 `sec_uid` 两种查询均返回 `status_code=0`。 +- 抖音作品第一页读取：通过，返回 13 条作品并标记存在下一页。 +- 独立 native runtime 的 control-plane works/comments checkpoint 已通过；自动响应、人工发送、代理和真实资源故障场景仍未执行，不能用只读读取结果替代。 +- 代理阻塞确认：2026-09-18 用户确认不提供合法 HTTP/HTTPS/SOCKS4/SOCKS5 代理端点；相关用例继续标记为“阻塞”，不宣称单节点目标完成。 + +## 补充：变更后自动检查复跑 + +- Python gateway：69 tests 通过；生产文件覆盖率分别为 `douyin.py 67%`、`gateway.py 68%`、`proxy.py 68%`、`runtime.py 76%`，均达到 65% 要求。 +- Go：`go test -p 1 -count=1 ./...`、`go test -race -p 1 -count=1 ./...`、`go vet ./...`、`go build ./cmd/control-plane` 通过。 +- Frontend：Vitest 101 tests 通过；coverage statements `66.31%`、lines `68.73%`；Vite build 通过。默认 worker 在机器高负载时曾启动超时，改用单 worker threads pool 后通过，未修改测试代码。 +- 部署：`git diff --check`、`docs/deployment_stop_test.sh` 通过。 +- 外部显示人工观察：临时 native gateway 复用已运行的 Xvfb `:99`，runtime 在 VNC `5900` 可见；停止/删除后恢复用户的 `chrome.service`，原 Profile 未被覆盖。该模式仅用于单节点人工观察，默认仍是每 runtime 独立 Xvfb。 +- 独立 native gateway + control plane 的真实抖音闭环记录见 [`native-control-plane-real-douyin-2026-09-18.md`](native-control-plane-real-douyin-2026-09-18.md)：runtime 启停、身份核对、works/comments checkpoint 均已通过；当前账号无公开作品，因此保存 0 条作品，不把空结果伪造为有数据。 diff --git a/docs/evidence/native-control-plane-real-douyin-2026-09-18.md b/docs/evidence/native-control-plane-real-douyin-2026-09-18.md new file mode 100644 index 0000000..043d74d --- /dev/null +++ b/docs/evidence/native-control-plane-real-douyin-2026-09-18.md @@ -0,0 +1,29 @@ +# Native gateway + control plane 抖音单节点真实闭环 + +- 日期：2026-09-18 +- 验证范围：独立 native gateway、独立 Profile、Xvfb、control plane、抖音真实账号只读采集。 +- 用户边界：未停止或接管用户当前 `chrome.service`、原 Profile、`:99` Xvfb 或 VNC `5900`。 +- 独立 control plane：`0.0.0.0:28082`，使用隔离 PostgreSQL schema。 +- 独立 gateway：`127.0.0.1:28187`，display `:122`，CDP `19000`，Profile 为独立目录。 + +## 结果 + +| 检查 | 结果 | 状态 | +| --- | --- | --- | +| control plane 创建 gateway/browser version/account/environment | API 返回成功；native browser version 使用 fingerprint Chromium `148.0.7778.215` | 通过 | +| 显式启停 | `POST /api/browsers/independent-douyin/start` 返回 204；runtime 保持 running/ready | 通过 | +| runtime 身份绑定 | control plane 记录 runtime ID、network ID、runtime instance 和 ready schedule | 通过 | +| 抖音账号核对 | control plane verify 返回 `logged_in`；账号标识不写入仓库 | 通过 | +| 抖音作品采集 | control plane scheduler 调用 native gateway，作品页返回 `status_code=0`；当前账号无公开作品，保存 0 条作品 | 通过 | +| 采集 checkpoint | works 与 comments 均为 `succeeded`，写入 `last_completed_at`，无错误 | 通过 | +| 资源状态 | runtime 运行期间为 running/ready，gateway 与 control plane 的 runtime lease 状态可见 | 通过 | +| 资源释放 | control plane stop 返回 204；重复 stop 返回 204；删除环境返回 204；停止后 cleanup_pending=false | 通过 | + +## 实施中发现并修复的问题 + +1. control plane scheduler 与手工启动同时绑定同一 runtime 时，旧逻辑会把并发胜出的 runtime 清理掉；`activateGatewayRuntime` 现会识别“同一 generation 已被另一执行者绑定”的情况并复用它。新增回归测试覆盖该竞态。 +2. fingerprint Chromium 在 Xvfb 下需要 `--disable-gpu` 与 `--disable-gpu-compositing`；否则 GPU 子进程会退出并造成 runtime 假 ready。启动命令已补齐参数。 +3. 抖音真实 works 接口对 `has_more` 返回数字 `0/1`；connector 现同时接受 JSON boolean 与 `0/1`，并新增解析测试。 +4. 第一次真实采集曾因身份核对与 scheduler 同时占用 alias 返回 `browser alias is busy`；该次被记录为失败并重试，未被伪造为成功。 + +原始账号标识、Cookie、密码、gateway token 和数据库凭据未写入仓库。 diff --git a/docs/evidence/native-douyin-service-browser-2026-09-18.md b/docs/evidence/native-douyin-service-browser-2026-09-18.md new file mode 100644 index 0000000..1efc470 --- /dev/null +++ b/docs/evidence/native-douyin-service-browser-2026-09-18.md @@ -0,0 +1,20 @@ +# 抖音真实浏览器连接验证记录 + +- 日期：2026-09-18（Asia/Shanghai） +- worktree：`/home/rogee/Workspace/creator-hub-native-browser` +- 浏览器归属：用户明确要求由 `chrome.service` 管理；本次不停止、不启动、不删除该 Chrome，仅通过本机 CDP 连接测试交互。 +- 显示：原有 Xvfb `:99` 与 VNC `5900` 保持不变。 +- CDP：`127.0.0.1:9222`。 +- 浏览器版本：fingerprint Chromium `148.0.7778.215`。 + +## 结果 + +| 检查 | 结果 | 状态 | +| --- | --- | --- | +| 抖音登录身份核对 | 用户提供的授权账号标识与 `/user/profile/self/` 返回的 UID 匹配；返回 `sec_uid`、昵称和平台时间 | 通过 | +| 抖音账号资料读取 | `/user/profile/other/` 使用 UID 与 `sec_uid` 均返回 HTTP 200、`status_code=0`、包含用户资料 | 通过 | +| 抖音作品第一页读取 | `/aweme/post/` 返回 HTTP 200、`status_code=0`、13 条作品、`has_more=1`，存在下一页游标 | 通过 | +| 写操作/自动响应 | 本次未执行 | 未执行 | +| 结果持久化 | 本次仅验证真实浏览器与平台读取，未伪造控制面持久化成功 | 未执行 | + +原始账号标识、Cookie、密码和 token 不写入仓库。native gateway 临时 runtime 已停止并删除；用户的 `chrome.service` 已恢复运行。 diff --git a/docs/native-browser-change-review.md b/docs/native-browser-change-review.md index 1c90458..1195815 100644 --- a/docs/native-browser-change-review.md +++ b/docs/native-browser-change-review.md @@ -2,11 +2,11 @@ ## 1. 状态与结论 -- 日期：2026-09-17。 -- 评审基线：`main` / `1f63b5b`，以及评审时工作区的现有未提交改动；不能将本文当作该提交单独具备的能力证明。 -- 用户已确认：本轮只交付评审、实施计划与验证文档，不修改业务代码。 +- 日期：2026-09-18。 +- 评审基线：已同步的 `main` / `origin/main` / `7e3808cf4ce453b3583079680a3b59ca2ed64ad4`；评审时的旧 Docker 入口仅作为历史对照，不能将本文当作真实平台能力证明。 +- 评审阶段的契约已在独立实施 worktree 执行。当前状态以实施计划、验证文档和证据记录为准。 - **方向成立，但不是把 Docker 启动命令换成 Xvfb 命令即可。**必须同时替换进程生命周期、运行代次、Profile 路径、端口分配、代理接入及版本管理。 -- 下文“目标/推荐”是待实施方案，不表示已实现或已通过真机验证。详细执行顺序见[实施计划](native-browser-implementation-plan.md)，验收见[验证文档](native-browser-verification.md)。 +- 下文保留必要的风险、边界和历史对照；不表示真实平台、真实代理或 LAN 已全部验收。详细实现范围见[实施计划](native-browser-implementation-plan.md)，验收见[验证文档](native-browser-verification.md)。 ### 已确认的需求 @@ -29,19 +29,19 @@ | 现有事实/入口 | 对本次变更的影响 | | --- | --- | -| `internal/hub/store.go` 的 `Gateway` 保存名称、Endpoint；`Env` 保存 `Gateway`、`ImageVersion` | 多机器注册与路由已存在，应复用，不新增第二套节点管理 | +| `internal/hub/store.go` 的 `Gateway` 保存名称、Endpoint；`Env` 保存 `Gateway`、`BrowserVersion` | 多机器注册与路由已存在，应复用，不新增第二套节点管理 | | `cmd/control-plane/hub.go` 的 `startBrowserRuntime`、`createGatewayRuntime`、`removeGatewayRuntime` 管理启停与清理 | 必须整体改造生命周期，不能仅替换 Python 中一个 Docker 调用 | | `gatewayCreatePayload` 下发 `creatorhub-profile-{alias}` 卷名、指纹、代理和绑定版本 | 卷改成 gateway 管理的持久目录；控制面不能下发任意宿主机路径 | | `runtimeCleanupStore` 及启动前清理逻辑已有 cleanup-pending 概念 | 保留“未清理完成不能冒充已停止”的语义，改用本机资源身份，不再依赖容器/网络 ID | -| `cmd/docker_gateway/gateway.py` 初始化 `DockerClient`；`load_config` 读取 `DOCKER_SOCKET`、`BROWSER_NETWORK` | 当前正常入口仍依赖 Docker。外部 CDP 配置并不等于完整的原生浏览器生命周期 | -| `docker/browser-wrapper/docker-entrypoint.sh` 包含 Xvfb、x11vnc、socat、Chromium，固定 `:99` 和容器内端口，Profile 为 `/data` | 已有容器内 Xvfb 脚本，但 gateway 强制入口与 wrapper 入口不一致，不能证明实际镜像执行了它。宿主机并发不能复用固定 display/端口 | -| 同一脚本传入 `--no-sandbox` | **当前脚本并未启用 Chromium sandbox。**不能以“浏览器本身就是沙盒”解释现有安全边界或直接照搬启动参数 | +| 历史 `cmd/docker_gateway/gateway.py` 曾初始化 `DockerClient` 并读取 `DOCKER_SOCKET`、`BROWSER_NETWORK` | 已由 `cmd/browser_gateway/gateway.py` 的 native runtime 管理替换；当前浏览器入口不依赖 Docker。该行只保留迁移前事实 | +| 历史 `docker/browser-wrapper/docker-entrypoint.sh` 曾固定容器内 Xvfb/端口/Profile | 浏览器 wrapper 已从生产入口删除；native gateway 为每个 runtime 独立分配 display、端口和 Profile，历史行不构成当前能力证明 | +| 历史 wrapper 曾传入 `--no-sandbox` | native gateway 拒绝该参数并以非 root 用户启动；无法满足 sandbox 时直接失败，不回退旧 wrapper | | `cmd/control-plane/creator.go` 的采集读取已有运行账号，没有按任务自动创建 runtime；`creator_events.go` 独立运行监听 | 需要新增任务使用权；现有 source lease 不是浏览器使用权，直接在采集末尾调用 stop 会误停共享会话 | | `creator.go` 的临时下载、`creator_material.go` 的 `audio.wav.tmp` 依赖正常退出删除；素材先写固定路径再提交数据库 token | 控制面本机也需要崩溃清理；旧执行可能覆盖新产物，必须先隔离执行目录与产物发布，不能只增加目录扫描 | | `internal/creator/source_lease.go` / `content.go` 的来源 lease 为固定十分钟；部分失败收尾仍使用已取消的 context | 长任务可能重复领取，取消后仍显示 running；资源使用权必须覆盖真实执行期并有独立收尾上下文 | | `internal/hub/store.go` 的 gateway endpoint 可更新，环境按 gateway 名称重新解析地址 | 不能把名称当稳定机器身份；原 owner 和待清理目标必须保留，禁止把仍有绑定资源的 gateway 改指另一机器 | | `internal/hub/store.go` 的 `Image`、前端镜像页、指纹/版本字段依赖 Docker 镜像语义 | 浏览器版本来源必须替换；只改界面名称会留下无效配置 | -| `scripts/dev-backend.mjs` / `compose*.yaml` 启动 Docker gateway | 联调脚本和部署说明必须同步；不能让“原生模式”后台仍偷偷启动旧 gateway | +| `scripts/dev-backend.mjs` / `compose*.yaml` 现在只保留 PostgreSQL（Compose）和 host-native gateway 注册 | 联调脚本和部署说明已同步；原生模式不启动 Docker browser gateway | | `requirements-gateway.lock` 当前锁定 `websocket-client` | 优先保留已有 CDP/WebSocket 与平台适配器，不因取消 Docker 就新增 Playwright/Patchright | ## 3. 方案选择 @@ -143,21 +143,21 @@ Xvfb 只负责显示；gateway 负责启动、就绪确认、终止、回收与 ## 6. 改造评审结论与放行项 -**建议采用：每台 gateway 管理本机独立 Xvfb/runtime，保留账号 Profile，采集结束回收任务资源，复用现有多机路由及业务层。**先形成一个可手工验收的抖音端到端切片，再完成多机故障与全业务回归。 +**已采用：每台 gateway 管理本机独立 Xvfb/runtime，保留账号 Profile，采集结束回收任务资源，复用现有多机路由及业务层。**单节点代码、自动检查和 native gateway smoke 已完成；真实账号、代理、LAN、多节点和性能验收按验证文档分开记录。 代码实施前须确认： -- [ ] 目标机器为满足 systemd、Xvfb、sandbox 条件的 Linux，原生指纹浏览器可合法部署且能力相同。 -- [ ] 接受删除 Docker 浏览器代码路径、镜像字段/接口及相关部署配置；开发数据可以受控重建，但删除用户数据仍需逐次授权。 -- [ ] 确认 Profile 不随采集删除、机器不自动切换、任务不得抢占人工登录/长期监听；gateway 重启按独立 runtime 核对恢复，机器重启不自动恢复已过期短任务。 -- [ ] 确认本次不把完整网页远程桌面当作已有能力；若要求补齐，应另列明确验收范围。 -- [ ] 批准[验证文档](native-browser-verification.md)中的资源预算与性能放行标准；没有旧基线时不能宣称已降低开销。 +- [x] 目标机器为满足 systemd、Xvfb、sandbox 条件的 Linux，原生指纹浏览器可合法部署且能力相同；已完成非 root native smoke。 +- [x] 已接受删除 Docker 浏览器代码路径、镜像字段/接口及相关部署配置；开发数据可受控重建，用户数据删除仍需逐次授权。 +- [x] Profile 不随采集删除、机器不自动切换、任务不得抢占人工登录/长期监听；gateway 重启按独立 runtime 核对恢复。 +- [x] 本次不把完整网页远程桌面当作已有能力；若要求补齐，另列明确验收范围。 +- [x] 验证文档中的资源预算已固定；性能改善和真实平台能力未以无旧基线的推测代替实测。 -以上是实施放行清单，不是本轮要求用户补齐运行环境后才能拿到文档。 +以上是已执行的实施放行记录；真实平台、代理、LAN 和破坏性资源用例仍需授权操作者按验证文档补齐，未执行项不视为通过。 ## 7. 专项评审处理记录 -本轮进行了两项独立只读代码评审：gateway/多机生命周期，以及采集/素材资源清理。它们审查的是现有实现与改造风险，不是新实现的测试通过证明；本方案由主审汇总，仍待用户批准实施。 +本轮进行了两项独立只读代码评审：gateway/多机生命周期，以及采集/素材资源清理。它们审查的是实现与改造风险，不是新实现的真实平台证明；自动检查和 native smoke 已记录，真实手工验收仍按验证文档执行。 | 发现 | 处理 | | --- | --- | @@ -168,7 +168,7 @@ Xvfb 只负责显示；gateway 负责启动、就绪确认、终止、回收与 | gateway 名称/Endpoint 可变，旧清理可能指向新机器 | 固定节点身份与资源 owner，阻止有资源时更换机器 | | 中央出口检查不能证明远端浏览器出网正确 | 从 A/B 实际执行路径分别验证，不复用中央结果冒充 | -尚待 P0 补齐的不是代码小修，而是原生浏览器发行物、目标机器条件、契约批准和性能基线。未执行真实平台或故障测试的项目全部保留为未验证。 +P0 的发行物、目标机器条件、契约和资源参数已记录；性能基线以及未执行的真实平台/代理/故障测试仍全部保留为未验证。 ## 8. 成熟方案参考与证据边界 diff --git a/docs/native-browser-implementation-plan.md b/docs/native-browser-implementation-plan.md index 95cd6dd..05060c0 100644 --- a/docs/native-browser-implementation-plan.md +++ b/docs/native-browser-implementation-plan.md @@ -1,14 +1,14 @@ # 原生浏览器环境：评审后的实施计划 -> 状态：待批准实施。仅文档，本轮未修改业务代码、数据库或运行环境。 -> 依据：[变更评审](native-browser-change-review.md)、[业务基线](plan01.md)。验收：[验证文档](native-browser-verification.md)。 +> 状态：实施 worktree 已按本计划完成 native gateway、控制面契约、runtime-use lease、前端和单机部署代码改造；自动检查与真实平台/代理/LAN 验收分开记录，缺少授权或真实资源时标为阻塞。 +> 依据：[变更评审](native-browser-change-review.md)、[业务基线](plan01.md)。验收：[验证文档](native-browser-verification.md)；记录：[单节点验证记录](evidence/native-browser-verification-2026-09-18.md)。 ## 1. 完成定义 满足以下条件才算完成改造，而不是“可以启动 Chromium”就结束： - 各 gateway 不访问 Docker socket，不创建浏览器容器、镜像层、卷或网络；每个 runtime 在本机运行 Xvfb 与指定版本浏览器。 -- 通过现有控制面管理至少两台机器；账号、运行代次、代理、Profile 与实际节点一致，不自动跨机重建。 +- 本目标先通过现有控制面管理单台机器；账号、运行代次、代理、Profile 与实际节点一致，不自动跨机重建。多节点仅保留稳定契约字段，另立目标。 - 采集任务可按需启停；任务自建进程和临时文件在所有终态回收。清理失败真实可见并可重试。 - 持久登录、Profile、稳定指纹及账号身份在重启和普通浏览器升级后保持；二维码登录、采集结果及逐次确认的人工发送不退化（A02、A10；AC-E1）。 - gateway、控制面重启及监听重连不清除自动响应的事件去重、基线或 UID 冷却；旧事件不重放，迟到事件默认只记录、不自动补发（C02、D04；AC-A9、AC-B1）。 @@ -20,11 +20,10 @@ ```text React → Go control-plane（账号/任务/绑定/业务结果/素材发布） - ├─ gateway A（已有节点登记与路由） - │ ├─ 预安装浏览器版本（所有任务共用只读二进制） - │ ├─ 账号 A 的持久 Profile - │ └─ runtime generation：Xvfb + 浏览器 + 临时目录 - └─ gateway B（同样结构，独立本地资源） + └─ native gateway G（单节点；保留后续节点登记契约） + ├─ 预安装浏览器版本（所有任务共用只读二进制） + ├─ 账号的持久 Profile + └─ runtime generation：Xvfb + 浏览器 + 临时目录 ``` - 继续使用 Go 控制面、Python gateway、PostgreSQL 与现有 CDP/WebSocket。 @@ -35,7 +34,7 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） ## 3. 必要数据和接口变化 -以下为**契约变更提案**，不是已存在的新字段/API。P0 批准后才能实施。 +以下为本实施 worktree 已采用的**单节点契约**；多节点字段保留稳定命名，但跨机调度与恢复不在本目标内。 | 概念 | 变更 | 约束 | | --- | --- | --- | @@ -65,8 +64,8 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） 交付： 1. 确认当前未提交工作由谁负责，将基线固定到可复现提交；在干净、明确基线之上新建实施分支，不擅自 stash/reset 用户改动。 -2. 核实两台 Linux 机器的 systemd、用户服务、Xvfb、字体、浏览器依赖、sandbox、可用磁盘；取得合法原生指纹浏览器并固定版本，指定用于 A02 普通升级验证的源版本与目标版本。 -3. 记录旧方式性能/磁盘基线。默认不运行 Docker；如无历史可信数据，需要用户另行批准旧方式对照运行。 +2. 核实单台 Linux 机器的 systemd、用户服务、Xvfb、字体、浏览器依赖、sandbox、可用磁盘；取得合法原生指纹浏览器并固定版本，指定用于 A02 普通升级验证的源版本与目标版本。 +3. 记录可用的旧方式性能/磁盘基线。默认不运行 Docker；如无历史可信数据，性能改善明确标记未验证，不用新建旧浏览器对照冒充基线。 4. 列出受影响状态码、载荷、字段、配置和移除项，批准第三节契约。明确 gateway 重启后受管 runtime 的恢复行为。 5. 固定验证参数：任务最长时间、清理时间预算、任务使用权续租/过期时间、磁盘下限、并发数、日志保留及资源对比指标。 @@ -127,23 +126,23 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） - 扩展 P2 的 Refine data provider 和交互测试，覆盖 P3 完整资源归属及多节点场景；保留业务结果、清理状态、错误和禁用原因分别可见，不把首个闭环必需的界面工作留到本阶段。 - 增加可重复部署的 gateway/原生 runner 用户服务配置和本机浏览器安装说明。跨重启保留节点 ID、Profile、版本配置和运行清单。 - 若 runtime 允许跨 gateway 重启存活，其临时目录必须由 runtime unit 归属，不能放在 gateway 重启就被 systemd 删除的 `RuntimeDirectory` 中。大型缓存/下载明确使用配置的数据磁盘，不因“临时”二字默认放进 `/run` 的 tmpfs 而转为大量内存占用。 -- 修改 `scripts/dev-backend.mjs`、开发脚本、Dockerfile/Compose 的旧 gateway 相关配置；默认裸启动 Go/Python/Vite，不悄悄 `compose up docker-gateway`。 +- 修改 `scripts/dev-backend.mjs`、开发脚本、Dockerfile/Compose 的旧 gateway 相关配置；默认裸启动 Go/Python/Vite，不悄悄启动 Docker browser gateway。 - 删除 Docker browser wrapper、拉镜像/构建/网络/卷管理及相关测试、配置；PostgreSQL 等仍使用 Docker 的独立部署资产可保留，逐项说明，不做无关清理。 - `AGENTS.md`、README、部署说明、架构说明、E2E 文档全部对齐。删除旧入口，不保留“失败就回退 Docker”。 -放行：验证文档全部必测项有证据；静态检查中浏览器链路没有 Docker 依赖；两节点局域网手工验收通过。 +放行：验证文档全部适用项有证据；静态检查中浏览器链路没有 Docker 依赖；单节点局域网手工验收通过。多节点项记录为后续目标。 ## 5. 改动定位与顺序约束 | 范围 | 主要定位 | 必须一起变化的内容 | | --- | --- | --- | -| gateway 执行 | `cmd/docker_gateway/gateway.py`、`proxy.py`、CDP/平台模块及其测试 | 原生 runner、版本/端口/Profile、unit 生命周期、snapshot、代理恢复 | +| gateway 执行 | `cmd/browser_gateway/gateway.py`、`runtime.py`、`proxy.py`、CDP/平台模块及其测试 | 原生 runner、版本/端口/Profile、unit 生命周期、snapshot、代理恢复 | | 控制面 lifecycle | `cmd/control-plane/hub.go`、`main.go`、hub tests | generation、owner、清理/心跳、多 gateway 错误隔离 | | 采集与监听 | `creator.go`、`creator_events.go`、`internal/creator/source_lease.go`、`content.go` | 使用权、续租、独立收尾、事件边界与身份核对 | | 素材 | `creator_material.go`、`internal/creator/material.go`、素材测试 | 临时目录、不可变发布、token、崩溃清理 | | 数据模型 | `internal/hub/`、`internal/creator/` | 删除 Docker 字段，明确开发重建；业务结果与清理状态分离 | | 前端 | 现有 gateway/环境/镜像/任务相关组件及 data provider | 对应契约、失败/禁用交互；不重做导航和视觉体系 | -| 部署/文档 | `scripts/`、`compose*.yaml`、Dockerfile、`docker/browser-wrapper/`、README、`docs/` | 原生安装和联调；旧路径移除；手工验收地址 | +| 部署/文档 | `scripts/`、`compose*.yaml`、Dockerfile、`deploy/`、README、`docs/` | 原生安装和联调；旧路径移除；手工验收地址 | 顺序限制：产物执行隔离先于孤儿文件清理；本地单任务闭环先于多机压测；所有权和代次先于通用自动清理；不能先删 Docker 实现再留下无法登录的中间交付。 @@ -151,9 +150,9 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） - 每个非平凡行为改动先有会在旧实现下失败的测试；单元覆盖率至少 65%。Go 全量测试、vet、build，生命周期/并发相关变更跑 race；Python 非交互测试与覆盖率；前端从 lockfile 安装并测试、构建。 - 更改 Docker/Compose 文件执行 `docker compose config --quiet`；不默认构建/运行任何 Docker 镜像。旧基线测试需另行得到用户同意。 -- 功能验收不使用浏览器自动化或批量 API 代替用户操作。提供 `0.0.0.0` 监听的本地环境和两台 gateway 地址，由用户按验证文档手工确认。 +- 功能验收不使用浏览器自动化或批量 API 代替用户操作。提供 `0.0.0.0` 监听的本地环境和单节点 gateway 地址，由用户按验证文档手工确认。 - 资源观测、日志读取和单元测试可以自动执行，但不能因此将真实登录、采集、发送或多机故障验收标为通过。 -- 本轮不会启动服务、重建数据库或删除遗留资源。实施完成后才启动可联调环境，并报告实测局域网地址。 +- 本文不把服务启动、数据库重建或遗留资源删除当作自动交付动作；需要真实联调时按部署和验证文档由授权操作者启动，并报告当次局域网地址。 ## 7. 切换与失败恢复 @@ -163,13 +162,13 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） 4. 如新环境验收失败，停止新任务、保留证据、修复再验。紧急恢复旧版只能是人工恢复到明确版本和匹配数据，须取得授权；不是运行时隐式回退。 5. 验收后，经授权回收旧浏览器容器/卷/网络与镜像。只删除确认属于旧 CreatorHub 浏览器的资源，禁止全机 `docker system prune` 或按相似名称批量删除。 -## 8. 实施前批准清单 +## 8. 实施与验收记录 -- [ ] Linux/systemd 与原生指纹浏览器部署条件满足。 -- [ ] 独立 runtime 跨 gateway 重启核对恢复、短任务有界存活策略获准。 -- [ ] 账号 Profile 保留、采集独占/借用边界、无自动跨机迁移获准。 -- [ ] API/schema/配置的破坏性变更及数据重建范围获准。 -- [ ] 明确完整网页远程桌面是否另立需求，不能把 wrapper 内有 x11vnc 当已具备该功能。 -- [ ] 验证参数、旧方式对照是否允许运行 Docker、实测性能放行标准获准。 +- [x] Linux/systemd、Xvfb 与原生指纹浏览器部署条件满足；非 root gateway 已完成 smoke。 +- [x] 独立 runtime 的 gateway 重启恢复和短任务有界清理已通过自动/真实 smoke；正式参数为租约 60 秒、续租 20 秒、清理 30 秒、恢复 60 秒。 +- [x] 账号 Profile 保留、采集独占/借用边界、无自动跨机迁移已写入契约；同 Profile 真实并发仍需授权操作者手工验收。 +- [x] API/schema/配置的破坏性开发切换范围已固定；不保留旧 Docker 浏览器兼容路径。 +- [x] 完整网页远程桌面未作为本目标能力；需要时另立需求，不能把 gateway 的 Xvfb 误称为远程桌面。 +- [x] 20 GB 磁盘下限、1 GB runtime 日志、20 GB Profile 缓存、并发 1/2 和 `0.0.0.0:8082` 已配置并记录。 -在上述批准之前，这份文档只作为实施依据，不构成已经完成的改造。 +自动检查结果见 [单节点验证记录](evidence/native-browser-verification-2026-09-18.md)。真实账号、代理、LAN 和破坏性资源用例仍须由授权操作者完成；未执行项不视为通过。 diff --git a/docs/native-browser-verification.md b/docs/native-browser-verification.md index e12d012..526b3d4 100644 --- a/docs/native-browser-verification.md +++ b/docs/native-browser-verification.md @@ -1,16 +1,17 @@ # 原生浏览器环境：验证与手工验收 -> **当前状态：仅验证计划，下面所有功能、故障、性能用例均未执行。** -> 本轮未构建镜像、启动服务、操作真实账号、终止进程或删除数据。 -> 方案：[变更评审](native-browser-change-review.md)；阶段：[实施计划](native-browser-implementation-plan.md)。 +> **当前状态：单节点代码改造与自动检查通过；真实平台、代理、LAN 和资源验收仍须由授权操作者手工完成。** +> 已完成 native gateway、控制面契约、runtime-use lease、Compose/P4 文档和本地单元测试；本文件只记录可复现的验收步骤，不把测试替身当作真实平台证据。 +> 方案：[变更评审](native-browser-change-review.md)；阶段：[实施计划](native-browser-implementation-plan.md)；当次记录：[单节点验证记录](evidence/native-browser-verification-2026-09-18.md)。 ## 1. 验证边界 - 开发者负责单元/契约检查、构建和启动可联调环境；真实功能由用户手工验证，不使用浏览器自动化或脚本代点网页。 - 读取日志、目录占用、进程/端口和系统指标可以使用命令。自动测试中的替身只证明代码行为，不能证明平台登录、监听、发送或真实代理能力。 -- 默认不构建/运行 Docker。旧方式对照、旧资源删除、故障注入必须单独取得用户同意。 +- 默认不构建/运行 Docker；Compose 仅可作为 PostgreSQL 独立依赖。旧方式对照、资源删除和故障注入必须单独取得用户同意。 - 测试专用账号、Profile、gateway、数据目录及出口；不能强杀用户正在使用的长期账号或清空全机 Docker 资源。 - 任何结果使用“未执行 / 通过 / 失败 / 阻塞”，不得把文档中的期望填成实际结果。 +- 若被测 Chrome 由用户的 `chrome.service` 管理，验证只能通过其已开放的 CDP 连接执行只读/交互检查，不得停止、启动、替换或接管该实例；这类结果不能替代 native gateway 的 runtime 生命周期、控制面持久化和资源清理验收。 ## 2. 前置条件和记录表 @@ -19,21 +20,21 @@ | 项目 | 必填记录 | | --- | --- | | 软件基线 | Git 提交、未提交改动清单、Go/Python/Node/systemd/Xvfb/浏览器版本 | -| 机器 | 控制面 C、gateway A、gateway B 的稳定节点 ID、IP、CPU、内存、磁盘、发行版 | +| 机器 | 单节点控制面 C 与 native gateway G 的稳定节点 ID、IP、CPU、内存、磁盘、发行版 | | 账号 | 专用采集账号、自有监听账号；预期平台 UID，勿记录密码/Cookie | | 浏览器 | 原生发行物来源、许可证、校验值、支持的指纹参数和 sandbox 证据；A02 普通升级的已安装源版本/目标版本 | | 路径 | 各机器 Profile 根目录、runtime 根目录、运行清单、日志、控制面素材目录 | -| 网络 | A/B 各节点的 HTTP/HTTPS/SOCKS4/SOCKS5 测试出口、协议支持的认证配置、预期出口 IP、CDP 可用情况；只记录配置标识，不记录密码 | +| 网络 | 单节点的 HTTP/HTTPS/SOCKS4/SOCKS5 测试出口、协议支持的认证配置、预期出口 IP、CDP 可用情况；只记录配置标识，不记录密码 | | 操作许可 | 是否允许旧 Docker 对照、普通浏览器升级、gateway/控制面重启、任务取消、磁盘故障和测试数据删除；C02/D04 自动响应的测试策略、可控互动/接收账号及授权范围，A10 人工发送另行逐次确认 | -| 时限 | task 最大执行时间、续租间隔/过期时间、清理预算 `T_cleanup`、恢复预算 `T_recover` | -| 资源预算 | 并发数、最低可用磁盘、日志上限；启动 p50/p95 和总内存目标，Profile 缓存预算 | +| 时限 | task 最大执行时间、续租间隔 20 秒、租约 60 秒、清理预算 30 秒、恢复预算 60 秒 | +| 资源预算 | 并发数 1/2、最低可用磁盘 20 GB、日志上限 1 GB；启动 p50/p95 和总内存目标，Profile 缓存预算 20 GB | | 证据位置 | 独立于 runtime 清理目录的本地证据目录，不提交账号敏感数据 | -建议首轮使用并发 1 和 2、短任务正常清理预算 30 秒、服务恢复预算 60 秒；这些是待批准的测试参数，不是已实现默认配置。复杂平台采集的最大时长单独确定，不能因为清理预算而强行截断正常业务。 +首轮使用并发 1 和 2、短任务正常清理预算 30 秒、服务恢复预算 60 秒。复杂平台采集的最大时长单独确定，不能因为清理预算而强行截断正常业务。 ## 3. 开发者检查命令 -以下是**实施完成后的命令**，不是本轮执行记录。gateway 重命名为 `cmd/browser_gateway/` 后才使用该路径；最终脚本需与实际交付一致，不允许测试目录为空仍算通过。 +以下命令是交付检查命令；执行结果另记在证据文件，不把未运行的命令当作通过。 ### Go @@ -53,10 +54,10 @@ go tool cover -func=/tmp/creatorhub-go.cover ```bash python3 -m venv .venv-gateway .venv-gateway/bin/python -m pip install -r requirements-gateway-dev.lock -.venv-gateway/bin/python -m unittest discover -s cmd/browser_gateway -p 'test_*.py' +.venv-gateway/bin/python -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' .venv-gateway/bin/python -m coverage erase -.venv-gateway/bin/python -m coverage run --source=cmd/browser_gateway \ - -m unittest discover -s cmd/browser_gateway -p 'test_*.py' +.venv-gateway/bin/python -m coverage run --source=cmd/browser_gateway --branch \ + -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' .venv-gateway/bin/python -m coverage report --omit='*/test_*.py' --fail-under=65 ``` @@ -79,22 +80,22 @@ npm --prefix web run build docker compose config --quiet ``` -仅在改动 Docker/Compose 时做配置校验，不执行 build/up；有保留的独立开发 Compose 文件时也逐份验证。另行检查浏览器 runtime 路径已无 DockerClient/socket、容器/镜像/卷/网络创建、容器 wrapper 入口、旧 image 字段和“原生失败回退 Docker”；PostgreSQL 的独立部署配置不算漏删。 +仅在改动 Compose 时做配置校验，不执行 build/up；保留的 PostgreSQL 开发 Compose 文件也逐份验证。另行检查浏览器 runtime 路径已无 DockerClient/socket、容器/镜像/卷/网络创建、容器 wrapper 入口、旧 image 字段和“原生失败回退 Docker”；PostgreSQL 的独立部署配置不算漏删。 ## 4. 联调启动与访问 -### 当前可确认的设置 +### 已交付设置 - Go 使用 `LISTEN_ADDR`；联调须设置 `0.0.0.0:8082`。 - Vite 当前 `/api` 代理到 `127.0.0.1:8082`，端口 5173。Vite 必须加 `--host 0.0.0.0`，不能只给 localhost 地址。 -- Python gateway 使用 `LISTEN_ADDR`，多机控制测试可采用各节点 `0.0.0.0:8081`；现有凭据约束保持不变。 -- 当前 `scripts/dev-backend.mjs` 会启动 Compose 依赖，因此在 P4 替换之前，**不能把现有 `pnpm dev` 当作已支持原生 gateway 的命令。** +- native Python gateway 使用 `LISTEN_ADDR=0.0.0.0:8081`，由非 root systemd user service 运行；稳定 `node_id` 从配置或主机身份加载。 +- `scripts/dev-backend.mjs` 只启动 PostgreSQL 并检查 `NATIVE_GATEWAY_ENDPOINT`，不会启动浏览器容器。 -### 实施完成后 +### 单节点执行步骤 -1. 按最终交付的原生安装说明，在 A/B 安装同一已确认浏览器、Xvfb 和 gateway 用户服务。服务名建议固定为 `creatorhub-gateway.service`；它目前尚未交付。 +1. 按部署说明在单节点安装已确认浏览器、Xvfb 和 `creatorhub-browser-gateway.service`。 2. 使用已有、经验证的 PostgreSQL 与凭据配置。不要重新生成现有主密钥；不要把秘密值粘贴到验收报告。 -3. 按交付脚本启动 A/B 的原生 gateway；在控制面登记真实 A/B 地址，并核对返回的节点身份。 +3. 按交付脚本启动单节点 native gateway；在控制面登记真实地址，并核对 `/v1/info` 返回的稳定节点身份。 4. 启动控制面及前端： ```bash @@ -104,9 +105,9 @@ LISTEN_ADDR=0.0.0.0:8082 go run ./cmd/control-plane npm --prefix web run dev -- --host 0.0.0.0 ``` -命令启动后，用 `ip -brief -4 addr` 记录当次实际局域网 IP。用户在另一台设备访问 `http://:5173`；开发者可查看 `http://:8082/healthz`。报告列出 A/B 的实际 Endpoint。 +命令启动后，用 `ip -brief -4 addr` 记录当次实际局域网 IP。用户在另一台设备访问 `http://:5173`；开发者可查看 `http://:8082/healthz`。报告列出单节点 gateway 的实际 Endpoint。 -本轮宿主机曾观察到 `10.1.1.104`，只可作为候选地址，**没有服务已启动的含义**；实施当天重新核实，不预先宣称该地址可用。 +历史观察到的地址不构成可用性证据；实施当天必须重新核实 IP、监听地址和防火墙。 ## 5. 手工功能与清理矩阵 @@ -116,7 +117,7 @@ npm --prefix web run dev -- --host 0.0.0.0 | ID | 手工步骤 | 通过条件 | | --- | --- | --- | -| A01 | 在无 Docker daemon/socket 可用的测试节点创建账号环境，选择原生版本，启动 | 正确 Xvfb/浏览器就绪；未登录时可进入人工登录，不永久停在“正在启动”；无镜像下载、容器、卷、Docker 网络操作 | +| A01 | 在 gateway 节点确认 Docker daemon/socket 不参与运行，创建账号环境，选择已安装 native 版本并启动 | 正确 Xvfb/浏览器就绪；未登录时可进入人工登录，不永久停在“正在启动”；无镜像下载、容器、卷或 Docker 浏览器网络操作 | | A02 | 人工完成二维码登录，记录账号、Profile 标识和指纹；先关闭再启动，再停止并从界面普通升级至另一已安装且获准的版本后启动 | 两个子项均保持原 Profile、指纹和登录身份，无意外重新登录；升级后实际浏览器版本等于所选目标版本，不通过重建环境/重新登录冒充保持（AC-E1） | | A03 | 浏览器停止时，从界面启动一次竞品账号采集，等待结束 | 自动创建任务 runtime，结果保存；`T_cleanup` 内进程、display、端口和临时文件释放 | | A04 | 连续采集作品/评论/线索，再打开结果查看 | 数据完整且仍可读；不能因 runtime 清理丢失业务结果或已发布素材 | @@ -127,9 +128,9 @@ npm --prefix web run dev -- --host 0.0.0.0 | A09 | 按下方代理矩阵分别配置四类代理及其支持的认证配置，改变代理或账号绑定后启动采集 | 每个配置下实际浏览器出口、代理版本、指纹时区/语言等正确；身份不符时拒绝动作；不能用单一无认证代理的成功代替其他配置（AC-E3） | | A10 | 仅在用户准备真实接收账号并逐次确认时验证人工回复/私信 | 保持原有确认与身份核对；重复结果不触发重复发送，不确定状态不冒充成功 | -A02 验证的是原生环境内的普通版本升级，不是首次 Docker 切换或 Profile 迁移；缺少获准版本时标为阻塞。 +A02 验证的是 native 环境内的普通版本升级，不是 Profile 迁移；缺少获准版本时标为阻塞。 -代理矩阵适用于 A09/B03/D05：A/B 每台节点分别列出 HTTP、HTTPS、SOCKS4、SOCKS5，按协议允许的配置验证无认证（如支持）及支持的认证方式；A09/D05 记录成功与实际浏览器出口，B03 记录认证失败（适用时）与网络失败，均不得静默直连或轮换出口。协议本身不支持的认证项说明依据，不作为通过项；缺少测试代理、凭据或尚未实现的承诺能力标为阻塞，不能免测。证据不得包含密码/Cookie。 +代理矩阵适用于 A09/B03：单节点分别列出 HTTP、HTTPS、SOCKS4、SOCKS5，按协议允许的配置验证无认证（如支持）及支持的认证方式；A09 记录成功与实际浏览器出口，B03 记录认证失败（适用时）与网络失败，均不得静默直连或轮换出口。协议本身不支持的认证项说明依据，不作为通过项；缺少测试代理、凭据或尚未实现的承诺能力标为阻塞，不能免测。证据不得包含密码/Cookie。 A10 不能由脚本自动发送，也不能因为只修改 runtime 就免除必要回归；人工逐次确认不能代替 C02/D04 的自动策略验收。 @@ -139,7 +140,7 @@ A10 不能由脚本自动发送，也不能因为只修改 runtime 就免除必 | --- | --- | --- | | B01 | 任务运行中点击取消 | 业务不永久 running；独立清理继续，`T_cleanup` 内完成或明确待清理 | | B02 | 让测试任务超过已批准最大时长 | 超时可见；lease 释放，任务进程/目录回收，不能继续后台写入 | -| B03 | 按代理矩阵逐节点、逐协议分别制造认证失败（协议支持时）和网络失败；另测无效浏览器依赖启动 | 每个适用子项明确失败，无静默直连或出口轮换；已创建的 Xvfb/端口/临时目录逆序回收（AC-E3） | +| B03 | 按代理矩阵逐协议分别制造认证失败（协议支持时）和网络失败；另测无效浏览器路径启动 | 每个适用子项明确失败，无静默直连或出口轮换；已创建的 Xvfb/端口/临时目录逆序回收（AC-E3） | | B04 | 经授权终止该测试 runtime 的浏览器，另测 Xvfb 退出 | 任务失败可见；其余子进程终止；其他 runtime 不受影响 | | B05 | 对测试专用目录制造无法删除条件后结束任务 | 显示“业务结果 + 清理失败”，保留可追溯记录；恢复条件后重试只清资源，不重跑业务 | | B06 | 测试磁盘低于启动阈值；用隔离测试盘验证写满 | 启动前拒绝或运行中明确失败；不删除登录资料/正式素材腾空间；收尾状态可靠 | @@ -169,16 +170,16 @@ C02/D04 必测恢复子项： C05/B09 的精确竞态先用确定性单元测试证明；真机无法可重复制造时标记该子项未执行，不能用“人工未复现”替代回归测试。测试不得添加面向生产的任意执行接口。 -### D. 多机控制 +### D. 多机控制（后续目标，全部未执行） | ID | 步骤 | 通过条件 | | --- | --- | --- | -| D01 | A/B 各创建合法唯一环境，分别启动；停止/清理一侧并观察另一侧 | 按稳定节点和代次执行，不误清另一机器；节点间资源 ID 碰撞另用契约测试覆盖，不为测试放松名称唯一性 | -| D02 | A 失联，B 正常采集/续租/监听 | B 不被 A 的超时拖住；A 显示不可达，不把 A 的账号迁到 B | -| D03 | A 仍有 Profile、运行实例或待清理资源时，尝试把登记地址改成 B | 明确阻止改变机器归属；旧 cleanup 目标仍是 A | -| D04 | A 恢复，旧停止响应/事件迟到，期间已有新代次；按 C02 恢复子项核对既有自动响应与冷却记录 | 旧响应不能关闭/污染新实例；事件和身份边界重新确认；原 UID 冷却保留，重复旧事件及迟到事件不触发额外动作，B 的记录与正常监听不受影响（AC-A9、AC-B1） | -| D05 | A/B 配置不同出口，按代理矩阵逐节点检查四类代理及支持的认证配置下目标浏览器的实际出网 | 各适用配置均有对应 gateway 的浏览器出口证据，不能只显示控制面测得 IP；失败子项与 B03 对照，无直连或轮换（AC-E3） | -| D06 | B 未安装账号指定浏览器版本 | 明确显示不可用，不自动换普通 Chromium、其他版本或另一节点 | +| D01 | 后续目标：两台节点各创建合法唯一环境，分别启动；停止/清理一侧并观察另一侧 | 本目标不执行；保留稳定节点和代数字段，后续目标按契约验收 | +| D02 | 后续目标：一台节点失联，另一台正常采集/续租/监听 | 本目标不执行；不得以单节点重启替代跨机证据 | +| D03 | 后续目标：带 Profile、运行实例或待清理资源时改变机器归属 | 本目标不执行；不得删除原节点资源 | +| D04 | 后续目标：跨节点恢复时旧响应/事件迟到且已有新代次 | 本目标不执行；单节点旧代次保护由 C07 和自动测试覆盖 | +| D05 | 后续目标：多节点配置不同出口并按代理矩阵核验 | 本目标不执行；单节点代理适用项在 A09/B03 验收 | +| D06 | 后续目标：另一节点未安装账号指定浏览器版本 | 本目标不执行；单节点版本缺失在 A01/B03 验收 | 抖音先完成全部适用用例；小红书按相同生命周期复验。尚未实现的平台能力记为阻塞，不用 Mock、轮询或另一平台成功代替。 @@ -187,10 +188,10 @@ C05/B09 的精确竞态先用确定性单元测试证明；真机无法可重复 ### 6.1 测量方法 1. 固定机器、浏览器版本、指纹、账号、代理、任务内容、并发和依赖状态；除生命周期方式外尽量保持一致。不要拿不同浏览器或不同网络的结果相减。 -2. 冷启动指本次机器/服务启动后的第一次浏览器启动，和后续热启动分开记录；不擅自全机清理 page cache。安装、镜像拉取或下载浏览器耗时另列，不混入日常任务启动。 +2. 冷启动指本次机器/服务启动后的第一次浏览器启动，和后续热启动分开记录；不擅自全机清理 page cache。安装或下载浏览器耗时另列，不混入日常任务启动。 3. 每种方式在并发 1 下至少 20 次有效启动，并在并发 2 下至少 10 组。按相同节奏执行，遵守平台限制。遇验证码/限流单独记录，不删去异常后只展示好看的数值。 4. 时间从用户启动请求被接收到 CDP+代理+账号就绪；另外记录完整任务、终止到资源清理完成耗时。不能只测进程创建或 Xvfb 启动时间。 -5. 内存统计整个 runtime 的浏览器子进程、Xvfb、可选桌面，以及 gateway 增量；旧方案同样统计完整进程树及 Docker/代理相关增量。优先使用 cgroup 峰值或一致口径的 PSS；RSS 合计会重复计算共享页，必须注明。 +5. 内存统计整个 runtime 的浏览器子进程、Xvfb、可选桌面，以及 gateway 增量；如有获准的旧基线，旧方案也必须按完整进程树和代理增量的同一口径统计。优先使用 cgroup 峰值或一致口径的 PSS；RSS 合计会重复计算共享页，必须注明。 6. 记录每轮之前、业务结束、清理完成三个时刻：活跃进程/unit、端口、runtime/Profile/日志/素材字节数。使用 `ps`、`ss`、`systemctl show`、`du -sb` 等只读工具；共享内存和 X socket 也须核对。 7. 原生路径连续完成至少 30 个短任务，覆盖取消和失败；再观察一轮长期监听与采集并行。平台受限时可用明确标识的本地测试页面单独测生命周期，但不得拿它替代真实平台功能验收。 diff --git a/docs/plan01.md b/docs/plan01.md index 2af9adb..ace8a95 100644 --- a/docs/plan01.md +++ b/docs/plan01.md @@ -44,9 +44,9 @@ | 页面入口 | 账号、任务、审计、环境、代理、镜像、网关等页面 | 竞品池、作品、素材仿写、评论线索、响应策略、私信会话尚无完整入口 | [页面路由](../web/src/main.jsx) | | 账号 | 平台账号 ID、标签、Cookie、暂停/恢复等 | 实名资料、登录用户名/密码、备注、人工业务状态、人工登录与大小号关系 | [账号页面](../web/src/AccountsPage.jsx)、[账号存储](../internal/phasea/store.go)、[账号接口](../cmd/control-plane/phasea.go) | | 平台范围 | 登记项还包含公众号、快手 | 新业务仅承诺抖音、小红书；其他登记项不是业务能力验收结果，也不因此要求删除无关已有功能 | [账号页面](../web/src/AccountsPage.jsx) | -| 作品读取 | 有抖音连接器、指标字段与测试 | 连接器只核验登录者自身、读取首批 20 条，未接成运行中的竞品采集链路；需目标账号解析、分页、保存、查询和定时更新 | [抖音连接器](../internal/douyin/connector.go)、[连接器测试](../internal/douyin/connector_test.go)、[网关读取限制](../cmd/docker_gateway/douyin.py) | +| 作品读取 | 有抖音连接器、指标字段与测试 | 连接器只核验登录者自身、读取首批 20 条，未接成运行中的竞品采集链路；需目标账号解析、分页、保存、查询和定时更新 | [抖音连接器](../internal/douyin/connector.go)、[连接器测试](../internal/douyin/connector_test.go)、[网关读取限制](../cmd/browser_gateway/douyin.py) | | 指纹与环境 | 结构化指纹、独立 Profile、启动/停止/升级及代理接入 | 默认固定 seed 不是自动分配；需首次自动生成、地区匹配及稳定性验证 | [环境页面](../web/src/BrowsersPage.jsx)、[指纹参数](../internal/hub/fingerprint.go)、[环境存储](../internal/hub/environment.go) | -| 代理 | 添加、列表、手动检测、停用、实际转发 | 补齐编辑、删除、重新启用及引用约束；不增加自动轮换 | [代理页面](../web/src/NetworkExitsPage.jsx)、[代理转发](../cmd/docker_gateway/proxy.py) | +| 代理 | 添加、列表、手动检测、停用、实际转发 | 补齐编辑、删除、重新启用及引用约束；不增加自动轮换 | [代理页面](../web/src/NetworkExitsPage.jsx)、[代理转发](../cmd/browser_gateway/proxy.py) | | 任务与发送 | 有草稿确认、任务记录、Mock 执行 | 不能作为真实回复、私信、点赞、关注、转发成功的证据；需接通实际平台执行及结果核验 | [任务页面](../web/src/TasksPage.jsx)、[任务接口](../cmd/control-plane/phasea.go) | | 事件、线索、私信、AI | 未发现完整可运行链路 | 均需新增业务能力，不能把旧文档、类型定义或模拟返回记为已实现 | [页面路由](../web/src/main.jsx)、[抖音连接器](../internal/douyin/connector.go) | diff --git a/docs/python-gateway-branch-review.md b/docs/python-gateway-branch-review.md index 3e792f8..d491d4e 100644 --- a/docs/python-gateway-branch-review.md +++ b/docs/python-gateway-branch-review.md @@ -1,6 +1,6 @@ # Python 网关分支审查与修正清单 -日期：2026-09-13。状态：**审查与代码修正已完成；本地门禁已通过，真实平台及供应商证据仍按外部验收项单独保留。** +日期：2026-09-13。状态：**历史分支审查已完成；本文保留迁移前 `cmd/docker_gateway`、Docker wrapper 和容器生命周期路径作为问题证据，不是当前生产入口。当前入口与状态以 `cmd/browser_gateway`、部署说明和 native 验证记录为准。** ## 1. 范围、结论与证据边界 diff --git a/docs/research/xhs-all-in-one.md b/docs/research/xhs-all-in-one.md index 855c76d..f89b4d0 100644 --- a/docs/research/xhs-all-in-one.md +++ b/docs/research/xhs-all-in-one.md @@ -230,7 +230,7 @@ CreatorHub 不应承诺“稳定”“不封号”或“无法识别”，也不 - `internal/creator/actions.go`：已有事件永久去重、策略顺序、同 UID 冷却，以及 AI 失败/结果不明不补发。 - `cmd/control-plane/creator.go`：已有写操作前身份核对、目标映射和结果证据保存。 - `cmd/control-plane/creator_material.go`：已有人工选取、原子媒体文件、大小/HTML 校验、ffprobe/ffmpeg 和显式转写失败步骤。 -- `cmd/control-plane/creator_events.go` 与 `cmd/docker_gateway/douyin.py`：已有监听代际、baseline、gap、断连恢复、delivery ACK/retry 的抖音实现；小红书不能直接沿用为已验证能力。 +- `cmd/control-plane/creator_events.go` 与 `cmd/browser_gateway/douyin.py`：已有监听代际、baseline、gap、断连恢复、delivery ACK/retry 的抖音实现；小红书不能直接沿用为已验证能力。 - `docs/plan01.md`：已明确先抖音后小红书、逐平台真实验收，以及读取/写入/事件的成功、失败、不明边界。 ### 5.2 当前小红书缺口 diff --git a/internal/creator/actions.go b/internal/creator/actions.go index 6cf1dfa..551df6f 100644 --- a/internal/creator/actions.go +++ b/internal/creator/actions.go @@ -834,7 +834,7 @@ func contains(values []string, want string) bool { } func (s *Store) scanAccountTx(ctx context.Context, tx *sql.Tx, id string, result *AccountProfile) error { var checkedAt sql.NullTime - if err := tx.QueryRowContext(ctx, accountProfileQuery(), id).Scan(&result.ID, &result.Name, &result.Platform, &result.PlatformAccountKey, &result.AuthorizationStatus, &result.RuntimeStatus, &result.LoginUsername, &result.PasswordConfigured, &result.RealNameStatus, &result.RealName, &result.IdentityNumber, &result.Note, &result.BusinessStatus, &result.BigAccount, &result.ReplyRequirements, &result.LoginStatus, &result.LoginReason, &checkedAt, &result.CooldownSeconds, &result.UpdatedAt); err != nil { + if err := tx.QueryRowContext(ctx, accountProfileQuery(), id).Scan(&result.ID, &result.Name, &result.Platform, &result.PlatformAccountKey, &result.AuthorizationKind, &result.AuthorizationStatus, &result.RuntimeStatus, &result.LoginUsername, &result.PasswordConfigured, &result.RealNameStatus, &result.RealName, &result.IdentityNumber, &result.Note, &result.BusinessStatus, &result.BigAccount, &result.ReplyRequirements, &result.LoginStatus, &result.LoginReason, &checkedAt, &result.CooldownSeconds, &result.UpdatedAt); err != nil { return rowError(err) } result.LoginCheckedAt = nullableTime(checkedAt) diff --git a/internal/creator/store.go b/internal/creator/store.go index cd31312..16d2b1d 100644 --- a/internal/creator/store.go +++ b/internal/creator/store.go @@ -67,9 +67,9 @@ var migration032 string //go:embed migrations/033_competitor_tags.sql var migration033 string -// schema_migration is shared by phase-a, hub, and creator. Hub already used -// version 33, so version 34 repairs creator competitor tags when creator -// migration 33 was skipped. +// Creator migrations use their own history table. Phase-a and hub retain a +// shared schema_migration table for their schemas, but their numeric versions +// overlap with the creator migration files and must not suppress one another. // //go:embed migrations/034_competitor_tags_repair.sql var migration034 string @@ -147,7 +147,7 @@ func (s *Store) migrate(ctx context.Context) error { if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1542738017)`); err != nil { return errors.New("lock creator schema migration") } - if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { + if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS creator_schema_migration (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { return errors.New("create creator schema migration table") } migrations := []struct { @@ -176,7 +176,7 @@ func (s *Store) migrate(ctx context.Context) error { } for _, migration := range migrations { var applied bool - if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM creator_schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { return errors.New("read creator schema migration state") } if applied { @@ -185,7 +185,7 @@ func (s *Store) migrate(ctx context.Context) error { if _, err := tx.ExecContext(ctx, migration.sql); err != nil { return fmt.Errorf("apply creator schema migration %d: %w", migration.version, err) } - if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migration (version) VALUES ($1)`, migration.version); err != nil { + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_schema_migration (version) VALUES ($1)`, migration.version); err != nil { return fmt.Errorf("record creator schema migration %d: %w", migration.version, err) } } diff --git a/internal/douyin/connector.go b/internal/douyin/connector.go index 0fd1365..0308be7 100644 --- a/internal/douyin/connector.go +++ b/internal/douyin/connector.go @@ -327,10 +327,26 @@ func parseIdentity(body []byte) (identityEnvelope, bool) { return identity, true } +type douyinBool bool + +func (value *douyinBool) UnmarshalJSON(data []byte) error { + var boolean bool + if err := json.Unmarshal(data, &boolean); err == nil { + *value = douyinBool(boolean) + return nil + } + var numeric int + if err := json.Unmarshal(data, &numeric); err == nil && (numeric == 0 || numeric == 1) { + *value = douyinBool(numeric == 1) + return nil + } + return errors.New("douyin boolean must be true, false, 0, or 1") +} + type worksEnvelope struct { - StatusCode *int `json:"status_code"` - HasMore *bool `json:"has_more"` - MaxCursor *int64 `json:"max_cursor"` + StatusCode *int `json:"status_code"` + HasMore *douyinBool `json:"has_more"` + MaxCursor *int64 `json:"max_cursor"` Works []struct { ID string `json:"aweme_id"` Description string `json:"desc"` @@ -365,7 +381,7 @@ func parseWorksPage(body []byte) ([]Work, bool, *int64, bool) { DiggCount: *candidate.Statistics.DiggCount, CommentCount: *candidate.Statistics.CommentCount, ShareCount: *candidate.Statistics.ShareCount, PlayCount: *candidate.Statistics.PlayCount}) } - return works, *envelope.HasMore, envelope.MaxCursor, true + return works, bool(*envelope.HasMore), envelope.MaxCursor, true } func parseWorks(body []byte) ([]Work, bool, bool) { diff --git a/internal/douyin/creator_collector.go b/internal/douyin/creator_collector.go index f773704..54b6d6e 100644 --- a/internal/douyin/creator_collector.go +++ b/internal/douyin/creator_collector.go @@ -258,7 +258,7 @@ func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) { } items = append(items, creator.CommentInput{Platform: creator.PlatformDouyin, CommentKey: item.ID, AuthorUID: item.UserUID, AuthorName: item.UserName, Content: item.Text, PublishedAt: published, CommentType: "top_level"}) } - page := creator.CommentPage{Items: items, HasMore: *envelope.HasMore} + page := creator.CommentPage{Items: items, HasMore: bool(*envelope.HasMore)} if envelope.Cursor != nil { page.NextCursor = strconv.FormatInt(*envelope.Cursor, 10) } @@ -283,7 +283,7 @@ func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bo return nil, false, nil, false } var envelope worksEnvelope - if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 || *envelope.HasMore && envelope.MaxCursor == nil { + if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 || bool(*envelope.HasMore) && envelope.MaxCursor == nil { return nil, false, nil, false } items := make([]creatorWorkPageItem, 0, len(envelope.Works)) @@ -312,13 +312,13 @@ func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bo } items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares}) } - return items, *envelope.HasMore, envelope.MaxCursor, true + return items, bool(*envelope.HasMore), envelope.MaxCursor, true } type commentEnvelope struct { - StatusCode *int `json:"status_code"` - HasMore *bool `json:"has_more"` - Cursor *int64 `json:"cursor"` + StatusCode *int `json:"status_code"` + HasMore *douyinBool `json:"has_more"` + Cursor *int64 `json:"cursor"` Comments []struct { ID string `json:"cid"` Text string `json:"text"` diff --git a/internal/douyin/creator_collector_test.go b/internal/douyin/creator_collector_test.go index 0b2d532..6b77033 100644 --- a/internal/douyin/creator_collector_test.go +++ b/internal/douyin/creator_collector_test.go @@ -92,6 +92,14 @@ func TestParseCreatorWorksPageMarksInvalidTimestamp(t *testing.T) { } } +func TestParseCreatorWorksPageAcceptsNumericHasMore(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":0,"aweme_list":[]}`) + works, hasMore, cursor, ok := parseCreatorWorksPage(body) + if !ok || hasMore || cursor != nil || len(works) != 0 { + t.Fatalf("numeric has_more was not normalized: ok=%v hasMore=%v cursor=%v works=%d", ok, hasMore, cursor, len(works)) + } +} + func TestParseCreatorWorksPageKeepsPartialMetadata(t *testing.T) { body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"partial","statistics":{"digg_count":7}}]}`) works, hasMore, cursor, ok := parseCreatorWorksPage(body) diff --git a/internal/hub/environment.go b/internal/hub/environment.go index bf7dc1a..4c6c844 100644 --- a/internal/hub/environment.go +++ b/internal/hub/environment.go @@ -16,6 +16,7 @@ import ( ) var exitIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`) +var nodeIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`) type NetworkExit struct { ID string `json:"id"` @@ -62,6 +63,16 @@ type EnvironmentContext struct { RuntimeInstanceID string `json:"runtime_instance_id,omitempty"` RuntimeID string `json:"runtime_id,omitempty"` RuntimeNetworkID string `json:"runtime_network_id,omitempty"` + RuntimeNodeID string `json:"runtime_node_id,omitempty"` +} + +type RuntimeUseLease struct { + Token string `json:"token"` + RuntimeInstanceID string `json:"runtime_instance_id"` + OwnerID string `json:"owner_id"` + Purpose string `json:"purpose"` + TaskID string `json:"task_id,omitempty"` + LeaseUntil time.Time `json:"lease_until"` } type EnvironmentAction struct { @@ -72,8 +83,8 @@ type EnvironmentAction struct { NetworkExitID string RuntimeInstanceID string BindingVersion int64 - OldImageVersion string - NewImageVersion string + OldBrowserVersion string + NewBrowserVersion string Outcome string ReasonCode string } @@ -430,7 +441,7 @@ func invalidateAccountsForExit(ctx context.Context, tx *sql.Tx, exitID string) ( func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID string) (EnvironmentContext, bool, error) { env.Alias, env.Name = strings.TrimSpace(env.Alias), strings.TrimSpace(env.Name) if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || !aliasPattern.MatchString(accountID) || - (exitID != "" && !exitIDPattern.MatchString(exitID)) || !gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) || + (exitID != "" && !exitIDPattern.MatchString(exitID)) || !gatewayNamePattern.MatchString(env.Gateway) || !browserVersionPattern.MatchString(env.BrowserVersion) || env.Fingerprint.ProxyServer != "" { return EnvironmentContext{}, false, ErrInvalid } @@ -454,7 +465,7 @@ func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID s return EnvironmentContext{}, false, errors.New("commit existing environment lookup") } context, err := s.GetEnvironmentContext(ctx, env.Alias) - if err != nil || context.Name != env.Name || context.Gateway != env.Gateway || context.ImageVersion != env.ImageVersion || context.Fingerprint != env.Fingerprint { + if err != nil || context.Name != env.Name || context.Gateway != env.Gateway || context.BrowserVersion != env.BrowserVersion || context.Fingerprint != env.Fingerprint { return EnvironmentContext{}, false, ErrConflict } return context, false, nil @@ -464,13 +475,13 @@ func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID s } var created string if err := tx.QueryRowContext(ctx, ` - INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) SELECT $1, $2, $3, image.version, $5 - FROM browser_image image, social_account account + FROM browser_version image, social_account account WHERE image.version = $4 AND image.enabled AND account.id = $6 AND account.status = 'paused' AND account.authorization_status = 'authorized' AND ($7 = '' OR EXISTS (SELECT 1 FROM network_exit WHERE id = $7 AND health_status = 'healthy')) - RETURNING alias`, env.Alias, env.Name, env.Gateway, env.ImageVersion, encoded, accountID, exitID).Scan(&created); err != nil { + RETURNING alias`, env.Alias, env.Name, env.Gateway, env.BrowserVersion, encoded, accountID, exitID).Scan(&created); err != nil { return EnvironmentContext{}, false, rowError(err) } if _, err := tx.ExecContext(ctx, ` @@ -504,7 +515,7 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro var runtimeInstanceID, runtimeID, runtimeNetworkID, cleanupInstanceID, cleanupRuntimeID, cleanupNetworkID sql.NullString var cleanupBindingVersion sql.NullInt64 err = tx.QueryRowContext(ctx, ` - SELECT environment.alias, environment.name, environment.gateway_name, environment.image_version, + SELECT environment.alias, environment.name, environment.gateway_name, environment.browser_version, environment.fingerprint, environment.created_at, binding.account_id, account.status, account.authorization_status, binding.id, binding.version, binding.runtime_cleanup_pending, binding.runtime_cleanup_binding_version, @@ -515,21 +526,21 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro COALESCE(network.health_status, 'unchecked'), COALESCE(network.last_check_reason, ''), COALESCE(network.version, 0), network.last_checked_at, COALESCE(network.created_at, to_timestamp(0)), COALESCE(network.updated_at, to_timestamp(0)), - runtime.id, runtime.runtime_id, runtime.network_id + runtime.id, runtime.runtime_id, runtime.network_id, COALESCE(runtime.node_id, '') FROM browser_env environment JOIN environment_binding binding ON binding.browser_env_alias = environment.alias JOIN social_account account ON account.id = binding.account_id LEFT JOIN network_exit network ON network.id = binding.network_exit_id LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL WHERE environment.alias = $1`, alias). - Scan(&result.Alias, &result.Name, &result.Gateway, &result.ImageVersion, &encoded, &result.CreatedAt, + Scan(&result.Alias, &result.Name, &result.Gateway, &result.BrowserVersion, &encoded, &result.CreatedAt, &result.AccountID, &result.AccountStatus, &result.AuthorizationStatus, &result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending, &cleanupBindingVersion, &cleanupInstanceID, &cleanupRuntimeID, &cleanupNetworkID, &result.Exit.ID, &result.Exit.Protocol, &result.Exit.Host, &result.Exit.Port, &expectedIP, &result.Exit.ExpectedRegion, &observedIP, &result.Exit.ObservedRegion, &result.Exit.HealthStatus, &result.Exit.LastCheckReason, &result.Exit.Version, &checked, &result.Exit.CreatedAt, &result.Exit.UpdatedAt, - &runtimeInstanceID, &runtimeID, &runtimeNetworkID) + &runtimeInstanceID, &runtimeID, &runtimeNetworkID, &result.RuntimeNodeID) if err != nil { return EnvironmentContext{}, rowError(err) } @@ -566,6 +577,143 @@ func (s *Store) GetEnvironmentContextForAccount(ctx context.Context, accountID s return s.GetEnvironmentContext(ctx, alias) } +const runtimeUseLeaseDuration = time.Minute + +func validRuntimeUse(purpose, ownerID, taskID string) bool { + if purpose != "task" && purpose != "listener" || ownerID == "" || len(ownerID) > 128 || !exitIDPattern.MatchString(ownerID) { + return false + } + return taskID == "" || exitIDPattern.MatchString(taskID) +} + +// AcquireRuntimeUse reserves a short-lived task or long-lived listener right on +// an already running browser. It deliberately does not own the browser runtime +// itself; stopping that runtime revokes active use leases and callers observe +// the loss on their next renewal. +func (s *Store) AcquireRuntimeUse(ctx context.Context, alias, purpose, ownerID, taskID string) (RuntimeUseLease, error) { + if !aliasPattern.MatchString(alias) || !validRuntimeUse(purpose, ownerID, taskID) { + return RuntimeUseLease{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return RuntimeUseLease{}, errors.New("begin runtime use acquisition") + } + defer tx.Rollback() + var runtimeInstanceID string + err = tx.QueryRowContext(ctx, ` + SELECT runtime.id + FROM environment_binding binding + JOIN runtime_instance runtime ON runtime.binding_id = binding.id + JOIN social_account account ON account.id = binding.account_id + WHERE binding.browser_env_alias = $1 AND runtime.released_at IS NULL + AND runtime.lease_until > now() AND NOT binding.runtime_cleanup_pending + AND account.status = 'active' AND account.authorization_status = 'authorized' + FOR UPDATE OF binding, runtime`, alias).Scan(&runtimeInstanceID) + if errors.Is(err, sql.ErrNoRows) { + return RuntimeUseLease{}, ErrConflict + } + if err != nil { + return RuntimeUseLease{}, publicDatabaseError(err) + } + token := "runtime-use-" + newHubID() + var lease RuntimeUseLease + err = tx.QueryRowContext(ctx, ` + INSERT INTO runtime_use_lease (token, runtime_instance_id, owner_id, purpose, task_id, lease_until) + VALUES ($1, $2, $3, $4, NULLIF($5, ''), now() + $6::interval) + RETURNING token, runtime_instance_id, owner_id, purpose, COALESCE(task_id, ''), lease_until`, + token, runtimeInstanceID, ownerID, purpose, taskID, runtimeUseLeaseDuration.String()).Scan( + &lease.Token, &lease.RuntimeInstanceID, &lease.OwnerID, &lease.Purpose, &lease.TaskID, &lease.LeaseUntil) + if err != nil { + return RuntimeUseLease{}, publicDatabaseError(err) + } + if err := commitHub(tx); err != nil { + return RuntimeUseLease{}, err + } + return lease, nil +} + +func (s *Store) RenewRuntimeUse(ctx context.Context, token string) (RuntimeUseLease, error) { + if !exitIDPattern.MatchString(token) { + return RuntimeUseLease{}, ErrInvalid + } + var lease RuntimeUseLease + err := s.db.QueryRowContext(ctx, ` + UPDATE runtime_use_lease use_lease + SET lease_until = now() + interval '1 minute' + FROM runtime_instance runtime + WHERE use_lease.token = $1 AND use_lease.released_at IS NULL + AND use_lease.lease_until > now() AND runtime.id = use_lease.runtime_instance_id + AND runtime.released_at IS NULL AND runtime.lease_until > now() + RETURNING use_lease.token, use_lease.runtime_instance_id, use_lease.owner_id, + use_lease.purpose, COALESCE(use_lease.task_id, ''), use_lease.lease_until`, token).Scan( + &lease.Token, &lease.RuntimeInstanceID, &lease.OwnerID, &lease.Purpose, &lease.TaskID, &lease.LeaseUntil) + if errors.Is(err, sql.ErrNoRows) { + return RuntimeUseLease{}, ErrConflict + } + if err != nil { + return RuntimeUseLease{}, publicDatabaseError(err) + } + return lease, nil +} + +func (s *Store) ReleaseRuntimeUse(ctx context.Context, token string) error { + if !exitIDPattern.MatchString(token) { + return ErrInvalid + } + result, err := s.db.ExecContext(ctx, ` + UPDATE runtime_use_lease SET released_at = now() + WHERE token = $1 AND released_at IS NULL`, token) + if err != nil { + return publicDatabaseError(err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 1 { + return nil + } + var exists bool + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM runtime_use_lease WHERE token = $1)`, token).Scan(&exists); err != nil { + return publicDatabaseError(err) + } + if !exists { + return ErrNotFound + } + return nil +} + +func (s *Store) SetRuntimeNode(ctx context.Context, alias, runtimeID, nodeID string) error { + if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || !nodeIDPattern.MatchString(nodeID) { + return ErrInvalid + } + result, err := s.db.ExecContext(ctx, ` + UPDATE runtime_instance runtime + SET node_id = $3 + FROM environment_binding binding + WHERE binding.browser_env_alias = $1 AND runtime.id = $2 AND runtime.binding_id = binding.id + AND runtime.released_at IS NULL`, alias, runtimeID, nodeID) + if err != nil { + return publicDatabaseError(err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return ErrConflict + } + return nil +} + +func releaseRuntimeUseLeases(ctx context.Context, tx *sql.Tx, runtimeInstanceID string) error { + if runtimeInstanceID == "" { + return nil + } + _, err := tx.ExecContext(ctx, `UPDATE runtime_use_lease SET released_at = now() WHERE runtime_instance_id = $1 AND released_at IS NULL`, runtimeInstanceID) + return err +} + func releaseExpiredRuntime(ctx context.Context, tx *sql.Tx, bindingID string) error { var accountID, alias, runtimeInstanceID string var exitID sql.NullString @@ -584,6 +732,9 @@ func releaseExpiredRuntime(ctx context.Context, tx *sql.Tx, bindingID string) er if err != nil { return err } + if err := releaseRuntimeUseLeases(ctx, tx, runtimeInstanceID); err != nil { + return errors.New("release runtime use leases") + } return appendRuntimeAudit(ctx, tx, "runtime_released", accountID, alias, exitID.String, runtimeInstanceID, bindingVersion) } @@ -670,8 +821,8 @@ func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID if runtimeID != "" { runtimeInstanceID := "runtime-" + newHubID() if _, err := tx.ExecContext(ctx, ` - INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, lease_until) - VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), now() + interval '1 minute')`, + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, owner_id, purpose, lease_token, node_id, lease_until) + VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), 'control-plane', 'session', 'runtime-lease-' || $1, '', now() + interval '1 minute')`, runtimeInstanceID, accountID, bindingID, expectedBindingVersion+1, runtimeID, networkID); err != nil { return EnvironmentContext{}, publicDatabaseError(err) } @@ -732,15 +883,15 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi if existingInstanceID == "" { existingInstanceID = "runtime-" + newHubID() if _, err := tx.ExecContext(ctx, ` - INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, lease_until) - VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), now() + interval '1 minute')`, + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, owner_id, purpose, lease_token, node_id, lease_until) + VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), 'control-plane', 'session', 'runtime-lease-' || $1, '', now() + interval '1 minute')`, existingInstanceID, accountID, bindingID, bindingVersion, runtimeID, networkID); err != nil { return EnvironmentContext{}, publicDatabaseError(err) } if err := appendRuntimeAudit(ctx, tx, "runtime_bound", accountID, alias, exitID, existingInstanceID, bindingVersion); err != nil { return EnvironmentContext{}, err } - } else if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 minute' WHERE id = $1`, existingInstanceID); err != nil { + } else if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET owner_id = 'control-plane', purpose = 'session', lease_until = now() + interval '1 minute' WHERE id = $1`, existingInstanceID); err != nil { return EnvironmentContext{}, errors.New("renew environment runtime") } if err := commitHub(tx); err != nil { @@ -780,6 +931,9 @@ func (s *Store) ReleaseRuntime(ctx context.Context, environment EnvironmentConte if err != nil { return errors.New("release environment runtime") } + if err := releaseRuntimeUseLeases(ctx, tx, runtimeInstanceID); err != nil { + return errors.New("release runtime use leases") + } if err := appendRuntimeAudit(ctx, tx, "runtime_released", accountID, alias, exitID.String, runtimeInstanceID, bindingVersion); err != nil { return err } @@ -855,6 +1009,9 @@ func (s *Store) SetRuntimeCleanupPending(ctx context.Context, environment Enviro if affected, err := result.RowsAffected(); err != nil || affected != 1 { return ErrConflict } + if err := releaseRuntimeUseLeases(ctx, tx, runtimeInstanceID); err != nil { + return errors.New("release runtime use leases") + } if err := appendRuntimeAudit(ctx, tx, "runtime_released", accountID, alias, exitID.String, runtimeInstanceID, environment.BindingVersion); err != nil { return err } @@ -895,20 +1052,20 @@ func appendRuntimeAudit(ctx context.Context, tx *sql.Tx, eventType, accountID, a func (s *Store) AppendEnvironmentAction(ctx context.Context, eventType string, action EnvironmentAction) error { if (eventType != "environment_action_requested" && eventType != "environment_action_finished") || !exitIDPattern.MatchString(action.OperationID) || action.Action == "" || action.ReasonCode == "" || - (action.OldImageVersion != "" && !imageVersionPattern.MatchString(action.OldImageVersion)) || - (action.NewImageVersion != "" && !imageVersionPattern.MatchString(action.NewImageVersion)) || + (action.OldBrowserVersion != "" && !browserVersionPattern.MatchString(action.OldBrowserVersion)) || + (action.NewBrowserVersion != "" && !browserVersionPattern.MatchString(action.NewBrowserVersion)) || (eventType == "environment_action_finished" && action.Outcome != "succeeded" && action.Outcome != "failed" && action.Outcome != "unknown") { return ErrInvalid } _, err := s.db.ExecContext(ctx, ` INSERT INTO audit_event (event_type, account_id, browser_env_alias, network_exit_id, runtime_instance_id, - binding_version, actor, reason_code, operation_id, action, outcome, old_image_version, new_image_version) + binding_version, actor, reason_code, operation_id, action, outcome, old_browser_version, new_browser_version) VALUES ($1, NULLIF($2, ''), NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''), NULLIF($6, 0), 'local-user', $7, $8, $9, NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, ''))`, eventType, action.AccountID, action.BrowserEnvAlias, action.NetworkExitID, action.RuntimeInstanceID, action.BindingVersion, action.ReasonCode, action.OperationID, action.Action, action.Outcome, - action.OldImageVersion, action.NewImageVersion) + action.OldBrowserVersion, action.NewBrowserVersion) if err != nil { return errors.New("append environment action") } diff --git a/internal/hub/migration_test.go b/internal/hub/migration_test.go index 929eb84..46fd2e0 100644 --- a/internal/hub/migration_test.go +++ b/internal/hub/migration_test.go @@ -33,8 +33,10 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal(err) } defer db.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version BETWEEN 1 AND 16`, 16) - assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('social_account', 'browser_env', 'network_exit', 'environment_binding')`, 4) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version BETWEEN 1 AND 17`, 17) + assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('social_account', 'browser_env', 'browser_version', 'network_exit', 'environment_binding')`, 5) + assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'browser_image'`, 0) + assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'browser_version' AND column_name = 'browser_path'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'social_account' AND column_name IN ('name', 'tags')`, 2) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'social_account' AND column_name = 'cookies'`, 0) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name = 'runtime_cleanup_pending'`, 1) @@ -43,7 +45,7 @@ func TestUnifiedAccountMigration(t *testing.T) { store = openFullyMigratedHub(t, ctx, testURL) store.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version BETWEEN 1 AND 16`, 16) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version BETWEEN 1 AND 17`, 17) }) t.Run("legacy migration 013 without account secrets is repaired forward", func(t *testing.T) { @@ -152,8 +154,8 @@ func TestUnifiedAccountMigration(t *testing.T) { VALUES ('upgrade', 'credential-upgrade', 'mock', 'upgrade', 'owned', 'authorized', 'paused'); INSERT INTO gateway (name, endpoint, token) VALUES ('upgrade-gateway', 'http://127.0.0.1:8081', 'upgrade-gateway-token'); - INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1'); - INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + INSERT INTO browser_version (version, browser_path) VALUES ('1', '/opt/creatorhub/browsers/1'); + INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES ('upgrade', 'Upgrade', 'upgrade-gateway', '1', '{"seed":1}'); INSERT INTO environment_binding (id, account_id, browser_env_alias) VALUES ('upgrade', 'upgrade', 'upgrade'); @@ -233,7 +235,7 @@ func TestUnifiedAccountMigration(t *testing.T) { ('legacy-task', 'legacy-task-key', 'mapped', 1, 'legacy-draft', 1, 'legacy-confirmation', 1, 'queued'), ('legacy-unknown-task', 'legacy-unknown-task-key', 'mapped', 1, 'legacy-draft', 1, 'legacy-confirmation', 1, 'needs_confirmation'); INSERT INTO gateway (name, endpoint, token) VALUES ('legacy-gateway', 'http://127.0.0.1:8081', 'legacy-gateway-token'); - INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1'); + INSERT INTO browser_image (version, image_ref) VALUES ('1', '/opt/creatorhub/browsers/1'); INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) VALUES ('mapped', 'Mapped', 'legacy-gateway', '1', '{"seed":1,"proxy_server":"http://legacy:secret@proxy.example:8080","disable_non_proxied_udp":true}'), ('orphan-env', 'Orphan', 'legacy-gateway', '1', '{"seed":2}'); @@ -318,7 +320,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal("one binding must not have two active runtime instances") } if _, err := db.Exec(` - INSERT INTO browser_image (version, image_ref) VALUES ('2', 'example/browser:2'); + INSERT INTO browser_version (version, browser_path) VALUES ('2', '/opt/creatorhub/browsers/2'); UPDATE social_account SET status = 'active' WHERE id = 'mapped'; INSERT INTO content_draft (id, account_id, version, content) VALUES ('upgrade-draft', 'mapped', 1, 'test'); INSERT INTO confirmation (id, account_id, account_version, draft_id, draft_version, version) @@ -344,7 +346,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal(err) } store.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND version = 2 AND image_version = '2'`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND version = 2 AND browser_version = '2'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE id = 'mapped' AND version = 2`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE id = 'mapped' AND version = 2 AND status = 'paused'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'upgrade-task' AND state = 'policy_hold' AND hold_reason = 'binding_version_changed'`, 1) diff --git a/internal/hub/migrations/003_unified_accounts.sql b/internal/hub/migrations/003_unified_accounts.sql index 0572241..1392570 100644 --- a/internal/hub/migrations/003_unified_accounts.sql +++ b/internal/hub/migrations/003_unified_accounts.sql @@ -41,7 +41,7 @@ ALTER TABLE browser_env ALTER TABLE browser_image DROP CONSTRAINT IF EXISTS browser_image_image_ref_check, ADD CONSTRAINT browser_image_image_ref_check - CHECK (length(image_ref) <= 301 AND image_ref ~ '^[A-Za-z0-9][A-Za-z0-9._:/@-]*$'); + CHECK (length(image_ref) <= 301 AND (image_ref ~ '^[A-Za-z0-9][A-Za-z0-9._:/@-]*$' OR image_ref ~ '^/[A-Za-z0-9._+~/-]+$')); CREATE TABLE network_exit ( id text PRIMARY KEY CHECK (id ~ '^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$'), diff --git a/internal/hub/migrations/017_native_browser_versions.sql b/internal/hub/migrations/017_native_browser_versions.sql new file mode 100644 index 0000000..fdec17f --- /dev/null +++ b/internal/hub/migrations/017_native_browser_versions.sql @@ -0,0 +1,13 @@ +-- Replace Docker image terminology with the pre-installed native browser contract. +ALTER TABLE browser_image RENAME TO browser_version; +ALTER TABLE browser_version RENAME COLUMN image_ref TO browser_path; +ALTER TABLE browser_env RENAME COLUMN image_version TO browser_version; +ALTER TABLE audit_event RENAME COLUMN old_image_version TO old_browser_version; +ALTER TABLE audit_event RENAME COLUMN new_image_version TO new_browser_version; + +ALTER TABLE browser_version DROP CONSTRAINT browser_image_image_ref_check; +ALTER TABLE browser_version + ADD CONSTRAINT browser_version_browser_path_check + CHECK (browser_path ~ '^/[A-Za-z0-9._+~/-]+$'); +ALTER TABLE browser_env + RENAME CONSTRAINT browser_env_image_version_fkey TO browser_env_browser_version_fkey; diff --git a/internal/hub/store.go b/internal/hub/store.go index 4400b7a..c9f7db0 100644 --- a/internal/hub/store.go +++ b/internal/hub/store.go @@ -66,6 +66,9 @@ var migration015 string //go:embed migrations/016_network_exit_plain_credentials.sql var migration016 string +//go:embed migrations/017_native_browser_versions.sql +var migration017 string + //go:embed migrations/033_unique_fingerprint_seed.sql var migration033 string @@ -76,16 +79,16 @@ var ( ErrReconcileRequired = errors.New("runtime cleanup generation is unknown; manual reconciliation required") ) -func ValidImageVersion(version string) bool { return imageVersionPattern.MatchString(version) } +func ValidBrowserVersion(version string) bool { return browserVersionPattern.MatchString(version) } func ValidNetworkExitID(id string) bool { return exitIDPattern.MatchString(id) } var ( - aliasPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) - gatewayNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) - tokenPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{15,127}$`) - imageVersionPattern = regexp.MustCompile(`^[0-9][A-Za-z0-9.+~-]{0,63}$`) - imageRefPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$`) + aliasPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) + gatewayNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) + tokenPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{15,127}$`) + browserVersionPattern = regexp.MustCompile(`^[0-9][A-Za-z0-9.+~-]{0,63}$`) + browserPathPattern = regexp.MustCompile(`^/[A-Za-z0-9._+~/-]+$`) ) type Store struct { @@ -94,7 +97,7 @@ type Store struct { notify taskstate.Notifier } -// Gateway 是平台注册的 docker-gateway 实例；Token 由平台生成，明文存储供页面复制（开发阶段约定）。 +// Gateway 是平台注册的 native browser gateway 节点；Token 由平台生成，明文存储供页面复制（开发阶段约定）。 type Gateway struct { Name string `json:"name"` Endpoint string `json:"endpoint"` @@ -103,24 +106,24 @@ type Gateway struct { UpdatedAt time.Time `json:"updated_at"` } -// Image 是页面维护的可用浏览器镜像版本。 -type Image struct { - Version string `json:"version"` - ImageRef string `json:"image_ref"` - Note string `json:"note"` - Enabled bool `json:"enabled"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` +// BrowserVersion 是页面维护的可用预安装浏览器版本。 +type BrowserVersion struct { + Version string `json:"version"` + BrowserPath string `json:"browser_path"` + Note string `json:"note"` + Enabled bool `json:"enabled"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } -// Env 是一个浏览器环境；Fingerprint 为全量启动参数，创建后随请求整体下发网关。 +// Env 是一个浏览器环境；Fingerprint 为全量启动参数，创建后随请求整体下发 gateway。 type Env struct { - Alias string `json:"alias"` - Name string `json:"name"` - Gateway string `json:"gateway"` - ImageVersion string `json:"image_version"` - Fingerprint Fingerprint `json:"fingerprint"` - CreatedAt time.Time `json:"created_at"` + Alias string `json:"alias"` + Name string `json:"name"` + Gateway string `json:"gateway"` + BrowserVersion string `json:"browser_version"` + Fingerprint Fingerprint `json:"fingerprint"` + CreatedAt time.Time `json:"created_at"` } func Open(ctx context.Context, databaseURL string) (*Store, error) { @@ -162,7 +165,7 @@ func (s *Store) notifyTransitions(transitions []taskstate.Transition) { // LockResources serializes lifecycle state across control-plane replicas. The // transaction carries no data changes; rolling it back only releases the locks. -func (s *Store) LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error) { +func (s *Store) LockResources(ctx context.Context, aliases, exitIDs, browserVersions []string) (func(), error) { for _, alias := range aliases { if !aliasPattern.MatchString(alias) { return nil, ErrInvalid @@ -173,12 +176,12 @@ func (s *Store) LockResources(ctx context.Context, aliases, exitIDs, imageVersio return nil, ErrInvalid } } - for _, version := range imageVersions { - if !imageVersionPattern.MatchString(version) { + for _, version := range browserVersions { + if !browserVersionPattern.MatchString(version) { return nil, ErrInvalid } } - if len(aliases)+len(exitIDs)+len(imageVersions) == 0 { + if len(aliases)+len(exitIDs)+len(browserVersions) == 0 { return func() {}, nil } select { @@ -194,7 +197,7 @@ func (s *Store) LockResources(ctx context.Context, aliases, exitIDs, imageVersio resources := []struct { namespace int keys []string - }{{1542738013, aliases}, {1542738015, exitIDs}, {1542738014, imageVersions}} + }{{1542738013, aliases}, {1542738015, exitIDs}, {1542738014, browserVersions}} for _, resource := range resources { keys := append([]string(nil), resource.keys...) sort.Strings(keys) @@ -227,7 +230,7 @@ func (s *Store) migrate(ctx context.Context) error { for _, migration := range []struct { version int sql string - }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}, {11, migration011}, {12, migration012}, {13, migration013}, {14, migration014}, {15, migration015}, {16, migration016}, {33, migration033}} { + }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}, {11, migration011}, {12, migration012}, {13, migration013}, {14, migration014}, {15, migration015}, {16, migration016}, {17, migration017}, {33, migration033}} { var applied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { return errors.New("read hub schema migration state") @@ -337,75 +340,75 @@ func (s *Store) DeleteGateway(ctx context.Context, name string) error { return nil } -func (s *Store) CreateImage(ctx context.Context, image Image) error { - image.Version = strings.TrimSpace(image.Version) - image.ImageRef = strings.TrimSpace(image.ImageRef) - image.Note = strings.TrimSpace(image.Note) - if !imageVersionPattern.MatchString(image.Version) || !imageRefPattern.MatchString(image.ImageRef) || - utf8.RuneCountInString(image.Note) > 200 { +func (s *Store) CreateBrowserVersion(ctx context.Context, version BrowserVersion) error { + version.Version = strings.TrimSpace(version.Version) + version.BrowserPath = strings.TrimSpace(version.BrowserPath) + version.Note = strings.TrimSpace(version.Note) + if !browserVersionPattern.MatchString(version.Version) || !validBrowserPath(version.BrowserPath) || + utf8.RuneCountInString(version.Note) > 200 { return ErrInvalid } _, err := s.db.ExecContext(ctx, ` - INSERT INTO browser_image (version, image_ref, note, enabled) VALUES ($1, $2, $3, $4)`, - image.Version, image.ImageRef, image.Note, image.Enabled) + INSERT INTO browser_version (version, browser_path, note, enabled) VALUES ($1, $2, $3, $4)`, + version.Version, version.BrowserPath, version.Note, version.Enabled) return publicDatabaseError(err) } -func (s *Store) UpdateImage(ctx context.Context, image Image) error { - image.ImageRef = strings.TrimSpace(image.ImageRef) - image.Note = strings.TrimSpace(image.Note) - if !imageVersionPattern.MatchString(image.Version) || !imageRefPattern.MatchString(image.ImageRef) || - utf8.RuneCountInString(image.Note) > 200 { +func (s *Store) UpdateBrowserVersion(ctx context.Context, version BrowserVersion) error { + version.BrowserPath = strings.TrimSpace(version.BrowserPath) + version.Note = strings.TrimSpace(version.Note) + if !browserVersionPattern.MatchString(version.Version) || !validBrowserPath(version.BrowserPath) || + utf8.RuneCountInString(version.Note) > 200 { return ErrInvalid } var updated string err := s.db.QueryRowContext(ctx, ` - UPDATE browser_image SET image_ref = $2, note = $3, enabled = $4, updated_at = now() - WHERE version = $1 RETURNING version`, image.Version, image.ImageRef, image.Note, image.Enabled). + UPDATE browser_version SET browser_path = $2, note = $3, enabled = $4, updated_at = now() + WHERE version = $1 RETURNING version`, version.Version, version.BrowserPath, version.Note, version.Enabled). Scan(&updated) return rowError(err) } -func (s *Store) ListImages(ctx context.Context, enabledOnly bool) ([]Image, error) { - query := `SELECT version, image_ref, note, enabled, created_at, updated_at FROM browser_image` +func (s *Store) ListBrowserVersions(ctx context.Context, enabledOnly bool) ([]BrowserVersion, error) { + query := `SELECT version, browser_path, note, enabled, created_at, updated_at FROM browser_version` if enabledOnly { query += ` WHERE enabled` } query += ` ORDER BY created_at DESC, version` rows, err := s.db.QueryContext(ctx, query) if err != nil { - return nil, errors.New("read browser images") + return nil, errors.New("read browser versions") } defer rows.Close() - images := []Image{} + versions := []BrowserVersion{} for rows.Next() { - var image Image - if err := rows.Scan(&image.Version, &image.ImageRef, &image.Note, &image.Enabled, &image.CreatedAt, &image.UpdatedAt); err != nil { - return nil, errors.New("decode browser image") + var version BrowserVersion + if err := rows.Scan(&version.Version, &version.BrowserPath, &version.Note, &version.Enabled, &version.CreatedAt, &version.UpdatedAt); err != nil { + return nil, errors.New("decode browser version") } - images = append(images, image) + versions = append(versions, version) } - return images, rows.Err() + return versions, rows.Err() } -func (s *Store) DeleteImage(ctx context.Context, version string) error { - if !imageVersionPattern.MatchString(version) { +func (s *Store) DeleteBrowserVersion(ctx context.Context, version string) error { + if !browserVersionPattern.MatchString(version) { return ErrInvalid } var deleted string - if err := s.db.QueryRowContext(ctx, `DELETE FROM browser_image WHERE version = $1 RETURNING version`, version).Scan(&deleted); err != nil { + if err := s.db.QueryRowContext(ctx, `DELETE FROM browser_version WHERE version = $1 RETURNING version`, version).Scan(&deleted); err != nil { return rowError(err) } return nil } -// ImageRef 返回可用（存在且启用）版本的镜像引用；缺失或禁用均视为不可用。 -func (s *Store) ImageRef(ctx context.Context, version string) (string, error) { - if !imageVersionPattern.MatchString(version) { +// BrowserPath 返回可用（存在且启用）版本的宿主机浏览器路径；缺失或禁用均视为不可用。 +func (s *Store) BrowserPath(ctx context.Context, version string) (string, error) { + if !browserVersionPattern.MatchString(version) { return "", ErrInvalid } var ref string - if err := s.db.QueryRowContext(ctx, `SELECT image_ref FROM browser_image WHERE version = $1 AND enabled`, version).Scan(&ref); err != nil { + if err := s.db.QueryRowContext(ctx, `SELECT browser_path FROM browser_version WHERE version = $1 AND enabled`, version).Scan(&ref); err != nil { return "", rowError(err) } return ref, nil @@ -415,7 +418,7 @@ func (s *Store) CreateEnv(ctx context.Context, env Env) error { env.Alias = strings.TrimSpace(env.Alias) env.Name = strings.TrimSpace(env.Name) if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || - !gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) || + !gatewayNamePattern.MatchString(env.Gateway) || !browserVersionPattern.MatchString(env.BrowserVersion) || env.Fingerprint.ProxyServer != "" { return ErrInvalid } @@ -428,15 +431,15 @@ func (s *Store) CreateEnv(ctx context.Context, env Env) error { } var created string err = s.db.QueryRowContext(ctx, ` - INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) - SELECT $1, $2, $3, version, $5 FROM browser_image WHERE version = $4 AND enabled - RETURNING alias`, env.Alias, env.Name, env.Gateway, env.ImageVersion, encoded).Scan(&created) + INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) + SELECT $1, $2, $3, version, $5 FROM browser_version WHERE version = $4 AND enabled + RETURNING alias`, env.Alias, env.Name, env.Gateway, env.BrowserVersion, encoded).Scan(&created) return rowError(err) } func (s *Store) ListEnvs(ctx context.Context) ([]Env, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT alias, name, gateway_name, image_version, fingerprint, created_at + SELECT alias, name, gateway_name, browser_version, fingerprint, created_at FROM browser_env ORDER BY created_at, alias`) if err != nil { return nil, errors.New("read browser envs") @@ -458,7 +461,7 @@ func (s *Store) GetEnv(ctx context.Context, alias string) (Env, error) { return Env{}, ErrInvalid } rows, err := s.db.QueryContext(ctx, ` - SELECT alias, name, gateway_name, image_version, fingerprint, created_at + SELECT alias, name, gateway_name, browser_version, fingerprint, created_at FROM browser_env WHERE alias = $1`, alias) if err != nil { return Env{}, errors.New("read browser env") @@ -476,7 +479,7 @@ func (s *Store) GetEnv(ctx context.Context, alias string) (Env, error) { // UpgradeEnv 将环境切换到指定可用镜像版本；参数与卷不变，容器重建由控制面编排网关完成。 func (s *Store) UpgradeEnv(ctx context.Context, alias, version string) error { - if !aliasPattern.MatchString(alias) || !imageVersionPattern.MatchString(version) { + if !aliasPattern.MatchString(alias) || !browserVersionPattern.MatchString(version) { return ErrInvalid } tx, err := s.db.BeginTx(ctx, nil) @@ -495,8 +498,8 @@ func (s *Store) UpgradeEnv(ctx context.Context, alias, version string) error { } var updated string if err := tx.QueryRowContext(ctx, ` - UPDATE browser_env SET image_version = $2, version = version + 1 - WHERE alias = $1 AND EXISTS (SELECT 1 FROM browser_image WHERE version = $2 AND enabled) + UPDATE browser_env SET browser_version = $2, version = version + 1 + WHERE alias = $1 AND EXISTS (SELECT 1 FROM browser_version WHERE version = $2 AND enabled) RETURNING alias`, alias, version).Scan(&updated); err != nil { return rowError(err) } @@ -581,7 +584,7 @@ func (s *Store) DeleteAccountEnvironment(ctx context.Context, accountID string) func scanEnv(rows *sql.Rows) (Env, error) { var env Env var encoded []byte - if err := rows.Scan(&env.Alias, &env.Name, &env.Gateway, &env.ImageVersion, &encoded, &env.CreatedAt); err != nil { + if err := rows.Scan(&env.Alias, &env.Name, &env.Gateway, &env.BrowserVersion, &encoded, &env.CreatedAt); err != nil { return Env{}, errors.New("decode browser env") } if len(encoded) > 0 { @@ -594,6 +597,10 @@ func scanEnv(rows *sql.Rows) (Env, error) { return env, nil } +func validBrowserPath(path string) bool { + return len(path) >= 2 && len(path) <= 4096 && browserPathPattern.MatchString(path) +} + func validDisplayName(name string) bool { if name == "" || utf8.RuneCountInString(name) > 64 { return false diff --git a/internal/hub/store_test.go b/internal/hub/store_test.go index 5a74bcc..5d7a972 100644 --- a/internal/hub/store_test.go +++ b/internal/hub/store_test.go @@ -258,22 +258,22 @@ func TestStoreValidationRejectsInvalidInputsBeforePersistence(t *testing.T) { t.Fatalf("expected invalid gateway update %s, got %v", test.name, err) } } - if err := store.CreateImage(ctx, Image{Version: "v1", ImageRef: "registry/img:1"}); !errors.Is(err, ErrInvalid) { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "v1", BrowserPath: "registry/img:1"}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid image version, got %v", err) } - if err := store.CreateImage(ctx, Image{Version: "148.0.0.1", ImageRef: "has space"}); !errors.Is(err, ErrInvalid) { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.0.1", BrowserPath: "has space"}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid image ref, got %v", err) } - if err := store.CreateImage(ctx, Image{Version: "148.0.0.1", ImageRef: "registry/img:1", Note: strings.Repeat("长", 201)}); !errors.Is(err, ErrInvalid) { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.0.1", BrowserPath: "registry/img:1", Note: strings.Repeat("长", 201)}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected overlong note to be rejected, got %v", err) } - if err := store.CreateEnv(ctx, Env{Alias: "UP", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrInvalid) { + if err := store.CreateEnv(ctx, Env{Alias: "UP", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid alias, got %v", err) } - if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: strings.Repeat("名", 65), Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrInvalid) { + if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: strings.Repeat("名", 65), Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected overlong name, got %v", err) } - if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 0}}); !errors.Is(err, ErrInvalid) { + if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 0}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid fingerprint, got %v", err) } for name, exit := range map[string]NetworkExit{ @@ -289,7 +289,7 @@ func TestStoreValidationRejectsInvalidInputsBeforePersistence(t *testing.T) { } }) } - if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1, ProxyServer: "socks5://proxy.example:1080"}}); !errors.Is(err, ErrInvalid) { + if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1, ProxyServer: "socks5://proxy.example:1080"}}); !errors.Is(err, ErrInvalid) { t.Fatalf("stored fingerprint proxy must be rejected, got %v", err) } } @@ -302,13 +302,13 @@ func TestFingerprintSeedIsGloballyUnique(t *testing.T) { ctx := context.Background() store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) t.Cleanup(func() { _ = store.Close() }) - if _, err := store.db.ExecContext(ctx, `TRUNCATE environment_binding, browser_env, browser_image, social_account, credential_reference, gateway CASCADE`); err != nil { + if _, err := store.db.ExecContext(ctx, `TRUNCATE environment_binding, browser_env, browser_version, social_account, credential_reference, gateway CASCADE`); err != nil { t.Fatal(err) } if _, err := store.CreateGateway(ctx, "gw-seed", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil { t.Fatal(err) } - if err := store.CreateImage(ctx, Image{Version: "148", ImageRef: "example/browser:148", Enabled: true}); err != nil { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` @@ -318,7 +318,7 @@ func TestFingerprintSeedIsGloballyUnique(t *testing.T) { ('seed-account-b', 'credential-seed-b', 'mock', 'seed-account-b', 'owned', 'authorized')`); err != nil { t.Fatal(err) } - env := Env{Alias: "seed-environment-a", Name: "Seed A", Gateway: "gw-seed", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 77}} + env := Env{Alias: "seed-environment-a", Name: "Seed A", Gateway: "gw-seed", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 77}} if _, created, err := store.CreateBoundEnv(ctx, env, "seed-account-a", ""); err != nil || !created { t.Fatalf("create first seeded environment: created=%v err=%v", created, err) } @@ -337,7 +337,7 @@ func TestHubWorkflow(t *testing.T) { ctx := context.Background() store := openFullyMigratedHub(t, ctx, databaseURL) t.Cleanup(func() { _ = store.Close() }) - if _, err := store.db.ExecContext(ctx, `TRUNCATE browser_env, browser_image, gateway CASCADE`); err != nil { + if _, err := store.db.ExecContext(ctx, `TRUNCATE browser_env, browser_version, gateway CASCADE`); err != nil { t.Fatal(err) } @@ -356,18 +356,18 @@ func TestHubWorkflow(t *testing.T) { t.Fatalf("expected duplicate gateway conflict, got %v", err) } - if err := store.CreateImage(ctx, Image{Version: "148.0.7778.215", ImageRef: "git.ipao.vip/rogee/fingerprint-chromium:148", Note: "主力版本", Enabled: true}); err != nil { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Note: "主力版本", Enabled: true}); err != nil { t.Fatal(err) } - if err := store.CreateImage(ctx, Image{Version: "144.0.7559.132", ImageRef: "git.ipao.vip/rogee/fingerprint-chromium:144", Enabled: false}); err != nil { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: false}); err != nil { t.Fatal(err) } - if err := store.CreateImage(ctx, Image{Version: "148.0.7778.215", ImageRef: "git.ipao.vip/rogee/fingerprint-chromium:148b"}); !errors.Is(err, ErrConflict) { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148b"}); !errors.Is(err, ErrConflict) { t.Fatalf("expected duplicate version conflict, got %v", err) } env := Env{ - Alias: "shop-01", Name: "店铺一号", Gateway: "gw-main", ImageVersion: "148.0.7778.215", + Alias: "shop-01", Name: "店铺一号", Gateway: "gw-main", BrowserVersion: "148.0.7778.215", Fingerprint: Fingerprint{Seed: 1000, Timezone: "Asia/Shanghai", Lang: "zh-CN"}, } if err := store.CreateEnv(ctx, env); err != nil { @@ -376,7 +376,7 @@ func TestHubWorkflow(t *testing.T) { if err := store.CreateEnv(ctx, env); !errors.Is(err, ErrConflict) { t.Fatalf("expected duplicate alias conflict, got %v", err) } - if err := store.CreateEnv(ctx, Env{Alias: "shop-02", Name: "店铺二号", Gateway: "missing", ImageVersion: "148.0.7778.215", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrConflict) { + if err := store.CreateEnv(ctx, Env{Alias: "shop-02", Name: "店铺二号", Gateway: "missing", BrowserVersion: "148.0.7778.215", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrConflict) { t.Fatalf("expected unknown gateway conflict, got %v", err) } @@ -408,16 +408,16 @@ func TestHubWorkflow(t *testing.T) { t.Fatalf("expected missing env, got %v", err) } - if _, err := store.ImageRef(ctx, "144.0.7559.132"); !errors.Is(err, ErrNotFound) { + if _, err := store.BrowserPath(ctx, "144.0.7559.132"); !errors.Is(err, ErrNotFound) { t.Fatalf("disabled version must not resolve, got %v", err) } - if err := store.UpdateImage(ctx, Image{Version: "144.0.7559.132", ImageRef: "git.ipao.vip/rogee/fingerprint-chromium:144", Enabled: true}); err != nil { + if err := store.UpdateBrowserVersion(ctx, BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: true}); err != nil { t.Fatal(err) } - if ref, err := store.ImageRef(ctx, "144.0.7559.132"); err != nil || !strings.HasSuffix(ref, ":144") { + if ref, err := store.BrowserPath(ctx, "144.0.7559.132"); err != nil || !strings.HasSuffix(ref, "/144") { t.Fatalf("enabled version must resolve: %v %q", err, ref) } - if _, err := store.ImageRef(ctx, "999"); !errors.Is(err, ErrNotFound) { + if _, err := store.BrowserPath(ctx, "999"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing version, got %v", err) } @@ -425,14 +425,14 @@ func TestHubWorkflow(t *testing.T) { t.Fatal(err) } upgraded, err := store.GetEnv(ctx, "shop-01") - if err != nil || upgraded.ImageVersion != "144.0.7559.132" || upgraded.Fingerprint.Seed != 1000 { + if err != nil || upgraded.BrowserVersion != "144.0.7559.132" || upgraded.Fingerprint.Seed != 1000 { t.Fatalf("upgrade must only change image version: %#v %v", upgraded, err) } if err := store.UpgradeEnv(ctx, "ghost", "144.0.7559.132"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing env on upgrade, got %v", err) } - if err := store.DeleteImage(ctx, "144.0.7559.132"); !errors.Is(err, ErrConflict) { + if err := store.DeleteBrowserVersion(ctx, "144.0.7559.132"); !errors.Is(err, ErrConflict) { t.Fatalf("referenced version must not be deletable, got %v", err) } if err := store.DeleteGateway(ctx, "gw-main"); !errors.Is(err, ErrConflict) { @@ -444,7 +444,7 @@ func TestHubWorkflow(t *testing.T) { if err := store.DeleteEnv(ctx, "shop-01"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing env on double delete, got %v", err) } - if err := store.DeleteImage(ctx, "144.0.7559.132"); err != nil { + if err := store.DeleteBrowserVersion(ctx, "144.0.7559.132"); err != nil { t.Fatal(err) } if err := store.DeleteGateway(ctx, "gw-main"); err != nil { @@ -455,6 +455,87 @@ func TestHubWorkflow(t *testing.T) { } } +func TestRuntimeUseLeaseLifecycle(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) + t.Cleanup(func() { _ = store.Close() }) + if _, err := store.db.ExecContext(ctx, `TRUNCATE runtime_use_lease, runtime_instance, environment_binding, browser_env, + browser_version, gateway, social_account, credential_reference CASCADE`); err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO credential_reference (id, provider, reference_key) + VALUES ('lease-credential', 'os_keyring', 'creatorhub/lease-account'); + INSERT INTO social_account + (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) + VALUES ('lease-account', 'lease-credential', 'mock', 'lease-account', 'owned', 'authorized')`); err != nil { + t.Fatal(err) + } + if _, err := store.CreateGateway(ctx, "lease-gateway", "http://127.0.0.1:8081", "lease-gateway-token"); err != nil { + t.Fatal(err) + } + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148-lease", BrowserPath: "/opt/creatorhub/browsers/lease", Enabled: true}); err != nil { + t.Fatal(err) + } + environment, created, err := store.CreateBoundEnv(ctx, Env{ + Alias: "lease-env", Name: "Lease environment", Gateway: "lease-gateway", BrowserVersion: "148-lease", + Fingerprint: Fingerprint{Seed: 1}, + }, "lease-account", "") + if err != nil || !created { + t.Fatalf("create lease environment: %#v created=%v err=%v", environment, created, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'lease-account'`); err != nil { + t.Fatal(err) + } + active, err := store.ActivateRuntime(ctx, environment.Alias, "runtime-lease-generation", environment.BindingVersion, "", "native-lease-network") + if err != nil { + t.Fatal(err) + } + lease, err := store.AcquireRuntimeUse(ctx, environment.Alias, "task", "lease-owner", "") + if err != nil || lease.RuntimeInstanceID != active.RuntimeInstanceID || lease.Purpose != "task" { + t.Fatalf("acquire runtime-use lease: %#v err=%v", lease, err) + } + if renewed, renewErr := store.RenewRuntimeUse(ctx, lease.Token); renewErr != nil || !renewed.LeaseUntil.After(lease.LeaseUntil) { + t.Fatalf("renew runtime-use lease: %#v err=%v", renewed, renewErr) + } + if err := store.ReleaseRuntimeUse(ctx, lease.Token); err != nil { + t.Fatal(err) + } + if err := store.ReleaseRuntimeUse(ctx, lease.Token); err != nil { + t.Fatalf("releasing an already released lease must be idempotent: %v", err) + } + listener, err := store.AcquireRuntimeUse(ctx, environment.Alias, "listener", "listener-owner", "") + if err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE runtime_use_lease SET lease_until = now() - interval '1 second' WHERE token = $1`, listener.Token); err != nil { + t.Fatal(err) + } + if _, err := store.RenewRuntimeUse(ctx, listener.Token); !errors.Is(err, ErrConflict) { + t.Fatalf("expired runtime-use lease renewed: %v", err) + } + if err := store.ReleaseRuntime(ctx, active); err != nil { + t.Fatal(err) + } + var released, runtimeReleased bool + if err := store.db.QueryRowContext(ctx, `SELECT released_at IS NOT NULL FROM runtime_use_lease WHERE token = $1`, listener.Token).Scan(&released); err != nil { + t.Fatal(err) + } + if err := store.db.QueryRowContext(ctx, `SELECT released_at IS NOT NULL FROM runtime_instance WHERE id = $1`, active.RuntimeInstanceID).Scan(&runtimeReleased); err != nil { + t.Fatal(err) + } + if !released || !runtimeReleased { + t.Fatalf("runtime cleanup did not revoke use leases: lease_released=%v runtime_released=%v", released, runtimeReleased) + } + if _, err := store.RenewRuntimeUse(ctx, listener.Token); !errors.Is(err, ErrConflict) { + t.Fatalf("runtime cleanup left an old lease renewable: %v", err) + } +} + func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { @@ -466,7 +547,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { var notifications []taskstate.Transition store.SetTaskNotifier(func(transition taskstate.Transition) { notifications = append(notifications, transition) }) if _, err := store.db.ExecContext(ctx, `TRUNCATE audit_event, runtime_instance, environment_binding, network_exit, - social_account, credential_reference, browser_env, browser_image, gateway CASCADE`); err != nil { + social_account, credential_reference, browser_env, browser_version, gateway CASCADE`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` @@ -480,7 +561,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if _, err := store.CreateGateway(ctx, "gw-main", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil { t.Fatal(err) } - if err := store.CreateImage(ctx, Image{Version: "148", ImageRef: "example/browser:148", Enabled: true}); err != nil { + if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}); err != nil { t.Fatal(err) } @@ -510,7 +591,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { t.Fatalf("matching identity must make the exit healthy: %#v reason=%s err=%v", exit, reason, err) } - env := Env{Alias: "environment-a", Name: "环境 A", Gateway: "gw-main", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1}} + env := Env{Alias: "environment-a", Name: "环境 A", Gateway: "gw-main", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1}} bound, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID) if err != nil || !created || bound.AccountID != "account-a" || bound.Exit.ID != exit.ID { t.Fatalf("create stable binding: %#v created=%v err=%v", bound, created, err) @@ -522,7 +603,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } - active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID, "network-a") + active, err := store.ActivateRuntime(ctx, env.Alias, "runtime-a", bound.BindingVersion, bound.Exit.ID, "network-a") if err != nil || active.RuntimeInstanceID == "" { t.Fatalf("activate runtime: %#v err=%v", active, err) } @@ -544,7 +625,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { INSERT INTO social_account (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) VALUES ('account-b', 'credential-account-b', 'mock', 'account-b', 'owned', 'authorized'); - INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES ('environment-b', '环境 B', 'gw-main', '148', '{"seed":2}'); INSERT INTO environment_binding (id, account_id, browser_env_alias) VALUES ('binding-b', 'account-b', 'environment-b')`); err != nil { @@ -554,13 +635,13 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if err != nil || legacyRebound.Exit.ID != second.ID { t.Fatalf("legacy binding without an exit must support explicit rebind: %#v err=%v", legacyRebound, err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID, "network-a"); !errors.Is(err, ErrConflict) { + if _, err := store.ActivateRuntime(ctx, env.Alias, "runtime-a", bound.BindingVersion+1, second.ID, "network-a"); !errors.Is(err, ErrConflict) { t.Fatalf("stale binding metadata must not activate a runtime: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '10 seconds' WHERE id = $1`, active.RuntimeInstanceID); err != nil { t.Fatal(err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID, "network-a"); err != nil { + if _, err := store.ActivateRuntime(ctx, env.Alias, "runtime-a", bound.BindingVersion, bound.Exit.ID, "network-a"); err != nil { t.Fatalf("runtime heartbeat failed: %v", err) } var renewed bool @@ -588,14 +669,14 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } - expiredBeforeActivation, err := store.ActivateRuntime(ctx, env.Alias, "expired-container", rebound.BindingVersion, rebound.Exit.ID, "network-expired") + expiredBeforeActivation, err := store.ActivateRuntime(ctx, env.Alias, "expired-runtime", rebound.BindingVersion, rebound.Exit.ID, "network-expired") if err != nil { t.Fatalf("activate runtime to expire: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, expiredBeforeActivation.RuntimeInstanceID); err != nil { t.Fatal(err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID, "network-same-exit"); err != nil { + if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-runtime", rebound.BindingVersion, rebound.Exit.ID, "network-same-exit"); err != nil { t.Fatalf("replace expired runtime before same-exit rebind: %v", err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE event_type = 'runtime_released' @@ -619,15 +700,15 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { AND reason_code = 'runtime_released' AND account_id = 'account-a' AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3 AND details = '{}'::jsonb`, 1, active.Exit.ID, active.RuntimeInstanceID, active.BindingVersion) - rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-container", rebound.BindingVersion) - if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-container" { + rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-runtime", rebound.BindingVersion) + if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-runtime" { t.Fatalf("same-exit rebind must atomically CAS the binding and runtime: %#v err=%v", rebound, err) } if err := store.ReleaseRuntime(ctx, active); !errors.Is(err, ErrConflict) { t.Fatalf("stale generation release must conflict: %v", err) } current, err := store.GetEnvironmentContext(ctx, env.Alias) - if err != nil || current.RuntimeInstanceID != rebound.RuntimeInstanceID || current.RuntimeID != "rebound-container" { + if err != nil || current.RuntimeInstanceID != rebound.RuntimeInstanceID || current.RuntimeID != "rebound-runtime" { t.Fatalf("stale release changed the current runtime: %#v err=%v", current, err) } cleanup := current @@ -641,7 +722,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3`, 1, current.Exit.ID, current.RuntimeInstanceID, current.BindingVersion) wrongCleanup := cleanup - wrongCleanup.RuntimeCleanupRuntimeID = "other-container" + wrongCleanup.RuntimeCleanupRuntimeID = "other-runtime" if err := store.SetRuntimeCleanupPending(ctx, wrongCleanup, false); !errors.Is(err, ErrConflict) { t.Fatalf("wrong cleanup generation cleared pending state: %v", err) } @@ -649,7 +730,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if err != nil || !pending.RuntimeCleanupPending || pending.RuntimeCleanupRuntimeID != current.RuntimeID { t.Fatalf("cleanup generation was not persisted: %#v err=%v", pending, err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "candidate-container", current.BindingVersion, current.Exit.ID, "network-candidate"); !errors.Is(err, ErrConflict) { + if _, err := store.ActivateRuntime(ctx, env.Alias, "candidate-runtime", current.BindingVersion, current.Exit.ID, "network-candidate"); !errors.Is(err, ErrConflict) { t.Fatalf("paused account activated a stale request: %v", err) } if err := store.SetRuntimeCleanupPending(ctx, pending, false); err != nil { @@ -675,7 +756,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { } activation := make(chan error, 1) go func() { - _, err := store.ActivateRuntime(ctx, env.Alias, "racing-container", newGeneration.BindingVersion, newGeneration.Exit.ID, "network-racing") + _, err := store.ActivateRuntime(ctx, env.Alias, "racing-runtime", newGeneration.BindingVersion, newGeneration.Exit.ID, "network-racing") activation <- err }() select { @@ -712,7 +793,7 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+action.OperationID+`'`, 2) invalidAction := action invalidAction.OperationID = NewOperationID() - invalidAction.NewImageVersion = "http://operator:secret@proxy.example" + invalidAction.NewBrowserVersion = "http://operator:secret@proxy.example" if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", invalidAction); !errors.Is(err, ErrInvalid) { t.Fatalf("invalid image version must not reach audit persistence: %v", err) } diff --git a/internal/phasea/migrations/036_runtime_use_leases.sql b/internal/phasea/migrations/036_runtime_use_leases.sql new file mode 100644 index 0000000..9a4a059 --- /dev/null +++ b/internal/phasea/migrations/036_runtime_use_leases.sql @@ -0,0 +1,36 @@ +ALTER TABLE runtime_instance + ADD COLUMN IF NOT EXISTS owner_id text NOT NULL DEFAULT 'control-plane', + ADD COLUMN IF NOT EXISTS purpose text NOT NULL DEFAULT 'session', + ADD COLUMN IF NOT EXISTS lease_token text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS node_id text NOT NULL DEFAULT ''; + +ALTER TABLE runtime_instance + DROP CONSTRAINT IF EXISTS runtime_instance_purpose_check; +ALTER TABLE runtime_instance + ADD CONSTRAINT runtime_instance_purpose_check + CHECK (purpose IN ('session', 'task', 'listener')); + +CREATE UNIQUE INDEX IF NOT EXISTS runtime_instance_active_lease_token_idx + ON runtime_instance (lease_token) + WHERE released_at IS NULL AND lease_token <> ''; + +CREATE TABLE IF NOT EXISTS runtime_use_lease ( + token text PRIMARY KEY, + runtime_instance_id text NOT NULL REFERENCES runtime_instance(id), + owner_id text NOT NULL, + purpose text NOT NULL CHECK (purpose IN ('task', 'listener')), + task_id text REFERENCES operation_task(id), + lease_until timestamptz NOT NULL, + acquired_at timestamptz NOT NULL DEFAULT now(), + released_at timestamptz, + CHECK (length(token) BETWEEN 16 AND 128), + CHECK (length(owner_id) BETWEEN 1 AND 128) +); + +CREATE INDEX IF NOT EXISTS runtime_use_lease_active_runtime_idx + ON runtime_use_lease (runtime_instance_id, lease_until) + WHERE released_at IS NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS runtime_use_lease_active_task_idx + ON runtime_use_lease (task_id) + WHERE released_at IS NULL AND task_id IS NOT NULL; diff --git a/internal/phasea/store.go b/internal/phasea/store.go index 7bedecc..473ebb3 100644 --- a/internal/phasea/store.go +++ b/internal/phasea/store.go @@ -26,6 +26,9 @@ import ( //go:embed migrations/001_phase_a.sql var migration001 string +//go:embed migrations/036_runtime_use_leases.sql +var migration036 string + var ( ErrConflict = errors.New("resource conflicts with existing state") ErrInvalid = errors.New("invalid phase A input") @@ -188,8 +191,8 @@ type AuditEvent struct { OperationID string `json:"operation_id,omitempty"` Action string `json:"action,omitempty"` Outcome string `json:"outcome,omitempty"` - OldImageVersion string `json:"old_image_version,omitempty"` - NewImageVersion string `json:"new_image_version,omitempty"` + OldBrowserVersion string `json:"old_browser_version,omitempty"` + NewBrowserVersion string `json:"new_browser_version,omitempty"` Details json.RawMessage `json:"details"` CreatedAt time.Time `json:"created_at"` } @@ -254,16 +257,22 @@ func (s *Store) migrate(ctx context.Context) error { if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { return errors.New("create schema migration table") } - var applied bool - if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = 1)`).Scan(&applied); err != nil { - return errors.New("read schema migration state") - } - if !applied { - if _, err := tx.ExecContext(ctx, migration001); err != nil { - return fmt.Errorf("apply schema migration 1: %w", err) + for _, migration := range []struct { + version int + sql string + }{{1, migration001}, {36, migration036}} { + var applied bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { + return errors.New("read schema migration state") } - if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migration (version) VALUES (1)`); err != nil { - return errors.New("record schema migration 1") + if applied { + continue + } + if _, err := tx.ExecContext(ctx, migration.sql); err != nil { + return fmt.Errorf("apply schema migration %d: %w", migration.version, err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migration (version) VALUES ($1)`, migration.version); err != nil { + return fmt.Errorf("record schema migration %d", migration.version) } } if err := tx.Commit(); err != nil { @@ -1824,7 +1833,7 @@ func (s *Store) ListAudit(ctx context.Context, filter AuditFilter) (AuditPage, e rows, err := s.db.QueryContext(ctx, ` SELECT id, event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id, browser_env_alias, network_exit_id, runtime_instance_id, binding_version, actor, reason_code, - operation_id, action, outcome, old_image_version, new_image_version, + operation_id, action, outcome, old_browser_version, new_browser_version, details, created_at FROM audit_event WHERE ($1 = '' OR account_id = $1) AND ($2 = '' OR task_id = $2) AND ($3 = '' OR attempt_id = $3) @@ -1854,7 +1863,7 @@ func (s *Store) ListAudit(ctx context.Context, filter AuditFilter) (AuditPage, e event.RuntimeInstanceID, event.BindingVersion = runtimeInstanceID.String, bindingVersion.Int64 event.Actor, event.ReasonCode = actor.String, reasonCode.String event.OperationID, event.Action, event.Outcome = operationID.String, action.String, outcome.String - event.OldImageVersion, event.NewImageVersion = oldImage.String, newImage.String + event.OldBrowserVersion, event.NewBrowserVersion = oldImage.String, newImage.String event.Details = safeDetails(event.Details) events = append(events, event) } diff --git a/internal/phasea/store_test.go b/internal/phasea/store_test.go index 37b49b1..cad1cbd 100644 --- a/internal/phasea/store_test.go +++ b/internal/phasea/store_test.go @@ -170,7 +170,7 @@ func TestPhaseAOfflineWorkflow(t *testing.T) { if _, err := store.db.ExecContext(ctx, ` TRUNCATE audit_event, execution_attempt, operation_task, confirmation, content_draft, runtime_instance, environment_binding, network_exit, social_account, credential_reference, - browser_env, browser_image, gateway RESTART IDENTITY CASCADE`); err != nil { + browser_env, browser_version, gateway RESTART IDENTITY CASCADE`); err != nil { t.Fatal(err) } @@ -215,11 +215,11 @@ func TestPhaseAOfflineWorkflow(t *testing.T) { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` - INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1')`); err != nil { + INSERT INTO browser_version (version, browser_path) VALUES ('1', '/opt/creatorhub/browsers/1')`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` - INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) VALUES + INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES ('account-a', 'Account A', 'test-gateway', '1', '{"seed":1}'), ('account-b', 'Account B', 'test-gateway', '1', '{"seed":2}'); INSERT INTO network_exit (id, protocol, host, port, health_status) @@ -407,7 +407,7 @@ func TestPhaseAOfflineWorkflow(t *testing.T) { // A held task is never revived in place: only a newly confirmed task with a new idempotency key may run. if _, err := store.db.ExecContext(ctx, ` INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) - VALUES ('runtime-recovered', 'gate-runtime-missing', 'binding-runtime-missing', 1, 'container-recovered', now() + interval '1 minute')`); err != nil { + VALUES ('runtime-recovered', 'gate-runtime-missing', 'binding-runtime-missing', 1, 'runtime-recovered', now() + interval '1 minute')`); err != nil { t.Fatal(err) } if execution, err := store.ExecuteMock(ctx, "worker-old-held", "succeeded"); err != nil || execution.WasClaimed { @@ -754,7 +754,7 @@ func TestCreateAccountWithoutCookiesSkipsCredentialStore(t *testing.T) { if _, err := store.db.ExecContext(ctx, ` TRUNCATE audit_event, execution_attempt, operation_task, confirmation, content_draft, runtime_instance, environment_binding, network_exit, social_account, credential_reference, - browser_env, browser_image, gateway RESTART IDENTITY CASCADE`); err != nil { + browser_env, browser_version, gateway RESTART IDENTITY CASCADE`); err != nil { t.Fatal(err) } credentials := &testCredentialBridge{values: map[string]string{}} @@ -785,7 +785,7 @@ func TestAccountCredentialCommitResult(t *testing.T) { if _, err := store.db.ExecContext(ctx, ` TRUNCATE audit_event, execution_attempt, operation_task, confirmation, content_draft, runtime_instance, environment_binding, network_exit, social_account, credential_reference, - browser_env, browser_image, gateway RESTART IDENTITY CASCADE`); err != nil { + browser_env, browser_version, gateway RESTART IDENTITY CASCADE`); err != nil { t.Fatal(err) } credentials := &testCredentialBridge{values: map[string]string{}} @@ -886,7 +886,7 @@ func seedGateTask(t *testing.T, store *Store, suffix, accountStatus, authorizati } if binding { exitID := "exit-gate-" + suffix - if _, err := tx.ExecContext(ctx, `INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + if _, err := tx.ExecContext(ctx, `INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES ($1, $1, 'test-gateway', '1', '{"seed":3}')`, accountID); err != nil { t.Fatal(err) } @@ -907,7 +907,7 @@ func seedGateTask(t *testing.T, store *Store, suffix, accountStatus, authorizati if _, err := tx.ExecContext(ctx, ` INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) VALUES ($1, $2, $3, 1, $4, now() + $5::interval)`, "runtime-gate-"+suffix, accountID, - "binding-"+suffix, "container-gate-"+suffix, interval); err != nil { + "binding-"+suffix, "runtime-gate-"+suffix, interval); err != nil { t.Fatal(err) } } @@ -958,7 +958,8 @@ func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) { }{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}, {6, "006_runtime_cleanup.sql"}, {7, "007_runtime_binding_version.sql"}, {8, "008_runtime_cleanup_generation.sql"}, {9, "009_runtime_cleanup_compatibility.sql"}, {10, "010_runtime_network_generation.sql"}, {11, "011_task_recovery.sql"}, - {12, "012_task_recovery_compatibility.sql"}, {13, "013_account_creation.sql"}, {14, "014_account_creation_compatibility.sql"}} { + {12, "012_task_recovery_compatibility.sql"}, {13, "013_account_creation.sql"}, {14, "014_account_creation_compatibility.sql"}, + {15, "015_gateway_rename_cascade.sql"}, {16, "016_network_exit_plain_credentials.sql"}, {17, "017_native_browser_versions.sql"}} { var applied bool if err := store.db.QueryRow(`SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migrationFile.version).Scan(&applied); err != nil { t.Fatal(err) diff --git a/package.json b/package.json index 372d393..0170da1 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "version": "0.1.0", "scripts": { - "dev:deps": "docker compose -f compose.yaml -f compose.dev.yaml up -d postgres docker-gateway", + "dev:deps": "docker compose -f compose.yaml -f compose.dev.yaml up -d postgres", "dev:backend": "node scripts/dev-backend.mjs", "dev:frontend": "pnpm -C web run dev", "dev": "node scripts/dev.mjs" diff --git a/scripts/dev-backend.mjs b/scripts/dev-backend.mjs old mode 100644 new mode 100755 index b84b75a..37e40ca --- a/scripts/dev-backend.mjs +++ b/scripts/dev-backend.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// dev-backend: 读取 .env，补齐开发默认值，确保 postgres/docker-gateway 容器在跑，然后用 air 热加载运行 control-plane。 +// dev-backend: 启动 PostgreSQL 依赖，并连接宿主机 native browser gateway 后运行 control-plane。 import { spawn, spawnSync } from "node:child_process"; import { readFileSync, existsSync } from "node:fs"; import { fileURLToPath } from "node:url"; @@ -20,63 +20,42 @@ for (const line of readFileSync(envPath, "utf8").split("\n")) { if (m && !(m[1] in overrides)) overrides[m[1]] = m[2]; } -if (!overrides.CONTROL_PLANE_USERNAME) - overrides.CONTROL_PLANE_USERNAME = "admin"; -if (!overrides.CONTROL_PLANE_PASSWORD) - overrides.CONTROL_PLANE_PASSWORD = "admin123"; -// 主密钥必须是 32 字节的 base64；无效或缺失时换成本地开发专用密钥（与 .env 里的 dev 凭据同级机密性）。 -const devMasterKey = Buffer.from( - "creatorhub dev local master key", - "utf8", -).toString("base64"); // 恰 32 字节 -if ( - Buffer.from(overrides.CREATORHUB_CREDENTIAL_MASTER_KEY ?? "", "base64") - .length !== 32 -) { +if (!overrides.CONTROL_PLANE_USERNAME) overrides.CONTROL_PLANE_USERNAME = "admin"; +if (!overrides.CONTROL_PLANE_PASSWORD) overrides.CONTROL_PLANE_PASSWORD = "admin123"; +const devMasterKey = Buffer.from("creatorhub dev local master key", "utf8").toString("base64"); +if (Buffer.from(overrides.CREATORHUB_CREDENTIAL_MASTER_KEY ?? "", "base64").length !== 32) { if (overrides.CREATORHUB_CREDENTIAL_MASTER_KEY) { - console.error( - "[dev-backend] .env 的 CREATORHUB_CREDENTIAL_MASTER_KEY 不是 32 字节 base64，本地开发改用固定开发密钥。", - ); + console.error("[dev-backend] .env 的 CREATORHUB_CREDENTIAL_MASTER_KEY 不是 32 字节 base64，本地开发改用固定开发密钥。"); } overrides.CREATORHUB_CREDENTIAL_MASTER_KEY = devMasterKey; } if (!overrides.LISTEN_ADDR) overrides.LISTEN_ADDR = ":8082"; -if (!overrides.DATABASE_URL) - overrides.DATABASE_URL = - "postgres://creatorhub@127.0.0.1:5432/creatorhub?sslmode=disable"; -if (!overrides.CREATORHUB_CREDENTIAL_STORE_DIR) - overrides.CREATORHUB_CREDENTIAL_STORE_DIR = path.join( - root, - ".dev-credentials", - ); +if (!overrides.DATABASE_URL) overrides.DATABASE_URL = "postgres://creatorhub@127.0.0.1:5432/creatorhub?sslmode=disable"; +if (!overrides.CREATORHUB_CREDENTIAL_STORE_DIR) overrides.CREATORHUB_CREDENTIAL_STORE_DIR = path.join(root, ".dev-credentials"); if (!overrides.WEB_DIR) overrides.WEB_DIR = path.join(root, "web", "dist"); if (!overrides.LOG_LEVEL) overrides.LOG_LEVEL = "debug"; -if (!overrides.DOCKER_GID) - overrides.DOCKER_GID = String(process.getgid?.() ?? 1000); +if (!overrides.NATIVE_GATEWAY_ENDPOINT) overrides.NATIVE_GATEWAY_ENDPOINT = "http://127.0.0.1:8081"; const env = { ...process.env, ...overrides }; - const compose = spawnSync( "docker", - [ - "compose", - "-f", - "compose.yaml", - "-f", - "compose.dev.yaml", - "up", - "-d", - "--build", - "postgres", - "docker-gateway", - ], + ["compose", "-f", "compose.yaml", "-f", "compose.dev.yaml", "up", "-d", "postgres"], { cwd: root, env, stdio: "inherit" }, ); if (compose.status !== 0) { - console.error("启动 postgres/docker-gateway 容器失败，请确认 docker 可用。"); + console.error("启动 PostgreSQL 容器失败，请确认 Docker 只用于数据库依赖。"); process.exit(compose.status ?? 1); } +try { + const response = await fetch(`${overrides.NATIVE_GATEWAY_ENDPOINT}/healthz`, { signal: AbortSignal.timeout(2000) }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); +} catch (error) { + console.error(`[dev-backend] native browser gateway 不可达：${overrides.NATIVE_GATEWAY_ENDPOINT} (${error})`); + console.error("请先以非 root 用户启动 cmd/browser_gateway.gateway 或对应的 systemd user service。"); + process.exit(1); +} + const air = spawn("air", ["-c", ".air.toml", ...process.argv.slice(2)], { cwd: root, env, diff --git a/scripts/install-native-browser-gateway.sh b/scripts/install-native-browser-gateway.sh new file mode 100755 index 0000000..7c2c076 --- /dev/null +++ b/scripts/install-native-browser-gateway.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$(id -u)" == 0 ]]; then + echo "run this installer as the non-root gateway user" >&2 + exit 1 +fi + +project_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +env_file="${HOME}/.config/creatorhub/browser-gateway.env" +unit_dir="${HOME}/.config/systemd/user" +unit_file="${unit_dir}/creatorhub-browser-gateway.service" + +if [[ ! -f "${env_file}" ]]; then + echo "missing ${env_file}; copy deploy/browser-gateway.env.example and set GATEWAY_TOKEN and BROWSER_PATH" >&2 + exit 1 +fi + +install -d -m 0700 "${unit_dir}" +sed "s|@PROJECT_DIR@|${project_dir}|g" \ + "${project_dir}/deploy/creatorhub-browser-gateway.service.in" > "${unit_file}.tmp" +chmod 0600 "${unit_file}.tmp" +mv -f "${unit_file}.tmp" "${unit_file}" +systemctl --user daemon-reload +systemctl --user enable --now creatorhub-browser-gateway.service +systemctl --user --no-pager --full status creatorhub-browser-gateway.service diff --git a/web/src/AccountEditPage.jsx b/web/src/AccountEditPage.jsx index fbe4d13..b3b1530 100644 --- a/web/src/AccountEditPage.jsx +++ b/web/src/AccountEditPage.jsx @@ -240,7 +240,7 @@ export function AccountEditPage() { variant: "destructive", text: conflictMessage( qrError, - "登录二维码获取失败；请检查运行环境、网关和镜像配置", + "登录二维码获取失败；请检查运行环境、网关和浏览器版本配置", ), }); } diff --git a/web/src/BrowserImagesPage.jsx b/web/src/BrowserVersionsPage.jsx similarity index 60% rename from web/src/BrowserImagesPage.jsx rename to web/src/BrowserVersionsPage.jsx index 04e81f5..1bf2a9a 100644 --- a/web/src/BrowserImagesPage.jsx +++ b/web/src/BrowserVersionsPage.jsx @@ -15,29 +15,30 @@ import { import { useTitle } from "./lib/hooks.js"; const versionPattern = /^\d[A-Za-z0-9._-]{0,63}$/; -const refPattern = /^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$/; +const pathPattern = /^\/[A-Za-z0-9._+~/-]{1,4095}$/; -function ImageCreateModal({ open, onClose, onSubmit, busy, error }) { - const [form, setForm] = useState({ version: "", image_ref: "", note: "" }); +function BrowserVersionCreateModal({ open, onClose, onSubmit, busy, error }) { + const [form, setForm] = useState({ version: "", browser_path: "", note: "" }); const update = (key, value) => setForm((current) => ({ ...current, [key]: value })); const valid = - versionPattern.test(form.version) && refPattern.test(form.image_ref); + versionPattern.test(form.version) && pathPattern.test(form.browser_path); const versionInvalid = form.version !== "" && !versionPattern.test(form.version); - const refInvalid = form.image_ref !== "" && !refPattern.test(form.image_ref); + const pathInvalid = + form.browser_path !== "" && !pathPattern.test(form.browser_path); async function submit(event) { event.preventDefault(); if (!valid) return; const created = await onSubmit({ version: form.version, - image_ref: form.image_ref, + browser_path: form.browser_path, note: form.note, enabled: true, }); if (created) { - setForm({ version: "", image_ref: "", note: "" }); + setForm({ version: "", browser_path: "", note: "" }); onClose(); } } @@ -46,8 +47,8 @@ function ImageCreateModal({ open, onClose, onSubmit, busy, error }) {