From 1ce0375414c5cd2ec95c686e1ba19e4f2870592f Mon Sep 17 00:00:00 2001 From: Rogee Date: Fri, 4 Sep 2026 16:39:53 +0800 Subject: [PATCH] fix: safely reconcile rejected runtime creates (#47) --- cmd/control-plane/hub.go | 56 +- cmd/control-plane/hub_test.go | 21 +- scripts/dev-backend.mjs | 1 + web/src/BrowsersPage.jsx | 931 ++++++++++++++++++++++++++-------- web/src/lib/ui.jsx | 9 +- 5 files changed, 800 insertions(+), 218 deletions(-) diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index 8635119..b3bb478 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -14,6 +14,7 @@ import ( "git.ipao.vip/rogee/creator-hub/internal/hub" "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" ) // hubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。 @@ -166,13 +167,40 @@ func gatewayNetworkID(body []byte) string { } func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, - environment hub.EnvironmentContext, body []byte) error { - return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion, gatewayNetworkID(body)) + environment hub.EnvironmentContext, status int, callErr error, body []byte) error { + // 网关 4xx 拒绝且未产生 network 代：创建在网关侧任何 Docker 变更之前就被确定性拒绝， + // 无需清理 fence；其余情况（5xx、断连、无效 201）结果未知，仍走 fail-closed。 + deterministicRejection := callErr == nil && status >= 400 && status < 500 + logEntry := logrus.WithFields(logrus.Fields{ + "service": "control-plane", + "event_type": "runtime_create_reconciliation", + "alias": environment.Alias, + "gateway": target.Name, + "binding_version": environment.BindingVersion, + "gateway_status": status, + "network_id_known": gatewayNetworkID(body) != "", + "deterministic": deterministicRejection, + }) + if callErr != nil { + logEntry = logEntry.WithError(callErr) + } + if deterministicRejection { + logEntry.Info("gateway rejected runtime create before Docker side effects") + } else { + logEntry.Warn("gateway runtime create result requires reconciliation") + } + return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion, + gatewayNetworkID(body), deterministicRejection) } func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, - environment hub.EnvironmentContext, bindingVersion int64, networkID string) error { + environment hub.EnvironmentContext, bindingVersion int64, networkID string, deterministicRejection bool) error { if networkID == "" { + if deterministicRejection { + // 网关明确拒绝：创建未发生、无副作用，不写 pending，让调用方透传真实失败原因， + // 绑定保持可重试（否则会把可判定的失败写成永远无法自动清理的“未知代”毒状态）。 + return nil + } pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "") return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true)) } @@ -1163,15 +1191,15 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) if callErr != nil { - return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, body)) + return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body)) } if status != http.StatusCreated { - return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, body)) + return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body)) } var created containerStatus if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) { return false, errors.Join(errors.New("gateway returned an invalid runtime generation"), - reconcileGatewayCreate(ctx, store, target, environment, body)) + reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body)) } _, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID) return err == nil, err @@ -1321,7 +1349,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw if callErr != nil { createErr = gatewayUnreachable(callErr) } - reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, errors.Join(createErr, reconcileErr)) @@ -1331,7 +1359,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw } var createdRuntime containerStatus if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) { - reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway create result unknown; environment retained for reconciliation"}, reconcileErr)) @@ -1562,7 +1590,7 @@ func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExit status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) if callErr != nil || status != http.StatusCreated { - reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr)) @@ -1575,7 +1603,7 @@ func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExit } var created containerStatus if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) { - reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr)) } @@ -1671,7 +1699,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) var createdRuntime containerStatus if callErr != nil || status != http.StatusCreated { - reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr)) @@ -1683,7 +1711,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx return hubError(c, gatewayRejected(status, body)) } if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) { - reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body) _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr)) } @@ -1892,7 +1920,7 @@ func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub. } created, err := createGatewayRuntime(ctx, target, environment, spec) if err != nil { - cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID) + cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID, false) return false, errors.Join(err, cleanupErr) } if previous.State == "running" && accountRunnable(environment) { @@ -2037,7 +2065,7 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw _ = finish("unknown", "gateway_result_unknown", before) return hubError(c, errors.Join(err, contextErr)) } - cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID) + cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID, false) if cleanupErr != nil { _ = finish("unknown", "gateway_result_unknown", before) return hubError(c, errors.Join(err, cleanupErr)) diff --git a/cmd/control-plane/hub_test.go b/cmd/control-plane/hub_test.go index c6d335d..8d780f6 100644 --- a/cmd/control-plane/hub_test.go +++ b/cmd/control-plane/hub_test.go @@ -1203,6 +1203,25 @@ func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testi } } +func TestCreateBrowserDeterministicRejectionDoesNotWedgeBinding(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadRequest} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers", createEnvBody) + if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "alias already in use") { + t.Fatalf("deterministic gateway rejection must surface its reason, got %d: %s", response.Code, response.Body.String()) + } + environment, err := store.GetEnvironmentContext(context.Background(), "account-a") + if err != nil || environment.RuntimeCleanupPending { + t.Fatalf("deterministic rejection wedged the binding for manual reconciliation: environment=%#v err=%v", environment, err) + } + if requests := gateway.recorded(); len(requests) != 1 || requests[0].method != http.MethodPost { + t.Fatalf("deterministic rejection must not reconcile or mutate the gateway: %#v", requests) + } +} + func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} @@ -1217,7 +1236,7 @@ func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) { defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} - err := reconcileGatewayCreate(context.Background(), store, target, environment, []byte(`{"error":"result unknown"}`)) + err := reconcileGatewayCreate(context.Background(), store, target, environment, http.StatusBadGateway, nil, []byte(`{"error":"result unknown"}`)) after := store.bindings[environment.Alias] if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.containers) != 1 || !after.RuntimeCleanupPending || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" { diff --git a/scripts/dev-backend.mjs b/scripts/dev-backend.mjs index c6d1c3b..b84b75a 100644 --- a/scripts/dev-backend.mjs +++ b/scripts/dev-backend.mjs @@ -66,6 +66,7 @@ const compose = spawnSync( "compose.dev.yaml", "up", "-d", + "--build", "postgres", "docker-gateway", ], diff --git a/web/src/BrowsersPage.jsx b/web/src/BrowsersPage.jsx index 87a1923..4cc9d40 100644 --- a/web/src/BrowsersPage.jsx +++ b/web/src/BrowsersPage.jsx @@ -1,87 +1,226 @@ -import { useEffect, useState } from 'react' -import { Link, useNavigate, useParams } from 'react-router' -import { useDataProvider, useList, useOne } from '@refinedev/core' +import { useEffect, useState } from "react"; +import { Link, useNavigate, useParams } from "react-router"; +import { useDataProvider, useList, useOne } from "@refinedev/core"; import { - Alert, Button, Card, CardContent, Copyable, DetailList, Field, Input, Modal, PageHeader, - PageState, Select, StatusPill, Table, -} from './lib/ui.jsx' -import { useTitle } from './lib/hooks.js' + Alert, + Button, + Card, + CardContent, + Copyable, + DetailList, + Field, + Input, + Modal, + PageHeader, + PageState, + Select, + StatusPill, + conflictMessage, +} from "./lib/ui.jsx"; +import { useTitle } from "./lib/hooks.js"; -const aliasPattern = /^[a-z0-9][a-z0-9-]{0,31}$/ -const platformOptions = ['windows', 'linux', 'macos'].map(value => ({ value, label: value })) -const brandOptions = ['Chrome', 'Edge', 'Opera', 'Vivaldi'].map(value => ({ value, label: value })) -const spoofingFields = ['font', 'audio', 'canvas', 'clientrects', 'gpu'] +const aliasPattern = /^[a-z0-9][a-z0-9-]{0,31}$/; +const platformOptions = ["windows", "linux", "macos"].map((value) => ({ + value, + label: value, +})); +const brandOptions = ["Chrome", "Edge", "Opera", "Vivaldi"].map((value) => ({ + value, + label: value, +})); +const spoofingFields = ["font", "audio", "canvas", "clientrects", "gpu"]; -const stateText = { created: '已创建', running: '运行中', exited: '已停止', restarting: '重启中', paused: '已暂停', missing: '未部署' } +const stateText = { + created: "已创建", + running: "运行中", + exited: "已停止", + restarting: "重启中", + paused: "已暂停", + missing: "未部署", +}; const blockText = { - account_revoked: '授权已撤销', account_paused: '账号已暂停', binding_missing: '绑定缺失', - network_exit_missing: '固定出口缺失', network_exit_unhealthy: '固定出口不健康', - runtime_stop_pending: '停止结果待确认', runtime_missing: '运行实例缺失', -} + account_revoked: "授权已撤销", + account_paused: "账号已暂停", + binding_missing: "绑定缺失", + network_exit_missing: "固定出口缺失", + network_exit_unhealthy: "固定出口不健康", + runtime_stop_pending: "停止结果待确认", + runtime_missing: "运行实例缺失", +}; -const scheduleBlock = runtime => runtime.schedule_status === 'ready' - ? '' - : blockText[runtime.schedule_block_reason] || runtime.schedule_block_reason || '不可调度' +const scheduleBlock = (runtime) => + runtime.schedule_status === "ready" + ? "" + : blockText[runtime.schedule_block_reason] || + runtime.schedule_block_reason || + "不可调度"; const createInitial = { - name: '', alias: '', gateway: '', image_version: '', account_id: '', network_exit_id: '', seed: '1000', - platform: '', platform_version: '', brand: '', brand_version: '', hardware_concurrency: '', lang: '', accept_lang: '', timezone: '', disable_spoofing: [], -} + name: "", + alias: "", + gateway: "", + image_version: "", + account_id: "", + network_exit_id: "", + seed: "1000", + platform: "", + platform_version: "", + brand: "", + brand_version: "", + hardware_concurrency: "", + lang: "", + accept_lang: "", + timezone: "", + disable_spoofing: [], +}; // 高级字段空值不提交，收敛到 seed 驱动。 function fingerprintPayload(form) { - const payload = { seed: Number(form.seed) } - const advanced = { platform: form.platform, platform_version: form.platform_version, brand: form.brand, brand_version: form.brand_version, lang: form.lang, accept_lang: form.accept_lang, timezone: form.timezone } - for (const [key, value] of Object.entries(advanced)) if (value !== '') payload[key] = value - if (form.hardware_concurrency !== '') payload.hardware_concurrency = Number(form.hardware_concurrency) - if (form.disable_spoofing.length > 0) payload.disable_spoofing = form.disable_spoofing.join(',') - return payload + const payload = { seed: Number(form.seed) }; + const advanced = { + platform: form.platform, + platform_version: form.platform_version, + brand: form.brand, + brand_version: form.brand_version, + lang: form.lang, + accept_lang: form.accept_lang, + timezone: form.timezone, + }; + for (const [key, value] of Object.entries(advanced)) + if (value !== "") payload[key] = value; + if (form.hardware_concurrency !== "") + payload.hardware_concurrency = Number(form.hardware_concurrency); + if (form.disable_spoofing.length > 0) + payload.disable_spoofing = form.disable_spoofing.join(","); + return payload; } function StatePill({ state }) { - const known = Object.hasOwn(stateText, state) - return + const known = Object.hasOwn(stateText, state); + return ( + + ); } function RuntimeActions({ runtime, busy, onAction }) { - const running = runtime.state === 'running' - const block = runtime.schedule_block_reason === 'runtime_missing' ? '' : scheduleBlock(runtime) - const startLabel = `启动 ${runtime.name}${block ? `（${block}）` : ''}` + const running = runtime.state === "running"; + const block = + runtime.schedule_block_reason === "runtime_missing" + ? "" + : scheduleBlock(runtime); + const startLabel = `启动 ${runtime.name}${block ? `（${block}）` : ""}`; return (
- - - - + + + +
- ) + ); } function SchedulePill({ runtime }) { - const block = scheduleBlock(runtime) - return + const block = scheduleBlock(runtime); + return ( + + ); } function UpgradeDialog({ target, images, busy, onClose, onConfirm }) { - const [version, setVersion] = useState('') - const enabled = images.filter(image => image.enabled) - useEffect(() => { setVersion('') }, [target]) + const [version, setVersion] = useState(""); + const enabled = images.filter((image) => image.enabled); + useEffect(() => { + setVersion(""); + }, [target]); return ( - - - - }> + + + + + } + >
-

容器将按新镜像重建，Profile 数据卷与指纹参数保持不变。

+

+ 容器将按新镜像重建，Profile 数据卷与指纹参数保持不变。 +

- setVersion(event.target.value)} + options={enabled.map((image) => ({ + value: image.version, + label: `${image.version}${image.version === target?.image_version ? "（当前）" : ""}`, + }))} + placeholder="选择版本" + />
- ) + ); } function RuntimeTable({ runtimes, busy, onAction, onRowClick }) { @@ -98,236 +237,575 @@ function RuntimeTable({ runtimes, busy, onAction, onRowClick }) { - {runtimes.map(runtime => ( - - onRowClick(runtime)}> + {runtimes.map((runtime) => ( + + onRowClick(runtime)} + >

{runtime.name}

-

{runtime.alias} · seed {runtime.fingerprint?.seed}

- - - -

{runtime.account_id || '未绑定账号'}

-

{runtime.network_exit_id || '当前机器直连'} · {runtime.image_version}

-
+

+ {runtime.alias} · seed {runtime.fingerprint?.seed} +

- {runtime.endpoint ? : —} + + + +

{runtime.account_id || "未绑定账号"}

+

+ {runtime.network_exit_id || "当前机器直连"} ·{" "} + {runtime.image_version} +

+
+ +
+ + + {runtime.endpoint ? ( + + ) : ( + — + )} + + + - ))} - ) + ); } function RuntimeCards({ runtimes, busy, onAction }) { return (
- {runtimes.map(runtime => ( + {runtimes.map((runtime) => (

{runtime.name}

-

{runtime.alias} · {runtime.image_version} · seed {runtime.fingerprint?.seed}

+

+ {runtime.alias} · {runtime.image_version} · seed{" "} + {runtime.fingerprint?.seed} +

-

账号：{runtime.account_id || '未绑定'} · 网络出口：{runtime.network_exit_id || '当前机器直连'}

+

+ 账号：{runtime.account_id || "未绑定"} · 网络出口： + {runtime.network_exit_id || "当前机器直连"} +

{runtime.endpoint ? : null} - + ))}
- ) + ); } export function BrowserList() { - const navigate = useNavigate() - const dataProvider = useDataProvider()('default') - const [busy, setBusy] = useState('') - const [actionError, setActionError] = useState('') - const [upgradeTarget, setUpgradeTarget] = useState(null) - const { result, query } = useList({ resource: 'browsers', queryOptions: { refetchInterval: 3000 } }) - const error = query.error - const isPending = query.isPending - const runtimes = result.data ?? [] - const { result: imageResult, query: imagesQuery } = useList({ resource: 'browser-images' }) - const images = imageResult.data ?? [] - useTitle('CreatorHub · 运行环境') + const navigate = useNavigate(); + const dataProvider = useDataProvider()("default"); + const [busy, setBusy] = useState(""); + const [actionError, setActionError] = useState(""); + const [upgradeTarget, setUpgradeTarget] = useState(null); + const { result, query } = useList({ + resource: "browsers", + queryOptions: { refetchInterval: 3000 }, + }); + const error = query.error; + const isPending = query.isPending; + const runtimes = result.data ?? []; + const { result: imageResult } = useList({ + resource: "browser-images", + }); + const images = imageResult.data ?? []; + useTitle("CreatorHub · 运行环境"); async function handleAction(runtime, action) { - if (action === 'recycle' && !window.confirm(`回收 ${runtime.name}（${runtime.alias}）？容器将被删除，Profile 数据卷保留。`)) return - if (action === 'upgrade') { setUpgradeTarget(runtime); return } - await runAction(runtime, action) + if ( + action === "recycle" && + !window.confirm( + `回收 ${runtime.name}（${runtime.alias}）？容器将被删除，Profile 数据卷保留。`, + ) + ) + return; + if (action === "upgrade") { + setUpgradeTarget(runtime); + return; + } + await runAction(runtime, action); } async function runAction(runtime, action, data) { - setBusy(runtime.alias) - setActionError('') + setBusy(runtime.alias); + setActionError(""); try { - await dataProvider.browserAction(runtime.alias, action, data) - await query.refetch() + await dataProvider.browserAction(runtime.alias, action, data); + await query.refetch(); } catch (reason) { - setActionError(reason.message) + setActionError(reason.message); } finally { - setBusy('') + setBusy(""); } } async function upgrade(version) { - await runAction(upgradeTarget, 'upgrade', { version }) - setUpgradeTarget(null) + await runAction(upgradeTarget, "upgrade", { version }); + setUpgradeTarget(null); } return (
- - + + {actionError ? ( {actionError} - + ) : null} - - navigate(`/browsers/${encodeURIComponent(runtime.alias)}`)} /> + + + navigate(`/browsers/${encodeURIComponent(runtime.alias)}`) + } + /> - setUpgradeTarget(null)} onConfirm={upgrade} /> -