rogee/creator-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: safely reconcile rejected runtime creates (#47)

Browse Source
This commit is contained in:
rogee
2026-09-04 16:39:53 +08:00
parent 62418b1f0d
commit 1ce0375414
5 changed files with 800 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/control-plane/hub.go
+42 -14
View File
@@ -14,6 +14,7 @@ import (
"git.ipao.vip/rogee/creator-hub/internal/hub"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
)
// hubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。
@@ -166,13 +167,40 @@ func gatewayNetworkID(body []byte) string {
}
func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
environment hub.EnvironmentContext, body []byte) error {
return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion, gatewayNetworkID(body))
environment hub.EnvironmentContext, status int, callErr error, body []byte) error {
// 网关 4xx 拒绝且未产生 network 代：创建在网关侧任何 Docker 变更之前就被确定性拒绝，
// 无需清理 fence；其余情况（5xx、断连、无效 201）结果未知，仍走 fail-closed。
deterministicRejection := callErr == nil && status >= 400 && status < 500
logEntry := logrus.WithFields(logrus.Fields{
"service": "control-plane",
"event_type": "runtime_create_reconciliation",
"alias": environment.Alias,
"gateway": target.Name,
"binding_version": environment.BindingVersion,
"gateway_status": status,
"network_id_known": gatewayNetworkID(body) != "",
"deterministic": deterministicRejection,
})
if callErr != nil {
logEntry = logEntry.WithError(callErr)
}
if deterministicRejection {
logEntry.Info("gateway rejected runtime create before Docker side effects")
} else {
logEntry.Warn("gateway runtime create result requires reconciliation")
}
return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion,
gatewayNetworkID(body), deterministicRejection)
}
func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
environment hub.EnvironmentContext, bindingVersion int64, networkID string) error {
environment hub.EnvironmentContext, bindingVersion int64, networkID string, deterministicRejection bool) error {
if networkID == "" {
if deterministicRejection {
// 网关明确拒绝：创建未发生、无副作用，不写 pending，让调用方透传真实失败原因，
// 绑定保持可重试（否则会把可判定的失败写成永远无法自动清理的“未知代”毒状态）。
return nil
}
pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "")
return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
}
@@ -1163,15 +1191,15 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE
status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
if callErr != nil {
return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, body))
return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
}
if status != http.StatusCreated {
return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, body))
return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
}
var created containerStatus
if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
return false, errors.Join(errors.New("gateway returned an invalid runtime generation"),
reconcileGatewayCreate(ctx, store, target, environment, body))
reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
}
_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID)
return err == nil, err
@@ -1321,7 +1349,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
if callErr != nil {
createErr = gatewayUnreachable(callErr)
}
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
if reconcileErr != nil {
_ = finish("unknown", "gateway_result_unknown", environment)
return hubError(c, errors.Join(createErr, reconcileErr))
@@ -1331,7 +1359,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
}
var createdRuntime containerStatus
if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
_ = finish("unknown", "gateway_result_unknown", environment)
return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway,
message: "gateway create result unknown; environment retained for reconciliation"}, reconcileErr))
@@ -1562,7 +1590,7 @@ func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExit
status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout)
if callErr != nil || status != http.StatusCreated {
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
if reconcileErr != nil {
_ = finish("unknown", "gateway_result_unknown", environment)
return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr))
@@ -1575,7 +1603,7 @@ func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExit
}
var created containerStatus
if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
_ = finish("unknown", "gateway_result_unknown", environment)
return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr))
}
@@ -1671,7 +1699,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx
gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
var createdRuntime containerStatus
if callErr != nil || status != http.StatusCreated {
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
if reconcileErr != nil {
_ = finish("unknown", "gateway_result_unknown", environment)
return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr))
@@ -1683,7 +1711,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkEx
return hubError(c, gatewayRejected(status, body))
}
if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
_ = finish("unknown", "gateway_result_unknown", environment)
return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr))
}
@@ -1892,7 +1920,7 @@ func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.
}
created, err := createGatewayRuntime(ctx, target, environment, spec)
if err != nil {
cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID)
cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID, false)
return false, errors.Join(err, cleanupErr)
}
if previous.State == "running" && accountRunnable(environment) {
@@ -2037,7 +2065,7 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
_ = finish("unknown", "gateway_result_unknown", before)
return hubError(c, errors.Join(err, contextErr))
}
cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID)
cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID, false)
if cleanupErr != nil {
_ = finish("unknown", "gateway_result_unknown", before)
return hubError(c, errors.Join(err, cleanupErr))
cmd/control-plane/hub_test.go
+20 -1
View File
@@ -1203,6 +1203,25 @@ func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testi
}
}
func TestCreateBrowserDeterministicRejectionDoesNotWedgeBinding(t *testing.T) {
store := newMemoryStore()
_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadRequest}
app := newTestApp(t, store, gateway)
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "alias already in use") {
t.Fatalf("deterministic gateway rejection must surface its reason, got %d: %s", response.Code, response.Body.String())
}
environment, err := store.GetEnvironmentContext(context.Background(), "account-a")
if err != nil || environment.RuntimeCleanupPending {
t.Fatalf("deterministic rejection wedged the binding for manual reconciliation: environment=%#v err=%v", environment, err)
}
if requests := gateway.recorded(); len(requests) != 1 || requests[0].method != http.MethodPost {
t.Fatalf("deterministic rejection must not reconcile or mutate the gateway: %#v", requests)
}
}
func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) {
store := newMemoryStore()
store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"}
@@ -1217,7 +1236,7 @@ func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) {
defer server.Close()
target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
err := reconcileGatewayCreate(context.Background(), store, target, environment, []byte(`{"error":"result unknown"}`))
err := reconcileGatewayCreate(context.Background(), store, target, environment, http.StatusBadGateway, nil, []byte(`{"error":"result unknown"}`))
after := store.bindings[environment.Alias]
if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.containers) != 1 ||
!after.RuntimeCleanupPending || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" {
scripts/dev-backend.mjs
+1
View File
@@ -66,6 +66,7 @@ const compose = spawnSync(
"compose.dev.yaml",
"up",
"-d",
"--build",
"postgres",
"docker-gateway",
],
web/src/BrowsersPage.jsx
+732 -199
View File
File diff suppressed because it is too large Load Diff
web/src/lib/ui.jsx
+5 -4
View File
@@ -12,7 +12,8 @@ export function apiMessage(error) {
}
export function conflictMessage(error, fallback) {
if (error?.status === 409) return `冲突（409）：${fallback}`;
if (error?.status === 409)
return `冲突（409）：${error?.body?.error || fallback}`;
if (error?.status === 503) return `环境不可用（503）：${error.message}`;
return apiMessage(error);
}
@@ -328,9 +329,9 @@ export function Select({
(value === "" ? null : { value, label: value });
const label = selected ? selected.label : placeholder;
const items =
emptyOption !== undefined
? [{ value: emptyOption.value, label: emptyOption.label }, ...options]
: options;
emptyOption === undefined
? options
: [{ value: emptyOption.value, label: emptyOption.label }, ...options];
const close = () => {
setOpen(false);
setActive(-1);