From 7ce5fbd6d20f5ad75d560ae2b3d34c722bd24664 Mon Sep 17 00:00:00 2001 From: Rogee Date: Mon, 7 Sep 2026 18:54:03 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=9C=AA=E7=9F=A5=E4=BB=A3=20pending=20?= =?UTF-8?q?=E6=B8=85=E7=90=86=E6=8C=89=E5=88=AB=E5=90=8D=E5=8F=8D=E6=9F=A5?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=94=B6=E6=95=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 启动创建结果未知时会登记 runtime-not-found + 空 network_id 的 pending（fail-closed fence）。此前该状态永久卡死：所有操作入口 直接 409 要求人工对账。 removeGatewayRuntime 现在对此状态先按别名反查网关： - 无容器：该代无实物，直接解除 pending； - 有同版本容器：采用其实物代继续统一清理； - 版本不符：别名下已是别的代，维持人工对账。 附回归测试覆盖三分支；真实环境验证卡死状态自动收敛。 --- cmd/control-plane/hub.go | 32 +++++++++++++-- cmd/control-plane/hub_test.go | 76 +++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index b3bb478..875edf8 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -1797,10 +1797,34 @@ func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, enviro func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) { removed := environment.RuntimeCleanupPending - if environment.RuntimeCleanupPending && - (environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" || - (environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "")) { - return false, hub.ErrReconcileRequired + if environment.RuntimeCleanupPending { + if environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "" { + // 未知代 pending：创建结果未知时登记的 runtime-not-found 没有实物 ID 可供清理。 + // 按别名反查网关：无容器则该代无实物，直接解除；有同版本容器则采用其实物代继续清理； + // 版本不符说明别名下已是别的代，维持人工对账。 + if environment.RuntimeCleanupBindingVersion < 1 { + return false, hub.ErrReconcileRequired + } + discovered, found, discoverErr := reconcileGatewayContainer(ctx, target, environment.Alias) + if discoverErr != nil { + return false, discoverErr + } + if !found { + return true, store.SetRuntimeCleanupPending(ctx, environment, false) + } + if discovered.BindingVersion != environment.RuntimeCleanupBindingVersion { + return false, hub.ErrReconcileRequired + } + if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil { + return false, err + } + environment = runtimeCleanupGeneration(environment, environment.RuntimeCleanupBindingVersion, discovered.ID, discovered.NetworkID) + if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil { + return false, err + } + } else if environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" { + return false, hub.ErrReconcileRequired + } } if !environment.RuntimeCleanupPending { container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias) diff --git a/cmd/control-plane/hub_test.go b/cmd/control-plane/hub_test.go index 8d780f6..c30fcb9 100644 --- a/cmd/control-plane/hub_test.go +++ b/cmd/control-plane/hub_test.go @@ -2494,6 +2494,82 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { }) } +func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) { + newPoisoned := func() (*memoryStore, *fakeGateway, hub.EnvironmentContext, hub.Gateway) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", ImageVersion: "148"} + environment := hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], + RuntimeCleanupPending: true, RuntimeCleanupBindingVersion: 1, + RuntimeCleanupRuntimeID: missingRuntimeID, + } + store.bindings[environment.Alias] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "orphan-container", Alias: environment.Alias, State: "running", ProxyReady: true, + BindingVersion: 1, NetworkExitID: environment.Exit.ID, NetworkID: "network-orphan", + }}} + server := httptest.NewServer(gateway.handler(t)) + t.Cleanup(server.Close) + return store, gateway, environment, hub.Gateway{Endpoint: server.URL, Token: gateway.token} + } + + t.Run("orphaned container adopts its real generation and converges", func(t *testing.T) { + store, gateway, environment, target := newPoisoned() + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err != nil || !removed { + t.Fatalf("unknown generation did not converge: removed=%v err=%v", removed, err) + } + after := store.bindings[environment.Alias] + if after.RuntimeCleanupPending || len(gateway.containers) != 0 { + t.Fatalf("cleanup did not finish: after=%#v containers=%#v", after, gateway.containers) + } + deleteBody := map[string]any{} + for _, request := range gateway.recorded() { + if request.method == http.MethodDelete { + deleteBody = request.body + } + } + if deleteBody["runtime_id"] != "orphan-container" || deleteBody["network_id"] != "network-orphan" { + t.Fatalf("cleanup did not adopt the discovered generation: %#v", deleteBody) + } + }) + + t.Run("absent runtime clears the unknown generation", func(t *testing.T) { + store, gateway, environment, target := newPoisoned() + gateway.containers = nil + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err != nil || !removed { + t.Fatalf("absent unknown generation did not converge: removed=%v err=%v", removed, err) + } + after := store.bindings[environment.Alias] + if after.RuntimeCleanupPending { + t.Fatalf("pending was not cleared: %#v", after) + } + for _, request := range gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("absent unknown generation mutated the gateway: %#v", gateway.recorded()) + } + } + }) + + t.Run("replacement generation keeps manual reconcile", func(t *testing.T) { + store, gateway, environment, target := newPoisoned() + gateway.containers[0].BindingVersion = 2 + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if !errors.Is(err, hub.ErrReconcileRequired) || removed { + t.Fatalf("replacement generation was not fail-closed: removed=%v err=%v", removed, err) + } + after := store.bindings[environment.Alias] + if !after.RuntimeCleanupPending || len(gateway.containers) != 1 { + t.Fatalf("fail-closed path mutated state: after=%#v containers=%#v", after, gateway.containers) + } + }) +} + func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { for _, test := range []struct { name string