fix: 未知代 pending 清理按别名反查自动收敛

启动创建结果未知时会登记 runtime-not-found + 空 network_id 的 pending（fail-closed fence）。此前该状态永久卡死：所有操作入口直接 409 要求人工对账。 removeGatewayRuntime 现在对此状态先按别名反查网关： - 无容器：该代无实物，直接解除 pending； - 有同版本容器：采用其实物代继续统一清理； - 版本不符：别名下已是别的代，维持人工对账。附回归测试覆盖三分支；真实环境验证卡死状态自动收敛。
2026-09-07 18:54:03 +08:00
parent 83397927c2
commit 7ce5fbd6d2
2 changed files with 104 additions and 4 deletions
@@ -1797,10 +1797,34 @@ func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, enviro

 func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) {
 	removed := environment.RuntimeCleanupPending
-	if environment.RuntimeCleanupPending &&
-		(environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" ||
-			(environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "")) {
-		return false, hub.ErrReconcileRequired
+	if environment.RuntimeCleanupPending {
+		if environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "" {
+			// 未知代 pending：创建结果未知时登记的 runtime-not-found 没有实物 ID 可供清理。
+			// 按别名反查网关：无容器则该代无实物，直接解除；有同版本容器则采用其实物代继续清理；
+			// 版本不符说明别名下已是别的代，维持人工对账。
+			if environment.RuntimeCleanupBindingVersion < 1 {
+				return false, hub.ErrReconcileRequired
+			}
+			discovered, found, discoverErr := reconcileGatewayContainer(ctx, target, environment.Alias)
+			if discoverErr != nil {
+				return false, discoverErr
+			}
+			if !found {
+				return true, store.SetRuntimeCleanupPending(ctx, environment, false)
+			}
+			if discovered.BindingVersion != environment.RuntimeCleanupBindingVersion {
+				return false, hub.ErrReconcileRequired
+			}
+			if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
+				return false, err
+			}
+			environment = runtimeCleanupGeneration(environment, environment.RuntimeCleanupBindingVersion, discovered.ID, discovered.NetworkID)
+			if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
+				return false, err
+			}
+		} else if environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" {
+			return false, hub.ErrReconcileRequired
+		}
 	}
 	if !environment.RuntimeCleanupPending {
 		container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias)
@@ -2494,6 +2494,82 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) {
 	})
 }

+func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) {
+	newPoisoned := func() (*memoryStore, *fakeGateway, hub.EnvironmentContext, hub.Gateway) {
+		store := newMemoryStore()
+		store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", ImageVersion: "148"}
+		environment := hub.EnvironmentContext{
+			Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
+			Exit: store.exits["exit-1"],
+			RuntimeCleanupPending: true, RuntimeCleanupBindingVersion: 1,
+			RuntimeCleanupRuntimeID: missingRuntimeID,
+		}
+		store.bindings[environment.Alias] = environment
+		gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
+			ID: "orphan-container", Alias: environment.Alias, State: "running", ProxyReady: true,
+			BindingVersion: 1, NetworkExitID: environment.Exit.ID, NetworkID: "network-orphan",
+		}}}
+		server := httptest.NewServer(gateway.handler(t))
+		t.Cleanup(server.Close)
+		return store, gateway, environment, hub.Gateway{Endpoint: server.URL, Token: gateway.token}
+	}
+
+	t.Run("orphaned container adopts its real generation and converges", func(t *testing.T) {
+		store, gateway, environment, target := newPoisoned()
+
+		removed, err := removeGatewayRuntime(context.Background(), store, target, environment)
+		if err != nil || !removed {
+			t.Fatalf("unknown generation did not converge: removed=%v err=%v", removed, err)
+		}
+		after := store.bindings[environment.Alias]
+		if after.RuntimeCleanupPending || len(gateway.containers) != 0 {
+			t.Fatalf("cleanup did not finish: after=%#v containers=%#v", after, gateway.containers)
+		}
+		deleteBody := map[string]any{}
+		for _, request := range gateway.recorded() {
+			if request.method == http.MethodDelete {
+				deleteBody = request.body
+			}
+		}
+		if deleteBody["runtime_id"] != "orphan-container" || deleteBody["network_id"] != "network-orphan" {
+			t.Fatalf("cleanup did not adopt the discovered generation: %#v", deleteBody)
+		}
+	})
+
+	t.Run("absent runtime clears the unknown generation", func(t *testing.T) {
+		store, gateway, environment, target := newPoisoned()
+		gateway.containers = nil
+
+		removed, err := removeGatewayRuntime(context.Background(), store, target, environment)
+		if err != nil || !removed {
+			t.Fatalf("absent unknown generation did not converge: removed=%v err=%v", removed, err)
+		}
+		after := store.bindings[environment.Alias]
+		if after.RuntimeCleanupPending {
+			t.Fatalf("pending was not cleared: %#v", after)
+		}
+		for _, request := range gateway.recorded() {
+			if request.method != http.MethodGet {
+				t.Fatalf("absent unknown generation mutated the gateway: %#v", gateway.recorded())
+			}
+		}
+	})
+
+	t.Run("replacement generation keeps manual reconcile", func(t *testing.T) {
+		store, gateway, environment, target := newPoisoned()
+		gateway.containers[0].BindingVersion = 2
+
+		removed, err := removeGatewayRuntime(context.Background(), store, target, environment)
+		if !errors.Is(err, hub.ErrReconcileRequired) || removed {
+			t.Fatalf("replacement generation was not fail-closed: removed=%v err=%v", removed, err)
+		}
+		after := store.bindings[environment.Alias]
+		if !after.RuntimeCleanupPending || len(gateway.containers) != 1 {
+			t.Fatalf("fail-closed path mutated state: after=%#v containers=%#v", after, gateway.containers)
+		}
+	})
+}
+
 func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) {
 	for _, test := range []struct {
 		name        string