diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index e8660df..5fc91b6 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "regexp" "strings" "sync" "time" @@ -39,20 +40,36 @@ type hubStore interface { DisableNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) + GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error - RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (hub.EnvironmentContext, error) - ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (hub.EnvironmentContext, error) - ReleaseRuntime(ctx context.Context, alias string) error - SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error + RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkID ...string) (hub.EnvironmentContext, error) + ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkID ...string) (hub.EnvironmentContext, error) + ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error + SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error } +type runtimeStopStore interface { + GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) + GetGateway(ctx context.Context, name string) (hub.Gateway, error) + ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error + SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error + AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error +} + +type runtimeCleanupStore interface { + SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error +} + const ( gatewayLongTimeout = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取 gatewayReconcileDelay = 100 * time.Millisecond gatewayReconcileAttempts = 10 + missingRuntimeID = "runtime-not-found" ) +var gatewayGenerationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`) + // gatewayCall 调用某个网关的 /v1 路由；ok 为 false 时 status/body 携带网关错误。 func gatewayCall(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration) (status int, responseBody []byte, err error) { callCtx, cancel := context.WithTimeout(ctx, timeout) @@ -99,14 +116,86 @@ func gatewayCreatePayload(environment hub.EnvironmentContext, imageRef string, n } } -func gatewayProxyPayload(environment hub.EnvironmentContext, networkExit gatewayNetworkExit) map[string]any { +func gatewayCreatePayloadForAccount(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any { + payload := gatewayCreatePayload(environment, imageRef, networkExit) + if !accountRunnable(environment) { + payload["network_exit_id"], payload["network_exit"], payload["stopped"] = "", gatewayNetworkExit{}, true + } + return payload +} + +func gatewayProxyPayload(environment hub.EnvironmentContext, runtimeID, networkID string, networkExit gatewayNetworkExit) map[string]any { return map[string]any{ "binding_version": environment.BindingVersion, + "runtime_id": runtimeID, + "network_id": networkID, "network_exit_id": environment.Exit.ID, "network_exit": networkExit, } } +func gatewayGenerationPayload(environment hub.EnvironmentContext) map[string]any { + if environment.RuntimeCleanupBindingVersion > 0 { + return map[string]any{"binding_version": environment.RuntimeCleanupBindingVersion, "runtime_id": environment.RuntimeCleanupRuntimeID, + "network_id": environment.RuntimeCleanupNetworkID} + } + return map[string]any{"binding_version": environment.BindingVersion, "runtime_id": environment.RuntimeID, "network_id": environment.RuntimeNetworkID} +} + +func runtimeCleanupGeneration(environment hub.EnvironmentContext, bindingVersion int64, runtimeID string, networkIDs ...string) hub.EnvironmentContext { + environment.RuntimeCleanupBindingVersion = bindingVersion + environment.RuntimeCleanupInstanceID = environment.RuntimeInstanceID + environment.RuntimeCleanupRuntimeID = runtimeID + environment.RuntimeCleanupNetworkID = environment.RuntimeNetworkID + if len(networkIDs) == 1 { + environment.RuntimeCleanupNetworkID = networkIDs[0] + } + return environment +} + +func gatewayNetworkID(body []byte) string { + var envelope struct { + NetworkID string `json:"network_id"` + } + if json.Unmarshal(body, &envelope) == nil && gatewayGenerationIDPattern.MatchString(envelope.NetworkID) { + return envelope.NetworkID + } + return "" +} + +func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, + environment hub.EnvironmentContext, body []byte) error { + return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion, gatewayNetworkID(body)) +} + +func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, + environment hub.EnvironmentContext, bindingVersion int64, networkID string) error { + if networkID == "" { + pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "") + return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true)) + } + container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias) + if err != nil { + runtimeID := missingRuntimeID + if bindingVersion == environment.BindingVersion && environment.RuntimeID != "" { + runtimeID = environment.RuntimeID + } + pending := runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID) + return errors.Join(err, store.SetRuntimeCleanupPending(ctx, pending, true)) + } + if found && (container.BindingVersion != bindingVersion || container.NetworkID != networkID) { + pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID) + return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true)) + } + if found { + environment = runtimeCleanupGeneration(environment, bindingVersion, container.ID, networkID) + } else { + environment = runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID) + } + _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment) + return cleanupErr +} + // gatewayFailure 将网关错误转换为对调用方可读的失败；4xx 保留原状态，其余一律 502。 type gatewayFailure struct { status int @@ -200,6 +289,7 @@ type containerStatus struct { Endpoint string `json:"endpoint"` BindingVersion int64 `json:"binding_version"` NetworkExitID string `json:"network_exit_id"` + NetworkID string `json:"network_id"` ProxyReady bool `json:"proxy_ready"` } @@ -216,7 +306,41 @@ type envView struct { } func containerMatchesBinding(container containerStatus, environment hub.EnvironmentContext) bool { - return container.BindingVersion == environment.BindingVersion && container.NetworkExitID == environment.Exit.ID + if !validCreatedRuntime(container, environment, container.State == "running") || container.BindingVersion != environment.BindingVersion || + (container.NetworkExitID != environment.Exit.ID && (container.State == "running" || container.NetworkExitID != "")) { + return false + } + return true +} + +func validCreatedRuntime(created containerStatus, environment hub.EnvironmentContext, running bool) bool { + if !gatewayGenerationIDPattern.MatchString(created.ID) || + (created.NetworkID != "" && !gatewayGenerationIDPattern.MatchString(created.NetworkID)) { + return false + } + if !running { + return true + } + if created.NetworkID == "" { + return false + } + if environment.RuntimeInstanceID != "" || environment.RuntimeID != "" { + return environment.RuntimeInstanceID != "" && environment.RuntimeID == created.ID && + environment.RuntimeNetworkID != "" && created.NetworkID == environment.RuntimeNetworkID + } + return environment.RuntimeNetworkID == "" || created.NetworkID == environment.RuntimeNetworkID +} + +func accountRunnable(environment hub.EnvironmentContext) bool { + return environment.AccountStatus == "active" && environment.AuthorizationStatus == "authorized" +} + +func releaseRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error { + err := store.ReleaseRuntime(ctx, environment) + if errors.Is(err, hub.ErrConflict) { + return nil + } + return err } func registerHub(app *fiber.App, store hubStore) { @@ -528,6 +652,18 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network continue } container, found := containers[env.Gateway][env.Alias] + if !accountRunnable(environment) { + if found && container.State == "running" { + if err := stopEnvironmentRuntime(ctx, store, environment); err != nil { + return err + } + } else if gatewayRead[env.Gateway] { + if err := releaseRuntime(ctx, store, environment); err != nil { + return err + } + } + continue + } if found && container.State == "running" { if environment.Exit.ID == "" { continue @@ -541,6 +677,16 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network } ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, container) if auditRecovery { + current, contextErr := store.GetEnvironmentContext(ctx, environment.Alias) + if contextErr == nil { + action.RuntimeInstanceID, action.BindingVersion, action.NetworkExitID = + current.RuntimeInstanceID, current.BindingVersion, current.Exit.ID + } else { + action.RuntimeInstanceID = "" + action.BindingVersion = 0 + action.NetworkExitID = "" + restoreErr = errors.Join(restoreErr, contextErr) + } action.Outcome, action.ReasonCode = "succeeded", "runtime_reconciled" if !ready { action.Outcome, action.ReasonCode = "failed", "runtime_unavailable" @@ -556,7 +702,7 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network return restoreErr } } else if gatewayRead[env.Gateway] { - if err := store.ReleaseRuntime(ctx, env.Alias); err != nil { + if err := releaseRuntime(ctx, store, environment); err != nil { return err } } @@ -594,7 +740,7 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE return false, err } if containerMatchesBinding(container, environment) && container.ProxyReady { - _, err := store.ActivateRuntime(ctx, environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + _, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID) return err == nil, err } networkExit, err := gatewayNetworkExitFor(access, resolve) @@ -603,12 +749,12 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE } if containerMatchesBinding(container, environment) { status, _, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/proxy", - gatewayProxyPayload(environment, networkExit), 30*time.Second) + gatewayProxyPayload(environment, container.ID, container.NetworkID, networkExit), 30*time.Second) if callErr == nil && status == http.StatusNoContent { container.ProxyReady = true } if container.ProxyReady { - _, err := store.ActivateRuntime(ctx, environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + _, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID) return err == nil, err } } @@ -616,28 +762,45 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE if removeErr != nil { return false, removeErr } + environment, err = store.GetEnvironmentContext(ctx, environment.Alias) + if err != nil { + return false, err + } imageRef, err := store.ImageRef(ctx, environment.ImageVersion) if err != nil { return false, err } status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", - gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) + gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) if callErr != nil { - return false, gatewayUnreachable(callErr) + return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, body)) } if status != http.StatusCreated { - return false, gatewayRejected(status, body) + return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, body)) } - var created struct { - ID string `json:"id"` + var created containerStatus + if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) { + return false, errors.Join(errors.New("gateway returned an invalid runtime generation"), + reconcileGatewayCreate(ctx, store, target, environment, body)) } - if json.Unmarshal(body, &created) != nil || created.ID == "" { - return false, errors.New("gateway returned an invalid runtime id") - } - _, err = store.ActivateRuntime(ctx, environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID) + _, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID) return err == nil, err } +func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string) (hub.EnvironmentContext, error) { + if !validCreatedRuntime(containerStatus{ID: runtimeID, NetworkID: networkID}, environment, true) { + return hub.EnvironmentContext{}, hub.ErrConflict + } + current, err := store.ActivateRuntime(ctx, environment.Alias, runtimeID, environment.BindingVersion, environment.Exit.ID, networkID) + if err == nil { + return current, nil + } + cleanup := environment + cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtimeID, networkID + _, cleanupErr := removeGatewayRuntime(ctx, store, target, cleanup) + return hub.EnvironmentContext{}, errors.Join(err, cleanupErr) +} + func discardRuntime(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error { target, targetErr := store.GetGateway(ctx, environment.Gateway) if targetErr != nil { @@ -686,7 +849,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw } finish := func(outcome, reason string, current hub.EnvironmentContext) error { action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID - action.BindingVersion = current.BindingVersion + action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) } access, reason, err := verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID) @@ -709,6 +872,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return hubError(c, err) } } + running := accountRunnable(environment) imageRef, err := store.ImageRef(c.Context(), env.ImageVersion) if err != nil { _ = finish("failed", "image_unavailable", environment) @@ -725,6 +889,24 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, gatewayUnreachable(reconcileErr)) } + if found && !running { + if !containerMatchesBinding(container, environment) { + _ = finish("failed", "runtime_unavailable", environment) + return hubError(c, hub.ErrConflict) + } + if container.State == "running" { + stopped := environment + stopped.RuntimeID, stopped.RuntimeNetworkID = container.ID, container.NetworkID + if err := stopEnvironmentRuntime(c.Context(), store, stopped); err != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, err) + } + } + if err := finish("succeeded", "environment_reused", environment); err != nil { + return hubError(c, err) + } + return c.JSON(map[string]string{"alias": env.Alias}) + } if found && container.State == "running" { ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, container) if restoreErr != nil { @@ -746,51 +928,37 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return c.JSON(map[string]string{"alias": env.Alias}) } } - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) - if callErr != nil || status >= http.StatusInternalServerError { + status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", + gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) + if callErr != nil || status != http.StatusCreated { createErr := gatewayRejected(status, body) if callErr != nil { createErr = gatewayUnreachable(callErr) } - container, exists, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, env.Alias) - if reconcileErr == nil && exists && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { - environment, err = store.ActivateRuntime(c.Context(), env.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) - if err != nil { - _ = finish("failed", "runtime_persistence_failed", environment) - return hubError(c, err) - } - if err := finish("succeeded", "gateway_reconciled", environment); err != nil { - return hubError(c, err) - } - return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias}) - } + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf( - "gateway create result unknown; environment retained for reconciliation: %v", createErr)}) + return hubError(c, errors.Join(createErr, reconcileErr)) } _ = finish("failed", "gateway_create_failed", environment) return hubError(c, createErr) } - if status != http.StatusCreated { - _ = finish("failed", "gateway_rejected", environment) - return hubError(c, gatewayRejected(status, body)) - } - var createdRuntime struct { - ID string `json:"id"` - } - if json.Unmarshal(body, &createdRuntime) != nil || createdRuntime.ID == "" { + var createdRuntime containerStatus + if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) { + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway create result unknown; environment retained for reconciliation"}) + return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, + message: "gateway create result unknown; environment retained for reconciliation"}, reconcileErr)) } - environment, err = store.ActivateRuntime(c.Context(), env.Alias, createdRuntime.ID, environment.BindingVersion, environment.Exit.ID) - if err != nil { - _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment) - if cleanupErr != nil { - _ = finish("unknown", "cleanup_result_unknown", environment) - } else { - _ = finish("failed", "runtime_persistence_failed", environment) + if !running { + if err := finish("succeeded", "environment_created", environment); err != nil { + return hubError(c, err) } + return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias}) + } + environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID) + if err != nil { + _ = finish("failed", "runtime_persistence_failed", environment) return hubError(c, err) } if err := finish("succeeded", "environment_created", environment); err != nil { @@ -821,69 +989,120 @@ func lifecycleAction(store hubStore, probe networkExitProbe, resolve func(hub.Ne return hubError(c, err) } actionName := strings.Clone(c.Params("action")) - action := actionForEnvironment(actionName, environment) - if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { - return hubError(c, err) - } - finish := func(outcome, reason string, current hub.EnvironmentContext) error { - action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID - action.BindingVersion = current.BindingVersion - return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) - } if actionName == "start" { - return startBrowser(store, probe, resolve, c, environment, finish) - } - gateway, err := store.GetGateway(c.Context(), environment.Gateway) - if err != nil { - _ = finish("failed", "gateway_unavailable", environment) - return hubError(c, err) - } - if environment.RuntimeCleanupPending { - if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil { - _ = finish("unknown", "cleanup_result_unknown", environment) - return hubError(c, cleanupErr) + if !accountRunnable(environment) { + return hubError(c, hub.ErrConflict) } - if err := finish("succeeded", "environment_stopped", environment); err != nil { + action := actionForEnvironment(actionName, environment) + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { return hubError(c, err) } - return c.SendStatus(fiber.StatusNoContent) - } - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/"+actionName, nil, 30*time.Second) - if callErr != nil { - container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) - if reconcileErr != nil { - _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayUnreachable(callErr)) + finish := func(outcome, reason string, current hub.EnvironmentContext) error { + action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID + action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID + return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) } - if !found || container.State != "running" { - if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil { - _ = finish("failed", "runtime_release_failed", environment) - return hubError(c, err) - } - if err := finish("succeeded", "gateway_reconciled", environment); err != nil { - return hubError(c, err) - } - return c.SendStatus(fiber.StatusNoContent) - } - _ = finish("failed", "gateway_action_failed", environment) - return hubError(c, gatewayUnreachable(callErr)) + return startBrowser(store, probe, resolve, c, environment, finish) } - if status != http.StatusNoContent && status != http.StatusNotModified { - _ = finish("failed", "gateway_rejected", environment) - return hubError(c, gatewayRejected(status, body)) - } - if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil { - _ = finish("failed", "runtime_release_failed", environment) - return hubError(c, err) - } - if err := finish("succeeded", "environment_stopped", environment); err != nil { + if err := stopEnvironmentRuntime(c.Context(), store, environment); err != nil { return hubError(c, err) } return c.SendStatus(fiber.StatusNoContent) } +func stopAccountRuntime(ctx context.Context, store runtimeStopStore, accountID string) error { + environment, err := store.GetEnvironmentContextForAccount(ctx, accountID) + if errors.Is(err, hub.ErrNotFound) { + return nil + } + if err != nil { + return err + } + return stopEnvironmentRuntime(ctx, store, environment) +} + +func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error { + action := actionForEnvironment("stop", environment) + if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { + return err + } + finish := func(outcome, reason string) error { + action.Outcome, action.ReasonCode = outcome, reason + return store.AppendEnvironmentAction(ctx, "environment_action_finished", action) + } + gateway, err := store.GetGateway(ctx, environment.Gateway) + if err != nil { + _ = finish("failed", "gateway_unavailable") + return err + } + if environment.RuntimeCleanupPending { + if _, cleanupErr := removeGatewayRuntime(ctx, store, gateway, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown") + return cleanupErr + } + return finish("succeeded", "environment_stopped") + } + if environment.RuntimeID == "" { + container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias) + if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown") + return gatewayUnreachable(reconcileErr) + } + if !found { + return finish("succeeded", "environment_stopped") + } + if container.BindingVersion != environment.BindingVersion { + _ = finish("failed", "runtime_generation_conflict") + return hub.ErrConflict + } + environment = runtimeCleanupGeneration(environment, environment.BindingVersion, container.ID, container.NetworkID) + } else { + environment = runtimeCleanupGeneration(environment, environment.BindingVersion, environment.RuntimeID) + } + if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil { + _ = finish("failed", "runtime_release_failed") + return err + } + status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop", + gatewayGenerationPayload(environment), 30*time.Second) + if callErr != nil || status >= http.StatusInternalServerError { + container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias) + if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown") + if callErr != nil { + return gatewayUnreachable(callErr) + } + return gatewayRejected(status, body) + } + if !found || container.State != "running" { + if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil { + _ = finish("failed", "runtime_release_failed") + return err + } + return finish("succeeded", "gateway_reconciled") + } + _ = finish("failed", "gateway_action_failed") + if callErr != nil { + return gatewayUnreachable(callErr) + } + return gatewayRejected(status, body) + } + if status != http.StatusNoContent && status != http.StatusNotModified && status != http.StatusNotFound { + _ = finish("failed", "gateway_rejected") + return gatewayRejected(status, body) + } + if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil { + _ = finish("failed", "runtime_release_failed") + return err + } + return finish("succeeded", "environment_stopped") +} + func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx, environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error { + if !accountRunnable(environment) { + return hubError(c, hub.ErrConflict) + } access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID) if err != nil { if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil { @@ -907,6 +1126,9 @@ func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netwo if err != nil { return hubError(c, err) } + if !accountRunnable(environment) { + return hubError(c, hub.ErrConflict) + } } imageRef, err := store.ImageRef(c.Context(), environment.ImageVersion) if err != nil { @@ -932,7 +1154,7 @@ func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netwo return hubError(c, gatewayUnreachable(err)) } if found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { - environment, err = store.ActivateRuntime(c.Context(), environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, container.ID, container.NetworkID) if err != nil { _ = finish("failed", "runtime_persistence_failed", environment) return hubError(c, err) @@ -947,40 +1169,31 @@ func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netwo _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, removeErr) } - } else if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil { + } else if err := releaseRuntime(c.Context(), store, environment); err != nil { _ = finish("failed", "runtime_release_failed", environment) return hubError(c, err) } status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) - if callErr != nil || status >= http.StatusInternalServerError { - container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) + if callErr != nil || status != http.StatusCreated { + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}) + return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr)) } - if !found || container.State != "running" || !container.ProxyReady || !containerMatchesBinding(container, environment) { - _ = finish("failed", "gateway_create_failed", environment) - if callErr != nil { - return hubError(c, gatewayUnreachable(callErr)) - } - return hubError(c, gatewayRejected(status, body)) + _ = finish("failed", "gateway_create_failed", environment) + if callErr != nil { + return hubError(c, gatewayUnreachable(callErr)) } - environment, err = store.ActivateRuntime(c.Context(), environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) - } else { - if status != http.StatusCreated { - _ = finish("failed", "gateway_rejected", environment) - return hubError(c, gatewayRejected(status, body)) - } - var created struct { - ID string `json:"id"` - } - if json.Unmarshal(body, &created) != nil || created.ID == "" { - _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}) - } - environment, err = store.ActivateRuntime(c.Context(), environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID) + return hubError(c, gatewayRejected(status, body)) } + var created containerStatus + if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) { + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr)) + } + environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, created.ID, created.NetworkID) if err != nil { _ = finish("failed", "runtime_persistence_failed", environment) return hubError(c, err) @@ -1028,7 +1241,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Net } finish := func(outcome, reason string, current hub.EnvironmentContext) error { action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID - action.BindingVersion = current.BindingVersion + action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) } access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID) @@ -1051,6 +1264,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Net return hubError(c, err) } } + running := accountRunnable(environment) networkExit, err := gatewayNetworkExitFor(access, resolve) if err != nil { _ = finish("failed", "credential_unavailable", environment) @@ -1070,42 +1284,36 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Net _ = finish("failed", "persistence_failed", environment) return hubError(c, err) } - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) - var createdRuntime struct { - ID string `json:"id"` - } - if callErr != nil || status >= http.StatusInternalServerError { - container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) + running = accountRunnable(environment) + status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", + gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout) + var createdRuntime containerStatus + if callErr != nil || status != http.StatusCreated { + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) if reconcileErr != nil { _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}) + return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr)) } - if !found || container.State != "running" || !container.ProxyReady || !containerMatchesBinding(container, environment) { - _ = finish("failed", "gateway_create_failed", environment) - if callErr != nil { - return hubError(c, gatewayUnreachable(callErr)) - } - return hubError(c, gatewayRejected(status, body)) - } - createdRuntime.ID = container.ID - } else { - if status != http.StatusCreated { - _ = finish("failed", "gateway_rejected", environment) - return hubError(c, gatewayRejected(status, body)) - } - if json.Unmarshal(body, &createdRuntime) != nil || createdRuntime.ID == "" { - _ = finish("unknown", "gateway_result_unknown", environment) - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}) + _ = finish("failed", "gateway_create_failed", environment) + if callErr != nil { + return hubError(c, gatewayUnreachable(callErr)) } + return hubError(c, gatewayRejected(status, body)) } - environment, err = store.ActivateRuntime(c.Context(), environment.Alias, createdRuntime.ID, environment.BindingVersion, environment.Exit.ID) - if err != nil { - _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment) - if cleanupErr != nil { - _ = finish("unknown", "cleanup_result_unknown", environment) - } else { - _ = finish("failed", "runtime_persistence_failed", environment) + if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) { + reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body) + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr)) + } + if !running { + if err := finish("succeeded", "environment_upgraded", environment); err != nil { + return hubError(c, err) } + return c.SendStatus(fiber.StatusNoContent) + } + environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID) + if err != nil { + _ = finish("failed", "runtime_persistence_failed", environment) return hubError(c, err) } if err := finish("succeeded", "environment_upgraded", environment); err != nil { @@ -1134,41 +1342,39 @@ func prepareRuntimeCreate(ctx context.Context, store hubStore, resolve func(hub. func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (containerStatus, error) { status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", - gatewayCreatePayload(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout) + gatewayCreatePayloadForAccount(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout) + unknown := containerStatus{NetworkID: gatewayNetworkID(body)} if callErr == nil && status == http.StatusCreated { var created containerStatus - if json.Unmarshal(body, &created) == nil && created.ID != "" { - created.Alias, created.State = environment.Alias, "running" - created.BindingVersion, created.NetworkExitID, created.ProxyReady = environment.BindingVersion, environment.Exit.ID, true + if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, accountRunnable(environment)) { + created.Alias, created.BindingVersion = environment.Alias, environment.BindingVersion + if accountRunnable(environment) { + created.State, created.NetworkExitID, created.ProxyReady = "running", environment.Exit.ID, true + } else { + created.State = "exited" + } return created, nil } } - container, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias) - if reconcileErr == nil && found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { - return container, nil - } if callErr != nil { - return containerStatus{}, gatewayUnreachable(callErr) + return unknown, gatewayUnreachable(callErr) } if status != http.StatusCreated { - return containerStatus{}, gatewayRejected(status, body) + return unknown, gatewayRejected(status, body) } - if reconcileErr != nil { - return containerStatus{}, gatewayUnreachable(reconcileErr) - } - return containerStatus{}, errors.New("gateway returned an invalid runtime id") + return unknown, errors.New("gateway returned an invalid runtime generation") } func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, imageRef string) error { payload := gatewayCreatePayload(environment, imageRef, gatewayNetworkExit{}) - payload["stopped"] = true + payload["network_exit_id"], payload["stopped"] = "", true status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", payload, gatewayLongTimeout) if callErr == nil && status == http.StatusCreated { - return nil - } - container, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias) - if reconcileErr == nil && found && container.State != "running" && containerMatchesBinding(container, environment) { - return nil + var created containerStatus + if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, false) { + return nil + } + return errors.New("gateway returned an invalid stopped runtime generation") } if callErr != nil { return gatewayUnreachable(callErr) @@ -1176,23 +1382,54 @@ func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, enviro if status != http.StatusCreated { return gatewayRejected(status, body) } - if reconcileErr != nil { - return gatewayUnreachable(reconcileErr) - } return errors.New("gateway did not preserve the stopped runtime") } -func removeGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) { +func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) { removed := environment.RuntimeCleanupPending + if environment.RuntimeCleanupPending && + (environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" || + (environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "")) { + return false, hub.ErrReconcileRequired + } if !environment.RuntimeCleanupPending { - if err := store.SetRuntimeCleanupPending(ctx, environment.Alias, true); err != nil { + container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias) + if err != nil { + return false, err + } + bindingVersion, runtimeID, networkID := environment.BindingVersion, environment.RuntimeID, environment.RuntimeNetworkID + if environment.RuntimeCleanupBindingVersion > 0 { + bindingVersion, runtimeID, networkID = environment.RuntimeCleanupBindingVersion, + environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID + } + if found { + if container.BindingVersion != bindingVersion || runtimeID == missingRuntimeID || + (runtimeID != "" && runtimeID != container.ID) || (runtimeID == "" && networkID == "" && container.State == "running") { + return false, hub.ErrConflict + } + runtimeID = container.ID + if networkID != "" { + if container.NetworkID != "" && container.NetworkID != networkID { + return false, hub.ErrConflict + } + } else { + networkID = container.NetworkID + } + } else if environment.RuntimeID == "" { + runtimeID = missingRuntimeID + } + environment = runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID) + } + if !environment.RuntimeCleanupPending { + if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil { return false, err } } for attempt := 0; attempt < 2; attempt++ { - status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias, nil, 30*time.Second) + status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias, + gatewayGenerationPayload(environment), 30*time.Second) if callErr == nil && (status == http.StatusNoContent || status == http.StatusNotFound) { - return true, store.SetRuntimeCleanupPending(ctx, environment.Alias, false) + return true, store.SetRuntimeCleanupPending(ctx, environment, false) } if callErr == nil && status == http.StatusAccepted { removed = true @@ -1226,10 +1463,10 @@ func removeGatewayRuntime(ctx context.Context, store hubStore, target hub.Gatewa func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error), target hub.Gateway, environment hub.EnvironmentContext, previous containerStatus, found bool, prepared *runtimeCreateSpec) (bool, error) { if !found { - return true, store.ReleaseRuntime(ctx, environment.Alias) + return true, releaseRuntime(ctx, store, environment) } if environment.Exit.ID == "" { - if err := store.ReleaseRuntime(ctx, environment.Alias); err != nil { + if err := releaseRuntime(ctx, store, environment); err != nil { return false, err } if previous.State == "running" { @@ -1264,18 +1501,25 @@ func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub. return false, err } } - if err := store.ReleaseRuntime(ctx, environment.Alias); err != nil { + if err := releaseRuntime(ctx, store, environment); err != nil { + return false, err + } + cleanupContext, err := store.GetEnvironmentContext(ctx, environment.Alias) + if err != nil { return false, err } created, err := createGatewayRuntime(ctx, target, environment, spec) if err != nil { - _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment) + cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID) return false, errors.Join(err, cleanupErr) } - if previous.State == "running" { - _, err = store.ActivateRuntime(ctx, environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID) - } else { - status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop", nil, 30*time.Second) + if previous.State == "running" && accountRunnable(environment) { + _, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID) + } else if accountRunnable(environment) { + stopped := environment + stopped.RuntimeID, stopped.RuntimeNetworkID = created.ID, created.NetworkID + status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop", + gatewayGenerationPayload(stopped), 30*time.Second) if callErr != nil { err = gatewayUnreachable(callErr) } else if status != http.StatusNoContent { @@ -1400,9 +1644,14 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw if wasRunning { candidateRuntime, err = createGatewayRuntime(c.Context(), target, candidate, nextSpec) if err != nil { - _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before) + cleanupContext, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias) + if contextErr != nil { + _ = finish("unknown", "gateway_result_unknown", before) + return hubError(c, errors.Join(err, contextErr)) + } + cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID) if cleanupErr != nil { - _ = finish("unknown", "rebind_recovery_failed", before) + _ = finish("unknown", "gateway_result_unknown", before) return hubError(c, errors.Join(err, cleanupErr)) } current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias) @@ -1419,10 +1668,20 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return hubError(c, err) } } - after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, candidateRuntime.ID, before.BindingVersion) + runtimeID := "" + if accountRunnable(candidate) { + runtimeID = candidateRuntime.ID + } + after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, runtimeID, before.BindingVersion, candidateRuntime.NetworkID) if err != nil { if candidateRuntime.ID != "" { - _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before) + cleanup, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias) + if contextErr != nil || cleanup.BindingVersion != before.BindingVersion || cleanup.Exit.ID != before.Exit.ID { + _ = finish("unknown", "rebind_recovery_failed", before) + return hubError(c, errors.Join(err, contextErr, hub.ErrConflict)) + } + cleanup.RuntimeID, cleanup.RuntimeNetworkID = candidateRuntime.ID, candidateRuntime.NetworkID + _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, cleanup) if cleanupErr != nil { _ = finish("unknown", "rebind_recovery_failed", before) return hubError(c, errors.Join(err, cleanupErr)) @@ -1513,6 +1772,8 @@ func hubError(c fiber.Ctx, err error) error { switch { case errors.Is(err, hub.ErrInvalid): status, message = fiber.StatusBadRequest, hub.ErrInvalid.Error() + case errors.Is(err, hub.ErrReconcileRequired): + status, message = fiber.StatusConflict, hub.ErrReconcileRequired.Error() case errors.Is(err, hub.ErrConflict): status, message = fiber.StatusConflict, hub.ErrConflict.Error() case errors.Is(err, hub.ErrNotFound): diff --git a/cmd/control-plane/hub_test.go b/cmd/control-plane/hub_test.go index 0ab69ba..2def095 100644 --- a/cmd/control-plane/hub_test.go +++ b/cmd/control-plane/hub_test.go @@ -39,6 +39,18 @@ type memoryStore struct { gatewayFn func(name string) (hub.Gateway, error) } +type blockingRuntimeStopStore struct { + *hub.Store + entered chan struct{} + release chan struct{} +} + +func (s *blockingRuntimeStopStore) GetGateway(context.Context, string) (hub.Gateway, error) { + s.entered <- struct{}{} + <-s.release + return hub.Gateway{}, errors.New("gateway unavailable") +} + func newMemoryStore() *memoryStore { return &memoryStore{ gateways: map[string]hub.Gateway{}, @@ -226,6 +238,9 @@ func (s *memoryStore) DisableNetworkExit(_ context.Context, id string) (hub.Netw func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) { s.mu.Lock() if existing, ok := s.bindings[env.Alias]; ok { + if existing.AccountStatus == "" { + existing.AccountStatus, existing.AuthorizationStatus = "active", "authorized" + } s.mu.Unlock() return existing, false, nil } @@ -235,7 +250,8 @@ func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID } s.mu.Lock() defer s.mu.Unlock() - bound := hub.EnvironmentContext{Env: env, AccountID: accountID, BindingID: accountID, BindingVersion: 1, Exit: s.exits[exitID]} + bound := hub.EnvironmentContext{Env: env, AccountID: accountID, AccountStatus: "active", AuthorizationStatus: "authorized", + BindingID: accountID, BindingVersion: 1, Exit: s.exits[exitID]} s.bindings[env.Alias] = bound return bound, true, nil } @@ -243,13 +259,28 @@ func (s *memoryStore) GetEnvironmentContext(_ context.Context, alias string) (hu s.mu.Lock() defer s.mu.Unlock() if bound, ok := s.bindings[alias]; ok { + if bound.AccountStatus == "" { + bound.AccountStatus, bound.AuthorizationStatus = "active", "authorized" + } return bound, nil } env, ok := s.envs[alias] if !ok { return hub.EnvironmentContext{}, hub.ErrNotFound } - return hub.EnvironmentContext{Env: env, AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}, nil + return hub.EnvironmentContext{Env: env, AccountID: alias, AccountStatus: "active", AuthorizationStatus: "authorized", + BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}, nil +} +func (s *memoryStore) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) { + s.mu.Lock() + for alias, bound := range s.bindings { + if bound.AccountID == accountID { + s.mu.Unlock() + return s.GetEnvironmentContext(ctx, alias) + } + } + s.mu.Unlock() + return hub.EnvironmentContext{}, hub.ErrNotFound } func (s *memoryStore) ValidateEnvironmentRebind(_ context.Context, alias, exitID string, expectedBindingVersion int64) error { s.mu.Lock() @@ -263,7 +294,7 @@ func (s *memoryStore) ValidateEnvironmentRebind(_ context.Context, alias, exitID } return nil } -func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (hub.EnvironmentContext, error) { +func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkIDs ...string) (hub.EnvironmentContext, error) { s.mu.Lock() defer s.mu.Unlock() bound, ok := s.bindings[alias] @@ -273,60 +304,103 @@ func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtim if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending { return hub.EnvironmentContext{}, hub.ErrConflict } - bound.RuntimeInstanceID, bound.RuntimeID = "", "" + bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", "" if runtimeID != "" { bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID + if len(networkIDs) == 1 { + bound.RuntimeNetworkID = networkIDs[0] + } } bound.Exit, bound.BindingVersion = s.exits[exitID], bound.BindingVersion+1 s.bindings[alias] = bound return bound, nil } -func (s *memoryStore) ActivateRuntime(_ context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (hub.EnvironmentContext, error) { +func (s *memoryStore) ActivateRuntime(_ context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkIDs ...string) (hub.EnvironmentContext, error) { s.mu.Lock() defer s.mu.Unlock() + if len(networkIDs) != 1 || networkIDs[0] == "" { + return hub.EnvironmentContext{}, hub.ErrInvalid + } bound, ok := s.bindings[alias] if !ok { bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} } - if bound.BindingVersion != bindingVersion || bound.Exit.ID != exitID || bound.RuntimeCleanupPending { + if bound.AccountStatus == "" { + bound.AccountStatus, bound.AuthorizationStatus = "active", "authorized" + } + if bound.AccountStatus != "active" || bound.AuthorizationStatus != "authorized" || + bound.BindingVersion != bindingVersion || bound.Exit.ID != exitID || bound.RuntimeCleanupPending { return hub.EnvironmentContext{}, hub.ErrConflict } bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID + bound.RuntimeNetworkID = networkIDs[0] s.bindings[alias] = bound return bound, nil } -func (s *memoryStore) ReleaseRuntime(_ context.Context, alias string) error { +func (s *memoryStore) ReleaseRuntime(_ context.Context, environment hub.EnvironmentContext) error { s.mu.Lock() defer s.mu.Unlock() if s.releaseErr != nil { return s.releaseErr } - bound, ok := s.bindings[alias] + bound, ok := s.bindings[environment.Alias] if !ok { - bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + bound = hub.EnvironmentContext{Env: s.envs[environment.Alias], AccountID: environment.Alias, + BindingID: environment.Alias, BindingVersion: 1, Exit: s.exits["exit-1"]} } - bound.RuntimeInstanceID, bound.RuntimeID = "", "" - s.bindings[alias] = bound + if environment.RuntimeInstanceID == "" { + return nil + } + if bound.BindingID != environment.BindingID || bound.BindingVersion != environment.BindingVersion || + bound.RuntimeInstanceID != environment.RuntimeInstanceID { + return hub.ErrConflict + } + bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", "" + s.bindings[environment.Alias] = bound return nil } -func (s *memoryStore) SetRuntimeCleanupPending(_ context.Context, alias string, pending bool) error { +func (s *memoryStore) SetRuntimeCleanupPending(_ context.Context, environment hub.EnvironmentContext, pending bool) error { s.mu.Lock() defer s.mu.Unlock() + if pending && environment.RuntimeCleanupRuntimeID == "" { + return hub.ErrInvalid + } if s.cleanupPendingErr != nil && !s.cleanupPendingErrAfterMutation { return s.cleanupPendingErr } if pending && s.releaseErr != nil { return s.releaseErr } - bound, ok := s.bindings[alias] + bound, ok := s.bindings[environment.Alias] if !ok { - bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + bound = hub.EnvironmentContext{Env: s.envs[environment.Alias], AccountID: environment.Alias, + BindingID: environment.Alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + if bound.BindingID != environment.BindingID || bound.BindingVersion != environment.BindingVersion { + return hub.ErrConflict + } + if bound.RuntimeCleanupPending { + if bound.RuntimeCleanupBindingVersion != environment.RuntimeCleanupBindingVersion || + bound.RuntimeCleanupInstanceID != environment.RuntimeCleanupInstanceID || + bound.RuntimeCleanupRuntimeID != environment.RuntimeCleanupRuntimeID || + bound.RuntimeCleanupNetworkID != environment.RuntimeCleanupNetworkID { + return hub.ErrConflict + } + } else if pending && bound.RuntimeInstanceID != environment.RuntimeCleanupInstanceID { + return hub.ErrConflict } bound.RuntimeCleanupPending = pending if pending { - bound.RuntimeInstanceID, bound.RuntimeID = "", "" + bound.RuntimeCleanupBindingVersion = environment.RuntimeCleanupBindingVersion + bound.RuntimeCleanupInstanceID = environment.RuntimeCleanupInstanceID + bound.RuntimeCleanupRuntimeID = environment.RuntimeCleanupRuntimeID + bound.RuntimeCleanupNetworkID = environment.RuntimeCleanupNetworkID + bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", "" + } else { + bound.RuntimeCleanupBindingVersion = 0 + bound.RuntimeCleanupInstanceID, bound.RuntimeCleanupRuntimeID, bound.RuntimeCleanupNetworkID = "", "", "" } - s.bindings[alias] = bound + s.bindings[environment.Alias] = bound return s.cleanupPendingErr } func (s *memoryStore) AppendEnvironmentAction(_ context.Context, _ string, action hub.EnvironmentAction) error { @@ -336,6 +410,69 @@ func (s *memoryStore) AppendEnvironmentAction(_ context.Context, _ string, actio return nil } +func TestPhaseARuntimeLeaseRoutesAreInternal(t *testing.T) { + app := fiber.New() + registerPhaseA(app, nil, nil) + for _, request := range []struct { + method, path string + }{ + {http.MethodPost, "/api/phase-a/runtimes"}, + {http.MethodDelete, "/api/phase-a/runtimes/runtime-a"}, + } { + if response := do(app, request.method, request.path, `{}`); response.Code != http.StatusNotFound { + t.Fatalf("%s %s remained public: %d", request.method, request.path, response.Code) + } + } +} + +func TestPauseSerializesResumeAndRetainsClaimGateOnUnknownStop(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + fixture := newPostgresRebindFixture(t, databaseURL) + accountStore, err := phasea.Open(ctx, fixture.databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = accountStore.Close() }) + if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil { + t.Fatal(err) + } + if _, err := fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old"); err != nil { + t.Fatal(err) + } + blocking := &blockingRuntimeStopStore{Store: fixture.store, entered: make(chan struct{}, 1), release: make(chan struct{})} + app := fiber.New() + registerPhaseA(app, accountStore, blocking) + pauseDone := make(chan *httptest.ResponseRecorder, 1) + resumeDone := make(chan *httptest.ResponseRecorder, 1) + go func() { pauseDone <- do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", "") }() + select { + case <-blocking.entered: + case <-time.After(time.Second): + t.Fatal("pause did not enter runtime stop") + } + go func() { resumeDone <- do(app, http.MethodPost, "/api/phase-a/accounts/account-a/resume", "") }() + select { + case response := <-resumeDone: + t.Fatalf("resume bypassed the runtime operation lock: %d", response.Code) + case <-time.After(50 * time.Millisecond): + } + close(blocking.release) + if response := <-pauseDone; response.Code != http.StatusInternalServerError { + t.Fatalf("unknown stop returned %d: %s", response.Code, response.Body.String()) + } + if response := <-resumeDone; response.Code != http.StatusConflict { + t.Fatalf("resume reopened an unconverged lease: %d: %s", response.Code, response.Body.String()) + } + account, err := accountStore.GetAccount(ctx, "account-a") + if err != nil || account.RuntimeStatus != "paused" { + t.Fatalf("resume reopened the claim gate: account=%#v err=%v", account, err) + } +} + type recordedRequest struct { method string path string @@ -351,6 +488,7 @@ type fakeGateway struct { requests []recordedRequest containers []containerStatus failCreate int // 前 N 次 create 返回失败 + failCreateStatus int failDelete int // 前 N 次 delete 返回 500 且保留容器 deleteNotFound int failProxy bool @@ -417,8 +555,12 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { g.mu.Lock() if g.failCreate > 0 { g.failCreate-- + status := http.StatusConflict + if g.failCreateStatus != 0 { + status = g.failCreateStatus + } g.mu.Unlock() - response.WriteHeader(http.StatusConflict) + response.WriteHeader(status) _, _ = response.Write([]byte(`{"error":"alias already in use"}`)) return } @@ -430,17 +572,19 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { <-g.releaseCreate } state, proxyReady := "running", true + networkID := "network-id" if stopped, _ := body["stopped"].(bool); stopped { - state, proxyReady = "exited", false + state, proxyReady, networkID = "exited", false, "" } g.mu.Lock() g.containers = []containerStatus{{ ID: "container-id", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady, BindingVersion: int64(body["binding_version"].(float64)), NetworkExitID: body["network_exit_id"].(string), + NetworkID: networkID, }} g.mu.Unlock() response.WriteHeader(http.StatusCreated) - _, _ = response.Write([]byte(`{"id":"container-id","alias":"account-a"}`)) + _, _ = response.Write([]byte(`{"id":"container-id","alias":"account-a","network_id":"` + networkID + `"}`)) case request.Method == http.MethodGet && request.URL.Path == "/v1/browsers": g.mu.Lock() if g.failList > 0 { @@ -480,6 +624,12 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { _ = json.NewEncoder(response).Encode(containers) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/v1/browsers/"): g.mu.Lock() + if len(g.containers) > 0 && (body["runtime_id"] != g.containers[0].ID || + int64(body["binding_version"].(float64)) != g.containers[0].BindingVersion) { + g.mu.Unlock() + response.WriteHeader(http.StatusConflict) + return + } if g.failDelete > 0 { g.failDelete-- g.mu.Unlock() @@ -529,6 +679,12 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"): g.mu.Lock() + if len(g.containers) > 0 && (body["runtime_id"] != g.containers[0].ID || + int64(body["binding_version"].(float64)) != g.containers[0].BindingVersion) { + g.mu.Unlock() + response.WriteHeader(http.StatusConflict) + return + } if len(g.containers) > 0 { g.containers[0].State, g.containers[0].Status = "exited", "Exited" } @@ -536,6 +692,11 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/proxy"): g.mu.Lock() + if len(g.containers) > 0 && body["runtime_id"] != g.containers[0].ID { + g.mu.Unlock() + response.WriteHeader(http.StatusConflict) + return + } if g.failProxy { g.mu.Unlock() response.WriteHeader(http.StatusBadGateway) @@ -766,13 +927,200 @@ func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) { } } -func TestCreateBrowserReconcilesDisconnectedGateway(t *testing.T) { +func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadGateway} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers", createEnvBody) + if response.Code != http.StatusBadGateway { + t.Fatalf("expected failed gateway create, got %d: %s", response.Code, response.Body.String()) + } + requests := gateway.recorded() + if len(requests) != 1 || requests[0].method != http.MethodPost { + t.Fatalf("unknown network generation was discovered or cleaned by mutable alias: %#v", requests) + } + environment, err := store.GetEnvironmentContext(context.Background(), "account-a") + if err != nil || !environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "" { + t.Fatalf("unknown network generation was not retained for manual reconciliation: environment=%#v err=%v", environment, err) + } +} + +func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} + environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", + AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], + RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"} + store.bindings[environment.Alias] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "container-c1", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-n1", ProxyReady: true, + }}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + + err := reconcileGatewayCreate(context.Background(), store, target, environment, []byte(`{"error":"result unknown"}`)) + after := store.bindings[environment.Alias] + if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.containers) != 1 || + !after.RuntimeCleanupPending || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" { + t.Fatalf("unknown create reused the old generation: err=%v requests=%#v containers=%#v environment=%#v", + err, gateway.recorded(), gateway.containers, after) + } +} + +func TestValidCreatedRuntimeGeneration(t *testing.T) { + environment := hub.EnvironmentContext{RuntimeNetworkID: "network-n1"} + for _, test := range []struct { + name string + created containerStatus + environment hub.EnvironmentContext + running bool + want bool + }{ + {name: "active matching generation", created: containerStatus{ID: "container-c1", NetworkID: "network-n1"}, running: true, want: true}, + {name: "active empty runtime", created: containerStatus{NetworkID: "network-n1"}, running: true}, + {name: "active invalid runtime", created: containerStatus{ID: "container c1", NetworkID: "network-n1"}, running: true}, + {name: "active empty network", created: containerStatus{ID: "container-c1"}, running: true}, + {name: "active invalid network", created: containerStatus{ID: "container-c1", NetworkID: "network n1"}, running: true}, + {name: "active replacement network", created: containerStatus{ID: "container-c1", NetworkID: "network-n2"}, running: true}, + {name: "active existing matching generation", created: containerStatus{ID: "container-c1", NetworkID: "network-n1"}, running: true, want: true, + environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"}}, + {name: "active existing successor container", created: containerStatus{ID: "container-c2", NetworkID: "network-n1"}, running: true, + environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"}}, + {name: "active legacy lease without network", created: containerStatus{ID: "container-c2", NetworkID: "network-n2"}, running: true, + environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1"}}, + {name: "stopped empty network", created: containerStatus{ID: "container-c1"}, want: true}, + {name: "stopped invalid network", created: containerStatus{ID: "container-c1", NetworkID: "network n1"}}, + } { + t.Run(test.name, func(t *testing.T) { + current := environment + if test.environment.RuntimeInstanceID != "" { + current = test.environment + } + if got := validCreatedRuntime(test.created, current, test.running); got != test.want { + t.Fatalf("validCreatedRuntime(%#v, running=%v)=%v, want %v", test.created, test.running, got, test.want) + } + }) + } +} + +func TestLegacyActiveRuntimeWithoutNetworkGenerationDoesNotTouchSuccessor(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} + environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", + AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], + RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "container-c1"} + store.bindings[environment.Alias] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "container-c2", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-n2", ProxyReady: true, + }}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + + if containerMatchesBinding(gateway.containers[0], environment) { + t.Fatal("legacy active lease accepted a successor network generation") + } + if _, err := activateGatewayRuntime(context.Background(), store, target, environment, "container-c2", "network-n2"); !errors.Is(err, hub.ErrConflict) { + t.Fatalf("successor activation did not fail closed: %v", err) + } + if _, err := removeGatewayRuntime(context.Background(), store, target, environment); !errors.Is(err, hub.ErrConflict) { + t.Fatalf("legacy cleanup did not fence the successor: %v", err) + } + after := store.bindings[environment.Alias] + requests := gateway.recorded() + if len(requests) != 1 || requests[0].method != http.MethodGet || len(gateway.containers) != 1 || + after.RuntimeID != "container-c1" || after.RuntimeNetworkID != "" || after.RuntimeCleanupPending { + t.Fatalf("legacy C1 cleanup touched C2/N2: requests=%#v containers=%#v environment=%#v", requests, gateway.containers, after) + } +} + +func TestCreateBrowserInvalid201TracksNetworkCleanupGeneration(t *testing.T) { + for _, networkID := range []string{"network-n1", ""} { + name := "known network" + if networkID == "" { + name = "unknown network" + } + t.Run(name, func(t *testing.T) { + var deleteBody map[string]any + gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" { + response.WriteHeader(http.StatusUnauthorized) + return + } + switch request.Method { + case http.MethodPost: + response.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(response).Encode(map[string]string{"network_id": networkID}) + case http.MethodGet: + _, _ = response.Write([]byte(`[]`)) + case http.MethodDelete: + _ = json.NewDecoder(request.Body).Decode(&deleteBody) + if networkID == "" { + response.WriteHeader(http.StatusConflict) + return + } + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path) + } + })) + defer gatewayServer.Close() + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + response := do(app, http.MethodPost, "/api/browsers", createEnvBody) + if response.Code != http.StatusBadGateway || (networkID != "" && + (deleteBody["runtime_id"] != missingRuntimeID || deleteBody["network_id"] != networkID)) { + t.Fatalf("invalid 201 did not reconcile its generation: status=%d delete=%v body=%s", response.Code, deleteBody, response.Body.String()) + } + environment := store.bindings["account-a"] + if networkID != "" && environment.RuntimeCleanupPending { + t.Fatalf("known network cleanup did not converge: %#v", environment) + } + if networkID == "" && (!environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "") { + t.Fatalf("unknown generation was not retained for manual reconciliation: %#v", environment) + } + }) + } +} + +func TestCreateBrowserRejectsActiveRuntimeWithoutNetworkGeneration(t *testing.T) { + requests := 0 + gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + requests++ + if request.Method != http.MethodPost { + t.Fatalf("active NULL network response triggered alias reconciliation: %s %s", request.Method, request.URL.Path) + } + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"id":"container-c1","network_id":""}`)) + })) + defer gatewayServer.Close() + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + response := do(app, http.MethodPost, "/api/browsers", createEnvBody) + environment := store.bindings["account-a"] + if response.Code != http.StatusBadGateway || requests != 1 || !environment.RuntimeCleanupPending || + environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "" || environment.RuntimeID != "" { + t.Fatalf("active NULL network generation was activated: status=%d requests=%d environment=%#v body=%s", + response.Code, requests, environment, response.Body.String()) + } +} + +func TestCreateBrowserDoesNotDiscoverDisconnectedCreateByAlias(t *testing.T) { for _, test := range []struct { name string reconcile bool wantStatus int }{ - {name: "completed create", reconcile: true, wantStatus: http.StatusCreated}, + {name: "completed create", reconcile: true, wantStatus: http.StatusBadGateway}, {name: "unknown result", wantStatus: http.StatusBadGateway}, } { t.Run(test.name, func(t *testing.T) { @@ -826,18 +1174,20 @@ func TestCreateBrowserReconcilesDisconnectedGateway(t *testing.T) { if _, err := store.GetEnv(context.Background(), "account-a"); err != nil { t.Fatalf("unknown gateway result must retain the environment for reconciliation: %v", err) } - wantOutcome := "unknown" - if test.reconcile { - wantOutcome = "succeeded" + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" { + t.Fatalf("gateway result without an immutable generation must be unknown: %#v", store.actions) } - if len(store.actions) != 2 || store.actions[1].Outcome != wantOutcome { - t.Fatalf("gateway reconciliation outcome must be audited as %s: %#v", wantOutcome, store.actions) + mu.Lock() + wasCreated := created + mu.Unlock() + if !wasCreated || store.bindings["account-a"].RuntimeCleanupNetworkID != "" { + t.Fatalf("disconnected create lost its manual reconcile marker: %#v", store.bindings["account-a"]) } }) } } -func TestCreateBrowserReconcilesGatewayBadGateway(t *testing.T) { +func TestCreateBrowserDoesNotDiscoverBadGatewayCreateByAlias(t *testing.T) { for _, test := range []struct { name string missingReads int @@ -877,8 +1227,8 @@ func TestCreateBrowserReconcilesGatewayBadGateway(t *testing.T) { registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) - if response.Code != http.StatusCreated { - t.Fatalf("expected reconciled create, status=%d body=%s", response.Code, response.Body.String()) + if response.Code != http.StatusBadGateway { + t.Fatalf("expected unknown create, status=%d body=%s", response.Code, response.Body.String()) } if _, err := store.GetEnv(context.Background(), "account-a"); err != nil { t.Fatalf("reconciled create must retain DB state: err=%v", err) @@ -886,8 +1236,8 @@ func TestCreateBrowserReconcilesGatewayBadGateway(t *testing.T) { mu.Lock() gotCalls := listCalls mu.Unlock() - if gotCalls != test.missingReads+1 { - t.Fatalf("expected %d reconciliation reads, got %d", test.missingReads+1, gotCalls) + if gotCalls != 0 { + t.Fatalf("unknown network generation was inspected by alias %d times", gotCalls) } }) } @@ -916,7 +1266,7 @@ func TestListBrowsersMergesLiveGatewayState(t *testing.T) { gateway := &fakeGateway{ token: "unit-test-gateway-token", containers: []containerStatus{ - {ID: "id-1", Alias: "account-a", State: "running", Status: "Up", Endpoint: "http://creatorhub-browser-account-a:9222", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}, + {ID: "id-1", Alias: "account-a", State: "running", Status: "Up", Endpoint: "http://creatorhub-browser-account-a:9222", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-id", ProxyReady: true}, }, } app := newTestApp(t, store, gateway) @@ -951,7 +1301,7 @@ func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) { Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 3, Exit: store.exits["exit-1"], } gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", ProxyReady: false, + ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", NetworkID: "network-id", ProxyReady: false, }}} app := newTestApp(t, store, gateway) @@ -966,7 +1316,8 @@ func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) { if len(requests) != 2 || requests[0].path != "/v1/browsers" || requests[1].path != "/v1/browsers/account-a/proxy" { t.Fatalf("expected list then proxy recovery without rebuild: %#v", requests) } - if requests[1].body["binding_version"] != float64(3) || requests[1].body["network_exit_id"] != "exit-1" { + if requests[1].body["binding_version"] != float64(3) || requests[1].body["runtime_id"] != "container-id" || + requests[1].body["network_exit_id"] != "exit-1" { t.Fatalf("proxy recovery did not use the current binding: %#v", requests[1].body) } } @@ -977,19 +1328,19 @@ func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing. store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"]} _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "network-old", }}} app := newTestApp(t, store, gateway) - if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { - t.Fatalf("gateway restart rebuild failed: %d %s", response.Code, response.Body.String()) + if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusConflict { + t.Fatalf("lease-free running orphan was not fenced: %d %s", response.Code, response.Body.String()) } requests := gateway.recorded() - if len(requests) != 4 || requests[1].path != "/v1/browsers/account-a/proxy" || requests[2].method != http.MethodDelete || requests[3].path != "/v1/browsers" { - t.Fatalf("failed proxy recovery must preserve Profile by rebuilding the container: %#v", requests) + if len(requests) != 3 || requests[1].path != "/v1/browsers/account-a/proxy" { + t.Fatalf("unproven orphan generation reached rebuild side effects: %#v", requests) } - if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" { - t.Fatalf("rebuilt runtime was not activated: %q", runtime) + if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { + t.Fatalf("unproven orphan generation was activated: %q", runtime) } } @@ -1008,11 +1359,11 @@ func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String()) } requests := gateway.recorded() - if len(requests) != 2 || requests[0].method != http.MethodDelete || requests[0].path != "/v1/browsers/account-a" || - requests[1].method != http.MethodPost || requests[1].path != "/v1/browsers" { + if len(requests) < 2 || requests[len(requests)-2].method != http.MethodDelete || requests[len(requests)-2].path != "/v1/browsers/account-a" || + requests[len(requests)-1].method != http.MethodPost || requests[len(requests)-1].path != "/v1/browsers" { t.Fatalf("upgrade must delete then recreate: %#v", requests) } - payload := requests[1].body + payload := requests[len(requests)-1].body if payload["image"] != "registry.example/browser:144" || payload["volume"] != "creatorhub-profile-account-a" { t.Fatalf("upgrade must reuse the profile volume and switch image: %#v", payload) } @@ -1104,20 +1455,25 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { if err != nil { t.Fatal(err) } - if after.ImageVersion != "149" || after.BindingVersion != before.BindingVersion+1 || after.RuntimeID != "container-id" { - t.Fatalf("upgrade did not activate the committed PostgreSQL binding: before=%#v after=%#v", before, after) + if after.ImageVersion != "149" || after.BindingVersion != before.BindingVersion+1 || after.RuntimeID != "" { + t.Fatalf("paused upgrade did not preserve the stopped-runtime contract: before=%#v after=%#v", before, after) } requests := gateway.recorded() - if len(requests) != 2 || requests[1].body["binding_version"] != float64(after.BindingVersion) || - requests[1].body["network_exit_id"] != after.Exit.ID { + if len(requests) < 2 || requests[len(requests)-1].body["binding_version"] != float64(after.BindingVersion) || + requests[len(requests)-1].body["network_exit_id"] != "" || requests[len(requests)-1].body["stopped"] != true { t.Fatalf("gateway labels diverged from the committed binding: %#v", requests) } + setFixtureAccountStatus(t, databaseURL, "active") + after, err = store.ActivateRuntime(ctx, "account-a", "stale-container", after.BindingVersion, after.Exit.ID, "network-old") + if err != nil { + t.Fatal(err) + } gateway.mu.Lock() gateway.failDelete = 1 gateway.containers = []containerStatus{{ - ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: after.BindingVersion - 1, - NetworkExitID: after.Exit.ID, ProxyReady: true, + ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: after.BindingVersion, + NetworkExitID: "stale-exit", NetworkID: "network-old", ProxyReady: true, }} gateway.mu.Unlock() response = do(app, http.MethodGet, "/api/browsers", "") @@ -1136,17 +1492,20 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { if err != nil || released.RuntimeCleanupPending { t.Fatalf("confirmed cleanup remained pending: %#v err=%v", released, err) } - if _, err := store.ActivateRuntime(ctx, "account-a", "coherent-container", released.BindingVersion, released.Exit.ID); err != nil { + active, err := store.ActivateRuntime(ctx, "account-a", "coherent-container", released.BindingVersion, released.Exit.ID, "network-coherent") + if err != nil { t.Fatal(err) } - if err := store.ReleaseRuntime(ctx, "account-a"); err != nil { + pending := runtimeCleanupGeneration(active, active.BindingVersion, active.RuntimeID, active.RuntimeNetworkID) + if err := store.SetRuntimeCleanupPending(ctx, pending, true); err != nil { t.Fatal(err) } + setFixtureAccountStatus(t, databaseURL, "paused") gateway.mu.Lock() gateway.failDelete = 1 gateway.containers = []containerStatus{{ ID: "coherent-container", Alias: "account-a", State: "running", BindingVersion: released.BindingVersion, - NetworkExitID: released.Exit.ID, ProxyReady: true, + NetworkExitID: released.Exit.ID, NetworkID: "network-coherent", ProxyReady: true, }} gateway.mu.Unlock() response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+released.Exit.ID+`"}`) @@ -1160,24 +1519,50 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { } type postgresRebindFixture struct { - store *hub.Store - db *sql.DB - gateway *fakeGateway - bound hub.EnvironmentContext - exit hub.NetworkExit + store *hub.Store + db *sql.DB + databaseURL string + gateway *fakeGateway + bound hub.EnvironmentContext + exit hub.NetworkExit } type cleanupCommitUnknownStore struct { hubStore } -func (s cleanupCommitUnknownStore) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error { - if err := s.hubStore.SetRuntimeCleanupPending(ctx, alias, pending); err != nil { +func setFixtureAccountStatus(t *testing.T, databaseURL, status string) { + t.Helper() + db, err := sql.Open("pgx", databaseURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`UPDATE social_account SET status = $1 WHERE id = 'account-a'`, status); err != nil { + t.Fatal(err) + } +} + +func (s cleanupCommitUnknownStore) SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error { + if err := s.hubStore.SetRuntimeCleanupPending(ctx, environment, pending); err != nil { return err } return errors.New("cleanup commit result unknown") } +type failContextRefreshStore struct { + hubStore + reads int +} + +func (s *failContextRefreshStore) GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) { + s.reads++ + if s.reads >= 3 { + return hub.EnvironmentContext{}, errors.New("context refresh unavailable") + } + return s.hubStore.GetEnvironmentContext(ctx, alias) +} + func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFixture { t.Helper() ctx := context.Background() @@ -1236,7 +1621,163 @@ func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFi t.Fatal(err) } t.Cleanup(func() { _ = db.Close() }) - return postgresRebindFixture{store: store, db: db, gateway: gateway, bound: bound, exit: exit} + return postgresRebindFixture{store: store, db: db, databaseURL: databaseURL, gateway: gateway, bound: bound, exit: exit} +} + +func TestPauseClosesClaimGateBeforeStoppingRuntime(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + for _, test := range []struct { + name string + gatewayEndpoint string + wantStatus int + wantOutcome string + wantReason string + }{ + {name: "confirmed stop", wantStatus: http.StatusNoContent, wantOutcome: "succeeded", wantReason: "environment_stopped"}, + {name: "gateway unreachable", gatewayEndpoint: "http://127.0.0.1:1", wantStatus: http.StatusBadGateway, wantOutcome: "unknown", wantReason: "gateway_result_unknown"}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + fixture := newPostgresRebindFixture(t, databaseURL) + accountStore, err := phasea.Open(ctx, fixture.databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = accountStore.Close() }) + if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil { + t.Fatal(err) + } + account, err := accountStore.GetAccount(ctx, "account-a") + if err != nil { + t.Fatal(err) + } + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "active-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-active") + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "active-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + if test.gatewayEndpoint != "" { + if _, err := fixture.db.ExecContext(ctx, `UPDATE gateway SET endpoint = $1 WHERE name = 'gw-1'`, test.gatewayEndpoint); err != nil { + t.Fatal(err) + } + } + if err := accountStore.CreateDraft(ctx, phasea.Draft{ID: "pause-draft", AccountID: "account-a", Version: 1, Content: "pause"}); err != nil { + t.Fatal(err) + } + if err := accountStore.Confirm(ctx, phasea.Confirmation{ + ID: "pause-confirmation", AccountID: "account-a", AccountVersion: account.Version, + DraftID: "pause-draft", DraftVersion: 1, Version: 1, + }); err != nil { + t.Fatal(err) + } + if _, inserted, err := accountStore.Enqueue(ctx, phasea.Task{ + ID: "pause-task", IdempotencyKey: "pause-idempotency", AccountID: "account-a", AccountVersion: account.Version, + DraftID: "pause-draft", DraftVersion: 1, ConfirmationID: "pause-confirmation", ConfirmationVersion: 1, + }); err != nil || !inserted { + t.Fatalf("enqueue pause task: inserted=%v err=%v", inserted, err) + } + if _, err := fixture.db.ExecContext(ctx, ` + INSERT INTO execution_attempt (id, task_id) VALUES ('pause-attempt', 'pause-task'); + UPDATE operation_task SET state = 'executing', current_attempt_id = 'pause-attempt', + lease_owner = 'worker-pause', lease_until = now() + interval '1 minute' + WHERE id = 'pause-task'`); err != nil { + t.Fatal(err) + } + + app := fiber.New() + registerPhaseA(app, accountStore, fixture.store) + response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", "") + if response.Code != test.wantStatus { + t.Fatalf("pause returned %d, want %d: %s", response.Code, test.wantStatus, response.Body.String()) + } + paused, err := accountStore.GetAccount(ctx, "account-a") + if err != nil || paused.RuntimeStatus != "paused" { + t.Fatalf("claim gate remained open: account=%#v err=%v", paused, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM operation_task WHERE id = 'pause-task' AND state = 'needs_confirmation'`, 1) + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM execution_attempt WHERE id = 'pause-attempt' AND outcome = 'uncertain'`, 1) + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM audit_event WHERE task_id = 'pause-task' AND attempt_id = 'pause-attempt' + AND browser_env_alias = 'account-a' AND network_exit_id IS NOT NULL AND runtime_instance_id IS NOT NULL + AND binding_version = 1 AND reason_code = 'account_paused'`, 1) + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, `SELECT outcome, reason_code FROM audit_event + WHERE account_id = 'account-a' AND action = 'stop' AND event_type = 'environment_action_finished' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil || outcome != test.wantOutcome || reason != test.wantReason { + t.Fatalf("stop audit: outcome=%q reason=%q err=%v", outcome, reason, err) + } + }) + } +} + +func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + fixture := newPostgresRebindFixture(t, databaseURL) + accountStore, err := phasea.Open(ctx, fixture.databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = accountStore.Close() }) + fixture.gateway.containers = []containerStatus{{ + ID: "stopped-container", Alias: fixture.bound.Alias, State: "exited", + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + app := fiber.New() + registerPhaseA(app, accountStore, fixture.store) + + if err := accountStore.ResumeAccount(ctx, fixture.bound.AccountID); err != nil { + t.Fatal(err) + } + if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", ""); response.Code != http.StatusNoContent { + t.Fatalf("pause stopped container without lease: %d %s", response.Code, response.Body.String()) + } + if err := accountStore.ResumeAccount(ctx, fixture.bound.AccountID); err != nil { + t.Fatal(err) + } + fixture.gateway.mu.Lock() + fixture.gateway.containers[0].State = "running" + fixture.gateway.mu.Unlock() + if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/revoke", ""); response.Code != http.StatusNoContent { + t.Fatalf("revoke running orphan without lease: %d %s", response.Code, response.Body.String()) + } + + after, err := fixture.store.GetEnvironmentContext(ctx, fixture.bound.Alias) + if err != nil || after.RuntimeID != "" || after.RuntimeCleanupPending { + t.Fatalf("lease-free stop did not converge: %#v err=%v", after, err) + } + account, err := accountStore.GetAccount(ctx, fixture.bound.AccountID) + if err != nil || account.AuthorizationStatus != "revoked" || account.RuntimeStatus != "paused" { + t.Fatalf("revoke gate did not remain closed: %#v err=%v", account, err) + } + fixture.gateway.mu.Lock() + container := fixture.gateway.containers[0] + fixture.gateway.mu.Unlock() + if container.State != "exited" { + t.Fatalf("running orphan was not stopped: %#v", container) + } + requests := fixture.gateway.recorded() + stopCalls := 0 + for _, request := range requests { + if request.method == http.MethodPost && request.path == "/v1/browsers/account-a/stop" { + stopCalls++ + if request.body["runtime_id"] != "stopped-container" || request.body["binding_version"] != float64(fixture.bound.BindingVersion) { + t.Fatalf("stop used the wrong discovered generation: %#v", request) + } + } + } + if stopCalls != 2 { + t.Fatalf("pause/revoke did not reconcile both lease-free containers: %#v", requests) + } } func installCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB, condition string) { @@ -1281,15 +1822,12 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { state string nullBinding bool }{ - {name: "active runtime after DELETE", race: "active", state: "running"}, - {name: "executing task after DELETE", race: "executing", state: "running"}, - {name: "binding version after DELETE", race: "version", state: "running"}, - {name: "image upgrade blocked during DELETE", race: "upgrade", state: "running"}, {name: "stopped container after DELETE", race: "version", state: "exited"}, {name: "stopped NULL binding after DELETE", race: "version", state: "exited", nullBinding: true}, } { t.Run(test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + var err error if test.race == "upgrade" { if err := fixture.store.CreateImage(ctx, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}); err != nil { t.Fatal(err) @@ -1299,7 +1837,22 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { if _, err := fixture.db.ExecContext(ctx, `UPDATE environment_binding SET network_exit_id = NULL WHERE browser_env_alias = 'account-a'`); err != nil { t.Fatal(err) } - var err error + fixture.bound, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil { + t.Fatal(err) + } + } + if test.state == "running" { + setFixtureAccountStatus(t, fixture.databaseURL, "active") + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, + fixture.bound.Exit.ID, "network-old") + if err != nil { + t.Fatal(err) + } + pending := runtimeCleanupGeneration(fixture.bound, fixture.bound.BindingVersion, "old-container", "network-old") + if err := fixture.store.SetRuntimeCleanupPending(ctx, pending, true); err != nil { + t.Fatal(err) + } fixture.bound, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil { t.Fatal(err) @@ -1311,6 +1864,9 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { ID: "old-container", Alias: "account-a", State: test.state, ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} + if test.state == "running" { + fixture.gateway.containers[0].NetworkID = "network-old" + } app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) server := httptest.NewServer(adaptor.FiberApp(app)) @@ -1344,8 +1900,8 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { switch test.race { case "active": _, err = tx.ExecContext(ctx, ` - INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) - VALUES ('runtime-race', 'account-a', 'account-a', 'old-container', now() + interval '1 minute')`) + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) + VALUES ('runtime-race', 'account-a', 'account-a', 1, 'old-container', now() + interval '1 minute')`) case "executing": _, err = tx.ExecContext(ctx, ` INSERT INTO content_draft (id, account_id, version, content) VALUES ('draft-rebind', 'account-a', 1, 'test'); @@ -1393,17 +1949,12 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { fixture.gateway.mu.Lock() containers := append([]containerStatus{}, fixture.gateway.containers...) fixture.gateway.mu.Unlock() - if test.nullBinding { - if after.RuntimeID != "" || len(containers) != 1 || containers[0].State == "running" || - !containerMatchesBinding(containers[0], after) { - t.Fatalf("NULL binding recovery must restore a network-disabled stopped container without a lease: after=%#v containers=%#v", after, containers) + if test.race == "version" { + if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(containers) != 0 { + t.Fatalf("stale cleanup clear crossed the new binding generation: after=%#v containers=%#v", after, containers) } - } else if test.state == "exited" { - if after.RuntimeID != "" || len(containers) != 1 || containers[0].State != "exited" || !containerMatchesBinding(containers[0], after) { - t.Fatalf("stopped runtime was not restored: after=%#v containers=%#v", after, containers) - } - } else if after.RuntimeID == "" || len(containers) != 1 || containers[0].State != "running" || !containerMatchesBinding(containers[0], after) { - t.Fatalf("running runtime was not restored coherently: after=%#v containers=%#v", after, containers) + } else if after.RuntimeID != "" || len(containers) != 1 || containers[0].State == "running" || !containerMatchesBinding(containers[0], after) { + t.Fatalf("paused recovery did not preserve a stopped container without a lease: after=%#v containers=%#v", after, containers) } if test.race == "upgrade" { requests := fixture.gateway.recorded() @@ -1427,6 +1978,8 @@ func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { expectedOutcome, expectedReason := "failed", "rebind_not_allowed" if test.race == "upgrade" { expectedOutcome, expectedReason = "succeeded", "environment_rebound" + } else if test.race == "version" { + expectedOutcome, expectedReason = "unknown", "gateway_result_unknown" } if outcome != expectedOutcome || reason != expectedReason { t.Fatalf("recovered conflict audit mismatch: outcome=%s reason=%s", outcome, reason) @@ -1452,14 +2005,15 @@ func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } fixture.gateway.containers = []containerStatus{{ ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", }} fixture.gateway.cleanupPending = test.cleanupPending fixture.gateway.disconnectDelete = test.disconnectDelete @@ -1492,6 +2046,7 @@ func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) { t.Fatalf("confirmed cleanup remained pending: %#v err=%v", after, err) } + setFixtureAccountStatus(t, fixture.databaseURL, "paused") response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+fixture.exit.ID+`"}`) if response.Code != http.StatusOK { t.Fatalf("rebind retry failed after cleanup confirmation: %d: %s", response.Code, response.Body.String()) @@ -1527,7 +2082,7 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { store.cleanupPendingErr = errors.New("injected rollback") removed, err := removeGatewayRuntime(context.Background(), store, target, environment) - if err == nil || removed || len(gateway.recorded()) != 0 { + if err == nil || removed || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { t.Fatalf("failed pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) } after := store.bindings[environment.Alias] @@ -1542,7 +2097,7 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { store.cleanupPendingErrAfterMutation = true removed, err := removeGatewayRuntime(context.Background(), store, target, environment) - if err == nil || removed || len(gateway.recorded()) != 0 { + if err == nil || removed || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { t.Fatalf("unknown pre-mark result touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) } after := store.bindings[environment.Alias] @@ -1562,7 +2117,8 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { for _, status := range []int{http.StatusNoContent, http.StatusNotFound} { t.Run(fmt.Sprintf("clear rollback after %d", status), func(t *testing.T) { store, gateway, environment, target := newRuntime() - if err := store.SetRuntimeCleanupPending(context.Background(), environment.Alias, true); err != nil { + environment = runtimeCleanupGeneration(environment, environment.BindingVersion, environment.RuntimeID) + if err := store.SetRuntimeCleanupPending(context.Background(), environment, true); err != nil { t.Fatal(err) } environment = store.bindings[environment.Alias] @@ -1587,6 +2143,66 @@ func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { } }) } + + t.Run("empty runtime rejects a discovered different binding generation", func(t *testing.T) { + store, gateway, environment, target := newRuntime() + environment.RuntimeInstanceID, environment.RuntimeID = "", "" + store.bindings[environment.Alias] = environment + gateway.containers[0].BindingVersion = environment.BindingVersion + 1 + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if !errors.Is(err, hub.ErrConflict) || removed || len(gateway.recorded()) != 1 || + store.bindings[environment.Alias].RuntimeCleanupPending { + t.Fatalf("mismatched discovered generation was adopted: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) + } + }) + + t.Run("confirmed absence uses a non-empty cleanup generation", func(t *testing.T) { + store, gateway, environment, target := newRuntime() + environment.RuntimeInstanceID, environment.RuntimeID = "", "" + store.bindings[environment.Alias] = environment + gateway.containers = nil + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending { + t.Fatalf("absent runtime cleanup failed: removed=%v err=%v after=%#v", removed, err, store.bindings[environment.Alias]) + } + requests := gateway.recorded() + if len(requests) == 0 || requests[len(requests)-1].method != http.MethodDelete || + requests[len(requests)-1].body["runtime_id"] != missingRuntimeID { + t.Fatalf("absent runtime reused an empty generation: %#v", requests) + } + }) + + t.Run("known database runtime survives missing gateway container", func(t *testing.T) { + store, gateway, environment, target := newRuntime() + gateway.containers = nil + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err != nil || !removed { + t.Fatalf("known runtime cleanup failed: removed=%v err=%v", removed, err) + } + after := store.bindings[environment.Alias] + if after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("known runtime cleanup did not converge: %#v", after) + } + requests := gateway.recorded() + if len(requests) == 0 || requests[len(requests)-1].method != http.MethodDelete || + requests[len(requests)-1].body["runtime_id"] != environment.RuntimeID { + t.Fatalf("known runtime id was replaced by an empty generation: %#v", requests) + } + }) + + t.Run("legacy pending requires manual reconcile before gateway mutation", func(t *testing.T) { + store, gateway, environment, target := newRuntime() + environment.RuntimeCleanupPending = true + store.bindings[environment.Alias] = environment + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if !errors.Is(err, hub.ErrReconcileRequired) || removed || len(gateway.recorded()) != 0 { + t.Fatalf("legacy pending reached the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) + } + }) } func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { @@ -1605,12 +2221,12 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", } store.cleanupPendingErr = errors.New("cleanup state unavailable") store.cleanupPendingErrAfterMutation = !test.commitKnown gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "network-old", }}} app := newTestApp(t, store, gateway) @@ -1626,9 +2242,14 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("commit-unknown lost its retry marker: %#v", after) } - if len(gateway.containers) != 1 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { + if len(gateway.containers) != 1 { t.Fatalf("failed cleanup mark touched the old container: containers=%#v requests=%#v", gateway.containers, gateway.recorded()) } + for _, request := range gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("failed cleanup mark reached a gateway side effect: %#v", gateway.recorded()) + } + } if len(store.actions) != 2 || store.actions[1].Outcome != test.outcome || store.actions[1].ReasonCode != test.reason { t.Fatalf("restore audit mismatch: %#v", store.actions) } @@ -1678,9 +2299,14 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("discard commit-unknown lost its retry marker: %#v", after) } - if len(gateway.containers) != 1 || len(gateway.recorded()) != 0 { + if len(gateway.containers) != 1 { t.Fatalf("failed discard mark touched the gateway: containers=%#v requests=%#v", gateway.containers, gateway.recorded()) } + for _, request := range gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("failed discard mark reached a gateway side effect: %#v", gateway.recorded()) + } + } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" { t.Fatalf("discard audit mismatch: %#v", store.actions) } @@ -1708,10 +2334,10 @@ func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", } gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, containers: []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "network-old", }}} app := newTestApp(t, store, gateway) body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}` @@ -1744,6 +2370,42 @@ func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) { } } +func TestRemoveGatewayRuntimePreservesKnownNetworkGeneration(t *testing.T) { + for _, test := range []struct { + name string + observedNetwork string + wantConflict bool + }{ + {name: "legacy gateway omits network", observedNetwork: ""}, + {name: "replacement network conflicts", observedNetwork: "network-n2", wantConflict: true}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} + environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", + AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], + RuntimeInstanceID: "runtime-instance", RuntimeID: "container-c1", RuntimeNetworkID: "network-n1"} + store.bindings["account-a"] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "container-c1", Alias: "account-a", + State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: test.observedNetwork, ProxyReady: true}}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + _, err := removeGatewayRuntime(context.Background(), store, target, environment) + if test.wantConflict { + if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 1 || store.bindings["account-a"].RuntimeID != "container-c1" { + t.Fatalf("replacement network was not fenced: err=%v requests=%#v environment=%#v", err, gateway.recorded(), store.bindings["account-a"]) + } + return + } + requests := gateway.recorded() + if err != nil || len(requests) != 2 || requests[1].body["network_id"] != "network-n1" || store.bindings["account-a"].RuntimeCleanupPending { + t.Fatalf("known N1 was not preserved through cleanup: err=%v requests=%#v environment=%#v", err, requests, store.bindings["account-a"]) + } + }) + } +} + func TestGatewayLookupFailurePreservesLease(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} @@ -1775,8 +2437,9 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { t.Run("pre-mark rollback skips DELETE", func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } @@ -1791,7 +2454,7 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { } removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) - if cleanupErr == nil || removed || len(fixture.gateway.recorded()) != 0 { + if cleanupErr == nil || removed || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet { t.Fatalf("rolled-back pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, cleanupErr, fixture.gateway.recorded()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") @@ -1815,8 +2478,9 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { t.Run("clear rollback remains pending and lifecycle retry recovers", func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } @@ -1842,6 +2506,7 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) dropCleanupTransitionFailure(t, ctx, fixture.db) + setFixtureAccountStatus(t, fixture.databaseURL, "active") app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { @@ -1860,6 +2525,66 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { }) } +func TestPostgresLegacyCleanupPendingRequiresManualReconcile(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + fixture := newPostgresRebindFixture(t, databaseURL) + if _, err := fixture.db.ExecContext(ctx, ` + UPDATE environment_binding SET runtime_cleanup_pending = true, + runtime_cleanup_binding_version = NULL, runtime_cleanup_instance_id = NULL, runtime_cleanup_runtime_id = NULL + WHERE id = $1`, fixture.bound.BindingID); err != nil { + t.Fatalf("write legacy pending state: %v", err) + } + environment, err := fixture.store.GetEnvironmentContext(ctx, fixture.bound.Alias) + if err != nil || !environment.RuntimeCleanupPending || environment.RuntimeCleanupBindingVersion != 0 || environment.RuntimeCleanupRuntimeID != "" { + t.Fatalf("read legacy pending state: %#v err=%v", environment, err) + } + target, err := fixture.store.GetGateway(ctx, environment.Gateway) + if err != nil { + t.Fatal(err) + } + removed, err := removeGatewayRuntime(ctx, fixture.store, target, environment) + if !errors.Is(err, hub.ErrReconcileRequired) || removed || len(fixture.gateway.recorded()) != 0 { + t.Fatalf("legacy pending reached gateway mutation: removed=%v err=%v requests=%#v", removed, err, fixture.gateway.recorded()) + } +} + +func TestPostgresCleanupMissingContainerReleasesKnownLease(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") + environment, err := fixture.store.ActivateRuntime(ctx, fixture.bound.Alias, "known-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-known") + if err != nil { + t.Fatal(err) + } + target, err := fixture.store.GetGateway(ctx, environment.Gateway) + if err != nil { + t.Fatal(err) + } + + removed, err := removeGatewayRuntime(ctx, fixture.store, target, environment) + if err != nil || !removed { + t.Fatalf("cleanup missing gateway container: removed=%v err=%v", removed, err) + } + after, err := fixture.store.GetEnvironmentContext(ctx, environment.Alias) + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("known PostgreSQL lease did not converge: %#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + requests := fixture.gateway.recorded() + if len(requests) == 0 || requests[len(requests)-1].method != http.MethodDelete || + requests[len(requests)-1].body["runtime_id"] != "known-container" { + t.Fatalf("cleanup did not preserve the known runtime generation: %#v", requests) + } +} + func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { @@ -1877,14 +2602,15 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } { t.Run("restore "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } fixture.gateway.containers = []containerStatus{{ ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion - 1, NetworkExitID: fixture.bound.Exit.ID, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} var store hubStore = fixture.store if test.commitUnknown { @@ -1911,9 +2637,14 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } - if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet { + if len(fixture.gateway.containers) != 1 { t.Fatalf("restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) } + for _, request := range fixture.gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("restore cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded()) + } + } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'reconcile' @@ -1949,14 +2680,15 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { t.Run("create restore "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } fixture.gateway.containers = []containerStatus{{ ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion - 1, NetworkExitID: fixture.bound.Exit.ID, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} var store hubStore = fixture.store if test.commitUnknown { @@ -1984,9 +2716,14 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } - if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet { + if len(fixture.gateway.containers) != 1 { t.Fatalf("create restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) } + for _, request := range fixture.gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("create restore cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded()) + } + } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'create' @@ -2018,14 +2755,15 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { t.Run("discard "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } fixture.gateway.containers = []containerStatus{{ ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", }} var store hubStore = fixture.store if test.commitUnknown { @@ -2052,9 +2790,14 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } - if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 0 { + if len(fixture.gateway.containers) != 1 { t.Fatalf("discard cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) } + for _, request := range fixture.gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("discard cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded()) + } + } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'start' @@ -2070,13 +2813,18 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { } retryApp := fiber.New() registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) - if response = do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusConflict { + if response = do(retryApp, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") - if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) } + for _, container := range fixture.gateway.containers { + if container.State == "running" { + t.Fatalf("discard retry retained a running container: %#v", fixture.gateway.containers) + } + } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) }) } @@ -2098,14 +2846,15 @@ func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry(t *testing.T) { } { t.Run("list "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } fixture.gateway.containers = []containerStatus{{ ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", }} test.set(fixture.gateway, 1) app := fiber.New() @@ -2130,14 +2879,15 @@ func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry(t *testing.T) { t.Run("create "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } fixture.gateway.containers = []containerStatus{{ ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "network-old", }} test.set(fixture.gateway, gatewayReconcileAttempts) app := fiber.New() @@ -2213,16 +2963,24 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) + if test.action == "start" || test.action == "" { + setFixtureAccountStatus(t, fixture.databaseURL, "active") + } if test.active { + setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error - fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "network-old") if err != nil { t.Fatal(err) } } + state, networkExitID, networkID := "running", fixture.bound.Exit.ID, "network-old" + if test.action == "rebind" { + state, networkExitID, networkID = "exited", "", "" + } fixture.gateway.containers = []containerStatus{{ - ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, - BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + ID: "old-container", Alias: "account-a", State: state, ProxyReady: state == "running", + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: networkExitID, NetworkID: networkID, }} attempts := gatewayReconcileAttempts if test.action == "" { @@ -2282,10 +3040,18 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) { if test.action == "rebind" { wantVersion++ } - if err != nil || after.BindingVersion != wantVersion || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { + validRuntime := after.RuntimeID != "" && len(containers) == 1 && containerMatchesBinding(containers[0], after) + if test.action == "rebind" { + validRuntime = after.RuntimeID == "" && len(containers) == 0 + } + if err != nil || after.BindingVersion != wantVersion || !validRuntime { t.Fatalf("strict list retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) } - assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + activeAfter := 1 + if test.action == "rebind" { + activeAfter = 0 + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeAfter) if test.action != "" { var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` @@ -2351,7 +3117,7 @@ func TestUpgradeBrowserStopsBeforeCreateWhenPersistenceFails(t *testing.T) { t.Fatalf("expected persistence failure, got %d: %s", response.Code, response.Body.String()) } requests := gateway.recorded() - if len(requests) != 1 || requests[0].method != http.MethodDelete || requests[0].path != "/v1/browsers/account-a" { + if len(requests) == 0 || requests[len(requests)-1].method != http.MethodDelete || requests[len(requests)-1].path != "/v1/browsers/account-a" { t.Fatalf("failed persistence must abort before creating the upgraded container: %#v", requests) } env, err := store.GetEnv(context.Background(), "account-a") @@ -2373,8 +3139,10 @@ func TestUpgradeStopsBeforeCreateWhenRuntimeReleaseFails(t *testing.T) { t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String()) } requests := gateway.recorded() - if len(requests) != 0 { - t.Fatalf("release failure must abort before delete: %#v", requests) + for _, request := range requests { + if request.method == http.MethodDelete || request.method == http.MethodPost { + t.Fatalf("release failure must abort before delete: %#v", requests) + } } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" { t.Fatalf("release failure must be audited: %#v", store.actions) @@ -2593,6 +3361,214 @@ func TestBrowserActionRoutesStartStopAndRejectsUnknown(t *testing.T) { } } +func TestStartRejectsPausedOrRevokedAccountBeforeGatewayCall(t *testing.T) { + for _, test := range []struct { + name, status, authorization string + }{ + {name: "paused", status: "paused", authorization: "authorized"}, + {name: "revoked", status: "paused", authorization: "revoked"}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: test.status, + AuthorizationStatus: test.authorization, BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], + } + gateway := &fakeGateway{token: "unit-test-gateway-token"} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/start", "") + if response.Code != http.StatusConflict { + t.Fatalf("start returned %d: %s", response.Code, response.Body.String()) + } + if len(gateway.recorded()) != 0 || len(store.actions) != 0 { + t.Fatalf("rejected start reached side effects: requests=%#v actions=%#v", gateway.recorded(), store.actions) + } + }) + } +} + +func TestReconcileStopsRuntimeForPausedOrRevokedAccount(t *testing.T) { + for _, authorization := range []string{"authorized", "revoked"} { + t.Run(authorization, func(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: authorization, + BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], + RuntimeInstanceID: "old-instance", RuntimeID: "old-container", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("reconcile returned %d: %s", response.Code, response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { + t.Fatalf("non-runnable account retained runtime %q", runtime) + } + requests := gateway.recorded() + if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost || + requests[1].path != "/v1/browsers/account-a/stop" { + t.Fatalf("reconcile did not stop the runtime: %#v", requests) + } + }) + } +} + +func TestActivationConflictCleanupIsGenerationSafe(t *testing.T) { + for _, test := range []struct { + name string + containerID string + wantDeleted bool + }{ + {name: "candidate remains current", containerID: "candidate-container", wantDeleted: true}, + {name: "alias points to newer container", containerID: "new-container"}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + environment := hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: "authorized", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], + } + store.bindings[environment.Alias] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: test.containerID, Alias: environment.Alias, State: "running", ProxyReady: true, + BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, NetworkID: "network-candidate", + }}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + + _, err := activateGatewayRuntime(context.Background(), store, target, environment, "candidate-container", "network-candidate") + if err == nil { + t.Fatal("paused account unexpectedly activated the candidate") + } + deleted := len(gateway.containers) == 0 + if deleted != test.wantDeleted { + t.Fatalf("generation-safe cleanup mismatch: deleted=%v requests=%#v", deleted, gateway.recorded()) + } + if !test.wantDeleted { + for _, request := range gateway.recorded() { + if request.method == http.MethodDelete { + t.Fatalf("stale candidate deleted a newer alias generation: %#v", gateway.recorded()) + } + } + } + }) + } +} + +func TestExplicitStopRejectsStaleBindingBeforeGatewaySideEffect(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "new-instance", RuntimeID: "new-container", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "new-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", + }}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + stale := store.bindings["account-a"] + stale.BindingVersion, stale.RuntimeInstanceID, stale.RuntimeID = 1, "old-instance", "old-container" + + if err := stopEnvironmentRuntime(context.Background(), store, stale); !errors.Is(err, hub.ErrConflict) { + t.Fatalf("stale explicit stop was not rejected: %v", err) + } + if len(gateway.recorded()) != 0 || len(gateway.containers) != 1 || gateway.containers[0].ID != "new-container" { + t.Fatalf("stale explicit stop reached the gateway: requests=%#v containers=%#v", gateway.recorded(), gateway.containers) + } +} + +func TestStopEnvironmentRuntimeReconcilesContainerWithoutLease(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148"} + environment := hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: "authorized", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], + } + store.bindings[environment.Alias] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "orphan-container", Alias: environment.Alias, State: "running", BindingVersion: environment.BindingVersion, + }}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + + if err := stopEnvironmentRuntime(context.Background(), store, environment); err != nil { + t.Fatalf("stop orphan without lease: %v", err) + } + after := store.bindings[environment.Alias] + if after.RuntimeCleanupPending || after.RuntimeID != "" || gateway.containers[0].State != "exited" { + t.Fatalf("lease-free orphan stop did not converge: after=%#v containers=%#v", after, gateway.containers) + } + requests := gateway.recorded() + if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost || + requests[1].body["runtime_id"] != "orphan-container" || requests[1].body["binding_version"] != float64(environment.BindingVersion) { + t.Fatalf("stop did not use the discovered generation: %#v", requests) + } +} + +func TestReconcileFinishedAuditUsesRebuiltRuntime(t *testing.T) { + store := newMemoryStore() + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-2"], RuntimeInstanceID: "old-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", + } + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "network-old", + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("reconcile returned %d: %s", response.Code, response.Body.String()) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "succeeded" || + store.actions[1].RuntimeInstanceID != "runtime-instance" || store.actions[1].BindingVersion != 2 || store.actions[1].NetworkExitID != "exit-2" { + t.Fatalf("finished audit used stale runtime context: %#v", store.actions) + } +} + +func TestReconcileContextRefreshFailureClearsAllAuditCorrelation(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", + BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "old-instance", RuntimeID: "old-container", RuntimeNetworkID: "network-old", + } + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "network-old", + }}} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + failing := &failContextRefreshStore{hubStore: store} + app := fiber.New() + registerHubWithNetwork(app, failing, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusInternalServerError { + t.Fatalf("context refresh failure returned %d: %s", response.Code, response.Body.String()) + } + finished := store.actions[len(store.actions)-1] + if finished.RuntimeInstanceID != "" || finished.BindingVersion != 0 || finished.NetworkExitID != "" { + t.Fatalf("failed context refresh retained partial stale correlation: %#v", finished) + } +} + func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) { gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet { @@ -2628,7 +3604,7 @@ func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", } _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ @@ -2645,11 +3621,11 @@ func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { t.Fatalf("running runtime was not rebuilt on the latest binding: %#v", bound) } requests := gateway.recorded() - if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete || requests[2].method != http.MethodPost { + if len(requests) != 4 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost { t.Fatalf("running rebind must inspect, delete and recreate: %#v", requests) } - if requests[2].body["binding_version"] != float64(2) || requests[2].body["network_exit_id"] != "exit-2" { - t.Fatalf("recreated runtime did not carry the latest binding CAS: %#v", requests[2].body) + if requests[3].body["binding_version"] != float64(2) || requests[3].body["network_exit_id"] != "exit-2" { + t.Fatalf("recreated runtime did not carry the latest binding CAS: %#v", requests[3].body) } } @@ -2675,11 +3651,11 @@ func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) { t.Fatalf("same-exit rebind did not rotate the binding generation and lease: %#v", bound) } requests := gateway.recorded() - if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete || requests[2].method != http.MethodPost { + if len(requests) != 4 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost { t.Fatalf("same-exit rebind bypassed gateway reconciliation: %#v", requests) } - if requests[2].body["binding_version"] != float64(2) || requests[2].body["network_exit_id"] != "exit-1" { - t.Fatalf("same-exit rebuild used stale metadata: %#v", requests[2].body) + if requests[3].body["binding_version"] != float64(2) || requests[3].body["network_exit_id"] != "exit-1" { + t.Fatalf("same-exit rebuild used stale metadata: %#v", requests[3].body) } } @@ -2704,7 +3680,7 @@ func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) { t.Fatalf("failed exit remained active: %q", runtime) } requests := gateway.recorded() - if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete { + if len(requests) != 3 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete { t.Fatalf("failed exit must be rechecked and removed instead of renewed: %#v", requests) } } @@ -2714,10 +3690,10 @@ func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, - Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", RuntimeNetworkID: "network-old", } gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{ - ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "network-old", ProxyReady: true, }}} app := newTestApp(t, store, gateway) @@ -2757,7 +3733,7 @@ func TestDisableExitImmediatelyDiscardsRuntime(t *testing.T) { t.Fatalf("disabled exit remained active: %q", runtime) } requests := gateway.recorded() - if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete { + if len(requests) != 3 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete { t.Fatalf("disable must reconcile and remove the runtime: %#v", requests) } } @@ -2819,7 +3795,7 @@ func TestLegacyNullBindingIsListableAndExplicitlyRecoverable(t *testing.T) { } _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ - ID: "legacy-container", Alias: "account-a", State: "running", + ID: "legacy-container", Alias: "account-a", State: "running", BindingVersion: 1, }}} app := newTestApp(t, store, gateway) @@ -2837,10 +3813,10 @@ func TestLegacyNullBindingIsListableAndExplicitlyRecoverable(t *testing.T) { t.Fatalf("legacy explicit recovery failed: %d: %s", response.Code, response.Body.String()) } requests := gateway.recorded() - if len(requests) != 4 || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost { + if len(requests) != 5 || requests[3].method != http.MethodDelete || requests[4].method != http.MethodPost { t.Fatalf("legacy recovery must inspect, remove, then recreate: %#v", requests) } - encoded, _ := json.Marshal(requests[3].body) + encoded, _ := json.Marshal(requests[4].body) if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") { t.Fatalf("legacy Config.Cmd credentials reached the recovered runtime: %s", encoded) } @@ -2962,7 +3938,8 @@ func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, - RuntimeCleanupPending: true, Exit: store.exits["exit-1"], + RuntimeCleanupPending: true, RuntimeCleanupBindingVersion: 1, + RuntimeCleanupRuntimeID: missingRuntimeID, RuntimeCleanupNetworkID: "network-old", Exit: store.exits["exit-1"], } _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) _ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}) @@ -3026,7 +4003,7 @@ func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) { } } -func TestRebindCandidateCreateFailureRestoresOldRuntime(t *testing.T) { +func TestRebindCandidateUnknownCreateLeavesGenerationPending(t *testing.T) { store := newMemoryStore() _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} @@ -3043,11 +4020,12 @@ func TestRebindCandidateCreateFailureRestoresOldRuntime(t *testing.T) { t.Fatalf("candidate create failure returned %d: %s", response.Code, response.Body.String()) } after := store.bindings["account-a"] - if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID == "" || - len(gateway.containers) != 1 || !containerMatchesBinding(gateway.containers[0], after) { - t.Fatalf("candidate create failure did not restore old runtime: after=%#v containers=%#v", after, gateway.containers) + if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" || !after.RuntimeCleanupPending || + after.RuntimeCleanupBindingVersion != 2 || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" || + len(gateway.containers) != 0 { + t.Fatalf("candidate create unknown result was not retained for manual reconcile: after=%#v containers=%#v", after, gateway.containers) } - if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "gateway_create_failed" { + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("candidate create failure audit mismatch: %#v", store.actions) } } @@ -3111,7 +4089,7 @@ func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) { store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"]} _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) - gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "old-container", Alias: "account-a", State: "exited"}}} + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "old-container", Alias: "account-a", State: "exited", BindingVersion: 1}}} app := newTestApp(t, store, gateway) if response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`); response.Code != http.StatusOK { diff --git a/cmd/control-plane/main.go b/cmd/control-plane/main.go index f931284..f931e46 100644 --- a/cmd/control-plane/main.go +++ b/cmd/control-plane/main.go @@ -176,7 +176,7 @@ func newHandlerWithStores(webDirectory string, phaseAStore *phasea.Store, hubSto registerHub(app, hubStore) } if phaseAStore != nil { - registerPhaseA(app, phaseAStore) + registerPhaseA(app, phaseAStore, hubStore) } app.Get("/*", spaHandler(webDirectory)) return app diff --git a/cmd/control-plane/phasea.go b/cmd/control-plane/phasea.go index 5faf6c4..41f4242 100644 --- a/cmd/control-plane/phasea.go +++ b/cmd/control-plane/phasea.go @@ -22,7 +22,7 @@ type accountRequest struct { } `json:"credential_reference"` } -func registerPhaseA(app *fiber.App, store *phasea.Store) { +func registerPhaseA(app *fiber.App, store *phasea.Store, runtimeStore runtimeStopStore) { app.Post("/api/phase-a/accounts", func(c fiber.Ctx) error { var input accountRequest if err := decodePhaseA(c, &input); err != nil { @@ -57,13 +57,22 @@ func registerPhaseA(app *fiber.App, store *phasea.Store) { }) app.Post("/api/phase-a/accounts/:id/pause", func(c fiber.Ctx) error { + runtimeOperations.Lock() + defer runtimeOperations.Unlock() if err := store.PauseAccount(c.Context(), c.Params("id")); err != nil { return phaseAError(c, err) } + if runtimeStore != nil { + if err := stopAccountRuntime(c.Context(), runtimeStore, c.Params("id")); err != nil { + return hubError(c, err) + } + } return c.SendStatus(fiber.StatusNoContent) }) app.Post("/api/phase-a/accounts/:id/resume", func(c fiber.Ctx) error { + runtimeOperations.Lock() + defer runtimeOperations.Unlock() if err := store.ResumeAccount(c.Context(), c.Params("id")); err != nil { return phaseAError(c, err) } @@ -71,26 +80,15 @@ func registerPhaseA(app *fiber.App, store *phasea.Store) { }) app.Post("/api/phase-a/accounts/:id/revoke", func(c fiber.Ctx) error { + runtimeOperations.Lock() + defer runtimeOperations.Unlock() if err := store.RevokeAccount(c.Context(), c.Params("id")); err != nil { return phaseAError(c, err) } - return c.SendStatus(fiber.StatusNoContent) - }) - - app.Post("/api/phase-a/runtimes", func(c fiber.Ctx) error { - var input phasea.RuntimeBinding - if err := decodePhaseA(c, &input); err != nil { - return phaseAError(c, err) - } - if err := store.BindRuntime(c.Context(), input); err != nil { - return phaseAError(c, err) - } - return c.Status(fiber.StatusCreated).JSON(map[string]string{"id": input.ID}) - }) - - app.Delete("/api/phase-a/runtimes/:id", func(c fiber.Ctx) error { - if err := store.ReleaseRuntime(c.Context(), c.Params("id")); err != nil { - return phaseAError(c, err) + if runtimeStore != nil { + if err := stopAccountRuntime(c.Context(), runtimeStore, c.Params("id")); err != nil { + return hubError(c, err) + } } return c.SendStatus(fiber.StatusNoContent) }) diff --git a/cmd/docker-gateway/main.go b/cmd/docker-gateway/main.go index e4e0c0b..02fa01f 100644 --- a/cmd/docker-gateway/main.go +++ b/cmd/docker-gateway/main.go @@ -3,7 +3,9 @@ package main import ( "bytes" "context" + "crypto/rand" "crypto/subtle" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -35,10 +37,15 @@ const ( bindingVersionLabel = "io.creatorhub.binding-version" networkExitLabel = "io.creatorhub.network-exit-id" proxyPortLabel = "io.creatorhub.proxy-port" + networkIDLabel = "io.creatorhub.network-id" networkRoleLabel = "io.creatorhub.network-role" + gatewayMemberLabel = "io.creatorhub.gateway-member" browserNetworkRole = "browser" controlNetworkName = "creatorhub_control" namePrefix = "creatorhub-browser-" + reservationPrefix = "creatorhub-reservation-" + reservationLabel = "io.creatorhub.alias-reservation" + reservationGenLabel = "io.creatorhub.reservation-generation" pullTimeout = 10 * time.Minute ) @@ -53,6 +60,7 @@ var ( var ( errInvalidRuntimeID = errors.New("invalid runtime id") errUnmanagedContainer = errors.New("refusing to operate on a container not owned by CreatorHub") + errGenerationConflict = errors.New("container generation does not match request") errUnauthorized = errors.New("gateway token rejected") ) @@ -70,12 +78,43 @@ type dockerClient struct { slow *http.Client // 镜像拉取等长操作，不设整体超时 } +type dockerAliasReservations struct { + docker dockerClient + self string +} + +type tenantNetworkGeneration struct { + ID string + Name string + Created bool + ConnectedSelf bool + ConnectedRuntime bool + GatewayMembers []string + SelfMember string + RuntimeAttached bool +} + +type dockerTenantNetwork struct { + ID string `json:"Id"` + Name string `json:"Name"` + Driver string `json:"Driver"` + Internal bool `json:"Internal"` + Attachable bool `json:"Attachable"` + Ingress bool `json:"Ingress"` + Labels map[string]string `json:"Labels"` + Containers map[string]struct { + Name string `json:"Name"` + IPv4Address string `json:"IPv4Address"` + } `json:"Containers"` +} + type gateway struct { docker dockerClient network string self string token string proxies *memoryProxyRegistry + locks *dockerAliasReservations } // createRequest 全量字段由平台下发；网关不做业务决策，只做输入合法性校验。 @@ -99,6 +138,20 @@ type gatewayProxyExit struct { Password string `json:"password"` } +type generationRequest struct { + BindingVersion int64 `json:"binding_version"` + RuntimeID string `json:"runtime_id"` + NetworkID string `json:"network_id"` +} + +type proxyRestoreRequest struct { + BindingVersion int64 `json:"binding_version"` + RuntimeID string `json:"runtime_id"` + NetworkID string `json:"network_id"` + NetworkExitID string `json:"network_exit_id"` + NetworkExit gatewayProxyExit `json:"network_exit"` +} + type browser struct { ID string `json:"id"` Alias string `json:"alias"` @@ -108,6 +161,7 @@ type browser struct { Endpoint string `json:"endpoint"` BindingVersion int64 `json:"binding_version"` NetworkExitID string `json:"network_exit_id"` + NetworkID string `json:"network_id"` ProxyReady bool `json:"proxy_ready"` } @@ -222,7 +276,8 @@ func newGateway(client dockerClient, network, token string) *fiber.App { } func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber.App { - api := gateway{docker: client, network: network, self: self, token: token, proxies: newMemoryProxyRegistry()} + api := gateway{docker: client, network: network, self: self, token: token, proxies: newMemoryProxyRegistry(), + locks: &dockerAliasReservations{docker: client, self: self}} app := fiber.New(fiber.Config{ AppName: "CreatorHub Docker gateway", BodyLimit: 1 << 20, @@ -300,7 +355,8 @@ func (api gateway) list(c fiber.Ctx) error { Endpoint: "http://" + namePrefix + alias + ":9222", BindingVersion: bindingVersion, NetworkExitID: container.Labels[networkExitLabel], - ProxyReady: api.proxies.ready(alias, proxyPort), + NetworkID: container.Labels[networkIDLabel], + ProxyReady: api.proxies.ready(alias, proxyPort, container.ID, container.Labels[networkIDLabel]), }) } return writeJSON(c, http.StatusOK, browsers) @@ -319,17 +375,52 @@ func (api gateway) create(c fiber.Ctx) error { if err := api.docker.pullIfMissing(c.Context(), input.Image); err != nil { return writeError(c, http.StatusBadGateway, err) } + if _, _, err := api.managedContainer(input.Alias); err == nil { + return writeError(c, http.StatusConflict, errors.New("browser alias is already in use")) + } else if !errors.Is(err, os.ErrNotExist) { + return writeError(c, statusFor(err), err) + } + _, release, err := api.locks.acquire(input.Alias) + if err != nil { + return writeError(c, statusFor(err), err) + } + defer release() + if _, _, err := api.managedContainer(input.Alias); err == nil { + return writeError(c, http.StatusConflict, errors.New("browser alias is already in use")) + } else if !errors.Is(err, os.ErrNotExist) { + return writeError(c, statusFor(err), err) + } network, proxyServer, undoProxy := "none", "", func() {} + var networkGeneration tenantNetworkGeneration + keepNetwork := input.Stopped if !input.Stopped { var err error var bindHost string - network, bindHost, err = api.docker.ensureTenantNetwork(api.network, input.Alias, api.self) - if err != nil { - return writeError(c, http.StatusBadGateway, errors.New("configure isolated browser network")) + networkGeneration, bindHost, err = api.docker.ensureTenantNetwork(api.network, input.Alias, api.self, input.BindingVersion, "", "", false) + if networkGeneration.ID != "" { + defer func() { + if keepNetwork { + return + } + var cleanupErr error + if networkGeneration.Created { + cleanupErr = api.removeTenantNetwork(input.Alias, input.BindingVersion, "", networkGeneration, nil, "", false) + } else if networkGeneration.ConnectedSelf { + _, cleanupErr = api.disconnectTenantNetworkMember(input.Alias, input.BindingVersion, "", networkGeneration, + networkGeneration.SelfMember, nil, "", false) + } + if cleanupErr != nil { + logrus.WithError(cleanupErr).WithField("alias", input.Alias).Error("rollback isolated browser network") + } + }() } - proxyServer, undoProxy, err = api.proxies.configure(input.Alias, bindHost, 0, input.NetworkExit) if err != nil { - return writeError(c, http.StatusBadGateway, errors.New("configure in-memory proxy")) + return writeNetworkError(c, http.StatusBadGateway, errors.New("configure isolated browser network"), networkGeneration.ID) + } + network = networkGeneration.ID + proxyServer, undoProxy, err = api.proxies.configure(input.Alias, input.BindingVersion, bindHost, 0, input.NetworkExit, networkGeneration.ID) + if err != nil { + return writeNetworkError(c, statusFor(err), errors.Join(errors.New("configure in-memory proxy"), err), networkGeneration.ID) } } keepProxy := false @@ -356,6 +447,7 @@ func (api gateway) create(c fiber.Ctx) error { nameLabel: input.Name, bindingVersionLabel: strconv.FormatInt(input.BindingVersion, 10), networkExitLabel: input.NetworkExitID, + networkIDLabel: networkGeneration.ID, proxyPortLabel: strconv.Itoa(proxyPort(proxyServer)), }, "ExposedPorts": map[string]any{"9222/tcp": map[string]any{}}, @@ -381,38 +473,52 @@ func (api gateway) create(c fiber.Ctx) error { }, } result, err := api.docker.request(http.MethodPost, "/containers/create?name="+url.QueryEscape(namePrefix+input.Alias), payload) - if err != nil { - return writeError(c, http.StatusBadGateway, err) - } - defer result.Body.Close() - if result.StatusCode != http.StatusCreated { - status := http.StatusBadGateway - if result.StatusCode == http.StatusConflict { - status = http.StatusConflict - } - return writeError(c, status, errors.New("Docker container creation failed")) - } var created struct { ID string `json:"Id"` } - if err := json.NewDecoder(result.Body).Decode(&created); err != nil || created.ID == "" { - cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(namePrefix+input.Alias)+"?force=1&v=0", nil, http.StatusNoContent, http.StatusNotFound) - if cleanupErr != nil { - return writeError(c, http.StatusBadGateway, fmt.Errorf("Docker returned an invalid container id and cleanup failed: %w", cleanupErr)) + status, createErr := http.StatusBadGateway, err + if result != nil { + if result.StatusCode == http.StatusConflict { + status = http.StatusConflict + } + if result.StatusCode == http.StatusCreated { + createErr = json.NewDecoder(result.Body).Decode(&created) + } else { + createErr = errors.New("Docker container creation failed") + } + result.Body.Close() + } + if createErr != nil || created.ID == "" { + containerID, labels, inspectErr := api.managedContainer(input.Alias) + if inspectErr == nil && labels[bindingVersionLabel] == strconv.FormatInt(input.BindingVersion, 10) && + labels[networkIDLabel] == networkGeneration.ID { + created.ID = containerID + } else { + if createErr == nil { + createErr = errors.New("Docker returned an invalid container id") + } + return writeNetworkError(c, status, createErr, networkGeneration.ID) } - return writeError(c, http.StatusBadGateway, errors.New("Docker returned an invalid container id; container was removed while preserving its Profile volume")) } if !input.Stopped { + if !api.proxies.bind(input.Alias, input.BindingVersion, proxyServer, created.ID, networkGeneration.ID) { + cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(created.ID)+"?force=1&v=0", nil, http.StatusNoContent) + if cleanupErr != nil { + return writeNetworkError(c, http.StatusBadGateway, fmt.Errorf("proxy generation changed and container cleanup failed: %w", cleanupErr), networkGeneration.ID) + } + return writeNetworkError(c, http.StatusConflict, errGenerationConflict, networkGeneration.ID) + } + undoProxy = func() { api.proxies.remove(input.Alias, input.BindingVersion, created.ID) } if err := api.docker.expect(http.MethodPost, "/containers/"+url.PathEscape(created.ID)+"/start", nil, http.StatusNoContent, http.StatusNotModified); err != nil { cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(created.ID)+"?force=1&v=0", nil, http.StatusNoContent) if cleanupErr != nil { - return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and cleanup failed: %w; cleanup: %v", err, cleanupErr)) + return writeNetworkError(c, http.StatusBadGateway, fmt.Errorf("container did not start and cleanup failed: %w; cleanup: %v", err, cleanupErr), networkGeneration.ID) } - return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and was removed while preserving its Profile volume: %w", err)) + return writeNetworkError(c, http.StatusBadGateway, fmt.Errorf("container did not start and was removed while preserving its Profile volume: %w", err), networkGeneration.ID) } } - keepProxy = !input.Stopped - return writeJSON(c, http.StatusCreated, map[string]string{"id": created.ID, "alias": input.Alias}) + keepProxy, keepNetwork = !input.Stopped, true + return writeJSON(c, http.StatusCreated, map[string]string{"id": created.ID, "alias": input.Alias, "network_id": networkGeneration.ID}) } func validateCreate(input createRequest) error { @@ -480,18 +586,50 @@ func hasControlRunes(value string) bool { func (api gateway) changeState(c fiber.Ctx) error { id := c.Params("id") action := c.Params("action") - if err := api.requireManaged(id); err != nil { - return writeError(c, statusFor(err), err) - } - var path string + var input generationRequest switch action { case "start": - path = "/containers/" + url.PathEscape(namePrefix+id) + "/start" + if err := api.requireManaged(id); err != nil { + return writeError(c, statusFor(err), err) + } case "stop": - path = "/containers/" + url.PathEscape(namePrefix+id) + "/stop?t=10" + var err error + input, err = decodeGeneration(c) + if err != nil { + return writeError(c, http.StatusBadRequest, err) + } + _, exists, err := api.requireGeneration(id, input) + if err != nil { + return writeError(c, statusFor(err), err) + } + if !exists { + return writeError(c, http.StatusNotFound, os.ErrNotExist) + } default: return writeError(c, http.StatusNotFound, errors.New("unknown action")) } + _, release, err := api.locks.acquire(id) + if err != nil { + return writeError(c, statusFor(err), err) + } + defer release() + var path string + if action == "start" { + containerID, _, err := api.managedContainer(id) + if err != nil { + return writeError(c, statusFor(err), err) + } + path = "/containers/" + url.PathEscape(containerID) + "/start" + } else { + containerID, exists, err := api.requireGeneration(id, input) + if err != nil { + return writeError(c, statusFor(err), err) + } + if !exists { + return writeError(c, http.StatusNotFound, os.ErrNotExist) + } + path = "/containers/" + url.PathEscape(containerID) + "/stop?t=10" + } if err := api.docker.expect(http.MethodPost, path, nil, http.StatusNoContent, http.StatusNotModified); err != nil { return writeError(c, http.StatusBadGateway, err) } @@ -501,90 +639,289 @@ func (api gateway) changeState(c fiber.Ctx) error { func (api gateway) remove(c fiber.Ctx) error { id := c.Params("id") - err := api.requireManaged(id) - if err != nil && !errors.Is(err, os.ErrNotExist) { + input, err := decodeGeneration(c) + if err != nil { + return writeError(c, http.StatusBadRequest, err) + } + containerID, exists, err := api.requireGeneration(id, input) + if err != nil { return writeError(c, statusFor(err), err) } - if err == nil { - path := "/containers/" + url.PathEscape(namePrefix+id) + "?force=1&v=0" - if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent, http.StatusNotFound); err != nil { - return writeError(c, http.StatusBadGateway, err) + _, release, err := api.locks.acquire(id) + if err != nil { + return writeError(c, statusFor(err), err) + } + defer release() + currentID, currentExists, err := api.requireGeneration(id, input) + if err != nil { + return writeError(c, statusFor(err), err) + } + if currentExists != exists || currentID != containerID { + return writeError(c, http.StatusConflict, errGenerationConflict) + } + expectedNetworkID := input.NetworkID + if currentExists { + _, labels, inspectErr := api.managedContainer(id) + if inspectErr != nil { + return writeError(c, statusFor(inspectErr), inspectErr) + } + containerNetworkID := labels[networkIDLabel] + if containerNetworkID != input.NetworkID { + return writeError(c, http.StatusConflict, errGenerationConflict) + } + } else if expectedNetworkID == "" { + return writeError(c, http.StatusConflict, errGenerationConflict) + } + var networkGeneration tenantNetworkGeneration + networkExists := false + if expectedNetworkID != "" { + networkGeneration, _, networkExists, err = api.docker.inspectTenantNetwork(api.network, id, input.BindingVersion, + input.RuntimeID, api.self, expectedNetworkID, false) + if err != nil { + return writeError(c, statusFor(err), err) } } - api.proxies.remove(id) - if err := api.docker.removeTenantNetwork(api.network, id, api.self); err != nil { + if networkExists { + err = api.removeTenantNetwork(id, input.BindingVersion, input.RuntimeID, networkGeneration, &input, containerID, exists) + } else if expectedNetworkID != "" { + return writeError(c, http.StatusConflict, errGenerationConflict) + } + if err != nil { + if errors.Is(err, errGenerationConflict) { + return writeError(c, http.StatusConflict, err) + } return c.Status(http.StatusAccepted).JSON(map[string]string{ - "status": "container_removed_network_cleanup_pending", + "status": "runtime_cleanup_pending", }) } + if !api.proxies.remove(id, input.BindingVersion, input.RuntimeID, input.NetworkID) { + return writeError(c, http.StatusConflict, errGenerationConflict) + } + if exists { + path := "/containers/" + url.PathEscape(containerID) + "?force=1&v=0" + if err := api.requireRuntimeState(id, input, containerID, true); err != nil { + return writeError(c, http.StatusConflict, err) + } + if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent); err != nil { + currentID, currentExists, generationErr := api.requireGeneration(id, input) + if errors.Is(err, os.ErrNotExist) && generationErr == nil && !currentExists { + return c.SendStatus(http.StatusNoContent) + } + if generationErr != nil || currentID != containerID { + err = errGenerationConflict + } + if errors.Is(err, errGenerationConflict) { + return writeError(c, http.StatusConflict, err) + } + return writeError(c, http.StatusBadGateway, err) + } + } c.Status(http.StatusNoContent) return nil } func (api gateway) restoreProxy(c fiber.Ctx) error { - input := struct { - BindingVersion int64 `json:"binding_version"` - NetworkExitID string `json:"network_exit_id"` - NetworkExit gatewayProxyExit `json:"network_exit"` - }{} + input := proxyRestoreRequest{} decoder := json.NewDecoder(bytes.NewReader(c.Body())) decoder.DisallowUnknownFields() - if err := decoder.Decode(&input); err != nil || input.BindingVersion < 1 || !exitIDPattern.MatchString(input.NetworkExitID) || + if err := decoder.Decode(&input); err != nil || input.BindingVersion < 1 || !exitIDPattern.MatchString(input.RuntimeID) || + !exitIDPattern.MatchString(input.NetworkID) || + !exitIDPattern.MatchString(input.NetworkExitID) || validateCreate(createRequest{Alias: c.Params("id"), Name: "x", Image: "x", Cmd: []string{"about:blank"}, Volume: "x", BindingVersion: input.BindingVersion, NetworkExitID: input.NetworkExitID, NetworkExit: input.NetworkExit}) != nil { return writeError(c, http.StatusBadRequest, errors.New("invalid proxy recovery request")) } - labels, err := api.managedLabels(c.Params("id")) + removeStaleProxy := func() { api.proxies.remove(c.Params("id"), input.BindingVersion, input.RuntimeID, input.NetworkID) } + runtimeID, labels, err := api.requireProxyGeneration(c.Params("id"), input) + if err != nil { + removeStaleProxy() + return writeError(c, statusFor(err), err) + } + port, _ := strconv.Atoi(labels[proxyPortLabel]) + if port < 1 { + removeStaleProxy() + return writeError(c, http.StatusConflict, errors.New("container binding does not match recovery request")) + } + _, release, err := api.locks.acquire(c.Params("id")) if err != nil { return writeError(c, statusFor(err), err) } - version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) - port, _ := strconv.Atoi(labels[proxyPortLabel]) - if version != input.BindingVersion || labels[networkExitLabel] != input.NetworkExitID || port < 1 { + defer release() + if runtimeID, labels, err = api.requireProxyGeneration(c.Params("id"), input); err != nil { + removeStaleProxy() + return writeError(c, statusFor(err), err) + } + port, _ = strconv.Atoi(labels[proxyPortLabel]) + if port < 1 { + removeStaleProxy() return writeError(c, http.StatusConflict, errors.New("container binding does not match recovery request")) } - _, bindHost, err := api.docker.ensureTenantNetwork(api.network, c.Params("id"), api.self) + networkGeneration, bindHost, err := api.docker.ensureTenantNetwork(api.network, c.Params("id"), api.self, input.BindingVersion, + input.RuntimeID, labels[networkIDLabel], true) + keepNetwork := false + if networkGeneration.ID != "" { + defer func() { + if keepNetwork { + return + } + var cleanupErr error + if networkGeneration.Created { + cleanupErr = api.removeTenantNetwork(c.Params("id"), input.BindingVersion, input.RuntimeID, networkGeneration, nil, "", false) + } else { + if networkGeneration.ConnectedRuntime { + _, cleanupErr = api.disconnectTenantNetworkMember(c.Params("id"), input.BindingVersion, input.RuntimeID, + networkGeneration, input.RuntimeID, nil, "", false) + } + if cleanupErr == nil && networkGeneration.ConnectedSelf { + _, cleanupErr = api.disconnectTenantNetworkMember(c.Params("id"), input.BindingVersion, input.RuntimeID, + networkGeneration, networkGeneration.SelfMember, nil, "", false) + } + } + if cleanupErr != nil { + logrus.WithError(cleanupErr).WithField("alias", c.Params("id")).Error("rollback restored browser network") + } + }() + } if err != nil { + removeStaleProxy() return writeError(c, http.StatusBadGateway, errors.New("restore isolated browser network")) } - if _, _, err := api.proxies.configure(c.Params("id"), bindHost, port, input.NetworkExit); err != nil { - return writeError(c, http.StatusBadGateway, errors.New("restore in-memory proxy")) + if err = api.requireProxyNetworkGeneration(c.Params("id"), input, networkGeneration); err != nil { + removeStaleProxy() + return writeError(c, statusFor(err), err) } + proxyServer, undoProxy, err := api.proxies.configure(c.Params("id"), input.BindingVersion, bindHost, port, input.NetworkExit, input.NetworkID) + if err != nil { + removeStaleProxy() + return writeError(c, statusFor(err), errors.Join(errors.New("restore in-memory proxy"), err)) + } + if err = api.requireProxyNetworkGeneration(c.Params("id"), input, networkGeneration); err != nil { + undoProxy() + return writeError(c, statusFor(err), err) + } + if !api.proxies.bind(c.Params("id"), input.BindingVersion, proxyServer, runtimeID, input.NetworkID) { + undoProxy() + return writeError(c, http.StatusConflict, errGenerationConflict) + } + if err = api.requireProxyNetworkGeneration(c.Params("id"), input, networkGeneration); err != nil { + undoProxy() + return writeError(c, statusFor(err), err) + } + keepNetwork = true return c.SendStatus(http.StatusNoContent) } +func (api gateway) requireProxyNetworkGeneration(alias string, input proxyRestoreRequest, expected tenantNetworkGeneration) error { + _, labels, err := api.requireProxyGeneration(alias, input) + if err != nil { + return err + } + if networkID := labels[networkIDLabel]; networkID != "" && networkID != expected.ID { + return errGenerationConflict + } + current, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, input.BindingVersion, + input.RuntimeID, api.self, expected.ID, false) + if err != nil || !exists || !sameTenantNetworkMembers(current, expected) { + return errGenerationConflict + } + return nil +} + +func sameTenantNetworkMembers(current, expected tenantNetworkGeneration) bool { + if current.ID != expected.ID || current.Name != expected.Name || current.RuntimeAttached != expected.RuntimeAttached || + current.SelfMember != expected.SelfMember || len(current.GatewayMembers) != len(expected.GatewayMembers) { + return false + } + members := make(map[string]struct{}, len(current.GatewayMembers)) + for _, member := range current.GatewayMembers { + members[member] = struct{}{} + } + for _, member := range expected.GatewayMembers { + if _, ok := members[member]; !ok { + return false + } + } + return true +} + +func (api gateway) requireProxyGeneration(alias string, input proxyRestoreRequest) (string, map[string]string, error) { + runtimeID, labels, err := api.managedContainer(alias) + if err != nil { + return "", nil, err + } + version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) + if runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkExitLabel] != input.NetworkExitID || + labels[networkIDLabel] != input.NetworkID { + return "", nil, errGenerationConflict + } + return runtimeID, labels, nil +} + func (api gateway) requireManaged(id string) error { _, err := api.managedLabels(id) return err } +func decodeGeneration(c fiber.Ctx) (generationRequest, error) { + var input generationRequest + decoder := json.NewDecoder(bytes.NewReader(c.Body())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil || input.BindingVersion < 1 || + (input.RuntimeID != "" && !exitIDPattern.MatchString(input.RuntimeID)) || + (input.NetworkID != "" && !exitIDPattern.MatchString(input.NetworkID)) { + return generationRequest{}, errors.New("binding_version, runtime_id and network_id must identify the expected generation") + } + return input, nil +} + +func (api gateway) requireGeneration(id string, input generationRequest) (string, bool, error) { + runtimeID, labels, err := api.managedContainer(id) + if errors.Is(err, os.ErrNotExist) { + return "", false, nil + } + if err != nil { + return "", false, err + } + version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) + if input.RuntimeID == "" || runtimeID != input.RuntimeID || version != input.BindingVersion { + return "", false, errGenerationConflict + } + return runtimeID, true, nil +} + func (api gateway) managedLabels(id string) (map[string]string, error) { + _, labels, err := api.managedContainer(id) + return labels, err +} + +func (api gateway) managedContainer(id string) (string, map[string]string, error) { if !runtimeIDPattern.MatchString(id) { - return nil, errInvalidRuntimeID + return "", nil, errInvalidRuntimeID } result, err := api.docker.request(http.MethodGet, "/containers/"+url.PathEscape(namePrefix+id)+"/json", nil) if err != nil { - return nil, err + return "", nil, err } defer result.Body.Close() if result.StatusCode == http.StatusNotFound { - return nil, os.ErrNotExist + return "", nil, os.ErrNotExist } if result.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Docker inspect returned %s", result.Status) + return "", nil, fmt.Errorf("Docker inspect returned %s", result.Status) } var inspected struct { + ID string `json:"Id"` Config struct { Labels map[string]string `json:"Labels"` } `json:"Config"` } if err := json.NewDecoder(result.Body).Decode(&inspected); err != nil { - return nil, fmt.Errorf("decode Docker inspect: %w", err) + return "", nil, fmt.Errorf("decode Docker inspect: %w", err) } if inspected.Config.Labels[managedLabel] != "true" || inspected.Config.Labels[idLabel] != id { - return nil, errUnmanagedContainer + return "", nil, errUnmanagedContainer } - return inspected.Config.Labels, nil + return inspected.ID, inspected.Config.Labels, nil } // pullIfMissing 在镜像不在本地时从远端仓库拉取；镜像缺失属于可恢复错误，调用方可直接重试。 @@ -669,10 +1006,138 @@ func (docker dockerClient) expect(method, path string, payload any, allowed ...i return nil } } + if response.StatusCode == http.StatusNotFound { + return os.ErrNotExist + } message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) return fmt.Errorf("Docker returned %s: %s", response.Status, strings.TrimSpace(string(message))) } +func (locks *dockerAliasReservations) acquire(alias string) (string, func(), error) { + inspected, err := locks.docker.request(http.MethodGet, "/containers/"+url.PathEscape(locks.self)+"/json", nil) + if err != nil { + return "", nil, err + } + var gatewayContainer struct { + Image string `json:"Image"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + } + if inspected.StatusCode != http.StatusOK || json.NewDecoder(inspected.Body).Decode(&gatewayContainer) != nil || gatewayContainer.Image == "" || + gatewayContainer.Config.Labels[gatewayMemberLabel] != "true" { + inspected.Body.Close() + return "", nil, errors.New("inspect trusted gateway for alias reservation") + } + inspected.Body.Close() + generationBytes := make([]byte, 16) + if _, err := rand.Read(generationBytes); err != nil { + return "", nil, errors.New("create alias reservation generation") + } + generation := hex.EncodeToString(generationBytes) + + response, err := locks.docker.request(http.MethodPost, "/containers/create?name="+url.QueryEscape(reservationPrefix+alias), map[string]any{ + "Image": gatewayContainer.Image, + "Labels": map[string]string{reservationLabel: "true", idLabel: alias, reservationGenLabel: generation}, + "HostConfig": map[string]any{"NetworkMode": "none"}, + }) + if err != nil { + return "", nil, locks.reconcile(alias, generation, "", fmt.Errorf("create alias reservation result unknown: %w", err)) + } + defer response.Body.Close() + if response.StatusCode == http.StatusConflict { + return "", nil, errGenerationConflict + } + if response.StatusCode != http.StatusCreated { + return "", nil, locks.reconcile(alias, generation, "", fmt.Errorf("create alias reservation returned %s", response.Status)) + } + var created struct { + ID string `json:"Id"` + } + if json.NewDecoder(response.Body).Decode(&created) != nil || created.ID == "" { + return "", nil, locks.reconcile(alias, generation, "", errors.New("Docker returned an invalid alias reservation id")) + } + actualID, inspectErr := locks.inspect(alias, generation) + if inspectErr != nil { + return "", nil, fmt.Errorf("Docker returned an unverified alias reservation id; generation %s requires manual reconcile: %w", generation, inspectErr) + } + if actualID != created.ID { + return "", nil, fmt.Errorf("Docker returned an alias reservation id that conflicts with generation %s; manual reconcile required", generation) + } + return created.ID, func() { + if err := locks.remove(alias, generation, created.ID); err != nil { + logrus.WithError(err).WithFields(logrus.Fields{"alias": alias, "reservation_id": created.ID, + "reservation_generation": generation}).Error("alias reservation cleanup pending; manual reconcile required") + } + }, nil +} + +func (locks *dockerAliasReservations) reconcile(alias, generation, expectedID string, cause error) error { + actualID, inspectErr := locks.inspect(alias, generation) + if inspectErr != nil { + return fmt.Errorf("%w; reservation generation %s requires manual reconcile: %v", cause, generation, inspectErr) + } + if expectedID != "" && actualID != expectedID { + return fmt.Errorf("%w; reservation generation %s conflicts with immutable id", cause, generation) + } + if cleanupErr := locks.remove(alias, generation, actualID); cleanupErr != nil { + return fmt.Errorf("%w; reservation %s generation %s cleanup pending: %v", cause, actualID, generation, cleanupErr) + } + return fmt.Errorf("%w; reservation %s was removed", cause, actualID) +} + +func (locks *dockerAliasReservations) remove(alias, generation, expectedID string) error { + var deleteErr error + for attempt := 0; attempt < 2; attempt++ { + actualID, inspectErr := locks.inspect(alias, generation) + if errors.Is(inspectErr, os.ErrNotExist) { + return nil + } + if inspectErr != nil { + return inspectErr + } + if actualID != expectedID { + return errors.New("reservation generation conflicts with immutable id") + } + deleteErr = locks.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(expectedID)+"?force=1&v=0", nil, + http.StatusNoContent) + if deleteErr == nil { + return nil + } + if errors.Is(deleteErr, os.ErrNotExist) { + _, confirmErr := locks.inspect(alias, generation) + if errors.Is(confirmErr, os.ErrNotExist) { + return nil + } + return errors.Join(deleteErr, confirmErr) + } + } + return deleteErr +} + +func (locks *dockerAliasReservations) inspect(alias, generation string) (string, error) { + response, err := locks.docker.request(http.MethodGet, "/containers/"+url.PathEscape(reservationPrefix+alias)+"/json", nil) + if err != nil { + return "", err + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return "", os.ErrNotExist + } + var reservation struct { + ID string `json:"Id"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + } + if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&reservation) != nil || reservation.ID == "" || + reservation.Config.Labels[reservationLabel] != "true" || reservation.Config.Labels[idLabel] != alias || + reservation.Config.Labels[reservationGenLabel] != generation { + return "", errors.New("reservation name does not identify the expected alias generation") + } + return reservation.ID, nil +} + func tenantNetworkName(base, alias string) (string, error) { name := base + "-" + alias if !networkNamePattern.MatchString(name) { @@ -681,118 +1146,306 @@ func tenantNetworkName(base, alias string) (string, error) { return name, nil } -func (docker dockerClient) ensureTenantNetwork(base, alias, self string) (string, string, error) { - name, err := tenantNetworkName(base, alias) - if err != nil || self == "" { - return "", "", errors.New("isolated browser network identity is invalid") +func sameContainerReference(id, name, reference string) bool { + if reference == "" { + return false } - inspect := func() (map[string]struct { - Name string `json:"Name"` - IPv4Address string `json:"IPv4Address"` - }, error) { - response, requestErr := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil) - if requestErr != nil { - return nil, requestErr - } - defer response.Body.Close() - if response.StatusCode == http.StatusNotFound { - return nil, os.ErrNotExist - } - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("inspect isolated browser network returned %s", response.Status) - } - var network struct { - Name string `json:"Name"` - Driver string `json:"Driver"` - Internal bool `json:"Internal"` - Attachable bool `json:"Attachable"` - Ingress bool `json:"Ingress"` - Labels map[string]string `json:"Labels"` - Containers map[string]struct { - Name string `json:"Name"` - IPv4Address string `json:"IPv4Address"` - } `json:"Containers"` - } - if err := json.NewDecoder(response.Body).Decode(&network); err != nil { - return nil, err - } - if network.Name != name || network.Driver != "bridge" || network.Internal || network.Attachable || network.Ingress || - network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias { - return nil, errors.New("isolated browser network is not owned by this runtime") - } - return network.Containers, nil - } - containers, err := inspect() - if errors.Is(err, os.ErrNotExist) { - if err := docker.expect(http.MethodPost, "/networks/create", map[string]any{ - "Name": name, "CheckDuplicate": true, "Driver": "bridge", - "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias}, - }, http.StatusCreated); err != nil { - return "", "", err - } - containers = map[string]struct { - Name string `json:"Name"` - IPv4Address string `json:"IPv4Address"` - }{} - } else if err != nil { - return "", "", err - } - findIP := func() string { - for id, container := range containers { - if id == self || strings.HasPrefix(id, self) || strings.HasPrefix(self, id) || container.Name == self { - host, _, _ := net.ParseCIDR(container.IPv4Address) - if host != nil { - return host.String() - } - } - } - return "" - } - if bindHost := findIP(); bindHost != "" { - return name, bindHost, nil - } - if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(name)+"/connect", map[string]any{ - "Container": self, "EndpointConfig": map[string]any{"Aliases": []string{browserProxyHost}}, - }, http.StatusOK); err != nil { - return "", "", err - } - containers, err = inspect() - if err != nil { - return "", "", err - } - if bindHost := findIP(); bindHost != "" { - return name, bindHost, nil - } - return "", "", errors.New("Docker did not assign the gateway an isolated network address") + return id == reference || name == reference || strings.HasPrefix(id, reference) || strings.HasPrefix(reference, id) } -func (docker dockerClient) removeTenantNetwork(base, alias, self string) error { +func (docker dockerClient) trustedGatewayMember(id string) bool { + response, err := docker.request(http.MethodGet, "/containers/"+url.PathEscape(id)+"/json", nil) + if err != nil { + return false + } + defer response.Body.Close() + var container struct { + ID string `json:"Id"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + } + return response.StatusCode == http.StatusOK && json.NewDecoder(response.Body).Decode(&container) == nil && + container.ID != "" && container.Config.Labels[gatewayMemberLabel] == "true" +} + +func (docker dockerClient) inspectTenantNetwork(base, alias string, bindingVersion int64, runtimeID, self, expectedID string, + allowUnversioned bool) (tenantNetworkGeneration, map[string]string, bool, error) { name, err := tenantNetworkName(base, alias) if err != nil { - return err + return tenantNetworkGeneration{}, nil, false, err } - response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil) + generation := tenantNetworkGeneration{ID: expectedID, Name: name} + reference := name + if expectedID != "" { + reference = expectedID + } + response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(reference), nil) if err != nil { - return err + return generation, nil, false, err } defer response.Body.Close() if response.StatusCode == http.StatusNotFound { + return generation, nil, false, nil + } + var network dockerTenantNetwork + if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&network) != nil || network.ID == "" || + network.Name != name || network.Driver != "bridge" || network.Internal || network.Attachable || network.Ingress || + network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias { + return generation, nil, false, errors.New("refusing to operate on an unowned browser network") + } + if expectedID != "" && network.ID != expectedID { + return generation, nil, false, errGenerationConflict + } + networkVersion := network.Labels[bindingVersionLabel] + if networkVersion != strconv.FormatInt(bindingVersion, 10) && !(allowUnversioned && networkVersion == "") { + return generation, nil, false, errGenerationConflict + } + generation.ID = network.ID + addresses := make(map[string]string, len(network.Containers)) + for id, member := range network.Containers { + addresses[id] = member.IPv4Address + if id == runtimeID { + generation.RuntimeAttached = true + continue + } + if !docker.trustedGatewayMember(id) { + return generation, nil, false, errGenerationConflict + } + generation.GatewayMembers = append(generation.GatewayMembers, id) + if sameContainerReference(id, member.Name, self) { + generation.SelfMember = id + } + } + return generation, addresses, true, nil +} + +func (docker dockerClient) ensureTenantNetwork(base, alias, self string, bindingVersion int64, runtimeID, expectedID string, + allowUnversioned bool) (tenantNetworkGeneration, string, error) { + if self == "" { + return tenantNetworkGeneration{}, "", errors.New("isolated browser network identity is invalid") + } + generation, addresses, exists, err := docker.inspectTenantNetwork(base, alias, bindingVersion, runtimeID, self, expectedID, allowUnversioned) + if err != nil { + return generation, "", err + } + if !exists { + if expectedID != "" { + return generation, "", errGenerationConflict + } + response, createErr := docker.request(http.MethodPost, "/networks/create", map[string]any{ + "Name": generation.Name, "CheckDuplicate": true, "Driver": "bridge", + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias, + bindingVersionLabel: strconv.FormatInt(bindingVersion, 10)}, + }) + if createErr != nil { + return generation, "", fmt.Errorf("create isolated browser network result unknown; immutable generation requires manual reconcile: %w", createErr) + } + var created struct { + ID string `json:"Id"` + } + if response.StatusCode != http.StatusCreated || json.NewDecoder(response.Body).Decode(&created) != nil || created.ID == "" { + response.Body.Close() + return generation, "", errors.New("create isolated browser network result has no immutable id; manual reconcile required") + } + response.Body.Close() + generation.ID, generation.Created = created.ID, true + observed, currentAddresses, observedExists, inspectErr := docker.inspectTenantNetwork(base, alias, bindingVersion, runtimeID, self, generation.ID, false) + generation = preserveTenantNetworkGeneration(generation, observed) + addresses, err = currentAddresses, inspectErr + if err != nil || !observedExists { + if err == nil { + err = errGenerationConflict + } + return generation, "", err + } + } + if runtimeID != "" && !generation.RuntimeAttached { + generation.ConnectedRuntime = true + if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(generation.ID)+"/connect", map[string]any{ + "Container": runtimeID, + }, http.StatusOK); err != nil { + observed, _, _, _ := docker.inspectTenantNetwork(base, alias, bindingVersion, runtimeID, self, generation.ID, allowUnversioned) + generation = preserveTenantNetworkGeneration(generation, observed) + return generation, "", err + } + generation.RuntimeAttached = true + } + if generation.SelfMember == "" { + generation.ConnectedSelf = true + if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(generation.ID)+"/connect", map[string]any{ + "Container": self, "EndpointConfig": map[string]any{"Aliases": []string{browserProxyHost}}, + }, http.StatusOK); err != nil { + observed, _, _, _ := docker.inspectTenantNetwork(base, alias, bindingVersion, runtimeID, self, generation.ID, allowUnversioned) + generation = preserveTenantNetworkGeneration(generation, observed) + return generation, "", err + } + } + observed, currentAddresses, observedExists, inspectErr := docker.inspectTenantNetwork(base, alias, bindingVersion, runtimeID, self, generation.ID, allowUnversioned) + generation = preserveTenantNetworkGeneration(generation, observed) + addresses, err = currentAddresses, inspectErr + if err != nil || !observedExists || generation.SelfMember == "" { + if err == nil && !observedExists { + err = errGenerationConflict + } + return generation, "", errors.Join(err, errors.New("Docker did not connect the gateway to the isolated network")) + } + host, _, _ := net.ParseCIDR(addresses[generation.SelfMember]) + if host == nil { + return generation, "", errors.New("Docker did not assign the gateway an isolated network address") + } + return generation, host.String(), nil +} + +func preserveTenantNetworkGeneration(known, observed tenantNetworkGeneration) tenantNetworkGeneration { + if observed.ID == "" { + observed.ID = known.ID + } + if observed.Name == "" { + observed.Name = known.Name + } + observed.Created = observed.Created || known.Created + observed.RuntimeAttached = observed.RuntimeAttached || known.RuntimeAttached + observed.ConnectedRuntime = observed.ConnectedRuntime || known.ConnectedRuntime + observed.ConnectedSelf = observed.ConnectedSelf || known.ConnectedSelf + if observed.SelfMember == "" { + observed.SelfMember = known.SelfMember + } + for _, member := range known.GatewayMembers { + if !memberPresent(observed, member, "") { + observed.GatewayMembers = append(observed.GatewayMembers, member) + } + } + return observed +} + +func (api gateway) requireRuntimeState(alias string, input generationRequest, expectedID string, expectedExists bool) error { + runtimeID, exists, err := api.requireGeneration(alias, input) + if err != nil || exists != expectedExists || runtimeID != expectedID { + return errGenerationConflict + } + return nil +} + +func memberPresent(generation tenantNetworkGeneration, id, runtimeID string) bool { + if generation.RuntimeAttached && runtimeID != "" && sameContainerReference(id, "", runtimeID) { + return true + } + for _, member := range generation.GatewayMembers { + if sameContainerReference(member, "", id) { + return true + } + } + return false +} + +func generationWithoutMember(generation tenantNetworkGeneration, member, runtimeID string) tenantNetworkGeneration { + if generation.RuntimeAttached && runtimeID != "" && sameContainerReference(member, "", runtimeID) { + generation.RuntimeAttached = false + } + if sameContainerReference(member, "", generation.SelfMember) { + generation.SelfMember = "" + } + members := generation.GatewayMembers[:0:0] + for _, current := range generation.GatewayMembers { + if !sameContainerReference(current, "", member) { + members = append(members, current) + } + } + generation.GatewayMembers = members + return generation +} + +func (api gateway) disconnectTenantNetworkMember(alias string, bindingVersion int64, runtimeID string, + generation tenantNetworkGeneration, member string, input *generationRequest, expectedRuntime string, expectedExists bool) (tenantNetworkGeneration, error) { + if input != nil { + if err := api.requireRuntimeState(alias, *input, expectedRuntime, expectedExists); err != nil { + return generation, err + } + } + current, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, bindingVersion, runtimeID, api.self, generation.ID, false) + if err != nil || !exists { + return current, err + } + if !sameTenantNetworkMembers(current, generation) { + return current, errGenerationConflict + } + if !memberPresent(current, member, runtimeID) { + return current, nil + } + err = api.docker.expect(http.MethodPost, "/networks/"+url.PathEscape(generation.ID)+"/disconnect", map[string]any{ + "Container": member, "Force": true, + }, http.StatusOK) + current, _, exists, fenceErr := api.docker.inspectTenantNetwork(api.network, alias, bindingVersion, runtimeID, api.self, generation.ID, false) + if fenceErr != nil || !exists { + return current, errGenerationConflict + } + if sameTenantNetworkMembers(current, generationWithoutMember(generation, member, runtimeID)) { + return current, nil + } + if err != nil { + if sameTenantNetworkMembers(current, generation) { + return current, err + } + return current, errGenerationConflict + } + if !sameTenantNetworkMembers(current, generation) { + return current, errGenerationConflict + } + return current, errors.New("Docker retained an isolated network member after disconnect") +} + +func (api gateway) deleteTenantNetwork(alias string, bindingVersion int64, runtimeID string, + generation tenantNetworkGeneration, input *generationRequest, expectedRuntime string, expectedExists bool) error { + if input != nil { + if err := api.requireRuntimeState(alias, *input, expectedRuntime, expectedExists); err != nil { + return err + } + } + current, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, bindingVersion, runtimeID, api.self, generation.ID, false) + if err != nil { + return err + } + if !exists { return nil } - var network struct { - Name string `json:"Name"` - Labels map[string]string `json:"Labels"` + if current.RuntimeAttached || len(current.GatewayMembers) != 0 { + return errGenerationConflict } - if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&network) != nil || network.Name != name || - network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias { - return errors.New("refusing to remove an unowned browser network") + err = api.docker.expect(http.MethodDelete, "/networks/"+url.PathEscape(generation.ID), nil, http.StatusNoContent) + _, _, exists, fenceErr := api.docker.inspectTenantNetwork(api.network, alias, bindingVersion, runtimeID, api.self, generation.ID, false) + if fenceErr != nil { + return errGenerationConflict } - if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(name)+"/disconnect", map[string]any{ - "Container": self, "Force": true, - }, http.StatusOK, http.StatusNotFound); err != nil { - return fmt.Errorf("disconnect isolated browser network: %w", err) + if !exists { + return nil } - if err := docker.expect(http.MethodDelete, "/networks/"+url.PathEscape(name), nil, http.StatusNoContent, http.StatusNotFound); err != nil { + if err != nil { + return err + } + return errors.New("Docker retained the isolated browser network after delete") +} + +func (api gateway) removeTenantNetwork(alias string, bindingVersion int64, runtimeID string, + generation tenantNetworkGeneration, input *generationRequest, expectedRuntime string, expectedExists bool) error { + if generation.RuntimeAttached || generation.ConnectedRuntime { + var err error + generation, err = api.disconnectTenantNetworkMember(alias, bindingVersion, runtimeID, generation, runtimeID, + input, expectedRuntime, expectedExists) + if err != nil { + return fmt.Errorf("disconnect browser from isolated network: %w", err) + } + } + for len(generation.GatewayMembers) > 0 { + gatewayID := generation.GatewayMembers[0] + var err error + generation, err = api.disconnectTenantNetworkMember(alias, bindingVersion, runtimeID, generation, gatewayID, + input, expectedRuntime, expectedExists) + if err != nil { + return fmt.Errorf("disconnect trusted gateway from isolated network: %w", err) + } + } + if err := api.deleteTenantNetwork(alias, bindingVersion, runtimeID, generation, input, expectedRuntime, expectedExists); err != nil { return fmt.Errorf("remove isolated browser network: %w", err) } return nil @@ -816,6 +1469,8 @@ func statusFor(err error) int { return http.StatusNotFound case errors.Is(err, errUnmanagedContainer): return http.StatusForbidden + case errors.Is(err, errGenerationConflict): + return http.StatusConflict default: return http.StatusBadGateway } @@ -834,6 +1489,10 @@ func writeError(c fiber.Ctx, status int, err error) error { return writeJSON(c, status, map[string]string{"error": err.Error()}) } +func writeNetworkError(c fiber.Ctx, status int, err error, networkID string) error { + return writeJSON(c, status, map[string]string{"error": err.Error(), "network_id": networkID}) +} + func writeJSON(c fiber.Ctx, status int, value any) error { return c.Status(status).JSON(value) } diff --git a/cmd/docker-gateway/main_test.go b/cmd/docker-gateway/main_test.go index 1ef6ac1..d6588ef 100644 --- a/cmd/docker-gateway/main_test.go +++ b/cmd/docker-gateway/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "fmt" "io" "net" "net/http" @@ -10,7 +11,9 @@ import ( "os" "strconv" "strings" + "sync" "testing" + "time" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" @@ -25,7 +28,59 @@ func authed(method, target string, body io.Reader) *http.Request { } func testDocker(handler http.HandlerFunc) (dockerClient, *httptest.Server) { - server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + self, _ := os.Hostname() + var networkMu sync.Mutex + networkMembers := map[string]map[string]bool{} + networkDeleted := map[string]bool{} + server := httptest.NewServer(withAliasReservations(self, func(response http.ResponseWriter, request *http.Request) { + if strings.HasPrefix(request.URL.Path, "/networks/network-") { + alias := strings.TrimPrefix(request.URL.Path, "/networks/network-") + alias = strings.TrimSuffix(strings.TrimSuffix(alias, "/disconnect"), "/connect") + networkMu.Lock() + if request.Method == http.MethodGet { + if networkDeleted[alias] { + networkMu.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + members := map[string]any{} + for id := range networkMembers[alias] { + members[id] = map[string]string{"Name": id, "IPv4Address": "127.0.0.1/8"} + } + networkMu.Unlock() + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-" + alias, "Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias, bindingVersionLabel: "1"}, + "Containers": members, + }) + return + } + if strings.HasSuffix(request.URL.Path, "/disconnect") { + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + delete(networkMembers[alias], body.Container) + } else if strings.HasSuffix(request.URL.Path, "/connect") { + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + if networkMembers[alias] == nil { + networkMembers[alias] = map[string]bool{} + } + networkMembers[alias][body.Container] = true + } else if request.Method == http.MethodDelete { + networkDeleted[alias] = true + } + networkMu.Unlock() + if request.Method == http.MethodDelete { + response.WriteHeader(http.StatusNoContent) + } else { + response.WriteHeader(http.StatusOK) + } + return + } if strings.HasPrefix(request.URL.Path, "/networks/creatorhub_browser-") { if request.Method != http.MethodGet { if request.Method == http.MethodDelete { @@ -36,11 +91,25 @@ func testDocker(handler http.HandlerFunc) (dockerClient, *httptest.Server) { return } alias := strings.TrimPrefix(request.URL.Path, "/networks/creatorhub_browser-") - self, _ := os.Hostname() + networkMu.Lock() + if networkDeleted[alias] { + networkMu.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + if networkMembers[alias] == nil { + networkMembers[alias] = map[string]bool{self: true} + } + members := map[string]any{} + for id := range networkMembers[alias] { + members[id] = map[string]string{"Name": id, "IPv4Address": "127.0.0.1/8"} + } + networkMu.Unlock() _ = json.NewEncoder(response).Encode(map[string]any{ - "Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, - "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias}, - "Containers": map[string]any{self: map[string]string{"Name": self, "IPv4Address": "127.0.0.1/8"}}, + "Id": "network-" + alias, "Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias, + bindingVersionLabel: "1"}, + "Containers": members, }) return } @@ -49,6 +118,66 @@ func testDocker(handler http.HandlerFunc) (dockerClient, *httptest.Server) { return dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, server } +func withAliasReservations(self string, next http.HandlerFunc) http.HandlerFunc { + var mu sync.Mutex + type reservation struct { + id, generation string + } + locks := map[string]reservation{} + return func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodGet && request.URL.Path == "/containers/"+self+"/json" { + _, _ = response.Write([]byte(`{"Id":"` + self + `","Image":"gateway-image-id","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + return + } + name := request.URL.Query().Get("name") + if request.Method == http.MethodPost && request.URL.Path == "/containers/create" && strings.HasPrefix(name, reservationPrefix) { + mu.Lock() + defer mu.Unlock() + if locks[name].id != "" { + response.WriteHeader(http.StatusConflict) + return + } + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + id := "reservation-" + strings.TrimPrefix(name, reservationPrefix) + locks[name] = reservation{id: id, generation: payload.Labels[reservationGenLabel]} + response.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(response).Encode(map[string]string{"Id": id}) + return + } + if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+reservationPrefix) && strings.HasSuffix(request.URL.Path, "/json") { + name := strings.TrimSuffix(strings.TrimPrefix(request.URL.Path, "/containers/"), "/json") + mu.Lock() + current, found := locks[name] + mu.Unlock() + if !found { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{"Id": current.id, "Config": map[string]any{"Labels": map[string]string{ + reservationLabel: "true", idLabel: strings.TrimPrefix(name, reservationPrefix), reservationGenLabel: current.generation, + }}}) + return + } + if request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/reservation-") { + id := strings.TrimPrefix(request.URL.Path, "/containers/") + mu.Lock() + for name, current := range locks { + if current.id == id { + delete(locks, name) + break + } + } + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + return + } + next(response, request) + } +} + func decodeJSONBody(t *testing.T, response *http.Response) map[string]any { t.Helper() var body map[string]any @@ -63,12 +192,296 @@ const testCreateBody = `{"alias":"account-a","name":"账号甲","image":"registr `"binding_version":1,"network_exit_id":"exit-1",` + `"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` +const testGenerationBody = `{"binding_version":1,"runtime_id":"container-id"}` + +func TestAliasReservationNameDoesNotOverlapRuntimeNamespace(t *testing.T) { + if strings.HasPrefix(reservationPrefix, namePrefix) || strings.HasPrefix(namePrefix, reservationPrefix) { + t.Fatalf("reservation and runtime prefixes overlap: reservation=%q runtime=%q", reservationPrefix, namePrefix) + } + generation := "" + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + if request.URL.Query().Get("name") == namePrefix+"lock-account-a" { + response.WriteHeader(http.StatusConflict) + return + } + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + _, release, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a") + if err != nil { + t.Fatalf("runtime alias lock-account-a occupied account-a reservation: %v", err) + } + release() +} + +func TestAliasReservationRecoversInvalidCreateAndReleaseResponses(t *testing.T) { + t.Run("create disconnect after apply", func(t *testing.T) { + generation := "" + removed := false + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + connection, _, _ := response.(http.Hijacker).Hijack() + _ = connection.Close() + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": + removed = true + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || !removed { + t.Fatalf("disconnected reservation create was not reconciled by immutable generation: removed=%v err=%v", removed, err) + } + }) + + t.Run("invalid create body", func(t *testing.T) { + reservationExists, removed := false, false + generation := "" + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + reservationExists = true + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + if !reservationExists { + response.WriteHeader(http.StatusNotFound) + return + } + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": + reservationExists, removed = false, true + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || !removed { + t.Fatalf("invalid reservation response was not reconciled: removed=%v err=%v", removed, err) + } + }) + + t.Run("nonempty foreign create id", func(t *testing.T) { + generation := "" + deletes := 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"foreign-id"}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + case request.Method == http.MethodDelete: + deletes++ + response.WriteHeader(http.StatusNotFound) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || deletes != 0 { + t.Fatalf("foreign 201 id was accepted or deleted: deletes=%d err=%v", deletes, err) + } + }) + + t.Run("201 inspect disconnect", func(t *testing.T) { + deletes := 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + connection, _, _ := response.(http.Hijacker).Hijack() + _ = connection.Close() + case request.Method == http.MethodDelete: + deletes++ + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || deletes != 0 { + t.Fatalf("unverified 201 reservation was deleted: deletes=%d err=%v", deletes, err) + } + }) + + t.Run("delete 404 requires generation absence", func(t *testing.T) { + generation := "" + deletes := 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": + deletes++ + response.WriteHeader(http.StatusNotFound) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + locks := &dockerAliasReservations{docker: docker, self: "gateway-self"} + id, _, err := locks.acquire("account-a") + if err != nil { + t.Fatal(err) + } + if err := locks.remove("account-a", generation, id); err == nil || deletes != 1 { + t.Fatalf("404 was accepted while the reservation generation remained: deletes=%d err=%v", deletes, err) + } + }) + + t.Run("release retry", func(t *testing.T) { + deletes := 0 + generation := "" + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": + deletes++ + if deletes == 1 { + response.WriteHeader(http.StatusInternalServerError) + return + } + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + _, release, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a") + if err != nil { + t.Fatal(err) + } + release() + if deletes != 2 { + t.Fatalf("reservation release did not retry the immutable id: deletes=%d", deletes) + } + }) + + t.Run("release does not delete successor", func(t *testing.T) { + generation := "" + deletes := 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": + _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + generation = payload.Labels[reservationGenLabel] + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": + deletes++ + response.WriteHeader(http.StatusInternalServerError) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": + if deletes == 0 { + _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) + return + } + _, _ = response.Write([]byte(`{"Id":"successor-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"successor"}}}`)) + default: + t.Fatalf("unexpected Docker request %s %s generation=%s", request.Method, request.URL.String(), generation) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + _, release, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a") + if err != nil { + t.Fatal(err) + } + release() + if deletes != 1 { + t.Fatalf("reservation release deleted a successor generation: deletes=%d", deletes) + } + }) +} + func TestGatewayCreatesNetworkDisabledStoppedRecoveryContainer(t *testing.T) { created := false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): _, _ = response.Write([]byte(`{}`)) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): + response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): var payload map[string]any _ = json.NewDecoder(request.Body).Decode(&payload) @@ -104,6 +517,8 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) _, _ = response.Write([]byte(`{}`)) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): + response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): if got := request.URL.Query().Get("name"); got != namePrefix+"account-a" { t.Fatalf("unexpected container name %q", got) @@ -140,7 +555,7 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { t.Fatalf("cmd must be passed through verbatim: %#v", created["Cmd"]) } host := created["HostConfig"].(map[string]any) - if host["NetworkMode"] != "creatorhub_browser-account-a" || host["ReadonlyRootfs"] != true { + if host["NetworkMode"] != "network-account-a" || host["ReadonlyRootfs"] != true { t.Fatalf("missing container isolation: %#v", host) } tmpfs := host["Tmpfs"].(map[string]any) @@ -157,12 +572,71 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { } } +func TestGatewayCreateUsesCapturedNetworkIDAcrossNameReplacement(t *testing.T) { + networkID := "" + members := map[string]string{} + usedNetworkID, touchedReplacement := "", false + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + response.WriteHeader(http.StatusNotFound) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + reference := strings.TrimPrefix(request.URL.Path, "/networks/") + if reference == "network-n2" { + touchedReplacement = true + } + if networkID == "" || (reference != "creatorhub_browser-account-a" && reference != networkID) { + response.WriteHeader(http.StatusNotFound) + return + } + containers := map[string]any{} + for id, name := range members { + containers[id] = map[string]string{"Name": name, "IPv4Address": "127.0.0.1/8"} + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": networkID, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": containers, + }) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + networkID = "network-n1" + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) + case request.Method == http.MethodPost && request.URL.Path == "/networks/network-n1/connect": + members["gateway-self"] = "gateway-self" + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + HostConfig map[string]any `json:"HostConfig"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + usedNetworkID, _ = payload.HostConfig["NetworkMode"].(string) + networkID, members = "network-n2", map[string]string{"replacement": "replacement"} + response.WriteHeader(http.StatusNotFound) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, + "creatorhub_browser", testToken, "gateway-self") + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) + if response.Code != http.StatusBadGateway || usedNetworkID != "network-n1" || touchedReplacement || networkID != "network-n2" || members["replacement"] == "" { + t.Fatalf("stale create crossed network generation: status=%d mode=%q touchedN2=%v network=%q members=%v body=%s", + response.Code, usedNetworkID, touchedReplacement, networkID, members, response.Body.String()) + } +} + func TestGatewayDockerInspectContainsNoProxyCredentials(t *testing.T) { var created map[string]any docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): + response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): if err := json.NewDecoder(request.Body).Decode(&created); err != nil { t.Fatal(err) @@ -226,6 +700,8 @@ func TestGatewayPullsMissingImageOnCreate(t *testing.T) { t.Fatalf("unexpected pull query %s", request.URL.RawQuery) } _, _ = response.Write([]byte(`{"status":"Download complete"}`)) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): + response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-id"}`)) @@ -326,18 +802,25 @@ func TestGatewayRejectsOversizedCreateRequest(t *testing.T) { } } -func TestGatewayRemovesContainerWhenCreateResponseHasNoID(t *testing.T) { - removed := false +func TestGatewayReconcilesContainerWhenCreateResponseHasNoID(t *testing.T) { + created, started := false, false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) _, _ = response.Write([]byte(`{}`)) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): + if !created { + response.WriteHeader(http.StatusNotFound) + return + } + _, _ = response.Write([]byte(`{"Id":"actual-container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-account-a"}}}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): + created = true response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":""}`)) - case request.Method == http.MethodDelete && strings.Contains(request.URL.Path, namePrefix+"account-a"): - removed = request.URL.Query().Get("force") == "1" && request.URL.Query().Get("v") == "0" + case request.Method == http.MethodPost && request.URL.Path == "/containers/actual-container-id/start": + started = true response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) @@ -349,8 +832,8 @@ func TestGatewayRemovesContainerWhenCreateResponseHasNoID(t *testing.T) { response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) - if response.Code != http.StatusBadGateway || !removed { - t.Fatalf("expected invalid create response cleanup, status=%d removed=%v body=%s", response.Code, removed, response.Body.String()) + if response.Code != http.StatusCreated || !started { + t.Fatalf("expected invalid create response reconciliation, status=%d started=%v body=%s", response.Code, started, response.Body.String()) } } @@ -361,6 +844,8 @@ func TestGatewayRemovesFailedContainerAndPreservesProfile(t *testing.T) { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) _, _ = response.Write([]byte(`{}`)) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): + response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"failed-id"}`)) @@ -390,6 +875,10 @@ func TestGatewayDoesNotEchoProxyCredentialsFromDockerErrors(t *testing.T) { response.WriteHeader(http.StatusOK) return } + if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix) { + response.WriteHeader(http.StatusNotFound) + return + } if request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create") { response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(`invalid cmd --proxy-server=http://operator:ephemeral@proxy.example:8080`)) @@ -440,12 +929,13 @@ func TestGatewayRestartRestoresExistingProxyListener(t *testing.T) { _ = reserved.Close() labels := map[string]string{ managedLabel: "true", idLabel: "account-a", nameLabel: "账号甲", - bindingVersionLabel: "3", networkExitLabel: "exit-1", proxyPortLabel: strconv.Itoa(port), + bindingVersionLabel: "1", networkExitLabel: "exit-1", proxyPortLabel: strconv.Itoa(port), + networkIDLabel: "network-account-a", } docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasSuffix(request.URL.Path, "/containers/creatorhub-browser-account-a/json"): - _ = json.NewEncoder(response).Encode(map[string]any{"Config": map[string]any{"Labels": labels}}) + _ = json.NewEncoder(response).Encode(map[string]any{"Id": "container-id", "Config": map[string]any{"Labels": labels}}) case request.Method == http.MethodGet && request.URL.Path == "/containers/json": _ = json.NewEncoder(response).Encode([]map[string]any{{"Id": "container-id", "State": "running", "Status": "Up", "Labels": labels}}) default: @@ -454,7 +944,7 @@ func TestGatewayRestartRestoresExistingProxyListener(t *testing.T) { }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) - recovery := `{"binding_version":3,"network_exit_id":"exit-1","network_exit":{"protocol":"http","host":"127.0.0.1","port":1}}` + recovery := `{"binding_version":1,"runtime_id":"container-id","network_id":"network-account-a","network_exit_id":"exit-1","network_exit":{"protocol":"http","host":"127.0.0.1","port":1}}` response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(recovery))) if response.Code != http.StatusNoContent { @@ -468,21 +958,91 @@ func TestGatewayRestartRestoresExistingProxyListener(t *testing.T) { } } -func TestGatewayLifecycle(t *testing.T) { +func TestGatewayRestoreFinalFenceRemovesStaleProxy(t *testing.T) { + reserved, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := reserved.Addr().(*net.TCPAddr).Port + _ = reserved.Close() + registry := newMemoryProxyRegistry() + networkID := "network-n1" + replaced := false + containerReads := 0 + exit := gatewayProxyExit{Protocol: "socks5", Host: "proxy.example", Port: 1080} + proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", port, exit, "network-n1") + if err != nil || !registry.bind("account-a", 1, proxyURL, "container-c1", "network-n1") { + t.Fatalf("seed existing proxy generation: %v", err) + } + defer cleanup() + labels := map[string]string{ + managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "1", networkExitLabel: "exit-1", + proxyPortLabel: strconv.Itoa(port), networkIDLabel: "network-n1", + } + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + containerReads++ + registry.mu.Lock() + proxy := registry.proxies["account-a"] + bound := proxy != nil && proxy.runtimeID == "container-c1" + registry.mu.Unlock() + if bound && containerReads == 4 && !replaced { + networkID, replaced = "network-n2", true + } + _ = json.NewEncoder(response).Encode(map[string]any{"Id": "container-c1", "Config": map[string]any{"Labels": labels}}) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + reference := strings.TrimPrefix(request.URL.Path, "/networks/") + if reference != networkID && reference != "creatorhub_browser-account-a" { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": networkID, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": map[string]any{ + "container-c1": map[string]string{"Name": namePrefix + "account-a", "IPv4Address": "127.0.0.3/8"}, + "gateway-self": map[string]string{"Name": "gateway-self", "IPv4Address": "127.0.0.1/8"}, + }, + }) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-self", token: testToken, proxies: registry, + locks: &dockerAliasReservations{docker: docker, self: "gateway-self"}} + app := fiber.New() + app.Use("/v1", api.authorize) + app.Post("/v1/browsers/:id/proxy", api.restoreProxy) + body := `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` + response := httptest.NewRecorder() + adaptor.FiberApp(app).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(body))) + registry.mu.Lock() + proxy := registry.proxies["account-a"] + registry.mu.Unlock() + if response.Code != http.StatusConflict || !replaced || networkID != "network-n2" || proxy != nil { + t.Fatalf("restore final fence accepted a replaced network: status=%d replaced=%v network=%q proxy=%v body=%s", + response.Code, replaced, networkID, proxy, response.Body.String()) + } +} + +func TestGatewayLifecycleUsesInspectedImmutableContainerID(t *testing.T) { tests := []struct { method string path string dockerPath string }{ - {http.MethodPost, "/v1/browsers/account-a/start", "/containers/creatorhub-browser-account-a/start"}, - {http.MethodPost, "/v1/browsers/account-a/stop", "/containers/creatorhub-browser-account-a/stop"}, - {http.MethodDelete, "/v1/browsers/account-a", "/containers/creatorhub-browser-account-a"}, + {http.MethodPost, "/v1/browsers/account-a/start", "/containers/container-id/start"}, + {http.MethodPost, "/v1/browsers/account-a/stop", "/containers/container-id/stop"}, + {http.MethodDelete, "/v1/browsers/account-a", "/containers/container-id"}, } for _, test := range tests { t.Run(test.method+" "+test.path, func(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet { - _, _ = response.Write([]byte(`{"Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a"}}}`)) + _, _ = response.Write([]byte(`{"Id":"container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1"}}}`)) return } if request.URL.Path != test.dockerPath { @@ -494,7 +1054,11 @@ func TestGatewayLifecycle(t *testing.T) { handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() - adaptor.FiberApp(handler).ServeHTTP(response, authed(test.method, test.path, nil)) + var body io.Reader + if test.path != "/v1/browsers/account-a/start" { + body = strings.NewReader(testGenerationBody) + } + adaptor.FiberApp(handler).ServeHTTP(response, authed(test.method, test.path, body)) if response.Code != http.StatusNoContent { t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String()) } @@ -502,32 +1066,54 @@ func TestGatewayLifecycle(t *testing.T) { } } -func TestGatewayDeleteDistinguishesContainerRemovalFromNetworkCleanup(t *testing.T) { +func TestGatewayDeleteUsesImmutableNetworkIDAcrossCleanupRetry(t *testing.T) { containerExists, cleanupFails, containerDeletes := true, true, 0 - server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + networkExists := true + networkMembers := map[string]any{ + "container-id": map[string]string{"Name": namePrefix + "account-a"}, + "gateway-self": map[string]string{"Name": "gateway-self"}, + } + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"): if !containerExists { response.WriteHeader(http.StatusNotFound) return } - _, _ = response.Write([]byte(`{"Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a"}}}`)) + _, _ = response.Write([]byte(`{"Id":"container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-id"}}}`)) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): containerExists = false containerDeletes++ response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + if !networkExists { + response.WriteHeader(http.StatusNotFound) + return + } _ = json.NewEncoder(response).Encode(map[string]any{ - "Name": "creatorhub_browser-account-a", - "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a"}, + "Id": "network-id", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": networkMembers, }) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + if request.URL.Path != "/networks/network-id/disconnect" { + t.Fatalf("network cleanup did not use immutable id: %s", request.URL.Path) + } if cleanupFails { response.WriteHeader(http.StatusInternalServerError) return } + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + delete(networkMembers, body.Container) response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): + if request.URL.Path != "/networks/network-id" { + t.Fatalf("network delete did not use immutable id: %s", request.URL.Path) + } + networkExists = false response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) @@ -538,19 +1124,1017 @@ func TestGatewayDeleteDistinguishesContainerRemovalFromNetworkCleanup(t *testing "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() - adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil)) - if response.Code != http.StatusAccepted || containerExists || containerDeletes != 1 { - t.Fatalf("expected definite container removal with pending cleanup, status=%d exists=%v deletes=%d body=%s", + deleteBody := `{"binding_version":1,"runtime_id":"container-id","network_id":"network-id"}` + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(deleteBody))) + if response.Code != http.StatusAccepted || !containerExists || containerDeletes != 0 { + t.Fatalf("expected alias reservation with pending cleanup, status=%d exists=%v deletes=%d body=%s", response.Code, containerExists, containerDeletes, response.Body.String()) } cleanupFails = false response = httptest.NewRecorder() - adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil)) - if response.Code != http.StatusNoContent || containerDeletes != 1 { + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(deleteBody))) + if response.Code != http.StatusNoContent || containerExists || containerDeletes != 1 { t.Fatalf("idempotent cleanup retry failed: status=%d deletes=%d body=%s", response.Code, containerDeletes, response.Body.String()) } } +func TestGatewayDeleteWithoutContainerOrNetworkGenerationFailsClosed(t *testing.T) { + networkRequests := 0 + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + response.WriteHeader(http.StatusNotFound) + case strings.HasPrefix(request.URL.Path, "/networks/"): + networkRequests++ + response.WriteHeader(http.StatusInternalServerError) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, + "creatorhub_browser", testToken, "gateway-self") + response := httptest.NewRecorder() + body := `{"binding_version":1,"runtime_id":"runtime-not-found"}` + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(body))) + if response.Code != http.StatusConflict || networkRequests != 0 { + t.Fatalf("legacy cleanup discovered a replacement network: status=%d networkRequests=%d body=%s", + response.Code, networkRequests, response.Body.String()) + } +} + +func TestGatewayRejectsStaleProxyRestoreAfterReplacementGeneration(t *testing.T) { + type dockerState struct { + sync.Mutex + containerID string + containerLabels map[string]string + networkID string + networkMembers map[string]string + containerReads int + cleanupMutations []string + } + state := &dockerState{ + containerID: "container-c1", + containerLabels: map[string]string{managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "1", networkExitLabel: "exit-1", proxyPortLabel: "12345", networkIDLabel: "network-n1"}, + networkID: "network-n1", + networkMembers: map[string]string{"container-c1": namePrefix + "account-a", "gateway-self": "gateway-self"}, + } + r1Captured := make(chan struct{}) + resumeR1 := make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(resumeR1) }) } + defer release() + dockerServer := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + state.Lock() + containerID := state.containerID + labels := state.containerLabels + state.containerReads++ + first := state.containerReads == 1 + state.Unlock() + if first { + close(r1Captured) + <-resumeR1 + } + if containerID == "" { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{"Id": containerID, "Config": map[string]any{"Labels": labels}}) + case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || + request.URL.Path == "/networks/network-n1" || request.URL.Path == "/networks/network-n2"): + state.Lock() + networkID := state.networkID + members := make(map[string]map[string]string, len(state.networkMembers)) + for id, name := range state.networkMembers { + members[id] = map[string]string{"Name": name, "IPv4Address": "127.0.0.1/8"} + } + state.Unlock() + if networkID == "" { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": networkID, "Name": "creatorhub_browser-account-a", "Driver": "bridge", + "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": members, + }) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + requestedNetwork := strings.TrimSuffix(strings.TrimPrefix(request.URL.Path, "/networks/"), "/disconnect") + state.Lock() + if requestedNetwork != state.networkID { + state.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + state.cleanupMutations = append(state.cleanupMutations, "disconnect:"+requestedNetwork+":"+body.Container) + delete(state.networkMembers, body.Container) + state.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): + requestedNetwork := strings.TrimPrefix(request.URL.Path, "/networks/") + state.Lock() + if requestedNetwork != state.networkID { + state.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + state.cleanupMutations = append(state.cleanupMutations, "delete-network:"+requestedNetwork) + state.networkID = "" + state.networkMembers = nil + state.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): + requestedContainer := strings.TrimPrefix(request.URL.Path, "/containers/") + state.Lock() + if requestedContainer != state.containerID { + state.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + state.cleanupMutations = append(state.cleanupMutations, "delete-container:"+requestedContainer) + state.containerID = "" + state.containerLabels = nil + state.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + state.Lock() + state.networkID = "network-n2" + state.networkMembers = map[string]string{} + state.Unlock() + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"network-n2"}`)) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): + state.Lock() + state.networkMembers["gateway-self"] = "gateway-self" + state.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + state.Lock() + state.containerID = "container-c2" + state.containerLabels = payload.Labels + state.networkMembers["container-c2"] = namePrefix + "account-a" + state.Unlock() + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"container-c2"}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/container-c2/start": + response.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected Docker request %s %s", request.Method, request.URL.String()) + response.WriteHeader(http.StatusInternalServerError) + } + })) + defer dockerServer.Close() + + registry := newMemoryProxyRegistry() + proxyURL, _, err := registry.configure("account-a", 1, "127.0.0.1", 0, + gatewayProxyExit{Protocol: "socks5", Host: "proxy.example", Port: 1080}) + if err != nil || !registry.bind("account-a", 1, proxyURL, "container-c1", "network-n1") { + t.Fatalf("seed C1 proxy: %v", err) + } + defer func() { + registry.remove("account-a", 1, "container-c2") + registry.remove("account-a", 1, "container-c1") + }() + api := gateway{ + docker: dockerClient{baseURL: dockerServer.URL, client: dockerServer.Client(), slow: dockerServer.Client()}, + network: "creatorhub_browser", self: "gateway-self", token: testToken, proxies: registry, + } + api.locks = &dockerAliasReservations{docker: api.docker, self: api.self} + app := fiber.New() + app.Use("/v1", api.authorize) + app.Post("/v1/browsers", api.create) + app.Post("/v1/browsers/:id/proxy", api.restoreProxy) + app.Delete("/v1/browsers/:id", api.remove) + gatewayServer := httptest.NewServer(adaptor.FiberApp(app)) + defer gatewayServer.Close() + defer release() + + type result struct { + status int + body string + } + call := func(method, path, body string) result { + request, _ := http.NewRequest(method, gatewayServer.URL+path, strings.NewReader(body)) + request.Header.Set("Authorization", "Bearer "+testToken) + response, err := gatewayServer.Client().Do(request) + if err != nil { + return result{body: err.Error()} + } + defer response.Body.Close() + responseBody, _ := io.ReadAll(response.Body) + return result{status: response.StatusCode, body: string(responseBody)} + } + r1Result := make(chan result, 1) + go func() { + r1Result <- call(http.MethodPost, "/v1/browsers/account-a/proxy", + `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}`) + }() + select { + case <-r1Captured: + case <-time.After(5 * time.Second): + t.Fatal("R1 did not capture C1/N1") + } + + r2Delete := call(http.MethodDelete, "/v1/browsers/account-a", `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1"}`) + if r2Delete.status != http.StatusNoContent { + t.Fatalf("R2 cleanup failed: status=%d body=%s", r2Delete.status, r2Delete.body) + } + r2Create := call(http.MethodPost, "/v1/browsers", testCreateBody) + if r2Create.status != http.StatusCreated { + t.Fatalf("R2 replacement create failed: status=%d body=%s", r2Create.status, r2Create.body) + } + release() + select { + case stale := <-r1Result: + if stale.status != http.StatusConflict { + t.Fatalf("R1 stale restore was not fenced: status=%d body=%s", stale.status, stale.body) + } + case <-time.After(5 * time.Second): + t.Fatal("R1 did not finish") + } + + state.Lock() + defer state.Unlock() + proxy := registry.proxies["account-a"] + wantCleanup := []string{ + "disconnect:network-n1:container-c1", + "disconnect:network-n1:gateway-self", + "delete-network:network-n1", + "delete-container:container-c1", + } + if state.containerID != "container-c2" || state.networkID != "network-n2" || state.networkMembers["container-c2"] == "" || + proxy == nil || proxy.runtimeID != "container-c2" || strings.Join(state.cleanupMutations, ",") != strings.Join(wantCleanup, ",") { + t.Fatalf("stale R1 affected replacement generation: container=%q network=%q members=%v proxy=%v mutations=%v", + state.containerID, state.networkID, state.networkMembers, proxy, state.cleanupMutations) + } +} + +func TestGatewayRejectsStaleCreateBeforeNetworkOrProxyMutation(t *testing.T) { + type dockerState struct { + sync.Mutex + containerID string + containerLabels map[string]string + containerReads int + networkExists bool + networkCreates int + networkConnects int + gatewayConnected bool + containerCreates int + } + state := &dockerState{} + r1Inspected := make(chan struct{}) + resumeR1 := make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(resumeR1) }) } + defer release() + dockerServer := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + state.Lock() + containerID, labels := state.containerID, state.containerLabels + state.containerReads++ + first := state.containerReads == 1 + state.Unlock() + if first { + close(r1Inspected) + <-resumeR1 + } + if containerID == "" { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{"Id": containerID, "Config": map[string]any{"Labels": labels}}) + case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1"): + state.Lock() + exists, connected := state.networkExists, state.gatewayConnected + state.Unlock() + if !exists { + response.WriteHeader(http.StatusNotFound) + return + } + members := map[string]any{} + if connected { + members["gateway-self"] = map[string]string{"Name": "gateway-self", "IPv4Address": "127.0.0.1/8"} + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": members, + }) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + state.Lock() + state.networkExists = true + state.networkCreates++ + state.Unlock() + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): + state.Lock() + state.networkConnects++ + state.gatewayConnected = true + state.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + state.Lock() + state.containerID = "container-c1" + state.containerLabels = payload.Labels + state.containerCreates++ + state.Unlock() + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"container-c1"}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/container-c1/start": + response.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected Docker request %s %s", request.Method, request.URL.String()) + response.WriteHeader(http.StatusInternalServerError) + } + })) + defer dockerServer.Close() + docker := dockerClient{baseURL: dockerServer.URL, client: dockerServer.Client(), slow: dockerServer.Client()} + registry := newMemoryProxyRegistry() + defer registry.remove("account-a", 1, "container-c1") + api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-self", token: testToken, proxies: registry, + locks: &dockerAliasReservations{docker: docker, self: "gateway-self"}} + app := fiber.New() + app.Use("/v1", api.authorize) + app.Post("/v1/browsers", api.create) + gatewayServer := httptest.NewServer(adaptor.FiberApp(app)) + defer gatewayServer.Close() + + call := func() (int, string) { + request, _ := http.NewRequest(http.MethodPost, gatewayServer.URL+"/v1/browsers", strings.NewReader(testCreateBody)) + request.Header.Set("Authorization", "Bearer "+testToken) + response, err := gatewayServer.Client().Do(request) + if err != nil { + return 0, err.Error() + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + return response.StatusCode, string(body) + } + r1Result := make(chan struct { + status int + body string + }, 1) + go func() { + status, body := call() + r1Result <- struct { + status int + body string + }{status, body} + }() + select { + case <-r1Inspected: + case <-time.After(5 * time.Second): + t.Fatal("R1 did not inspect the empty alias") + } + status, body := call() + if status != http.StatusCreated { + release() + t.Fatalf("R2 create failed: status=%d body=%s", status, body) + } + release() + select { + case stale := <-r1Result: + if stale.status != http.StatusConflict { + t.Fatalf("R1 stale create was not fenced: status=%d body=%s", stale.status, stale.body) + } + case <-time.After(5 * time.Second): + t.Fatal("R1 did not finish") + } + + state.Lock() + defer state.Unlock() + proxy := registry.proxies["account-a"] + if state.containerID != "container-c1" || state.networkCreates != 1 || state.networkConnects != 1 || state.containerCreates != 1 || + proxy == nil || proxy.runtimeID != "container-c1" { + t.Fatalf("stale create left side effects: container=%q networkCreates=%d connects=%d containerCreates=%d proxy=%v", + state.containerID, state.networkCreates, state.networkConnects, state.containerCreates, proxy) + } +} + +func TestGatewayTwoReplicaCreateRestoreRemoveProxyContract(t *testing.T) { + type member struct{ name, ip string } + type dockerState struct { + sync.Mutex + containerID string + containerLabels map[string]string + networkID string + networkMembers map[string]member + sequence int + } + state := &dockerState{} + dockerServer := httptest.NewServer(withAliasReservations("gateway-g1", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-g2/json": + _, _ = response.Write([]byte(`{"Id":"gateway-g2","Image":"gateway-image-id","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + state.Lock() + id, labels := state.containerID, state.containerLabels + state.Unlock() + if id == "" { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{"Id": id, "Config": map[string]any{"Labels": labels}}) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + state.Lock() + id := state.networkID + members := map[string]any{} + for memberID, current := range state.networkMembers { + members[memberID] = map[string]string{"Name": current.name, "IPv4Address": current.ip} + } + state.Unlock() + if id == "" { + response.WriteHeader(http.StatusNotFound) + return + } + reference := strings.TrimPrefix(request.URL.Path, "/networks/") + if reference != "creatorhub_browser-account-a" && reference != id { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": id, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": members, + }) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + state.Lock() + state.sequence++ + state.networkID = fmt.Sprintf("network-n%d", state.sequence) + state.networkMembers = map[string]member{} + id := state.networkID + state.Unlock() + response.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(response).Encode(map[string]string{"Id": id}) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + state.Lock() + ip := "127.0.0.3/8" + if body.Container == "gateway-g1" { + ip = "127.0.0.1/8" + } else if body.Container == "gateway-g2" { + ip = "127.0.0.2/8" + } + state.networkMembers[body.Container] = member{name: body.Container, ip: ip} + state.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + var payload struct { + Labels map[string]string `json:"Labels"` + } + _ = json.NewDecoder(request.Body).Decode(&payload) + state.Lock() + id := fmt.Sprintf("container-c%d", state.sequence) + state.containerID, state.containerLabels = id, payload.Labels + state.networkMembers[id] = member{name: namePrefix + "account-a", ip: "127.0.0.3/8"} + state.Unlock() + response.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(response).Encode(map[string]string{"Id": id}) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"): + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + state.Lock() + delete(state.networkMembers, body.Container) + state.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): + state.Lock() + if len(state.networkMembers) != 0 { + state.Unlock() + response.WriteHeader(http.StatusConflict) + return + } + state.networkID = "" + state.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): + id := strings.TrimPrefix(request.URL.Path, "/containers/") + state.Lock() + if id != state.containerID { + state.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + delete(state.networkMembers, id) + state.containerID, state.containerLabels = "", nil + state.Unlock() + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer dockerServer.Close() + docker := dockerClient{baseURL: dockerServer.URL, client: dockerServer.Client(), slow: dockerServer.Client()} + newReplica := func(self string) (*fiber.App, *memoryProxyRegistry) { + registry := newMemoryProxyRegistry() + api := gateway{docker: docker, network: "creatorhub_browser", self: self, token: testToken, proxies: registry, + locks: &dockerAliasReservations{docker: docker, self: self}} + app := fiber.New() + app.Use("/v1", api.authorize) + app.Post("/v1/browsers", api.create) + app.Post("/v1/browsers/:id/proxy", api.restoreProxy) + app.Delete("/v1/browsers/:id", api.remove) + return app, registry + } + g1, proxiesG1 := newReplica("gateway-g1") + g2, proxiesG2 := newReplica("gateway-g2") + call := func(app *fiber.App, method, path, body string) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + adaptor.FiberApp(app).ServeHTTP(response, authed(method, path, strings.NewReader(body))) + return response + } + created := call(g1, http.MethodPost, "/v1/browsers", testCreateBody) + if created.Code != http.StatusCreated { + t.Fatalf("G1 create failed: %d %s", created.Code, created.Body.String()) + } + state.Lock() + c1, n1, port := state.containerID, state.networkID, state.containerLabels[proxyPortLabel] + state.Unlock() + restoreBody := `{"binding_version":1,"runtime_id":"` + c1 + `","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` + if restored := call(g2, http.MethodPost, "/v1/browsers/account-a/proxy", restoreBody); restored.Code != http.StatusNoContent { + t.Fatalf("G2 restore failed: %d %s", restored.Code, restored.Body.String()) + } + if proxiesG1.proxies["account-a"].runtimeID != c1 || proxiesG2.proxies["account-a"].runtimeID != c1 || port == "" { + t.Fatal("both replicas did not bind the same runtime generation") + } + removeBody := `{"binding_version":1,"runtime_id":"` + c1 + `","network_id":"` + n1 + `"}` + if removed := call(g2, http.MethodDelete, "/v1/browsers/account-a", removeBody); removed.Code != http.StatusNoContent { + t.Fatalf("G2 remove failed: %d %s", removed.Code, removed.Body.String()) + } + if replacement := call(g2, http.MethodPost, "/v1/browsers", testCreateBody); replacement.Code != http.StatusCreated { + t.Fatalf("G2 replacement create failed: %d %s", replacement.Code, replacement.Body.String()) + } + state.Lock() + c2 := state.containerID + _, g1Attached := state.networkMembers["gateway-g1"] + _, g2Attached := state.networkMembers["gateway-g2"] + _, c2Attached := state.networkMembers[c2] + state.Unlock() + if c2 == c1 || g1Attached || !g2Attached || !c2Attached || proxiesG1.proxies["account-a"].runtimeID != c1 || + proxiesG2.proxies["account-a"].runtimeID != c2 { + t.Fatalf("cross-process proxy release contract failed: c1=%q c2=%q g1=%v g2=%v c2Attached=%v", c1, c2, g1Attached, g2Attached, c2Attached) + } + proxiesG1.remove("account-a", 1, c1) + proxiesG2.remove("account-a", 1, c2) +} + +func TestGatewayRemoveFencesNetworkReplacementAndMemberChanges(t *testing.T) { + for _, test := range []struct { + name string + replaceOnDisconnect bool + addMemberOnDisconnect bool + wantNetworkID string + }{ + {name: "N1 replaced by N2 after inspect", replaceOnDisconnect: true, wantNetworkID: "network-n2"}, + {name: "trusted member joins before delete", addMemberOnDisconnect: true, wantNetworkID: "network-n1"}, + } { + t.Run(test.name, func(t *testing.T) { + var mu sync.Mutex + networkID := "network-n1" + members := map[string]string{"container-c1": namePrefix + "account-a", "gateway-g1": "gateway-g1"} + containerDeleted, networkDeletes := false, 0 + server := httptest.NewServer(withAliasReservations("gateway-g1", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-n1"}}}`)) + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-g2/json": + _, _ = response.Write([]byte(`{"Id":"gateway-g2","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) + case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || + request.URL.Path == "/networks/network-n1" || request.URL.Path == "/networks/network-n2"): + mu.Lock() + id := networkID + current := map[string]any{} + for memberID, name := range members { + current[memberID] = map[string]string{"Name": name} + } + mu.Unlock() + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": id, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": current, + }) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + mu.Lock() + if test.replaceOnDisconnect { + networkID = "network-n2" + mu.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + delete(members, body.Container) + if test.addMemberOnDisconnect && body.Container == "gateway-g1" { + members["gateway-g2"] = "gateway-g2" + } + mu.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): + mu.Lock() + networkDeletes++ + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/container-c1": + mu.Lock() + containerDeleted = true + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + registry := newMemoryProxyRegistry() + proxyURL, _, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{Protocol: "socks5", Host: "proxy.example", Port: 1080}) + if err != nil || !registry.bind("account-a", 1, proxyURL, "container-c1") { + t.Fatal("seed proxy generation") + } + defer registry.remove("account-a", 1, "container-c1") + api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-g1", token: testToken, proxies: registry, + locks: &dockerAliasReservations{docker: docker, self: "gateway-g1"}} + app := fiber.New() + app.Use("/v1", api.authorize) + app.Delete("/v1/browsers/:id", api.remove) + response := httptest.NewRecorder() + adaptor.FiberApp(app).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", + strings.NewReader(`{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1"}`))) + mu.Lock() + defer mu.Unlock() + if response.Code != http.StatusConflict || networkID != test.wantNetworkID || containerDeleted || networkDeletes != 0 || + registry.proxies["account-a"] == nil { + t.Fatalf("remove crossed network fence: status=%d network=%q containerDeleted=%v networkDeletes=%d proxy=%v body=%s", + response.Code, networkID, containerDeleted, networkDeletes, registry.proxies["account-a"], response.Body.String()) + } + }) + } +} + +func TestGatewayCreateFailureRemovesCreatedNetworkGeneration(t *testing.T) { + for _, failure := range []string{"network-create-id", "configure", "container-create", "start"} { + t.Run(failure, func(t *testing.T) { + var mu sync.Mutex + networkExists, containerExists := false, false + members := map[string]string{} + networkDeletes := 0 + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + mu.Lock() + exists := containerExists + mu.Unlock() + if !exists { + response.WriteHeader(http.StatusNotFound) + return + } + _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-n1"}}}`)) + case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1"): + mu.Lock() + exists := networkExists + current := map[string]any{} + for id, name := range members { + ip := "127.0.0.1/8" + if failure == "configure" && id == "gateway-self" { + ip = "192.0.2.1/24" + } + current[id] = map[string]string{"Name": name, "IPv4Address": ip} + } + mu.Unlock() + if !exists { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": current, + }) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + mu.Lock() + networkExists = true + mu.Unlock() + response.WriteHeader(http.StatusCreated) + if failure == "network-create-id" { + _, _ = response.Write([]byte(`{"Id":""}`)) + } else { + _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) + } + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect") && + !strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + mu.Lock() + members[body.Container] = body.Container + mu.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && request.URL.Path == "/containers/create": + if failure == "container-create" { + response.WriteHeader(http.StatusInternalServerError) + return + } + mu.Lock() + containerExists = true + members["container-c1"] = namePrefix + "account-a" + mu.Unlock() + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"container-c1"}`)) + case request.Method == http.MethodPost && request.URL.Path == "/containers/container-c1/start": + if failure == "start" { + response.WriteHeader(http.StatusInternalServerError) + return + } + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodDelete && request.URL.Path == "/containers/container-c1": + mu.Lock() + containerExists = false + delete(members, "container-c1") + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + mu.Lock() + delete(members, body.Container) + mu.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && request.URL.Path == "/networks/network-n1": + mu.Lock() + networkExists = false + networkDeletes++ + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + handler := newGatewayWithSelf(docker, "creatorhub_browser", testToken, "gateway-self") + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) + mu.Lock() + defer mu.Unlock() + wantNetwork, wantDeletes := false, 1 + if failure == "network-create-id" { + wantNetwork, wantDeletes = true, 0 + } + if response.Code != http.StatusBadGateway || networkExists != wantNetwork || networkDeletes != wantDeletes || containerExists { + t.Fatalf("%s failure left managed resources: status=%d network=%v deletes=%d container=%v body=%s", + failure, response.Code, networkExists, networkDeletes, containerExists, response.Body.String()) + } + }) + } +} + +func TestGatewayNetworkCreateDisconnectDoesNotDiscoverReplacementByName(t *testing.T) { + networkReads, networkDeletes := 0, 0 + networkCreated := false + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + response.WriteHeader(http.StatusNotFound) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + networkReads++ + if networkCreated { + _, _ = response.Write([]byte(`{"Id":"network-n2","Name":"creatorhub_browser-account-a","Driver":"bridge","Labels":{"` + managedLabel + `":"true","` + networkRoleLabel + `":"` + browserNetworkRole + `","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1"},"Containers":{}}`)) + return + } + response.WriteHeader(http.StatusNotFound) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + networkCreated = true + connection, _, _ := response.(http.Hijacker).Hijack() + _ = connection.Close() + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): + networkDeletes++ + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, + "creatorhub_browser", testToken, "gateway-self") + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) + if response.Code != http.StatusBadGateway || !networkCreated || networkReads != 1 || networkDeletes != 0 { + t.Fatalf("unknown N1 was discovered or cleaned as N2: status=%d created=%v reads=%d deletes=%d body=%s", + response.Code, networkCreated, networkReads, networkDeletes, response.Body.String()) + } +} + +func TestGatewayCompensatesConnectThatAppliedBeforeError(t *testing.T) { + for _, restore := range []bool{false, true} { + name := "create self" + failedMember := "gateway-self" + if restore { + name, failedMember = "restore runtime", "container-c1" + } + t.Run(name, func(t *testing.T) { + networkExists := false + members := map[string]string{} + disconnected := []string{} + networkDeletes := 0 + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + if !restore { + response.WriteHeader(http.StatusNotFound) + return + } + _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkExitLabel + `":"exit-1","` + networkIDLabel + `":"network-n1","` + proxyPortLabel + `":"12345"}}}`)) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + if !networkExists { + response.WriteHeader(http.StatusNotFound) + return + } + containers := map[string]any{} + for id, memberName := range members { + containers[id] = map[string]string{"Name": memberName, "IPv4Address": "127.0.0.1/8"} + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": containers, + }) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + networkExists = true + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect") && + !strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + members[body.Container] = body.Container + if body.Container == failedMember { + response.WriteHeader(http.StatusInternalServerError) + return + } + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + delete(members, body.Container) + disconnected = append(disconnected, body.Container) + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && request.URL.Path == "/networks/network-n1": + networkExists = false + networkDeletes++ + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, + "creatorhub_browser", testToken, "gateway-self") + method, path, body := http.MethodPost, "/v1/browsers", testCreateBody + if restore { + path = "/v1/browsers/account-a/proxy" + body = `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` + } + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(method, path, strings.NewReader(body))) + wantDeletes, wantDisconnects := 1, 1 + if restore { + wantDeletes, wantDisconnects = 0, 0 + } + if response.Code != http.StatusBadGateway || networkExists || networkDeletes != wantDeletes || + len(disconnected) != wantDisconnects || (wantDisconnects == 1 && disconnected[0] != failedMember) { + t.Fatalf("applied connect was not compensated: status=%d network=%v deletes=%d disconnected=%v body=%s", + response.Code, networkExists, networkDeletes, disconnected, response.Body.String()) + } + }) + } +} + +func TestGatewayRestoreFailureRemovesCreatedNetworkGeneration(t *testing.T) { + var mu sync.Mutex + networkExists := false + members := map[string]string{} + networkDeletes, containerDeletes := 0, 0 + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": + _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkExitLabel + `":"exit-1","` + networkIDLabel + `":"network-n1","` + proxyPortLabel + `":"12345"}}}`)) + case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1"): + mu.Lock() + exists := networkExists + current := map[string]any{} + for id, name := range members { + ip := "127.0.0.3/8" + if id == "gateway-self" { + ip = "192.0.2.1/24" + } + current[id] = map[string]string{"Name": name, "IPv4Address": ip} + } + mu.Unlock() + if !exists { + response.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, + "Containers": current, + }) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + mu.Lock() + networkExists = true + mu.Unlock() + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + mu.Lock() + members[body.Container] = body.Container + mu.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + var body struct { + Container string `json:"Container"` + } + _ = json.NewDecoder(request.Body).Decode(&body) + mu.Lock() + delete(members, body.Container) + mu.Unlock() + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && request.URL.Path == "/networks/network-n1": + mu.Lock() + networkExists = false + networkDeletes++ + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): + mu.Lock() + containerDeletes++ + mu.Unlock() + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + handler := newGatewayWithSelf(docker, "creatorhub_browser", testToken, "gateway-self") + response := httptest.NewRecorder() + body := `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(body))) + mu.Lock() + defer mu.Unlock() + if response.Code != http.StatusBadGateway || networkExists || networkDeletes != 0 || containerDeletes != 0 { + t.Fatalf("restore failure left managed network: status=%d network=%v networkDeletes=%d containerDeletes=%d body=%s", + response.Code, networkExists, networkDeletes, containerDeletes, response.Body.String()) + } +} + func TestGatewayMapsDockerServiceFailureToBadGateway(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, _ *http.Request) { http.Error(response, "daemon unavailable", http.StatusInternalServerError) @@ -559,7 +2143,7 @@ func TestGatewayMapsDockerServiceFailureToBadGateway(t *testing.T) { handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() - adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil)) + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(testGenerationBody))) if response.Code != http.StatusBadGateway { t.Fatalf("expected 502 for Docker failure, got %d: %s", response.Code, response.Body.String()) } @@ -579,7 +2163,7 @@ func TestGatewayRefusesUnmanagedContainer(t *testing.T) { defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) - request := authed(http.MethodDelete, "/v1/browsers/foreign", nil) + request := authed(http.MethodDelete, "/v1/browsers/foreign", strings.NewReader(testGenerationBody)) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, request) @@ -588,6 +2172,59 @@ func TestGatewayRefusesUnmanagedContainer(t *testing.T) { } } +func TestGatewayRejectsStaleGenerationBeforeDockerMutation(t *testing.T) { + for _, request := range []struct { + method, path string + }{ + {http.MethodPost, "/v1/browsers/account-a/stop"}, + {http.MethodDelete, "/v1/browsers/account-a"}, + } { + t.Run(request.method, func(t *testing.T) { + mutations := 0 + docker, server := testDocker(func(response http.ResponseWriter, dockerRequest *http.Request) { + if dockerRequest.Method != http.MethodGet { + mutations++ + } + _, _ = response.Write([]byte(`{"Id":"new-container","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"2"}}}`)) + }) + defer server.Close() + handler := newGateway(docker, "creatorhub_browser", testToken) + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(request.method, request.path, strings.NewReader(testGenerationBody))) + if response.Code != http.StatusConflict || mutations != 0 { + t.Fatalf("stale generation reached Docker mutation: status=%d mutations=%d body=%s", response.Code, mutations, response.Body.String()) + } + }) + } +} + +func TestGatewayRejectsStaleDeleteDuringNewNetworkCreation(t *testing.T) { + mutations := 0 + server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"): + response.WriteHeader(http.StatusNotFound) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-new", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", + bindingVersionLabel: "2"}, + }) + default: + mutations++ + response.WriteHeader(http.StatusNoContent) + } + })) + defer server.Close() + handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, + "creatorhub_browser", testToken, "gateway-self") + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(testGenerationBody))) + if response.Code != http.StatusConflict || mutations != 0 { + t.Fatalf("stale delete crossed the new network generation: status=%d mutations=%d body=%s", response.Code, mutations, response.Body.String()) + } +} + func TestEnsureTenantNetworkConnectsGatewayOnlyToRuntimeNetwork(t *testing.T) { created, connected := false, false server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { @@ -603,14 +2240,22 @@ func TestEnsureTenantNetworkConnectsGatewayOnlyToRuntimeNetwork(t *testing.T) { } created = true response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"network-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): connected = true response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-id/json": + _, _ = response.Write([]byte(`{"Id":"gateway-id","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodGet: + members := map[string]any{} + if connected { + members["gateway-id"] = map[string]string{"Name": "gateway-id", "IPv4Address": "127.0.0.3/8"} + } _ = json.NewEncoder(response).Encode(map[string]any{ - "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, - "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a"}, - "Containers": map[string]any{"gateway-id": map[string]string{"Name": "gateway-id", "IPv4Address": "127.0.0.3/8"}}, + "Id": "network-id", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", + bindingVersionLabel: "1"}, + "Containers": members, }) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path) @@ -618,9 +2263,9 @@ func TestEnsureTenantNetworkConnectsGatewayOnlyToRuntimeNetwork(t *testing.T) { })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} - name, bindHost, err := docker.ensureTenantNetwork("creatorhub_browser", "account-a", "gateway-id") - if err != nil || !created || !connected || name != "creatorhub_browser-account-a" || bindHost != "127.0.0.3" { - t.Fatalf("isolated network was not created and connected: name=%q host=%q created=%v connected=%v err=%v", name, bindHost, created, connected, err) + generation, bindHost, err := docker.ensureTenantNetwork("creatorhub_browser", "account-a", "gateway-id", 1, "", "", false) + if err != nil || !created || !connected || generation.Name != "creatorhub_browser-account-a" || bindHost != "127.0.0.3" { + t.Fatalf("isolated network was not created and connected: generation=%#v host=%q created=%v connected=%v err=%v", generation, bindHost, created, connected, err) } } diff --git a/cmd/docker-gateway/proxy.go b/cmd/docker-gateway/proxy.go index 9d1e3fd..a58dde5 100644 --- a/cmd/docker-gateway/proxy.go +++ b/cmd/docker-gateway/proxy.go @@ -25,65 +25,106 @@ type memoryProxyRegistry struct { } type memoryProxy struct { - mu sync.RWMutex - exit gatewayProxyExit - bindHost string - listener net.Listener - server *http.Server - url string + mu sync.RWMutex + exit gatewayProxyExit + bindingVersion int64 + runtimeID string + networkID string + bindHost string + listener net.Listener + server *http.Server + url string } func newMemoryProxyRegistry() *memoryProxyRegistry { return &memoryProxyRegistry{proxies: map[string]*memoryProxy{}} } -func (registry *memoryProxyRegistry) configure(alias, bindHost string, port int, exit gatewayProxyExit) (string, func(), error) { +func (registry *memoryProxyRegistry) configure(alias string, bindingVersion int64, bindHost string, port int, exit gatewayProxyExit, + networkIDs ...string) (string, func(), error) { registry.mu.Lock() defer registry.mu.Unlock() + networkID := "" + if len(networkIDs) == 1 { + networkID = networkIDs[0] + } if proxy := registry.proxies[alias]; proxy != nil { - if proxy.bindHost == bindHost && (port == 0 || proxy.listener.Addr().(*net.TCPAddr).Port == port) && proxy.exit == exit { - return proxy.url, func() {}, nil + if proxy.bindingVersion == bindingVersion && proxy.bindHost == bindHost && + (port == 0 || proxy.listener.Addr().(*net.TCPAddr).Port == port) && proxy.exit == exit && proxy.networkID == networkID { + return proxy.url, func() { registry.removeObject(alias, proxy) }, nil } - delete(registry.proxies, alias) - _ = proxy.server.Close() + return "", nil, errGenerationConflict } listener, err := net.Listen("tcp4", net.JoinHostPort(bindHost, strconv.Itoa(port))) if err != nil { return "", nil, err } actualPort := listener.Addr().(*net.TCPAddr).Port - proxy := &memoryProxy{exit: exit, bindHost: bindHost, listener: listener, url: "http://" + net.JoinHostPort(browserProxyHost, strconv.Itoa(actualPort))} + proxy := &memoryProxy{exit: exit, bindingVersion: bindingVersion, networkID: networkID, bindHost: bindHost, listener: listener, + url: "http://" + net.JoinHostPort(browserProxyHost, strconv.Itoa(actualPort))} proxy.server = &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second} registry.proxies[alias] = proxy go func() { _ = proxy.server.Serve(listener) }() undo := func() { - registry.mu.Lock() - defer registry.mu.Unlock() - if registry.proxies[alias] == proxy { - delete(registry.proxies, alias) - _ = proxy.server.Close() - } + registry.removeObject(alias, proxy) } return proxy.url, undo, nil } -func (registry *memoryProxyRegistry) ready(alias string, port int) bool { +func (registry *memoryProxyRegistry) removeObject(alias string, proxy *memoryProxy) { registry.mu.Lock() - defer registry.mu.Unlock() - proxy := registry.proxies[alias] - return proxy != nil && proxy.listener.Addr().(*net.TCPAddr).Port == port -} - -func (registry *memoryProxyRegistry) remove(alias string) { - registry.mu.Lock() - proxy := registry.proxies[alias] - delete(registry.proxies, alias) + if registry.proxies[alias] == proxy { + delete(registry.proxies, alias) + } else { + proxy = nil + } registry.mu.Unlock() if proxy != nil { _ = proxy.server.Close() } } +func (registry *memoryProxyRegistry) ready(alias string, port int, runtimeID string, networkIDs ...string) bool { + registry.mu.Lock() + defer registry.mu.Unlock() + proxy := registry.proxies[alias] + return proxy != nil && runtimeID != "" && proxy.runtimeID == runtimeID && proxy.listener.Addr().(*net.TCPAddr).Port == port && + (len(networkIDs) == 0 || proxy.networkID == networkIDs[0]) +} + +func (registry *memoryProxyRegistry) bind(alias string, bindingVersion int64, proxyURL, runtimeID string, networkIDs ...string) bool { + registry.mu.Lock() + defer registry.mu.Unlock() + proxy := registry.proxies[alias] + if proxy == nil || runtimeID == "" || proxy.bindingVersion != bindingVersion || proxy.url != proxyURL { + return false + } + proxy.runtimeID = runtimeID + if len(networkIDs) == 1 { + proxy.networkID = networkIDs[0] + } + return true +} + +func (registry *memoryProxyRegistry) remove(alias string, bindingVersion int64, runtimeID string, networkIDs ...string) bool { + registry.mu.Lock() + proxy := registry.proxies[alias] + if proxy != nil && runtimeID != "" && proxy.bindingVersion == bindingVersion && proxy.runtimeID == runtimeID && + (len(networkIDs) == 0 || proxy.networkID == networkIDs[0]) { + delete(registry.proxies, alias) + } else if proxy != nil { + registry.mu.Unlock() + return false + } else { + proxy = nil + } + registry.mu.Unlock() + if proxy != nil { + _ = proxy.server.Close() + } + return true +} + func (proxy *memoryProxy) ServeHTTP(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodConnect { proxy.tunnel(response, request) diff --git a/cmd/docker-gateway/proxy_test.go b/cmd/docker-gateway/proxy_test.go index e53e3d4..56264a4 100644 --- a/cmd/docker-gateway/proxy_test.go +++ b/cmd/docker-gateway/proxy_test.go @@ -73,7 +73,7 @@ func TestMemoryProxyUsesSOCKS5Credentials(t *testing.T) { host, portText, _ := net.SplitHostPort(listener.Addr().String()) port, _ := net.LookupPort("tcp", portText) registry := newMemoryProxyRegistry() - proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{ + proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{ Protocol: "socks5", Host: host, Port: port, Username: "operator", Password: "ephemeral", }) if err != nil { @@ -119,7 +119,7 @@ func TestMemoryProxyUsesAbsoluteFormForHTTPUpstream(t *testing.T) { address, _ := url.Parse(upstream.URL) port, _ := strconv.Atoi(address.Port()) registry := newMemoryProxyRegistry() - proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{ + proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{ Protocol: "http", Host: address.Hostname(), Port: port, Username: "operator", Password: "ephemeral", }) if err != nil { @@ -141,7 +141,7 @@ func TestMemoryProxyUsesAbsoluteFormForHTTPUpstream(t *testing.T) { func TestMemoryProxyRejectsCrossAliasAddress(t *testing.T) { registry := newMemoryProxyRegistry() - proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1}) + proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1}) if err != nil { t.Fatal(err) } @@ -152,3 +152,19 @@ func TestMemoryProxyRejectsCrossAliasAddress(t *testing.T) { t.Fatal("another tenant address could reach account-a proxy") } } + +func TestMemoryProxyRemoveRequiresMatchingGeneration(t *testing.T) { + registry := newMemoryProxyRegistry() + proxyURL, cleanup, err := registry.configure("account-a", 2, "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1}) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if !registry.bind("account-a", 2, proxyURL, "container-c2") { + t.Fatal("bind proxy generation") + } + proxy := registry.proxies["account-a"] + if registry.remove("account-a", 2, "") || registry.remove("account-a", 2, "container-c1") || registry.proxies["account-a"] != proxy { + t.Fatal("stale generation removed the current proxy") + } +} diff --git a/compose.yaml b/compose.yaml index f65fd99..5a56ec5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -44,6 +44,8 @@ services: docker-gateway: build: . command: ["/app/docker-gateway"] + labels: + io.creatorhub.gateway-member: "true" environment: BROWSER_NETWORK: creatorhub_browser GATEWAY_TOKEN: ${GATEWAY_TOKEN:-dev-creatorhub-gateway-token} diff --git a/internal/hub/environment.go b/internal/hub/environment.go index ad3216a..ccf2e76 100644 --- a/internal/hub/environment.go +++ b/internal/hub/environment.go @@ -51,13 +51,20 @@ type ExitObservation struct { type EnvironmentContext struct { Env - AccountID string `json:"account_id"` - BindingID string `json:"binding_id"` - BindingVersion int64 `json:"binding_version"` - RuntimeCleanupPending bool `json:"runtime_cleanup_pending,omitempty"` - Exit NetworkExit `json:"network_exit"` - RuntimeInstanceID string `json:"runtime_instance_id,omitempty"` - RuntimeID string `json:"runtime_id,omitempty"` + AccountID string `json:"account_id"` + AccountStatus string `json:"account_status"` + AuthorizationStatus string `json:"authorization_status"` + BindingID string `json:"binding_id"` + BindingVersion int64 `json:"binding_version"` + RuntimeCleanupPending bool `json:"runtime_cleanup_pending,omitempty"` + RuntimeCleanupBindingVersion int64 `json:"runtime_cleanup_binding_version,omitempty"` + RuntimeCleanupInstanceID string `json:"runtime_cleanup_instance_id,omitempty"` + RuntimeCleanupRuntimeID string `json:"runtime_cleanup_runtime_id,omitempty"` + RuntimeCleanupNetworkID string `json:"runtime_cleanup_network_id,omitempty"` + Exit NetworkExit `json:"network_exit"` + RuntimeInstanceID string `json:"runtime_instance_id,omitempty"` + RuntimeID string `json:"runtime_id,omitempty"` + RuntimeNetworkID string `json:"runtime_network_id,omitempty"` } type EnvironmentAction struct { @@ -396,40 +403,39 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro return EnvironmentContext{}, errors.New("begin environment context read") } defer tx.Rollback() - if _, err := tx.ExecContext(ctx, ` - UPDATE runtime_instance runtime SET released_at = now() - FROM environment_binding binding - WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id - AND runtime.released_at IS NULL AND runtime.lease_until <= now()`, alias); err != nil { - return EnvironmentContext{}, errors.New("expire environment runtime") - } var result EnvironmentContext var encoded []byte var expectedIP, observedIP string var checked sql.NullTime - var runtimeInstanceID, runtimeID sql.NullString + var runtimeInstanceID, runtimeID, runtimeNetworkID, cleanupInstanceID, cleanupRuntimeID, cleanupNetworkID sql.NullString + var cleanupBindingVersion sql.NullInt64 err = tx.QueryRowContext(ctx, ` SELECT environment.alias, environment.name, environment.gateway_name, environment.image_version, - environment.fingerprint, environment.created_at, binding.account_id, binding.id, binding.version, - binding.runtime_cleanup_pending, + environment.fingerprint, environment.created_at, binding.account_id, account.status, account.authorization_status, + binding.id, binding.version, + binding.runtime_cleanup_pending, binding.runtime_cleanup_binding_version, + binding.runtime_cleanup_instance_id, binding.runtime_cleanup_runtime_id, binding.runtime_cleanup_network_id, COALESCE(network.id, ''), COALESCE(network.protocol, ''), COALESCE(network.host, ''), COALESCE(network.port, 0), COALESCE(host(network.expected_public_ip), ''), COALESCE(network.expected_region, ''), COALESCE(host(network.observed_public_ip), ''), COALESCE(network.observed_region, ''), COALESCE(network.health_status, 'unchecked'), COALESCE(network.last_check_reason, ''), COALESCE(network.version, 0), network.last_checked_at, COALESCE(network.created_at, to_timestamp(0)), COALESCE(network.updated_at, to_timestamp(0)), - runtime.id, runtime.runtime_id + runtime.id, runtime.runtime_id, runtime.network_id FROM browser_env environment JOIN environment_binding binding ON binding.browser_env_alias = environment.alias + JOIN social_account account ON account.id = binding.account_id LEFT JOIN network_exit network ON network.id = binding.network_exit_id LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL WHERE environment.alias = $1`, alias). Scan(&result.Alias, &result.Name, &result.Gateway, &result.ImageVersion, &encoded, &result.CreatedAt, - &result.AccountID, &result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending, + &result.AccountID, &result.AccountStatus, &result.AuthorizationStatus, + &result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending, &cleanupBindingVersion, + &cleanupInstanceID, &cleanupRuntimeID, &cleanupNetworkID, &result.Exit.ID, &result.Exit.Protocol, &result.Exit.Host, &result.Exit.Port, &expectedIP, &result.Exit.ExpectedRegion, &observedIP, &result.Exit.ObservedRegion, &result.Exit.HealthStatus, &result.Exit.LastCheckReason, &result.Exit.Version, &checked, &result.Exit.CreatedAt, &result.Exit.UpdatedAt, - &runtimeInstanceID, &runtimeID) + &runtimeInstanceID, &runtimeID, &runtimeNetworkID) if err != nil { return EnvironmentContext{}, rowError(err) } @@ -442,13 +448,30 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro if checked.Valid { result.Exit.LastCheckedAt = &checked.Time } - result.RuntimeInstanceID, result.RuntimeID = runtimeInstanceID.String, runtimeID.String + result.RuntimeInstanceID, result.RuntimeID, result.RuntimeNetworkID = runtimeInstanceID.String, runtimeID.String, runtimeNetworkID.String + if result.RuntimeCleanupPending { + result.RuntimeCleanupBindingVersion = cleanupBindingVersion.Int64 + result.RuntimeCleanupInstanceID, result.RuntimeCleanupRuntimeID = cleanupInstanceID.String, cleanupRuntimeID.String + result.RuntimeCleanupNetworkID = cleanupNetworkID.String + } if err := commitHub(tx); err != nil { return EnvironmentContext{}, err } return result, nil } +func (s *Store) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (EnvironmentContext, error) { + if !aliasPattern.MatchString(accountID) { + return EnvironmentContext{}, ErrInvalid + } + var alias string + if err := s.db.QueryRowContext(ctx, ` + SELECT browser_env_alias FROM environment_binding WHERE account_id = $1`, accountID).Scan(&alias); err != nil { + return EnvironmentContext{}, rowError(err) + } + return s.GetEnvironmentContext(ctx, alias) +} + func validateEnvironmentRebind(ctx context.Context, tx *sql.Tx, alias, exitID string, expectedBindingVersion int64) (string, string, error) { var accountID, bindingID string var bindingVersion int64 @@ -506,9 +529,14 @@ func (s *Store) ValidateEnvironmentRebind(ctx context.Context, alias, exitID str return nil } -func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (EnvironmentContext, error) { +func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkIDs ...string) (EnvironmentContext, error) { + networkID := "" + if len(networkIDs) == 1 { + networkID = networkIDs[0] + } if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(exitID) || - (runtimeID != "" && !exitIDPattern.MatchString(runtimeID)) || expectedBindingVersion < 1 { + (runtimeID != "" && !exitIDPattern.MatchString(runtimeID)) || (networkID != "" && !exitIDPattern.MatchString(networkID)) || + len(networkIDs) > 1 || expectedBindingVersion < 1 { return EnvironmentContext{}, ErrInvalid } tx, err := s.db.BeginTx(ctx, nil) @@ -528,8 +556,9 @@ func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID } if runtimeID != "" { if _, err := tx.ExecContext(ctx, ` - INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) - VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil { + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, lease_until) + VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), now() + interval '1 minute')`, + "runtime-"+newHubID(), accountID, bindingID, expectedBindingVersion+1, runtimeID, networkID); err != nil { return EnvironmentContext{}, publicDatabaseError(err) } } @@ -539,8 +568,13 @@ func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID return s.GetEnvironmentContext(ctx, alias) } -func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (EnvironmentContext, error) { - if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || bindingVersion < 1 || !exitIDPattern.MatchString(exitID) { +func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkIDs ...string) (EnvironmentContext, error) { + networkID := "" + if len(networkIDs) == 1 { + networkID = networkIDs[0] + } + if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || bindingVersion < 1 || !exitIDPattern.MatchString(exitID) || + !exitIDPattern.MatchString(networkID) || len(networkIDs) != 1 { return EnvironmentContext{}, ErrInvalid } tx, err := s.db.BeginTx(ctx, nil) @@ -548,18 +582,21 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi return EnvironmentContext{}, errors.New("begin runtime activation") } defer tx.Rollback() - var accountID, bindingID, currentExitID string + var accountID, bindingID, currentExitID, accountStatus, authorizationStatus string var currentBindingVersion int64 var cleanupPending bool err = tx.QueryRowContext(ctx, ` - SELECT binding.account_id, binding.id, binding.version, COALESCE(binding.network_exit_id, ''), binding.runtime_cleanup_pending + SELECT binding.account_id, binding.id, binding.version, COALESCE(binding.network_exit_id, ''), binding.runtime_cleanup_pending, + account.status, account.authorization_status FROM environment_binding binding - WHERE binding.browser_env_alias = $1 FOR UPDATE OF binding`, alias). - Scan(&accountID, &bindingID, ¤tBindingVersion, ¤tExitID, &cleanupPending) + JOIN social_account account ON account.id = binding.account_id + WHERE binding.browser_env_alias = $1 FOR UPDATE OF binding, account`, alias). + Scan(&accountID, &bindingID, ¤tBindingVersion, ¤tExitID, &cleanupPending, &accountStatus, &authorizationStatus) if err != nil { return EnvironmentContext{}, rowError(err) } - if cleanupPending || currentBindingVersion != bindingVersion || currentExitID != exitID { + if cleanupPending || accountStatus != "active" || authorizationStatus != "authorized" || + currentBindingVersion != bindingVersion || currentExitID != exitID { return EnvironmentContext{}, ErrConflict } if _, err := tx.ExecContext(ctx, ` @@ -567,19 +604,21 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi WHERE binding_id = $1 AND released_at IS NULL AND lease_until <= now()`, bindingID); err != nil { return EnvironmentContext{}, errors.New("expire runtime before activation") } - var existingInstanceID, existingRuntimeID string + var existingInstanceID, existingRuntimeID, existingNetworkID string + var existingBindingVersion int64 if err := tx.QueryRowContext(ctx, ` - SELECT COALESCE(id, ''), COALESCE(runtime_id, '') FROM runtime_instance - WHERE binding_id = $1 AND released_at IS NULL`, bindingID).Scan(&existingInstanceID, &existingRuntimeID); err != nil && !errors.Is(err, sql.ErrNoRows) { + SELECT COALESCE(id, ''), COALESCE(runtime_id, ''), COALESCE(network_id, ''), COALESCE(binding_version, 0) FROM runtime_instance + WHERE binding_id = $1 AND released_at IS NULL`, bindingID).Scan(&existingInstanceID, &existingRuntimeID, &existingNetworkID, &existingBindingVersion); err != nil && !errors.Is(err, sql.ErrNoRows) { return EnvironmentContext{}, publicDatabaseError(err) } - if existingInstanceID != "" && existingRuntimeID != runtimeID { + if existingInstanceID != "" && (existingRuntimeID != runtimeID || existingNetworkID != networkID || existingBindingVersion != bindingVersion) { return EnvironmentContext{}, ErrConflict } if existingInstanceID == "" { if _, err := tx.ExecContext(ctx, ` - INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) - VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil { + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, lease_until) + VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), now() + interval '1 minute')`, + "runtime-"+newHubID(), accountID, bindingID, bindingVersion, runtimeID, networkID); err != nil { return EnvironmentContext{}, publicDatabaseError(err) } } else if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 minute' WHERE id = $1`, existingInstanceID); err != nil { @@ -591,22 +630,37 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi return s.GetEnvironmentContext(ctx, alias) } -func (s *Store) ReleaseRuntime(ctx context.Context, alias string) error { - if !aliasPattern.MatchString(alias) { +func (s *Store) ReleaseRuntime(ctx context.Context, environment EnvironmentContext) error { + if !aliasPattern.MatchString(environment.Alias) || !exitIDPattern.MatchString(environment.BindingID) || + environment.BindingVersion < 1 || (environment.RuntimeInstanceID != "" && !exitIDPattern.MatchString(environment.RuntimeInstanceID)) { return ErrInvalid } - _, err := s.db.ExecContext(ctx, ` + if environment.RuntimeInstanceID == "" { + return nil + } + result, err := s.db.ExecContext(ctx, ` UPDATE runtime_instance runtime SET released_at = now() FROM environment_binding binding - WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id AND runtime.released_at IS NULL`, alias) + WHERE binding.browser_env_alias = $1 AND binding.id = $2 AND binding.version = $3 + AND runtime.binding_id = binding.id AND runtime.binding_version = binding.version + AND runtime.id = $4 AND runtime.released_at IS NULL`, environment.Alias, environment.BindingID, + environment.BindingVersion, environment.RuntimeInstanceID) if err != nil { return errors.New("release environment runtime") } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + return ErrConflict + } return nil } -func (s *Store) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error { - if !aliasPattern.MatchString(alias) { +func (s *Store) SetRuntimeCleanupPending(ctx context.Context, environment EnvironmentContext, pending bool) error { + if !aliasPattern.MatchString(environment.Alias) || !exitIDPattern.MatchString(environment.BindingID) || + environment.BindingVersion < 1 || environment.RuntimeCleanupBindingVersion < 1 || + (pending && environment.RuntimeCleanupRuntimeID == "") || + (environment.RuntimeCleanupInstanceID != "" && !exitIDPattern.MatchString(environment.RuntimeCleanupInstanceID)) || + (environment.RuntimeCleanupRuntimeID != "" && !exitIDPattern.MatchString(environment.RuntimeCleanupRuntimeID)) || + (environment.RuntimeCleanupNetworkID != "" && !exitIDPattern.MatchString(environment.RuntimeCleanupNetworkID)) { return ErrInvalid } tx, err := s.db.BeginTx(ctx, nil) @@ -614,18 +668,75 @@ func (s *Store) SetRuntimeCleanupPending(ctx context.Context, alias string, pend return errors.New("begin runtime cleanup state update") } defer tx.Rollback() - var bindingID string + var currentPending bool + var cleanupBindingVersion sql.NullInt64 + var cleanupInstanceID, cleanupRuntimeID, cleanupNetworkID sql.NullString if err := tx.QueryRowContext(ctx, ` - UPDATE environment_binding SET runtime_cleanup_pending = $2, updated_at = now() - WHERE browser_env_alias = $1 RETURNING id`, alias, pending).Scan(&bindingID); err != nil { - return rowError(err) + SELECT runtime_cleanup_pending, runtime_cleanup_binding_version, + runtime_cleanup_instance_id, runtime_cleanup_runtime_id, runtime_cleanup_network_id + FROM environment_binding + WHERE browser_env_alias = $1 AND id = $2 AND version = $3 FOR UPDATE`, environment.Alias, + environment.BindingID, environment.BindingVersion). + Scan(¤tPending, &cleanupBindingVersion, &cleanupInstanceID, &cleanupRuntimeID, &cleanupNetworkID); errors.Is(err, sql.ErrNoRows) { + return ErrConflict + } else if err != nil { + return publicDatabaseError(err) + } + if currentPending { + if cleanupBindingVersion.Int64 != environment.RuntimeCleanupBindingVersion || + cleanupInstanceID.String != environment.RuntimeCleanupInstanceID || cleanupRuntimeID.String != environment.RuntimeCleanupRuntimeID || + cleanupNetworkID.String != environment.RuntimeCleanupNetworkID { + return ErrConflict + } + if pending { + return commitHub(tx) + } + } else if !pending { + return commitHub(tx) + } + if pending && !currentPending { + var runtimeInstanceID string + err := tx.QueryRowContext(ctx, ` + SELECT id FROM runtime_instance + WHERE binding_id = $1 AND binding_version = $2 AND released_at IS NULL`, environment.BindingID, + environment.BindingVersion).Scan(&runtimeInstanceID) + if errors.Is(err, sql.ErrNoRows) { + runtimeInstanceID = "" + } else if err != nil { + return publicDatabaseError(err) + } + if runtimeInstanceID != environment.RuntimeCleanupInstanceID { + return ErrConflict + } + if runtimeInstanceID != "" { + result, err := tx.ExecContext(ctx, ` + UPDATE runtime_instance SET released_at = now() + WHERE id = $1 AND binding_id = $2 AND binding_version = $3 AND released_at IS NULL`, runtimeInstanceID, + environment.BindingID, environment.BindingVersion) + if err != nil { + return errors.New("release runtime for pending cleanup") + } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + return ErrConflict + } + } } if pending { - if _, err := tx.ExecContext(ctx, ` - UPDATE runtime_instance SET released_at = now() - WHERE binding_id = $1 AND released_at IS NULL`, bindingID); err != nil { - return errors.New("release runtime for pending cleanup") - } + _, err = tx.ExecContext(ctx, ` + UPDATE environment_binding SET runtime_cleanup_pending = true, + runtime_cleanup_binding_version = $2, runtime_cleanup_instance_id = NULLIF($3, ''), + runtime_cleanup_runtime_id = NULLIF($4, ''), runtime_cleanup_network_id = NULLIF($5, ''), updated_at = now() + WHERE id = $1`, environment.BindingID, environment.RuntimeCleanupBindingVersion, + environment.RuntimeCleanupInstanceID, environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID) + } else { + _, err = tx.ExecContext(ctx, ` + UPDATE environment_binding SET runtime_cleanup_pending = false, + runtime_cleanup_binding_version = NULL, runtime_cleanup_instance_id = NULL, + runtime_cleanup_runtime_id = NULL, runtime_cleanup_network_id = NULL, updated_at = now() + WHERE id = $1`, environment.BindingID) + } + if err != nil { + return errors.New("update runtime cleanup state") } return commitHub(tx) } diff --git a/internal/hub/migration_test.go b/internal/hub/migration_test.go index 4197556..d9cf3c9 100644 --- a/internal/hub/migration_test.go +++ b/internal/hub/migration_test.go @@ -29,13 +29,78 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal(err) } defer db.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)`, 10) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('social_account', 'browser_env', 'network_exit', 'environment_binding')`, 4) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name = 'runtime_cleanup_pending'`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name LIKE 'runtime_cleanup_%'`, 5) store = openFullyMigratedHub(t, ctx, testURL) store.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)`, 10) + }) + + t.Run("previous migration 008 already applied", func(t *testing.T) { + ctx := context.Background() + testURL := isolatedDatabaseURL(t, databaseURL) + store := openFullyMigratedHub(t, ctx, testURL) + store.Close() + db, err := sql.Open("pgx", testURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(` + INSERT INTO credential_reference (id, provider, reference_key) + VALUES ('credential-upgrade', 'os_keyring', 'creatorhub/upgrade'); + INSERT INTO social_account + (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status, status) + VALUES ('upgrade', 'credential-upgrade', 'mock', 'upgrade', 'owned', 'authorized', 'paused'); + INSERT INTO gateway (name, endpoint, token) + VALUES ('upgrade-gateway', 'http://127.0.0.1:8081', 'upgrade-gateway-token'); + INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1'); + INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + VALUES ('upgrade', 'Upgrade', 'upgrade-gateway', '1', '{"seed":1}'); + INSERT INTO environment_binding (id, account_id, browser_env_alias) + VALUES ('upgrade', 'upgrade', 'upgrade'); + UPDATE environment_binding SET runtime_cleanup_pending = true, + runtime_cleanup_binding_version = version WHERE id = 'upgrade'; + DELETE FROM schema_migration WHERE version = 9; + ALTER TABLE environment_binding ADD CONSTRAINT runtime_cleanup_generation_consistent CHECK ( + (runtime_cleanup_pending AND runtime_cleanup_binding_version IS NOT NULL AND runtime_cleanup_binding_version > 0) + OR + (NOT runtime_cleanup_pending AND runtime_cleanup_binding_version IS NULL + AND runtime_cleanup_instance_id IS NULL AND runtime_cleanup_runtime_id IS NULL) + )`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = false WHERE id = 'upgrade'`); err == nil { + t.Fatal("old migration 008 constraint unexpectedly allowed a legacy writer to clear pending") + } + + store, err = Open(ctx, testURL) + if err != nil { + t.Fatalf("apply forward compatibility migration: %v", err) + } + defer store.Close() + if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = false WHERE id = 'upgrade'`); err != nil { + t.Fatalf("migration 009 did not unblock the old writer: %v", err) + } + context, err := store.GetEnvironmentContext(ctx, "upgrade") + if err != nil || context.RuntimeCleanupPending || context.RuntimeCleanupBindingVersion != 0 || context.RuntimeCleanupRuntimeID != "" { + t.Fatalf("new runner reused stale cleanup generation after old writer clear: %#v err=%v", context, err) + } + if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = true WHERE id = 'upgrade'`); err != nil { + t.Fatalf("migration 009 did not allow the old writer to set pending: %v", err) + } + context, err = store.GetEnvironmentContext(ctx, "upgrade") + if err != nil || !context.RuntimeCleanupPending || context.RuntimeCleanupBindingVersion != 1 || context.RuntimeCleanupRuntimeID != "" { + t.Fatalf("new runner did not expose legacy pending for manual reconcile: %#v err=%v", context, err) + } + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (9, 10)`, 2) + assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND + ((table_name = 'runtime_instance' AND column_name = 'network_id') OR + (table_name = 'environment_binding' AND column_name = 'runtime_cleanup_network_id'))`, 2) + assertDatabaseCount(t, db, `SELECT count(*) FROM pg_constraint WHERE conname = 'runtime_cleanup_generation_consistent'`, 0) }) t.Run("v1 and v2 data", func(t *testing.T) { @@ -94,10 +159,18 @@ func TestUnifiedAccountMigration(t *testing.T) { assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'mapped' AND NOT runtime_cleanup_pending`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'unbound'`, 0) assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-mapped' AND binding_id = 'mapped'`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-mapped' AND binding_version = 1`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-unbound' AND binding_id IS NULL`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM audit_event WHERE event_type = 'legacy_event'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'legacy-task' AND state = 'policy_hold'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND NOT (fingerprint ?| ARRAY['proxy_server', 'disable_non_proxied_udp'])`, 1) + if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = true WHERE id = 'mapped'`); err != nil { + t.Fatalf("migration 008 blocked an old writer setting cleanup pending: %v", err) + } + assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE id = 'mapped' AND runtime_cleanup_pending AND runtime_cleanup_binding_version IS NULL AND runtime_cleanup_runtime_id IS NULL`, 1) + if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = false WHERE id = 'mapped'`); err != nil { + t.Fatalf("migration 008 blocked an old writer clearing cleanup pending: %v", err) + } store, err = Open(ctx, testURL) if err != nil { t.Fatal(err) @@ -175,7 +248,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal(err) } store.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (3, 4, 5, 6)`, 4) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (3, 4, 5, 6, 7, 8, 9)`, 7) }) } diff --git a/internal/hub/migrations/007_runtime_binding_version.sql b/internal/hub/migrations/007_runtime_binding_version.sql new file mode 100644 index 0000000..9a236e4 --- /dev/null +++ b/internal/hub/migrations/007_runtime_binding_version.sql @@ -0,0 +1,7 @@ +ALTER TABLE runtime_instance + ADD COLUMN binding_version bigint CHECK (binding_version > 0); + +UPDATE runtime_instance runtime +SET binding_version = binding.version +FROM environment_binding binding +WHERE runtime.binding_id = binding.id; diff --git a/internal/hub/migrations/008_runtime_cleanup_generation.sql b/internal/hub/migrations/008_runtime_cleanup_generation.sql new file mode 100644 index 0000000..15470fc --- /dev/null +++ b/internal/hub/migrations/008_runtime_cleanup_generation.sql @@ -0,0 +1,4 @@ +ALTER TABLE environment_binding + ADD COLUMN runtime_cleanup_binding_version bigint, + ADD COLUMN runtime_cleanup_instance_id text, + ADD COLUMN runtime_cleanup_runtime_id text; diff --git a/internal/hub/migrations/009_runtime_cleanup_compatibility.sql b/internal/hub/migrations/009_runtime_cleanup_compatibility.sql new file mode 100644 index 0000000..e7193d5 --- /dev/null +++ b/internal/hub/migrations/009_runtime_cleanup_compatibility.sql @@ -0,0 +1,2 @@ +ALTER TABLE environment_binding + DROP CONSTRAINT IF EXISTS runtime_cleanup_generation_consistent; diff --git a/internal/hub/migrations/010_runtime_network_generation.sql b/internal/hub/migrations/010_runtime_network_generation.sql new file mode 100644 index 0000000..e21ab16 --- /dev/null +++ b/internal/hub/migrations/010_runtime_network_generation.sql @@ -0,0 +1,5 @@ +ALTER TABLE runtime_instance + ADD COLUMN network_id text; + +ALTER TABLE environment_binding + ADD COLUMN runtime_cleanup_network_id text; diff --git a/internal/hub/store.go b/internal/hub/store.go index bc33442..7adc9ea 100644 --- a/internal/hub/store.go +++ b/internal/hub/store.go @@ -34,10 +34,23 @@ var migration005 string //go:embed migrations/006_runtime_cleanup.sql var migration006 string +//go:embed migrations/007_runtime_binding_version.sql +var migration007 string + +//go:embed migrations/008_runtime_cleanup_generation.sql +var migration008 string + +//go:embed migrations/009_runtime_cleanup_compatibility.sql +var migration009 string + +//go:embed migrations/010_runtime_network_generation.sql +var migration010 string + var ( - ErrConflict = errors.New("resource conflicts with existing state") - ErrInvalid = errors.New("invalid hub input") - ErrNotFound = errors.New("resource not found") + ErrConflict = errors.New("resource conflicts with existing state") + ErrInvalid = errors.New("invalid hub input") + ErrNotFound = errors.New("resource not found") + ErrReconcileRequired = errors.New("runtime cleanup generation is unknown; manual reconciliation required") ) func ValidImageVersion(version string) bool { return imageVersionPattern.MatchString(version) } @@ -118,7 +131,7 @@ func (s *Store) migrate(ctx context.Context) error { for _, migration := range []struct { version int sql string - }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}} { + }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}} { var applied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { return errors.New("read hub schema migration state") diff --git a/internal/hub/store_test.go b/internal/hub/store_test.go index 5fb18ad..55cf2d9 100644 --- a/internal/hub/store_test.go +++ b/internal/hub/store_test.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" "testing" + "time" ) func TestFingerprintArgsFollowUpstreamCommandLineContract(t *testing.T) { @@ -303,7 +304,10 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if err != nil || created || reused.Alias != bound.Alias || reused.BindingID != bound.BindingID { t.Fatalf("same account must reuse its environment: %#v created=%v err=%v", reused, created, err) } - active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID) + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { + t.Fatal(err) + } + active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID, "network-a") if err != nil || active.RuntimeInstanceID == "" { t.Fatalf("activate runtime: %#v err=%v", active, err) } @@ -331,13 +335,13 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if err != nil || legacyRebound.Exit.ID != second.ID { t.Fatalf("legacy binding without an exit must support explicit rebind: %#v err=%v", legacyRebound, err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID); !errors.Is(err, ErrConflict) { + if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID, "network-a"); !errors.Is(err, ErrConflict) { t.Fatalf("stale binding metadata must not activate a runtime: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil { t.Fatal(err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID); err != nil { + if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID, "network-a"); err != nil { t.Fatalf("runtime heartbeat failed: %v", err) } var renewed bool @@ -347,23 +351,110 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil { t.Fatal(err) } + expired, err := store.GetEnvironmentContext(ctx, env.Alias) + if err != nil || expired.RuntimeInstanceID != active.RuntimeInstanceID || expired.RuntimeID != active.RuntimeID { + t.Fatalf("context read discarded expired cleanup generation: %#v err=%v", expired, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil { + t.Fatal(err) + } rebound, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", bound.BindingVersion) if err != nil || rebound.Exit.ID != second.ID || rebound.BindingVersion != 2 { t.Fatalf("expired runtime must be transactionally released before rebind: %#v err=%v", rebound, err) } - if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID); err != nil { + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { + t.Fatal(err) + } + if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID, "network-same-exit"); err != nil { t.Fatalf("activate runtime before same-exit rebind: %v", err) } + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil { + t.Fatal(err) + } if _, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", rebound.BindingVersion); !errors.Is(err, ErrConflict) { t.Fatalf("active runtime must block same-exit rebind: %v", err) } - if err := store.ReleaseRuntime(ctx, env.Alias); err != nil { + active, err = store.GetEnvironmentContext(ctx, env.Alias) + if err != nil { + t.Fatal(err) + } + if err := store.ReleaseRuntime(ctx, active); err != nil { t.Fatal(err) } rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-container", rebound.BindingVersion) if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-container" { t.Fatalf("same-exit rebind must atomically CAS the binding and runtime: %#v err=%v", rebound, err) } + if err := store.ReleaseRuntime(ctx, active); !errors.Is(err, ErrConflict) { + t.Fatalf("stale generation release must conflict: %v", err) + } + current, err := store.GetEnvironmentContext(ctx, env.Alias) + if err != nil || current.RuntimeInstanceID != rebound.RuntimeInstanceID || current.RuntimeID != "rebound-container" { + t.Fatalf("stale release changed the current runtime: %#v err=%v", current, err) + } + cleanup := current + cleanup.RuntimeCleanupBindingVersion = current.BindingVersion + cleanup.RuntimeCleanupInstanceID = current.RuntimeInstanceID + cleanup.RuntimeCleanupRuntimeID = current.RuntimeID + if err := store.SetRuntimeCleanupPending(ctx, cleanup, true); err != nil { + t.Fatalf("set generation cleanup pending: %v", err) + } + wrongCleanup := cleanup + wrongCleanup.RuntimeCleanupRuntimeID = "other-container" + if err := store.SetRuntimeCleanupPending(ctx, wrongCleanup, false); !errors.Is(err, ErrConflict) { + t.Fatalf("wrong cleanup generation cleared pending state: %v", err) + } + pending, err := store.GetEnvironmentContext(ctx, env.Alias) + if err != nil || !pending.RuntimeCleanupPending || pending.RuntimeCleanupRuntimeID != current.RuntimeID { + t.Fatalf("cleanup generation was not persisted: %#v err=%v", pending, err) + } + if _, err := store.ActivateRuntime(ctx, env.Alias, "candidate-container", current.BindingVersion, current.Exit.ID, "network-candidate"); !errors.Is(err, ErrConflict) { + t.Fatalf("paused account activated a stale request: %v", err) + } + if err := store.SetRuntimeCleanupPending(ctx, pending, false); err != nil { + t.Fatalf("clear matching cleanup generation: %v", err) + } + newGeneration, err := store.RebindEnvironment(ctx, env.Alias, current.Exit.ID, "", current.BindingVersion) + if err != nil { + t.Fatalf("advance binding generation: %v", err) + } + if err := store.SetRuntimeCleanupPending(ctx, cleanup, true); !errors.Is(err, ErrConflict) { + t.Fatalf("stale binding set cleanup pending on version %d: %v", newGeneration.BindingVersion, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { + t.Fatal(err) + } + pauseTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + var locked string + if err := pauseTx.QueryRowContext(ctx, `SELECT id FROM social_account WHERE id = 'account-a' FOR UPDATE`).Scan(&locked); err != nil { + t.Fatal(err) + } + activation := make(chan error, 1) + go func() { + _, err := store.ActivateRuntime(ctx, env.Alias, "racing-container", newGeneration.BindingVersion, newGeneration.Exit.ID, "network-racing") + activation <- err + }() + select { + case err := <-activation: + t.Fatalf("activation bypassed the locked account row: %v", err) + case <-time.After(50 * time.Millisecond): + } + if _, err := pauseTx.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil { + t.Fatal(err) + } + if err := pauseTx.Commit(); err != nil { + t.Fatal(err) + } + if err := <-activation; !errors.Is(err, ErrConflict) { + t.Fatalf("activation won the pause race: %v", err) + } + latest, err := store.GetEnvironmentContext(ctx, env.Alias) + if err != nil || latest.RuntimeID != "" { + t.Fatalf("pause race left an active runtime: %#v err=%v", latest, err) + } action := EnvironmentAction{ OperationID: NewOperationID(), Action: "start", AccountID: rebound.AccountID, diff --git a/internal/phasea/store.go b/internal/phasea/store.go index 5e4876e..0ab385e 100644 --- a/internal/phasea/store.go +++ b/internal/phasea/store.go @@ -49,13 +49,6 @@ type CredentialReference struct { Provider string `json:"provider"` } -type RuntimeBinding struct { - ID string `json:"id"` - AccountID string `json:"account_id"` - RuntimeID string `json:"runtime_id"` - LeaseUntil time.Time `json:"lease_until"` -} - type Draft struct { ID string `json:"id"` AccountID string `json:"account_id"` @@ -193,7 +186,7 @@ func (s *Store) CreateAccount(ctx context.Context, account Account) error { account.Platform, account.PlatformAccountKey, account.AuthorizationKind); err != nil { return publicDatabaseError(err) } - if err := appendAudit(ctx, tx, "account_created", account.ID, "", 0, "", "", map[string]string{"platform": account.Platform}); err != nil { + if err := appendAudit(ctx, tx, "account_created", "account_created", account.ID, "", 0, "", "", map[string]string{"platform": account.Platform}); err != nil { return err } return commit(tx) @@ -245,57 +238,6 @@ func scanAccount(row accountScanner) (Account, error) { return account, nil } -func (s *Store) BindRuntime(ctx context.Context, binding RuntimeBinding) error { - if !refPattern.MatchString(binding.ID) || !idPattern.MatchString(binding.AccountID) || - !refPattern.MatchString(binding.RuntimeID) || !binding.LeaseUntil.After(time.Now()) { - return ErrInvalid - } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return errors.New("begin runtime transaction") - } - defer tx.Rollback() - if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET released_at = now() WHERE released_at IS NULL AND lease_until < now()`); err != nil { - return errors.New("expire runtime bindings") - } - var accountID string - if err := tx.QueryRowContext(ctx, ` - INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) - SELECT $1, account.id, binding.id, $3, $4 - FROM social_account account - JOIN environment_binding binding ON binding.account_id = account.id - WHERE account.id = $2 AND account.status = 'active' - RETURNING account_id`, binding.ID, binding.AccountID, binding.RuntimeID, binding.LeaseUntil).Scan(&accountID); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return ErrConflict - } - return rowError(err) - } - if err := appendAudit(ctx, tx, "runtime_bound", accountID, "", 0, "", "", map[string]string{"runtime_id": binding.RuntimeID}); err != nil { - return err - } - return commit(tx) -} - -func (s *Store) ReleaseRuntime(ctx context.Context, id string) error { - if !refPattern.MatchString(id) { - return ErrInvalid - } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return errors.New("begin runtime release") - } - defer tx.Rollback() - var accountID string - if err := tx.QueryRowContext(ctx, `UPDATE runtime_instance SET released_at = now() WHERE id = $1 AND released_at IS NULL RETURNING account_id`, id).Scan(&accountID); err != nil { - return rowError(err) - } - if err := appendAudit(ctx, tx, "runtime_released", accountID, "", 0, "", "", map[string]string{"binding_id": id}); err != nil { - return err - } - return commit(tx) -} - func (s *Store) CreateDraft(ctx context.Context, draft Draft) error { if !refPattern.MatchString(draft.ID) || !idPattern.MatchString(draft.AccountID) || draft.Version < 1 || strings.TrimSpace(draft.Content) == "" { return ErrInvalid @@ -332,7 +274,7 @@ func (s *Store) Confirm(ctx context.Context, confirmation Confirmation) error { confirmation.DraftID, confirmation.DraftVersion, confirmation.Version); err != nil { return publicDatabaseError(err) } - if err := appendAudit(ctx, tx, "draft_confirmed", confirmation.AccountID, confirmation.ID, confirmation.Version, "", "", map[string]any{ + if err := appendAudit(ctx, tx, "draft_confirmed", "draft_confirmed", confirmation.AccountID, confirmation.ID, confirmation.Version, "", "", map[string]any{ "account_version": confirmation.AccountVersion, "draft_id": confirmation.DraftID, "draft_version": confirmation.DraftVersion, }); err != nil { return err @@ -363,7 +305,7 @@ func (s *Store) Enqueue(ctx context.Context, task Task) (Task, bool, error) { } if insertedID != "" { task.State = "queued" - if err := appendAudit(ctx, tx, "task_queued", task.AccountID, task.ConfirmationID, task.ConfirmationVersion, "", task.ID, nil); err != nil { + if err := appendAudit(ctx, tx, "task_queued", "task_queued", task.AccountID, task.ConfirmationID, task.ConfirmationVersion, "", task.ID, nil); err != nil { return Task{}, false, err } if err := commit(tx); err != nil { @@ -432,6 +374,14 @@ func (s *Store) disableAccount(ctx context.Context, accountID string, revoke boo if err != nil { return err } + reason := "account_paused" + if revoke { + reason = "account_revoked" + } + interrupted, err := interruptExecutingTasks(ctx, tx, accountID, reason) + if err != nil { + return err + } if unchanged { return commit(tx) } @@ -439,7 +389,9 @@ func (s *Store) disableAccount(ctx context.Context, accountID string, revoke boo if revoke { eventType = "account_revoked" } - if err := appendAudit(ctx, tx, eventType, accountID, "", 0, "", "", map[string]any{"account_version": version, "tasks_held": held}); err != nil { + if err := appendAudit(ctx, tx, eventType, reason, accountID, "", 0, "", "", map[string]any{ + "account_version": version, "tasks_held": held, "attempts_interrupted": interrupted, + }); err != nil { return err } return commit(tx) @@ -468,6 +420,11 @@ func (s *Store) ResumeAccount(ctx context.Context, accountID string) error { SELECT 1 FROM environment_binding binding JOIN network_exit network ON network.id = binding.network_exit_id WHERE binding.account_id = $1 AND network.health_status = 'healthy' + AND NOT binding.runtime_cleanup_pending + AND NOT EXISTS ( + SELECT 1 FROM runtime_instance runtime + WHERE runtime.binding_id = binding.id AND runtime.released_at IS NULL + ) )`, accountID).Scan(&ready); err != nil { return errors.New("validate account binding") } @@ -483,7 +440,7 @@ func (s *Store) ResumeAccount(ctx context.Context, accountID string) error { WHERE id = $1 RETURNING version`, accountID).Scan(&version); err != nil { return errors.New("resume account") } - if err := appendAudit(ctx, tx, "account_resumed", accountID, "", 0, "", "", map[string]any{"account_version": version}); err != nil { + if err := appendAudit(ctx, tx, "account_resumed", "account_resumed", accountID, "", 0, "", "", map[string]any{"account_version": version}); err != nil { return err } return commit(tx) @@ -500,6 +457,49 @@ func holdQueuedTasks(ctx context.Context, tx *sql.Tx, accountID string) (int64, return held, nil } +func interruptExecutingTasks(ctx context.Context, tx *sql.Tx, accountID, reason string) (int64, error) { + rows, err := tx.QueryContext(ctx, ` + UPDATE operation_task SET state = 'needs_confirmation', lease_owner = NULL, lease_until = NULL, updated_at = now() + WHERE account_id = $1 AND state = 'executing' + RETURNING id, current_attempt_id, confirmation_id, confirmation_version`, accountID) + if err != nil { + return 0, errors.New("interrupt executing account tasks") + } + type interruptedTask struct { + taskID, attemptID, confirmationID string + confirmationVersion int64 + } + var tasks []interruptedTask + for rows.Next() { + var task interruptedTask + var attemptID, confirmationID sql.NullString + var confirmationVersion sql.NullInt64 + if err := rows.Scan(&task.taskID, &attemptID, &confirmationID, &confirmationVersion); err != nil { + rows.Close() + return 0, errors.New("read interrupted account task") + } + task.attemptID, task.confirmationID, task.confirmationVersion = attemptID.String, confirmationID.String, confirmationVersion.Int64 + tasks = append(tasks, task) + } + if err := rows.Close(); err != nil { + return 0, errors.New("close interrupted account task rows") + } + for _, task := range tasks { + if task.attemptID != "" { + if _, err := tx.ExecContext(ctx, ` + UPDATE execution_attempt SET finished_at = now(), outcome = 'uncertain' + WHERE id = $1 AND finished_at IS NULL`, task.attemptID); err != nil { + return 0, errors.New("mark interrupted attempt uncertain") + } + } + if err := appendAudit(ctx, tx, "task_interrupted", reason, accountID, task.confirmationID, + task.confirmationVersion, task.attemptID, task.taskID, nil); err != nil { + return 0, err + } + } + return int64(len(tasks)), nil +} + func (s *Store) CancelTask(ctx context.Context, taskID string) error { if !refPattern.MatchString(taskID) { return ErrInvalid @@ -529,7 +529,7 @@ func (s *Store) CancelTask(ctx context.Context, taskID string) error { return errors.New("mark cancelled attempt uncertain") } } - if err := appendAudit(ctx, tx, "task_cancel_requested", accountID, confirmationID.String, confirmationVersion.Int64, attemptID.String, taskID, map[string]string{"state": state}); err != nil { + if err := appendAudit(ctx, tx, "task_cancel_requested", "task_cancel_requested", accountID, confirmationID.String, confirmationVersion.Int64, attemptID.String, taskID, map[string]string{"state": state}); err != nil { return err } return commit(tx) @@ -566,14 +566,20 @@ func (s *Store) claim(ctx context.Context, workerID string) (Execution, error) { JOIN social_account a ON a.id = t.account_id JOIN content_draft d ON d.id = t.draft_id JOIN confirmation c ON c.id = t.confirmation_id - WHERE t.state = 'queued' AND a.status = 'active' + JOIN environment_binding binding ON binding.account_id = a.id + JOIN browser_env environment ON environment.alias = binding.browser_env_alias + JOIN network_exit network ON network.id = binding.network_exit_id + JOIN runtime_instance runtime ON runtime.binding_id = binding.id + AND runtime.released_at IS NULL AND runtime.lease_until > now() AND runtime.binding_version = binding.version + WHERE t.state = 'queued' AND a.status = 'active' AND a.authorization_status = 'authorized' AND a.version = t.account_version AND d.account_id = t.account_id AND d.version = t.draft_version AND c.account_id = t.account_id AND c.account_version = t.account_version AND c.draft_id = t.draft_id AND c.draft_version = t.draft_version AND c.version = t.confirmation_version + AND network.health_status = 'healthy' AND NOT binding.runtime_cleanup_pending ORDER BY t.created_at, t.id - FOR UPDATE OF t SKIP LOCKED LIMIT 1 + FOR UPDATE OF t, a, binding, network, runtime SKIP LOCKED LIMIT 1 ) UPDATE operation_task t SET state = 'executing', lease_owner = $1, lease_until = now() + interval '1 minute', updated_at = now() @@ -598,7 +604,7 @@ func (s *Store) claim(ctx context.Context, workerID string) (Execution, error) { if _, err := tx.ExecContext(ctx, `UPDATE operation_task SET current_attempt_id = $1 WHERE id = $2`, execution.AttemptID, execution.TaskID); err != nil { return Execution{}, errors.New("attach execution attempt") } - if err := appendAudit(ctx, tx, "task_claimed", execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"worker_id": workerID}); err != nil { + if err := appendAudit(ctx, tx, "task_claimed", "task_claimed", execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"worker_id": workerID}); err != nil { return Execution{}, err } if err := commit(tx); err != nil { @@ -633,7 +639,11 @@ func (s *Store) complete(ctx context.Context, workerID string, execution Executi UPDATE operation_task SET state = $1, lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE id = $2`, state, execution.TaskID); err != nil { return Execution{}, errors.New("finish task") } - if err := appendAudit(ctx, tx, "task_finished", execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"state": state}); err != nil { + reason := map[string]string{ + "succeeded": "task_succeeded", "failed": "task_failed", + "needs_confirmation": "task_result_uncertain", "policy_hold": "task_policy_hold", + }[state] + if err := appendAudit(ctx, tx, "task_finished", reason, execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"state": state}); err != nil { return Execution{}, err } if err := commit(tx); err != nil { @@ -676,7 +686,7 @@ func quarantineExpired(ctx context.Context, tx *sql.Tx) error { return errors.New("mark expired attempt uncertain") } } - if err := appendAudit(ctx, tx, "task_lease_expired", task.accountID, task.confirmationID, task.confirmationVersion, task.attemptID, task.taskID, nil); err != nil { + if err := appendAudit(ctx, tx, "task_lease_expired", "execution_lease_expired", task.accountID, task.confirmationID, task.confirmationVersion, task.attemptID, task.taskID, nil); err != nil { return err } } @@ -685,46 +695,84 @@ func quarantineExpired(ctx context.Context, tx *sql.Tx) error { func quarantineInvalid(ctx context.Context, tx *sql.Tx) error { rows, err := tx.QueryContext(ctx, ` - UPDATE operation_task t SET - state = CASE WHEN EXISTS ( - SELECT 1 FROM social_account paused WHERE paused.id = t.account_id AND paused.status = 'paused' - ) THEN 'policy_hold' ELSE 'needs_confirmation' END, - updated_at = now() - WHERE t.state = 'queued' AND NOT EXISTS ( - SELECT 1 - FROM social_account a - JOIN content_draft d ON d.id = t.draft_id - JOIN confirmation c ON c.id = t.confirmation_id - WHERE a.id = t.account_id AND a.status = 'active' AND a.version = t.account_version - AND d.account_id = t.account_id AND d.version = t.draft_version - AND c.account_id = t.account_id AND c.account_version = t.account_version - AND c.draft_id = t.draft_id AND c.draft_version = t.draft_version - AND c.version = t.confirmation_version + WITH invalid AS ( + SELECT t.id, + CASE + WHEN account.id IS NULL OR account.status <> 'active' OR account.authorization_status <> 'authorized' THEN 'policy_hold' + WHEN account.version <> t.account_version THEN 'needs_confirmation' + WHEN draft.id IS NULL OR draft.account_id <> t.account_id OR draft.version <> t.draft_version THEN 'needs_confirmation' + WHEN confirmation.id IS NULL OR confirmation.account_id <> t.account_id + OR confirmation.account_version <> t.account_version OR confirmation.draft_id <> t.draft_id + OR confirmation.draft_version <> t.draft_version OR confirmation.version <> t.confirmation_version THEN 'needs_confirmation' + WHEN runtime.id IS NOT NULL AND runtime.binding_version IS DISTINCT FROM binding.version THEN 'needs_confirmation' + ELSE 'policy_hold' + END AS state, + CASE + WHEN account.id IS NULL THEN 'account_missing' + WHEN account.authorization_status = 'revoked' THEN 'account_revoked' + WHEN account.status <> 'active' THEN 'account_paused' + WHEN account.version <> t.account_version THEN 'account_version_changed' + WHEN draft.id IS NULL OR draft.account_id <> t.account_id OR draft.version <> t.draft_version THEN 'draft_version_changed' + WHEN confirmation.id IS NULL THEN 'confirmation_missing' + WHEN confirmation.account_id <> t.account_id OR confirmation.account_version <> t.account_version + OR confirmation.draft_id <> t.draft_id OR confirmation.draft_version <> t.draft_version + OR confirmation.version <> t.confirmation_version THEN 'confirmation_version_changed' + WHEN binding.id IS NULL THEN 'binding_missing' + WHEN environment.alias IS NULL THEN 'environment_missing' + WHEN network.id IS NULL THEN 'exit_missing' + WHEN network.health_status <> 'healthy' THEN 'exit_unhealthy' + WHEN binding.runtime_cleanup_pending THEN 'runtime_stop_pending' + WHEN runtime.id IS NULL THEN 'runtime_missing' + WHEN runtime.binding_version IS DISTINCT FROM binding.version THEN 'binding_version_changed' + WHEN runtime.lease_until <= now() THEN 'runtime_lease_expired' + END AS reason_code + FROM operation_task t + LEFT JOIN social_account account ON account.id = t.account_id + LEFT JOIN content_draft draft ON draft.id = t.draft_id + LEFT JOIN confirmation confirmation ON confirmation.id = t.confirmation_id + LEFT JOIN environment_binding binding ON binding.account_id = t.account_id + LEFT JOIN browser_env environment ON environment.alias = binding.browser_env_alias + LEFT JOIN network_exit network ON network.id = binding.network_exit_id + LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL + WHERE t.state = 'queued' AND ( + account.id IS NULL OR account.status <> 'active' OR account.authorization_status <> 'authorized' + OR account.version <> t.account_version + OR draft.id IS NULL OR draft.account_id <> t.account_id OR draft.version <> t.draft_version + OR confirmation.id IS NULL OR confirmation.account_id <> t.account_id + OR confirmation.account_version <> t.account_version OR confirmation.draft_id <> t.draft_id + OR confirmation.draft_version <> t.draft_version OR confirmation.version <> t.confirmation_version + OR binding.id IS NULL OR environment.alias IS NULL OR network.id IS NULL OR network.health_status <> 'healthy' + OR binding.runtime_cleanup_pending OR runtime.id IS NULL + OR runtime.binding_version IS DISTINCT FROM binding.version OR runtime.lease_until <= now() + ) + FOR UPDATE OF t SKIP LOCKED ) - RETURNING t.id, t.account_id, t.confirmation_id, t.confirmation_version, t.state`) + UPDATE operation_task task SET state = invalid.state, updated_at = now() + FROM invalid WHERE task.id = invalid.id + RETURNING task.id, task.account_id, task.confirmation_id, task.confirmation_version, task.state, invalid.reason_code`) if err != nil { return errors.New("quarantine invalid tasks") } type invalidTask struct { - taskID, accountID, confirmationID, state string - confirmationVersion int64 + taskID, accountID, confirmationID, state, reasonCode string + confirmationVersion int64 } var invalid []invalidTask for rows.Next() { - var taskID, accountID, state string + var taskID, accountID, state, reasonCode string var confirmationID sql.NullString var confirmationVersion sql.NullInt64 - if err := rows.Scan(&taskID, &accountID, &confirmationID, &confirmationVersion, &state); err != nil { + if err := rows.Scan(&taskID, &accountID, &confirmationID, &confirmationVersion, &state, &reasonCode); err != nil { rows.Close() return errors.New("read quarantined task") } - invalid = append(invalid, invalidTask{taskID, accountID, confirmationID.String, state, confirmationVersion.Int64}) + invalid = append(invalid, invalidTask{taskID, accountID, confirmationID.String, state, reasonCode, confirmationVersion.Int64}) } if err := rows.Close(); err != nil { return errors.New("close quarantined task rows") } for _, task := range invalid { - if err := appendAudit(ctx, tx, "task_quarantined", task.accountID, task.confirmationID, task.confirmationVersion, "", task.taskID, map[string]string{"state": task.state}); err != nil { + if err := appendAudit(ctx, tx, "task_quarantined", task.reasonCode, task.accountID, task.confirmationID, task.confirmationVersion, "", task.taskID, map[string]string{"state": task.state}); err != nil { return err } } @@ -766,7 +814,7 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) { return events, rows.Err() } -func appendAudit(ctx context.Context, tx *sql.Tx, eventType, accountID, confirmationID string, confirmationVersion int64, attemptID, taskID string, details any) error { +func appendAudit(ctx context.Context, tx *sql.Tx, eventType, reasonCode, accountID, confirmationID string, confirmationVersion int64, attemptID, taskID string, details any) error { if details == nil { details = map[string]any{} } @@ -776,9 +824,14 @@ func appendAudit(ctx context.Context, tx *sql.Tx, eventType, accountID, confirma } _, err = tx.ExecContext(ctx, ` INSERT INTO audit_event - (event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id, actor, reason_code, details) - VALUES ($1, NULLIF($2, ''), NULLIF($3, ''), NULLIF($4, 0), NULLIF($5, ''), NULLIF($6, ''), 'local-user', $1, $7)`, - eventType, accountID, confirmationID, confirmationVersion, attemptID, taskID, encoded) + (event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id, + browser_env_alias, network_exit_id, runtime_instance_id, binding_version, actor, reason_code, details) + SELECT $1, NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, 0), NULLIF($6, ''), NULLIF($7, ''), + binding.browser_env_alias, binding.network_exit_id, runtime.id, binding.version, 'local-user', $2, $8 + FROM (VALUES (1)) AS singleton(value) + LEFT JOIN environment_binding binding ON binding.account_id = NULLIF($3, '') + LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL`, + eventType, reasonCode, accountID, confirmationID, confirmationVersion, attemptID, taskID, encoded) if err != nil { return errors.New("append audit event") } diff --git a/internal/phasea/store_test.go b/internal/phasea/store_test.go index a563c3e..79eba80 100644 --- a/internal/phasea/store_test.go +++ b/internal/phasea/store_test.go @@ -131,23 +131,21 @@ func TestPhaseAOfflineWorkflow(t *testing.T) { t.Fatal(err) } - lease := time.Now().Add(time.Minute) - if err := store.BindRuntime(ctx, RuntimeBinding{ID: "binding-a", AccountID: "account-a", RuntimeID: "runtime-a", LeaseUntil: lease}); err != nil { - t.Fatal(err) - } - if err := store.BindRuntime(ctx, RuntimeBinding{ID: "binding-a-2", AccountID: "account-a", RuntimeID: "runtime-b", LeaseUntil: lease}); !errors.Is(err, ErrConflict) { - t.Fatalf("expected one active runtime per account, got %v", err) - } - if err := store.BindRuntime(ctx, RuntimeBinding{ID: "binding-b", AccountID: "account-b", RuntimeID: "runtime-a", LeaseUntil: lease}); !errors.Is(err, ErrConflict) { - t.Fatalf("expected one account per runtime, got %v", err) - } - if err := store.ReleaseRuntime(ctx, "binding-a"); err != nil { + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) VALUES + ('runtime-instance-a', 'account-a', 'binding-a', 1, 'runtime-a', now() + interval '1 minute'), + ('runtime-instance-b', 'account-b', 'binding-b', 1, 'runtime-b', now() + interval '1 minute')`); err != nil { t.Fatal(err) } createApprovedDraft(t, store, "account-a", accountA.Version, "draft-a", "confirmation-a") + createApprovedDraft(t, store, "account-b", accountB.Version, "draft-b", "confirmation-b") for index := range 20 { - task := approvedTask(index, "account-a", accountA.Version, "draft-a", "confirmation-a") + accountID, accountVersion, draftID, confirmationID := "account-a", accountA.Version, "draft-a", "confirmation-a" + if index%2 == 1 { + accountID, accountVersion, draftID, confirmationID = "account-b", accountB.Version, "draft-b", "confirmation-b" + } + task := approvedTask(index, accountID, accountVersion, draftID, confirmationID) if _, inserted, err := store.Enqueue(ctx, task); err != nil || !inserted { t.Fatalf("enqueue task %d: inserted=%v err=%v", index, inserted, err) } @@ -188,7 +186,51 @@ func TestPhaseAOfflineWorkflow(t *testing.T) { assertCount(t, store, `SELECT count(*) FROM operation_task WHERE state = 'succeeded'`, 20) assertCount(t, store, `SELECT count(*) FROM execution_attempt`, 20) - createApprovedDraft(t, store, "account-b", accountB.Version, "draft-b", "confirmation-b") + for _, test := range []struct { + name, accountStatus, authorizationStatus, exitStatus, runtimeLease, wantState, wantReason string + binding, versionChanged, confirmationChanged bool + }{ + {name: "binding-missing", accountStatus: "active", authorizationStatus: "authorized", wantState: "policy_hold", wantReason: "binding_missing"}, + {name: "exit-unhealthy", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "unhealthy", wantState: "policy_hold", wantReason: "exit_unhealthy"}, + {name: "runtime-missing", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", wantState: "policy_hold", wantReason: "runtime_missing"}, + {name: "runtime-expired", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "expired", wantState: "policy_hold", wantReason: "runtime_lease_expired"}, + {name: "binding-version", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", versionChanged: true, wantState: "needs_confirmation", wantReason: "binding_version_changed"}, + {name: "confirmation-version", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", confirmationChanged: true, wantState: "needs_confirmation", wantReason: "confirmation_version_changed"}, + {name: "paused", accountStatus: "paused", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", wantState: "policy_hold", wantReason: "account_paused"}, + {name: "revoked", accountStatus: "paused", authorizationStatus: "revoked", binding: true, exitStatus: "healthy", runtimeLease: "active", wantState: "policy_hold", wantReason: "account_revoked"}, + } { + t.Run("gate "+test.name, func(t *testing.T) { + taskID := seedGateTask(t, store, test.name, test.accountStatus, test.authorizationStatus, test.exitStatus, + test.runtimeLease, test.binding, test.versionChanged, test.confirmationChanged) + if execution, err := store.ExecuteMock(ctx, "worker-gate-"+test.name, "succeeded"); err != nil || execution.WasClaimed { + t.Fatalf("invalid task was claimed: execution=%+v err=%v", execution, err) + } + assertTaskGate(t, store, taskID, test.wantState, test.wantReason) + }) + } + + // A held task is never revived in place: only a newly confirmed task with a new idempotency key may run. + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) + VALUES ('runtime-recovered', 'gate-runtime-missing', 'binding-runtime-missing', 1, 'container-recovered', now() + interval '1 minute')`); err != nil { + t.Fatal(err) + } + if execution, err := store.ExecuteMock(ctx, "worker-old-held", "succeeded"); err != nil || execution.WasClaimed { + t.Fatalf("held task was revived after health recovery: execution=%+v err=%v", execution, err) + } + createApprovedDraft(t, store, "gate-runtime-missing", 1, "draft-recovered", "confirmation-recovered") + recovered := Task{ + ID: "task-recovered", IdempotencyKey: "idempotency-recovered", AccountID: "gate-runtime-missing", AccountVersion: 1, + DraftID: "draft-recovered", DraftVersion: 1, ConfirmationID: "confirmation-recovered", ConfirmationVersion: 1, + } + if _, inserted, err := store.Enqueue(ctx, recovered); err != nil || !inserted { + t.Fatalf("enqueue recovered task: inserted=%v err=%v", inserted, err) + } + if execution, err := store.ExecuteMock(ctx, "worker-recovered", "succeeded"); err != nil || !execution.WasClaimed || execution.State != "succeeded" { + t.Fatalf("newly confirmed task did not run after recovery: execution=%+v err=%v", execution, err) + } + assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-gate-runtime-missing' AND state = 'policy_hold'`, 1) + unconfirmed := approvedTask(30, "account-b", accountB.Version, "draft-b", "") unconfirmed.ConfirmationVersion = 0 if _, _, err := store.Enqueue(ctx, unconfirmed); err != nil { @@ -338,12 +380,107 @@ func approvedTask(index int, accountID string, accountVersion int64, draftID, co } } +func seedGateTask(t *testing.T, store *Store, suffix, accountStatus, authorizationStatus, exitStatus, runtimeLease string, + binding, versionChanged, confirmationChanged bool) string { + t.Helper() + ctx := context.Background() + accountID, taskID := "gate-"+suffix, "task-gate-"+suffix + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `INSERT INTO credential_reference (id, provider, reference_key) VALUES ($1, 'os_keyring', $2)`, + "credential-"+suffix, "creatorhub/gate-"+suffix); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO social_account + (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status, status, version) + VALUES ($1, $2, 'mock', $1, 'owned', $3, $4, 1)`, accountID, "credential-"+suffix, authorizationStatus, accountStatus); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO content_draft (id, account_id, version, content) VALUES ($1, $2, 1, 'gate test')`, + "draft-gate-"+suffix, accountID); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO confirmation (id, account_id, account_version, draft_id, draft_version, version) + VALUES ($1, $2, 1, $3, 1, 1)`, "confirmation-gate-"+suffix, accountID, "draft-gate-"+suffix); err != nil { + t.Fatal(err) + } + if binding { + exitID := "exit-gate-" + suffix + if _, err := tx.ExecContext(ctx, `INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + VALUES ($1, $1, 'test-gateway', '1', '{"seed":3}')`, accountID); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO network_exit (id, protocol, host, port, health_status) + VALUES ($1, 'socks5', '127.0.0.1', 1080, $2)`, exitID, exitStatus); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO environment_binding (id, account_id, browser_env_alias, network_exit_id, version) + VALUES ($1, $2, $2, $3, 1)`, "binding-"+suffix, accountID, exitID); err != nil { + t.Fatal(err) + } + if runtimeLease != "" { + interval := "1 minute" + if runtimeLease == "expired" { + interval = "-1 second" + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) + VALUES ($1, $2, $3, 1, $4, now() + $5::interval)`, "runtime-gate-"+suffix, accountID, + "binding-"+suffix, "container-gate-"+suffix, interval); err != nil { + t.Fatal(err) + } + } + } + if versionChanged { + if _, err := tx.ExecContext(ctx, `UPDATE environment_binding SET version = 2 WHERE account_id = $1`, accountID); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + confirmationVersion := int64(1) + if confirmationChanged { + confirmationVersion = 2 + } + if _, _, err := store.Enqueue(ctx, Task{ + ID: taskID, IdempotencyKey: "idempotency-gate-" + suffix, AccountID: accountID, AccountVersion: 1, + DraftID: "draft-gate-" + suffix, DraftVersion: 1, ConfirmationID: "confirmation-gate-" + suffix, + ConfirmationVersion: confirmationVersion, + }); err != nil { + t.Fatal(err) + } + return taskID +} + +func assertTaskGate(t *testing.T, store *Store, taskID, wantState, wantReason string) { + t.Helper() + var state string + if err := store.db.QueryRow(`SELECT state FROM operation_task WHERE id = $1`, taskID).Scan(&state); err != nil || state != wantState { + t.Fatalf("task gate state: got=%q want=%q err=%v", state, wantState, err) + } + var attempts int + if err := store.db.QueryRow(`SELECT count(*) FROM execution_attempt WHERE task_id = $1`, taskID).Scan(&attempts); err != nil || attempts != 0 { + t.Fatalf("gated task executed: attempts=%d err=%v", attempts, err) + } + var reason string + if err := store.db.QueryRow(`SELECT reason_code FROM audit_event WHERE task_id = $1 AND event_type = 'task_quarantined' ORDER BY id DESC LIMIT 1`, taskID).Scan(&reason); err != nil || reason != wantReason { + t.Fatalf("task gate reason: got=%q want=%q err=%v", reason, wantReason, err) + } +} + func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) { t.Helper() for _, migrationFile := range []struct { version int name string - }{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}} { + }{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}, {6, "006_runtime_cleanup.sql"}, {7, "007_runtime_binding_version.sql"}} { var applied bool if err := store.db.QueryRow(`SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migrationFile.version).Scan(&applied); err != nil { t.Fatal(err)