HH-802: enforce scheduling gates and runtime leases (#20)

cmd/control-plane/hub.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -39,20 +40,36 @@ type hubStore interface {
 	DisableNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error)
 	CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error)
 	GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error)
+	GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error)
 	ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error
-	RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (hub.EnvironmentContext, error)
-	ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (hub.EnvironmentContext, error)
-	ReleaseRuntime(ctx context.Context, alias string) error
-	SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error
+	RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkID ...string) (hub.EnvironmentContext, error)
+	ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkID ...string) (hub.EnvironmentContext, error)
+	ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error
+	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
 	AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error
 }
 
+type runtimeStopStore interface {
+	GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error)
+	GetGateway(ctx context.Context, name string) (hub.Gateway, error)
+	ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error
+	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
+	AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error
+}
+
+type runtimeCleanupStore interface {
+	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
+}
+
 const (
 	gatewayLongTimeout       = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取
 	gatewayReconcileDelay    = 100 * time.Millisecond
 	gatewayReconcileAttempts = 10
+	missingRuntimeID         = "runtime-not-found"
 )
 
+var gatewayGenerationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`)
+
 // gatewayCall 调用某个网关的 /v1 路由；ok 为 false 时 status/body 携带网关错误。
 func gatewayCall(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration) (status int, responseBody []byte, err error) {
 	callCtx, cancel := context.WithTimeout(ctx, timeout)
@@ -99,14 +116,86 @@ func gatewayCreatePayload(environment hub.EnvironmentContext, imageRef string, n
 	}
 }
 
-func gatewayProxyPayload(environment hub.EnvironmentContext, networkExit gatewayNetworkExit) map[string]any {
+func gatewayCreatePayloadForAccount(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any {
+	payload := gatewayCreatePayload(environment, imageRef, networkExit)
+	if !accountRunnable(environment) {
+		payload["network_exit_id"], payload["network_exit"], payload["stopped"] = "", gatewayNetworkExit{}, true
+	}
+	return payload
+}
+
+func gatewayProxyPayload(environment hub.EnvironmentContext, runtimeID, networkID string, networkExit gatewayNetworkExit) map[string]any {
 	return map[string]any{
 		"binding_version": environment.BindingVersion,
+		"runtime_id":      runtimeID,
+		"network_id":      networkID,
 		"network_exit_id": environment.Exit.ID,
 		"network_exit":    networkExit,
 	}
 }
 
+func gatewayGenerationPayload(environment hub.EnvironmentContext) map[string]any {
+	if environment.RuntimeCleanupBindingVersion > 0 {
+		return map[string]any{"binding_version": environment.RuntimeCleanupBindingVersion, "runtime_id": environment.RuntimeCleanupRuntimeID,
+			"network_id": environment.RuntimeCleanupNetworkID}
+	}
+	return map[string]any{"binding_version": environment.BindingVersion, "runtime_id": environment.RuntimeID, "network_id": environment.RuntimeNetworkID}
+}
+
+func runtimeCleanupGeneration(environment hub.EnvironmentContext, bindingVersion int64, runtimeID string, networkIDs ...string) hub.EnvironmentContext {
+	environment.RuntimeCleanupBindingVersion = bindingVersion
+	environment.RuntimeCleanupInstanceID = environment.RuntimeInstanceID
+	environment.RuntimeCleanupRuntimeID = runtimeID
+	environment.RuntimeCleanupNetworkID = environment.RuntimeNetworkID
+	if len(networkIDs) == 1 {
+		environment.RuntimeCleanupNetworkID = networkIDs[0]
+	}
+	return environment
+}
+
+func gatewayNetworkID(body []byte) string {
+	var envelope struct {
+		NetworkID string `json:"network_id"`
+	}
+	if json.Unmarshal(body, &envelope) == nil && gatewayGenerationIDPattern.MatchString(envelope.NetworkID) {
+		return envelope.NetworkID
+	}
+	return ""
+}
+
+func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
+	environment hub.EnvironmentContext, body []byte) error {
+	return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion, gatewayNetworkID(body))
+}
+
+func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
+	environment hub.EnvironmentContext, bindingVersion int64, networkID string) error {
+	if networkID == "" {
+		pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "")
+		return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
+	}
+	container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias)
+	if err != nil {
+		runtimeID := missingRuntimeID
+		if bindingVersion == environment.BindingVersion && environment.RuntimeID != "" {
+			runtimeID = environment.RuntimeID
+		}
+		pending := runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID)
+		return errors.Join(err, store.SetRuntimeCleanupPending(ctx, pending, true))
+	}
+	if found && (container.BindingVersion != bindingVersion || container.NetworkID != networkID) {
+		pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID)
+		return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
+	}
+	if found {
+		environment = runtimeCleanupGeneration(environment, bindingVersion, container.ID, networkID)
+	} else {
+		environment = runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID)
+	}
+	_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
+	return cleanupErr
+}
+
 // gatewayFailure 将网关错误转换为对调用方可读的失败；4xx 保留原状态，其余一律 502。
 type gatewayFailure struct {
 	status  int
@@ -200,6 +289,7 @@ type containerStatus struct {
 	Endpoint       string `json:"endpoint"`
 	BindingVersion int64  `json:"binding_version"`
 	NetworkExitID  string `json:"network_exit_id"`
+	NetworkID      string `json:"network_id"`
 	ProxyReady     bool   `json:"proxy_ready"`
 }
 
@@ -216,7 +306,41 @@ type envView struct {
 }
 
 func containerMatchesBinding(container containerStatus, environment hub.EnvironmentContext) bool {
-	return container.BindingVersion == environment.BindingVersion && container.NetworkExitID == environment.Exit.ID
+	if !validCreatedRuntime(container, environment, container.State == "running") || container.BindingVersion != environment.BindingVersion ||
+		(container.NetworkExitID != environment.Exit.ID && (container.State == "running" || container.NetworkExitID != "")) {
+		return false
+	}
+	return true
+}
+
+func validCreatedRuntime(created containerStatus, environment hub.EnvironmentContext, running bool) bool {
+	if !gatewayGenerationIDPattern.MatchString(created.ID) ||
+		(created.NetworkID != "" && !gatewayGenerationIDPattern.MatchString(created.NetworkID)) {
+		return false
+	}
+	if !running {
+		return true
+	}
+	if created.NetworkID == "" {
+		return false
+	}
+	if environment.RuntimeInstanceID != "" || environment.RuntimeID != "" {
+		return environment.RuntimeInstanceID != "" && environment.RuntimeID == created.ID &&
+			environment.RuntimeNetworkID != "" && created.NetworkID == environment.RuntimeNetworkID
+	}
+	return environment.RuntimeNetworkID == "" || created.NetworkID == environment.RuntimeNetworkID
+}
+
+func accountRunnable(environment hub.EnvironmentContext) bool {
+	return environment.AccountStatus == "active" && environment.AuthorizationStatus == "authorized"
+}
+
+func releaseRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error {
+	err := store.ReleaseRuntime(ctx, environment)
+	if errors.Is(err, hub.ErrConflict) {
+		return nil
+	}
+	return err
 }
 
 func registerHub(app *fiber.App, store hubStore) {
@@ -528,6 +652,18 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network
 			continue
 		}
 		container, found := containers[env.Gateway][env.Alias]
+		if !accountRunnable(environment) {
+			if found && container.State == "running" {
+				if err := stopEnvironmentRuntime(ctx, store, environment); err != nil {
+					return err
+				}
+			} else if gatewayRead[env.Gateway] {
+				if err := releaseRuntime(ctx, store, environment); err != nil {
+					return err
+				}
+			}
+			continue
+		}
 		if found && container.State == "running" {
 			if environment.Exit.ID == "" {
 				continue
@@ -541,6 +677,16 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network
 			}
 			ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, container)
 			if auditRecovery {
+				current, contextErr := store.GetEnvironmentContext(ctx, environment.Alias)
+				if contextErr == nil {
+					action.RuntimeInstanceID, action.BindingVersion, action.NetworkExitID =
+						current.RuntimeInstanceID, current.BindingVersion, current.Exit.ID
+				} else {
+					action.RuntimeInstanceID = ""
+					action.BindingVersion = 0
+					action.NetworkExitID = ""
+					restoreErr = errors.Join(restoreErr, contextErr)
+				}
 				action.Outcome, action.ReasonCode = "succeeded", "runtime_reconciled"
 				if !ready {
 					action.Outcome, action.ReasonCode = "failed", "runtime_unavailable"
@@ -556,7 +702,7 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network
 				return restoreErr
 			}
 		} else if gatewayRead[env.Gateway] {
-			if err := store.ReleaseRuntime(ctx, env.Alias); err != nil {
+			if err := releaseRuntime(ctx, store, environment); err != nil {
 				return err
 			}
 		}
@@ -594,7 +740,7 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE
 		return false, err
 	}
 	if containerMatchesBinding(container, environment) && container.ProxyReady {
-		_, err := store.ActivateRuntime(ctx, environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID)
+		_, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID)
 		return err == nil, err
 	}
 	networkExit, err := gatewayNetworkExitFor(access, resolve)
@@ -603,12 +749,12 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE
 	}
 	if containerMatchesBinding(container, environment) {
 		status, _, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/proxy",
-			gatewayProxyPayload(environment, networkExit), 30*time.Second)
+			gatewayProxyPayload(environment, container.ID, container.NetworkID, networkExit), 30*time.Second)
 		if callErr == nil && status == http.StatusNoContent {
 			container.ProxyReady = true
 		}
 		if container.ProxyReady {
-			_, err := store.ActivateRuntime(ctx, environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID)
+			_, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID)
 			return err == nil, err
 		}
 	}
@@ -616,28 +762,45 @@ func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkE
 	if removeErr != nil {
 		return false, removeErr
 	}
+	environment, err = store.GetEnvironmentContext(ctx, environment.Alias)
+	if err != nil {
+		return false, err
+	}
 	imageRef, err := store.ImageRef(ctx, environment.ImageVersion)
 	if err != nil {
 		return false, err
 	}
 	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
-		gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout)
+		gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
 	if callErr != nil {
-		return false, gatewayUnreachable(callErr)
+		return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, body))
 	}
 	if status != http.StatusCreated {
-		return false, gatewayRejected(status, body)
+		return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, body))
 	}
-	var created struct {
-		ID string `json:"id"`
+	var created containerStatus
+	if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
+		return false, errors.Join(errors.New("gateway returned an invalid runtime generation"),
+			reconcileGatewayCreate(ctx, store, target, environment, body))
 	}
-	if json.Unmarshal(body, &created) != nil || created.ID == "" {
-		return false, errors.New("gateway returned an invalid runtime id")
-	}
-	_, err = store.ActivateRuntime(ctx, environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID)
+	_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID)
 	return err == nil, err
 }
 
+func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string) (hub.EnvironmentContext, error) {
+	if !validCreatedRuntime(containerStatus{ID: runtimeID, NetworkID: networkID}, environment, true) {
+		return hub.EnvironmentContext{}, hub.ErrConflict
+	}
+	current, err := store.ActivateRuntime(ctx, environment.Alias, runtimeID, environment.BindingVersion, environment.Exit.ID, networkID)
+	if err == nil {
+		return current, nil
+	}
+	cleanup := environment
+	cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtimeID, networkID
+	_, cleanupErr := removeGatewayRuntime(ctx, store, target, cleanup)
+	return hub.EnvironmentContext{}, errors.Join(err, cleanupErr)
+}
+
 func discardRuntime(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error {
 	target, targetErr := store.GetGateway(ctx, environment.Gateway)
 	if targetErr != nil {
@@ -686,7 +849,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
 		}
 		finish := func(outcome, reason string, current hub.EnvironmentContext) error {
 			action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
-			action.BindingVersion = current.BindingVersion
+			action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
 			return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
 		}
 		access, reason, err := verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID)
@@ -709,6 +872,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
 				return hubError(c, err)
 			}
 		}
+		running := accountRunnable(environment)
 		imageRef, err := store.ImageRef(c.Context(), env.ImageVersion)
 		if err != nil {
 			_ = finish("failed", "image_unavailable", environment)
@@ -725,6 +889,24 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
 				_ = finish("unknown", "gateway_result_unknown", environment)
 				return hubError(c, gatewayUnreachable(reconcileErr))
 			}
+			if found && !running {
+				if !containerMatchesBinding(container, environment) {
+					_ = finish("failed", "runtime_unavailable", environment)
+					return hubError(c, hub.ErrConflict)
+				}
+				if container.State == "running" {
+					stopped := environment
+					stopped.RuntimeID, stopped.RuntimeNetworkID = container.ID, container.NetworkID
+					if err := stopEnvironmentRuntime(c.Context(), store, stopped); err != nil {
+						_ = finish("unknown", "cleanup_result_unknown", environment)
+						return hubError(c, err)
+					}
+				}
+				if err := finish("succeeded", "environment_reused", environment); err != nil {
+					return hubError(c, err)
+				}
+				return c.JSON(map[string]string{"alias": env.Alias})
+			}
 			if found && container.State == "running" {
 				ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, container)
 				if restoreErr != nil {
@@ -746,51 +928,37 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
 				return c.JSON(map[string]string{"alias": env.Alias})
 			}
 		}
-		status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout)
-		if callErr != nil || status >= http.StatusInternalServerError {
+		status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
+			gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
+		if callErr != nil || status != http.StatusCreated {
 			createErr := gatewayRejected(status, body)
 			if callErr != nil {
 				createErr = gatewayUnreachable(callErr)
 			}
-			container, exists, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, env.Alias)
-			if reconcileErr == nil && exists && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) {
-				environment, err = store.ActivateRuntime(c.Context(), env.Alias, container.ID, environment.BindingVersion, environment.Exit.ID)
-				if err != nil {
-					_ = finish("failed", "runtime_persistence_failed", environment)
-					return hubError(c, err)
-				}
-				if err := finish("succeeded", "gateway_reconciled", environment); err != nil {
-					return hubError(c, err)
-				}
-				return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias})
-			}
+			reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
 			if reconcileErr != nil {
 				_ = finish("unknown", "gateway_result_unknown", environment)
-				return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf(
-					"gateway create result unknown; environment retained for reconciliation: %v", createErr)})
+				return hubError(c, errors.Join(createErr, reconcileErr))
 			}
 			_ = finish("failed", "gateway_create_failed", environment)
 			return hubError(c, createErr)
 		}
-		if status != http.StatusCreated {
-			_ = finish("failed", "gateway_rejected", environment)
-			return hubError(c, gatewayRejected(status, body))
-		}
-		var createdRuntime struct {
-			ID string `json:"id"`
-		}
-		if json.Unmarshal(body, &createdRuntime) != nil || createdRuntime.ID == "" {
+		var createdRuntime containerStatus
+		if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
+			reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
 			_ = finish("unknown", "gateway_result_unknown", environment)
-			return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway create result unknown; environment retained for reconciliation"})
+			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway,
+				message: "gateway create result unknown; environment retained for reconciliation"}, reconcileErr))
 		}
-		environment, err = store.ActivateRuntime(c.Context(), env.Alias, createdRuntime.ID, environment.BindingVersion, environment.Exit.ID)
-		if err != nil {
-			_, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment)
-			if cleanupErr != nil {
-				_ = finish("unknown", "cleanup_result_unknown", environment)
-			} else {
-				_ = finish("failed", "runtime_persistence_failed", environment)
+		if !running {
+			if err := finish("succeeded", "environment_created", environment); err != nil {
+				return hubError(c, err)
 			}
+			return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias})
+		}
+		environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID)
+		if err != nil {
+			_ = finish("failed", "runtime_persistence_failed", environment)
 			return hubError(c, err)
 		}
 		if err := finish("succeeded", "environment_created", environment); err != nil {
@@ -821,69 +989,120 @@ func lifecycleAction(store hubStore, probe networkExitProbe, resolve func(hub.Ne
 		return hubError(c, err)
 	}
 	actionName := strings.Clone(c.Params("action"))
-	action := actionForEnvironment(actionName, environment)
-	if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
-		return hubError(c, err)
-	}
-	finish := func(outcome, reason string, current hub.EnvironmentContext) error {
-		action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
-		action.BindingVersion = current.BindingVersion
-		return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
-	}
 	if actionName == "start" {
-		return startBrowser(store, probe, resolve, c, environment, finish)
-	}
-	gateway, err := store.GetGateway(c.Context(), environment.Gateway)
-	if err != nil {
-		_ = finish("failed", "gateway_unavailable", environment)
-		return hubError(c, err)
-	}
-	if environment.RuntimeCleanupPending {
-		if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil {
-			_ = finish("unknown", "cleanup_result_unknown", environment)
-			return hubError(c, cleanupErr)
+		if !accountRunnable(environment) {
+			return hubError(c, hub.ErrConflict)
 		}
-		if err := finish("succeeded", "environment_stopped", environment); err != nil {
+		action := actionForEnvironment(actionName, environment)
+		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
 			return hubError(c, err)
 		}
-		return c.SendStatus(fiber.StatusNoContent)
-	}
-	status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/"+actionName, nil, 30*time.Second)
-	if callErr != nil {
-		container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias)
-		if reconcileErr != nil {
-			_ = finish("unknown", "gateway_result_unknown", environment)
-			return hubError(c, gatewayUnreachable(callErr))
+		finish := func(outcome, reason string, current hub.EnvironmentContext) error {
+			action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
+			action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
+			return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
 		}
-		if !found || container.State != "running" {
-			if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil {
-				_ = finish("failed", "runtime_release_failed", environment)
-				return hubError(c, err)
-			}
-			if err := finish("succeeded", "gateway_reconciled", environment); err != nil {
-				return hubError(c, err)
-			}
-			return c.SendStatus(fiber.StatusNoContent)
-		}
-		_ = finish("failed", "gateway_action_failed", environment)
-		return hubError(c, gatewayUnreachable(callErr))
+		return startBrowser(store, probe, resolve, c, environment, finish)
 	}
-	if status != http.StatusNoContent && status != http.StatusNotModified {
-		_ = finish("failed", "gateway_rejected", environment)
-		return hubError(c, gatewayRejected(status, body))
-	}
-	if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil {
-		_ = finish("failed", "runtime_release_failed", environment)
-		return hubError(c, err)
-	}
-	if err := finish("succeeded", "environment_stopped", environment); err != nil {
+	if err := stopEnvironmentRuntime(c.Context(), store, environment); err != nil {
 		return hubError(c, err)
 	}
 	return c.SendStatus(fiber.StatusNoContent)
 }
 
+func stopAccountRuntime(ctx context.Context, store runtimeStopStore, accountID string) error {
+	environment, err := store.GetEnvironmentContextForAccount(ctx, accountID)
+	if errors.Is(err, hub.ErrNotFound) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	return stopEnvironmentRuntime(ctx, store, environment)
+}
+
+func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error {
+	action := actionForEnvironment("stop", environment)
+	if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
+		return err
+	}
+	finish := func(outcome, reason string) error {
+		action.Outcome, action.ReasonCode = outcome, reason
+		return store.AppendEnvironmentAction(ctx, "environment_action_finished", action)
+	}
+	gateway, err := store.GetGateway(ctx, environment.Gateway)
+	if err != nil {
+		_ = finish("failed", "gateway_unavailable")
+		return err
+	}
+	if environment.RuntimeCleanupPending {
+		if _, cleanupErr := removeGatewayRuntime(ctx, store, gateway, environment); cleanupErr != nil {
+			_ = finish("unknown", "cleanup_result_unknown")
+			return cleanupErr
+		}
+		return finish("succeeded", "environment_stopped")
+	}
+	if environment.RuntimeID == "" {
+		container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias)
+		if reconcileErr != nil {
+			_ = finish("unknown", "gateway_result_unknown")
+			return gatewayUnreachable(reconcileErr)
+		}
+		if !found {
+			return finish("succeeded", "environment_stopped")
+		}
+		if container.BindingVersion != environment.BindingVersion {
+			_ = finish("failed", "runtime_generation_conflict")
+			return hub.ErrConflict
+		}
+		environment = runtimeCleanupGeneration(environment, environment.BindingVersion, container.ID, container.NetworkID)
+	} else {
+		environment = runtimeCleanupGeneration(environment, environment.BindingVersion, environment.RuntimeID)
+	}
+	if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
+		_ = finish("failed", "runtime_release_failed")
+		return err
+	}
+	status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop",
+		gatewayGenerationPayload(environment), 30*time.Second)
+	if callErr != nil || status >= http.StatusInternalServerError {
+		container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias)
+		if reconcileErr != nil {
+			_ = finish("unknown", "gateway_result_unknown")
+			if callErr != nil {
+				return gatewayUnreachable(callErr)
+			}
+			return gatewayRejected(status, body)
+		}
+		if !found || container.State != "running" {
+			if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
+				_ = finish("failed", "runtime_release_failed")
+				return err
+			}
+			return finish("succeeded", "gateway_reconciled")
+		}
+		_ = finish("failed", "gateway_action_failed")
+		if callErr != nil {
+			return gatewayUnreachable(callErr)
+		}
+		return gatewayRejected(status, body)
+	}
+	if status != http.StatusNoContent && status != http.StatusNotModified && status != http.StatusNotFound {
+		_ = finish("failed", "gateway_rejected")
+		return gatewayRejected(status, body)
+	}
+	if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
+		_ = finish("failed", "runtime_release_failed")
+		return err
+	}
+	return finish("succeeded", "environment_stopped")
+}
+
 func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx,
 	environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error {
+	if !accountRunnable(environment) {
+		return hubError(c, hub.ErrConflict)
+	}
 	access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID)
 	if err != nil {
 		if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil {
@@ -907,6 +1126,9 @@ func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netwo
 		if err != nil {
 			return hubError(c, err)
 		}
+		if !accountRunnable(environment) {
+			return hubError(c, hub.ErrConflict)
+		}
 	}
 	imageRef, err := store.ImageRef(c.Context(), environment.ImageVersion)
 	if err != nil {
@@ -932,7 +1154,7 @@ func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netwo
 		return hubError(c, gatewayUnreachable(err))
 	}
 	if found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) {
-		environment, err = store.ActivateRuntime(c.Context(), environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID)
+		environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, container.ID, container.NetworkID)
 		if err != nil {
 			_ = finish("failed", "runtime_persistence_failed", environment)
 			return hubError(c, err)
@@ -947,40 +1169,31 @@ func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netwo
 			_ = finish("unknown", "gateway_result_unknown", environment)
 			return hubError(c, removeErr)
 		}
-	} else if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil {
+	} else if err := releaseRuntime(c.Context(), store, environment); err != nil {
 		_ = finish("failed", "runtime_release_failed", environment)
 		return hubError(c, err)
 	}
 	status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
 		gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout)
-	if callErr != nil || status >= http.StatusInternalServerError {
-		container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias)
+	if callErr != nil || status != http.StatusCreated {
+		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
 		if reconcileErr != nil {
 			_ = finish("unknown", "gateway_result_unknown", environment)
-			return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"})
+			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr))
 		}
-		if !found || container.State != "running" || !container.ProxyReady || !containerMatchesBinding(container, environment) {
-			_ = finish("failed", "gateway_create_failed", environment)
-			if callErr != nil {
-				return hubError(c, gatewayUnreachable(callErr))
-			}
-			return hubError(c, gatewayRejected(status, body))
+		_ = finish("failed", "gateway_create_failed", environment)
+		if callErr != nil {
+			return hubError(c, gatewayUnreachable(callErr))
 		}
-		environment, err = store.ActivateRuntime(c.Context(), environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID)
-	} else {
-		if status != http.StatusCreated {
-			_ = finish("failed", "gateway_rejected", environment)
-			return hubError(c, gatewayRejected(status, body))
-		}
-		var created struct {
-			ID string `json:"id"`
-		}
-		if json.Unmarshal(body, &created) != nil || created.ID == "" {
-			_ = finish("unknown", "gateway_result_unknown", environment)
-			return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"})
-		}
-		environment, err = store.ActivateRuntime(c.Context(), environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID)
+		return hubError(c, gatewayRejected(status, body))
 	}
+	var created containerStatus
+	if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
+		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
+		_ = finish("unknown", "gateway_result_unknown", environment)
+		return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr))
+	}
+	environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, created.ID, created.NetworkID)
 	if err != nil {
 		_ = finish("failed", "runtime_persistence_failed", environment)
 		return hubError(c, err)
@@ -1028,7 +1241,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Net
 	}
 	finish := func(outcome, reason string, current hub.EnvironmentContext) error {
 		action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
-		action.BindingVersion = current.BindingVersion
+		action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
 		return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
 	}
 	access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID)
@@ -1051,6 +1264,7 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Net
 			return hubError(c, err)
 		}
 	}
+	running := accountRunnable(environment)
 	networkExit, err := gatewayNetworkExitFor(access, resolve)
 	if err != nil {
 		_ = finish("failed", "credential_unavailable", environment)
@@ -1070,42 +1284,36 @@ func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Net
 		_ = finish("failed", "persistence_failed", environment)
 		return hubError(c, err)
 	}
-	status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout)
-	var createdRuntime struct {
-		ID string `json:"id"`
-	}
-	if callErr != nil || status >= http.StatusInternalServerError {
-		container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias)
+	running = accountRunnable(environment)
+	status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
+		gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
+	var createdRuntime containerStatus
+	if callErr != nil || status != http.StatusCreated {
+		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
 		if reconcileErr != nil {
 			_ = finish("unknown", "gateway_result_unknown", environment)
-			return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"})
+			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr))
 		}
-		if !found || container.State != "running" || !container.ProxyReady || !containerMatchesBinding(container, environment) {
-			_ = finish("failed", "gateway_create_failed", environment)
-			if callErr != nil {
-				return hubError(c, gatewayUnreachable(callErr))
-			}
-			return hubError(c, gatewayRejected(status, body))
-		}
-		createdRuntime.ID = container.ID
-	} else {
-		if status != http.StatusCreated {
-			_ = finish("failed", "gateway_rejected", environment)
-			return hubError(c, gatewayRejected(status, body))
-		}
-		if json.Unmarshal(body, &createdRuntime) != nil || createdRuntime.ID == "" {
-			_ = finish("unknown", "gateway_result_unknown", environment)
-			return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"})
+		_ = finish("failed", "gateway_create_failed", environment)
+		if callErr != nil {
+			return hubError(c, gatewayUnreachable(callErr))
 		}
+		return hubError(c, gatewayRejected(status, body))
 	}
-	environment, err = store.ActivateRuntime(c.Context(), environment.Alias, createdRuntime.ID, environment.BindingVersion, environment.Exit.ID)
-	if err != nil {
-		_, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment)
-		if cleanupErr != nil {
-			_ = finish("unknown", "cleanup_result_unknown", environment)
-		} else {
-			_ = finish("failed", "runtime_persistence_failed", environment)
+	if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
+		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, body)
+		_ = finish("unknown", "gateway_result_unknown", environment)
+		return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr))
+	}
+	if !running {
+		if err := finish("succeeded", "environment_upgraded", environment); err != nil {
+			return hubError(c, err)
 		}
+		return c.SendStatus(fiber.StatusNoContent)
+	}
+	environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID)
+	if err != nil {
+		_ = finish("failed", "runtime_persistence_failed", environment)
 		return hubError(c, err)
 	}
 	if err := finish("succeeded", "environment_upgraded", environment); err != nil {
@@ -1134,41 +1342,39 @@ func prepareRuntimeCreate(ctx context.Context, store hubStore, resolve func(hub.
 
 func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (containerStatus, error) {
 	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
-		gatewayCreatePayload(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout)
+		gatewayCreatePayloadForAccount(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout)
+	unknown := containerStatus{NetworkID: gatewayNetworkID(body)}
 	if callErr == nil && status == http.StatusCreated {
 		var created containerStatus
-		if json.Unmarshal(body, &created) == nil && created.ID != "" {
-			created.Alias, created.State = environment.Alias, "running"
-			created.BindingVersion, created.NetworkExitID, created.ProxyReady = environment.BindingVersion, environment.Exit.ID, true
+		if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, accountRunnable(environment)) {
+			created.Alias, created.BindingVersion = environment.Alias, environment.BindingVersion
+			if accountRunnable(environment) {
+				created.State, created.NetworkExitID, created.ProxyReady = "running", environment.Exit.ID, true
+			} else {
+				created.State = "exited"
+			}
 			return created, nil
 		}
 	}
-	container, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias)
-	if reconcileErr == nil && found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) {
-		return container, nil
-	}
 	if callErr != nil {
-		return containerStatus{}, gatewayUnreachable(callErr)
+		return unknown, gatewayUnreachable(callErr)
 	}
 	if status != http.StatusCreated {
-		return containerStatus{}, gatewayRejected(status, body)
+		return unknown, gatewayRejected(status, body)
 	}
-	if reconcileErr != nil {
-		return containerStatus{}, gatewayUnreachable(reconcileErr)
-	}
-	return containerStatus{}, errors.New("gateway returned an invalid runtime id")
+	return unknown, errors.New("gateway returned an invalid runtime generation")
 }
 
 func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, imageRef string) error {
 	payload := gatewayCreatePayload(environment, imageRef, gatewayNetworkExit{})
-	payload["stopped"] = true
+	payload["network_exit_id"], payload["stopped"] = "", true
 	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", payload, gatewayLongTimeout)
 	if callErr == nil && status == http.StatusCreated {
-		return nil
-	}
-	container, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias)
-	if reconcileErr == nil && found && container.State != "running" && containerMatchesBinding(container, environment) {
-		return nil
+		var created containerStatus
+		if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, false) {
+			return nil
+		}
+		return errors.New("gateway returned an invalid stopped runtime generation")
 	}
 	if callErr != nil {
 		return gatewayUnreachable(callErr)
@@ -1176,23 +1382,54 @@ func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, enviro
 	if status != http.StatusCreated {
 		return gatewayRejected(status, body)
 	}
-	if reconcileErr != nil {
-		return gatewayUnreachable(reconcileErr)
-	}
 	return errors.New("gateway did not preserve the stopped runtime")
 }
 
-func removeGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) {
+func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) {
 	removed := environment.RuntimeCleanupPending
+	if environment.RuntimeCleanupPending &&
+		(environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" ||
+			(environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "")) {
+		return false, hub.ErrReconcileRequired
+	}
 	if !environment.RuntimeCleanupPending {
-		if err := store.SetRuntimeCleanupPending(ctx, environment.Alias, true); err != nil {
+		container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias)
+		if err != nil {
+			return false, err
+		}
+		bindingVersion, runtimeID, networkID := environment.BindingVersion, environment.RuntimeID, environment.RuntimeNetworkID
+		if environment.RuntimeCleanupBindingVersion > 0 {
+			bindingVersion, runtimeID, networkID = environment.RuntimeCleanupBindingVersion,
+				environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID
+		}
+		if found {
+			if container.BindingVersion != bindingVersion || runtimeID == missingRuntimeID ||
+				(runtimeID != "" && runtimeID != container.ID) || (runtimeID == "" && networkID == "" && container.State == "running") {
+				return false, hub.ErrConflict
+			}
+			runtimeID = container.ID
+			if networkID != "" {
+				if container.NetworkID != "" && container.NetworkID != networkID {
+					return false, hub.ErrConflict
+				}
+			} else {
+				networkID = container.NetworkID
+			}
+		} else if environment.RuntimeID == "" {
+			runtimeID = missingRuntimeID
+		}
+		environment = runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID)
+	}
+	if !environment.RuntimeCleanupPending {
+		if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
 			return false, err
 		}
 	}
 	for attempt := 0; attempt < 2; attempt++ {
-		status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias, nil, 30*time.Second)
+		status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias,
+			gatewayGenerationPayload(environment), 30*time.Second)
 		if callErr == nil && (status == http.StatusNoContent || status == http.StatusNotFound) {
-			return true, store.SetRuntimeCleanupPending(ctx, environment.Alias, false)
+			return true, store.SetRuntimeCleanupPending(ctx, environment, false)
 		}
 		if callErr == nil && status == http.StatusAccepted {
 			removed = true
@@ -1226,10 +1463,10 @@ func removeGatewayRuntime(ctx context.Context, store hubStore, target hub.Gatewa
 func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error),
 	target hub.Gateway, environment hub.EnvironmentContext, previous containerStatus, found bool, prepared *runtimeCreateSpec) (bool, error) {
 	if !found {
-		return true, store.ReleaseRuntime(ctx, environment.Alias)
+		return true, releaseRuntime(ctx, store, environment)
 	}
 	if environment.Exit.ID == "" {
-		if err := store.ReleaseRuntime(ctx, environment.Alias); err != nil {
+		if err := releaseRuntime(ctx, store, environment); err != nil {
 			return false, err
 		}
 		if previous.State == "running" {
@@ -1264,18 +1501,25 @@ func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.
 			return false, err
 		}
 	}
-	if err := store.ReleaseRuntime(ctx, environment.Alias); err != nil {
+	if err := releaseRuntime(ctx, store, environment); err != nil {
+		return false, err
+	}
+	cleanupContext, err := store.GetEnvironmentContext(ctx, environment.Alias)
+	if err != nil {
 		return false, err
 	}
 	created, err := createGatewayRuntime(ctx, target, environment, spec)
 	if err != nil {
-		_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
+		cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID)
 		return false, errors.Join(err, cleanupErr)
 	}
-	if previous.State == "running" {
-		_, err = store.ActivateRuntime(ctx, environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID)
-	} else {
-		status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop", nil, 30*time.Second)
+	if previous.State == "running" && accountRunnable(environment) {
+		_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID)
+	} else if accountRunnable(environment) {
+		stopped := environment
+		stopped.RuntimeID, stopped.RuntimeNetworkID = created.ID, created.NetworkID
+		status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop",
+			gatewayGenerationPayload(stopped), 30*time.Second)
 		if callErr != nil {
 			err = gatewayUnreachable(callErr)
 		} else if status != http.StatusNoContent {
@@ -1400,9 +1644,14 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
 	if wasRunning {
 		candidateRuntime, err = createGatewayRuntime(c.Context(), target, candidate, nextSpec)
 		if err != nil {
-			_, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before)
+			cleanupContext, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
+			if contextErr != nil {
+				_ = finish("unknown", "gateway_result_unknown", before)
+				return hubError(c, errors.Join(err, contextErr))
+			}
+			cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID)
 			if cleanupErr != nil {
-				_ = finish("unknown", "rebind_recovery_failed", before)
+				_ = finish("unknown", "gateway_result_unknown", before)
 				return hubError(c, errors.Join(err, cleanupErr))
 			}
 			current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
@@ -1419,10 +1668,20 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw
 			return hubError(c, err)
 		}
 	}
-	after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, candidateRuntime.ID, before.BindingVersion)
+	runtimeID := ""
+	if accountRunnable(candidate) {
+		runtimeID = candidateRuntime.ID
+	}
+	after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, runtimeID, before.BindingVersion, candidateRuntime.NetworkID)
 	if err != nil {
 		if candidateRuntime.ID != "" {
-			_, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before)
+			cleanup, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
+			if contextErr != nil || cleanup.BindingVersion != before.BindingVersion || cleanup.Exit.ID != before.Exit.ID {
+				_ = finish("unknown", "rebind_recovery_failed", before)
+				return hubError(c, errors.Join(err, contextErr, hub.ErrConflict))
+			}
+			cleanup.RuntimeID, cleanup.RuntimeNetworkID = candidateRuntime.ID, candidateRuntime.NetworkID
+			_, cleanupErr := removeGatewayRuntime(c.Context(), store, target, cleanup)
 			if cleanupErr != nil {
 				_ = finish("unknown", "rebind_recovery_failed", before)
 				return hubError(c, errors.Join(err, cleanupErr))
@@ -1513,6 +1772,8 @@ func hubError(c fiber.Ctx, err error) error {
 	switch {
 	case errors.Is(err, hub.ErrInvalid):
 		status, message = fiber.StatusBadRequest, hub.ErrInvalid.Error()
+	case errors.Is(err, hub.ErrReconcileRequired):
+		status, message = fiber.StatusConflict, hub.ErrReconcileRequired.Error()
 	case errors.Is(err, hub.ErrConflict):
 		status, message = fiber.StatusConflict, hub.ErrConflict.Error()
 	case errors.Is(err, hub.ErrNotFound):

cmd/control-plane/main.go

@@ -176,7 +176,7 @@ func newHandlerWithStores(webDirectory string, phaseAStore *phasea.Store, hubSto
 		registerHub(app, hubStore)
 	}
 	if phaseAStore != nil {
-		registerPhaseA(app, phaseAStore)
+		registerPhaseA(app, phaseAStore, hubStore)
 	}
 	app.Get("/*", spaHandler(webDirectory))
 	return app

cmd/control-plane/phasea.go

@@ -22,7 +22,7 @@ type accountRequest struct {
 	} `json:"credential_reference"`
 }
 
-func registerPhaseA(app *fiber.App, store *phasea.Store) {
+func registerPhaseA(app *fiber.App, store *phasea.Store, runtimeStore runtimeStopStore) {
 	app.Post("/api/phase-a/accounts", func(c fiber.Ctx) error {
 		var input accountRequest
 		if err := decodePhaseA(c, &input); err != nil {
@@ -57,13 +57,22 @@ func registerPhaseA(app *fiber.App, store *phasea.Store) {
 	})
 
 	app.Post("/api/phase-a/accounts/:id/pause", func(c fiber.Ctx) error {
+		runtimeOperations.Lock()
+		defer runtimeOperations.Unlock()
 		if err := store.PauseAccount(c.Context(), c.Params("id")); err != nil {
 			return phaseAError(c, err)
 		}
+		if runtimeStore != nil {
+			if err := stopAccountRuntime(c.Context(), runtimeStore, c.Params("id")); err != nil {
+				return hubError(c, err)
+			}
+		}
 		return c.SendStatus(fiber.StatusNoContent)
 	})
 
 	app.Post("/api/phase-a/accounts/:id/resume", func(c fiber.Ctx) error {
+		runtimeOperations.Lock()
+		defer runtimeOperations.Unlock()
 		if err := store.ResumeAccount(c.Context(), c.Params("id")); err != nil {
 			return phaseAError(c, err)
 		}
@@ -71,26 +80,15 @@ func registerPhaseA(app *fiber.App, store *phasea.Store) {
 	})
 
 	app.Post("/api/phase-a/accounts/:id/revoke", func(c fiber.Ctx) error {
+		runtimeOperations.Lock()
+		defer runtimeOperations.Unlock()
 		if err := store.RevokeAccount(c.Context(), c.Params("id")); err != nil {
 			return phaseAError(c, err)
 		}
-		return c.SendStatus(fiber.StatusNoContent)
-	})
-
-	app.Post("/api/phase-a/runtimes", func(c fiber.Ctx) error {
-		var input phasea.RuntimeBinding
-		if err := decodePhaseA(c, &input); err != nil {
-			return phaseAError(c, err)
-		}
-		if err := store.BindRuntime(c.Context(), input); err != nil {
-			return phaseAError(c, err)
-		}
-		return c.Status(fiber.StatusCreated).JSON(map[string]string{"id": input.ID})
-	})
-
-	app.Delete("/api/phase-a/runtimes/:id", func(c fiber.Ctx) error {
-		if err := store.ReleaseRuntime(c.Context(), c.Params("id")); err != nil {
-			return phaseAError(c, err)
+		if runtimeStore != nil {
+			if err := stopAccountRuntime(c.Context(), runtimeStore, c.Params("id")); err != nil {
+				return hubError(c, err)
+			}
 		}
 		return c.SendStatus(fiber.StatusNoContent)
 	})

cmd/docker-gateway/proxy.go

@@ -25,65 +25,106 @@ type memoryProxyRegistry struct {
 }
 
 type memoryProxy struct {
-	mu       sync.RWMutex
-	exit     gatewayProxyExit
-	bindHost string
-	listener net.Listener
-	server   *http.Server
-	url      string
+	mu             sync.RWMutex
+	exit           gatewayProxyExit
+	bindingVersion int64
+	runtimeID      string
+	networkID      string
+	bindHost       string
+	listener       net.Listener
+	server         *http.Server
+	url            string
 }
 
 func newMemoryProxyRegistry() *memoryProxyRegistry {
 	return &memoryProxyRegistry{proxies: map[string]*memoryProxy{}}
 }
 
-func (registry *memoryProxyRegistry) configure(alias, bindHost string, port int, exit gatewayProxyExit) (string, func(), error) {
+func (registry *memoryProxyRegistry) configure(alias string, bindingVersion int64, bindHost string, port int, exit gatewayProxyExit,
+	networkIDs ...string) (string, func(), error) {
 	registry.mu.Lock()
 	defer registry.mu.Unlock()
+	networkID := ""
+	if len(networkIDs) == 1 {
+		networkID = networkIDs[0]
+	}
 	if proxy := registry.proxies[alias]; proxy != nil {
-		if proxy.bindHost == bindHost && (port == 0 || proxy.listener.Addr().(*net.TCPAddr).Port == port) && proxy.exit == exit {
-			return proxy.url, func() {}, nil
+		if proxy.bindingVersion == bindingVersion && proxy.bindHost == bindHost &&
+			(port == 0 || proxy.listener.Addr().(*net.TCPAddr).Port == port) && proxy.exit == exit && proxy.networkID == networkID {
+			return proxy.url, func() { registry.removeObject(alias, proxy) }, nil
 		}
-		delete(registry.proxies, alias)
-		_ = proxy.server.Close()
+		return "", nil, errGenerationConflict
 	}
 	listener, err := net.Listen("tcp4", net.JoinHostPort(bindHost, strconv.Itoa(port)))
 	if err != nil {
 		return "", nil, err
 	}
 	actualPort := listener.Addr().(*net.TCPAddr).Port
-	proxy := &memoryProxy{exit: exit, bindHost: bindHost, listener: listener, url: "http://" + net.JoinHostPort(browserProxyHost, strconv.Itoa(actualPort))}
+	proxy := &memoryProxy{exit: exit, bindingVersion: bindingVersion, networkID: networkID, bindHost: bindHost, listener: listener,
+		url: "http://" + net.JoinHostPort(browserProxyHost, strconv.Itoa(actualPort))}
 	proxy.server = &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second}
 	registry.proxies[alias] = proxy
 	go func() { _ = proxy.server.Serve(listener) }()
 	undo := func() {
-		registry.mu.Lock()
-		defer registry.mu.Unlock()
-		if registry.proxies[alias] == proxy {
-			delete(registry.proxies, alias)
-			_ = proxy.server.Close()
-		}
+		registry.removeObject(alias, proxy)
 	}
 	return proxy.url, undo, nil
 }
 
-func (registry *memoryProxyRegistry) ready(alias string, port int) bool {
+func (registry *memoryProxyRegistry) removeObject(alias string, proxy *memoryProxy) {
 	registry.mu.Lock()
-	defer registry.mu.Unlock()
-	proxy := registry.proxies[alias]
-	return proxy != nil && proxy.listener.Addr().(*net.TCPAddr).Port == port
-}
-
-func (registry *memoryProxyRegistry) remove(alias string) {
-	registry.mu.Lock()
-	proxy := registry.proxies[alias]
-	delete(registry.proxies, alias)
+	if registry.proxies[alias] == proxy {
+		delete(registry.proxies, alias)
+	} else {
+		proxy = nil
+	}
 	registry.mu.Unlock()
 	if proxy != nil {
 		_ = proxy.server.Close()
 	}
 }
 
+func (registry *memoryProxyRegistry) ready(alias string, port int, runtimeID string, networkIDs ...string) bool {
+	registry.mu.Lock()
+	defer registry.mu.Unlock()
+	proxy := registry.proxies[alias]
+	return proxy != nil && runtimeID != "" && proxy.runtimeID == runtimeID && proxy.listener.Addr().(*net.TCPAddr).Port == port &&
+		(len(networkIDs) == 0 || proxy.networkID == networkIDs[0])
+}
+
+func (registry *memoryProxyRegistry) bind(alias string, bindingVersion int64, proxyURL, runtimeID string, networkIDs ...string) bool {
+	registry.mu.Lock()
+	defer registry.mu.Unlock()
+	proxy := registry.proxies[alias]
+	if proxy == nil || runtimeID == "" || proxy.bindingVersion != bindingVersion || proxy.url != proxyURL {
+		return false
+	}
+	proxy.runtimeID = runtimeID
+	if len(networkIDs) == 1 {
+		proxy.networkID = networkIDs[0]
+	}
+	return true
+}
+
+func (registry *memoryProxyRegistry) remove(alias string, bindingVersion int64, runtimeID string, networkIDs ...string) bool {
+	registry.mu.Lock()
+	proxy := registry.proxies[alias]
+	if proxy != nil && runtimeID != "" && proxy.bindingVersion == bindingVersion && proxy.runtimeID == runtimeID &&
+		(len(networkIDs) == 0 || proxy.networkID == networkIDs[0]) {
+		delete(registry.proxies, alias)
+	} else if proxy != nil {
+		registry.mu.Unlock()
+		return false
+	} else {
+		proxy = nil
+	}
+	registry.mu.Unlock()
+	if proxy != nil {
+		_ = proxy.server.Close()
+	}
+	return true
+}
+
 func (proxy *memoryProxy) ServeHTTP(response http.ResponseWriter, request *http.Request) {
 	if request.Method == http.MethodConnect {
 		proxy.tunnel(response, request)

cmd/docker-gateway/proxy_test.go

@@ -73,7 +73,7 @@ func TestMemoryProxyUsesSOCKS5Credentials(t *testing.T) {
 	host, portText, _ := net.SplitHostPort(listener.Addr().String())
 	port, _ := net.LookupPort("tcp", portText)
 	registry := newMemoryProxyRegistry()
-	proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{
+	proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{
 		Protocol: "socks5", Host: host, Port: port, Username: "operator", Password: "ephemeral",
 	})
 	if err != nil {
@@ -119,7 +119,7 @@ func TestMemoryProxyUsesAbsoluteFormForHTTPUpstream(t *testing.T) {
 	address, _ := url.Parse(upstream.URL)
 	port, _ := strconv.Atoi(address.Port())
 	registry := newMemoryProxyRegistry()
-	proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{
+	proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{
 		Protocol: "http", Host: address.Hostname(), Port: port, Username: "operator", Password: "ephemeral",
 	})
 	if err != nil {
@@ -141,7 +141,7 @@ func TestMemoryProxyUsesAbsoluteFormForHTTPUpstream(t *testing.T) {
 
 func TestMemoryProxyRejectsCrossAliasAddress(t *testing.T) {
 	registry := newMemoryProxyRegistry()
-	proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1})
+	proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -152,3 +152,19 @@ func TestMemoryProxyRejectsCrossAliasAddress(t *testing.T) {
 		t.Fatal("another tenant address could reach account-a proxy")
 	}
 }
+
+func TestMemoryProxyRemoveRequiresMatchingGeneration(t *testing.T) {
+	registry := newMemoryProxyRegistry()
+	proxyURL, cleanup, err := registry.configure("account-a", 2, "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+	if !registry.bind("account-a", 2, proxyURL, "container-c2") {
+		t.Fatal("bind proxy generation")
+	}
+	proxy := registry.proxies["account-a"]
+	if registry.remove("account-a", 2, "") || registry.remove("account-a", 2, "container-c1") || registry.proxies["account-a"] != proxy {
+		t.Fatal("stale generation removed the current proxy")
+	}
+}

compose.yaml

@@ -44,6 +44,8 @@ services:
   docker-gateway:
     build: .
     command: ["/app/docker-gateway"]
+    labels:
+      io.creatorhub.gateway-member: "true"
     environment:
       BROWSER_NETWORK: creatorhub_browser
       GATEWAY_TOKEN: ${GATEWAY_TOKEN:-dev-creatorhub-gateway-token}

internal/hub/environment.go

@@ -51,13 +51,20 @@ type ExitObservation struct {
 
 type EnvironmentContext struct {
 	Env
-	AccountID             string      `json:"account_id"`
-	BindingID             string      `json:"binding_id"`
-	BindingVersion        int64       `json:"binding_version"`
-	RuntimeCleanupPending bool        `json:"runtime_cleanup_pending,omitempty"`
-	Exit                  NetworkExit `json:"network_exit"`
-	RuntimeInstanceID     string      `json:"runtime_instance_id,omitempty"`
-	RuntimeID             string      `json:"runtime_id,omitempty"`
+	AccountID                    string      `json:"account_id"`
+	AccountStatus                string      `json:"account_status"`
+	AuthorizationStatus          string      `json:"authorization_status"`
+	BindingID                    string      `json:"binding_id"`
+	BindingVersion               int64       `json:"binding_version"`
+	RuntimeCleanupPending        bool        `json:"runtime_cleanup_pending,omitempty"`
+	RuntimeCleanupBindingVersion int64       `json:"runtime_cleanup_binding_version,omitempty"`
+	RuntimeCleanupInstanceID     string      `json:"runtime_cleanup_instance_id,omitempty"`
+	RuntimeCleanupRuntimeID      string      `json:"runtime_cleanup_runtime_id,omitempty"`
+	RuntimeCleanupNetworkID      string      `json:"runtime_cleanup_network_id,omitempty"`
+	Exit                         NetworkExit `json:"network_exit"`
+	RuntimeInstanceID            string      `json:"runtime_instance_id,omitempty"`
+	RuntimeID                    string      `json:"runtime_id,omitempty"`
+	RuntimeNetworkID             string      `json:"runtime_network_id,omitempty"`
 }
 
 type EnvironmentAction struct {
@@ -396,40 +403,39 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro
 		return EnvironmentContext{}, errors.New("begin environment context read")
 	}
 	defer tx.Rollback()
-	if _, err := tx.ExecContext(ctx, `
-		UPDATE runtime_instance runtime SET released_at = now()
-		FROM environment_binding binding
-		WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id
-			AND runtime.released_at IS NULL AND runtime.lease_until <= now()`, alias); err != nil {
-		return EnvironmentContext{}, errors.New("expire environment runtime")
-	}
 	var result EnvironmentContext
 	var encoded []byte
 	var expectedIP, observedIP string
 	var checked sql.NullTime
-	var runtimeInstanceID, runtimeID sql.NullString
+	var runtimeInstanceID, runtimeID, runtimeNetworkID, cleanupInstanceID, cleanupRuntimeID, cleanupNetworkID sql.NullString
+	var cleanupBindingVersion sql.NullInt64
 	err = tx.QueryRowContext(ctx, `
 		SELECT environment.alias, environment.name, environment.gateway_name, environment.image_version,
-			environment.fingerprint, environment.created_at, binding.account_id, binding.id, binding.version,
-			binding.runtime_cleanup_pending,
+			environment.fingerprint, environment.created_at, binding.account_id, account.status, account.authorization_status,
+			binding.id, binding.version,
+			binding.runtime_cleanup_pending, binding.runtime_cleanup_binding_version,
+			binding.runtime_cleanup_instance_id, binding.runtime_cleanup_runtime_id, binding.runtime_cleanup_network_id,
 			COALESCE(network.id, ''), COALESCE(network.protocol, ''), COALESCE(network.host, ''), COALESCE(network.port, 0),
 			COALESCE(host(network.expected_public_ip), ''), COALESCE(network.expected_region, ''),
 			COALESCE(host(network.observed_public_ip), ''), COALESCE(network.observed_region, ''),
 			COALESCE(network.health_status, 'unchecked'), COALESCE(network.last_check_reason, ''),
 			COALESCE(network.version, 0), network.last_checked_at,
 			COALESCE(network.created_at, to_timestamp(0)), COALESCE(network.updated_at, to_timestamp(0)),
-			runtime.id, runtime.runtime_id
+			runtime.id, runtime.runtime_id, runtime.network_id
 		FROM browser_env environment
 		JOIN environment_binding binding ON binding.browser_env_alias = environment.alias
+		JOIN social_account account ON account.id = binding.account_id
 		LEFT JOIN network_exit network ON network.id = binding.network_exit_id
 		LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL
 		WHERE environment.alias = $1`, alias).
 		Scan(&result.Alias, &result.Name, &result.Gateway, &result.ImageVersion, &encoded, &result.CreatedAt,
-			&result.AccountID, &result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending,
+			&result.AccountID, &result.AccountStatus, &result.AuthorizationStatus,
+			&result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending, &cleanupBindingVersion,
+			&cleanupInstanceID, &cleanupRuntimeID, &cleanupNetworkID,
 			&result.Exit.ID, &result.Exit.Protocol, &result.Exit.Host, &result.Exit.Port,
 			&expectedIP, &result.Exit.ExpectedRegion, &observedIP, &result.Exit.ObservedRegion,
 			&result.Exit.HealthStatus, &result.Exit.LastCheckReason, &result.Exit.Version, &checked, &result.Exit.CreatedAt, &result.Exit.UpdatedAt,
-			&runtimeInstanceID, &runtimeID)
+			&runtimeInstanceID, &runtimeID, &runtimeNetworkID)
 	if err != nil {
 		return EnvironmentContext{}, rowError(err)
 	}
@@ -442,13 +448,30 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro
 	if checked.Valid {
 		result.Exit.LastCheckedAt = &checked.Time
 	}
-	result.RuntimeInstanceID, result.RuntimeID = runtimeInstanceID.String, runtimeID.String
+	result.RuntimeInstanceID, result.RuntimeID, result.RuntimeNetworkID = runtimeInstanceID.String, runtimeID.String, runtimeNetworkID.String
+	if result.RuntimeCleanupPending {
+		result.RuntimeCleanupBindingVersion = cleanupBindingVersion.Int64
+		result.RuntimeCleanupInstanceID, result.RuntimeCleanupRuntimeID = cleanupInstanceID.String, cleanupRuntimeID.String
+		result.RuntimeCleanupNetworkID = cleanupNetworkID.String
+	}
 	if err := commitHub(tx); err != nil {
 		return EnvironmentContext{}, err
 	}
 	return result, nil
 }
 
+func (s *Store) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (EnvironmentContext, error) {
+	if !aliasPattern.MatchString(accountID) {
+		return EnvironmentContext{}, ErrInvalid
+	}
+	var alias string
+	if err := s.db.QueryRowContext(ctx, `
+		SELECT browser_env_alias FROM environment_binding WHERE account_id = $1`, accountID).Scan(&alias); err != nil {
+		return EnvironmentContext{}, rowError(err)
+	}
+	return s.GetEnvironmentContext(ctx, alias)
+}
+
 func validateEnvironmentRebind(ctx context.Context, tx *sql.Tx, alias, exitID string, expectedBindingVersion int64) (string, string, error) {
 	var accountID, bindingID string
 	var bindingVersion int64
@@ -506,9 +529,14 @@ func (s *Store) ValidateEnvironmentRebind(ctx context.Context, alias, exitID str
 	return nil
 }
 
-func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (EnvironmentContext, error) {
+func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkIDs ...string) (EnvironmentContext, error) {
+	networkID := ""
+	if len(networkIDs) == 1 {
+		networkID = networkIDs[0]
+	}
 	if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(exitID) ||
-		(runtimeID != "" && !exitIDPattern.MatchString(runtimeID)) || expectedBindingVersion < 1 {
+		(runtimeID != "" && !exitIDPattern.MatchString(runtimeID)) || (networkID != "" && !exitIDPattern.MatchString(networkID)) ||
+		len(networkIDs) > 1 || expectedBindingVersion < 1 {
 		return EnvironmentContext{}, ErrInvalid
 	}
 	tx, err := s.db.BeginTx(ctx, nil)
@@ -528,8 +556,9 @@ func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID
 	}
 	if runtimeID != "" {
 		if _, err := tx.ExecContext(ctx, `
-			INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until)
-			VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil {
+			INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, lease_until)
+			VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), now() + interval '1 minute')`,
+			"runtime-"+newHubID(), accountID, bindingID, expectedBindingVersion+1, runtimeID, networkID); err != nil {
 			return EnvironmentContext{}, publicDatabaseError(err)
 		}
 	}
@@ -539,8 +568,13 @@ func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID
 	return s.GetEnvironmentContext(ctx, alias)
 }
 
-func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (EnvironmentContext, error) {
-	if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || bindingVersion < 1 || !exitIDPattern.MatchString(exitID) {
+func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkIDs ...string) (EnvironmentContext, error) {
+	networkID := ""
+	if len(networkIDs) == 1 {
+		networkID = networkIDs[0]
+	}
+	if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || bindingVersion < 1 || !exitIDPattern.MatchString(exitID) ||
+		!exitIDPattern.MatchString(networkID) || len(networkIDs) != 1 {
 		return EnvironmentContext{}, ErrInvalid
 	}
 	tx, err := s.db.BeginTx(ctx, nil)
@@ -548,18 +582,21 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi
 		return EnvironmentContext{}, errors.New("begin runtime activation")
 	}
 	defer tx.Rollback()
-	var accountID, bindingID, currentExitID string
+	var accountID, bindingID, currentExitID, accountStatus, authorizationStatus string
 	var currentBindingVersion int64
 	var cleanupPending bool
 	err = tx.QueryRowContext(ctx, `
-		SELECT binding.account_id, binding.id, binding.version, COALESCE(binding.network_exit_id, ''), binding.runtime_cleanup_pending
+		SELECT binding.account_id, binding.id, binding.version, COALESCE(binding.network_exit_id, ''), binding.runtime_cleanup_pending,
+			account.status, account.authorization_status
 		FROM environment_binding binding
-		WHERE binding.browser_env_alias = $1 FOR UPDATE OF binding`, alias).
-		Scan(&accountID, &bindingID, &currentBindingVersion, &currentExitID, &cleanupPending)
+		JOIN social_account account ON account.id = binding.account_id
+		WHERE binding.browser_env_alias = $1 FOR UPDATE OF binding, account`, alias).
+		Scan(&accountID, &bindingID, &currentBindingVersion, &currentExitID, &cleanupPending, &accountStatus, &authorizationStatus)
 	if err != nil {
 		return EnvironmentContext{}, rowError(err)
 	}
-	if cleanupPending || currentBindingVersion != bindingVersion || currentExitID != exitID {
+	if cleanupPending || accountStatus != "active" || authorizationStatus != "authorized" ||
+		currentBindingVersion != bindingVersion || currentExitID != exitID {
 		return EnvironmentContext{}, ErrConflict
 	}
 	if _, err := tx.ExecContext(ctx, `
@@ -567,19 +604,21 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi
 		WHERE binding_id = $1 AND released_at IS NULL AND lease_until <= now()`, bindingID); err != nil {
 		return EnvironmentContext{}, errors.New("expire runtime before activation")
 	}
-	var existingInstanceID, existingRuntimeID string
+	var existingInstanceID, existingRuntimeID, existingNetworkID string
+	var existingBindingVersion int64
 	if err := tx.QueryRowContext(ctx, `
-		SELECT COALESCE(id, ''), COALESCE(runtime_id, '') FROM runtime_instance
-		WHERE binding_id = $1 AND released_at IS NULL`, bindingID).Scan(&existingInstanceID, &existingRuntimeID); err != nil && !errors.Is(err, sql.ErrNoRows) {
+		SELECT COALESCE(id, ''), COALESCE(runtime_id, ''), COALESCE(network_id, ''), COALESCE(binding_version, 0) FROM runtime_instance
+		WHERE binding_id = $1 AND released_at IS NULL`, bindingID).Scan(&existingInstanceID, &existingRuntimeID, &existingNetworkID, &existingBindingVersion); err != nil && !errors.Is(err, sql.ErrNoRows) {
 		return EnvironmentContext{}, publicDatabaseError(err)
 	}
-	if existingInstanceID != "" && existingRuntimeID != runtimeID {
+	if existingInstanceID != "" && (existingRuntimeID != runtimeID || existingNetworkID != networkID || existingBindingVersion != bindingVersion) {
 		return EnvironmentContext{}, ErrConflict
 	}
 	if existingInstanceID == "" {
 		if _, err := tx.ExecContext(ctx, `
-			INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until)
-			VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil {
+			INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, network_id, lease_until)
+			VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), now() + interval '1 minute')`,
+			"runtime-"+newHubID(), accountID, bindingID, bindingVersion, runtimeID, networkID); err != nil {
 			return EnvironmentContext{}, publicDatabaseError(err)
 		}
 	} else if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 minute' WHERE id = $1`, existingInstanceID); err != nil {
@@ -591,22 +630,37 @@ func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bi
 	return s.GetEnvironmentContext(ctx, alias)
 }
 
-func (s *Store) ReleaseRuntime(ctx context.Context, alias string) error {
-	if !aliasPattern.MatchString(alias) {
+func (s *Store) ReleaseRuntime(ctx context.Context, environment EnvironmentContext) error {
+	if !aliasPattern.MatchString(environment.Alias) || !exitIDPattern.MatchString(environment.BindingID) ||
+		environment.BindingVersion < 1 || (environment.RuntimeInstanceID != "" && !exitIDPattern.MatchString(environment.RuntimeInstanceID)) {
 		return ErrInvalid
 	}
-	_, err := s.db.ExecContext(ctx, `
+	if environment.RuntimeInstanceID == "" {
+		return nil
+	}
+	result, err := s.db.ExecContext(ctx, `
 		UPDATE runtime_instance runtime SET released_at = now()
 		FROM environment_binding binding
-		WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id AND runtime.released_at IS NULL`, alias)
+		WHERE binding.browser_env_alias = $1 AND binding.id = $2 AND binding.version = $3
+			AND runtime.binding_id = binding.id AND runtime.binding_version = binding.version
+			AND runtime.id = $4 AND runtime.released_at IS NULL`, environment.Alias, environment.BindingID,
+		environment.BindingVersion, environment.RuntimeInstanceID)
 	if err != nil {
 		return errors.New("release environment runtime")
 	}
+	if affected, err := result.RowsAffected(); err != nil || affected != 1 {
+		return ErrConflict
+	}
 	return nil
 }
 
-func (s *Store) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error {
-	if !aliasPattern.MatchString(alias) {
+func (s *Store) SetRuntimeCleanupPending(ctx context.Context, environment EnvironmentContext, pending bool) error {
+	if !aliasPattern.MatchString(environment.Alias) || !exitIDPattern.MatchString(environment.BindingID) ||
+		environment.BindingVersion < 1 || environment.RuntimeCleanupBindingVersion < 1 ||
+		(pending && environment.RuntimeCleanupRuntimeID == "") ||
+		(environment.RuntimeCleanupInstanceID != "" && !exitIDPattern.MatchString(environment.RuntimeCleanupInstanceID)) ||
+		(environment.RuntimeCleanupRuntimeID != "" && !exitIDPattern.MatchString(environment.RuntimeCleanupRuntimeID)) ||
+		(environment.RuntimeCleanupNetworkID != "" && !exitIDPattern.MatchString(environment.RuntimeCleanupNetworkID)) {
 		return ErrInvalid
 	}
 	tx, err := s.db.BeginTx(ctx, nil)
@@ -614,18 +668,75 @@ func (s *Store) SetRuntimeCleanupPending(ctx context.Context, alias string, pend
 		return errors.New("begin runtime cleanup state update")
 	}
 	defer tx.Rollback()
-	var bindingID string
+	var currentPending bool
+	var cleanupBindingVersion sql.NullInt64
+	var cleanupInstanceID, cleanupRuntimeID, cleanupNetworkID sql.NullString
 	if err := tx.QueryRowContext(ctx, `
-		UPDATE environment_binding SET runtime_cleanup_pending = $2, updated_at = now()
-		WHERE browser_env_alias = $1 RETURNING id`, alias, pending).Scan(&bindingID); err != nil {
-		return rowError(err)
+		SELECT runtime_cleanup_pending, runtime_cleanup_binding_version,
+			runtime_cleanup_instance_id, runtime_cleanup_runtime_id, runtime_cleanup_network_id
+		FROM environment_binding
+		WHERE browser_env_alias = $1 AND id = $2 AND version = $3 FOR UPDATE`, environment.Alias,
+		environment.BindingID, environment.BindingVersion).
+		Scan(&currentPending, &cleanupBindingVersion, &cleanupInstanceID, &cleanupRuntimeID, &cleanupNetworkID); errors.Is(err, sql.ErrNoRows) {
+		return ErrConflict
+	} else if err != nil {
+		return publicDatabaseError(err)
+	}
+	if currentPending {
+		if cleanupBindingVersion.Int64 != environment.RuntimeCleanupBindingVersion ||
+			cleanupInstanceID.String != environment.RuntimeCleanupInstanceID || cleanupRuntimeID.String != environment.RuntimeCleanupRuntimeID ||
+			cleanupNetworkID.String != environment.RuntimeCleanupNetworkID {
+			return ErrConflict
+		}
+		if pending {
+			return commitHub(tx)
+		}
+	} else if !pending {
+		return commitHub(tx)
+	}
+	if pending && !currentPending {
+		var runtimeInstanceID string
+		err := tx.QueryRowContext(ctx, `
+			SELECT id FROM runtime_instance
+			WHERE binding_id = $1 AND binding_version = $2 AND released_at IS NULL`, environment.BindingID,
+			environment.BindingVersion).Scan(&runtimeInstanceID)
+		if errors.Is(err, sql.ErrNoRows) {
+			runtimeInstanceID = ""
+		} else if err != nil {
+			return publicDatabaseError(err)
+		}
+		if runtimeInstanceID != environment.RuntimeCleanupInstanceID {
+			return ErrConflict
+		}
+		if runtimeInstanceID != "" {
+			result, err := tx.ExecContext(ctx, `
+				UPDATE runtime_instance SET released_at = now()
+				WHERE id = $1 AND binding_id = $2 AND binding_version = $3 AND released_at IS NULL`, runtimeInstanceID,
+				environment.BindingID, environment.BindingVersion)
+			if err != nil {
+				return errors.New("release runtime for pending cleanup")
+			}
+			if affected, err := result.RowsAffected(); err != nil || affected != 1 {
+				return ErrConflict
+			}
+		}
 	}
 	if pending {
-		if _, err := tx.ExecContext(ctx, `
-			UPDATE runtime_instance SET released_at = now()
-			WHERE binding_id = $1 AND released_at IS NULL`, bindingID); err != nil {
-			return errors.New("release runtime for pending cleanup")
-		}
+		_, err = tx.ExecContext(ctx, `
+			UPDATE environment_binding SET runtime_cleanup_pending = true,
+				runtime_cleanup_binding_version = $2, runtime_cleanup_instance_id = NULLIF($3, ''),
+				runtime_cleanup_runtime_id = NULLIF($4, ''), runtime_cleanup_network_id = NULLIF($5, ''), updated_at = now()
+			WHERE id = $1`, environment.BindingID, environment.RuntimeCleanupBindingVersion,
+			environment.RuntimeCleanupInstanceID, environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID)
+	} else {
+		_, err = tx.ExecContext(ctx, `
+			UPDATE environment_binding SET runtime_cleanup_pending = false,
+				runtime_cleanup_binding_version = NULL, runtime_cleanup_instance_id = NULL,
+				runtime_cleanup_runtime_id = NULL, runtime_cleanup_network_id = NULL, updated_at = now()
+			WHERE id = $1`, environment.BindingID)
+	}
+	if err != nil {
+		return errors.New("update runtime cleanup state")
 	}
 	return commitHub(tx)
 }

internal/hub/migration_test.go

@@ -29,13 +29,78 @@ func TestUnifiedAccountMigration(t *testing.T) {
 			t.Fatal(err)
 		}
 		defer db.Close()
-		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)`, 10)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('social_account', 'browser_env', 'network_exit', 'environment_binding')`, 4)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name = 'runtime_cleanup_pending'`, 1)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name LIKE 'runtime_cleanup_%'`, 5)
 
 		store = openFullyMigratedHub(t, ctx, testURL)
 		store.Close()
-		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)`, 10)
+	})
+
+	t.Run("previous migration 008 already applied", func(t *testing.T) {
+		ctx := context.Background()
+		testURL := isolatedDatabaseURL(t, databaseURL)
+		store := openFullyMigratedHub(t, ctx, testURL)
+		store.Close()
+		db, err := sql.Open("pgx", testURL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer db.Close()
+		if _, err := db.Exec(`
+			INSERT INTO credential_reference (id, provider, reference_key)
+			VALUES ('credential-upgrade', 'os_keyring', 'creatorhub/upgrade');
+			INSERT INTO social_account
+				(id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status, status)
+			VALUES ('upgrade', 'credential-upgrade', 'mock', 'upgrade', 'owned', 'authorized', 'paused');
+			INSERT INTO gateway (name, endpoint, token)
+			VALUES ('upgrade-gateway', 'http://127.0.0.1:8081', 'upgrade-gateway-token');
+			INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1');
+			INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint)
+			VALUES ('upgrade', 'Upgrade', 'upgrade-gateway', '1', '{"seed":1}');
+			INSERT INTO environment_binding (id, account_id, browser_env_alias)
+			VALUES ('upgrade', 'upgrade', 'upgrade');
+			UPDATE environment_binding SET runtime_cleanup_pending = true,
+				runtime_cleanup_binding_version = version WHERE id = 'upgrade';
+			DELETE FROM schema_migration WHERE version = 9;
+			ALTER TABLE environment_binding ADD CONSTRAINT runtime_cleanup_generation_consistent CHECK (
+				(runtime_cleanup_pending AND runtime_cleanup_binding_version IS NOT NULL AND runtime_cleanup_binding_version > 0)
+				OR
+				(NOT runtime_cleanup_pending AND runtime_cleanup_binding_version IS NULL
+					AND runtime_cleanup_instance_id IS NULL AND runtime_cleanup_runtime_id IS NULL)
+			)`); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = false WHERE id = 'upgrade'`); err == nil {
+			t.Fatal("old migration 008 constraint unexpectedly allowed a legacy writer to clear pending")
+		}
+
+		store, err = Open(ctx, testURL)
+		if err != nil {
+			t.Fatalf("apply forward compatibility migration: %v", err)
+		}
+		defer store.Close()
+		if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = false WHERE id = 'upgrade'`); err != nil {
+			t.Fatalf("migration 009 did not unblock the old writer: %v", err)
+		}
+		context, err := store.GetEnvironmentContext(ctx, "upgrade")
+		if err != nil || context.RuntimeCleanupPending || context.RuntimeCleanupBindingVersion != 0 || context.RuntimeCleanupRuntimeID != "" {
+			t.Fatalf("new runner reused stale cleanup generation after old writer clear: %#v err=%v", context, err)
+		}
+		if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = true WHERE id = 'upgrade'`); err != nil {
+			t.Fatalf("migration 009 did not allow the old writer to set pending: %v", err)
+		}
+		context, err = store.GetEnvironmentContext(ctx, "upgrade")
+		if err != nil || !context.RuntimeCleanupPending || context.RuntimeCleanupBindingVersion != 1 || context.RuntimeCleanupRuntimeID != "" {
+			t.Fatalf("new runner did not expose legacy pending for manual reconcile: %#v err=%v", context, err)
+		}
+		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (9, 10)`, 2)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND
+			((table_name = 'runtime_instance' AND column_name = 'network_id') OR
+			 (table_name = 'environment_binding' AND column_name = 'runtime_cleanup_network_id'))`, 2)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM pg_constraint WHERE conname = 'runtime_cleanup_generation_consistent'`, 0)
 	})
 
 	t.Run("v1 and v2 data", func(t *testing.T) {
@@ -94,10 +159,18 @@ func TestUnifiedAccountMigration(t *testing.T) {
 		assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'mapped' AND NOT runtime_cleanup_pending`, 1)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'unbound'`, 0)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-mapped' AND binding_id = 'mapped'`, 1)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-mapped' AND binding_version = 1`, 1)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-unbound' AND binding_id IS NULL`, 1)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM audit_event WHERE event_type = 'legacy_event'`, 1)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'legacy-task' AND state = 'policy_hold'`, 1)
 		assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND NOT (fingerprint ?| ARRAY['proxy_server', 'disable_non_proxied_udp'])`, 1)
+		if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = true WHERE id = 'mapped'`); err != nil {
+			t.Fatalf("migration 008 blocked an old writer setting cleanup pending: %v", err)
+		}
+		assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE id = 'mapped' AND runtime_cleanup_pending AND runtime_cleanup_binding_version IS NULL AND runtime_cleanup_runtime_id IS NULL`, 1)
+		if _, err := db.Exec(`UPDATE environment_binding SET runtime_cleanup_pending = false WHERE id = 'mapped'`); err != nil {
+			t.Fatalf("migration 008 blocked an old writer clearing cleanup pending: %v", err)
+		}
 		store, err = Open(ctx, testURL)
 		if err != nil {
 			t.Fatal(err)
@@ -175,7 +248,7 @@ func TestUnifiedAccountMigration(t *testing.T) {
 			t.Fatal(err)
 		}
 		store.Close()
-		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (3, 4, 5, 6)`, 4)
+		assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (3, 4, 5, 6, 7, 8, 9)`, 7)
 	})
 }
 

internal/hub/migrations/007_runtime_binding_version.sql

@@ -0,0 +1,7 @@
+ALTER TABLE runtime_instance
+    ADD COLUMN binding_version bigint CHECK (binding_version > 0);
+
+UPDATE runtime_instance runtime
+SET binding_version = binding.version
+FROM environment_binding binding
+WHERE runtime.binding_id = binding.id;

internal/hub/migrations/008_runtime_cleanup_generation.sql

@@ -0,0 +1,4 @@
+ALTER TABLE environment_binding
+    ADD COLUMN runtime_cleanup_binding_version bigint,
+    ADD COLUMN runtime_cleanup_instance_id text,
+    ADD COLUMN runtime_cleanup_runtime_id text;

internal/hub/migrations/009_runtime_cleanup_compatibility.sql

@@ -0,0 +1,2 @@
+ALTER TABLE environment_binding
+    DROP CONSTRAINT IF EXISTS runtime_cleanup_generation_consistent;

internal/hub/migrations/010_runtime_network_generation.sql

@@ -0,0 +1,5 @@
+ALTER TABLE runtime_instance
+    ADD COLUMN network_id text;
+
+ALTER TABLE environment_binding
+    ADD COLUMN runtime_cleanup_network_id text;

internal/hub/store.go

@@ -34,10 +34,23 @@ var migration005 string
 //go:embed migrations/006_runtime_cleanup.sql
 var migration006 string
 
+//go:embed migrations/007_runtime_binding_version.sql
+var migration007 string
+
+//go:embed migrations/008_runtime_cleanup_generation.sql
+var migration008 string
+
+//go:embed migrations/009_runtime_cleanup_compatibility.sql
+var migration009 string
+
+//go:embed migrations/010_runtime_network_generation.sql
+var migration010 string
+
 var (
-	ErrConflict = errors.New("resource conflicts with existing state")
-	ErrInvalid  = errors.New("invalid hub input")
-	ErrNotFound = errors.New("resource not found")
+	ErrConflict          = errors.New("resource conflicts with existing state")
+	ErrInvalid           = errors.New("invalid hub input")
+	ErrNotFound          = errors.New("resource not found")
+	ErrReconcileRequired = errors.New("runtime cleanup generation is unknown; manual reconciliation required")
 )
 
 func ValidImageVersion(version string) bool { return imageVersionPattern.MatchString(version) }
@@ -118,7 +131,7 @@ func (s *Store) migrate(ctx context.Context) error {
 	for _, migration := range []struct {
 		version int
 		sql     string
-	}{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}} {
+	}{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}} {
 		var applied bool
 		if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil {
 			return errors.New("read hub schema migration state")

internal/hub/store_test.go

@@ -8,6 +8,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestFingerprintArgsFollowUpstreamCommandLineContract(t *testing.T) {
@@ -303,7 +304,10 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) {
 	if err != nil || created || reused.Alias != bound.Alias || reused.BindingID != bound.BindingID {
 		t.Fatalf("same account must reuse its environment: %#v created=%v err=%v", reused, created, err)
 	}
-	active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID)
+	if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil {
+		t.Fatal(err)
+	}
+	active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID, "network-a")
 	if err != nil || active.RuntimeInstanceID == "" {
 		t.Fatalf("activate runtime: %#v err=%v", active, err)
 	}
@@ -331,13 +335,13 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) {
 	if err != nil || legacyRebound.Exit.ID != second.ID {
 		t.Fatalf("legacy binding without an exit must support explicit rebind: %#v err=%v", legacyRebound, err)
 	}
-	if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID); !errors.Is(err, ErrConflict) {
+	if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID, "network-a"); !errors.Is(err, ErrConflict) {
 		t.Fatalf("stale binding metadata must not activate a runtime: %v", err)
 	}
 	if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID); err != nil {
+	if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID, "network-a"); err != nil {
 		t.Fatalf("runtime heartbeat failed: %v", err)
 	}
 	var renewed bool
@@ -347,23 +351,110 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) {
 	if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil {
 		t.Fatal(err)
 	}
+	expired, err := store.GetEnvironmentContext(ctx, env.Alias)
+	if err != nil || expired.RuntimeInstanceID != active.RuntimeInstanceID || expired.RuntimeID != active.RuntimeID {
+		t.Fatalf("context read discarded expired cleanup generation: %#v err=%v", expired, err)
+	}
+	if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil {
+		t.Fatal(err)
+	}
 	rebound, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", bound.BindingVersion)
 	if err != nil || rebound.Exit.ID != second.ID || rebound.BindingVersion != 2 {
 		t.Fatalf("expired runtime must be transactionally released before rebind: %#v err=%v", rebound, err)
 	}
-	if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID); err != nil {
+	if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID, "network-same-exit"); err != nil {
 		t.Fatalf("activate runtime before same-exit rebind: %v", err)
 	}
+	if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil {
+		t.Fatal(err)
+	}
 	if _, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", rebound.BindingVersion); !errors.Is(err, ErrConflict) {
 		t.Fatalf("active runtime must block same-exit rebind: %v", err)
 	}
-	if err := store.ReleaseRuntime(ctx, env.Alias); err != nil {
+	active, err = store.GetEnvironmentContext(ctx, env.Alias)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := store.ReleaseRuntime(ctx, active); err != nil {
 		t.Fatal(err)
 	}
 	rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-container", rebound.BindingVersion)
 	if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-container" {
 		t.Fatalf("same-exit rebind must atomically CAS the binding and runtime: %#v err=%v", rebound, err)
 	}
+	if err := store.ReleaseRuntime(ctx, active); !errors.Is(err, ErrConflict) {
+		t.Fatalf("stale generation release must conflict: %v", err)
+	}
+	current, err := store.GetEnvironmentContext(ctx, env.Alias)
+	if err != nil || current.RuntimeInstanceID != rebound.RuntimeInstanceID || current.RuntimeID != "rebound-container" {
+		t.Fatalf("stale release changed the current runtime: %#v err=%v", current, err)
+	}
+	cleanup := current
+	cleanup.RuntimeCleanupBindingVersion = current.BindingVersion
+	cleanup.RuntimeCleanupInstanceID = current.RuntimeInstanceID
+	cleanup.RuntimeCleanupRuntimeID = current.RuntimeID
+	if err := store.SetRuntimeCleanupPending(ctx, cleanup, true); err != nil {
+		t.Fatalf("set generation cleanup pending: %v", err)
+	}
+	wrongCleanup := cleanup
+	wrongCleanup.RuntimeCleanupRuntimeID = "other-container"
+	if err := store.SetRuntimeCleanupPending(ctx, wrongCleanup, false); !errors.Is(err, ErrConflict) {
+		t.Fatalf("wrong cleanup generation cleared pending state: %v", err)
+	}
+	pending, err := store.GetEnvironmentContext(ctx, env.Alias)
+	if err != nil || !pending.RuntimeCleanupPending || pending.RuntimeCleanupRuntimeID != current.RuntimeID {
+		t.Fatalf("cleanup generation was not persisted: %#v err=%v", pending, err)
+	}
+	if _, err := store.ActivateRuntime(ctx, env.Alias, "candidate-container", current.BindingVersion, current.Exit.ID, "network-candidate"); !errors.Is(err, ErrConflict) {
+		t.Fatalf("paused account activated a stale request: %v", err)
+	}
+	if err := store.SetRuntimeCleanupPending(ctx, pending, false); err != nil {
+		t.Fatalf("clear matching cleanup generation: %v", err)
+	}
+	newGeneration, err := store.RebindEnvironment(ctx, env.Alias, current.Exit.ID, "", current.BindingVersion)
+	if err != nil {
+		t.Fatalf("advance binding generation: %v", err)
+	}
+	if err := store.SetRuntimeCleanupPending(ctx, cleanup, true); !errors.Is(err, ErrConflict) {
+		t.Fatalf("stale binding set cleanup pending on version %d: %v", newGeneration.BindingVersion, err)
+	}
+	if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil {
+		t.Fatal(err)
+	}
+	pauseTx, err := store.db.BeginTx(ctx, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var locked string
+	if err := pauseTx.QueryRowContext(ctx, `SELECT id FROM social_account WHERE id = 'account-a' FOR UPDATE`).Scan(&locked); err != nil {
+		t.Fatal(err)
+	}
+	activation := make(chan error, 1)
+	go func() {
+		_, err := store.ActivateRuntime(ctx, env.Alias, "racing-container", newGeneration.BindingVersion, newGeneration.Exit.ID, "network-racing")
+		activation <- err
+	}()
+	select {
+	case err := <-activation:
+		t.Fatalf("activation bypassed the locked account row: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+	if _, err := pauseTx.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil {
+		t.Fatal(err)
+	}
+	if err := pauseTx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+	if err := <-activation; !errors.Is(err, ErrConflict) {
+		t.Fatalf("activation won the pause race: %v", err)
+	}
+	latest, err := store.GetEnvironmentContext(ctx, env.Alias)
+	if err != nil || latest.RuntimeID != "" {
+		t.Fatalf("pause race left an active runtime: %#v err=%v", latest, err)
+	}
 
 	action := EnvironmentAction{
 		OperationID: NewOperationID(), Action: "start", AccountID: rebound.AccountID,

internal/phasea/store.go

@@ -49,13 +49,6 @@ type CredentialReference struct {
 	Provider string `json:"provider"`
 }
 
-type RuntimeBinding struct {
-	ID         string    `json:"id"`
-	AccountID  string    `json:"account_id"`
-	RuntimeID  string    `json:"runtime_id"`
-	LeaseUntil time.Time `json:"lease_until"`
-}
-
 type Draft struct {
 	ID        string `json:"id"`
 	AccountID string `json:"account_id"`
@@ -193,7 +186,7 @@ func (s *Store) CreateAccount(ctx context.Context, account Account) error {
 		account.Platform, account.PlatformAccountKey, account.AuthorizationKind); err != nil {
 		return publicDatabaseError(err)
 	}
-	if err := appendAudit(ctx, tx, "account_created", account.ID, "", 0, "", "", map[string]string{"platform": account.Platform}); err != nil {
+	if err := appendAudit(ctx, tx, "account_created", "account_created", account.ID, "", 0, "", "", map[string]string{"platform": account.Platform}); err != nil {
 		return err
 	}
 	return commit(tx)
@@ -245,57 +238,6 @@ func scanAccount(row accountScanner) (Account, error) {
 	return account, nil
 }
 
-func (s *Store) BindRuntime(ctx context.Context, binding RuntimeBinding) error {
-	if !refPattern.MatchString(binding.ID) || !idPattern.MatchString(binding.AccountID) ||
-		!refPattern.MatchString(binding.RuntimeID) || !binding.LeaseUntil.After(time.Now()) {
-		return ErrInvalid
-	}
-	tx, err := s.db.BeginTx(ctx, nil)
-	if err != nil {
-		return errors.New("begin runtime transaction")
-	}
-	defer tx.Rollback()
-	if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET released_at = now() WHERE released_at IS NULL AND lease_until < now()`); err != nil {
-		return errors.New("expire runtime bindings")
-	}
-	var accountID string
-	if err := tx.QueryRowContext(ctx, `
-		INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until)
-		SELECT $1, account.id, binding.id, $3, $4
-		FROM social_account account
-		JOIN environment_binding binding ON binding.account_id = account.id
-		WHERE account.id = $2 AND account.status = 'active'
-		RETURNING account_id`, binding.ID, binding.AccountID, binding.RuntimeID, binding.LeaseUntil).Scan(&accountID); err != nil {
-		if errors.Is(err, sql.ErrNoRows) {
-			return ErrConflict
-		}
-		return rowError(err)
-	}
-	if err := appendAudit(ctx, tx, "runtime_bound", accountID, "", 0, "", "", map[string]string{"runtime_id": binding.RuntimeID}); err != nil {
-		return err
-	}
-	return commit(tx)
-}
-
-func (s *Store) ReleaseRuntime(ctx context.Context, id string) error {
-	if !refPattern.MatchString(id) {
-		return ErrInvalid
-	}
-	tx, err := s.db.BeginTx(ctx, nil)
-	if err != nil {
-		return errors.New("begin runtime release")
-	}
-	defer tx.Rollback()
-	var accountID string
-	if err := tx.QueryRowContext(ctx, `UPDATE runtime_instance SET released_at = now() WHERE id = $1 AND released_at IS NULL RETURNING account_id`, id).Scan(&accountID); err != nil {
-		return rowError(err)
-	}
-	if err := appendAudit(ctx, tx, "runtime_released", accountID, "", 0, "", "", map[string]string{"binding_id": id}); err != nil {
-		return err
-	}
-	return commit(tx)
-}
-
 func (s *Store) CreateDraft(ctx context.Context, draft Draft) error {
 	if !refPattern.MatchString(draft.ID) || !idPattern.MatchString(draft.AccountID) || draft.Version < 1 || strings.TrimSpace(draft.Content) == "" {
 		return ErrInvalid
@@ -332,7 +274,7 @@ func (s *Store) Confirm(ctx context.Context, confirmation Confirmation) error {
 		confirmation.DraftID, confirmation.DraftVersion, confirmation.Version); err != nil {
 		return publicDatabaseError(err)
 	}
-	if err := appendAudit(ctx, tx, "draft_confirmed", confirmation.AccountID, confirmation.ID, confirmation.Version, "", "", map[string]any{
+	if err := appendAudit(ctx, tx, "draft_confirmed", "draft_confirmed", confirmation.AccountID, confirmation.ID, confirmation.Version, "", "", map[string]any{
 		"account_version": confirmation.AccountVersion, "draft_id": confirmation.DraftID, "draft_version": confirmation.DraftVersion,
 	}); err != nil {
 		return err
@@ -363,7 +305,7 @@ func (s *Store) Enqueue(ctx context.Context, task Task) (Task, bool, error) {
 	}
 	if insertedID != "" {
 		task.State = "queued"
-		if err := appendAudit(ctx, tx, "task_queued", task.AccountID, task.ConfirmationID, task.ConfirmationVersion, "", task.ID, nil); err != nil {
+		if err := appendAudit(ctx, tx, "task_queued", "task_queued", task.AccountID, task.ConfirmationID, task.ConfirmationVersion, "", task.ID, nil); err != nil {
 			return Task{}, false, err
 		}
 		if err := commit(tx); err != nil {
@@ -432,6 +374,14 @@ func (s *Store) disableAccount(ctx context.Context, accountID string, revoke boo
 	if err != nil {
 		return err
 	}
+	reason := "account_paused"
+	if revoke {
+		reason = "account_revoked"
+	}
+	interrupted, err := interruptExecutingTasks(ctx, tx, accountID, reason)
+	if err != nil {
+		return err
+	}
 	if unchanged {
 		return commit(tx)
 	}
@@ -439,7 +389,9 @@ func (s *Store) disableAccount(ctx context.Context, accountID string, revoke boo
 	if revoke {
 		eventType = "account_revoked"
 	}
-	if err := appendAudit(ctx, tx, eventType, accountID, "", 0, "", "", map[string]any{"account_version": version, "tasks_held": held}); err != nil {
+	if err := appendAudit(ctx, tx, eventType, reason, accountID, "", 0, "", "", map[string]any{
+		"account_version": version, "tasks_held": held, "attempts_interrupted": interrupted,
+	}); err != nil {
 		return err
 	}
 	return commit(tx)
@@ -468,6 +420,11 @@ func (s *Store) ResumeAccount(ctx context.Context, accountID string) error {
 			SELECT 1 FROM environment_binding binding
 			JOIN network_exit network ON network.id = binding.network_exit_id
 			WHERE binding.account_id = $1 AND network.health_status = 'healthy'
+				AND NOT binding.runtime_cleanup_pending
+				AND NOT EXISTS (
+					SELECT 1 FROM runtime_instance runtime
+					WHERE runtime.binding_id = binding.id AND runtime.released_at IS NULL
+				)
 		)`, accountID).Scan(&ready); err != nil {
 		return errors.New("validate account binding")
 	}
@@ -483,7 +440,7 @@ func (s *Store) ResumeAccount(ctx context.Context, accountID string) error {
 		WHERE id = $1 RETURNING version`, accountID).Scan(&version); err != nil {
 		return errors.New("resume account")
 	}
-	if err := appendAudit(ctx, tx, "account_resumed", accountID, "", 0, "", "", map[string]any{"account_version": version}); err != nil {
+	if err := appendAudit(ctx, tx, "account_resumed", "account_resumed", accountID, "", 0, "", "", map[string]any{"account_version": version}); err != nil {
 		return err
 	}
 	return commit(tx)
@@ -500,6 +457,49 @@ func holdQueuedTasks(ctx context.Context, tx *sql.Tx, accountID string) (int64,
 	return held, nil
 }
 
+func interruptExecutingTasks(ctx context.Context, tx *sql.Tx, accountID, reason string) (int64, error) {
+	rows, err := tx.QueryContext(ctx, `
+		UPDATE operation_task SET state = 'needs_confirmation', lease_owner = NULL, lease_until = NULL, updated_at = now()
+		WHERE account_id = $1 AND state = 'executing'
+		RETURNING id, current_attempt_id, confirmation_id, confirmation_version`, accountID)
+	if err != nil {
+		return 0, errors.New("interrupt executing account tasks")
+	}
+	type interruptedTask struct {
+		taskID, attemptID, confirmationID string
+		confirmationVersion               int64
+	}
+	var tasks []interruptedTask
+	for rows.Next() {
+		var task interruptedTask
+		var attemptID, confirmationID sql.NullString
+		var confirmationVersion sql.NullInt64
+		if err := rows.Scan(&task.taskID, &attemptID, &confirmationID, &confirmationVersion); err != nil {
+			rows.Close()
+			return 0, errors.New("read interrupted account task")
+		}
+		task.attemptID, task.confirmationID, task.confirmationVersion = attemptID.String, confirmationID.String, confirmationVersion.Int64
+		tasks = append(tasks, task)
+	}
+	if err := rows.Close(); err != nil {
+		return 0, errors.New("close interrupted account task rows")
+	}
+	for _, task := range tasks {
+		if task.attemptID != "" {
+			if _, err := tx.ExecContext(ctx, `
+				UPDATE execution_attempt SET finished_at = now(), outcome = 'uncertain'
+				WHERE id = $1 AND finished_at IS NULL`, task.attemptID); err != nil {
+				return 0, errors.New("mark interrupted attempt uncertain")
+			}
+		}
+		if err := appendAudit(ctx, tx, "task_interrupted", reason, accountID, task.confirmationID,
+			task.confirmationVersion, task.attemptID, task.taskID, nil); err != nil {
+			return 0, err
+		}
+	}
+	return int64(len(tasks)), nil
+}
+
 func (s *Store) CancelTask(ctx context.Context, taskID string) error {
 	if !refPattern.MatchString(taskID) {
 		return ErrInvalid
@@ -529,7 +529,7 @@ func (s *Store) CancelTask(ctx context.Context, taskID string) error {
 			return errors.New("mark cancelled attempt uncertain")
 		}
 	}
-	if err := appendAudit(ctx, tx, "task_cancel_requested", accountID, confirmationID.String, confirmationVersion.Int64, attemptID.String, taskID, map[string]string{"state": state}); err != nil {
+	if err := appendAudit(ctx, tx, "task_cancel_requested", "task_cancel_requested", accountID, confirmationID.String, confirmationVersion.Int64, attemptID.String, taskID, map[string]string{"state": state}); err != nil {
 		return err
 	}
 	return commit(tx)
@@ -566,14 +566,20 @@ func (s *Store) claim(ctx context.Context, workerID string) (Execution, error) {
 			JOIN social_account a ON a.id = t.account_id
 			JOIN content_draft d ON d.id = t.draft_id
 			JOIN confirmation c ON c.id = t.confirmation_id
-			WHERE t.state = 'queued' AND a.status = 'active'
+			JOIN environment_binding binding ON binding.account_id = a.id
+			JOIN browser_env environment ON environment.alias = binding.browser_env_alias
+			JOIN network_exit network ON network.id = binding.network_exit_id
+			JOIN runtime_instance runtime ON runtime.binding_id = binding.id
+				AND runtime.released_at IS NULL AND runtime.lease_until > now() AND runtime.binding_version = binding.version
+			WHERE t.state = 'queued' AND a.status = 'active' AND a.authorization_status = 'authorized'
 			  AND a.version = t.account_version
 			  AND d.account_id = t.account_id AND d.version = t.draft_version
 			  AND c.account_id = t.account_id AND c.account_version = t.account_version
 			  AND c.draft_id = t.draft_id AND c.draft_version = t.draft_version
 			  AND c.version = t.confirmation_version
+			  AND network.health_status = 'healthy' AND NOT binding.runtime_cleanup_pending
 			ORDER BY t.created_at, t.id
-			FOR UPDATE OF t SKIP LOCKED LIMIT 1
+			FOR UPDATE OF t, a, binding, network, runtime SKIP LOCKED LIMIT 1
 		)
 		UPDATE operation_task t SET state = 'executing', lease_owner = $1,
 			lease_until = now() + interval '1 minute', updated_at = now()
@@ -598,7 +604,7 @@ func (s *Store) claim(ctx context.Context, workerID string) (Execution, error) {
 	if _, err := tx.ExecContext(ctx, `UPDATE operation_task SET current_attempt_id = $1 WHERE id = $2`, execution.AttemptID, execution.TaskID); err != nil {
 		return Execution{}, errors.New("attach execution attempt")
 	}
-	if err := appendAudit(ctx, tx, "task_claimed", execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"worker_id": workerID}); err != nil {
+	if err := appendAudit(ctx, tx, "task_claimed", "task_claimed", execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"worker_id": workerID}); err != nil {
 		return Execution{}, err
 	}
 	if err := commit(tx); err != nil {
@@ -633,7 +639,11 @@ func (s *Store) complete(ctx context.Context, workerID string, execution Executi
 		UPDATE operation_task SET state = $1, lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE id = $2`, state, execution.TaskID); err != nil {
 		return Execution{}, errors.New("finish task")
 	}
-	if err := appendAudit(ctx, tx, "task_finished", execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"state": state}); err != nil {
+	reason := map[string]string{
+		"succeeded": "task_succeeded", "failed": "task_failed",
+		"needs_confirmation": "task_result_uncertain", "policy_hold": "task_policy_hold",
+	}[state]
+	if err := appendAudit(ctx, tx, "task_finished", reason, execution.AccountID, execution.ConfirmationID, execution.ConfirmationVersion, execution.AttemptID, execution.TaskID, map[string]string{"state": state}); err != nil {
 		return Execution{}, err
 	}
 	if err := commit(tx); err != nil {
@@ -676,7 +686,7 @@ func quarantineExpired(ctx context.Context, tx *sql.Tx) error {
 				return errors.New("mark expired attempt uncertain")
 			}
 		}
-		if err := appendAudit(ctx, tx, "task_lease_expired", task.accountID, task.confirmationID, task.confirmationVersion, task.attemptID, task.taskID, nil); err != nil {
+		if err := appendAudit(ctx, tx, "task_lease_expired", "execution_lease_expired", task.accountID, task.confirmationID, task.confirmationVersion, task.attemptID, task.taskID, nil); err != nil {
 			return err
 		}
 	}
@@ -685,46 +695,84 @@ func quarantineExpired(ctx context.Context, tx *sql.Tx) error {
 
 func quarantineInvalid(ctx context.Context, tx *sql.Tx) error {
 	rows, err := tx.QueryContext(ctx, `
-		UPDATE operation_task t SET
-			state = CASE WHEN EXISTS (
-				SELECT 1 FROM social_account paused WHERE paused.id = t.account_id AND paused.status = 'paused'
-			) THEN 'policy_hold' ELSE 'needs_confirmation' END,
-			updated_at = now()
-		WHERE t.state = 'queued' AND NOT EXISTS (
-			SELECT 1
-			FROM social_account a
-			JOIN content_draft d ON d.id = t.draft_id
-			JOIN confirmation c ON c.id = t.confirmation_id
-			WHERE a.id = t.account_id AND a.status = 'active' AND a.version = t.account_version
-			  AND d.account_id = t.account_id AND d.version = t.draft_version
-			  AND c.account_id = t.account_id AND c.account_version = t.account_version
-			  AND c.draft_id = t.draft_id AND c.draft_version = t.draft_version
-			  AND c.version = t.confirmation_version
+		WITH invalid AS (
+			SELECT t.id,
+				CASE
+					WHEN account.id IS NULL OR account.status <> 'active' OR account.authorization_status <> 'authorized' THEN 'policy_hold'
+					WHEN account.version <> t.account_version THEN 'needs_confirmation'
+					WHEN draft.id IS NULL OR draft.account_id <> t.account_id OR draft.version <> t.draft_version THEN 'needs_confirmation'
+					WHEN confirmation.id IS NULL OR confirmation.account_id <> t.account_id
+						OR confirmation.account_version <> t.account_version OR confirmation.draft_id <> t.draft_id
+						OR confirmation.draft_version <> t.draft_version OR confirmation.version <> t.confirmation_version THEN 'needs_confirmation'
+					WHEN runtime.id IS NOT NULL AND runtime.binding_version IS DISTINCT FROM binding.version THEN 'needs_confirmation'
+					ELSE 'policy_hold'
+				END AS state,
+				CASE
+					WHEN account.id IS NULL THEN 'account_missing'
+					WHEN account.authorization_status = 'revoked' THEN 'account_revoked'
+					WHEN account.status <> 'active' THEN 'account_paused'
+					WHEN account.version <> t.account_version THEN 'account_version_changed'
+					WHEN draft.id IS NULL OR draft.account_id <> t.account_id OR draft.version <> t.draft_version THEN 'draft_version_changed'
+					WHEN confirmation.id IS NULL THEN 'confirmation_missing'
+					WHEN confirmation.account_id <> t.account_id OR confirmation.account_version <> t.account_version
+						OR confirmation.draft_id <> t.draft_id OR confirmation.draft_version <> t.draft_version
+						OR confirmation.version <> t.confirmation_version THEN 'confirmation_version_changed'
+					WHEN binding.id IS NULL THEN 'binding_missing'
+					WHEN environment.alias IS NULL THEN 'environment_missing'
+					WHEN network.id IS NULL THEN 'exit_missing'
+					WHEN network.health_status <> 'healthy' THEN 'exit_unhealthy'
+					WHEN binding.runtime_cleanup_pending THEN 'runtime_stop_pending'
+					WHEN runtime.id IS NULL THEN 'runtime_missing'
+					WHEN runtime.binding_version IS DISTINCT FROM binding.version THEN 'binding_version_changed'
+					WHEN runtime.lease_until <= now() THEN 'runtime_lease_expired'
+				END AS reason_code
+			FROM operation_task t
+			LEFT JOIN social_account account ON account.id = t.account_id
+			LEFT JOIN content_draft draft ON draft.id = t.draft_id
+			LEFT JOIN confirmation confirmation ON confirmation.id = t.confirmation_id
+			LEFT JOIN environment_binding binding ON binding.account_id = t.account_id
+			LEFT JOIN browser_env environment ON environment.alias = binding.browser_env_alias
+			LEFT JOIN network_exit network ON network.id = binding.network_exit_id
+			LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL
+			WHERE t.state = 'queued' AND (
+				account.id IS NULL OR account.status <> 'active' OR account.authorization_status <> 'authorized'
+				OR account.version <> t.account_version
+				OR draft.id IS NULL OR draft.account_id <> t.account_id OR draft.version <> t.draft_version
+				OR confirmation.id IS NULL OR confirmation.account_id <> t.account_id
+				OR confirmation.account_version <> t.account_version OR confirmation.draft_id <> t.draft_id
+				OR confirmation.draft_version <> t.draft_version OR confirmation.version <> t.confirmation_version
+				OR binding.id IS NULL OR environment.alias IS NULL OR network.id IS NULL OR network.health_status <> 'healthy'
+				OR binding.runtime_cleanup_pending OR runtime.id IS NULL
+				OR runtime.binding_version IS DISTINCT FROM binding.version OR runtime.lease_until <= now()
+			)
+			FOR UPDATE OF t SKIP LOCKED
 		)
-		RETURNING t.id, t.account_id, t.confirmation_id, t.confirmation_version, t.state`)
+		UPDATE operation_task task SET state = invalid.state, updated_at = now()
+		FROM invalid WHERE task.id = invalid.id
+		RETURNING task.id, task.account_id, task.confirmation_id, task.confirmation_version, task.state, invalid.reason_code`)
 	if err != nil {
 		return errors.New("quarantine invalid tasks")
 	}
 	type invalidTask struct {
-		taskID, accountID, confirmationID, state string
-		confirmationVersion                      int64
+		taskID, accountID, confirmationID, state, reasonCode string
+		confirmationVersion                                  int64
 	}
 	var invalid []invalidTask
 	for rows.Next() {
-		var taskID, accountID, state string
+		var taskID, accountID, state, reasonCode string
 		var confirmationID sql.NullString
 		var confirmationVersion sql.NullInt64
-		if err := rows.Scan(&taskID, &accountID, &confirmationID, &confirmationVersion, &state); err != nil {
+		if err := rows.Scan(&taskID, &accountID, &confirmationID, &confirmationVersion, &state, &reasonCode); err != nil {
 			rows.Close()
 			return errors.New("read quarantined task")
 		}
-		invalid = append(invalid, invalidTask{taskID, accountID, confirmationID.String, state, confirmationVersion.Int64})
+		invalid = append(invalid, invalidTask{taskID, accountID, confirmationID.String, state, reasonCode, confirmationVersion.Int64})
 	}
 	if err := rows.Close(); err != nil {
 		return errors.New("close quarantined task rows")
 	}
 	for _, task := range invalid {
-		if err := appendAudit(ctx, tx, "task_quarantined", task.accountID, task.confirmationID, task.confirmationVersion, "", task.taskID, map[string]string{"state": task.state}); err != nil {
+		if err := appendAudit(ctx, tx, "task_quarantined", task.reasonCode, task.accountID, task.confirmationID, task.confirmationVersion, "", task.taskID, map[string]string{"state": task.state}); err != nil {
 			return err
 		}
 	}
@@ -766,7 +814,7 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) {
 	return events, rows.Err()
 }
 
-func appendAudit(ctx context.Context, tx *sql.Tx, eventType, accountID, confirmationID string, confirmationVersion int64, attemptID, taskID string, details any) error {
+func appendAudit(ctx context.Context, tx *sql.Tx, eventType, reasonCode, accountID, confirmationID string, confirmationVersion int64, attemptID, taskID string, details any) error {
 	if details == nil {
 		details = map[string]any{}
 	}
@@ -776,9 +824,14 @@ func appendAudit(ctx context.Context, tx *sql.Tx, eventType, accountID, confirma
 	}
 	_, err = tx.ExecContext(ctx, `
 		INSERT INTO audit_event
-			(event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id, actor, reason_code, details)
-		VALUES ($1, NULLIF($2, ''), NULLIF($3, ''), NULLIF($4, 0), NULLIF($5, ''), NULLIF($6, ''), 'local-user', $1, $7)`,
-		eventType, accountID, confirmationID, confirmationVersion, attemptID, taskID, encoded)
+			(event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id,
+			 browser_env_alias, network_exit_id, runtime_instance_id, binding_version, actor, reason_code, details)
+		SELECT $1, NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, 0), NULLIF($6, ''), NULLIF($7, ''),
+			binding.browser_env_alias, binding.network_exit_id, runtime.id, binding.version, 'local-user', $2, $8
+		FROM (VALUES (1)) AS singleton(value)
+		LEFT JOIN environment_binding binding ON binding.account_id = NULLIF($3, '')
+		LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL`,
+		eventType, reasonCode, accountID, confirmationID, confirmationVersion, attemptID, taskID, encoded)
 	if err != nil {
 		return errors.New("append audit event")
 	}

internal/phasea/store_test.go

@@ -131,23 +131,21 @@ func TestPhaseAOfflineWorkflow(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	lease := time.Now().Add(time.Minute)
-	if err := store.BindRuntime(ctx, RuntimeBinding{ID: "binding-a", AccountID: "account-a", RuntimeID: "runtime-a", LeaseUntil: lease}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.BindRuntime(ctx, RuntimeBinding{ID: "binding-a-2", AccountID: "account-a", RuntimeID: "runtime-b", LeaseUntil: lease}); !errors.Is(err, ErrConflict) {
-		t.Fatalf("expected one active runtime per account, got %v", err)
-	}
-	if err := store.BindRuntime(ctx, RuntimeBinding{ID: "binding-b", AccountID: "account-b", RuntimeID: "runtime-a", LeaseUntil: lease}); !errors.Is(err, ErrConflict) {
-		t.Fatalf("expected one account per runtime, got %v", err)
-	}
-	if err := store.ReleaseRuntime(ctx, "binding-a"); err != nil {
+	if _, err := store.db.ExecContext(ctx, `
+		INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) VALUES
+			('runtime-instance-a', 'account-a', 'binding-a', 1, 'runtime-a', now() + interval '1 minute'),
+			('runtime-instance-b', 'account-b', 'binding-b', 1, 'runtime-b', now() + interval '1 minute')`); err != nil {
 		t.Fatal(err)
 	}
 
 	createApprovedDraft(t, store, "account-a", accountA.Version, "draft-a", "confirmation-a")
+	createApprovedDraft(t, store, "account-b", accountB.Version, "draft-b", "confirmation-b")
 	for index := range 20 {
-		task := approvedTask(index, "account-a", accountA.Version, "draft-a", "confirmation-a")
+		accountID, accountVersion, draftID, confirmationID := "account-a", accountA.Version, "draft-a", "confirmation-a"
+		if index%2 == 1 {
+			accountID, accountVersion, draftID, confirmationID = "account-b", accountB.Version, "draft-b", "confirmation-b"
+		}
+		task := approvedTask(index, accountID, accountVersion, draftID, confirmationID)
 		if _, inserted, err := store.Enqueue(ctx, task); err != nil || !inserted {
 			t.Fatalf("enqueue task %d: inserted=%v err=%v", index, inserted, err)
 		}
@@ -188,7 +186,51 @@ func TestPhaseAOfflineWorkflow(t *testing.T) {
 	assertCount(t, store, `SELECT count(*) FROM operation_task WHERE state = 'succeeded'`, 20)
 	assertCount(t, store, `SELECT count(*) FROM execution_attempt`, 20)
 
-	createApprovedDraft(t, store, "account-b", accountB.Version, "draft-b", "confirmation-b")
+	for _, test := range []struct {
+		name, accountStatus, authorizationStatus, exitStatus, runtimeLease, wantState, wantReason string
+		binding, versionChanged, confirmationChanged                                              bool
+	}{
+		{name: "binding-missing", accountStatus: "active", authorizationStatus: "authorized", wantState: "policy_hold", wantReason: "binding_missing"},
+		{name: "exit-unhealthy", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "unhealthy", wantState: "policy_hold", wantReason: "exit_unhealthy"},
+		{name: "runtime-missing", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", wantState: "policy_hold", wantReason: "runtime_missing"},
+		{name: "runtime-expired", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "expired", wantState: "policy_hold", wantReason: "runtime_lease_expired"},
+		{name: "binding-version", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", versionChanged: true, wantState: "needs_confirmation", wantReason: "binding_version_changed"},
+		{name: "confirmation-version", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", confirmationChanged: true, wantState: "needs_confirmation", wantReason: "confirmation_version_changed"},
+		{name: "paused", accountStatus: "paused", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", wantState: "policy_hold", wantReason: "account_paused"},
+		{name: "revoked", accountStatus: "paused", authorizationStatus: "revoked", binding: true, exitStatus: "healthy", runtimeLease: "active", wantState: "policy_hold", wantReason: "account_revoked"},
+	} {
+		t.Run("gate "+test.name, func(t *testing.T) {
+			taskID := seedGateTask(t, store, test.name, test.accountStatus, test.authorizationStatus, test.exitStatus,
+				test.runtimeLease, test.binding, test.versionChanged, test.confirmationChanged)
+			if execution, err := store.ExecuteMock(ctx, "worker-gate-"+test.name, "succeeded"); err != nil || execution.WasClaimed {
+				t.Fatalf("invalid task was claimed: execution=%+v err=%v", execution, err)
+			}
+			assertTaskGate(t, store, taskID, test.wantState, test.wantReason)
+		})
+	}
+
+	// A held task is never revived in place: only a newly confirmed task with a new idempotency key may run.
+	if _, err := store.db.ExecContext(ctx, `
+		INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until)
+		VALUES ('runtime-recovered', 'gate-runtime-missing', 'binding-runtime-missing', 1, 'container-recovered', now() + interval '1 minute')`); err != nil {
+		t.Fatal(err)
+	}
+	if execution, err := store.ExecuteMock(ctx, "worker-old-held", "succeeded"); err != nil || execution.WasClaimed {
+		t.Fatalf("held task was revived after health recovery: execution=%+v err=%v", execution, err)
+	}
+	createApprovedDraft(t, store, "gate-runtime-missing", 1, "draft-recovered", "confirmation-recovered")
+	recovered := Task{
+		ID: "task-recovered", IdempotencyKey: "idempotency-recovered", AccountID: "gate-runtime-missing", AccountVersion: 1,
+		DraftID: "draft-recovered", DraftVersion: 1, ConfirmationID: "confirmation-recovered", ConfirmationVersion: 1,
+	}
+	if _, inserted, err := store.Enqueue(ctx, recovered); err != nil || !inserted {
+		t.Fatalf("enqueue recovered task: inserted=%v err=%v", inserted, err)
+	}
+	if execution, err := store.ExecuteMock(ctx, "worker-recovered", "succeeded"); err != nil || !execution.WasClaimed || execution.State != "succeeded" {
+		t.Fatalf("newly confirmed task did not run after recovery: execution=%+v err=%v", execution, err)
+	}
+	assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-gate-runtime-missing' AND state = 'policy_hold'`, 1)
+
 	unconfirmed := approvedTask(30, "account-b", accountB.Version, "draft-b", "")
 	unconfirmed.ConfirmationVersion = 0
 	if _, _, err := store.Enqueue(ctx, unconfirmed); err != nil {
@@ -338,12 +380,107 @@ func approvedTask(index int, accountID string, accountVersion int64, draftID, co
 	}
 }
 
+func seedGateTask(t *testing.T, store *Store, suffix, accountStatus, authorizationStatus, exitStatus, runtimeLease string,
+	binding, versionChanged, confirmationChanged bool) string {
+	t.Helper()
+	ctx := context.Background()
+	accountID, taskID := "gate-"+suffix, "task-gate-"+suffix
+	tx, err := store.db.BeginTx(ctx, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tx.Rollback()
+	if _, err := tx.ExecContext(ctx, `INSERT INTO credential_reference (id, provider, reference_key) VALUES ($1, 'os_keyring', $2)`,
+		"credential-"+suffix, "creatorhub/gate-"+suffix); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tx.ExecContext(ctx, `
+		INSERT INTO social_account
+			(id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status, status, version)
+		VALUES ($1, $2, 'mock', $1, 'owned', $3, $4, 1)`, accountID, "credential-"+suffix, authorizationStatus, accountStatus); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tx.ExecContext(ctx, `INSERT INTO content_draft (id, account_id, version, content) VALUES ($1, $2, 1, 'gate test')`,
+		"draft-gate-"+suffix, accountID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tx.ExecContext(ctx, `
+		INSERT INTO confirmation (id, account_id, account_version, draft_id, draft_version, version)
+		VALUES ($1, $2, 1, $3, 1, 1)`, "confirmation-gate-"+suffix, accountID, "draft-gate-"+suffix); err != nil {
+		t.Fatal(err)
+	}
+	if binding {
+		exitID := "exit-gate-" + suffix
+		if _, err := tx.ExecContext(ctx, `INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint)
+			VALUES ($1, $1, 'test-gateway', '1', '{"seed":3}')`, accountID); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tx.ExecContext(ctx, `INSERT INTO network_exit (id, protocol, host, port, health_status)
+			VALUES ($1, 'socks5', '127.0.0.1', 1080, $2)`, exitID, exitStatus); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tx.ExecContext(ctx, `
+			INSERT INTO environment_binding (id, account_id, browser_env_alias, network_exit_id, version)
+			VALUES ($1, $2, $2, $3, 1)`, "binding-"+suffix, accountID, exitID); err != nil {
+			t.Fatal(err)
+		}
+		if runtimeLease != "" {
+			interval := "1 minute"
+			if runtimeLease == "expired" {
+				interval = "-1 second"
+			}
+			if _, err := tx.ExecContext(ctx, `
+				INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until)
+				VALUES ($1, $2, $3, 1, $4, now() + $5::interval)`, "runtime-gate-"+suffix, accountID,
+				"binding-"+suffix, "container-gate-"+suffix, interval); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	if versionChanged {
+		if _, err := tx.ExecContext(ctx, `UPDATE environment_binding SET version = 2 WHERE account_id = $1`, accountID); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+	confirmationVersion := int64(1)
+	if confirmationChanged {
+		confirmationVersion = 2
+	}
+	if _, _, err := store.Enqueue(ctx, Task{
+		ID: taskID, IdempotencyKey: "idempotency-gate-" + suffix, AccountID: accountID, AccountVersion: 1,
+		DraftID: "draft-gate-" + suffix, DraftVersion: 1, ConfirmationID: "confirmation-gate-" + suffix,
+		ConfirmationVersion: confirmationVersion,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	return taskID
+}
+
+func assertTaskGate(t *testing.T, store *Store, taskID, wantState, wantReason string) {
+	t.Helper()
+	var state string
+	if err := store.db.QueryRow(`SELECT state FROM operation_task WHERE id = $1`, taskID).Scan(&state); err != nil || state != wantState {
+		t.Fatalf("task gate state: got=%q want=%q err=%v", state, wantState, err)
+	}
+	var attempts int
+	if err := store.db.QueryRow(`SELECT count(*) FROM execution_attempt WHERE task_id = $1`, taskID).Scan(&attempts); err != nil || attempts != 0 {
+		t.Fatalf("gated task executed: attempts=%d err=%v", attempts, err)
+	}
+	var reason string
+	if err := store.db.QueryRow(`SELECT reason_code FROM audit_event WHERE task_id = $1 AND event_type = 'task_quarantined' ORDER BY id DESC LIMIT 1`, taskID).Scan(&reason); err != nil || reason != wantReason {
+		t.Fatalf("task gate reason: got=%q want=%q err=%v", reason, wantReason, err)
+	}
+}
+
 func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) {
 	t.Helper()
 	for _, migrationFile := range []struct {
 		version int
 		name    string
-	}{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}} {
+	}{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}, {6, "006_runtime_cleanup.sql"}, {7, "007_runtime_binding_version.sql"}} {
 		var applied bool
 		if err := store.db.QueryRow(`SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migrationFile.version).Scan(&applied); err != nil {
 			t.Fatal(err)