creator-hub/cmd/control-plane/hub.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"

	"git.ipao.vip/rogee/creator-hub/internal/hub"
	"github.com/gofiber/fiber/v3"
	"github.com/sirupsen/logrus"
)

// hubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。
type hubStore interface {
	LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error)
	CreateGateway(ctx context.Context, name, endpoint, token string) (hub.Gateway, error)
	UpdateGateway(ctx context.Context, currentName, name, endpoint, token string) (hub.Gateway, error)
	ListGateways(ctx context.Context) ([]hub.Gateway, error)
	GetGateway(ctx context.Context, name string) (hub.Gateway, error)
	DeleteGateway(ctx context.Context, name string) error
	CreateImage(ctx context.Context, image hub.Image) error
	UpdateImage(ctx context.Context, image hub.Image) error
	ListImages(ctx context.Context, enabledOnly bool) ([]hub.Image, error)
	DeleteImage(ctx context.Context, version string) error
	ImageRef(ctx context.Context, version string) (string, error)
	CreateEnv(ctx context.Context, env hub.Env) error
	ListEnvs(ctx context.Context) ([]hub.Env, error)
	GetEnv(ctx context.Context, alias string) (hub.Env, error)
	UpgradeEnv(ctx context.Context, alias, version string) error
	CreateNetworkExit(ctx context.Context, exit hub.NetworkExit) (hub.NetworkExit, error)
	ListNetworkExits(ctx context.Context) ([]hub.NetworkExit, error)
	GetNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error)
	GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error)
	RecordNetworkExitCheck(ctx context.Context, id string, observation hub.ExitObservation, failureReason string) (hub.NetworkExit, string, error)
	DisableNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error)
	CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error)
	GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error)
	GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error)
	ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error
	RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkID ...string) (hub.EnvironmentContext, error)
	ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkID ...string) (hub.EnvironmentContext, error)
	ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error
	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
	AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error
}

type runtimeStopStore interface {
	LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error)
	GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error)
	GetGateway(ctx context.Context, name string) (hub.Gateway, error)
	ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error
	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
	AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error
}

type runtimeCleanupStore interface {
	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
}

const (
	gatewayLongTimeout       = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取
	gatewayReconcileDelay    = 100 * time.Millisecond
	gatewayReconcileAttempts = 10
	missingRuntimeID         = "runtime-not-found"
)

var gatewayGenerationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`)

// gatewayCall 调用某个网关的 /v1 路由；ok 为 false 时 status/body 携带网关错误。
func gatewayCall(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration) (status int, responseBody []byte, err error) {
	callCtx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()
	var payload io.Reader
	if body != nil {
		encoded, marshalErr := json.Marshal(body)
		if marshalErr != nil {
			return 0, nil, marshalErr
		}
		payload = bytes.NewReader(encoded)
	}
	request, requestErr := http.NewRequestWithContext(callCtx, method, target.Endpoint+path, payload)
	if requestErr != nil {
		return 0, nil, requestErr
	}
	request.Header.Set("Authorization", "Bearer "+target.Token)
	if body != nil {
		request.Header.Set("Content-Type", "application/json")
	}
	response, requestErr := http.DefaultClient.Do(request)
	if requestErr != nil {
		return 0, nil, requestErr
	}
	defer response.Body.Close()
	responseBody, err = io.ReadAll(io.LimitReader(response.Body, 1<<20))
	return response.StatusCode, responseBody, err
}

func gatewayCreatePayload(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any {
	fingerprint := environment.Fingerprint
	fingerprint.ProxyServer = ""
	fingerprint.DisableNonProxiedUDP = false
	cmd := append(fingerprint.Args(), "--remote-allow-origins=*", "about:blank")
	return map[string]any{
		"alias":           environment.Alias,
		"name":            environment.Name,
		"image":           imageRef,
		"cmd":             cmd,
		"volume":          "creatorhub-profile-" + environment.Alias,
		"binding_version": environment.BindingVersion,
		"network_exit_id": environment.Exit.ID,
		"network_exit":    networkExit,
	}
}

func gatewayCreatePayloadForAccount(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any {
	payload := gatewayCreatePayload(environment, imageRef, networkExit)
	if !accountRunnable(environment) {
		payload["network_exit_id"], payload["network_exit"], payload["stopped"] = "", gatewayNetworkExit{}, true
	}
	return payload
}

func gatewayProxyPayload(environment hub.EnvironmentContext, runtimeID, networkID string, networkExit gatewayNetworkExit) map[string]any {
	return map[string]any{
		"binding_version": environment.BindingVersion,
		"runtime_id":      runtimeID,
		"network_id":      networkID,
		"network_exit_id": environment.Exit.ID,
		"network_exit":    networkExit,
	}
}

func gatewayGenerationPayload(environment hub.EnvironmentContext) map[string]any {
	return map[string]any{"binding_version": environment.BindingVersion, "runtime_id": environment.RuntimeID, "network_id": environment.RuntimeNetworkID, "network_exit_id": environment.Exit.ID}
}

func gatewayCleanupGenerationPayload(environment hub.EnvironmentContext) map[string]any {
	bindingVersion, runtimeID, networkID := environment.RuntimeCleanupBindingVersion, environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID
	if bindingVersion == 0 {
		bindingVersion, runtimeID, networkID = environment.BindingVersion, environment.RuntimeID, environment.RuntimeNetworkID
	}
	return map[string]any{"binding_version": bindingVersion, "runtime_id": runtimeID, "network_id": networkID}
}

func runtimeCleanupGeneration(environment hub.EnvironmentContext, bindingVersion int64, runtimeID string, networkIDs ...string) hub.EnvironmentContext {
	environment.RuntimeCleanupBindingVersion = bindingVersion
	environment.RuntimeCleanupInstanceID = environment.RuntimeInstanceID
	environment.RuntimeCleanupRuntimeID = runtimeID
	environment.RuntimeCleanupNetworkID = environment.RuntimeNetworkID
	if len(networkIDs) == 1 {
		environment.RuntimeCleanupNetworkID = networkIDs[0]
	}
	return environment
}

func gatewayNetworkID(body []byte) string {
	var envelope struct {
		NetworkID string `json:"network_id"`
	}
	if json.Unmarshal(body, &envelope) == nil && gatewayGenerationIDPattern.MatchString(envelope.NetworkID) {
		return envelope.NetworkID
	}
	return ""
}

func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
	environment hub.EnvironmentContext, status int, callErr error, body []byte) error {
	// 网关 4xx 拒绝且未产生 network 代：创建在网关侧任何 Docker 变更之前就被确定性拒绝，
	// 无需清理 fence；其余情况（5xx、断连、无效 201）结果未知，仍走 fail-closed。
	deterministicRejection := callErr == nil && status >= 400 && status < 500
	logEntry := logrus.WithFields(logrus.Fields{
		"service":          "control-plane",
		"event_type":       "runtime_create_reconciliation",
		"alias":            environment.Alias,
		"gateway":          target.Name,
		"binding_version":  environment.BindingVersion,
		"gateway_status":   status,
		"network_id_known": gatewayNetworkID(body) != "",
		"deterministic":    deterministicRejection,
	})
	if callErr != nil {
		logEntry = logEntry.WithError(callErr)
	}
	if deterministicRejection {
		logEntry.Info("gateway rejected runtime create before Docker side effects")
	} else {
		logEntry.Warn("gateway runtime create result requires reconciliation")
	}
	return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion,
		gatewayNetworkID(body), deterministicRejection)
}

func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
	environment hub.EnvironmentContext, bindingVersion int64, networkID string, deterministicRejection bool) error {
	if networkID == "" {
		if deterministicRejection {
			// 网关明确拒绝：创建未发生、无副作用，不写 pending，让调用方透传真实失败原因，
			// 绑定保持可重试（否则会把可判定的失败写成永远无法自动清理的“未知代”毒状态）。
			return nil
		}
		pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "")
		return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
	}
	container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias)
	if err != nil {
		runtimeID := missingRuntimeID
		if bindingVersion == environment.BindingVersion && environment.RuntimeID != "" {
			runtimeID = environment.RuntimeID
		}
		pending := runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID)
		return errors.Join(err, store.SetRuntimeCleanupPending(ctx, pending, true))
	}
	if found && (container.BindingVersion != bindingVersion || container.NetworkID != networkID) {
		pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID)
		return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
	}
	if found {
		environment = runtimeCleanupGeneration(environment, bindingVersion, container.ID, networkID)
	} else {
		environment = runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID)
	}
	_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
	return cleanupErr
}

// gatewayFailure 将网关错误转换为对调用方可读的失败；4xx 保留原状态，其余一律 502。
type gatewayFailure struct {
	status  int
	message string
}

func (e gatewayFailure) Error() string { return e.message }

func gatewayRejected(status int, body []byte) error {
	if status >= 400 && status < 500 {
		return gatewayFailure{status: status, message: "gateway rejected: " + errorFromBody(body, status)}
	}
	return gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf("gateway call failed with status %d", status)}
}

func gatewayUnreachable(err error) error {
	return gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf("gateway unreachable: %v", err)}
}

func reconcileGatewayContainer(ctx context.Context, target hub.Gateway, alias string) (containerStatus, bool, error) {
	var lastErr error
	for attempt := 0; attempt < gatewayReconcileAttempts; attempt++ {
		if attempt > 0 {
			timer := time.NewTimer(gatewayReconcileDelay)
			select {
			case <-ctx.Done():
				timer.Stop()
				return containerStatus{}, false, ctx.Err()
			case <-timer.C:
			}
		}
		status, body, err := gatewayCall(ctx, target, http.MethodGet, "/v1/browsers", nil, 30*time.Second)
		if err != nil {
			lastErr = err
			continue
		}
		if status != http.StatusOK {
			lastErr = gatewayRejected(status, body)
			continue
		}
		browsers, parseErr := parseGatewayBrowserList(body)
		if parseErr != nil {
			lastErr = parseErr
			continue
		}
		lastErr = nil
		for _, browser := range browsers {
			if browser.Alias == alias {
				return browser, true, nil
			}
		}
	}
	return containerStatus{}, false, lastErr
}

func parseGatewayBrowserList(body []byte) ([]containerStatus, error) {
	var entries []json.RawMessage
	if err := json.Unmarshal(body, &entries); err != nil || entries == nil {
		return nil, errors.New("gateway returned an invalid browser list")
	}
	browsers := make([]containerStatus, 0, len(entries))
	aliases := make(map[string]bool, len(entries))
	for _, entry := range entries {
		var browser *containerStatus
		if err := json.Unmarshal(entry, &browser); err != nil || browser == nil || browser.ID == "" || browser.Alias == "" || browser.State == "" ||
			browser.BindingVersion < 0 || aliases[browser.Alias] {
			return nil, errors.New("gateway returned an invalid browser list")
		}
		aliases[browser.Alias] = true
		browsers = append(browsers, *browser)
	}
	return browsers, nil
}

func errorFromBody(body []byte, status int) string {
	var envelope struct {
		Error string `json:"error"`
	}
	if json.Unmarshal(body, &envelope) == nil && envelope.Error != "" {
		return envelope.Error
	}
	return http.StatusText(status)
}

type containerStatus struct {
	ID             string `json:"id"`
	Alias          string `json:"alias"`
	Name           string `json:"name"`
	State          string `json:"state"`
	Status         string `json:"status"`
	Endpoint       string `json:"endpoint"`
	BindingVersion int64  `json:"binding_version"`
	NetworkExitID  string `json:"network_exit_id"`
	NetworkID      string `json:"network_id"`
	ProxyReady     bool   `json:"proxy_ready"`
}

type envView struct {
	hub.Env
	State               string `json:"state"`
	Status              string `json:"status"`
	ContainerID         string `json:"container_id"`
	Endpoint            string `json:"endpoint"`
	AccountID           string `json:"account_id"`
	AccountStatus       string `json:"account_status"`
	AuthorizationStatus string `json:"authorization_status"`
	NetworkExitID       string `json:"network_exit_id"`
	NetworkExitHealth   string `json:"network_exit_health"`
	BindingVersion      int64  `json:"binding_version"`
	RuntimeInstanceID   string `json:"runtime_instance_id"`
	ScheduleStatus      string `json:"schedule_status"`
	ScheduleBlockReason string `json:"schedule_block_reason,omitempty"`
	RecoveryRequired    bool   `json:"recovery_required"`
	CleanupPending      bool   `json:"cleanup_pending"`
}

func environmentScheduleReadiness(environment hub.EnvironmentContext) (string, string) {
	switch {
	case environment.AuthorizationStatus != "authorized":
		return "blocked", "account_revoked"
	case environment.AccountStatus != "active":
		return "blocked", "account_paused"
	case environment.Exit.ID != "" && environment.Exit.HealthStatus != "healthy":
		return "blocked", "network_exit_unhealthy"
	case environment.RuntimeCleanupPending:
		return "blocked", "runtime_stop_pending"
	case environment.RuntimeInstanceID == "":
		return "blocked", "runtime_missing"
	default:
		return "ready", ""
	}
}

func containerMatchesBinding(container containerStatus, environment hub.EnvironmentContext) bool {
	if !validCreatedRuntime(container, environment, container.State == "running") || container.BindingVersion != environment.BindingVersion ||
		(container.NetworkExitID != environment.Exit.ID && (container.State == "running" || container.NetworkExitID != "")) {
		return false
	}
	return true
}

func validCreatedRuntime(created containerStatus, environment hub.EnvironmentContext, running bool) bool {
	if !gatewayGenerationIDPattern.MatchString(created.ID) ||
		(created.NetworkID != "" && !gatewayGenerationIDPattern.MatchString(created.NetworkID)) {
		return false
	}
	if !running {
		return true
	}
	if created.NetworkID == "" {
		return false
	}
	if environment.RuntimeInstanceID != "" || environment.RuntimeID != "" {
		return environment.RuntimeInstanceID != "" && environment.RuntimeID == created.ID &&
			environment.RuntimeNetworkID != "" && created.NetworkID == environment.RuntimeNetworkID
	}
	return environment.RuntimeNetworkID == "" || created.NetworkID == environment.RuntimeNetworkID
}

func accountRunnable(environment hub.EnvironmentContext) bool {
	return environment.AccountStatus == "active" && environment.AuthorizationStatus == "authorized"
}

func releaseRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error {
	err := store.ReleaseRuntime(ctx, environment)
	if errors.Is(err, hub.ErrConflict) {
		return nil
	}
	return err
}

func releaseRuntimeWithReconcileAudit(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error {
	action := actionForEnvironment("reconcile", environment)
	if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
		return err
	}
	releaseErr := releaseRuntime(ctx, store, environment)
	action.Outcome, action.ReasonCode = "succeeded", "runtime_released"
	if releaseErr != nil {
		action.Outcome, action.ReasonCode = "failed", "runtime_release_failed"
	}
	if auditErr := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); auditErr != nil {
		return errors.Join(releaseErr, auditErr)
	}
	return releaseErr
}

func registerHub(app *fiber.App, store hubStore) {
	registerHubWithNetwork(app, store, defaultNetworkExitProbe(), nil)
}

func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) {
	app.Get("/api/browsers", listBrowsers(store))
	app.Get("/api/browsers/:alias", getBrowser(store))
	app.Post("/api/browsers", lockBrowserCreate(store, createBrowser(store, probe, resolve)))
	app.Post("/api/browsers/:alias/:action", lockBrowserAlias(store, browserAction(store, probe, resolve)))
	app.Delete("/api/browsers/:alias", lockBrowserAlias(store, deleteBrowser(store)))
	app.Get("/api/network-exits", listNetworkExits(store))
	app.Get("/api/network-exits/:id", getNetworkExit(store))
	app.Post("/api/network-exits", createNetworkExit(store))
	app.Post("/api/network-exits/:id/check", checkNetworkExit(store, probe))
	app.Post("/api/network-exits/:id/disable", disableNetworkExit(store, probe, resolve))

	app.Get("/api/browser-images", func(c fiber.Ctx) error {
		images, err := store.ListImages(c.Context(), false)
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(images)
	})
	app.Post("/api/browser-images", func(c fiber.Ctx) error {
		input := struct {
			Version  string `json:"version"`
			ImageRef string `json:"image_ref"`
			Note     string `json:"note"`
			Enabled  *bool  `json:"enabled"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		enabled := true
		if input.Enabled != nil {
			enabled = *input.Enabled
		}
		if err := store.CreateImage(c.Context(), hub.Image{Version: input.Version, ImageRef: input.ImageRef, Note: input.Note, Enabled: enabled}); err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(map[string]any{
			"version": input.Version, "image_ref": input.ImageRef, "note": input.Note, "enabled": enabled,
		})
	})
	app.Put("/api/browser-images/:version", lockBrowserImage(store, func(c fiber.Ctx) error {
		input := struct {
			ImageRef string `json:"image_ref"`
			Note     string `json:"note"`
			Enabled  *bool  `json:"enabled"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		enabled := true
		if input.Enabled != nil {
			enabled = *input.Enabled
		}
		if err := store.UpdateImage(c.Context(), hub.Image{Version: c.Params("version"), ImageRef: input.ImageRef, Note: input.Note, Enabled: enabled}); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	}))
	app.Delete("/api/browser-images/:version", lockBrowserImage(store, func(c fiber.Ctx) error {
		if err := store.DeleteImage(c.Context(), c.Params("version")); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	}))

	app.Get("/api/gateways", func(c fiber.Ctx) error {
		gateways, err := store.ListGateways(c.Context())
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(gateways)
	})
	app.Post("/api/gateways", func(c fiber.Ctx) error {
		input := struct {
			Name     string `json:"name"`
			Endpoint string `json:"endpoint"`
			Token    string `json:"token"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		gateway, err := store.CreateGateway(c.Context(), input.Name, input.Endpoint, input.Token)
		if err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(gateway)
	})
	app.Put("/api/gateways/:name", func(c fiber.Ctx) error {
		input := struct {
			Name     string `json:"name"`
			Endpoint string `json:"endpoint"`
			Token    string `json:"token"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		gateway, err := store.UpdateGateway(c.Context(), c.Params("name"), input.Name, input.Endpoint, input.Token)
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(gateway)
	})
	app.Delete("/api/gateways/:name", func(c fiber.Ctx) error {
		if err := store.DeleteGateway(c.Context(), c.Params("name")); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	})
}

func getBrowser(store hubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(environment)
	}
}

func lockBrowserAlias(store hubStore, handler fiber.Handler) fiber.Handler {
	return func(c fiber.Ctx) error {
		requestedExitID, imageVersion := "", ""
		if c.Params("action") == "rebind" {
			var input struct {
				NetworkExitID string `json:"network_exit_id"`
			}
			if json.Unmarshal(c.Body(), &input) == nil {
				if hub.ValidNetworkExitID(input.NetworkExitID) {
					requestedExitID = input.NetworkExitID
				}
			}
		}
		if c.Params("action") == "upgrade" {
			var input struct {
				Version string `json:"version"`
			}
			if json.Unmarshal(c.Body(), &input) == nil {
				if hub.ValidImageVersion(input.Version) {
					imageVersion = input.Version
				}
			}
		}
		alias := c.Params("alias")
		unlock, err := lockAliasResources(c.Context(), store, alias, requestedExitID, imageVersion)
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		return handler(c)
	}
}

func lockBrowserImage(store hubStore, handler fiber.Handler) fiber.Handler {
	return func(c fiber.Ctx) error {
		unlock, err := store.LockResources(c.Context(), nil, nil, []string{c.Params("version")})
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		return handler(c)
	}
}

func lockBrowserCreate(store hubStore, handler fiber.Handler) fiber.Handler {
	return func(c fiber.Ctx) error {
		var input struct {
			Alias         string `json:"alias"`
			AccountID     string `json:"account_id"`
			NetworkExitID string `json:"network_exit_id"`
			ImageVersion  string `json:"image_version"`
		}
		if json.Unmarshal(c.Body(), &input) != nil || input.Alias == "" {
			return handler(c)
		}
		exitIDs, imageVersions := []string(nil), []string(nil)
		if hub.ValidNetworkExitID(input.NetworkExitID) {
			exitIDs = []string{input.NetworkExitID}
		}
		if hub.ValidImageVersion(input.ImageVersion) {
			imageVersions = []string{input.ImageVersion}
		}
		unlock, err := store.LockResources(c.Context(), nonEmpty(input.Alias, input.AccountID), exitIDs, imageVersions)
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		return handler(c)
	}
}

func lockEnvironmentResources(ctx context.Context, store hubStore, envs []hub.Env) (func(), error) {
	aliases := make([]string, 0, len(envs))
	seen := make(map[string]bool, len(envs))
	for _, env := range envs {
		if !seen[env.Alias] {
			seen[env.Alias] = true
			aliases = append(aliases, env.Alias)
		}
	}
	for {
		before, err := environmentResourceMap(ctx, store, aliases)
		if err != nil {
			return nil, err
		}
		exitIDs, imageVersions := environmentResourceValues(before)
		unlock, err := store.LockResources(ctx, aliases, exitIDs, imageVersions)
		if err != nil {
			return nil, err
		}
		after, err := environmentResourceMap(ctx, store, aliases)
		if err != nil {
			unlock()
			return nil, err
		}
		if equalEnvironmentResourceMaps(before, after) {
			return unlock, nil
		}
		unlock()
	}
}

type environmentResource struct {
	alias        string
	exitID       string
	imageVersion string
}

type resourceLockStore interface {
	LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error)
	GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error)
}

func lockAliasResources(ctx context.Context, store resourceLockStore, alias, requestedExitID, requestedImageVersion string) (func(), error) {
	for {
		before, beforeFound, err := environmentResources(ctx, store, alias)
		if err != nil {
			return nil, err
		}
		unlock, err := store.LockResources(ctx, []string{alias}, nonEmpty(before.exitID, requestedExitID), nonEmpty(before.imageVersion, requestedImageVersion))
		if err != nil {
			return nil, err
		}
		after, afterFound, err := environmentResources(ctx, store, alias)
		if err != nil {
			unlock()
			return nil, err
		}
		if beforeFound == afterFound && before == after {
			return unlock, nil
		}
		unlock()
	}
}

func lockAccountResources(ctx context.Context, store runtimeStopStore, accountID string) (func(), error) {
	if store == nil {
		return func() {}, nil
	}
	for {
		before, beforeFound, err := accountEnvironmentResources(ctx, store, accountID)
		if err != nil {
			return nil, err
		}
		unlock, err := store.LockResources(ctx, nonEmpty(accountID, before.alias), nonEmpty(before.exitID), nonEmpty(before.imageVersion))
		if err != nil {
			return nil, err
		}
		after, afterFound, err := accountEnvironmentResources(ctx, store, accountID)
		if err != nil {
			unlock()
			return nil, err
		}
		if beforeFound == afterFound && before == after {
			return unlock, nil
		}
		unlock()
	}
}

func accountEnvironmentResources(ctx context.Context, store runtimeStopStore, accountID string) (environmentResource, bool, error) {
	environment, err := store.GetEnvironmentContextForAccount(ctx, accountID)
	if errors.Is(err, hub.ErrNotFound) {
		return environmentResource{}, false, nil
	}
	if err != nil {
		return environmentResource{}, false, err
	}
	return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, imageVersion: environment.ImageVersion}, true, nil
}

func environmentResources(ctx context.Context, store resourceLockStore, alias string) (environmentResource, bool, error) {
	environment, err := store.GetEnvironmentContext(ctx, alias)
	if errors.Is(err, hub.ErrNotFound) {
		return environmentResource{}, false, nil
	}
	if err != nil {
		return environmentResource{}, false, err
	}
	return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, imageVersion: environment.ImageVersion}, true, nil
}

func environmentResourceMap(ctx context.Context, store hubStore, aliases []string) (map[string]environmentResource, error) {
	result := make(map[string]environmentResource, len(aliases))
	for _, alias := range aliases {
		resources, found, err := environmentResources(ctx, store, alias)
		if err != nil {
			return nil, err
		}
		if found {
			result[alias] = resources
		}
	}
	return result, nil
}

func nonEmpty(values ...string) []string {
	result := make([]string, 0, len(values))
	seen := make(map[string]bool, len(values))
	for _, value := range values {
		if value != "" && !seen[value] {
			seen[value] = true
			result = append(result, value)
		}
	}
	return result
}

func environmentResourceValues(values map[string]environmentResource) ([]string, []string) {
	exitIDs, imageVersions := make([]string, 0, len(values)), make([]string, 0, len(values))
	for _, value := range values {
		exitIDs = append(exitIDs, value.exitID)
		imageVersions = append(imageVersions, value.imageVersion)
	}
	return nonEmpty(exitIDs...), nonEmpty(imageVersions...)
}

func equalEnvironmentResourceMaps(left, right map[string]environmentResource) bool {
	if len(left) != len(right) {
		return false
	}
	for key, value := range left {
		other, ok := right[key]
		if !ok || other != value {
			return false
		}
	}
	return true
}

func listNetworkExits(store hubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		exits, err := store.ListNetworkExits(c.Context())
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(exits)
	}
}

func getNetworkExit(store hubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		exit, err := store.GetNetworkExit(c.Context(), c.Params("id"))
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(exit)
	}
}

func createNetworkExit(store hubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		var input struct {
			Protocol         string `json:"protocol"`
			Host             string `json:"host"`
			Port             int    `json:"port"`
			Username         string `json:"username"`
			Password         string `json:"password"`
			ExpectedPublicIP string `json:"expected_public_ip"`
			ExpectedRegion   string `json:"expected_region"`
		}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		exit, err := store.CreateNetworkExit(c.Context(), hub.NetworkExit{
			Protocol: input.Protocol, Host: input.Host, Port: input.Port,
			Username: input.Username, Password: input.Password,
			ExpectedPublicIP: input.ExpectedPublicIP, ExpectedRegion: input.ExpectedRegion,
		})
		if err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(exit)
	}
}

func checkNetworkExit(store hubStore, probe networkExitProbe) fiber.Handler {
	return func(c fiber.Ctx) error {
		unlock, err := store.LockResources(c.Context(), nil, []string{c.Params("id")}, nil)
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		access, err := store.GetNetworkExitAccess(c.Context(), c.Params("id"))
		if err != nil {
			return hubError(c, err)
		}
		if access.HealthStatus == "disabled" {
			return hubError(c, hub.ErrConflict)
		}
		observation, failureReason := probe.Check(c.Context(), access)
		exit, reason, err := store.RecordNetworkExitCheck(c.Context(), access.ID, observation, failureReason)
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(struct {
			hub.NetworkExit
			ReasonCode string `json:"reason_code"`
		}{exit, reason})
	}
}

func disableNetworkExit(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler {
	return func(c fiber.Ctx) error {
		unlock, err := store.LockResources(c.Context(), nil, []string{c.Params("id")}, nil)
		if err != nil {
			return hubError(c, err)
		}
		exit, err := store.DisableNetworkExit(c.Context(), c.Params("id"))
		unlock()
		if err != nil {
			return hubError(c, err)
		}
		if err := reconcileRuntimeLeasesUnlocked(c.Context(), store, probe, resolve); err != nil {
			return hubError(c, err)
		}
		return c.JSON(exit)
	}
}

func verifyNetworkExit(ctx context.Context, store hubStore, probe networkExitProbe, id string) (hub.NetworkExitAccess, string, error) {
	access, err := store.GetNetworkExitAccess(ctx, id)
	if err != nil {
		return hub.NetworkExitAccess{}, "exit_unavailable", err
	}
	if access.HealthStatus == "disabled" {
		return hub.NetworkExitAccess{}, "exit_disabled", hub.ErrConflict
	}
	observation, failureReason := probe.Check(ctx, access)
	exit, reason, err := store.RecordNetworkExitCheck(ctx, id, observation, failureReason)
	if err != nil {
		return hub.NetworkExitAccess{}, reason, err
	}
	if exit.HealthStatus != "healthy" {
		return hub.NetworkExitAccess{}, reason, hub.ErrConflict
	}
	access, err = store.GetNetworkExitAccess(ctx, id)
	return access, reason, err
}

func listBrowsers(store hubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		envs, err := store.ListEnvs(c.Context())
		if err != nil {
			return hubError(c, err)
		}
		views := make([]envView, 0, len(envs))
		for _, env := range envs {
			view := envView{Env: env, State: "missing", Status: "未记录运行实例", ScheduleStatus: "blocked", ScheduleBlockReason: "binding_missing", RecoveryRequired: true}
			if environment, contextErr := store.GetEnvironmentContext(c.Context(), env.Alias); contextErr == nil {
				view.AccountID, view.AccountStatus, view.AuthorizationStatus = environment.AccountID, environment.AccountStatus, environment.AuthorizationStatus
				view.NetworkExitID, view.NetworkExitHealth, view.BindingVersion = environment.Exit.ID, environment.Exit.HealthStatus, environment.BindingVersion
				view.RuntimeInstanceID = environment.RuntimeInstanceID
				view.ContainerID = environment.RuntimeID
				view.ScheduleStatus, view.ScheduleBlockReason = environmentScheduleReadiness(environment)
				view.CleanupPending = environment.RuntimeCleanupPending
				view.RecoveryRequired = (environment.Exit.ID != "" && environment.Exit.HealthStatus != "healthy") || environment.RuntimeCleanupPending
				if environment.RuntimeInstanceID != "" {
					view.State, view.Status = "running", "已记录运行实例"
				}
			} else if !errors.Is(contextErr, hub.ErrNotFound) {
				return hubError(c, contextErr)
			}
			views = append(views, view)
		}
		return c.JSON(views)
	}
}

func reconcileRuntimeLeases(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error {
	return reconcileRuntimeLeasesUnlocked(ctx, store, probe, resolve)
}

func reconcileRuntimeLeasesUnlocked(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error {
	envs, err := store.ListEnvs(ctx)
	if err != nil {
		return err
	}
	unlock, err := lockEnvironmentResources(ctx, store, envs)
	if err != nil {
		return err
	}
	defer unlock()
	containers, gatewayRead, gatewayErrors := gatewayContainerSnapshot(ctx, store, envs)
	return reconcileRuntimeSnapshot(ctx, store, probe, resolve, envs, containers, gatewayRead, gatewayErrors)
}

func gatewayContainerSnapshot(ctx context.Context, store hubStore, envs []hub.Env) (map[string]map[string]containerStatus, map[string]bool, map[string]error) {
	gateways := map[string]bool{}
	containers := map[string]map[string]containerStatus{}
	gatewayRead := map[string]bool{}
	gatewayErrors := map[string]error{}
	for _, env := range envs {
		if gateways[env.Gateway] {
			continue
		}
		gateways[env.Gateway] = true
		gateway, err := store.GetGateway(ctx, env.Gateway)
		if err != nil {
			gatewayErrors[env.Gateway] = gatewayUnreachable(err)
			continue
		}
		status, body, callErr := gatewayCall(ctx, gateway, http.MethodGet, "/v1/browsers", nil, 30*time.Second)
		if callErr != nil {
			gatewayErrors[env.Gateway] = gatewayUnreachable(callErr)
			continue
		}
		if status != http.StatusOK {
			gatewayErrors[env.Gateway] = gatewayRejected(status, body)
			continue
		}
		list, parseErr := parseGatewayBrowserList(body)
		if parseErr != nil {
			gatewayErrors[env.Gateway] = gatewayFailure{status: http.StatusBadGateway, message: "gateway returned an invalid browser list"}
			continue
		}
		byAlias := map[string]containerStatus{}
		for _, container := range list {
			byAlias[container.Alias] = container
		}
		containers[env.Gateway] = byAlias
		gatewayRead[env.Gateway] = true
	}
	return containers, gatewayRead, gatewayErrors
}

func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), envs []hub.Env,
	containers map[string]map[string]containerStatus, gatewayRead map[string]bool, gatewayErrors map[string]error) error {
	var firstErr error
	for _, env := range envs {
		if readErr := gatewayErrors[env.Gateway]; readErr != nil {
			if firstErr == nil {
				firstErr = readErr
			}
			continue
		}
		environment, err := store.GetEnvironmentContext(ctx, env.Alias)
		if errors.Is(err, hub.ErrNotFound) {
			continue
		}
		if err != nil {
			return err
		}
		if environment.RuntimeCleanupPending {
			target, targetErr := store.GetGateway(ctx, environment.Gateway)
			if targetErr != nil {
				return targetErr
			}
			if _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment); cleanupErr != nil {
				return cleanupErr
			}
			continue
		}
		container, found := containers[env.Gateway][env.Alias]
		if !accountRunnable(environment) {
			if found && container.State == "running" {
				if err := stopEnvironmentRuntime(ctx, store, environment); err != nil {
					return err
				}
			} else if gatewayRead[env.Gateway] && environment.RuntimeInstanceID != "" {
				if err := releaseRuntimeWithReconcileAudit(ctx, store, environment); err != nil {
					return err
				}
			}
			continue
		}
		if found && container.State == "running" {
			auditRecovery := !containerMatchesBinding(container, environment) || !container.ProxyReady
			action := actionForEnvironment("reconcile", environment)
			if auditRecovery {
				if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
					return err
				}
			}
			ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, container)
			if auditRecovery {
				current, contextErr := store.GetEnvironmentContext(ctx, environment.Alias)
				if contextErr == nil {
					action.RuntimeInstanceID, action.BindingVersion, action.NetworkExitID =
						current.RuntimeInstanceID, current.BindingVersion, current.Exit.ID
				} else {
					action.RuntimeInstanceID = ""
					action.BindingVersion = 0
					action.NetworkExitID = ""
					restoreErr = errors.Join(restoreErr, contextErr)
				}
				action.Outcome, action.ReasonCode = "succeeded", "runtime_reconciled"
				if !ready {
					action.Outcome, action.ReasonCode = "failed", "runtime_unavailable"
				}
				if restoreErr != nil {
					action.Outcome, action.ReasonCode = runtimeRecoveryFailure(ctx, store, environment.Alias, restoreErr)
				}
				if err := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); err != nil {
					return errors.Join(restoreErr, err)
				}
			}
			if restoreErr != nil {
				return restoreErr
			}
		} else if gatewayRead[env.Gateway] && environment.RuntimeInstanceID != "" {
			if err := releaseRuntimeWithReconcileAudit(ctx, store, environment); err != nil {
				return err
			}
		}
	}
	return firstErr
}

func runtimeRecoveryFailure(ctx context.Context, store hubStore, alias string, err error) (string, string) {
	var gatewayErr gatewayFailure
	if errors.As(err, &gatewayErr) {
		return "unknown", "gateway_result_unknown"
	}
	if environment, contextErr := store.GetEnvironmentContext(ctx, alias); contextErr == nil && environment.RuntimeCleanupPending {
		return "unknown", "cleanup_result_unknown"
	}
	return "failed", "runtime_persistence_failed"
}

func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkExitProbe,
	_ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, container containerStatus) (bool, error) {
	if environment.RuntimeCleanupPending {
		target, err := store.GetGateway(ctx, environment.Gateway)
		if err != nil {
			return false, err
		}
		_, err = removeGatewayRuntime(ctx, store, target, environment)
		return false, err
	}
	var access hub.NetworkExitAccess
	var err error
	if environment.Exit.ID != "" {
		access, _, err = verifyNetworkExit(ctx, store, probe, environment.Exit.ID)
		if err != nil {
			return false, discardRuntime(ctx, store, environment)
		}
	}
	target, err := store.GetGateway(ctx, environment.Gateway)
	if err != nil {
		return false, err
	}
	if containerMatchesBinding(container, environment) && container.ProxyReady {
		_, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID)
		return err == nil, err
	}
	networkExit := gatewayNetworkExit{}
	if environment.Exit.ID != "" {
		networkExit = gatewayNetworkExitFor(access)
	}
	if containerMatchesBinding(container, environment) {
		if environment.Exit.ID == "" {
			container.ProxyReady = true
		} else {
			status, _, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/proxy",
				gatewayProxyPayload(environment, container.ID, container.NetworkID, networkExit), 30*time.Second)
			if callErr == nil && status == http.StatusNoContent {
				container.ProxyReady = true
			}
		}
		if container.ProxyReady {
			_, err := activateGatewayRuntime(ctx, store, target, environment, container.ID, container.NetworkID)
			return err == nil, err
		}
	}
	_, removeErr := removeGatewayRuntime(ctx, store, target, environment)
	if removeErr != nil {
		return false, removeErr
	}
	environment, err = store.GetEnvironmentContext(ctx, environment.Alias)
	if err != nil {
		return false, err
	}
	imageRef, err := store.ImageRef(ctx, environment.ImageVersion)
	if err != nil {
		return false, err
	}
	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
		gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
	if callErr != nil {
		return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
	}
	if status != http.StatusCreated {
		return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
	}
	var created containerStatus
	if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
		return false, errors.Join(errors.New("gateway returned an invalid runtime generation"),
			reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
	}
	_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID)
	return err == nil, err
}

func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string) (hub.EnvironmentContext, error) {
	if !validCreatedRuntime(containerStatus{ID: runtimeID, NetworkID: networkID}, environment, true) {
		return hub.EnvironmentContext{}, hub.ErrConflict
	}
	current, err := store.ActivateRuntime(ctx, environment.Alias, runtimeID, environment.BindingVersion, environment.Exit.ID, networkID)
	if err == nil {
		return current, nil
	}
	cleanup := environment
	cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtimeID, networkID
	_, cleanupErr := removeGatewayRuntime(ctx, store, target, cleanup)
	return hub.EnvironmentContext{}, errors.Join(err, cleanupErr)
}

func discardRuntime(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error {
	target, targetErr := store.GetGateway(ctx, environment.Gateway)
	if targetErr != nil {
		return targetErr
	}
	_, err := removeGatewayRuntime(ctx, store, target, environment)
	return err
}

func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler {
	return func(c fiber.Ctx) error {
		input := struct {
			Alias         string          `json:"alias"`
			Name          string          `json:"name"`
			Gateway       string          `json:"gateway"`
			ImageVersion  string          `json:"image_version"`
			Fingerprint   hub.Fingerprint `json:"fingerprint"`
			AccountID     string          `json:"account_id"`
			NetworkExitID string          `json:"network_exit_id"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		if input.Fingerprint.ProxyServer != "" {
			return hubError(c, hub.ErrInvalid)
		}
		input.Fingerprint.DisableNonProxiedUDP = false
		env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, ImageVersion: input.ImageVersion, Fingerprint: input.Fingerprint}
		if err := env.Fingerprint.Validate(); err != nil {
			return c.Status(fiber.StatusBadRequest).JSON(map[string]string{"error": err.Error()})
		}
		environment, created, err := store.CreateBoundEnv(c.Context(), env, input.AccountID, input.NetworkExitID)
		if err != nil {
			return hubError(c, err)
		}
		action := actionForEnvironment("create", environment)
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		finish := func(outcome, reason string, current hub.EnvironmentContext) error {
			action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
			action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
			return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
		}
		var access hub.NetworkExitAccess
		if input.NetworkExitID != "" {
			var reason string
			access, reason, err = verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID)
			if err != nil {
				_ = finish("failed", reason, environment)
				return hubError(c, err)
			}
		}
		gateway, err := store.GetGateway(c.Context(), env.Gateway)
		if err != nil {
			_ = finish("failed", "gateway_unavailable", environment)
			return hubError(c, err)
		}
		if environment.RuntimeCleanupPending {
			if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil {
				_ = finish("unknown", "cleanup_result_unknown", environment)
				return hubError(c, cleanupErr)
			}
			environment, err = store.GetEnvironmentContext(c.Context(), env.Alias)
			if err != nil {
				return hubError(c, err)
			}
		}
		running := accountRunnable(environment)
		imageRef, err := store.ImageRef(c.Context(), env.ImageVersion)
		if err != nil {
			_ = finish("failed", "image_unavailable", environment)
			return hubError(c, err)
		}
		networkExit := gatewayNetworkExit{}
		if input.NetworkExitID != "" {
			networkExit = gatewayNetworkExitFor(access)
		}
		if !created {
			container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, env.Alias)
			if reconcileErr != nil {
				_ = finish("unknown", "gateway_result_unknown", environment)
				return hubError(c, gatewayUnreachable(reconcileErr))
			}
			if found && !running {
				if !containerMatchesBinding(container, environment) {
					_ = finish("failed", "runtime_unavailable", environment)
					return hubError(c, hub.ErrConflict)
				}
				if container.State == "running" {
					stopped := environment
					stopped.RuntimeID, stopped.RuntimeNetworkID = container.ID, container.NetworkID
					if err := stopEnvironmentRuntime(c.Context(), store, stopped); err != nil {
						_ = finish("unknown", "cleanup_result_unknown", environment)
						return hubError(c, err)
					}
				}
				if err := finish("succeeded", "environment_reused", environment); err != nil {
					return hubError(c, err)
				}
				return c.JSON(map[string]string{"alias": env.Alias})
			}
			if found && container.State == "running" {
				ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, container)
				if restoreErr != nil {
					outcome, reason := runtimeRecoveryFailure(c.Context(), store, environment.Alias, restoreErr)
					_ = finish(outcome, reason, environment)
					return hubError(c, restoreErr)
				}
				if !ready {
					_ = finish("failed", "runtime_unavailable", environment)
					return c.Status(fiber.StatusConflict).JSON(map[string]string{"error": "environment exists but runtime is not running"})
				}
				environment, err = store.GetEnvironmentContext(c.Context(), env.Alias)
				if err != nil {
					return hubError(c, err)
				}
				if err := finish("succeeded", "environment_reused", environment); err != nil {
					return hubError(c, err)
				}
				return c.JSON(map[string]string{"alias": env.Alias})
			}
		}
		status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
			gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
		if callErr != nil || status != http.StatusCreated {
			createErr := gatewayRejected(status, body)
			if callErr != nil {
				createErr = gatewayUnreachable(callErr)
			}
			reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
			if reconcileErr != nil {
				_ = finish("unknown", "gateway_result_unknown", environment)
				return hubError(c, errors.Join(createErr, reconcileErr))
			}
			_ = finish("failed", "gateway_create_failed", environment)
			return hubError(c, createErr)
		}
		var createdRuntime containerStatus
		if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
			reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
			_ = finish("unknown", "gateway_result_unknown", environment)
			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway,
				message: "gateway create result unknown; environment retained for reconciliation"}, reconcileErr))
		}
		if !running {
			if err := finish("succeeded", "environment_created", environment); err != nil {
				return hubError(c, err)
			}
			return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias})
		}
		environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID)
		if err != nil {
			_ = finish("failed", "runtime_persistence_failed", environment)
			return hubError(c, err)
		}
		if err := finish("succeeded", "environment_created", environment); err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias})
	}
}

func browserAction(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler {
	return func(c fiber.Ctx) error {
		switch c.Params("action") {
		case "start", "stop":
			return lifecycleAction(store, probe, resolve, c)
		case "upgrade":
			return upgradeBrowser(store, probe, resolve, c)
		case "rebind":
			return rebindBrowser(store, probe, resolve, c)
		default:
			return hubError(c, hub.ErrInvalid)
		}
	}
}

func lifecycleAction(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error {
	environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
	if err != nil {
		return hubError(c, err)
	}
	actionName := strings.Clone(c.Params("action"))
	if actionName == "start" {
		if !accountRunnable(environment) {
			return hubError(c, hub.ErrConflict)
		}
		action := actionForEnvironment(actionName, environment)
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		finish := func(outcome, reason string, current hub.EnvironmentContext) error {
			action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
			action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
			return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
		}
		return startBrowser(store, probe, resolve, c, environment, finish)
	}
	if err := stopEnvironmentRuntime(c.Context(), store, environment); err != nil {
		return hubError(c, err)
	}
	return c.SendStatus(fiber.StatusNoContent)
}

func stopAccountRuntime(ctx context.Context, store runtimeStopStore, accountID string) error {
	environment, err := store.GetEnvironmentContextForAccount(ctx, accountID)
	if errors.Is(err, hub.ErrNotFound) {
		return nil
	}
	if err != nil {
		return err
	}
	return stopEnvironmentRuntime(ctx, store, environment)
}

func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error {
	action := actionForEnvironment("stop", environment)
	if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
		return err
	}
	finish := func(outcome, reason string) error {
		action.Outcome, action.ReasonCode = outcome, reason
		return store.AppendEnvironmentAction(ctx, "environment_action_finished", action)
	}
	gateway, err := store.GetGateway(ctx, environment.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable")
		return err
	}
	if environment.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(ctx, store, gateway, environment); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown")
			return cleanupErr
		}
		return finish("succeeded", "environment_stopped")
	}
	if environment.RuntimeID == "" {
		container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown")
			return gatewayUnreachable(reconcileErr)
		}
		if !found {
			return finish("succeeded", "environment_stopped")
		}
		if container.BindingVersion != environment.BindingVersion {
			_ = finish("failed", "runtime_generation_conflict")
			return hub.ErrConflict
		}
		environment = runtimeCleanupGeneration(environment, environment.BindingVersion, container.ID, container.NetworkID)
	} else {
		environment = runtimeCleanupGeneration(environment, environment.BindingVersion, environment.RuntimeID)
	}
	if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
		_ = finish("failed", "runtime_release_failed")
		return err
	}
	status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop",
		gatewayCleanupGenerationPayload(environment), 30*time.Second)
	if callErr != nil || status >= http.StatusInternalServerError {
		container, found, reconcileErr := reconcileGatewayContainer(ctx, gateway, environment.Alias)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown")
			if callErr != nil {
				return gatewayUnreachable(callErr)
			}
			return gatewayRejected(status, body)
		}
		if !found || container.State != "running" {
			if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
				_ = finish("failed", "runtime_release_failed")
				return err
			}
			return finish("succeeded", "gateway_reconciled")
		}
		_ = finish("failed", "gateway_action_failed")
		if callErr != nil {
			return gatewayUnreachable(callErr)
		}
		return gatewayRejected(status, body)
	}
	if status != http.StatusNoContent && status != http.StatusNotModified && status != http.StatusNotFound {
		_ = finish("failed", "gateway_rejected")
		return gatewayRejected(status, body)
	}
	if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
		_ = finish("failed", "runtime_release_failed")
		return err
	}
	return finish("succeeded", "environment_stopped")
}

func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx,
	environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error {
	if !accountRunnable(environment) {
		return hubError(c, hub.ErrConflict)
	}
	var access hub.NetworkExitAccess
	var err error
	if environment.Exit.ID != "" {
		var reason string
		access, reason, err = verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID)
		if err != nil {
			if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil {
				_ = finish("unknown", "cleanup_result_unknown", environment)
				return hubError(c, cleanupErr)
			}
			_ = finish("failed", reason, environment)
			return hubError(c, err)
		}
	}
	gateway, err := store.GetGateway(c.Context(), environment.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable", environment)
		return hubError(c, err)
	}
	if environment.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown", environment)
			return hubError(c, cleanupErr)
		}
		environment, err = store.GetEnvironmentContext(c.Context(), environment.Alias)
		if err != nil {
			return hubError(c, err)
		}
		if !accountRunnable(environment) {
			return hubError(c, hub.ErrConflict)
		}
	}
	imageRef, err := store.ImageRef(c.Context(), environment.ImageVersion)
	if err != nil {
		if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown", environment)
			return hubError(c, cleanupErr)
		}
		_ = finish("failed", "image_unavailable", environment)
		return hubError(c, err)
	}
	networkExit := gatewayNetworkExit{}
	if environment.Exit.ID != "" {
		networkExit = gatewayNetworkExitFor(access)
	}
	container, found, err := reconcileGatewayContainer(c.Context(), gateway, environment.Alias)
	if err != nil {
		_ = finish("unknown", "gateway_result_unknown", environment)
		return hubError(c, gatewayUnreachable(err))
	}
	if found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) {
		environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, container.ID, container.NetworkID)
		if err != nil {
			_ = finish("failed", "runtime_persistence_failed", environment)
			return hubError(c, err)
		}
		if err := finish("succeeded", "gateway_reconciled", environment); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	}
	if found {
		if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil {
			_ = finish("unknown", "gateway_result_unknown", environment)
			return hubError(c, removeErr)
		}
	} else if err := releaseRuntime(c.Context(), store, environment); err != nil {
		_ = finish("failed", "runtime_release_failed", environment)
		return hubError(c, err)
	}
	status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
		gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout)
	if callErr != nil || status != http.StatusCreated {
		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown", environment)
			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr))
		}
		_ = finish("failed", "gateway_create_failed", environment)
		if callErr != nil {
			return hubError(c, gatewayUnreachable(callErr))
		}
		return hubError(c, gatewayRejected(status, body))
	}
	var created containerStatus
	if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
		_ = finish("unknown", "gateway_result_unknown", environment)
		return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr))
	}
	environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, created.ID, created.NetworkID)
	if err != nil {
		_ = finish("failed", "runtime_persistence_failed", environment)
		return hubError(c, err)
	}
	if err := finish("succeeded", "environment_started", environment); err != nil {
		return hubError(c, err)
	}
	return c.SendStatus(fiber.StatusNoContent)
}

func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error {
	var input struct {
		Version string `json:"version"`
	}
	if err := decodeHubJSON(c, &input); err != nil {
		return hubError(c, err)
	}
	environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
	if err != nil {
		return hubError(c, err)
	}
	var imageRef string
	var imageErr error
	if !hub.ValidImageVersion(input.Version) {
		imageErr = hub.ErrInvalid
	} else {
		imageRef, imageErr = store.ImageRef(c.Context(), input.Version)
	}
	action := actionForEnvironment("upgrade", environment)
	action.OldImageVersion = environment.ImageVersion
	if imageErr != nil {
		action.NewImageVersion, action.ReasonCode = "", "upgrade_input_rejected"
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		action.Outcome = "failed"
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action); err != nil {
			return hubError(c, err)
		}
		return hubError(c, imageErr)
	}
	action.NewImageVersion = input.Version
	if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
		return hubError(c, err)
	}
	finish := func(outcome, reason string, current hub.EnvironmentContext) error {
		action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
		action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
		return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
	}
	access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID)
	if err != nil {
		_ = finish("failed", reason, environment)
		return hubError(c, err)
	}
	gateway, err := store.GetGateway(c.Context(), environment.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable", environment)
		return hubError(c, err)
	}
	if environment.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown", environment)
			return hubError(c, cleanupErr)
		}
		environment, err = store.GetEnvironmentContext(c.Context(), environment.Alias)
		if err != nil {
			return hubError(c, err)
		}
	}
	running := accountRunnable(environment)
	networkExit := gatewayNetworkExitFor(access)
	// 先删容器（保留卷）；404 视为已删除，保证升级可重试。
	if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil {
		_ = finish("unknown", "cleanup_result_unknown", environment)
		return hubError(c, removeErr)
	}
	if err := store.UpgradeEnv(c.Context(), environment.Alias, input.Version); err != nil {
		_ = finish("failed", "persistence_failed", environment)
		return hubError(c, err)
	}
	environment, err = store.GetEnvironmentContext(c.Context(), environment.Alias)
	if err != nil {
		_ = finish("failed", "persistence_failed", environment)
		return hubError(c, err)
	}
	running = accountRunnable(environment)
	status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
		gatewayCreatePayloadForAccount(environment, imageRef, networkExit), gatewayLongTimeout)
	var createdRuntime containerStatus
	if callErr != nil || status != http.StatusCreated {
		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown", environment)
			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr))
		}
		_ = finish("failed", "gateway_create_failed", environment)
		if callErr != nil {
			return hubError(c, gatewayUnreachable(callErr))
		}
		return hubError(c, gatewayRejected(status, body))
	}
	if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
		reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
		_ = finish("unknown", "gateway_result_unknown", environment)
		return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}, reconcileErr))
	}
	if !running {
		if err := finish("succeeded", "environment_upgraded", environment); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	}
	environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID)
	if err != nil {
		_ = finish("failed", "runtime_persistence_failed", environment)
		return hubError(c, err)
	}
	if err := finish("succeeded", "environment_upgraded", environment); err != nil {
		return hubError(c, err)
	}
	return c.SendStatus(fiber.StatusNoContent)
}

type runtimeCreateSpec struct {
	imageRef    string
	networkExit gatewayNetworkExit
}

func prepareRuntimeCreate(ctx context.Context, store hubStore, _ func(hub.NetworkExitAccess) (string, error),
	environment hub.EnvironmentContext, access hub.NetworkExitAccess) (runtimeCreateSpec, error) {
	imageRef, err := store.ImageRef(ctx, environment.ImageVersion)
	if err != nil {
		return runtimeCreateSpec{}, err
	}
	networkExit := gatewayNetworkExit{}
	if environment.Exit.ID != "" {
		networkExit = gatewayNetworkExitFor(access)
	}
	return runtimeCreateSpec{imageRef: imageRef, networkExit: networkExit}, nil
}

func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (containerStatus, error) {
	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
		gatewayCreatePayloadForAccount(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout)
	unknown := containerStatus{NetworkID: gatewayNetworkID(body)}
	if callErr == nil && status == http.StatusCreated {
		var created containerStatus
		if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, accountRunnable(environment)) {
			created.Alias, created.BindingVersion = environment.Alias, environment.BindingVersion
			if accountRunnable(environment) {
				created.State, created.NetworkExitID, created.ProxyReady = "running", environment.Exit.ID, true
			} else {
				created.State = "exited"
			}
			return created, nil
		}
	}
	if callErr != nil {
		return unknown, gatewayUnreachable(callErr)
	}
	if status != http.StatusCreated {
		return unknown, gatewayRejected(status, body)
	}
	return unknown, errors.New("gateway returned an invalid runtime generation")
}

func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, imageRef string) error {
	payload := gatewayCreatePayload(environment, imageRef, gatewayNetworkExit{})
	payload["network_exit_id"], payload["stopped"] = "", true
	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", payload, gatewayLongTimeout)
	if callErr == nil && status == http.StatusCreated {
		var created containerStatus
		if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, false) {
			return nil
		}
		return errors.New("gateway returned an invalid stopped runtime generation")
	}
	if callErr != nil {
		return gatewayUnreachable(callErr)
	}
	if status != http.StatusCreated {
		return gatewayRejected(status, body)
	}
	return errors.New("gateway did not preserve the stopped runtime")
}

func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) {
	removed := environment.RuntimeCleanupPending
	if environment.RuntimeCleanupPending {
		if environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "" {
			// 未知代 pending：创建结果未知时登记的 runtime-not-found 没有实物 ID 可供清理。
			// 按别名反查网关：无容器则该代无实物，直接解除；有同版本容器则采用其实物代继续清理；
			// 版本不符说明别名下已是别的代，维持人工对账。
			if environment.RuntimeCleanupBindingVersion < 1 {
				return false, hub.ErrReconcileRequired
			}
			discovered, found, discoverErr := reconcileGatewayContainer(ctx, target, environment.Alias)
			if discoverErr != nil {
				return false, discoverErr
			}
			if !found {
				return true, store.SetRuntimeCleanupPending(ctx, environment, false)
			}
			if discovered.BindingVersion != environment.RuntimeCleanupBindingVersion {
				return false, hub.ErrReconcileRequired
			}
			if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
				return false, err
			}
			environment = runtimeCleanupGeneration(environment, environment.RuntimeCleanupBindingVersion, discovered.ID, discovered.NetworkID)
			if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
				return false, err
			}
		} else if environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" {
			return false, hub.ErrReconcileRequired
		}
	}
	if !environment.RuntimeCleanupPending {
		container, found, err := reconcileGatewayContainer(ctx, target, environment.Alias)
		if err != nil {
			return false, err
		}
		bindingVersion, runtimeID, networkID := environment.BindingVersion, environment.RuntimeID, environment.RuntimeNetworkID
		if environment.RuntimeCleanupBindingVersion > 0 {
			bindingVersion, runtimeID, networkID = environment.RuntimeCleanupBindingVersion,
				environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID
		}
		if found {
			if container.BindingVersion != bindingVersion || runtimeID == missingRuntimeID ||
				(runtimeID != "" && runtimeID != container.ID) || (runtimeID == "" && networkID == "" && container.State == "running") {
				return false, hub.ErrConflict
			}
			runtimeID = container.ID
			if networkID != "" {
				if container.NetworkID != "" && container.NetworkID != networkID {
					return false, hub.ErrConflict
				}
			} else {
				networkID = container.NetworkID
			}
		} else if environment.RuntimeID == "" {
			runtimeID = missingRuntimeID
		}
		environment = runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID)
	}
	if !environment.RuntimeCleanupPending {
		if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
			return false, err
		}
	}
	for attempt := 0; attempt < 2; attempt++ {
		status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias,
			gatewayCleanupGenerationPayload(environment), 30*time.Second)
		if callErr == nil && (status == http.StatusNoContent || status == http.StatusNotFound) {
			return true, store.SetRuntimeCleanupPending(ctx, environment, false)
		}
		if callErr == nil && status == http.StatusAccepted {
			removed = true
			continue
		}
		_, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias)
		if reconcileErr != nil {
			if callErr != nil {
				return removed, errors.Join(gatewayUnreachable(callErr), gatewayUnreachable(reconcileErr))
			}
			return removed, errors.Join(gatewayRejected(status, body), gatewayUnreachable(reconcileErr))
		}
		if found {
			if callErr != nil {
				return removed, gatewayUnreachable(callErr)
			}
			return removed, gatewayRejected(status, body)
		}
		removed = true
		if callErr != nil {
			return true, gatewayUnreachable(callErr)
		}
		return true, gatewayRejected(status, body)
	}
	if removed {
		return true, gatewayFailure{status: http.StatusBadGateway, message: "gateway removed the container but network cleanup is pending"}
	}
	return false, gatewayFailure{status: http.StatusBadGateway, message: "gateway runtime cleanup is pending"}
}

func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error),
	target hub.Gateway, environment hub.EnvironmentContext, previous containerStatus, found bool, prepared *runtimeCreateSpec) (bool, error) {
	if !found {
		return true, releaseRuntime(ctx, store, environment)
	}
	if environment.Exit.ID == "" {
		if err := releaseRuntime(ctx, store, environment); err != nil {
			return false, err
		}
		if previous.State == "running" {
			return false, nil
		}
		imageRef := ""
		if prepared != nil {
			imageRef = prepared.imageRef
		} else {
			var err error
			imageRef, err = store.ImageRef(ctx, environment.ImageVersion)
			if err != nil {
				return false, err
			}
		}
		if err := createStoppedGatewayRuntime(ctx, target, environment, imageRef); err != nil {
			_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
			return false, errors.Join(err, cleanupErr)
		}
		return true, nil
	}
	spec := runtimeCreateSpec{}
	if prepared != nil {
		spec = *prepared
	} else {
		access, err := store.GetNetworkExitAccess(ctx, environment.Exit.ID)
		if err != nil {
			return false, err
		}
		spec, err = prepareRuntimeCreate(ctx, store, resolve, environment, access)
		if err != nil {
			return false, err
		}
	}
	if err := releaseRuntime(ctx, store, environment); err != nil {
		return false, err
	}
	cleanupContext, err := store.GetEnvironmentContext(ctx, environment.Alias)
	if err != nil {
		return false, err
	}
	created, err := createGatewayRuntime(ctx, target, environment, spec)
	if err != nil {
		cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID, false)
		return false, errors.Join(err, cleanupErr)
	}
	if previous.State == "running" && accountRunnable(environment) {
		_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID)
	} else if accountRunnable(environment) {
		stopped := environment
		stopped.RuntimeID, stopped.RuntimeNetworkID = created.ID, created.NetworkID
		status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop",
			gatewayGenerationPayload(stopped), 30*time.Second)
		if callErr != nil {
			err = gatewayUnreachable(callErr)
		} else if status != http.StatusNoContent {
			err = gatewayRejected(status, body)
		}
	}
	if err != nil {
		_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
		return false, errors.Join(err, cleanupErr)
	}
	return true, nil
}

func rebindRecoveryError(ready bool, err error) error {
	if err != nil {
		return err
	}
	if !ready {
		return gatewayFailure{status: http.StatusBadGateway, message: "rebind recovery is incomplete; retry reconciliation"}
	}
	return nil
}

func runtimeCreateSpecMatches(ctx context.Context, store hubStore, current, previous hub.EnvironmentContext, prepared *runtimeCreateSpec) bool {
	if prepared == nil || current.BindingVersion != previous.BindingVersion || current.ImageVersion != previous.ImageVersion || current.Exit.ID != previous.Exit.ID {
		return false
	}
	imageRef, err := store.ImageRef(ctx, current.ImageVersion)
	return err == nil && imageRef == prepared.imageRef
}

func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error {
	var input struct {
		NetworkExitID string `json:"network_exit_id"`
	}
	if err := decodeHubJSON(c, &input); err != nil {
		return hubError(c, err)
	}
	before, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
	if err != nil {
		return hubError(c, err)
	}
	action := actionForEnvironment("rebind", before)
	if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
		return hubError(c, err)
	}
	finish := func(outcome, reason string, current hub.EnvironmentContext) error {
		action.Outcome, action.ReasonCode = outcome, reason
		action.NetworkExitID, action.BindingVersion, action.RuntimeInstanceID = current.Exit.ID, current.BindingVersion, current.RuntimeInstanceID
		return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
	}
	if !hub.ValidNetworkExitID(input.NetworkExitID) {
		if err := finish("failed", "rebind_input_rejected", before); err != nil {
			return hubError(c, err)
		}
		return hubError(c, hub.ErrInvalid)
	}
	access, reason, err := verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID)
	if err != nil {
		_ = finish("failed", reason, before)
		return hubError(c, err)
	}
	target, err := store.GetGateway(c.Context(), before.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable", before)
		return hubError(c, err)
	}
	if before.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown", before)
			return hubError(c, cleanupErr)
		}
		before, err = store.GetEnvironmentContext(c.Context(), before.Alias)
		if err != nil {
			return hubError(c, err)
		}
	}
	container, found, err := reconcileGatewayContainer(c.Context(), target, before.Alias)
	if err != nil {
		_ = finish("unknown", "gateway_result_unknown", before)
		return hubError(c, gatewayUnreachable(err))
	}
	wasRunning := found && container.State == "running"
	if err := store.ValidateEnvironmentRebind(c.Context(), before.Alias, input.NetworkExitID, before.BindingVersion); err != nil {
		_ = finish("failed", "rebind_not_allowed", before)
		return hubError(c, err)
	}
	var previousSpec *runtimeCreateSpec
	if found && before.Exit.ID == "" {
		imageRef, imageErr := store.ImageRef(c.Context(), before.ImageVersion)
		if imageErr != nil {
			_ = finish("failed", "runtime_prepare_failed", before)
			return hubError(c, imageErr)
		}
		previousSpec = &runtimeCreateSpec{imageRef: imageRef}
	} else if found {
		previousAccess, accessErr := store.GetNetworkExitAccess(c.Context(), before.Exit.ID)
		if accessErr != nil {
			_ = finish("failed", "exit_unavailable", before)
			return hubError(c, accessErr)
		}
		prepared, prepareErr := prepareRuntimeCreate(c.Context(), store, resolve, before, previousAccess)
		if prepareErr != nil {
			_ = finish("failed", "runtime_prepare_failed", before)
			return hubError(c, prepareErr)
		}
		previousSpec = &prepared
	}
	candidate := before
	candidate.Exit, candidate.BindingVersion = access.NetworkExit, before.BindingVersion+1
	candidate.RuntimeInstanceID, candidate.RuntimeID = "", ""
	var nextSpec runtimeCreateSpec
	if wasRunning {
		nextSpec, err = prepareRuntimeCreate(c.Context(), store, resolve, candidate, access)
		if err != nil {
			_ = finish("failed", "runtime_prepare_failed", before)
			return hubError(c, err)
		}
	}
	if found {
		_, removeErr := removeGatewayRuntime(c.Context(), store, target, before)
		if removeErr != nil {
			_ = finish("unknown", "gateway_result_unknown", before)
			return hubError(c, removeErr)
		}
	}
	var candidateRuntime containerStatus
	if wasRunning {
		candidateRuntime, err = createGatewayRuntime(c.Context(), target, candidate, nextSpec)
		if err != nil {
			cleanupContext, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			if contextErr != nil {
				_ = finish("unknown", "gateway_result_unknown", before)
				return hubError(c, errors.Join(err, contextErr))
			}
			cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID, false)
			if cleanupErr != nil {
				_ = finish("unknown", "gateway_result_unknown", before)
				return hubError(c, errors.Join(err, cleanupErr))
			}
			current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			ready := false
			if contextErr == nil {
				ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, container, found, previousSpec)
			}
			contextErr = rebindRecoveryError(ready, contextErr)
			if contextErr != nil {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, contextErr))
			}
			_ = finish("failed", "gateway_create_failed", before)
			return hubError(c, err)
		}
	}
	runtimeID := ""
	if accountRunnable(candidate) {
		runtimeID = candidateRuntime.ID
	}
	after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, runtimeID, before.BindingVersion, candidateRuntime.NetworkID)
	if err != nil {
		if candidateRuntime.ID != "" {
			cleanup, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			if contextErr != nil || cleanup.BindingVersion != before.BindingVersion || cleanup.Exit.ID != before.Exit.ID {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, contextErr, hub.ErrConflict))
			}
			cleanup.RuntimeID, cleanup.RuntimeNetworkID = candidateRuntime.ID, candidateRuntime.NetworkID
			_, cleanupErr := removeGatewayRuntime(c.Context(), store, target, cleanup)
			if cleanupErr != nil {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, cleanupErr))
			}
		}
		if found {
			current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			ready := false
			if contextErr == nil {
				prepared := previousSpec
				if !runtimeCreateSpecMatches(c.Context(), store, current, before, previousSpec) {
					prepared = nil
				}
				ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, container, true, prepared)
			}
			contextErr = rebindRecoveryError(ready, contextErr)
			if contextErr != nil {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, contextErr))
			}
		}
		_ = finish("failed", "rebind_not_allowed", before)
		return hubError(c, err)
	}
	if err := finish("succeeded", "environment_rebound", after); err != nil {
		return hubError(c, err)
	}
	return c.JSON(after)
}

func deleteBrowser(store hubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
		if err != nil {
			return hubError(c, err)
		}
		action := actionForEnvironment("recycle", environment)
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		gateway, err := store.GetGateway(c.Context(), environment.Gateway)
		if err != nil {
			action.Outcome, action.ReasonCode = "failed", "gateway_unavailable"
			_ = store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
			return hubError(c, err)
		}
		if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil {
			action.Outcome, action.ReasonCode = "unknown", "cleanup_result_unknown"
			_ = store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
			return hubError(c, removeErr)
		}
		action.Outcome, action.ReasonCode = "succeeded", "environment_recycled"
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	}
}

func actionForEnvironment(actionName string, environment hub.EnvironmentContext) hub.EnvironmentAction {
	return hub.EnvironmentAction{
		OperationID: hub.NewOperationID(), Action: actionName, ReasonCode: "action_requested",
		AccountID: environment.AccountID, BrowserEnvAlias: environment.Alias, NetworkExitID: environment.Exit.ID,
		RuntimeInstanceID: environment.RuntimeInstanceID, BindingVersion: environment.BindingVersion,
		OldImageVersion: environment.ImageVersion, NewImageVersion: environment.ImageVersion,
	}
}

func decodeHubJSON(c fiber.Ctx, destination any) error {
	decoder := json.NewDecoder(bytes.NewReader(c.Body()))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(destination); err != nil {
		return hub.ErrInvalid
	}
	if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
		return hub.ErrInvalid
	}
	return nil
}

func hubError(c fiber.Ctx, err error) error {
	var failure gatewayFailure
	if errors.As(err, &failure) {
		return c.Status(failure.status).JSON(map[string]string{"error": failure.message})
	}
	status := fiber.StatusInternalServerError
	message := "hub operation failed"
	switch {
	case errors.Is(err, hub.ErrInvalid):
		status, message = fiber.StatusBadRequest, hub.ErrInvalid.Error()
	case errors.Is(err, hub.ErrReconcileRequired):
		status, message = fiber.StatusConflict, hub.ErrReconcileRequired.Error()
	case errors.Is(err, hub.ErrConflict):
		status, message = fiber.StatusConflict, hub.ErrConflict.Error()
	case errors.Is(err, hub.ErrNotFound):
		status, message = fiber.StatusNotFound, hub.ErrNotFound.Error()
	default:
		return c.Status(status).JSON(map[string]string{"error": err.Error()})
	}
	return c.Status(status).JSON(map[string]string{"error": message})
}