creator-hub/internal/controlplane/api/environments.go

package api

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"

	"git.ipao.vip/rogee/creator-hub/internal/controlplane/gatewayclient"
	hub "git.ipao.vip/rogee/creator-hub/internal/environment"
	"github.com/gofiber/fiber/v3"
	"github.com/sirupsen/logrus"
)

// HubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。
type HubStore interface {
	LockResources(ctx context.Context, aliases, exitIDs []string, _ ...[]string) (func(), error)
	CreateGateway(ctx context.Context, name, endpoint, token string) (hub.Gateway, error)
	UpdateGateway(ctx context.Context, currentName, name, endpoint, token string) (hub.Gateway, error)
	ListGateways(ctx context.Context) ([]hub.Gateway, error)
	GetGateway(ctx context.Context, name string) (hub.Gateway, error)
	DeleteGateway(ctx context.Context, name string) error
	CreateEnv(ctx context.Context, env hub.Env) error
	ListEnvs(ctx context.Context) ([]hub.Env, error)
	GetEnv(ctx context.Context, alias string) (hub.Env, error)
	CreateNetworkExit(ctx context.Context, exit hub.NetworkExit) (hub.NetworkExit, error)
	ListNetworkExits(ctx context.Context) ([]hub.NetworkExit, error)
	GetNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error)
	GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error)
	RecordNetworkExitCheck(ctx context.Context, id string, observation hub.ExitObservation, failureReason string) (hub.NetworkExit, string, error)
	DisableNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error)
	CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error)
	GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error)
	GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error)
	ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error
	RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkID ...string) (hub.EnvironmentContext, error)
	ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkID ...string) (hub.EnvironmentContext, error)
	ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error
	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
	AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error
}

type networkExitAdminStore interface {
	UpdateNetworkExit(context.Context, string, hub.NetworkExit) (hub.NetworkExit, error)
	EnableNetworkExit(context.Context, string) (hub.NetworkExit, error)
	DeleteNetworkExit(context.Context, string) error
}

type RuntimeStopStore interface {
	LockResources(ctx context.Context, aliases, exitIDs []string, _ ...[]string) (func(), error)
	GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error)
	GetGateway(ctx context.Context, name string) (hub.Gateway, error)
	ReleaseRuntime(ctx context.Context, environment hub.EnvironmentContext) error
	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
	AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error
}

type runtimeCleanupStore interface {
	SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error
}

const (
	gatewayLongTimeout             = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取
	gatewayBrowserOperationTimeout = 90 * time.Second
	gatewayReconcileDelay          = 100 * time.Millisecond
	gatewayReconcileAttempts       = 10
	missingRuntimeID               = "runtime-not-found"
)

var (
	// Runtime and network identifiers are opaque, gateway-issued generations.
	// Their exact value is fenced and persisted; only control characters and
	// oversized values are rejected here.
	gatewayGenerationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`)
	gatewayRuntimeIDPattern    = gatewayGenerationIDPattern
	gatewayNetworkIDPattern    = gatewayGenerationIDPattern
)

const (
	defaultGatewayResponseLimit = 1 << 20
	largeGatewayResponseLimit   = 16 << 20
)

// gatewayCall 调用某个网关的 /v1 路由；ok 为 false 时 status/body 携带网关错误。
func gatewayCall(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration) (status int, responseBody []byte, err error) {
	return gatewayclient.Call(ctx, target, method, path, body, timeout)
}

func gatewayCallWithLimit(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration, responseLimit int) (status int, responseBody []byte, err error) {
	return gatewayclient.CallWithLimit(ctx, target, method, path, body, timeout, responseLimit)
}

func gatewayCreatePayload(environment hub.EnvironmentContext, _ string, networkExit gatewayNetworkExit) map[string]any {
	fingerprint := environment.Fingerprint
	fingerprint.ProxyServer = ""
	fingerprint.DisableNonProxiedUDP = false
	profileID := environment.AccountID
	if profileID == "" {
		profileID = environment.Alias
	}
	payload := map[string]any{
		"alias":           environment.Alias,
		"name":            environment.Name,
		"profile_id":      profileID,
		"cmd":             append(fingerprint.Args(), "about:blank"),
		"binding_version": environment.BindingVersion,
		"network_exit_id": environment.Exit.ID,
		"network_exit":    networkExit,
	}
	return payload
}

func gatewayCreatePayloadForAccount(environment hub.EnvironmentContext, browserPath string, networkExit gatewayNetworkExit) map[string]any {
	payload := gatewayCreatePayload(environment, browserPath, networkExit)
	if !accountRunnable(environment) {
		payload["network_exit_id"], payload["network_exit"], payload["stopped"] = "", gatewayNetworkExit{}, true
	}
	return payload
}

func gatewayProxyPayload(environment hub.EnvironmentContext, runtimeID, networkID string, networkExit gatewayNetworkExit) map[string]any {
	return map[string]any{
		"binding_version": environment.BindingVersion,
		"runtime_id":      runtimeID,
		"network_id":      networkID,
		"network_exit_id": environment.Exit.ID,
		"network_exit":    networkExit,
	}
}

func gatewayGenerationPayload(environment hub.EnvironmentContext) map[string]any {
	return map[string]any{"binding_version": environment.BindingVersion, "runtime_id": environment.RuntimeID, "network_id": environment.RuntimeNetworkID, "network_exit_id": environment.Exit.ID}
}

func gatewayCleanupGenerationPayload(environment hub.EnvironmentContext) map[string]any {
	bindingVersion, runtimeID, networkID := environment.RuntimeCleanupBindingVersion, environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID
	if bindingVersion == 0 {
		bindingVersion, runtimeID, networkID = environment.BindingVersion, environment.RuntimeID, environment.RuntimeNetworkID
	}
	return map[string]any{"binding_version": bindingVersion, "runtime_id": runtimeID, "network_id": networkID}
}

func runtimeCleanupGeneration(environment hub.EnvironmentContext, bindingVersion int64, runtimeID string, networkIDs ...string) hub.EnvironmentContext {
	environment.RuntimeCleanupBindingVersion = bindingVersion
	environment.RuntimeCleanupInstanceID = environment.RuntimeInstanceID
	environment.RuntimeCleanupRuntimeID = runtimeID
	environment.RuntimeCleanupNetworkID = environment.RuntimeNetworkID
	if len(networkIDs) == 1 {
		environment.RuntimeCleanupNetworkID = networkIDs[0]
	}
	return environment
}

func gatewayNetworkID(body []byte) string {
	var envelope struct {
		NetworkID string `json:"network_id"`
	}
	if json.Unmarshal(body, &envelope) == nil && gatewayNetworkIDPattern.MatchString(envelope.NetworkID) {
		return envelope.NetworkID
	}
	return ""
}

func reconcileGatewayCreate(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
	environment hub.EnvironmentContext, status int, callErr error, body []byte) error {
	// 网关 4xx 拒绝且未产生 network 代：创建在 gateway 侧任何 runtime 变更之前就被确定性拒绝，
	// 无需清理 fence；其余情况（5xx、断连、无效 201）结果未知，仍走 fail-closed。
	deterministicRejection := callErr == nil && status >= 400 && status < 500
	logEntry := logrus.WithFields(logrus.Fields{
		"service":          "control-plane",
		"event_type":       "runtime_create_reconciliation",
		"alias":            environment.Alias,
		"gateway":          target.Name,
		"binding_version":  environment.BindingVersion,
		"gateway_status":   status,
		"network_id_known": gatewayNetworkID(body) != "",
		"deterministic":    deterministicRejection,
	})
	if callErr != nil {
		logEntry = logEntry.WithError(callErr)
	}
	if deterministicRejection {
		logEntry.Info("gateway rejected runtime create before native runtime side effects")
	} else {
		logEntry.Warn("gateway runtime create result requires reconciliation")
	}
	return reconcileGatewayCreateGeneration(ctx, store, target, environment, environment.BindingVersion,
		gatewayNetworkID(body), deterministicRejection)
}

func reconcileGatewayCreateGeneration(ctx context.Context, store runtimeCleanupStore, target hub.Gateway,
	environment hub.EnvironmentContext, bindingVersion int64, networkID string, deterministicRejection bool) error {
	if networkID == "" {
		if deterministicRejection {
			// 网关明确拒绝：创建未发生、无副作用，不写 pending，让调用方透传真实失败原因，
			// 绑定保持可重试（否则会把可判定的失败写成永远无法自动清理的“未知代”毒状态）。
			return nil
		}
		pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, "")
		return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
	}
	runtime, found, err := reconcileGatewayRuntime(ctx, target, environment.Alias)
	if err != nil {
		runtimeID := missingRuntimeID
		if bindingVersion == environment.BindingVersion && environment.RuntimeID != "" {
			runtimeID = environment.RuntimeID
		}
		pending := runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID)
		return errors.Join(err, store.SetRuntimeCleanupPending(ctx, pending, true))
	}
	if found && (runtime.BindingVersion != bindingVersion || runtime.NetworkID != networkID) {
		pending := runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID)
		return errors.Join(hub.ErrConflict, store.SetRuntimeCleanupPending(ctx, pending, true))
	}
	if found {
		environment = runtimeCleanupGeneration(environment, bindingVersion, runtime.ID, networkID)
	} else {
		environment = runtimeCleanupGeneration(environment, bindingVersion, missingRuntimeID, networkID)
	}
	_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
	return cleanupErr
}

// gatewayFailure 将网关错误转换为对调用方可读的失败；4xx 保留原状态，其余一律 502。
type gatewayFailure struct {
	status  int
	message string
}

func (e gatewayFailure) Error() string { return e.message }

func gatewayRejected(status int, body []byte) error {
	if status >= 400 && status < 500 {
		return gatewayFailure{status: status, message: "gateway rejected: " + errorFromBody(body, status)}
	}
	return gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf("gateway call failed with status %d", status)}
}

func gatewayUnreachable(err error) error {
	return gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf("gateway unreachable: %v", err)}
}

func reconcileGatewayRuntime(ctx context.Context, target hub.Gateway, alias string) (runtimeStatus, bool, error) {
	var lastErr error
	for attempt := 0; attempt < gatewayReconcileAttempts; attempt++ {
		if attempt > 0 {
			timer := time.NewTimer(gatewayReconcileDelay)
			select {
			case <-ctx.Done():
				timer.Stop()
				return runtimeStatus{}, false, ctx.Err()
			case <-timer.C:
			}
		}
		status, body, err := gatewayCall(ctx, target, http.MethodGet, "/v1/browsers", nil, 30*time.Second)
		if err != nil {
			lastErr = err
			continue
		}
		if status != http.StatusOK {
			lastErr = gatewayRejected(status, body)
			continue
		}
		browsers, parseErr := parseGatewayBrowserList(body)
		if parseErr != nil {
			lastErr = parseErr
			continue
		}
		lastErr = nil
		for _, browser := range browsers {
			if browser.Alias == alias {
				return browser, true, nil
			}
		}
	}
	return runtimeStatus{}, false, lastErr
}

func parseGatewayBrowserList(body []byte) ([]runtimeStatus, error) {
	var entries []json.RawMessage
	if err := json.Unmarshal(body, &entries); err != nil || entries == nil {
		return nil, errors.New("gateway returned an invalid browser list")
	}
	browsers := make([]runtimeStatus, 0, len(entries))
	aliases := make(map[string]bool, len(entries))
	for _, entry := range entries {
		var browser *runtimeStatus
		if err := json.Unmarshal(entry, &browser); err != nil || browser == nil || browser.ID == "" || browser.Alias == "" || browser.State == "" ||
			browser.BindingVersion < 0 || aliases[browser.Alias] {
			return nil, errors.New("gateway returned an invalid browser list")
		}
		aliases[browser.Alias] = true
		browsers = append(browsers, *browser)
	}
	return browsers, nil
}

func errorFromBody(body []byte, status int) string {
	var envelope struct {
		Error string `json:"error"`
	}
	if json.Unmarshal(body, &envelope) == nil && envelope.Error != "" {
		return envelope.Error
	}
	return http.StatusText(status)
}

type runtimeStatus struct {
	ID             string `json:"id"`
	Alias          string `json:"alias"`
	Name           string `json:"name"`
	State          string `json:"state"`
	Status         string `json:"status"`
	Endpoint       string `json:"endpoint"`
	BindingVersion int64  `json:"binding_version"`
	NetworkExitID  string `json:"network_exit_id"`
	NetworkID      string `json:"network_id"`
	NodeID         string `json:"node_id"`
	CleanupState   string `json:"cleanup_state"`
	CleanupError   string `json:"cleanup_error"`
	ProxyReady     bool   `json:"proxy_ready"`
}

type envView struct {
	hub.Env
	State               string `json:"state"`
	Status              string `json:"status"`
	RuntimeID           string `json:"runtime_id"`
	Endpoint            string `json:"endpoint"`
	AccountID           string `json:"account_id"`
	AccountStatus       string `json:"account_status"`
	AuthorizationStatus string `json:"authorization_status"`
	NetworkExitID       string `json:"network_exit_id"`
	NetworkExitHealth   string `json:"network_exit_health"`
	BindingVersion      int64  `json:"binding_version"`
	RuntimeInstanceID   string `json:"runtime_instance_id"`
	RuntimeNodeID       string `json:"runtime_node_id,omitempty"`
	RuntimeStatus       string `json:"runtime_status,omitempty"`
	RuntimeCleanupState string `json:"runtime_cleanup_state,omitempty"`
	RuntimeCleanupError string `json:"runtime_cleanup_error,omitempty"`
	GatewayReachable    bool   `json:"gateway_reachable"`
	ScheduleStatus      string `json:"schedule_status"`
	ScheduleBlockReason string `json:"schedule_block_reason,omitempty"`
	RecoveryRequired    bool   `json:"recovery_required"`
	CleanupPending      bool   `json:"cleanup_pending"`
}

func environmentScheduleReadiness(environment hub.EnvironmentContext) (string, string) {
	switch {
	case environment.AuthorizationStatus != "authorized":
		return "blocked", "account_revoked"
	case environment.AccountStatus != "active":
		return "blocked", "account_paused"
	case environment.Exit.ID != "" && environment.Exit.HealthStatus != "healthy":
		return "blocked", "network_exit_unhealthy"
	case environment.RuntimeCleanupPending:
		return "blocked", "runtime_stop_pending"
	case environment.RuntimeInstanceID == "":
		return "blocked", "runtime_missing"
	default:
		return "ready", ""
	}
}

func runtimeMatchesBinding(runtime runtimeStatus, environment hub.EnvironmentContext) bool {
	if !validCreatedRuntime(runtime, environment, runtime.State == "running") || runtime.BindingVersion != environment.BindingVersion ||
		(runtime.NetworkExitID != environment.Exit.ID && (runtime.State == "running" || runtime.NetworkExitID != "")) {
		return false
	}
	return true
}

func validCreatedRuntime(created runtimeStatus, environment hub.EnvironmentContext, running bool) bool {
	if !gatewayRuntimeIDPattern.MatchString(created.ID) ||
		(created.NetworkID != "" && !gatewayNetworkIDPattern.MatchString(created.NetworkID)) {
		return false
	}
	if !running {
		return true
	}
	if created.NetworkID == "" {
		return false
	}
	if environment.RuntimeInstanceID != "" || environment.RuntimeID != "" {
		return environment.RuntimeInstanceID != "" && environment.RuntimeID == created.ID &&
			environment.RuntimeNetworkID != "" && created.NetworkID == environment.RuntimeNetworkID
	}
	return environment.RuntimeNetworkID == "" || created.NetworkID == environment.RuntimeNetworkID
}

func accountRunnable(environment hub.EnvironmentContext) bool {
	return environment.AccountStatus == "active" && environment.AuthorizationStatus == "authorized"
}

func releaseRuntime(ctx context.Context, store RuntimeStopStore, environment hub.EnvironmentContext) error {
	err := store.ReleaseRuntime(ctx, environment)
	if errors.Is(err, hub.ErrConflict) {
		return nil
	}
	return err
}

func releaseRuntimeWithReconcileAudit(ctx context.Context, store HubStore, environment hub.EnvironmentContext) error {
	action := actionForEnvironment("reconcile", environment)
	if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
		return err
	}
	releaseErr := releaseRuntime(ctx, store, environment)
	action.Outcome, action.ReasonCode = "succeeded", "runtime_released"
	if releaseErr != nil {
		action.Outcome, action.ReasonCode = "failed", "runtime_release_failed"
	}
	if auditErr := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); auditErr != nil {
		return errors.Join(releaseErr, auditErr)
	}
	return releaseErr
}

func registerHub(app *fiber.App, store HubStore) {
	registerHubWithNetwork(app, store, defaultNetworkExitProbe(), nil)
}

func registerHubWithNetwork(app *fiber.App, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) {
	app.Get("/api/browsers", listBrowsers(store))
	app.Get("/api/browsers/:alias", getBrowser(store))
	app.Post("/api/browsers", lockBrowserCreate(store, createBrowser(store, probe, resolve)))
	app.Post("/api/browsers/:alias/:action", lockBrowserAlias(store, browserAction(store, probe, resolve)))
	app.Delete("/api/browsers/:alias", lockBrowserAlias(store, deleteBrowser(store)))
	app.Get("/api/network-exits", listNetworkExits(store))
	app.Get("/api/network-exits/:id", getNetworkExit(store))
	app.Post("/api/network-exits", createNetworkExit(store))
	app.Post("/api/network-exits/:id/check", checkNetworkExit(store, probe))
	app.Post("/api/network-exits/:id/disable", disableNetworkExit(store, probe, resolve))
	app.Put("/api/network-exits/:id", func(c fiber.Ctx) error {
		admin, ok := store.(networkExitAdminStore)
		if !ok {
			return hubError(c, hub.ErrConflict)
		}
		input := hub.NetworkExit{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		exit, err := admin.UpdateNetworkExit(c.Context(), c.Params("id"), input)
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(exit)
	})
	app.Post("/api/network-exits/:id/enable", func(c fiber.Ctx) error {
		admin, ok := store.(networkExitAdminStore)
		if !ok {
			return hubError(c, hub.ErrConflict)
		}
		exit, err := admin.EnableNetworkExit(c.Context(), c.Params("id"))
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(exit)
	})
	app.Delete("/api/network-exits/:id", func(c fiber.Ctx) error {
		admin, ok := store.(networkExitAdminStore)
		if !ok {
			return hubError(c, hub.ErrConflict)
		}
		if err := admin.DeleteNetworkExit(c.Context(), c.Params("id")); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	})

	app.Get("/api/gateways", func(c fiber.Ctx) error {
		gateways, err := store.ListGateways(c.Context())
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(gateways)
	})
	app.Post("/api/gateways", func(c fiber.Ctx) error {
		input := struct {
			Name     string `json:"name"`
			Endpoint string `json:"endpoint"`
			Token    string `json:"token"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		gateway, err := store.CreateGateway(c.Context(), input.Name, input.Endpoint, input.Token)
		if err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(gateway)
	})
	app.Put("/api/gateways/:name", func(c fiber.Ctx) error {
		input := struct {
			Name     string `json:"name"`
			Endpoint string `json:"endpoint"`
			Token    string `json:"token"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		gateway, err := store.UpdateGateway(c.Context(), c.Params("name"), input.Name, input.Endpoint, input.Token)
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(gateway)
	})
	app.Delete("/api/gateways/:name", func(c fiber.Ctx) error {
		if err := store.DeleteGateway(c.Context(), c.Params("name")); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	})
}

func getBrowser(store HubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
		if err != nil {
			return hubError(c, err)
		}
		view := envView{
			Env: environment.Env, State: "recorded", Status: "已记录运行实例",
			AccountID: environment.AccountID, AccountStatus: environment.AccountStatus,
			AuthorizationStatus: environment.AuthorizationStatus, NetworkExitID: environment.Exit.ID,
			NetworkExitHealth: environment.Exit.HealthStatus, BindingVersion: environment.BindingVersion,
			RuntimeInstanceID: environment.RuntimeInstanceID, RuntimeID: environment.RuntimeID, RuntimeNodeID: environment.RuntimeNodeID,
			ScheduleStatus: "blocked", ScheduleBlockReason: "binding_missing",
			RecoveryRequired: true, CleanupPending: environment.RuntimeCleanupPending,
		}
		view.ScheduleStatus, view.ScheduleBlockReason = environmentScheduleReadiness(environment)
		if environment.RuntimeInstanceID == "" {
			view.State, view.Status = "missing", "未记录运行实例"
		}
		runtimes, gatewayRead, gatewayErrors := gatewayRuntimeSnapshot(c.Context(), store, []hub.Env{environment.Env})
		if gatewayErrors[environment.Gateway] != nil {
			view.State, view.Status, view.GatewayReachable = "gateway_unreachable", "网关不可达", false
		} else if gatewayRead[environment.Gateway] {
			view.GatewayReachable = true
			if runtime, found := runtimes[environment.Gateway][environment.Alias]; found {
				view.RuntimeID, view.RuntimeStatus, view.RuntimeNodeID = runtime.ID, runtime.Status, runtime.NodeID
				view.State, view.Status = runtime.State, runtime.Status
				view.RuntimeCleanupState, view.RuntimeCleanupError = runtime.CleanupState, runtime.CleanupError
				view.CleanupPending = view.CleanupPending || runtime.CleanupState == "pending"
				view.RecoveryRequired = view.RecoveryRequired || runtime.State != "running" || runtime.CleanupState == "pending"
				if runtime.State == "running" && runtime.Status == "ready" {
					view.Endpoint = runtime.Endpoint
				}
			} else if view.RuntimeInstanceID != "" {
				view.State, view.Status = "runtime_missing", "网关中未找到运行实例"
			}
		}
		return c.JSON(view)
	}
}

func lockBrowserAlias(store HubStore, handler fiber.Handler) fiber.Handler {
	return func(c fiber.Ctx) error {
		requestedExitID := ""
		if c.Params("action") == "rebind" {
			var input struct {
				NetworkExitID string `json:"network_exit_id"`
			}
			if json.Unmarshal(c.Body(), &input) == nil {
				if hub.ValidNetworkExitID(input.NetworkExitID) {
					requestedExitID = input.NetworkExitID
				}
			}
		}
		alias := c.Params("alias")
		unlock, err := lockAliasResources(c.Context(), store, alias, requestedExitID)
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		return handler(c)
	}
}

func lockBrowserCreate(store HubStore, handler fiber.Handler) fiber.Handler {
	return func(c fiber.Ctx) error {
		var input struct {
			Alias         string `json:"alias"`
			AccountID     string `json:"account_id"`
			NetworkExitID string `json:"network_exit_id"`
		}
		if json.Unmarshal(c.Body(), &input) != nil || input.Alias == "" {
			return handler(c)
		}
		exitIDs := []string(nil)
		if hub.ValidNetworkExitID(input.NetworkExitID) {
			exitIDs = []string{input.NetworkExitID}
		}
		unlock, err := store.LockResources(c.Context(), nonEmpty(input.Alias, input.AccountID), exitIDs)
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		return handler(c)
	}
}

func lockEnvironmentResources(ctx context.Context, store HubStore, envs []hub.Env) (func(), error) {
	aliases := make([]string, 0, len(envs))
	seen := make(map[string]bool, len(envs))
	for _, env := range envs {
		if !seen[env.Alias] {
			seen[env.Alias] = true
			aliases = append(aliases, env.Alias)
		}
	}
	for {
		before, err := environmentResourceMap(ctx, store, aliases)
		if err != nil {
			return nil, err
		}
		exitIDs := environmentResourceValues(before)
		unlock, err := store.LockResources(ctx, aliases, exitIDs)
		if err != nil {
			return nil, err
		}
		after, err := environmentResourceMap(ctx, store, aliases)
		if err != nil {
			unlock()
			return nil, err
		}
		if equalEnvironmentResourceMaps(before, after) {
			return unlock, nil
		}
		unlock()
	}
}

type environmentResource struct {
	alias  string
	exitID string
}

type resourceLockStore interface {
	LockResources(ctx context.Context, aliases, exitIDs []string, _ ...[]string) (func(), error)
	GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error)
}

func lockAliasResources(ctx context.Context, store resourceLockStore, alias, requestedExitID string) (func(), error) {
	for {
		before, beforeFound, err := environmentResources(ctx, store, alias)
		if err != nil {
			return nil, err
		}
		unlock, err := store.LockResources(ctx, []string{alias}, nonEmpty(before.exitID, requestedExitID))
		if err != nil {
			return nil, err
		}
		after, afterFound, err := environmentResources(ctx, store, alias)
		if err != nil {
			unlock()
			return nil, err
		}
		if beforeFound == afterFound && before == after {
			return unlock, nil
		}
		unlock()
	}
}

func lockAccountResources(ctx context.Context, store RuntimeStopStore, accountID string) (func(), error) {
	if store == nil {
		return func() {}, nil
	}
	for {
		before, beforeFound, err := accountEnvironmentResources(ctx, store, accountID)
		if err != nil {
			return nil, err
		}
		unlock, err := store.LockResources(ctx, nonEmpty(accountID, before.alias), nonEmpty(before.exitID))
		if err != nil {
			return nil, err
		}
		after, afterFound, err := accountEnvironmentResources(ctx, store, accountID)
		if err != nil {
			unlock()
			return nil, err
		}
		if beforeFound == afterFound && before == after {
			return unlock, nil
		}
		unlock()
	}
}

func accountEnvironmentResources(ctx context.Context, store RuntimeStopStore, accountID string) (environmentResource, bool, error) {
	environment, err := store.GetEnvironmentContextForAccount(ctx, accountID)
	if errors.Is(err, hub.ErrNotFound) {
		return environmentResource{}, false, nil
	}
	if err != nil {
		return environmentResource{}, false, err
	}
	return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID}, true, nil
}

func environmentResources(ctx context.Context, store resourceLockStore, alias string) (environmentResource, bool, error) {
	environment, err := store.GetEnvironmentContext(ctx, alias)
	if errors.Is(err, hub.ErrNotFound) {
		return environmentResource{}, false, nil
	}
	if err != nil {
		return environmentResource{}, false, err
	}
	return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID}, true, nil
}

func environmentResourceMap(ctx context.Context, store HubStore, aliases []string) (map[string]environmentResource, error) {
	result := make(map[string]environmentResource, len(aliases))
	for _, alias := range aliases {
		resources, found, err := environmentResources(ctx, store, alias)
		if err != nil {
			return nil, err
		}
		if found {
			result[alias] = resources
		}
	}
	return result, nil
}

func nonEmpty(values ...string) []string {
	result := make([]string, 0, len(values))
	seen := make(map[string]bool, len(values))
	for _, value := range values {
		if value != "" && !seen[value] {
			seen[value] = true
			result = append(result, value)
		}
	}
	return result
}

func environmentResourceValues(values map[string]environmentResource) []string {
	exitIDs := make([]string, 0, len(values))
	for _, value := range values {
		exitIDs = append(exitIDs, value.exitID)
	}
	return nonEmpty(exitIDs...)
}

func equalEnvironmentResourceMaps(left, right map[string]environmentResource) bool {
	if len(left) != len(right) {
		return false
	}
	for key, value := range left {
		other, ok := right[key]
		if !ok || other != value {
			return false
		}
	}
	return true
}

func listNetworkExits(store HubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		exits, err := store.ListNetworkExits(c.Context())
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(exits)
	}
}

func getNetworkExit(store HubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		exit, err := store.GetNetworkExit(c.Context(), c.Params("id"))
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(exit)
	}
}

func createNetworkExit(store HubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		var input struct {
			Protocol         string `json:"protocol"`
			Host             string `json:"host"`
			Port             int    `json:"port"`
			Username         string `json:"username"`
			Password         string `json:"password"`
			ExpectedPublicIP string `json:"expected_public_ip"`
			ExpectedRegion   string `json:"expected_region"`
		}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		exit, err := store.CreateNetworkExit(c.Context(), hub.NetworkExit{
			Protocol: input.Protocol, Host: input.Host, Port: input.Port,
			Username: input.Username, Password: input.Password,
			ExpectedPublicIP: input.ExpectedPublicIP, ExpectedRegion: input.ExpectedRegion,
		})
		if err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(exit)
	}
}

func checkNetworkExit(store HubStore, probe NetworkExitProbe) fiber.Handler {
	return func(c fiber.Ctx) error {
		unlock, err := store.LockResources(c.Context(), nil, []string{c.Params("id")}, nil)
		if err != nil {
			return hubError(c, err)
		}
		defer unlock()
		access, err := store.GetNetworkExitAccess(c.Context(), c.Params("id"))
		if err != nil {
			return hubError(c, err)
		}
		if access.HealthStatus == "disabled" {
			return hubError(c, hub.ErrConflict)
		}
		observation, failureReason := probe.Check(c.Context(), access)
		exit, reason, err := store.RecordNetworkExitCheck(c.Context(), access.ID, observation, failureReason)
		if err != nil {
			return hubError(c, err)
		}
		return c.JSON(struct {
			hub.NetworkExit
			ReasonCode string `json:"reason_code"`
		}{exit, reason})
	}
}

func disableNetworkExit(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler {
	return func(c fiber.Ctx) error {
		unlock, err := store.LockResources(c.Context(), nil, []string{c.Params("id")}, nil)
		if err != nil {
			return hubError(c, err)
		}
		exit, err := store.DisableNetworkExit(c.Context(), c.Params("id"))
		unlock()
		if err != nil {
			return hubError(c, err)
		}
		if err := reconcileRuntimeLeasesUnlocked(c.Context(), store, probe, resolve); err != nil {
			return hubError(c, err)
		}
		return c.JSON(exit)
	}
}

func verifyNetworkExit(ctx context.Context, store HubStore, probe NetworkExitProbe, id string) (hub.NetworkExitAccess, string, error) {
	access, err := store.GetNetworkExitAccess(ctx, id)
	if err != nil {
		return hub.NetworkExitAccess{}, "exit_unavailable", err
	}
	if access.HealthStatus == "disabled" {
		return hub.NetworkExitAccess{}, "exit_disabled", hub.ErrConflict
	}
	observation, failureReason := probe.Check(ctx, access)
	exit, reason, err := store.RecordNetworkExitCheck(ctx, id, observation, failureReason)
	if err != nil {
		return hub.NetworkExitAccess{}, reason, err
	}
	if exit.HealthStatus != "healthy" {
		return hub.NetworkExitAccess{}, reason, hub.ErrConflict
	}
	access, err = store.GetNetworkExitAccess(ctx, id)
	return access, reason, err
}

func listBrowsers(store HubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		envs, err := store.ListEnvs(c.Context())
		if err != nil {
			return hubError(c, err)
		}
		runtimes, gatewayRead, gatewayErrors := gatewayRuntimeSnapshot(c.Context(), store, envs)
		views := make([]envView, 0, len(envs))
		for _, env := range envs {
			view := envView{
				Env: env, State: "missing", Status: "未记录运行实例",
				ScheduleStatus: "blocked", ScheduleBlockReason: "binding_missing",
				RecoveryRequired: true, GatewayReachable: false,
			}
			if environment, contextErr := store.GetEnvironmentContext(c.Context(), env.Alias); contextErr == nil {
				view.AccountID, view.AccountStatus, view.AuthorizationStatus = environment.AccountID, environment.AccountStatus, environment.AuthorizationStatus
				view.NetworkExitID, view.NetworkExitHealth, view.BindingVersion = environment.Exit.ID, environment.Exit.HealthStatus, environment.BindingVersion
				view.RuntimeInstanceID = environment.RuntimeInstanceID
				view.RuntimeID = environment.RuntimeID
				view.RuntimeNodeID = environment.RuntimeNodeID
				view.ScheduleStatus, view.ScheduleBlockReason = environmentScheduleReadiness(environment)
				view.CleanupPending = environment.RuntimeCleanupPending
				view.RecoveryRequired = (environment.Exit.ID != "" && environment.Exit.HealthStatus != "healthy") || environment.RuntimeCleanupPending
				if environment.RuntimeCleanupPending {
					view.State, view.Status = "cleanup_pending", "运行资源待清理"
				} else if environment.RuntimeInstanceID != "" {
					view.State, view.Status = "recorded", "已记录运行实例"
				}
			} else if !errors.Is(contextErr, hub.ErrNotFound) {
				return hubError(c, contextErr)
			}
			if gatewayErrors[env.Gateway] != nil {
				view.State, view.Status = "gateway_unreachable", "网关不可达"
				view.RecoveryRequired = true
			} else if gatewayRead[env.Gateway] {
				view.GatewayReachable = true
				if runtime, found := runtimes[env.Gateway][env.Alias]; found {
					view.RuntimeID, view.RuntimeStatus, view.RuntimeNodeID = runtime.ID, runtime.Status, runtime.NodeID
					view.State, view.Status = runtime.State, runtime.Status
					view.RuntimeCleanupState, view.RuntimeCleanupError = runtime.CleanupState, runtime.CleanupError
					view.CleanupPending = view.CleanupPending || runtime.CleanupState == "pending"
					view.RecoveryRequired = view.RecoveryRequired || runtime.State != "running" || runtime.CleanupState == "pending"
					if runtime.State == "running" && runtime.Status == "ready" {
						view.Endpoint = runtime.Endpoint
					}
				} else if view.RuntimeInstanceID != "" {
					view.State, view.Status = "runtime_missing", "网关中未找到运行实例"
					view.RecoveryRequired = true
				}
			}
			views = append(views, view)
		}
		return c.JSON(views)
	}
}

func reconcileRuntimeLeases(ctx context.Context, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error {
	return reconcileRuntimeLeasesUnlocked(ctx, store, probe, resolve)
}

func reconcileRuntimeLeasesUnlocked(ctx context.Context, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error {
	envs, err := store.ListEnvs(ctx)
	if err != nil {
		return err
	}
	unlock, err := lockEnvironmentResources(ctx, store, envs)
	if err != nil {
		return err
	}
	defer unlock()
	runtimes, gatewayRead, gatewayErrors := gatewayRuntimeSnapshot(ctx, store, envs)
	return reconcileRuntimeSnapshot(ctx, store, probe, resolve, envs, runtimes, gatewayRead, gatewayErrors)
}

func gatewayRuntimeSnapshot(ctx context.Context, store HubStore, envs []hub.Env) (map[string]map[string]runtimeStatus, map[string]bool, map[string]error) {
	gateways := map[string]bool{}
	runtimes := map[string]map[string]runtimeStatus{}
	gatewayRead := map[string]bool{}
	gatewayErrors := map[string]error{}
	for _, env := range envs {
		if gateways[env.Gateway] {
			continue
		}
		gateways[env.Gateway] = true
		gateway, err := store.GetGateway(ctx, env.Gateway)
		if err != nil {
			gatewayErrors[env.Gateway] = gatewayUnreachable(err)
			continue
		}
		status, body, callErr := gatewayCall(ctx, gateway, http.MethodGet, "/v1/browsers", nil, 30*time.Second)
		if callErr != nil {
			gatewayErrors[env.Gateway] = gatewayUnreachable(callErr)
			continue
		}
		if status != http.StatusOK {
			gatewayErrors[env.Gateway] = gatewayRejected(status, body)
			continue
		}
		list, parseErr := parseGatewayBrowserList(body)
		if parseErr != nil {
			gatewayErrors[env.Gateway] = gatewayFailure{status: http.StatusBadGateway, message: "gateway returned an invalid browser list"}
			continue
		}
		byAlias := map[string]runtimeStatus{}
		for _, runtime := range list {
			byAlias[runtime.Alias] = runtime
		}
		runtimes[env.Gateway] = byAlias
		gatewayRead[env.Gateway] = true
	}
	return runtimes, gatewayRead, gatewayErrors
}

func reconcileRuntimeSnapshot(ctx context.Context, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), envs []hub.Env,
	runtimes map[string]map[string]runtimeStatus, gatewayRead map[string]bool, gatewayErrors map[string]error) error {
	var firstErr error
	for _, env := range envs {
		if readErr := gatewayErrors[env.Gateway]; readErr != nil {
			if firstErr == nil {
				firstErr = readErr
			}
			continue
		}
		environment, err := store.GetEnvironmentContext(ctx, env.Alias)
		if errors.Is(err, hub.ErrNotFound) {
			continue
		}
		if err != nil {
			return err
		}
		if environment.RuntimeCleanupPending {
			target, targetErr := store.GetGateway(ctx, environment.Gateway)
			if targetErr != nil {
				return targetErr
			}
			if _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment); cleanupErr != nil {
				return cleanupErr
			}
			continue
		}
		runtime, found := runtimes[env.Gateway][env.Alias]
		if !accountRunnable(environment) {
			if found && runtime.State == "running" {
				if err := stopEnvironmentRuntime(ctx, store, environment); err != nil {
					return err
				}
			} else if gatewayRead[env.Gateway] && environment.RuntimeInstanceID != "" {
				if err := releaseRuntimeWithReconcileAudit(ctx, store, environment); err != nil {
					return err
				}
			}
			continue
		}
		if found && runtime.State == "running" {
			auditRecovery := !runtimeMatchesBinding(runtime, environment) || !runtime.ProxyReady
			action := actionForEnvironment("reconcile", environment)
			if auditRecovery {
				if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
					return err
				}
			}
			ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, runtime)
			if auditRecovery {
				current, contextErr := store.GetEnvironmentContext(ctx, environment.Alias)
				if contextErr == nil {
					action.RuntimeInstanceID, action.BindingVersion, action.NetworkExitID =
						current.RuntimeInstanceID, current.BindingVersion, current.Exit.ID
				} else {
					action.RuntimeInstanceID = ""
					action.BindingVersion = 0
					action.NetworkExitID = ""
					restoreErr = errors.Join(restoreErr, contextErr)
				}
				action.Outcome, action.ReasonCode = "succeeded", "runtime_reconciled"
				if !ready {
					action.Outcome, action.ReasonCode = "failed", "runtime_unavailable"
				}
				if restoreErr != nil {
					action.Outcome, action.ReasonCode = runtimeRecoveryFailure(ctx, store, environment.Alias, restoreErr)
				}
				if err := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); err != nil {
					return errors.Join(restoreErr, err)
				}
			}
			if restoreErr != nil {
				return restoreErr
			}
		} else if gatewayRead[env.Gateway] && environment.RuntimeInstanceID != "" {
			if err := releaseRuntimeWithReconcileAudit(ctx, store, environment); err != nil {
				return err
			}
		}
	}
	return firstErr
}

func runtimeRecoveryFailure(ctx context.Context, store HubStore, alias string, err error) (string, string) {
	var gatewayErr gatewayFailure
	if errors.As(err, &gatewayErr) {
		return "unknown", "gateway_result_unknown"
	}
	if environment, contextErr := store.GetEnvironmentContext(ctx, alias); contextErr == nil && environment.RuntimeCleanupPending {
		return "unknown", "cleanup_result_unknown"
	}
	return "failed", "runtime_persistence_failed"
}

func restoreOrRebuildRuntime(ctx context.Context, store HubStore, probe NetworkExitProbe,
	_ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, runtime runtimeStatus) (bool, error) {
	if environment.RuntimeCleanupPending {
		target, err := store.GetGateway(ctx, environment.Gateway)
		if err != nil {
			return false, err
		}
		_, err = removeGatewayRuntime(ctx, store, target, environment)
		return false, err
	}
	var access hub.NetworkExitAccess
	var err error
	if environment.Exit.ID != "" {
		access, _, err = verifyNetworkExit(ctx, store, probe, environment.Exit.ID)
		if err != nil {
			return false, discardRuntime(ctx, store, environment)
		}
	}
	target, err := store.GetGateway(ctx, environment.Gateway)
	if err != nil {
		return false, err
	}
	if runtimeMatchesBinding(runtime, environment) && runtime.ProxyReady {
		_, err := activateGatewayRuntime(ctx, store, target, environment, runtime.ID, runtime.NetworkID, runtime.NodeID)
		return err == nil, err
	}
	networkExit := gatewayNetworkExit{}
	if environment.Exit.ID != "" {
		networkExit = gatewayNetworkExitFor(access)
	}
	if runtimeMatchesBinding(runtime, environment) {
		if environment.Exit.ID == "" {
			runtime.ProxyReady = true
		} else {
			status, _, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/proxy",
				gatewayProxyPayload(environment, runtime.ID, runtime.NetworkID, networkExit), 30*time.Second)
			if callErr == nil && status == http.StatusNoContent {
				runtime.ProxyReady = true
			}
		}
		if runtime.ProxyReady {
			_, err := activateGatewayRuntime(ctx, store, target, environment, runtime.ID, runtime.NetworkID, runtime.NodeID)
			return err == nil, err
		}
	}
	_, removeErr := removeGatewayRuntime(ctx, store, target, environment)
	if removeErr != nil {
		return false, removeErr
	}
	environment, err = store.GetEnvironmentContext(ctx, environment.Alias)
	if err != nil {
		return false, err
	}
	browserPath := ""
	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
		gatewayCreatePayloadForAccount(environment, browserPath, networkExit), gatewayLongTimeout)
	if callErr != nil {
		return false, errors.Join(gatewayUnreachable(callErr), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
	}
	if status != http.StatusCreated {
		return false, errors.Join(gatewayRejected(status, body), reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
	}
	var created runtimeStatus
	if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
		return false, errors.Join(errors.New("gateway returned an invalid runtime generation"),
			reconcileGatewayCreate(ctx, store, target, environment, status, callErr, body))
	}
	_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID, created.NodeID)
	return err == nil, err
}

type runtimeNodeStore interface {
	SetRuntimeNode(context.Context, string, string, string) error
}

func activateGatewayRuntime(ctx context.Context, store HubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string, nodeIDs ...string) (hub.EnvironmentContext, error) {
	if !validCreatedRuntime(runtimeStatus{ID: runtimeID, NetworkID: networkID}, environment, true) {
		return hub.EnvironmentContext{}, hub.ErrConflict
	}
	current, err := store.ActivateRuntime(ctx, environment.Alias, runtimeID, environment.BindingVersion, environment.Exit.ID, networkID)
	if err == nil && len(nodeIDs) > 0 && nodeIDs[0] != "" {
		if recorder, ok := store.(runtimeNodeStore); ok {
			if nodeErr := recorder.SetRuntimeNode(ctx, environment.Alias, runtimeID, nodeIDs[0]); nodeErr != nil {
				err = nodeErr
			} else {
				current.RuntimeNodeID = nodeIDs[0]
			}
		}
	}
	if err == nil {
		return current, nil
	}
	// A scheduler heartbeat can win the activation race after the action has
	// created the runtime. If it persisted this exact generation, keep the
	// runtime and treat the activation as successful instead of deleting the
	// winner from the gateway.
	if errors.Is(err, hub.ErrConflict) {
		if reconciled, readErr := store.GetEnvironmentContext(ctx, environment.Alias); readErr == nil &&
			reconciled.BindingVersion == environment.BindingVersion &&
			reconciled.RuntimeID == runtimeID &&
			reconciled.RuntimeNetworkID == networkID &&
			reconciled.RuntimeInstanceID != "" {
			return reconciled, nil
		}
	}
	cleanup := environment
	cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtimeID, networkID
	_, cleanupErr := removeGatewayRuntime(ctx, store, target, cleanup)
	return hub.EnvironmentContext{}, errors.Join(err, cleanupErr)
}

func discardRuntime(ctx context.Context, store HubStore, environment hub.EnvironmentContext) error {
	target, targetErr := store.GetGateway(ctx, environment.Gateway)
	if targetErr != nil {
		return targetErr
	}
	_, err := removeGatewayRuntime(ctx, store, target, environment)
	return err
}

func createBrowser(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler {
	return func(c fiber.Ctx) error {
		input := struct {
			Alias         string          `json:"alias"`
			Name          string          `json:"name"`
			Gateway       string          `json:"gateway"`
			Fingerprint   hub.Fingerprint `json:"fingerprint"`
			AccountID     string          `json:"account_id"`
			NetworkExitID string          `json:"network_exit_id"`
		}{}
		if err := decodeHubJSON(c, &input); err != nil {
			return hubError(c, err)
		}
		if input.Fingerprint.ProxyServer != "" {
			return hubError(c, hub.ErrInvalid)
		}
		input.Fingerprint.DisableNonProxiedUDP = false
		env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, Fingerprint: input.Fingerprint}
		if err := env.Fingerprint.Validate(); err != nil {
			return c.Status(fiber.StatusBadRequest).JSON(map[string]string{"error": err.Error()})
		}
		environment, created, err := store.CreateBoundEnv(c.Context(), env, input.AccountID, input.NetworkExitID)
		if err != nil {
			return hubError(c, err)
		}
		action := actionForEnvironment("create", environment)
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		finish := func(outcome, reason string, current hub.EnvironmentContext) error {
			action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
			action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
			return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
		}
		var access hub.NetworkExitAccess
		if input.NetworkExitID != "" {
			var reason string
			access, reason, err = verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID)
			if err != nil {
				_ = finish("failed", reason, environment)
				return hubError(c, err)
			}
		}
		gateway, err := store.GetGateway(c.Context(), env.Gateway)
		if err != nil {
			_ = finish("failed", "gateway_unavailable", environment)
			return hubError(c, err)
		}
		if environment.RuntimeCleanupPending {
			if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil {
				_ = finish("unknown", "cleanup_result_unknown", environment)
				return hubError(c, cleanupErr)
			}
			environment, err = store.GetEnvironmentContext(c.Context(), env.Alias)
			if err != nil {
				return hubError(c, err)
			}
		}
		running := accountRunnable(environment)
		browserPath := ""
		networkExit := gatewayNetworkExit{}
		if input.NetworkExitID != "" {
			networkExit = gatewayNetworkExitFor(access)
		}
		if !created {
			runtime, found, reconcileErr := reconcileGatewayRuntime(c.Context(), gateway, env.Alias)
			if reconcileErr != nil {
				_ = finish("unknown", "gateway_result_unknown", environment)
				return hubError(c, gatewayUnreachable(reconcileErr))
			}
			if found && !running {
				if !runtimeMatchesBinding(runtime, environment) {
					_ = finish("failed", "runtime_unavailable", environment)
					return hubError(c, hub.ErrConflict)
				}
				if runtime.State == "running" {
					stopped := environment
					stopped.RuntimeID, stopped.RuntimeNetworkID = runtime.ID, runtime.NetworkID
					if err := stopEnvironmentRuntime(c.Context(), store, stopped); err != nil {
						_ = finish("unknown", "cleanup_result_unknown", environment)
						return hubError(c, err)
					}
				}
				if err := finish("succeeded", "environment_reused", environment); err != nil {
					return hubError(c, err)
				}
				return c.JSON(map[string]string{"alias": env.Alias})
			}
			if found && runtime.State == "running" {
				ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, runtime)
				if restoreErr != nil {
					outcome, reason := runtimeRecoveryFailure(c.Context(), store, environment.Alias, restoreErr)
					_ = finish(outcome, reason, environment)
					return hubError(c, restoreErr)
				}
				if !ready {
					_ = finish("failed", "runtime_unavailable", environment)
					return c.Status(fiber.StatusConflict).JSON(map[string]string{"error": "environment exists but runtime is not running"})
				}
				environment, err = store.GetEnvironmentContext(c.Context(), env.Alias)
				if err != nil {
					return hubError(c, err)
				}
				if err := finish("succeeded", "environment_reused", environment); err != nil {
					return hubError(c, err)
				}
				return c.JSON(map[string]string{"alias": env.Alias})
			}
		}
		status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers",
			gatewayCreatePayloadForAccount(environment, browserPath, networkExit), gatewayLongTimeout)
		if callErr != nil || status != http.StatusCreated {
			createErr := gatewayRejected(status, body)
			if callErr != nil {
				createErr = gatewayUnreachable(callErr)
			}
			reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
			if reconcileErr != nil {
				_ = finish("unknown", "gateway_result_unknown", environment)
				return hubError(c, errors.Join(createErr, reconcileErr))
			}
			_ = finish("failed", "gateway_create_failed", environment)
			return hubError(c, createErr)
		}
		var createdRuntime runtimeStatus
		if json.Unmarshal(body, &createdRuntime) != nil || !validCreatedRuntime(createdRuntime, environment, running) {
			reconcileErr := reconcileGatewayCreate(c.Context(), store, gateway, environment, status, callErr, body)
			_ = finish("unknown", "gateway_result_unknown", environment)
			return hubError(c, errors.Join(gatewayFailure{status: http.StatusBadGateway,
				message: "gateway create result unknown; environment retained for reconciliation"}, reconcileErr))
		}
		if !running {
			if err := finish("succeeded", "environment_created", environment); err != nil {
				return hubError(c, err)
			}
			return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias})
		}
		environment, err = activateGatewayRuntime(c.Context(), store, gateway, environment, createdRuntime.ID, createdRuntime.NetworkID)
		if err != nil {
			_ = finish("failed", "runtime_persistence_failed", environment)
			return hubError(c, err)
		}
		if err := finish("succeeded", "environment_created", environment); err != nil {
			return hubError(c, err)
		}
		return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias})
	}
}

func browserAction(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler {
	return func(c fiber.Ctx) error {
		switch c.Params("action") {
		case "start", "stop":
			return lifecycleAction(store, probe, resolve, c)
		case "rebind":
			return rebindBrowser(store, probe, resolve, c)
		default:
			return hubError(c, hub.ErrInvalid)
		}
	}
}

func lifecycleAction(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error {
	environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
	if err != nil {
		return hubError(c, err)
	}
	actionName := strings.Clone(c.Params("action"))
	if actionName == "start" {
		if !accountRunnable(environment) {
			return hubError(c, hub.ErrConflict)
		}
		action := actionForEnvironment(actionName, environment)
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		finish := func(outcome, reason string, current hub.EnvironmentContext) error {
			action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID
			action.BindingVersion, action.NetworkExitID = current.BindingVersion, current.Exit.ID
			return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
		}
		return startBrowser(store, probe, resolve, c, environment, finish)
	}
	if err := stopEnvironmentRuntime(c.Context(), store, environment); err != nil {
		return hubError(c, err)
	}
	return c.SendStatus(fiber.StatusNoContent)
}

func stopAccountRuntime(ctx context.Context, store RuntimeStopStore, accountID string) error {
	environment, err := store.GetEnvironmentContextForAccount(ctx, accountID)
	if errors.Is(err, hub.ErrNotFound) {
		return nil
	}
	if err != nil {
		return err
	}
	return stopEnvironmentRuntime(ctx, store, environment)
}

func stopEnvironmentRuntime(ctx context.Context, store RuntimeStopStore, environment hub.EnvironmentContext) error {
	action := actionForEnvironment("stop", environment)
	if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
		return err
	}
	finish := func(outcome, reason string) error {
		action.Outcome, action.ReasonCode = outcome, reason
		return store.AppendEnvironmentAction(ctx, "environment_action_finished", action)
	}
	gateway, err := store.GetGateway(ctx, environment.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable")
		return err
	}
	if environment.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(ctx, store, gateway, environment); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown")
			return cleanupErr
		}
		return finish("succeeded", "environment_stopped")
	}
	if environment.RuntimeID == "" {
		runtime, found, reconcileErr := reconcileGatewayRuntime(ctx, gateway, environment.Alias)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown")
			return gatewayUnreachable(reconcileErr)
		}
		if !found {
			return finish("succeeded", "environment_stopped")
		}
		if runtime.BindingVersion != environment.BindingVersion {
			_ = finish("failed", "runtime_generation_conflict")
			return hub.ErrConflict
		}
		environment = runtimeCleanupGeneration(environment, environment.BindingVersion, runtime.ID, runtime.NetworkID)
	} else {
		environment = runtimeCleanupGeneration(environment, environment.BindingVersion, environment.RuntimeID)
	}
	if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
		_ = finish("failed", "runtime_release_failed")
		return err
	}
	status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop",
		gatewayCleanupGenerationPayload(environment), 30*time.Second)
	if callErr != nil || status >= http.StatusInternalServerError {
		runtime, found, reconcileErr := reconcileGatewayRuntime(ctx, gateway, environment.Alias)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown")
			if callErr != nil {
				return gatewayUnreachable(callErr)
			}
			return gatewayRejected(status, body)
		}
		if !found || runtime.State != "running" {
			if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
				_ = finish("failed", "runtime_release_failed")
				return err
			}
			return finish("succeeded", "gateway_reconciled")
		}
		_ = finish("failed", "gateway_action_failed")
		if callErr != nil {
			return gatewayUnreachable(callErr)
		}
		return gatewayRejected(status, body)
	}
	if status != http.StatusNoContent && status != http.StatusNotModified && status != http.StatusNotFound {
		_ = finish("failed", "gateway_rejected")
		return gatewayRejected(status, body)
	}
	if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
		_ = finish("failed", "runtime_release_failed")
		return err
	}
	return finish("succeeded", "environment_stopped")
}

func startBrowser(store HubStore, probe NetworkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx,
	environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error {
	if err := startBrowserRuntime(c.Context(), store, probe, environment, finish); err != nil {
		return hubError(c, err)
	}
	return c.SendStatus(fiber.StatusNoContent)
}

func startBrowserRuntime(ctx context.Context, store HubStore, probe NetworkExitProbe,
	environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error {
	if !accountRunnable(environment) {
		return hub.ErrConflict
	}
	var access hub.NetworkExitAccess
	var err error
	if environment.Exit.ID != "" {
		var reason string
		access, reason, err = verifyNetworkExit(ctx, store, probe, environment.Exit.ID)
		if err != nil {
			if cleanupErr := discardRuntime(ctx, store, environment); cleanupErr != nil {
				_ = finish("unknown", "cleanup_result_unknown", environment)
				return cleanupErr
			}
			_ = finish("failed", reason, environment)
			return err
		}
	}
	gateway, err := store.GetGateway(ctx, environment.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable", environment)
		return err
	}
	if environment.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(ctx, store, gateway, environment); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown", environment)
			return cleanupErr
		}
		environment, err = store.GetEnvironmentContext(ctx, environment.Alias)
		if err != nil {
			return err
		}
		if !accountRunnable(environment) {
			return hub.ErrConflict
		}
	}
	browserPath := ""
	networkExit := gatewayNetworkExit{}
	if environment.Exit.ID != "" {
		networkExit = gatewayNetworkExitFor(access)
	}
	runtime, found, err := reconcileGatewayRuntime(ctx, gateway, environment.Alias)
	if err != nil {
		_ = finish("unknown", "gateway_result_unknown", environment)
		return gatewayUnreachable(err)
	}
	if found && runtime.State == "running" && runtime.ProxyReady && runtimeMatchesBinding(runtime, environment) {
		environment, err = activateGatewayRuntime(ctx, store, gateway, environment, runtime.ID, runtime.NetworkID, runtime.NodeID)
		if err != nil {
			_ = finish("failed", "runtime_persistence_failed", environment)
			return err
		}
		if err := finish("succeeded", "gateway_reconciled", environment); err != nil {
			return err
		}
		return nil
	}
	if found {
		cleanup := environment
		cleanup.RuntimeID, cleanup.RuntimeNetworkID = runtime.ID, runtime.NetworkID
		if _, removeErr := removeGatewayRuntime(ctx, store, gateway, cleanup); removeErr != nil {
			_ = finish("unknown", "gateway_result_unknown", environment)
			return removeErr
		}
	} else if err := releaseRuntime(ctx, store, environment); err != nil {
		_ = finish("failed", "runtime_release_failed", environment)
		return err
	}
	status, body, callErr := gatewayCall(ctx, gateway, http.MethodPost, "/v1/browsers",
		gatewayCreatePayload(environment, browserPath, networkExit), gatewayLongTimeout)
	if callErr != nil || status != http.StatusCreated {
		reconcileErr := reconcileGatewayCreate(ctx, store, gateway, environment, status, callErr, body)
		if reconcileErr != nil {
			_ = finish("unknown", "gateway_result_unknown", environment)
			return errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr)
		}
		_ = finish("failed", "gateway_create_failed", environment)
		if callErr != nil {
			return gatewayUnreachable(callErr)
		}
		return gatewayRejected(status, body)
	}
	var created runtimeStatus
	if json.Unmarshal(body, &created) != nil || !validCreatedRuntime(created, environment, true) {
		reconcileErr := reconcileGatewayCreate(ctx, store, gateway, environment, status, callErr, body)
		_ = finish("unknown", "gateway_result_unknown", environment)
		return errors.Join(gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}, reconcileErr)
	}
	environment, err = activateGatewayRuntime(ctx, store, gateway, environment, created.ID, created.NetworkID, created.NodeID)
	if err != nil {
		_ = finish("failed", "runtime_persistence_failed", environment)
		return err
	}
	if err := finish("succeeded", "environment_started", environment); err != nil {
		return err
	}
	return nil
}

type runtimeCreateSpec struct {
	browserPath string
	networkExit gatewayNetworkExit
}

func prepareRuntimeCreate(_ context.Context, _ HubStore, _ func(hub.NetworkExitAccess) (string, error),
	environment hub.EnvironmentContext, access hub.NetworkExitAccess) (runtimeCreateSpec, error) {
	networkExit := gatewayNetworkExit{}
	if environment.Exit.ID != "" {
		networkExit = gatewayNetworkExitFor(access)
	}
	return runtimeCreateSpec{networkExit: networkExit}, nil
}

func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (runtimeStatus, error) {
	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers",
		gatewayCreatePayloadForAccount(environment, spec.browserPath, spec.networkExit), gatewayLongTimeout)
	unknown := runtimeStatus{NetworkID: gatewayNetworkID(body)}
	if callErr == nil && status == http.StatusCreated {
		var created runtimeStatus
		if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, accountRunnable(environment)) {
			created.Alias, created.BindingVersion = environment.Alias, environment.BindingVersion
			if accountRunnable(environment) {
				created.State, created.NetworkExitID, created.ProxyReady = "running", environment.Exit.ID, true
			} else {
				created.State = "exited"
			}
			return created, nil
		}
	}
	if callErr != nil {
		return unknown, gatewayUnreachable(callErr)
	}
	if status != http.StatusCreated {
		return unknown, gatewayRejected(status, body)
	}
	return unknown, errors.New("gateway returned an invalid runtime generation")
}

func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, browserPath string) error {
	payload := gatewayCreatePayload(environment, browserPath, gatewayNetworkExit{})
	payload["network_exit_id"], payload["stopped"] = "", true
	status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", payload, gatewayLongTimeout)
	if callErr == nil && status == http.StatusCreated {
		var created runtimeStatus
		if json.Unmarshal(body, &created) == nil && validCreatedRuntime(created, environment, false) {
			return nil
		}
		return errors.New("gateway returned an invalid stopped runtime generation")
	}
	if callErr != nil {
		return gatewayUnreachable(callErr)
	}
	if status != http.StatusCreated {
		return gatewayRejected(status, body)
	}
	return errors.New("gateway did not preserve the stopped runtime")
}

func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) {
	removed := environment.RuntimeCleanupPending
	if environment.RuntimeCleanupPending {
		if environment.RuntimeCleanupRuntimeID == missingRuntimeID && environment.RuntimeCleanupNetworkID == "" {
			// 未知代 pending：创建结果未知时登记的 runtime-not-found 没有实物 ID 可供清理。
			// 按别名反查网关：无容器则该代无实物，直接解除；有同版本容器则采用其实物代继续清理；
			// 版本不符说明别名下已是别的代，维持人工对账。
			if environment.RuntimeCleanupBindingVersion < 1 {
				return false, hub.ErrReconcileRequired
			}
			discovered, found, discoverErr := reconcileGatewayRuntime(ctx, target, environment.Alias)
			if discoverErr != nil {
				return false, discoverErr
			}
			if !found {
				return true, store.SetRuntimeCleanupPending(ctx, environment, false)
			}
			if discovered.BindingVersion != environment.RuntimeCleanupBindingVersion {
				return false, hub.ErrReconcileRequired
			}
			if err := store.SetRuntimeCleanupPending(ctx, environment, false); err != nil {
				return false, err
			}
			environment = runtimeCleanupGeneration(environment, environment.RuntimeCleanupBindingVersion, discovered.ID, discovered.NetworkID)
			if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
				return false, err
			}
		} else if environment.RuntimeCleanupBindingVersion < 1 || environment.RuntimeCleanupRuntimeID == "" {
			return false, hub.ErrReconcileRequired
		}
	}
	if !environment.RuntimeCleanupPending {
		runtime, found, err := reconcileGatewayRuntime(ctx, target, environment.Alias)
		if err != nil {
			return false, err
		}
		bindingVersion, runtimeID, networkID := environment.BindingVersion, environment.RuntimeID, environment.RuntimeNetworkID
		if environment.RuntimeCleanupBindingVersion > 0 {
			bindingVersion, runtimeID, networkID = environment.RuntimeCleanupBindingVersion,
				environment.RuntimeCleanupRuntimeID, environment.RuntimeCleanupNetworkID
		}
		if found {
			if runtime.BindingVersion != bindingVersion || runtimeID == missingRuntimeID ||
				(runtimeID != "" && runtimeID != runtime.ID) || (runtimeID == "" && networkID == "" && runtime.State == "running") {
				return false, hub.ErrConflict
			}
			runtimeID = runtime.ID
			if networkID != "" {
				if runtime.NetworkID != "" && runtime.NetworkID != networkID {
					return false, hub.ErrConflict
				}
			} else {
				networkID = runtime.NetworkID
			}
		} else if environment.RuntimeID == "" {
			runtimeID = missingRuntimeID
		}
		environment = runtimeCleanupGeneration(environment, bindingVersion, runtimeID, networkID)
	}
	if !environment.RuntimeCleanupPending {
		if err := store.SetRuntimeCleanupPending(ctx, environment, true); err != nil {
			return false, err
		}
	}
	for attempt := 0; attempt < 2; attempt++ {
		status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias,
			gatewayCleanupGenerationPayload(environment), 30*time.Second)
		if callErr == nil && (status == http.StatusNoContent || status == http.StatusNotFound) {
			return true, store.SetRuntimeCleanupPending(ctx, environment, false)
		}
		if callErr == nil && status == http.StatusAccepted {
			removed = true
			continue
		}
		_, found, reconcileErr := reconcileGatewayRuntime(ctx, target, environment.Alias)
		if reconcileErr != nil {
			if callErr != nil {
				return removed, errors.Join(gatewayUnreachable(callErr), gatewayUnreachable(reconcileErr))
			}
			return removed, errors.Join(gatewayRejected(status, body), gatewayUnreachable(reconcileErr))
		}
		if found {
			if callErr != nil {
				return removed, gatewayUnreachable(callErr)
			}
			return removed, gatewayRejected(status, body)
		}
		removed = true
		if callErr != nil {
			return true, gatewayUnreachable(callErr)
		}
		return true, gatewayRejected(status, body)
	}
	if removed {
		return true, gatewayFailure{status: http.StatusBadGateway, message: "gateway removed the runtime but network cleanup is pending"}
	}
	return false, gatewayFailure{status: http.StatusBadGateway, message: "gateway runtime cleanup is pending"}
}

func restoreRebindRuntime(ctx context.Context, store HubStore, resolve func(hub.NetworkExitAccess) (string, error),
	target hub.Gateway, environment hub.EnvironmentContext, previous runtimeStatus, found bool, prepared *runtimeCreateSpec) (bool, error) {
	if !found {
		return true, releaseRuntime(ctx, store, environment)
	}
	if environment.Exit.ID == "" {
		if err := releaseRuntime(ctx, store, environment); err != nil {
			return false, err
		}
		if previous.State == "running" {
			return false, nil
		}
		if err := createStoppedGatewayRuntime(ctx, target, environment, ""); err != nil {
			_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
			return false, errors.Join(err, cleanupErr)
		}
		return true, nil
	}
	spec := runtimeCreateSpec{}
	if prepared != nil {
		spec = *prepared
	} else {
		access, err := store.GetNetworkExitAccess(ctx, environment.Exit.ID)
		if err != nil {
			return false, err
		}
		spec, err = prepareRuntimeCreate(ctx, store, resolve, environment, access)
		if err != nil {
			return false, err
		}
	}
	if err := releaseRuntime(ctx, store, environment); err != nil {
		return false, err
	}
	cleanupContext, err := store.GetEnvironmentContext(ctx, environment.Alias)
	if err != nil {
		return false, err
	}
	created, err := createGatewayRuntime(ctx, target, environment, spec)
	if err != nil {
		cleanupErr := reconcileGatewayCreateGeneration(ctx, store, target, cleanupContext, environment.BindingVersion, created.NetworkID, false)
		return false, errors.Join(err, cleanupErr)
	}
	if previous.State == "running" && accountRunnable(environment) {
		_, err = activateGatewayRuntime(ctx, store, target, environment, created.ID, created.NetworkID)
	} else if accountRunnable(environment) {
		stopped := environment
		stopped.RuntimeID, stopped.RuntimeNetworkID = created.ID, created.NetworkID
		status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop",
			gatewayGenerationPayload(stopped), 30*time.Second)
		if callErr != nil {
			err = gatewayUnreachable(callErr)
		} else if status != http.StatusNoContent {
			err = gatewayRejected(status, body)
		}
	}
	if err != nil {
		_, cleanupErr := removeGatewayRuntime(ctx, store, target, environment)
		return false, errors.Join(err, cleanupErr)
	}
	return true, nil
}

func rebindRecoveryError(ready bool, err error) error {
	if err != nil {
		return err
	}
	if !ready {
		return gatewayFailure{status: http.StatusBadGateway, message: "rebind recovery is incomplete; retry reconciliation"}
	}
	return nil
}

func runtimeCreateSpecMatches(_ context.Context, _ HubStore, current, previous hub.EnvironmentContext, prepared *runtimeCreateSpec) bool {
	return prepared != nil && current.BindingVersion == previous.BindingVersion && current.Exit.ID == previous.Exit.ID
}

func rebindBrowser(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error {
	var input struct {
		NetworkExitID string `json:"network_exit_id"`
	}
	if err := decodeHubJSON(c, &input); err != nil {
		return hubError(c, err)
	}
	before, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
	if err != nil {
		return hubError(c, err)
	}
	action := actionForEnvironment("rebind", before)
	if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
		return hubError(c, err)
	}
	finish := func(outcome, reason string, current hub.EnvironmentContext) error {
		action.Outcome, action.ReasonCode = outcome, reason
		action.NetworkExitID, action.BindingVersion, action.RuntimeInstanceID = current.Exit.ID, current.BindingVersion, current.RuntimeInstanceID
		return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
	}
	if !hub.ValidNetworkExitID(input.NetworkExitID) {
		if err := finish("failed", "rebind_input_rejected", before); err != nil {
			return hubError(c, err)
		}
		return hubError(c, hub.ErrInvalid)
	}
	access, reason, err := verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID)
	if err != nil {
		_ = finish("failed", reason, before)
		return hubError(c, err)
	}
	target, err := store.GetGateway(c.Context(), before.Gateway)
	if err != nil {
		_ = finish("failed", "gateway_unavailable", before)
		return hubError(c, err)
	}
	if before.RuntimeCleanupPending {
		if _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before); cleanupErr != nil {
			_ = finish("unknown", "cleanup_result_unknown", before)
			return hubError(c, cleanupErr)
		}
		before, err = store.GetEnvironmentContext(c.Context(), before.Alias)
		if err != nil {
			return hubError(c, err)
		}
	}
	runtime, found, err := reconcileGatewayRuntime(c.Context(), target, before.Alias)
	if err != nil {
		_ = finish("unknown", "gateway_result_unknown", before)
		return hubError(c, gatewayUnreachable(err))
	}
	wasRunning := found && runtime.State == "running"
	if err := store.ValidateEnvironmentRebind(c.Context(), before.Alias, input.NetworkExitID, before.BindingVersion); err != nil {
		_ = finish("failed", "rebind_not_allowed", before)
		return hubError(c, err)
	}
	var previousSpec *runtimeCreateSpec
	if found && before.Exit.ID == "" {
		previousSpec = &runtimeCreateSpec{}
	} else if found {
		previousAccess, accessErr := store.GetNetworkExitAccess(c.Context(), before.Exit.ID)
		if accessErr != nil {
			_ = finish("failed", "exit_unavailable", before)
			return hubError(c, accessErr)
		}
		prepared, prepareErr := prepareRuntimeCreate(c.Context(), store, resolve, before, previousAccess)
		if prepareErr != nil {
			_ = finish("failed", "runtime_prepare_failed", before)
			return hubError(c, prepareErr)
		}
		previousSpec = &prepared
	}
	candidate := before
	candidate.Exit, candidate.BindingVersion = access.NetworkExit, before.BindingVersion+1
	candidate.RuntimeInstanceID, candidate.RuntimeID = "", ""
	var nextSpec runtimeCreateSpec
	if wasRunning {
		nextSpec, err = prepareRuntimeCreate(c.Context(), store, resolve, candidate, access)
		if err != nil {
			_ = finish("failed", "runtime_prepare_failed", before)
			return hubError(c, err)
		}
	}
	if found {
		_, removeErr := removeGatewayRuntime(c.Context(), store, target, before)
		if removeErr != nil {
			_ = finish("unknown", "gateway_result_unknown", before)
			return hubError(c, removeErr)
		}
	}
	var candidateRuntime runtimeStatus
	if wasRunning {
		candidateRuntime, err = createGatewayRuntime(c.Context(), target, candidate, nextSpec)
		if err != nil {
			cleanupContext, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			if contextErr != nil {
				_ = finish("unknown", "gateway_result_unknown", before)
				return hubError(c, errors.Join(err, contextErr))
			}
			cleanupErr := reconcileGatewayCreateGeneration(c.Context(), store, target, cleanupContext, candidate.BindingVersion, candidateRuntime.NetworkID, false)
			if cleanupErr != nil {
				_ = finish("unknown", "gateway_result_unknown", before)
				return hubError(c, errors.Join(err, cleanupErr))
			}
			current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			ready := false
			if contextErr == nil {
				ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, runtime, found, previousSpec)
			}
			contextErr = rebindRecoveryError(ready, contextErr)
			if contextErr != nil {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, contextErr))
			}
			_ = finish("failed", "gateway_create_failed", before)
			return hubError(c, err)
		}
	}
	runtimeID := ""
	if accountRunnable(candidate) {
		runtimeID = candidateRuntime.ID
	}
	after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, runtimeID, before.BindingVersion, candidateRuntime.NetworkID)
	if err != nil {
		if candidateRuntime.ID != "" {
			cleanup, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			if contextErr != nil || cleanup.BindingVersion != before.BindingVersion || cleanup.Exit.ID != before.Exit.ID {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, contextErr, hub.ErrConflict))
			}
			cleanup.RuntimeID, cleanup.RuntimeNetworkID = candidateRuntime.ID, candidateRuntime.NetworkID
			_, cleanupErr := removeGatewayRuntime(c.Context(), store, target, cleanup)
			if cleanupErr != nil {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, cleanupErr))
			}
		}
		if found {
			current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias)
			ready := false
			if contextErr == nil {
				prepared := previousSpec
				if !runtimeCreateSpecMatches(c.Context(), store, current, before, previousSpec) {
					prepared = nil
				}
				ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, runtime, true, prepared)
			}
			contextErr = rebindRecoveryError(ready, contextErr)
			if contextErr != nil {
				_ = finish("unknown", "rebind_recovery_failed", before)
				return hubError(c, errors.Join(err, contextErr))
			}
		}
		_ = finish("failed", "rebind_not_allowed", before)
		return hubError(c, err)
	}
	if err := finish("succeeded", "environment_rebound", after); err != nil {
		return hubError(c, err)
	}
	return c.JSON(after)
}

func deleteBrowser(store HubStore) fiber.Handler {
	return func(c fiber.Ctx) error {
		environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias"))
		if err != nil {
			return hubError(c, err)
		}
		action := actionForEnvironment("recycle", environment)
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil {
			return hubError(c, err)
		}
		gateway, err := store.GetGateway(c.Context(), environment.Gateway)
		if err != nil {
			action.Outcome, action.ReasonCode = "failed", "gateway_unavailable"
			_ = store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
			return hubError(c, err)
		}
		if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil {
			action.Outcome, action.ReasonCode = "unknown", "cleanup_result_unknown"
			_ = store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action)
			return hubError(c, removeErr)
		}
		action.Outcome, action.ReasonCode = "succeeded", "environment_recycled"
		if err := store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action); err != nil {
			return hubError(c, err)
		}
		return c.SendStatus(fiber.StatusNoContent)
	}
}

func actionForEnvironment(actionName string, environment hub.EnvironmentContext) hub.EnvironmentAction {
	return hub.EnvironmentAction{
		OperationID: hub.NewOperationID(), Action: actionName, ReasonCode: "action_requested",
		AccountID: environment.AccountID, BrowserEnvAlias: environment.Alias, NetworkExitID: environment.Exit.ID,
		RuntimeInstanceID: environment.RuntimeInstanceID, BindingVersion: environment.BindingVersion,
	}
}

func decodeHubJSON(c fiber.Ctx, destination any) error {
	decoder := json.NewDecoder(bytes.NewReader(c.Body()))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(destination); err != nil {
		return hub.ErrInvalid
	}
	if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
		return hub.ErrInvalid
	}
	return nil
}

func hubError(c fiber.Ctx, err error) error {
	var failure gatewayFailure
	if errors.As(err, &failure) {
		return c.Status(failure.status).JSON(map[string]string{"error": failure.message})
	}
	status := fiber.StatusInternalServerError
	message := "hub operation failed"
	switch {
	case errors.Is(err, hub.ErrInvalid):
		status, message = fiber.StatusBadRequest, hub.ErrInvalid.Error()
	case errors.Is(err, hub.ErrReconcileRequired):
		status, message = fiber.StatusConflict, hub.ErrReconcileRequired.Error()
	case errors.Is(err, hub.ErrConflict):
		status, message = fiber.StatusConflict, hub.ErrConflict.Error()
	case errors.Is(err, hub.ErrNotFound):
		status, message = fiber.StatusNotFound, hub.ErrNotFound.Error()
	default:
		return c.Status(status).JSON(map[string]string{"error": err.Error()})
	}
	return c.Status(status).JSON(map[string]string{"error": message})
}

// RegisterEnvironments exposes environment routes to the grouped API package.
func RegisterEnvironments(app *fiber.App, store *hub.Store) {
	registerHub(app, store)
}

// ReconcileRuntimeLeases performs one native runtime lease reconciliation pass.
func ReconcileRuntimeLeases(ctx context.Context, store *hub.Store) error {
	return reconcileRuntimeLeases(ctx, store, defaultNetworkExitProbe(), nil)
}

// RegisterHubWithNetwork exposes environment route registration with explicit network dependencies for tests and composition.
func RegisterHubWithNetwork(app *fiber.App, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) {
	registerHubWithNetwork(app, store, probe, resolve)
}

// hubStore remains the package-local name used by lifecycle test doubles; HubStore is its composition boundary.
type hubStore = HubStore