creator-hub/cmd/control-plane/hub_test.go

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"git.ipao.vip/rogee/creator-hub/internal/hub"
	"git.ipao.vip/rogee/creator-hub/internal/phasea"
	"github.com/gofiber/fiber/v3"
	"github.com/gofiber/fiber/v3/middleware/adaptor"
)

// memoryStore 是 hubStore 的内存桩，记录写入以便断言编排副作用。
type memoryStore struct {
	mu                             sync.Mutex
	gateways                       map[string]hub.Gateway
	images                         map[string]hub.Image
	envs                           map[string]hub.Env
	exits                          map[string]hub.NetworkExit
	bindings                       map[string]hub.EnvironmentContext
	actions                        []hub.EnvironmentAction
	upgraded                       map[string]string
	upgradeErr                     error
	releaseErr                     error
	cleanupPendingErr              error
	cleanupPendingErrAfterMutation bool
	gatewayFn                      func(name string) (hub.Gateway, error)
}

func newMemoryStore() *memoryStore {
	return &memoryStore{
		gateways: map[string]hub.Gateway{},
		images:   map[string]hub.Image{},
		envs:     map[string]hub.Env{},
		exits: map[string]hub.NetworkExit{
			"exit-1": {ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1},
		},
		bindings: map[string]hub.EnvironmentContext{},
		upgraded: map[string]string{},
	}
}

func (s *memoryStore) CreateGateway(_ context.Context, _, _, _ string) (hub.Gateway, error) {
	return hub.Gateway{}, nil
}
func (s *memoryStore) ListGateways(context.Context) ([]hub.Gateway, error) { return nil, nil }
func (s *memoryStore) GetGateway(_ context.Context, name string) (hub.Gateway, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.gatewayFn != nil {
		return s.gatewayFn(name)
	}
	gateway, ok := s.gateways[name]
	if !ok {
		return hub.Gateway{}, hub.ErrNotFound
	}
	return gateway, nil
}
func (s *memoryStore) DeleteGateway(context.Context, string) error { return nil }
func (s *memoryStore) CreateImage(_ context.Context, image hub.Image) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.images[image.Version] = image
	return nil
}
func (s *memoryStore) UpdateImage(_ context.Context, image hub.Image) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.images[image.Version] = image
	return nil
}
func (s *memoryStore) ListImages(context.Context, bool) ([]hub.Image, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	images := make([]hub.Image, 0, len(s.images))
	for _, image := range s.images {
		images = append(images, image)
	}
	return images, nil
}
func (s *memoryStore) DeleteImage(context.Context, string) error { return nil }
func (s *memoryStore) ImageRef(_ context.Context, version string) (string, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	image, ok := s.images[version]
	if !ok || !image.Enabled {
		return "", hub.ErrNotFound
	}
	return image.ImageRef, nil
}
func (s *memoryStore) CreateEnv(_ context.Context, env hub.Env) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if _, exists := s.envs[env.Alias]; exists {
		return hub.ErrConflict
	}
	if image, exists := s.images[env.ImageVersion]; !exists || !image.Enabled {
		return hub.ErrNotFound
	}
	s.envs[env.Alias] = env
	return nil
}
func (s *memoryStore) ListEnvs(context.Context) ([]hub.Env, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	envs := make([]hub.Env, 0, len(s.envs))
	for _, env := range s.envs {
		envs = append(envs, env)
	}
	return envs, nil
}
func (s *memoryStore) GetEnv(_ context.Context, alias string) (hub.Env, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	env, ok := s.envs[alias]
	if !ok {
		return hub.Env{}, hub.ErrNotFound
	}
	return env, nil
}
func (s *memoryStore) UpgradeEnv(_ context.Context, alias, version string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.upgradeErr != nil {
		return s.upgradeErr
	}
	if _, ok := s.envs[alias]; !ok {
		return hub.ErrNotFound
	}
	if s.bindings[alias].RuntimeCleanupPending {
		return hub.ErrConflict
	}
	if image, exists := s.images[version]; !exists || !image.Enabled {
		return hub.ErrNotFound
	}
	s.upgraded[alias] = version
	env := s.envs[alias]
	env.ImageVersion = version
	s.envs[alias] = env
	bound, ok := s.bindings[alias]
	if !ok {
		bound = hub.EnvironmentContext{Env: env, AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
	}
	bound.Env, bound.BindingVersion = env, bound.BindingVersion+1
	s.bindings[alias] = bound
	return nil
}
func (s *memoryStore) CreateNetworkExit(_ context.Context, exit hub.NetworkExit, credentialID string) (hub.NetworkExit, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	exit.ID, exit.HealthStatus, exit.Version = "exit-created", "unchecked", 1
	if credentialID != "" {
		exit.CredentialReference = &hub.CredentialReference{ID: credentialID, Provider: "os_keyring"}
	}
	s.exits[exit.ID] = exit
	return exit, nil
}
func (s *memoryStore) ListNetworkExits(context.Context) ([]hub.NetworkExit, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	result := make([]hub.NetworkExit, 0, len(s.exits))
	for _, exit := range s.exits {
		result = append(result, exit)
	}
	return result, nil
}
func (s *memoryStore) GetNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	exit, ok := s.exits[id]
	if !ok {
		return hub.NetworkExit{}, hub.ErrNotFound
	}
	return exit, nil
}
func (s *memoryStore) GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error) {
	exit, err := s.GetNetworkExit(ctx, id)
	return hub.NetworkExitAccess{NetworkExit: exit}, err
}
func (s *memoryStore) RecordNetworkExitCheck(_ context.Context, id string, observation hub.ExitObservation, failure string) (hub.NetworkExit, string, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	exit, ok := s.exits[id]
	if !ok {
		return hub.NetworkExit{}, "exit_unavailable", hub.ErrNotFound
	}
	reason := failure
	if reason == "" && exit.ExpectedPublicIP != "" && exit.ExpectedPublicIP != observation.PublicIP {
		reason = "exit_ip_drift"
	}
	if reason == "" && exit.ExpectedRegion != "" && exit.ExpectedRegion != observation.Region {
		reason = "exit_region_drift"
	}
	exit.ObservedPublicIP, exit.ObservedRegion = observation.PublicIP, observation.Region
	if reason == "" {
		exit.HealthStatus, reason = "healthy", "exit_healthy"
	} else {
		exit.HealthStatus = "unhealthy"
	}
	s.exits[id] = exit
	return exit, reason, nil
}
func (s *memoryStore) DisableNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	exit, ok := s.exits[id]
	if !ok {
		return hub.NetworkExit{}, hub.ErrNotFound
	}
	exit.HealthStatus = "disabled"
	s.exits[id] = exit
	return exit, nil
}
func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) {
	s.mu.Lock()
	if existing, ok := s.bindings[env.Alias]; ok {
		s.mu.Unlock()
		return existing, false, nil
	}
	s.mu.Unlock()
	if err := s.CreateEnv(ctx, env); err != nil {
		return hub.EnvironmentContext{}, false, err
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	bound := hub.EnvironmentContext{Env: env, AccountID: accountID, BindingID: accountID, BindingVersion: 1, Exit: s.exits[exitID]}
	s.bindings[env.Alias] = bound
	return bound, true, nil
}
func (s *memoryStore) GetEnvironmentContext(_ context.Context, alias string) (hub.EnvironmentContext, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if bound, ok := s.bindings[alias]; ok {
		return bound, nil
	}
	env, ok := s.envs[alias]
	if !ok {
		return hub.EnvironmentContext{}, hub.ErrNotFound
	}
	return hub.EnvironmentContext{Env: env, AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}, nil
}
func (s *memoryStore) ValidateEnvironmentRebind(_ context.Context, alias, exitID string, expectedBindingVersion int64) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	bound, ok := s.bindings[alias]
	if !ok {
		bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
	}
	if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending || s.exits[exitID].HealthStatus != "healthy" {
		return hub.ErrConflict
	}
	return nil
}
func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (hub.EnvironmentContext, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	bound, ok := s.bindings[alias]
	if !ok {
		bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
	}
	if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending {
		return hub.EnvironmentContext{}, hub.ErrConflict
	}
	bound.RuntimeInstanceID, bound.RuntimeID = "", ""
	if runtimeID != "" {
		bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID
	}
	bound.Exit, bound.BindingVersion = s.exits[exitID], bound.BindingVersion+1
	s.bindings[alias] = bound
	return bound, nil
}
func (s *memoryStore) ActivateRuntime(_ context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (hub.EnvironmentContext, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	bound, ok := s.bindings[alias]
	if !ok {
		bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
	}
	if bound.BindingVersion != bindingVersion || bound.Exit.ID != exitID || bound.RuntimeCleanupPending {
		return hub.EnvironmentContext{}, hub.ErrConflict
	}
	bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID
	s.bindings[alias] = bound
	return bound, nil
}
func (s *memoryStore) ReleaseRuntime(_ context.Context, alias string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.releaseErr != nil {
		return s.releaseErr
	}
	bound, ok := s.bindings[alias]
	if !ok {
		bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
	}
	bound.RuntimeInstanceID, bound.RuntimeID = "", ""
	s.bindings[alias] = bound
	return nil
}
func (s *memoryStore) SetRuntimeCleanupPending(_ context.Context, alias string, pending bool) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.cleanupPendingErr != nil && !s.cleanupPendingErrAfterMutation {
		return s.cleanupPendingErr
	}
	if pending && s.releaseErr != nil {
		return s.releaseErr
	}
	bound, ok := s.bindings[alias]
	if !ok {
		bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
	}
	bound.RuntimeCleanupPending = pending
	if pending {
		bound.RuntimeInstanceID, bound.RuntimeID = "", ""
	}
	s.bindings[alias] = bound
	return s.cleanupPendingErr
}
func (s *memoryStore) AppendEnvironmentAction(_ context.Context, _ string, action hub.EnvironmentAction) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.actions = append(s.actions, action)
	return nil
}

type recordedRequest struct {
	method string
	path   string
	body   map[string]any
}

// fakeGateway 模拟 docker-gateway：按路由表应答并记录请求。
type fakeGateway struct {
	mu                        sync.Mutex
	createOnce                sync.Once
	deleteOnce                sync.Once
	token                     string
	requests                  []recordedRequest
	containers                []containerStatus
	failCreate                int // 前 N 次 create 返回失败
	failDelete                int // 前 N 次 delete 返回 500 且保留容器
	deleteNotFound            int
	failProxy                 bool
	createStarted             chan struct{}
	releaseCreate             <-chan struct{}
	deleteDone                chan struct{}
	releaseDelete             <-chan struct{}
	cleanupPending            int
	disconnectDelete          int
	disconnectList            int
	failList                  int
	invalidList               int
	invalidListBody           string
	readErrorList             int
	disconnectListAfterDelete bool
}

type fakeExitProbe struct {
	observation hub.ExitObservation
	failure     string
}

func (probe fakeExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) {
	if probe.observation.PublicIP == "" && probe.failure == "" {
		probe.observation = hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}
	}
	return probe.observation, probe.failure
}

type sequenceExitProbe struct {
	calls    int
	failures []string
}

func (probe *sequenceExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) {
	failure := ""
	if probe.calls < len(probe.failures) {
		failure = probe.failures[probe.calls]
	}
	probe.calls++
	return hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, failure
}

func (g *fakeGateway) handler(t *testing.T) http.Handler {
	return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
		if request.Header.Get("Authorization") != "Bearer "+g.token {
			response.WriteHeader(http.StatusUnauthorized)
			_, _ = response.Write([]byte(`{"error":"gateway token rejected"}`))
			return
		}
		var body map[string]any
		if request.Body != nil {
			raw, _ := io.ReadAll(request.Body)
			if len(raw) > 0 {
				_ = json.Unmarshal(raw, &body)
			}
		}
		g.mu.Lock()
		g.requests = append(g.requests, recordedRequest{method: request.Method, path: request.URL.Path, body: body})
		g.mu.Unlock()

		switch {
		case request.Method == http.MethodPost && request.URL.Path == "/v1/browsers":
			g.mu.Lock()
			if g.failCreate > 0 {
				g.failCreate--
				g.mu.Unlock()
				response.WriteHeader(http.StatusConflict)
				_, _ = response.Write([]byte(`{"error":"alias already in use"}`))
				return
			}
			g.mu.Unlock()
			if g.createStarted != nil {
				g.createOnce.Do(func() { close(g.createStarted) })
			}
			if g.releaseCreate != nil {
				<-g.releaseCreate
			}
			state, proxyReady := "running", true
			if stopped, _ := body["stopped"].(bool); stopped {
				state, proxyReady = "exited", false
			}
			g.mu.Lock()
			g.containers = []containerStatus{{
				ID: "container-id", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady,
				BindingVersion: int64(body["binding_version"].(float64)), NetworkExitID: body["network_exit_id"].(string),
			}}
			g.mu.Unlock()
			response.WriteHeader(http.StatusCreated)
			_, _ = response.Write([]byte(`{"id":"container-id","alias":"account-a"}`))
		case request.Method == http.MethodGet && request.URL.Path == "/v1/browsers":
			g.mu.Lock()
			if g.failList > 0 {
				g.failList--
				g.mu.Unlock()
				response.WriteHeader(http.StatusInternalServerError)
				_, _ = response.Write([]byte(`{"error":"docker unavailable"}`))
				return
			}
			if g.invalidList > 0 {
				g.invalidList--
				body := g.invalidListBody
				if body == "" {
					body = `{"not":"a browser list"}`
				}
				g.mu.Unlock()
				_, _ = response.Write([]byte(body))
				return
			}
			if g.readErrorList > 0 {
				g.readErrorList--
				g.mu.Unlock()
				connection, _, _ := response.(http.Hijacker).Hijack()
				_, _ = connection.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n[]"))
				_ = connection.Close()
				return
			}
			if g.disconnectList > 0 {
				g.disconnectList--
				g.mu.Unlock()
				connection, _, _ := response.(http.Hijacker).Hijack()
				_ = connection.Close()
				return
			}
			containers := append([]containerStatus{}, g.containers...)
			g.mu.Unlock()
			_ = json.NewEncoder(response).Encode(containers)
		case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/v1/browsers/"):
			g.mu.Lock()
			if g.failDelete > 0 {
				g.failDelete--
				g.mu.Unlock()
				response.WriteHeader(http.StatusInternalServerError)
				_, _ = response.Write([]byte(`{"error":"docker delete failed"}`))
				return
			}
			g.containers = nil
			if g.deleteNotFound > 0 {
				g.deleteNotFound--
				g.mu.Unlock()
				response.WriteHeader(http.StatusNotFound)
				return
			}
			g.mu.Unlock()
			if g.deleteDone != nil {
				g.deleteOnce.Do(func() { close(g.deleteDone) })
			}
			if g.releaseDelete != nil {
				<-g.releaseDelete
			}
			g.mu.Lock()
			if g.cleanupPending > 0 {
				g.cleanupPending--
				g.mu.Unlock()
				response.WriteHeader(http.StatusAccepted)
				return
			}
			if g.disconnectDelete > 0 {
				g.disconnectDelete--
				if g.disconnectListAfterDelete {
					g.disconnectList++
				}
				g.mu.Unlock()
				connection, _, _ := response.(http.Hijacker).Hijack()
				_ = connection.Close()
				return
			}
			g.mu.Unlock()
			response.WriteHeader(http.StatusNoContent)
		case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"):
			g.mu.Lock()
			if len(g.containers) > 0 {
				g.containers[0].State, g.containers[0].Status = "running", "Up"
			}
			g.mu.Unlock()
			response.WriteHeader(http.StatusNoContent)
		case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"):
			g.mu.Lock()
			if len(g.containers) > 0 {
				g.containers[0].State, g.containers[0].Status = "exited", "Exited"
			}
			g.mu.Unlock()
			response.WriteHeader(http.StatusNoContent)
		case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/proxy"):
			g.mu.Lock()
			if g.failProxy {
				g.mu.Unlock()
				response.WriteHeader(http.StatusBadGateway)
				return
			}
			if len(g.containers) > 0 {
				g.containers[0].ProxyReady = true
			}
			g.mu.Unlock()
			response.WriteHeader(http.StatusNoContent)
		default:
			t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path)
		}
	})
}

func (g *fakeGateway) recorded() []recordedRequest {
	g.mu.Lock()
	defer g.mu.Unlock()
	return append([]recordedRequest{}, g.requests...)
}

func newTestApp(t *testing.T, store *memoryStore, gateway *fakeGateway) *fiber.App {
	return newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
}

func newTestAppWithNetwork(t *testing.T, store *memoryStore, gateway *fakeGateway, probe networkExitProbe,
	resolve func(hub.NetworkExitAccess) (string, error)) *fiber.App {
	t.Helper()
	server := httptest.NewServer(gateway.handler(t))
	t.Cleanup(server.Close)
	if store.gateways == nil {
		store.gateways = map[string]hub.Gateway{}
	}
	store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
	app := fiber.New()
	registerHubWithNetwork(app, store, probe, resolve)
	return app
}

func do(app *fiber.App, method, path, body string) *httptest.ResponseRecorder {
	response := httptest.NewRecorder()
	var reader io.Reader
	if body != "" {
		reader = strings.NewReader(body)
	}
	adaptor.FiberApp(app).ServeHTTP(response, httptest.NewRequest(method, path, reader))
	return response
}

const createEnvBody = `{"alias":"account-a","name":"店铺一号","gateway":"gw-1","image_version":"148.0.7778.215",` +
	`"fingerprint":{"seed":2024,"platform":"windows","timezone":"Asia/Shanghai"},"account_id":"account-a","network_exit_id":"exit-1"}`

func TestParseGatewayBrowserListStrict(t *testing.T) {
	for _, test := range []struct {
		name string
		body string
		ok   bool
	}{
		{name: "empty", body: `[]`, ok: true},
		{name: "minimal browser", body: `[{"id":"container-id","alias":"account-a","state":"running"}]`, ok: true},
		{name: "top-level null", body: `null`},
		{name: "null element", body: `[null]`},
		{name: "missing id", body: `[{"alias":"account-a","state":"running"}]`},
		{name: "missing alias", body: `[{"id":"container-id","state":"running"}]`},
		{name: "missing state", body: `[{"id":"container-id","alias":"account-a"}]`},
		{name: "duplicate alias", body: `[{"id":"one","alias":"account-a","state":"running"},{"id":"two","alias":"account-a","state":"exited"}]`},
	} {
		t.Run(test.name, func(t *testing.T) {
			browsers, err := parseGatewayBrowserList([]byte(test.body))
			if (err == nil) != test.ok {
				t.Fatalf("parse result browsers=%#v err=%v", browsers, err)
			}
		})
	}
}

func TestGatewayCallPropagatesBodyReadError(t *testing.T) {
	gateway := &fakeGateway{token: "unit-test-gateway-token", readErrorList: 1}
	server := httptest.NewServer(gateway.handler(t))
	defer server.Close()

	status, body, err := gatewayCall(context.Background(), hub.Gateway{Endpoint: server.URL, Token: gateway.token},
		http.MethodGet, "/v1/browsers", nil, time.Second)
	if status != http.StatusOK || string(body) != "[]" || err == nil {
		t.Fatalf("partial response was accepted: status=%d body=%q err=%v", status, body, err)
	}
}

func TestGatewayCreatePayloadStripsLegacyProxyFingerprint(t *testing.T) {
	payload := gatewayCreatePayload(hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Name: "甲", Fingerprint: hub.Fingerprint{
		Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true,
	}}, BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1"}}, "registry.example/browser:1", gatewayNetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080})
	encoded, _ := json.Marshal(payload)
	if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") {
		t.Fatalf("legacy proxy URI entered the gateway contract: %s", encoded)
	}
}

func TestCreateBrowserOrchestratesGateway(t *testing.T) {
	store := newMemoryStore()
	_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser@sha256:abc", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
	if response.Code != http.StatusCreated {
		t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String())
	}
	if _, kept := store.envs["account-a"]; !kept {
		t.Fatal("env must be persisted after successful gateway create")
	}
	requests := gateway.recorded()
	if len(requests) != 1 || requests[0].method != http.MethodPost || requests[0].path != "/v1/browsers" {
		t.Fatalf("unexpected gateway calls: %#v", requests)
	}
	payload := requests[0].body
	if payload["alias"] != "account-a" || payload["name"] != "店铺一号" ||
		payload["image"] != "registry.example/browser@sha256:abc" ||
		payload["volume"] != "creatorhub-profile-account-a" {
		t.Fatalf("platform must fully specify the gateway payload: %#v", payload)
	}
	exit := payload["network_exit"].(map[string]any)
	if exit["protocol"] != "socks5" || exit["host"] != "127.0.0.1" || exit["port"] != float64(1080) {
		t.Fatalf("platform must force the bound exit: %#v", payload)
	}
	cmd := payload["cmd"].([]any)
	if len(cmd) != 4 || cmd[0] != "--fingerprint=2024" || cmd[1] != "--fingerprint-platform=windows" ||
		cmd[2] != "--timezone=Asia/Shanghai" || cmd[3] != "about:blank" {
		t.Fatalf("cmd must carry fingerprint args plus start url: %#v", cmd)
	}
	if stored := store.envs["account-a"].Fingerprint; stored.ProxyServer != "" || stored.DisableNonProxiedUDP {
		t.Fatalf("persistent fingerprint must not contain proxy material: %#v", stored)
	}
	if len(store.actions) != 2 || store.actions[0].OperationID != store.actions[1].OperationID ||
		store.actions[0].Action != "create" || store.actions[1].Outcome != "succeeded" {
		t.Fatalf("create must emit a correlated audit pair: %#v", store.actions)
	}
}

func TestExitFailuresStopCreateBeforeGateway(t *testing.T) {
	for _, test := range []struct {
		name   string
		probe  fakeExitProbe
		reason string
	}{
		{name: "identity drift", probe: fakeExitProbe{observation: hub.ExitObservation{PublicIP: "203.0.113.11", Region: "test"}}, reason: "exit_ip_drift"},
		{name: "authentication rejected", probe: fakeExitProbe{failure: "proxy_auth_failed"}, reason: "proxy_auth_failed"},
		{name: "proxy disconnected", probe: fakeExitProbe{failure: "proxy_check_failed"}, reason: "proxy_check_failed"},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := newMemoryStore()
			store.exits["exit-1"] = hub.NetworkExit{
				ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080,
				ExpectedPublicIP: "203.0.113.10", HealthStatus: "healthy", Version: 1,
			}
			_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
			gateway := &fakeGateway{token: "unit-test-gateway-token"}
			server := httptest.NewServer(gateway.handler(t))
			defer server.Close()
			store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
			app := fiber.New()
			registerHubWithNetwork(app, store, test.probe, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
			if response.Code != http.StatusConflict {
				t.Fatalf("expected fail-close 409, got %d: %s", response.Code, response.Body.String())
			}
			if len(gateway.recorded()) != 0 {
				t.Fatalf("exit failure must stop before any gateway start/create call: %#v", gateway.recorded())
			}
			bound, err := store.GetEnvironmentContext(context.Background(), "account-a")
			if err != nil || bound.RuntimeInstanceID != "" || store.exits["exit-1"].HealthStatus != "unhealthy" {
				t.Fatalf("failed create must retain only the stable inactive binding: %#v err=%v", bound, err)
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != test.reason {
				t.Fatalf("exit failure must be auditable without secrets: %#v", store.actions)
			}
		})
	}
}

func TestRepeatedCreateAndStartReuseStableEnvironment(t *testing.T) {
	store := newMemoryStore()
	_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	if response := do(app, http.MethodPost, "/api/browsers", createEnvBody); response.Code != http.StatusCreated {
		t.Fatalf("initial create failed: %d %s", response.Code, response.Body.String())
	}
	if response := do(app, http.MethodPost, "/api/browsers", createEnvBody); response.Code != http.StatusOK {
		t.Fatalf("idempotent create failed: %d %s", response.Code, response.Body.String())
	}
	if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
		t.Fatalf("idempotent start failed: %d %s", response.Code, response.Body.String())
	}
	bound := store.bindings["account-a"]
	if bound.Alias != "account-a" || bound.Exit.ID != "exit-1" || bound.RuntimeID != "container-id" {
		t.Fatalf("repeated actions changed stable environment identity: %#v", bound)
	}
	createCalls := 0
	for _, request := range gateway.recorded() {
		if request.method == http.MethodPost && request.path == "/v1/browsers" {
			createCalls++
			if request.body["volume"] != "creatorhub-profile-account-a" || request.body["network_exit"].(map[string]any)["host"] != "127.0.0.1" {
				t.Fatalf("create changed Profile volume or exit: %#v", request.body)
			}
		}
	}
	if createCalls != 1 {
		t.Fatalf("idempotent create called gateway create %d times", createCalls)
	}
}

func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) {
	store := newMemoryStore()
	_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
	if response.Code != http.StatusConflict {
		t.Fatalf("expected gateway conflict to pass through as 409, got %d: %s", response.Code, response.Body.String())
	}
	if _, kept := store.envs["account-a"]; !kept {
		t.Fatal("stable environment and Profile anchor must remain retryable")
	}
}

func TestCreateBrowserReconcilesDisconnectedGateway(t *testing.T) {
	for _, test := range []struct {
		name       string
		reconcile  bool
		wantStatus int
	}{
		{name: "completed create", reconcile: true, wantStatus: http.StatusCreated},
		{name: "unknown result", wantStatus: http.StatusBadGateway},
	} {
		t.Run(test.name, func(t *testing.T) {
			var created bool
			var mu sync.Mutex
			gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
				if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" {
					response.WriteHeader(http.StatusUnauthorized)
					return
				}
				if request.Method == http.MethodGet && test.reconcile {
					mu.Lock()
					exists := created
					mu.Unlock()
					if exists {
						_ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}})
						return
					}
					_ = json.NewEncoder(response).Encode([]containerStatus{})
					return
				}
				if request.Method == http.MethodPost {
					mu.Lock()
					created = true
					mu.Unlock()
				}
				hijacker, ok := response.(http.Hijacker)
				if !ok {
					t.Error("test server does not support hijacking")
					return
				}
				connection, _, err := hijacker.Hijack()
				if err != nil {
					t.Errorf("hijack gateway response: %v", err)
					return
				}
				_ = connection.Close()
			}))
			defer gatewayServer.Close()

			store := newMemoryStore()
			_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
			store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
			app := fiber.New()
			registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
			if response.Code != test.wantStatus {
				t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
			}
			if _, err := store.GetEnv(context.Background(), "account-a"); err != nil {
				t.Fatalf("unknown gateway result must retain the environment for reconciliation: %v", err)
			}
			wantOutcome := "unknown"
			if test.reconcile {
				wantOutcome = "succeeded"
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != wantOutcome {
				t.Fatalf("gateway reconciliation outcome must be audited as %s: %#v", wantOutcome, store.actions)
			}
		})
	}
}

func TestCreateBrowserReconcilesGatewayBadGateway(t *testing.T) {
	for _, test := range []struct {
		name         string
		missingReads int
	}{
		{name: "Docker create disconnect returned 502"},
		{name: "container becomes visible after query window", missingReads: 2},
	} {
		t.Run(test.name, func(t *testing.T) {
			var mu sync.Mutex
			listCalls := 0
			gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
				if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" {
					response.WriteHeader(http.StatusUnauthorized)
					return
				}
				if request.Method == http.MethodPost {
					response.WriteHeader(http.StatusBadGateway)
					_, _ = response.Write([]byte(`{"error":"create container: context deadline exceeded"}`))
					return
				}
				mu.Lock()
				listCalls++
				missing := listCalls <= test.missingReads
				mu.Unlock()
				if missing {
					_ = json.NewEncoder(response).Encode([]containerStatus{})
					return
				}
				_ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}})
			}))
			defer gatewayServer.Close()

			store := newMemoryStore()
			_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
			store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
			app := fiber.New()
			registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
			if response.Code != http.StatusCreated {
				t.Fatalf("expected reconciled create, status=%d body=%s", response.Code, response.Body.String())
			}
			if _, err := store.GetEnv(context.Background(), "account-a"); err != nil {
				t.Fatalf("reconciled create must retain DB state: err=%v", err)
			}
			mu.Lock()
			gotCalls := listCalls
			mu.Unlock()
			if gotCalls != test.missingReads+1 {
				t.Fatalf("expected %d reconciliation reads, got %d", test.missingReads+1, gotCalls)
			}
		})
	}
}

func TestCreateBrowserRejectsInvalidFingerprintBeforeSideEffects(t *testing.T) {
	store := newMemoryStore()
	_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers",
		`{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148.0.7778.215","fingerprint":{"seed":0}}`)
	if response.Code != http.StatusBadRequest {
		t.Fatalf("expected 400 for invalid fingerprint, got %d: %s", response.Code, response.Body.String())
	}
	if len(gateway.recorded()) != 0 || len(store.envs) != 0 {
		t.Fatal("invalid input must not reach the gateway or the store")
	}
}

func TestListBrowsersMergesLiveGatewayState(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148"}
	store.envs["account-b"] = hub.Env{Alias: "account-b", Name: "店铺二号", Gateway: "gw-1", ImageVersion: "148"}
	gateway := &fakeGateway{
		token: "unit-test-gateway-token",
		containers: []containerStatus{
			{ID: "id-1", Alias: "account-a", State: "running", Status: "Up", Endpoint: "http://creatorhub-browser-account-a:9222", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true},
		},
	}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", response.Code, response.Body.String())
	}
	var views []envView
	if err := json.NewDecoder(response.Body).Decode(&views); err != nil || len(views) != 2 {
		t.Fatalf("expected two env views, err=%v body=%s", err, response.Body.String())
	}
	byAlias := map[string]envView{}
	for _, view := range views {
		byAlias[view.Alias] = view
	}
	if byAlias["account-a"].State != "running" || byAlias["account-a"].ContainerID != "id-1" {
		t.Fatalf("running container state must be merged: %#v", byAlias["account-a"])
	}
	if byAlias["account-b"].State != "missing" {
		t.Fatalf("env without container must report missing: %#v", byAlias["account-b"])
	}
	if runtime := store.bindings["account-a"].RuntimeID; runtime != "id-1" {
		t.Fatalf("list reconciliation must heartbeat the running runtime, got %q", runtime)
	}
}

func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 3, Exit: store.exits["exit-1"],
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
		ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", ProxyReady: false,
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusOK {
		t.Fatalf("gateway restart recovery failed: %d %s", response.Code, response.Body.String())
	}
	if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" {
		t.Fatalf("runtime was activated before proxy recovery completed: %q", runtime)
	}
	requests := gateway.recorded()
	if len(requests) != 2 || requests[0].path != "/v1/browsers" || requests[1].path != "/v1/browsers/account-a/proxy" {
		t.Fatalf("expected list then proxy recovery without rebuild: %#v", requests)
	}
	if requests[1].body["binding_version"] != float64(3) || requests[1].body["network_exit_id"] != "exit-1" {
		t.Fatalf("proxy recovery did not use the current binding: %#v", requests[1].body)
	}
}

func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"]}
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, containers: []containerStatus{{
		ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1",
	}}}
	app := newTestApp(t, store, gateway)

	if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK {
		t.Fatalf("gateway restart rebuild failed: %d %s", response.Code, response.Body.String())
	}
	requests := gateway.recorded()
	if len(requests) != 4 || requests[1].path != "/v1/browsers/account-a/proxy" || requests[2].method != http.MethodDelete || requests[3].path != "/v1/browsers" {
		t.Fatalf("failed proxy recovery must preserve Profile by rebuilding the container: %#v", requests)
	}
	if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" {
		t.Fatalf("rebuilt runtime was not activated: %q", runtime)
	}
}

func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{
		Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148",
		Fingerprint: hub.Fingerprint{Seed: 2024, Timezone: "Asia/Shanghai"},
	}
	_ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"144.0.7559.132"}`)
	if response.Code != http.StatusNoContent {
		t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String())
	}
	requests := gateway.recorded()
	if len(requests) != 2 || requests[0].method != http.MethodDelete || requests[0].path != "/v1/browsers/account-a" ||
		requests[1].method != http.MethodPost || requests[1].path != "/v1/browsers" {
		t.Fatalf("upgrade must delete then recreate: %#v", requests)
	}
	payload := requests[1].body
	if payload["image"] != "registry.example/browser:144" || payload["volume"] != "creatorhub-profile-account-a" {
		t.Fatalf("upgrade must reuse the profile volume and switch image: %#v", payload)
	}
	if payload["binding_version"] != float64(2) || payload["network_exit_id"] != "exit-1" {
		t.Fatalf("upgrade must create from the committed binding generation: %#v", payload)
	}
	cmd := payload["cmd"].([]any)
	if cmd[0] != "--fingerprint=2024" || cmd[len(cmd)-1] != "about:blank" {
		t.Fatalf("upgrade must reuse stored fingerprint params: %#v", cmd)
	}
	if store.upgraded["account-a"] != "144.0.7559.132" || store.envs["account-a"].ImageVersion != "144.0.7559.132" {
		t.Fatal("image version must be persisted after successful upgrade")
	}
	if len(store.actions) != 2 || store.actions[0].OldImageVersion != "148" ||
		store.actions[1].NewImageVersion != "144.0.7559.132" || store.actions[1].Outcome != "succeeded" {
		t.Fatalf("upgrade must emit image-aware audit evidence: %#v", store.actions)
	}
}

func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()
	databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL)
	accountStore, err := phasea.Open(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	if err := accountStore.Close(); err != nil {
		t.Fatal(err)
	}
	store, err := hub.Open(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	accountStore, err = phasea.Open(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	if err := accountStore.CreateAccount(ctx, phasea.Account{
		ID: "account-a", Platform: "mock", PlatformAccountKey: "account-a", AuthorizationKind: "owned",
		CredentialReference: phasea.CredentialReference{ID: "credential-account", Provider: "os_keyring"},
		CredentialKey:       "creatorhub/account-a",
	}); err != nil {
		t.Fatal(err)
	}
	if err := accountStore.Close(); err != nil {
		t.Fatal(err)
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	gatewayServer := httptest.NewServer(gateway.handler(t))
	t.Cleanup(gatewayServer.Close)
	if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil {
		t.Fatal(err)
	}
	for _, image := range []hub.Image{
		{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true},
		{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true},
	} {
		if err := store.CreateImage(ctx, image); err != nil {
			t.Fatal(err)
		}
	}
	exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080}, "")
	if err != nil {
		t.Fatal(err)
	}
	exit, _, err = store.RecordNetworkExitCheck(ctx, exit.ID, hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "")
	if err != nil {
		t.Fatal(err)
	}
	before, created, err := store.CreateBoundEnv(ctx, hub.Env{
		Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1},
	}, "account-a", exit.ID)
	if err != nil || !created {
		t.Fatalf("create bound environment: created=%v err=%v", created, err)
	}
	app := fiber.New()
	registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

	response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"149"}`)
	if response.Code != http.StatusNoContent {
		t.Fatalf("PostgreSQL-backed upgrade failed: %d: %s", response.Code, response.Body.String())
	}
	after, err := store.GetEnvironmentContext(ctx, "account-a")
	if err != nil {
		t.Fatal(err)
	}
	if after.ImageVersion != "149" || after.BindingVersion != before.BindingVersion+1 || after.RuntimeID != "container-id" {
		t.Fatalf("upgrade did not activate the committed PostgreSQL binding: before=%#v after=%#v", before, after)
	}
	requests := gateway.recorded()
	if len(requests) != 2 || requests[1].body["binding_version"] != float64(after.BindingVersion) ||
		requests[1].body["network_exit_id"] != after.Exit.ID {
		t.Fatalf("gateway labels diverged from the committed binding: %#v", requests)
	}

	gateway.mu.Lock()
	gateway.failDelete = 1
	gateway.containers = []containerStatus{{
		ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: after.BindingVersion - 1,
		NetworkExitID: after.Exit.ID, ProxyReady: true,
	}}
	gateway.mu.Unlock()
	response = do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusBadGateway {
		t.Fatalf("PostgreSQL-backed reconcile delete failure returned %d: %s", response.Code, response.Body.String())
	}
	released, err := store.GetEnvironmentContext(ctx, "account-a")
	if err != nil || released.RuntimeID != "" || released.BindingVersion != after.BindingVersion || !released.RuntimeCleanupPending {
		t.Fatalf("delete failure must release the real Store lease without changing binding: %#v err=%v", released, err)
	}
	response = do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusOK {
		t.Fatalf("PostgreSQL-backed cleanup retry returned %d: %s", response.Code, response.Body.String())
	}
	released, err = store.GetEnvironmentContext(ctx, "account-a")
	if err != nil || released.RuntimeCleanupPending {
		t.Fatalf("confirmed cleanup remained pending: %#v err=%v", released, err)
	}
	if _, err := store.ActivateRuntime(ctx, "account-a", "coherent-container", released.BindingVersion, released.Exit.ID); err != nil {
		t.Fatal(err)
	}
	if err := store.ReleaseRuntime(ctx, "account-a"); err != nil {
		t.Fatal(err)
	}
	gateway.mu.Lock()
	gateway.failDelete = 1
	gateway.containers = []containerStatus{{
		ID: "coherent-container", Alias: "account-a", State: "running", BindingVersion: released.BindingVersion,
		NetworkExitID: released.Exit.ID, ProxyReady: true,
	}}
	gateway.mu.Unlock()
	response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+released.Exit.ID+`"}`)
	if response.Code != http.StatusBadGateway {
		t.Fatalf("PostgreSQL-backed rebind delete failure returned %d: %s", response.Code, response.Body.String())
	}
	unchanged, err := store.GetEnvironmentContext(ctx, "account-a")
	if err != nil || unchanged.BindingVersion != released.BindingVersion || unchanged.Exit.ID != released.Exit.ID || unchanged.RuntimeID != "" || !unchanged.RuntimeCleanupPending {
		t.Fatalf("rebind committed before delete confirmation: %#v err=%v", unchanged, err)
	}
}

type postgresRebindFixture struct {
	store   *hub.Store
	db      *sql.DB
	gateway *fakeGateway
	bound   hub.EnvironmentContext
	exit    hub.NetworkExit
}

type cleanupCommitUnknownStore struct {
	hubStore
}

func (s cleanupCommitUnknownStore) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error {
	if err := s.hubStore.SetRuntimeCleanupPending(ctx, alias, pending); err != nil {
		return err
	}
	return errors.New("cleanup commit result unknown")
}

func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFixture {
	t.Helper()
	ctx := context.Background()
	databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL)
	accountStore, err := phasea.Open(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	if err := accountStore.Close(); err != nil {
		t.Fatal(err)
	}
	store, err := hub.Open(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	accountStore, err = phasea.Open(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	if err := accountStore.CreateAccount(ctx, phasea.Account{
		ID: "account-a", Platform: "mock", PlatformAccountKey: "account-a", AuthorizationKind: "owned",
		CredentialReference: phasea.CredentialReference{ID: "credential-account", Provider: "os_keyring"},
		CredentialKey:       "creatorhub/account-a",
	}); err != nil {
		t.Fatal(err)
	}
	if err := accountStore.Close(); err != nil {
		t.Fatal(err)
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	gatewayServer := httptest.NewServer(gateway.handler(t))
	t.Cleanup(gatewayServer.Close)
	if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil {
		t.Fatal(err)
	}
	if err := store.CreateImage(ctx, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}); err != nil {
		t.Fatal(err)
	}
	exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080}, "")
	if err != nil {
		t.Fatal(err)
	}
	exit, _, err = store.RecordNetworkExitCheck(ctx, exit.ID, hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "")
	if err != nil {
		t.Fatal(err)
	}
	bound, created, err := store.CreateBoundEnv(ctx, hub.Env{
		Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1},
	}, "account-a", exit.ID)
	if err != nil || !created {
		t.Fatalf("create bound environment: created=%v err=%v", created, err)
	}
	db, err := sql.Open("pgx", databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = db.Close() })
	return postgresRebindFixture{store: store, db: db, gateway: gateway, bound: bound, exit: exit}
}

func installCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB, condition string) {
	t.Helper()
	if _, err := db.ExecContext(ctx, `
		CREATE FUNCTION fail_cleanup_transition() RETURNS trigger AS $$
		BEGIN
			IF `+condition+` THEN
				RAISE EXCEPTION 'injected cleanup state failure';
			END IF;
			RETURN NEW;
		END
		$$ LANGUAGE plpgsql`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.ExecContext(ctx, `
		CREATE TRIGGER fail_cleanup_transition BEFORE UPDATE OF runtime_cleanup_pending ON environment_binding
		FOR EACH ROW EXECUTE FUNCTION fail_cleanup_transition()`); err != nil {
		t.Fatal(err)
	}
}

func dropCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB) {
	t.Helper()
	if _, err := db.ExecContext(ctx, `DROP TRIGGER fail_cleanup_transition ON environment_binding`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.ExecContext(ctx, `DROP FUNCTION fail_cleanup_transition()`); err != nil {
		t.Fatal(err)
	}
}

func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()
	for _, test := range []struct {
		name        string
		race        string
		state       string
		nullBinding bool
	}{
		{name: "active runtime after DELETE", race: "active", state: "running"},
		{name: "executing task after DELETE", race: "executing", state: "running"},
		{name: "binding version after DELETE", race: "version", state: "running"},
		{name: "image upgrade blocked during DELETE", race: "upgrade", state: "running"},
		{name: "stopped container after DELETE", race: "version", state: "exited"},
		{name: "stopped NULL binding after DELETE", race: "version", state: "exited", nullBinding: true},
	} {
		t.Run(test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			if test.race == "upgrade" {
				if err := fixture.store.CreateImage(ctx, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}); err != nil {
					t.Fatal(err)
				}
			}
			if test.nullBinding {
				if _, err := fixture.db.ExecContext(ctx, `UPDATE environment_binding SET network_exit_id = NULL WHERE browser_env_alias = 'account-a'`); err != nil {
					t.Fatal(err)
				}
				var err error
				fixture.bound, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
				if err != nil {
					t.Fatal(err)
				}
			}
			deleteDone, releaseDelete := make(chan struct{}), make(chan struct{})
			fixture.gateway.deleteDone, fixture.gateway.releaseDelete = deleteDone, releaseDelete
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: test.state, ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
			}}
			app := fiber.New()
			registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			server := httptest.NewServer(adaptor.FiberApp(app))
			defer server.Close()
			type result struct {
				status int
				body   string
				err    error
			}
			done := make(chan result, 1)
			go func() {
				response, err := server.Client().Post(server.URL+"/api/browsers/account-a/rebind", "application/json",
					strings.NewReader(`{"network_exit_id":"`+fixture.exit.ID+`"}`))
				if err != nil {
					done <- result{err: err}
					return
				}
				defer response.Body.Close()
				body, _ := io.ReadAll(response.Body)
				done <- result{status: response.StatusCode, body: string(body)}
			}()
			select {
			case <-deleteDone:
			case <-time.After(time.Second):
				close(releaseDelete)
				t.Fatal("rebind did not reach the DELETE barrier")
			}
			tx, err := fixture.db.BeginTx(ctx, nil)
			upgradeBlocked := false
			if err == nil {
				switch test.race {
				case "active":
					_, err = tx.ExecContext(ctx, `
						INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until)
						VALUES ('runtime-race', 'account-a', 'account-a', 'old-container', now() + interval '1 minute')`)
				case "executing":
					_, err = tx.ExecContext(ctx, `
						INSERT INTO content_draft (id, account_id, version, content) VALUES ('draft-rebind', 'account-a', 1, 'test');
						INSERT INTO operation_task (id, idempotency_key, account_id, account_version, draft_id, draft_version, state)
						SELECT 'task-rebind', 'task-rebind', id, version, 'draft-rebind', 1, 'executing'
						FROM social_account WHERE id = 'account-a'`)
				case "version":
					_, err = tx.ExecContext(ctx, `UPDATE environment_binding SET version = version + 1 WHERE browser_env_alias = 'account-a'`)
				case "upgrade":
					err = fixture.store.UpgradeEnv(ctx, "account-a", "149")
					if errors.Is(err, hub.ErrConflict) {
						upgradeBlocked, err = true, nil
					}
				}
			}
			if err == nil {
				err = tx.Commit()
			} else if tx != nil {
				_ = tx.Rollback()
			}
			close(releaseDelete)
			response := <-done
			if err != nil {
				t.Fatal(err)
			}
			expectedStatus := http.StatusConflict
			if test.race == "upgrade" {
				expectedStatus = http.StatusOK
			}
			if response.err != nil || response.status != expectedStatus {
				t.Fatalf("rebind race returned status=%d body=%s err=%v", response.status, response.body, response.err)
			}
			if test.race == "upgrade" && !upgradeBlocked {
				t.Fatal("upgrade advanced while runtime cleanup was pending")
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			expectedVersion := fixture.bound.BindingVersion
			if test.race == "version" || test.race == "upgrade" {
				expectedVersion++
			}
			expectedImage := fixture.bound.ImageVersion
			if err != nil || after.BindingVersion != expectedVersion || after.Exit.ID != fixture.bound.Exit.ID || after.ImageVersion != expectedImage {
				t.Fatalf("failed rebind changed PostgreSQL binding: before=%#v after=%#v err=%v", fixture.bound, after, err)
			}
			fixture.gateway.mu.Lock()
			containers := append([]containerStatus{}, fixture.gateway.containers...)
			fixture.gateway.mu.Unlock()
			if test.nullBinding {
				if after.RuntimeID != "" || len(containers) != 1 || containers[0].State == "running" ||
					!containerMatchesBinding(containers[0], after) {
					t.Fatalf("NULL binding recovery must restore a network-disabled stopped container without a lease: after=%#v containers=%#v", after, containers)
				}
			} else if test.state == "exited" {
				if after.RuntimeID != "" || len(containers) != 1 || containers[0].State != "exited" || !containerMatchesBinding(containers[0], after) {
					t.Fatalf("stopped runtime was not restored: after=%#v containers=%#v", after, containers)
				}
			} else if after.RuntimeID == "" || len(containers) != 1 || containers[0].State != "running" || !containerMatchesBinding(containers[0], after) {
				t.Fatalf("running runtime was not restored coherently: after=%#v containers=%#v", after, containers)
			}
			if test.race == "upgrade" {
				requests := fixture.gateway.recorded()
				var lastCreate map[string]any
				for _, request := range requests {
					if request.method == http.MethodPost && request.path == "/v1/browsers" {
						lastCreate = request.body
					}
				}
				if lastCreate["image"] != "registry.example/browser:148" || int64(lastCreate["binding_version"].(float64)) != after.BindingVersion {
					t.Fatalf("blocked upgrade changed the rebind generation: %#v", lastCreate)
				}
			}
			var outcome, reason string
			if err := fixture.db.QueryRowContext(ctx, `
				SELECT outcome, reason_code FROM audit_event
				WHERE event_type = 'environment_action_finished' AND action = 'rebind'
				ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
				t.Fatal(err)
			}
			expectedOutcome, expectedReason := "failed", "rebind_not_allowed"
			if test.race == "upgrade" {
				expectedOutcome, expectedReason = "succeeded", "environment_rebound"
			}
			if outcome != expectedOutcome || reason != expectedReason {
				t.Fatalf("recovered conflict audit mismatch: outcome=%s reason=%s", outcome, reason)
			}
		})
	}
}

func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()
	for _, test := range []struct {
		name                      string
		cleanupPending            int
		disconnectDelete          int
		disconnectListAfterDelete bool
	}{
		{name: "accepted cleanup pending", cleanupPending: 2},
		{name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true},
	} {
		t.Run(test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			var err error
			fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
			if err != nil {
				t.Fatal(err)
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
			}}
			fixture.gateway.cleanupPending = test.cleanupPending
			fixture.gateway.disconnectDelete = test.disconnectDelete
			fixture.gateway.disconnectListAfterDelete = test.disconnectListAfterDelete
			app := fiber.New()
			registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			target, err := fixture.store.GetGateway(ctx, "gw-1")
			if err != nil {
				t.Fatal(err)
			}
			removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound)
			if !removed || cleanupErr == nil {
				t.Fatalf("non-final cleanup lost the removed fact: removed=%v err=%v", removed, cleanupErr)
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || !after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion ||
				after.Exit.ID != fixture.bound.Exit.ID || after.RuntimeID != "" {
				t.Fatalf("PostgreSQL did not persist cleanup pending atomically: %#v err=%v", after, err)
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM environment_binding WHERE browser_env_alias = 'account-a' AND runtime_cleanup_pending`, 1)
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)

			response := do(app, http.MethodGet, "/api/browsers", "")
			if response.Code != http.StatusOK {
				t.Fatalf("list reconcile did not confirm cleanup: %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeCleanupPending {
				t.Fatalf("confirmed cleanup remained pending: %#v err=%v", after, err)
			}

			response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+fixture.exit.ID+`"}`)
			if response.Code != http.StatusOK {
				t.Fatalf("rebind retry failed after cleanup confirmation: %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion+1 || after.RuntimeID != "" {
				t.Fatalf("retry did not commit a consistent stopped generation: %#v err=%v", after, err)
			}
		})
	}
}

func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) {
	newRuntime := func() (*memoryStore, *fakeGateway, hub.EnvironmentContext, hub.Gateway) {
		store := newMemoryStore()
		store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", ImageVersion: "148"}
		environment := hub.EnvironmentContext{
			Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
			Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
		}
		store.bindings[environment.Alias] = environment
		gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
			ID: "old-container", Alias: environment.Alias, State: "running", ProxyReady: true,
			BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID,
		}}}
		server := httptest.NewServer(gateway.handler(t))
		t.Cleanup(server.Close)
		return store, gateway, environment, hub.Gateway{Endpoint: server.URL, Token: gateway.token}
	}

	t.Run("rollback keeps the lease and skips DELETE", func(t *testing.T) {
		store, gateway, environment, target := newRuntime()
		store.cleanupPendingErr = errors.New("injected rollback")

		removed, err := removeGatewayRuntime(context.Background(), store, target, environment)
		if err == nil || removed || len(gateway.recorded()) != 0 {
			t.Fatalf("failed pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded())
		}
		after := store.bindings[environment.Alias]
		if after.RuntimeCleanupPending || after.RuntimeID != environment.RuntimeID || len(gateway.containers) != 1 {
			t.Fatalf("rollback did not preserve the active generation: after=%#v containers=%#v", after, gateway.containers)
		}
	})

	t.Run("commit unknown remains retryable", func(t *testing.T) {
		store, gateway, environment, target := newRuntime()
		store.cleanupPendingErr = errors.New("commit result unknown")
		store.cleanupPendingErrAfterMutation = true

		removed, err := removeGatewayRuntime(context.Background(), store, target, environment)
		if err == nil || removed || len(gateway.recorded()) != 0 {
			t.Fatalf("unknown pre-mark result touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded())
		}
		after := store.bindings[environment.Alias]
		if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 1 {
			t.Fatalf("committed pre-mark was not retryable: after=%#v containers=%#v", after, gateway.containers)
		}

		store.cleanupPendingErr = nil
		store.cleanupPendingErrAfterMutation = false
		removed, err = removeGatewayRuntime(context.Background(), store, target, after)
		if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending || len(gateway.containers) != 0 {
			t.Fatalf("retry did not reconcile the committed pre-mark: removed=%v err=%v after=%#v containers=%#v",
				removed, err, store.bindings[environment.Alias], gateway.containers)
		}
	})

	for _, status := range []int{http.StatusNoContent, http.StatusNotFound} {
		t.Run(fmt.Sprintf("clear rollback after %d", status), func(t *testing.T) {
			store, gateway, environment, target := newRuntime()
			if err := store.SetRuntimeCleanupPending(context.Background(), environment.Alias, true); err != nil {
				t.Fatal(err)
			}
			environment = store.bindings[environment.Alias]
			store.cleanupPendingErr = errors.New("injected clear rollback")
			if status == http.StatusNotFound {
				gateway.deleteNotFound = 1
			}

			removed, err := removeGatewayRuntime(context.Background(), store, target, environment)
			if err == nil || !removed {
				t.Fatalf("clear rollback lost the delete fact: removed=%v err=%v", removed, err)
			}
			after := store.bindings[environment.Alias]
			if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 0 {
				t.Fatalf("clear rollback was not retryable: after=%#v containers=%#v", after, gateway.containers)
			}

			store.cleanupPendingErr = nil
			removed, err = removeGatewayRuntime(context.Background(), store, target, after)
			if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending {
				t.Fatalf("clear retry failed: removed=%v err=%v after=%#v", removed, err, store.bindings[environment.Alias])
			}
		})
	}
}

func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) {
	for _, test := range []struct {
		name        string
		commitKnown bool
		outcome     string
		reason      string
	}{
		{name: "rollback", commitKnown: true, outcome: "failed", reason: "runtime_persistence_failed"},
		{name: "commit unknown", outcome: "unknown", reason: "cleanup_result_unknown"},
	} {
		t.Run("restore "+test.name, func(t *testing.T) {
			store := newMemoryStore()
			_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			store.bindings["account-a"] = hub.EnvironmentContext{
				Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2,
				Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
			}
			store.cleanupPendingErr = errors.New("cleanup state unavailable")
			store.cleanupPendingErrAfterMutation = !test.commitKnown
			gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1",
			}}}
			app := newTestApp(t, store, gateway)

			response := do(app, http.MethodGet, "/api/browsers", "")
			if response.Code != http.StatusInternalServerError {
				t.Fatalf("failed cleanup mark returned %d: %s", response.Code, response.Body.String())
			}
			after := store.bindings["account-a"]
			if test.commitKnown {
				if after.RuntimeCleanupPending || after.RuntimeID != "old-container" {
					t.Fatalf("rollback changed the old generation: %#v", after)
				}
			} else if !after.RuntimeCleanupPending || after.RuntimeID != "" {
				t.Fatalf("commit-unknown lost its retry marker: %#v", after)
			}
			if len(gateway.containers) != 1 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet {
				t.Fatalf("failed cleanup mark touched the old container: containers=%#v requests=%#v", gateway.containers, gateway.recorded())
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != test.outcome || store.actions[1].ReasonCode != test.reason {
				t.Fatalf("restore audit mismatch: %#v", store.actions)
			}

			store.cleanupPendingErr = nil
			store.cleanupPendingErrAfterMutation = false
			if response = do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK {
				t.Fatalf("list retry failed: %d: %s", response.Code, response.Body.String())
			}
			if test.commitKnown {
				after = store.bindings["account-a"]
			} else {
				if response = do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
					t.Fatalf("start retry failed: %d: %s", response.Code, response.Body.String())
				}
				after = store.bindings["account-a"]
			}
			if after.RuntimeCleanupPending || after.RuntimeID == "" || len(gateway.containers) != 1 || !containerMatchesBinding(gateway.containers[0], after) {
				t.Fatalf("retry ended inconsistently: after=%#v containers=%#v", after, gateway.containers)
			}
		})

		t.Run("discard "+test.name, func(t *testing.T) {
			store := newMemoryStore()
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			store.bindings["account-a"] = hub.EnvironmentContext{
				Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
				Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
			}
			store.cleanupPendingErr = errors.New("cleanup state unavailable")
			store.cleanupPendingErrAfterMutation = !test.commitKnown
			gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1",
			}}}
			app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "proxy_auth_failed"},
				func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodPost, "/api/browsers/account-a/start", "")
			if response.Code != http.StatusInternalServerError {
				t.Fatalf("failed discard mark returned %d: %s", response.Code, response.Body.String())
			}
			after := store.bindings["account-a"]
			if test.commitKnown {
				if after.RuntimeCleanupPending || after.RuntimeID != "old-container" {
					t.Fatalf("discard rollback changed the old generation: %#v", after)
				}
			} else if !after.RuntimeCleanupPending || after.RuntimeID != "" {
				t.Fatalf("discard commit-unknown lost its retry marker: %#v", after)
			}
			if len(gateway.containers) != 1 || len(gateway.recorded()) != 0 {
				t.Fatalf("failed discard mark touched the gateway: containers=%#v requests=%#v", gateway.containers, gateway.recorded())
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" {
				t.Fatalf("discard audit mismatch: %#v", store.actions)
			}

			store.cleanupPendingErr = nil
			store.cleanupPendingErrAfterMutation = false
			response = do(app, http.MethodPost, "/api/browsers/account-a/start", "")
			if response.Code != http.StatusConflict {
				t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String())
			}
			after = store.bindings["account-a"]
			if after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 0 {
				t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v", after, gateway.containers)
			}
			if len(store.actions) != 4 || store.actions[3].Outcome != "failed" || store.actions[3].ReasonCode != "proxy_auth_failed" {
				t.Fatalf("discard retry audit mismatch: %#v", store.actions)
			}
		})
	}
}

func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) {
	store := newMemoryStore()
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, containers: []containerStatus{{
		ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1",
	}}}
	app := newTestApp(t, store, gateway)
	body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`

	response := do(app, http.MethodPost, "/api/browsers", body)
	if response.Code != http.StatusBadGateway {
		t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String())
	}
	if after := store.bindings["account-a"]; after.RuntimeID != "old-container" || after.RuntimeCleanupPending {
		t.Fatalf("gateway unknown changed the existing lease: %#v", after)
	}
	for _, request := range gateway.recorded() {
		if request.method == http.MethodPost {
			t.Fatalf("gateway unknown created a duplicate container: %#v", gateway.recorded())
		}
	}
	if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
		t.Fatalf("gateway unknown create audit mismatch: %#v", store.actions)
	}

	response = do(app, http.MethodPost, "/api/browsers", body)
	if response.Code != http.StatusOK {
		t.Fatalf("create retry did not reuse the runtime: %d: %s", response.Code, response.Body.String())
	}
	if after := store.bindings["account-a"]; after.RuntimeID != "old-container" || after.RuntimeCleanupPending {
		t.Fatalf("create retry changed the matching generation: %#v", after)
	}
	if len(gateway.containers) != 1 || len(store.actions) != 4 || store.actions[3].Outcome != "succeeded" || store.actions[3].ReasonCode != "environment_reused" {
		t.Fatalf("create retry ended inconsistently: containers=%#v actions=%#v", gateway.containers, store.actions)
	}
}

func TestGatewayLookupFailurePreservesLease(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	environment := hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
	}
	store.bindings[environment.Alias] = environment
	store.gatewayFn = func(string) (hub.Gateway, error) { return hub.Gateway{}, errors.New("gateway lookup unavailable") }

	if _, err := restoreOrRebuildRuntime(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil },
		environment, containerStatus{ID: "old-container", Alias: environment.Alias, State: "running"}); err == nil {
		t.Fatal("restore accepted an unknown gateway")
	}
	if err := discardRuntime(context.Background(), store, environment); err == nil {
		t.Fatal("discard accepted an unknown gateway")
	}
	if after := store.bindings[environment.Alias]; after.RuntimeCleanupPending || after.RuntimeID != "old-container" {
		t.Fatalf("gateway lookup failure changed the old generation: %#v", after)
	}
}

func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()

	t.Run("pre-mark rollback skips DELETE", func(t *testing.T) {
		fixture := newPostgresRebindFixture(t, databaseURL)
		var err error
		fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
		if err != nil {
			t.Fatal(err)
		}
		fixture.gateway.containers = []containerStatus{{
			ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
			BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
		}}
		installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
		target, err := fixture.store.GetGateway(ctx, "gw-1")
		if err != nil {
			t.Fatal(err)
		}

		removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound)
		if cleanupErr == nil || removed || len(fixture.gateway.recorded()) != 0 {
			t.Fatalf("rolled-back pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, cleanupErr, fixture.gateway.recorded())
		}
		after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
		if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 {
			t.Fatalf("PostgreSQL rollback did not preserve the active generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
		}
		assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)

		dropCleanupTransitionFailure(t, ctx, fixture.db)
		fixture.gateway.cleanupPending = 2
		removed, cleanupErr = removeGatewayRuntime(ctx, fixture.store, target, after)
		if cleanupErr == nil || !removed {
			t.Fatalf("202 retry did not retain pending cleanup: removed=%v err=%v", removed, cleanupErr)
		}
		after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
		if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 {
			t.Fatalf("202 retry was not durable: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
		}
		assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
	})

	t.Run("clear rollback remains pending and lifecycle retry recovers", func(t *testing.T) {
		fixture := newPostgresRebindFixture(t, databaseURL)
		var err error
		fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
		if err != nil {
			t.Fatal(err)
		}
		fixture.gateway.containers = []containerStatus{{
			ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
			BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
		}}
		fixture.gateway.deleteNotFound = 1
		installCleanupTransitionFailure(t, ctx, fixture.db, `OLD.runtime_cleanup_pending AND NOT NEW.runtime_cleanup_pending`)
		target, err := fixture.store.GetGateway(ctx, "gw-1")
		if err != nil {
			t.Fatal(err)
		}

		removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound)
		if cleanupErr == nil || !removed {
			t.Fatalf("404 clear rollback lost the delete fact: removed=%v err=%v", removed, cleanupErr)
		}
		after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
		if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 {
			t.Fatalf("clear rollback was not durably pending: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
		}
		assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)

		dropCleanupTransitionFailure(t, ctx, fixture.db)
		app := fiber.New()
		registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
		if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK {
			t.Fatalf("list retry did not confirm cleanup: %d: %s", response.Code, response.Body.String())
		}
		if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
			t.Fatalf("lifecycle retry did not rebuild runtime: %d: %s", response.Code, response.Body.String())
		}
		after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
		fixture.gateway.mu.Lock()
		containers := append([]containerStatus{}, fixture.gateway.containers...)
		fixture.gateway.mu.Unlock()
		if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) {
			t.Fatalf("lifecycle retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err)
		}
	})
}

func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()
	for _, test := range []struct {
		name          string
		commitUnknown bool
		outcome       string
		reason        string
	}{
		{name: "rollback", outcome: "failed", reason: "runtime_persistence_failed"},
		{name: "commit unknown", commitUnknown: true, outcome: "unknown", reason: "cleanup_result_unknown"},
	} {
		t.Run("restore "+test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			var err error
			fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
			if err != nil {
				t.Fatal(err)
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion - 1, NetworkExitID: fixture.bound.Exit.ID,
			}}
			var store hubStore = fixture.store
			if test.commitUnknown {
				store = cleanupCommitUnknownStore{hubStore: fixture.store}
			} else {
				installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
			}
			app := fiber.New()
			registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodGet, "/api/browsers", "")
			if response.Code != http.StatusInternalServerError {
				t.Fatalf("restore cleanup failure returned %d: %s", response.Code, response.Body.String())
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if test.commitUnknown {
				if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" {
					t.Fatalf("restore commit-unknown lost pending state: after=%#v err=%v", after, err)
				}
				assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
			} else {
				if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" {
					t.Fatalf("restore rollback changed the old generation: after=%#v err=%v", after, err)
				}
				assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
			}
			if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet {
				t.Fatalf("restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded())
			}
			var outcome, reason string
			if err := fixture.db.QueryRowContext(ctx, `
				SELECT outcome, reason_code FROM audit_event WHERE action = 'reconcile'
				ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
				t.Fatal(err)
			}
			if outcome != test.outcome || reason != test.reason {
				t.Fatalf("restore audit mismatch: outcome=%s reason=%s", outcome, reason)
			}

			if !test.commitUnknown {
				dropCleanupTransitionFailure(t, ctx, fixture.db)
			}
			retryApp := fiber.New()
			registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			if response = do(retryApp, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK {
				t.Fatalf("restore list retry failed: %d: %s", response.Code, response.Body.String())
			}
			if test.commitUnknown {
				if response = do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
					t.Fatalf("restore start retry failed: %d: %s", response.Code, response.Body.String())
				}
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			fixture.gateway.mu.Lock()
			containers := append([]containerStatus{}, fixture.gateway.containers...)
			fixture.gateway.mu.Unlock()
			if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) {
				t.Fatalf("restore retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err)
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
		})

		t.Run("create restore "+test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			var err error
			fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
			if err != nil {
				t.Fatal(err)
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion - 1, NetworkExitID: fixture.bound.Exit.ID,
			}}
			var store hubStore = fixture.store
			if test.commitUnknown {
				store = cleanupCommitUnknownStore{hubStore: fixture.store}
			} else {
				installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
			}
			app := fiber.New()
			registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}`

			response := do(app, http.MethodPost, "/api/browsers", body)
			if response.Code != http.StatusInternalServerError {
				t.Fatalf("create restore cleanup failure returned %d: %s", response.Code, response.Body.String())
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if test.commitUnknown {
				if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" {
					t.Fatalf("create restore commit-unknown lost pending state: after=%#v err=%v", after, err)
				}
				assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
			} else {
				if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" {
					t.Fatalf("create restore rollback changed the old generation: after=%#v err=%v", after, err)
				}
				assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
			}
			if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet {
				t.Fatalf("create restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded())
			}
			var outcome, reason string
			if err := fixture.db.QueryRowContext(ctx, `
				SELECT outcome, reason_code FROM audit_event WHERE action = 'create'
				ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
				t.Fatal(err)
			}
			if outcome != test.outcome || reason != test.reason {
				t.Fatalf("create restore audit mismatch: outcome=%s reason=%s", outcome, reason)
			}

			if !test.commitUnknown {
				dropCleanupTransitionFailure(t, ctx, fixture.db)
			}
			retryApp := fiber.New()
			registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			response = do(retryApp, http.MethodPost, "/api/browsers", body)
			if response.Code != http.StatusOK && response.Code != http.StatusCreated {
				t.Fatalf("create restore retry failed: %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			fixture.gateway.mu.Lock()
			containers := append([]containerStatus{}, fixture.gateway.containers...)
			fixture.gateway.mu.Unlock()
			if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) {
				t.Fatalf("create restore retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err)
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
		})

		t.Run("discard "+test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			var err error
			fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
			if err != nil {
				t.Fatal(err)
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
			}}
			var store hubStore = fixture.store
			if test.commitUnknown {
				store = cleanupCommitUnknownStore{hubStore: fixture.store}
			} else {
				installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
			}
			app := fiber.New()
			registerHubWithNetwork(app, store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodPost, "/api/browsers/account-a/start", "")
			if response.Code != http.StatusInternalServerError {
				t.Fatalf("discard cleanup failure returned %d: %s", response.Code, response.Body.String())
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if test.commitUnknown {
				if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" {
					t.Fatalf("discard commit-unknown lost pending state: after=%#v err=%v", after, err)
				}
				assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
			} else {
				if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" {
					t.Fatalf("discard rollback changed the old generation: after=%#v err=%v", after, err)
				}
				assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
			}
			if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 0 {
				t.Fatalf("discard cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded())
			}
			var outcome, reason string
			if err := fixture.db.QueryRowContext(ctx, `
				SELECT outcome, reason_code FROM audit_event WHERE action = 'start'
				ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
				t.Fatal(err)
			}
			if outcome != "unknown" || reason != "cleanup_result_unknown" {
				t.Fatalf("discard audit mismatch: outcome=%s reason=%s", outcome, reason)
			}

			if !test.commitUnknown {
				dropCleanupTransitionFailure(t, ctx, fixture.db)
			}
			retryApp := fiber.New()
			registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			if response = do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusConflict {
				t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 {
				t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
		})
	}
}

func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()
	for _, test := range []struct {
		name string
		set  func(*fakeGateway, int)
	}{
		{name: "disconnect", set: func(gateway *fakeGateway, count int) { gateway.disconnectList = count }},
		{name: "status 500", set: func(gateway *fakeGateway, count int) { gateway.failList = count }},
		{name: "invalid JSON", set: func(gateway *fakeGateway, count int) { gateway.invalidList = count }},
	} {
		t.Run("list "+test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			var err error
			fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
			if err != nil {
				t.Fatal(err)
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
			}}
			test.set(fixture.gateway, 1)
			app := fiber.New()
			registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusBadGateway {
				t.Fatalf("gateway unknown list returned %d: %s", response.Code, response.Body.String())
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 {
				t.Fatalf("gateway unknown list changed the generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
			if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK {
				t.Fatalf("gateway list retry failed: %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 {
				t.Fatalf("gateway list retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
			}
		})

		t.Run("create "+test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			var err error
			fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
			if err != nil {
				t.Fatal(err)
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
			}}
			test.set(fixture.gateway, gatewayReconcileAttempts)
			app := fiber.New()
			registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}`

			if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusBadGateway {
				t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String())
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 {
				t.Fatalf("gateway unknown create changed the generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
			}
			for _, request := range fixture.gateway.recorded() {
				if request.method == http.MethodPost {
					t.Fatalf("gateway unknown create sent a duplicate create: %#v", fixture.gateway.recorded())
				}
			}
			var outcome, reason string
			if err := fixture.db.QueryRowContext(ctx, `
				SELECT outcome, reason_code FROM audit_event WHERE action = 'create'
				ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
				t.Fatal(err)
			}
			if outcome != "unknown" || reason != "gateway_result_unknown" {
				t.Fatalf("gateway unknown create audit mismatch: outcome=%s reason=%s", outcome, reason)
			}

			if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusOK {
				t.Fatalf("gateway create retry failed: %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 {
				t.Fatalf("gateway create retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err)
			}
			if err := fixture.db.QueryRowContext(ctx, `
				SELECT outcome, reason_code FROM audit_event WHERE action = 'create'
				ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
				t.Fatal(err)
			}
			if outcome != "succeeded" || reason != "environment_reused" {
				t.Fatalf("gateway create retry audit mismatch: outcome=%s reason=%s", outcome, reason)
			}
		})
	}
}

func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) {
	databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
	if databaseURL == "" {
		t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
	}
	ctx := context.Background()
	for _, test := range []struct {
		name, method, path, action, successReason string
		configure                                 func(*fakeGateway, int)
		active                                    bool
		wantStatus                                int
	}{
		{name: "list rejects top-level null", method: http.MethodGet, path: "/api/browsers", active: true, wantStatus: http.StatusOK,
			configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `null` }},
		{name: "create rejects null element", method: http.MethodPost, path: "/api/browsers", action: "create", active: true,
			wantStatus: http.StatusOK, successReason: "environment_reused",
			configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `[null]` }},
		{name: "start rejects missing fields", method: http.MethodPost, path: "/api/browsers/account-a/start", action: "start", active: true,
			wantStatus: http.StatusNoContent, successReason: "gateway_reconciled",
			configure: func(gateway *fakeGateway, count int) {
				gateway.invalidList, gateway.invalidListBody = count, `[{"alias":"account-a","state":"running"}]`
			}},
		{name: "rebind rejects partial body", method: http.MethodPost, path: "/api/browsers/account-a/rebind", action: "rebind",
			wantStatus: http.StatusOK, successReason: "environment_rebound",
			configure: func(gateway *fakeGateway, count int) { gateway.readErrorList = count }},
	} {
		t.Run(test.name, func(t *testing.T) {
			fixture := newPostgresRebindFixture(t, databaseURL)
			if test.active {
				var err error
				fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID)
				if err != nil {
					t.Fatal(err)
				}
			}
			fixture.gateway.containers = []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true,
				BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
			}}
			attempts := gatewayReconcileAttempts
			if test.action == "" {
				attempts = 1
			}
			test.configure(fixture.gateway, attempts)
			app := fiber.New()
			registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
			body := ""
			if test.action == "create" {
				body = `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}`
			} else if test.action == "rebind" {
				body = `{"network_exit_id":"` + fixture.exit.ID + `"}`
			}

			response := do(app, test.method, test.path, body)
			if response.Code != http.StatusBadGateway {
				t.Fatalf("strict list failure returned %d: %s", response.Code, response.Body.String())
			}
			after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
			if err != nil || after.BindingVersion != fixture.bound.BindingVersion || after.Exit.ID != fixture.bound.Exit.ID ||
				after.RuntimeCleanupPending || after.RuntimeID != fixture.bound.RuntimeID || len(fixture.gateway.containers) != 1 {
				t.Fatalf("strict list failure changed the generation: before=%#v after=%#v containers=%#v err=%v",
					fixture.bound, after, fixture.gateway.containers, err)
			}
			for _, request := range fixture.gateway.recorded() {
				if request.method != http.MethodGet {
					t.Fatalf("strict list failure advanced the lifecycle: %#v", fixture.gateway.recorded())
				}
			}
			activeCount := 0
			if test.active {
				activeCount = 1
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeCount)
			if test.action != "" {
				var outcome, reason string
				if err := fixture.db.QueryRowContext(ctx, `
					SELECT outcome, reason_code FROM audit_event WHERE action = $1
					ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil {
					t.Fatal(err)
				}
				if outcome != "unknown" || reason != "gateway_result_unknown" {
					t.Fatalf("strict list audit mismatch: outcome=%s reason=%s", outcome, reason)
				}
			}

			response = do(app, test.method, test.path, body)
			if response.Code != test.wantStatus {
				t.Fatalf("strict list retry returned %d: %s", response.Code, response.Body.String())
			}
			after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
			fixture.gateway.mu.Lock()
			containers := append([]containerStatus{}, fixture.gateway.containers...)
			fixture.gateway.mu.Unlock()
			wantVersion := fixture.bound.BindingVersion
			if test.action == "rebind" {
				wantVersion++
			}
			if err != nil || after.BindingVersion != wantVersion || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) {
				t.Fatalf("strict list retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err)
			}
			assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
			if test.action != "" {
				var outcome, reason string
				if err := fixture.db.QueryRowContext(ctx, `
					SELECT outcome, reason_code FROM audit_event WHERE action = $1
					ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil {
					t.Fatal(err)
				}
				if outcome != "succeeded" || reason != test.successReason {
					t.Fatalf("strict list retry audit mismatch: outcome=%s reason=%s", outcome, reason)
				}
			}
		})
	}
}

func isolatedControlPlaneDatabaseURL(t *testing.T, databaseURL string) string {
	t.Helper()
	admin, err := sql.Open("pgx", databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = admin.Close() })
	schema := fmt.Sprintf("creatorhub_hh803_%d", time.Now().UnixNano())
	if _, err := admin.Exec("CREATE SCHEMA " + schema); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		if _, err := admin.Exec("DROP SCHEMA " + schema + " CASCADE"); err != nil {
			t.Errorf("drop test schema: %v", err)
		}
	})
	parsed, err := url.Parse(databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	query := parsed.Query()
	query.Set("search_path", schema)
	parsed.RawQuery = query.Encode()
	return parsed.String()
}

func assertControlPlaneDatabaseCount(t *testing.T, db *sql.DB, query string, want int) {
	t.Helper()
	var got int
	if err := db.QueryRow(query).Scan(&got); err != nil {
		t.Fatal(err)
	}
	if got != want {
		t.Fatalf("query %q returned %d, want %d", query, got, want)
	}
}

func TestUpgradeBrowserStopsBeforeCreateWhenPersistenceFails(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}}
	store.upgradeErr = hub.ErrNotFound
	_ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"144.0.7559.132"}`)
	if response.Code != http.StatusNotFound {
		t.Fatalf("expected persistence failure, got %d: %s", response.Code, response.Body.String())
	}
	requests := gateway.recorded()
	if len(requests) != 1 || requests[0].method != http.MethodDelete || requests[0].path != "/v1/browsers/account-a" {
		t.Fatalf("failed persistence must abort before creating the upgraded container: %#v", requests)
	}
	env, err := store.GetEnv(context.Background(), "account-a")
	if err != nil || env.ImageVersion != "148" {
		t.Fatalf("failed upgrade must preserve the stored version: %#v %v", env, err)
	}
}

func TestUpgradeStopsBeforeCreateWhenRuntimeReleaseFails(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}}
	store.releaseErr = errors.New("database unavailable")
	_ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"144.0.7559.132"}`)
	if response.Code != http.StatusInternalServerError {
		t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String())
	}
	requests := gateway.recorded()
	if len(requests) != 0 {
		t.Fatalf("release failure must abort before delete: %#v", requests)
	}
	if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" {
		t.Fatalf("release failure must be audited: %#v", store.actions)
	}
}

func TestUpgradeRejectsInvalidVersionWithoutAuditingRawInput(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}}
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"http://operator:secret@proxy.example"}`)
	if response.Code != http.StatusBadRequest {
		t.Fatalf("expected invalid version rejection, got %d: %s", response.Code, response.Body.String())
	}
	if len(gateway.recorded()) != 0 || len(store.actions) != 2 {
		t.Fatalf("invalid version must be rejected before gateway side effects: requests=%#v actions=%#v", gateway.recorded(), store.actions)
	}
	for _, action := range store.actions {
		if action.NewImageVersion != "" || action.ReasonCode != "upgrade_input_rejected" {
			t.Fatalf("raw invalid version reached audit: %#v", store.actions)
		}
	}
}

func TestImageDisableWaitsForUpgradeCommit(t *testing.T) {
	releaseCreate := make(chan struct{})
	gateway := &fakeGateway{
		token:         "unit-test-gateway-token",
		createStarted: make(chan struct{}),
		releaseCreate: releaseCreate,
	}
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}}
	_ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true})
	gatewayServer := httptest.NewServer(gateway.handler(t))
	defer gatewayServer.Close()
	store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token}
	disableArrived := make(chan struct{})
	var disableOnce sync.Once
	app := fiber.New()
	app.Use(func(c fiber.Ctx) error {
		if c.Method() == http.MethodPut {
			disableOnce.Do(func() { close(disableArrived) })
		}
		return c.Next()
	})
	registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
	server := httptest.NewServer(adaptor.FiberApp(app))
	defer server.Close()

	type result struct {
		status int
		err    error
	}
	upgradeDone := make(chan result, 1)
	go func() {
		response, err := server.Client().Post(server.URL+"/api/browsers/account-a/upgrade", "application/json", strings.NewReader(`{"version":"144.0.7559.132"}`))
		if err != nil {
			upgradeDone <- result{err: err}
			return
		}
		defer response.Body.Close()
		upgradeDone <- result{status: response.StatusCode}
	}()
	select {
	case <-gateway.createStarted:
	case <-time.After(time.Second):
		t.Fatal("upgrade did not reach gateway create")
	}

	disableDone := make(chan result, 1)
	go func() {
		request, err := http.NewRequest(http.MethodPut, server.URL+"/api/browser-images/144.0.7559.132", strings.NewReader(
			`{"image_ref":"registry.example/browser:144","enabled":false}`))
		if err != nil {
			disableDone <- result{err: err}
			return
		}
		request.Header.Set("Content-Type", "application/json")
		response, err := server.Client().Do(request)
		if err != nil {
			disableDone <- result{err: err}
			return
		}
		defer response.Body.Close()
		disableDone <- result{status: response.StatusCode}
	}()
	<-disableArrived
	select {
	case result := <-disableDone:
		t.Fatalf("disable completed before upgrade commit: %#v", result)
	case <-time.After(50 * time.Millisecond):
	}

	close(releaseCreate)
	if result := <-upgradeDone; result.err != nil || result.status != http.StatusNoContent {
		t.Fatalf("upgrade failed: %#v", result)
	}
	if result := <-disableDone; result.err != nil || result.status != http.StatusNoContent {
		t.Fatalf("disable failed: %#v", result)
	}
	env, err := store.GetEnv(context.Background(), "account-a")
	if err != nil || env.ImageVersion != "144.0.7559.132" {
		t.Fatalf("upgrade must commit before disable: %#v %v", env, err)
	}
	if _, err := store.ImageRef(context.Background(), "144.0.7559.132"); !errors.Is(err, hub.ErrNotFound) {
		t.Fatalf("disable must apply after upgrade: %v", err)
	}
}

func TestListAndHeartbeatWaitForUpgradeCoordination(t *testing.T) {
	releaseCreate := make(chan struct{})
	gateway := &fakeGateway{
		token:         "unit-test-gateway-token",
		createStarted: make(chan struct{}),
		releaseCreate: releaseCreate,
	}
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	_ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true})
	gatewayServer := httptest.NewServer(gateway.handler(t))
	defer gatewayServer.Close()
	store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token}
	listArrived := make(chan struct{})
	var listOnce sync.Once
	app := fiber.New()
	app.Use(func(c fiber.Ctx) error {
		if c.Method() == http.MethodGet && c.Path() == "/api/browsers" {
			listOnce.Do(func() { close(listArrived) })
		}
		return c.Next()
	})
	registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
	server := httptest.NewServer(adaptor.FiberApp(app))
	defer server.Close()

	type result struct {
		status int
		err    error
	}
	upgradeDone := make(chan result, 1)
	go func() {
		response, err := server.Client().Post(server.URL+"/api/browsers/account-a/upgrade", "application/json", strings.NewReader(`{"version":"149"}`))
		if err != nil {
			upgradeDone <- result{err: err}
			return
		}
		defer response.Body.Close()
		upgradeDone <- result{status: response.StatusCode}
	}()
	select {
	case <-gateway.createStarted:
	case <-time.After(time.Second):
		t.Fatal("upgrade did not reach gateway create")
	}

	listDone := make(chan result, 1)
	go func() {
		response, err := server.Client().Get(server.URL + "/api/browsers")
		if err != nil {
			listDone <- result{err: err}
			return
		}
		defer response.Body.Close()
		listDone <- result{status: response.StatusCode}
	}()
	<-listArrived
	heartbeatDone := make(chan error, 1)
	heartbeatStarted := make(chan struct{})
	go func() {
		close(heartbeatStarted)
		heartbeatDone <- reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
	}()
	<-heartbeatStarted
	select {
	case result := <-listDone:
		t.Fatalf("list reconciled a stale snapshot during upgrade: %#v", result)
	case err := <-heartbeatDone:
		t.Fatalf("heartbeat reconciled a stale snapshot during upgrade: %v", err)
	case <-time.After(50 * time.Millisecond):
	}

	close(releaseCreate)
	if result := <-upgradeDone; result.err != nil || result.status != http.StatusNoContent {
		t.Fatalf("upgrade failed: %#v", result)
	}
	if result := <-listDone; result.err != nil || result.status != http.StatusOK {
		t.Fatalf("coordinated list failed: %#v", result)
	}
	if err := <-heartbeatDone; err != nil {
		t.Fatalf("coordinated heartbeat failed: %v", err)
	}
}

func TestBrowserActionRoutesStartStopAndRejectsUnknown(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
		t.Fatalf("expected 204 for start, got %d: %s", response.Code, response.Body.String())
	}
	if response := do(app, http.MethodPost, "/api/browsers/account-a/stop", ""); response.Code != http.StatusNoContent {
		t.Fatalf("expected 204 for stop, got %d: %s", response.Code, response.Body.String())
	}
	if response := do(app, http.MethodPost, "/api/browsers/account-a/pause", ""); response.Code != http.StatusBadRequest {
		t.Fatalf("expected 400 for unknown action, got %d", response.Code)
	}
	if len(store.actions) != 4 || store.actions[0].Action != "start" || store.actions[1].Outcome != "succeeded" ||
		store.actions[2].Action != "stop" || store.actions[3].Outcome != "succeeded" {
		t.Fatalf("start and stop must each emit an audit pair: %#v", store.actions)
	}
}

func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) {
	gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
		if request.Method == http.MethodGet {
			_ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "exited"}})
			return
		}
		connection, _, err := response.(http.Hijacker).Hijack()
		if err == nil {
			_ = connection.Close()
		}
	}))
	defer gatewayServer.Close()
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id"}
	store.releaseErr = errors.New("database unavailable")
	store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
	app := fiber.New()
	registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

	response := do(app, http.MethodPost, "/api/browsers/account-a/stop", "")
	if response.Code != http.StatusInternalServerError {
		t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String())
	}
	if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_release_failed" {
		t.Fatalf("stopped reconcile release failure must be audited: %#v", store.actions)
	}
}

func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) {
	store := newMemoryStore()
	store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id",
	}
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
		ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
	if response.Code != http.StatusOK {
		t.Fatalf("running runtime rebind failed: %d: %s", response.Code, response.Body.String())
	}
	bound := store.bindings["account-a"]
	if bound.Exit.ID != "exit-2" || bound.BindingVersion != 2 || bound.RuntimeID != "container-id" {
		t.Fatalf("running runtime was not rebuilt on the latest binding: %#v", bound)
	}
	requests := gateway.recorded()
	if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete || requests[2].method != http.MethodPost {
		t.Fatalf("running rebind must inspect, delete and recreate: %#v", requests)
	}
	if requests[2].body["binding_version"] != float64(2) || requests[2].body["network_exit_id"] != "exit-2" {
		t.Fatalf("recreated runtime did not carry the latest binding CAS: %#v", requests[2].body)
	}
}

func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "expired-runtime", RuntimeID: "old-container",
	}
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
		ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-1"}`)
	if response.Code != http.StatusOK {
		t.Fatalf("same-exit rebind failed: %d: %s", response.Code, response.Body.String())
	}
	bound := store.bindings["account-a"]
	if bound.BindingVersion != 2 || bound.RuntimeID != "container-id" {
		t.Fatalf("same-exit rebind did not rotate the binding generation and lease: %#v", bound)
	}
	requests := gateway.recorded()
	if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete || requests[2].method != http.MethodPost {
		t.Fatalf("same-exit rebind bypassed gateway reconciliation: %#v", requests)
	}
	if requests[2].body["binding_version"] != float64(2) || requests[2].body["network_exit_id"] != "exit-1" {
		t.Fatalf("same-exit rebuild used stale metadata: %#v", requests[2].body)
	}
}

func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id",
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
		ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
	}}}
	app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "exit_auth_failed"},
		func(hub.NetworkExitAccess) (string, error) { return "", nil })

	response := do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusOK {
		t.Fatalf("unhealthy runtime reconciliation failed: %d: %s", response.Code, response.Body.String())
	}
	if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
		t.Fatalf("failed exit remained active: %q", runtime)
	}
	requests := gateway.recorded()
	if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete {
		t.Fatalf("failed exit must be rechecked and removed instead of renewed: %#v", requests)
	}
}

func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container",
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{
		ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusBadGateway {
		t.Fatalf("delete failure must remain unknown, got %d: %s", response.Code, response.Body.String())
	}
	if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
		t.Fatalf("delete failure retained DB runtime lease: %q", runtime)
	}
	if len(store.actions) != 2 || store.actions[0].Action != "reconcile" || store.actions[1].Outcome != "unknown" ||
		store.actions[1].ReasonCode != "gateway_result_unknown" {
		t.Fatalf("delete failure was not audited as retryable unknown: %#v", store.actions)
	}
	if len(gateway.containers) != 1 {
		t.Fatal("failed delete unexpectedly removed the gateway container")
	}
}

func TestDisableExitImmediatelyDiscardsRuntime(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id",
	}
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
		ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "")
	if response.Code != http.StatusOK {
		t.Fatalf("disable cleanup failed: %d: %s", response.Code, response.Body.String())
	}
	if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
		t.Fatalf("disabled exit remained active: %q", runtime)
	}
	requests := gateway.recorded()
	if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete {
		t.Fatalf("disable must reconcile and remove the runtime: %#v", requests)
	}
}

func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) {
	for _, test := range []struct {
		name    string
		handler http.HandlerFunc
	}{
		{name: "disconnect", handler: func(response http.ResponseWriter, _ *http.Request) {
			connection, _, err := response.(http.Hijacker).Hijack()
			if err == nil {
				_ = connection.Close()
			}
		}},
		{name: "status 500", handler: func(response http.ResponseWriter, _ *http.Request) {
			response.WriteHeader(http.StatusInternalServerError)
			_, _ = response.Write([]byte(`{"error":"docker unavailable"}`))
		}},
		{name: "invalid JSON", handler: func(response http.ResponseWriter, _ *http.Request) {
			_, _ = response.Write([]byte(`{"not":"a browser list"}`))
		}},
	} {
		t.Run(test.name, func(t *testing.T) {
			gatewayServer := httptest.NewServer(test.handler)
			defer gatewayServer.Close()
			store := newMemoryStore()
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			store.bindings["account-a"] = hub.EnvironmentContext{
				Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
				Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id",
			}
			store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
			app := fiber.New()
			registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })

			response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "")
			if response.Code != http.StatusBadGateway {
				t.Fatalf("gateway unknown must not return 200: %d: %s", response.Code, response.Body.String())
			}
			if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" {
				t.Fatalf("gateway unknown changed the unconfirmed runtime lease: %q", runtime)
			}
			if store.exits["exit-1"].HealthStatus != "disabled" {
				t.Fatal("disable state was not retained for retryable reconciliation")
			}
		})
	}
}

func TestLegacyNullBindingIsListableAndExplicitlyRecoverable(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{
		Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true,
	}}
	store.bindings["account-a"] = hub.EnvironmentContext{
		Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
		RuntimeInstanceID: "legacy-runtime", RuntimeID: "legacy-container",
	}
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
		ID: "legacy-container", Alias: "account-a", State: "running",
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodGet, "/api/browsers", "")
	if response.Code != http.StatusOK {
		t.Fatalf("legacy NULL binding broke browser listing: %d: %s", response.Code, response.Body.String())
	}
	var views []envView
	if err := json.Unmarshal(response.Body.Bytes(), &views); err != nil || len(views) != 1 || !views[0].RecoveryRequired || views[0].NetworkExitID != "" {
		t.Fatalf("legacy recovery state was not visible: %#v err=%v", views, err)
	}

	response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-1"}`)
	if response.Code != http.StatusOK {
		t.Fatalf("legacy explicit recovery failed: %d: %s", response.Code, response.Body.String())
	}
	requests := gateway.recorded()
	if len(requests) != 4 || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost {
		t.Fatalf("legacy recovery must inspect, remove, then recreate: %#v", requests)
	}
	encoded, _ := json.Marshal(requests[3].body)
	if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") {
		t.Fatalf("legacy Config.Cmd credentials reached the recovered runtime: %s", encoded)
	}
}

func TestRebindDeleteFailureKeepsOriginalBinding(t *testing.T) {
	for _, test := range []struct {
		name     string
		binding  hub.EnvironmentContext
		targetID string
	}{
		{name: "running binding", binding: hub.EnvironmentContext{
			AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
			Exit:              hub.NetworkExit{ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy"},
			RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
		}, targetID: "exit-2"},
		{name: "legacy NULL binding", binding: hub.EnvironmentContext{
			AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
			RuntimeInstanceID: "legacy-runtime", RuntimeID: "legacy-container",
		}, targetID: "exit-1"},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := newMemoryStore()
			_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
			store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			before := test.binding
			before.Env = store.envs["account-a"]
			store.bindings["account-a"] = before
			gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{
				ID: before.RuntimeID, Alias: "account-a", State: "running", BindingVersion: before.BindingVersion, NetworkExitID: before.Exit.ID, ProxyReady: true,
			}}}
			app := newTestApp(t, store, gateway)

			response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+test.targetID+`"}`)
			if response.Code != http.StatusBadGateway {
				t.Fatalf("delete failure must remain unknown: %d: %s", response.Code, response.Body.String())
			}
			after := store.bindings["account-a"]
			if after.BindingVersion != before.BindingVersion || after.Exit.ID != before.Exit.ID || !after.RuntimeCleanupPending || after.RuntimeID != "" {
				t.Fatalf("binding committed before old container deletion: before=%#v after=%#v", before, after)
			}
			if len(gateway.containers) != 1 {
				t.Fatalf("non-final delete result lost the existing container: %#v", gateway.containers)
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
				t.Fatalf("delete failure was not audited as unknown: %#v", store.actions)
			}
		})
	}
}

func TestRebindNetworkCleanupPendingBlocksUntilConfirmed(t *testing.T) {
	for _, test := range []struct {
		name                      string
		cleanupPending            int
		disconnectDelete          int
		disconnectListAfterDelete bool
	}{
		{name: "cleanup pending", cleanupPending: 2},
		{name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := newMemoryStore()
			_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
			store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			store.bindings["account-a"] = hub.EnvironmentContext{
				Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
				Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container",
			}
			gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: test.cleanupPending,
				disconnectDelete: test.disconnectDelete, disconnectListAfterDelete: test.disconnectListAfterDelete,
				containers: []containerStatus{{
					ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
				}}}
			app := newTestApp(t, store, gateway)

			response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
			if response.Code != http.StatusBadGateway {
				t.Fatalf("network cleanup uncertainty returned %d: %s", response.Code, response.Body.String())
			}
			after := store.bindings["account-a"]
			if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || !after.RuntimeCleanupPending || after.RuntimeID != "" {
				t.Fatalf("network cleanup uncertainty changed the binding: %#v", after)
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
				t.Fatalf("network cleanup uncertainty audit mismatch: %#v", store.actions)
			}
			if len(gateway.containers) != 0 {
				t.Fatalf("container-removed fact was lost: %#v", gateway.containers)
			}

			response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
			if response.Code != http.StatusOK {
				t.Fatalf("confirmed cleanup did not allow retry: %d: %s", response.Code, response.Body.String())
			}
			after = store.bindings["account-a"]
			if after.RuntimeCleanupPending || after.BindingVersion != 2 || after.Exit.ID != "exit-2" || after.RuntimeID != "" {
				t.Fatalf("retry did not commit a clean stopped generation: %#v", after)
			}
		})
	}
}

func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) {
	for _, test := range []struct {
		name, method, path, body string
	}{
		{name: "list reconcile", method: http.MethodGet, path: "/api/browsers"},
		{name: "create reuse", method: http.MethodPost, path: "/api/browsers", body: `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`},
		{name: "start", method: http.MethodPost, path: "/api/browsers/account-a/start"},
		{name: "upgrade", method: http.MethodPost, path: "/api/browsers/account-a/upgrade", body: `{"version":"149"}`},
		{name: "rebind", method: http.MethodPost, path: "/api/browsers/account-a/rebind", body: `{"network_exit_id":"exit-1"}`},
		{name: "recycle", method: http.MethodDelete, path: "/api/browsers/account-a"},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := newMemoryStore()
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			store.bindings["account-a"] = hub.EnvironmentContext{
				Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
				RuntimeCleanupPending: true, Exit: store.exits["exit-1"],
			}
			_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
			_ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true})
			gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: 2}
			app := newTestApp(t, store, gateway)

			response := do(app, test.method, test.path, test.body)
			if response.Code != http.StatusBadGateway {
				t.Fatalf("pending cleanup returned %d: %s", response.Code, response.Body.String())
			}
			after := store.bindings["account-a"]
			if !after.RuntimeCleanupPending || after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" {
				t.Fatalf("lifecycle path advanced a pending generation: %#v", after)
			}
			for _, request := range gateway.recorded() {
				if request.method == http.MethodPost && request.path == "/v1/browsers" {
					t.Fatalf("lifecycle path created before cleanup confirmation: %#v", gateway.recorded())
				}
			}
		})
	}
}

func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) {
	for _, test := range []struct {
		name       string
		withImage  bool
		credential bool
	}{
		{name: "image unavailable"},
		{name: "credential unavailable", withImage: true, credential: true},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := newMemoryStore()
			if test.withImage {
				_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
			}
			store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
			if test.credential {
				store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1,
					CredentialReference: &hub.CredentialReference{ID: "credential-exit", Provider: "os_keyring"}}
			}
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
			store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a",
				BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container"}
			gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
				ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
			}}}
			app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) {
				return "", errors.New("credential unavailable")
			})

			response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
			if response.Code < 400 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet {
				t.Fatalf("runtime preparation failure touched the old container: status=%d requests=%#v", response.Code, gateway.recorded())
			}
			if after := store.bindings["account-a"]; after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "old-container" {
				t.Fatalf("runtime preparation failure changed state: %#v", after)
			}
		})
	}
}

func TestRebindCandidateCreateFailureRestoresOldRuntime(t *testing.T) {
	store := newMemoryStore()
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a",
		BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container"}
	gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, containers: []containerStatus{{
		ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
	}}}
	app := newTestApp(t, store, gateway)

	response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
	if response.Code != http.StatusConflict {
		t.Fatalf("candidate create failure returned %d: %s", response.Code, response.Body.String())
	}
	after := store.bindings["account-a"]
	if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID == "" ||
		len(gateway.containers) != 1 || !containerMatchesBinding(gateway.containers[0], after) {
		t.Fatalf("candidate create failure did not restore old runtime: after=%#v containers=%#v", after, gateway.containers)
	}
	if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "gateway_create_failed" {
		t.Fatalf("candidate create failure audit mismatch: %#v", store.actions)
	}
}

func TestExistingEnvironmentCleanupNeverReturnsReusedSuccess(t *testing.T) {
	for _, test := range []struct {
		name    string
		probe   *sequenceExitProbe
		resolve func(hub.NetworkExitAccess) (string, error)
	}{
		{name: "second probe fails", probe: &sequenceExitProbe{failures: []string{"", "exit_auth_failed"}},
			resolve: func(hub.NetworkExitAccess) (string, error) { return "username:password", nil }},
		{name: "second credential restore fails", probe: &sequenceExitProbe{}, resolve: func() func(hub.NetworkExitAccess) (string, error) {
			calls := 0
			return func(hub.NetworkExitAccess) (string, error) {
				calls++
				if calls == 2 {
					return "", errors.New("credential unavailable")
				}
				return "username:password", nil
			}
		}()},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := newMemoryStore()
			store.exits["exit-1"] = hub.NetworkExit{
				ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1,
				CredentialReference: &hub.CredentialReference{ID: "credential-exit", Provider: "os_keyring"},
			}
			store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148.0.7778.215", Fingerprint: hub.Fingerprint{Seed: 2024, Platform: "windows", Timezone: "Asia/Shanghai"}}
			store.bindings["account-a"] = hub.EnvironmentContext{
				Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
				Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id",
			}
			_ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true})
			gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{
				ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1",
			}}}
			app := newTestAppWithNetwork(t, store, gateway, test.probe, test.resolve)

			response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
			if response.Code != http.StatusConflict {
				t.Fatalf("cleaned runtime must not return reused success: %d: %s", response.Code, response.Body.String())
			}
			if !strings.Contains(response.Body.String(), "not running") {
				t.Fatalf("cleaned runtime response did not state the stopped result: %s", response.Body.String())
			}
			if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
				t.Fatalf("failed recovery retained runtime: %q probe_calls=%d actions=%#v requests=%#v", runtime, test.probe.calls, store.actions, gateway.recorded())
			}
			if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_unavailable" {
				t.Fatalf("cleaned runtime was audited as reused success: %#v", store.actions)
			}
		})
	}
}

func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) {
	store := newMemoryStore()
	store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1}
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"]}
	_ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true})
	gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "old-container", Alias: "account-a", State: "exited"}}}
	app := newTestApp(t, store, gateway)

	if response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`); response.Code != http.StatusOK {
		t.Fatalf("rebind failed: %d %s", response.Code, response.Body.String())
	}
	if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
		t.Fatalf("start failed: %d %s", response.Code, response.Body.String())
	}
	requests := gateway.recorded()
	var createRequest recordedRequest
	for _, request := range requests {
		if request.method == http.MethodPost && request.path == "/v1/browsers" {
			createRequest = request
		}
	}
	if createRequest.body == nil {
		t.Fatalf("stopped container was not recreated: %#v", requests)
	}
	exit := createRequest.body["network_exit"].(map[string]any)
	if exit["host"] != "proxy-2.example" {
		t.Fatalf("recreated container did not use rebound exit: %#v", createRequest.body)
	}
}

func TestRecycleBrowserKeepsStableEnvironment(t *testing.T) {
	store := newMemoryStore()
	store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}}
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	if response := do(app, http.MethodDelete, "/api/browsers/account-a", ""); response.Code != http.StatusNoContent {
		t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String())
	}
	_, kept := store.envs["account-a"]
	if !kept {
		t.Fatal("recycle must preserve the stable environment and Profile anchor")
	}
	if len(store.actions) != 2 || store.actions[0].Action != "recycle" || store.actions[1].Outcome != "succeeded" {
		t.Fatalf("recycle must emit an audit pair: %#v", store.actions)
	}
}

func TestGatewayAndImageCRUDRoutes(t *testing.T) {
	store := newMemoryStore()
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	if response := do(app, http.MethodPost, "/api/browser-images",
		`{"version":"148.0.7778.215","image_ref":"registry.example/browser:148","note":"main","enabled":true}`); response.Code != http.StatusCreated {
		t.Fatalf("expected 201 for image create, got %d: %s", response.Code, response.Body.String())
	}
	if image, ok := store.images["148.0.7778.215"]; !ok || image.ImageRef != "registry.example/browser:148" || !image.Enabled {
		t.Fatalf("image must be stored: %#v", store.images)
	}
	if response := do(app, http.MethodPost, "/api/browser-images",
		`{"version":"148.0.7778.215","image_ref":"registry.example/browser:148"}`); response.Code != http.StatusCreated {
		t.Fatalf("enabled must default to true, got %d", response.Code)
	}
}

func TestNetworkExitRoutesAreStrictAndSecretFree(t *testing.T) {
	store := newMemoryStore()
	gateway := &fakeGateway{token: "unit-test-gateway-token"}
	app := newTestApp(t, store, gateway)

	invalid := do(app, http.MethodPost, "/api/network-exits",
		`{"protocol":"socks5","host":"proxy.example","port":1080,"credential_reference":{"id":"credential-a","key":"raw-value"}}`)
	if invalid.Code != http.StatusBadRequest || len(store.exits) != 1 {
		t.Fatalf("raw credential fields must be rejected before persistence: status=%d exits=%#v", invalid.Code, store.exits)
	}
	created := do(app, http.MethodPost, "/api/network-exits",
		`{"protocol":"socks5","host":"proxy.example","port":1080,"credential_reference":{"id":"credential-a"},"expected_public_ip":"203.0.113.1","expected_region":"test"}`)
	if created.Code != http.StatusCreated || strings.Contains(created.Body.String(), "raw-value") {
		t.Fatalf("unexpected secret-bearing network exit response: status=%d body=%s", created.Code, created.Body.String())
	}
	checked := do(app, http.MethodPost, "/api/network-exits/exit-created/check", "")
	if checked.Code != http.StatusOK || !strings.Contains(checked.Body.String(), `"health_status":"healthy"`) {
		t.Fatalf("network exit check did not record health: status=%d body=%s", checked.Code, checked.Body.String())
	}
	disabled := do(app, http.MethodPost, "/api/network-exits/exit-created/disable", "")
	if disabled.Code != http.StatusOK || !strings.Contains(disabled.Body.String(), `"health_status":"disabled"`) {
		t.Fatalf("network exit disable failed: status=%d body=%s", disabled.Code, disabled.Body.String())
	}
}

func TestCreateImageReturnsJSONOverHTTP(t *testing.T) {
	store := newMemoryStore()
	app := fiber.New()
	registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
	server := httptest.NewServer(adaptor.FiberApp(app))
	defer server.Close()

	response, err := server.Client().Post(server.URL+"/api/browser-images", "application/json", strings.NewReader(
		`{"version":"148.0.7778.215","image_ref":"registry.example/browser:148","note":"main","enabled":true}`))
	if err != nil {
		t.Fatal(err)
	}
	defer response.Body.Close()
	var image hub.Image
	if err := json.NewDecoder(response.Body).Decode(&image); err != nil {
		t.Fatalf("201 response must be JSON: %v", err)
	}
	if response.StatusCode != http.StatusCreated || !strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") ||
		image.Version != "148.0.7778.215" || image.ImageRef != "registry.example/browser:148" || !image.Enabled {
		t.Fatalf("unexpected create response: status=%d content-type=%q image=%#v", response.StatusCode, response.Header.Get("Content-Type"), image)
	}
}