package api import ( "context" "database/sql" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "net/url" "os" "sort" "strings" "sync" "testing" "time" accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" ) // memoryStore 是 hubStore 的内存桩,记录写入以便断言编排副作用。 type memoryStore struct { mu sync.Mutex locksMu sync.Mutex locks map[string]*sync.Mutex gateways map[string]hub.Gateway envs map[string]hub.Env exits map[string]hub.NetworkExit bindings map[string]hub.EnvironmentContext actions []hub.EnvironmentAction upgraded map[string]string upgradeErr error releaseErr error cleanupPendingErr error cleanupPendingErrAfterMutation bool gatewayFn func(name string) (hub.Gateway, error) } type blockingRuntimeStopStore struct { *hub.Store entered chan struct{} release chan struct{} } func (s *blockingRuntimeStopStore) GetGateway(context.Context, string) (hub.Gateway, error) { s.entered <- struct{}{} <-s.release return hub.Gateway{}, errors.New("gateway unavailable") } func newMemoryStore() *memoryStore { return &memoryStore{ locks: map[string]*sync.Mutex{}, gateways: map[string]hub.Gateway{}, envs: map[string]hub.Env{}, exits: map[string]hub.NetworkExit{ "exit-1": {ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1}, }, bindings: map[string]hub.EnvironmentContext{}, upgraded: map[string]string{}, } } func TestResumeBlockReasonIsStable(t *testing.T) { account := accountdomain.Account{AuthorizationStatus: "authorized"} healthy := hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "healthy"}} for name, test := range map[string]struct { account accountdomain.Account env hub.EnvironmentContext found bool want string }{ "revoked": {accountdomain.Account{AuthorizationStatus: "revoked"}, healthy, true, "account_revoked"}, "missing binding": {account, hub.EnvironmentContext{}, false, "binding_missing"}, "direct exit": {account, hub.EnvironmentContext{}, true, "account_conflict"}, "unhealthy exit": {account, hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "unhealthy"}}, true, "network_exit_unhealthy"}, "cleanup pending": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeCleanupPending: true}, true, "runtime_stop_pending"}, "runtime active": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeInstanceID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, true, "runtime_active"}, } { t.Run(name, func(t *testing.T) { if got := resumeBlockReason(test.account, test.env, test.found); got != test.want { t.Fatalf("got %q, want %q", got, test.want) } }) } } func TestPhaseAReadinessErrorsAreStructured(t *testing.T) { for _, test := range []struct { name, reason string unavailable bool status int }{ {name: "version conflict", reason: "draft_version_changed", status: http.StatusConflict}, {name: "resource unavailable", reason: "network_exit_unhealthy", unavailable: true, status: http.StatusServiceUnavailable}, } { t.Run(test.name, func(t *testing.T) { app := fiber.New() app.Get("/", func(c fiber.Ctx) error { return phaseAError(c, &accountdomain.ReadinessError{Reason: test.reason, Unavailable: test.unavailable}) }) response := do(app, http.MethodGet, "/", "") if response.Code != test.status || !strings.Contains(response.Body.String(), `"reason_code":"`+test.reason+`"`) { t.Fatalf("unexpected response: %d %s", response.Code, response.Body.String()) } }) } } func (s *memoryStore) LockResources(_ context.Context, aliases, exitIDs []string, imageVersionSets ...[]string) (func(), error) { imageVersions := []string(nil) if len(imageVersionSets) > 0 { imageVersions = imageVersionSets[0] } keys := make([]string, 0, len(aliases)+len(exitIDs)+len(imageVersions)) for _, alias := range aliases { keys = append(keys, "environment:"+alias) } for _, id := range exitIDs { keys = append(keys, "network-exit:"+id) } for _, version := range imageVersions { keys = append(keys, "image:"+version) } sort.Strings(keys) unlocks := make([]func(), 0, len(keys)) seen := map[string]bool{} for _, key := range keys { if !seen[key] { seen[key] = true unlocks = append(unlocks, s.lock(key)) } } return func() { for index := len(unlocks) - 1; index >= 0; index-- { unlocks[index]() } }, nil } func (s *memoryStore) lock(key string) func() { s.locksMu.Lock() lock := s.locks[key] if lock == nil { lock = &sync.Mutex{} s.locks[key] = lock } s.locksMu.Unlock() lock.Lock() return lock.Unlock } func (s *memoryStore) CreateGateway(_ context.Context, _, _, _ string) (hub.Gateway, error) { return hub.Gateway{}, nil } func (s *memoryStore) UpdateGateway(_ context.Context, currentName, name, endpoint, token string) (hub.Gateway, error) { s.mu.Lock() defer s.mu.Unlock() gateway, exists := s.gateways[currentName] if !exists { return hub.Gateway{}, hub.ErrNotFound } if currentName != name { if _, exists := s.gateways[name]; exists { return hub.Gateway{}, hub.ErrConflict } delete(s.gateways, currentName) for alias, env := range s.envs { if env.Gateway == currentName { env.Gateway = name s.envs[alias] = env } } } gateway.Name, gateway.Endpoint = name, endpoint if token != "" { gateway.Token = token } s.gateways[name] = gateway return gateway, nil } func (s *memoryStore) ListGateways(context.Context) ([]hub.Gateway, error) { s.mu.Lock() defer s.mu.Unlock() gateways := make([]hub.Gateway, 0, len(s.gateways)) for _, gateway := range s.gateways { gateways = append(gateways, gateway) } return gateways, nil } func (s *memoryStore) GetGateway(_ context.Context, name string) (hub.Gateway, error) { s.mu.Lock() defer s.mu.Unlock() if s.gatewayFn != nil { return s.gatewayFn(name) } gateway, ok := s.gateways[name] if !ok { return hub.Gateway{}, hub.ErrNotFound } return gateway, nil } func (s *memoryStore) DeleteGateway(context.Context, string) error { return nil } func (s *memoryStore) CreateEnv(_ context.Context, env hub.Env) error { s.mu.Lock() defer s.mu.Unlock() if _, exists := s.envs[env.Alias]; exists { return hub.ErrConflict } s.envs[env.Alias] = env return nil } func (s *memoryStore) ListEnvs(context.Context) ([]hub.Env, error) { s.mu.Lock() defer s.mu.Unlock() envs := make([]hub.Env, 0, len(s.envs)) for _, env := range s.envs { envs = append(envs, env) } return envs, nil } func (s *memoryStore) GetEnv(_ context.Context, alias string) (hub.Env, error) { s.mu.Lock() defer s.mu.Unlock() env, ok := s.envs[alias] if !ok { return hub.Env{}, hub.ErrNotFound } return env, nil } func (s *memoryStore) CreateNetworkExit(_ context.Context, exit hub.NetworkExit) (hub.NetworkExit, error) { s.mu.Lock() defer s.mu.Unlock() if len(exit.Username) > 255 || len(exit.Password) > 255 || (exit.Username == "" && exit.Password != "") { return hub.NetworkExit{}, hub.ErrInvalid } exit.ID, exit.HealthStatus, exit.Version = "exit-created", "unchecked", 1 s.exits[exit.ID] = exit return exit, nil } func (s *memoryStore) ListNetworkExits(context.Context) ([]hub.NetworkExit, error) { s.mu.Lock() defer s.mu.Unlock() result := make([]hub.NetworkExit, 0, len(s.exits)) for _, exit := range s.exits { result = append(result, exit) } return result, nil } func (s *memoryStore) GetNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) { s.mu.Lock() defer s.mu.Unlock() exit, ok := s.exits[id] if !ok { return hub.NetworkExit{}, hub.ErrNotFound } return exit, nil } func (s *memoryStore) GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error) { exit, err := s.GetNetworkExit(ctx, id) return hub.NetworkExitAccess{NetworkExit: exit}, err } func (s *memoryStore) RecordNetworkExitCheck(_ context.Context, id string, observation hub.ExitObservation, failure string) (hub.NetworkExit, string, error) { s.mu.Lock() defer s.mu.Unlock() exit, ok := s.exits[id] if !ok { return hub.NetworkExit{}, "exit_unavailable", hub.ErrNotFound } reason := failure if reason == "" && exit.ExpectedPublicIP != "" && exit.ExpectedPublicIP != observation.PublicIP { reason = "exit_ip_drift" } if reason == "" && exit.ExpectedRegion != "" && exit.ExpectedRegion != observation.Region { reason = "exit_region_drift" } exit.ObservedPublicIP, exit.ObservedRegion = observation.PublicIP, observation.Region if reason == "" { exit.HealthStatus, reason = "healthy", "exit_healthy" } else { exit.HealthStatus = "unhealthy" } s.exits[id] = exit return exit, reason, nil } func (s *memoryStore) DisableNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) { s.mu.Lock() defer s.mu.Unlock() exit, ok := s.exits[id] if !ok { return hub.NetworkExit{}, hub.ErrNotFound } exit.HealthStatus = "disabled" s.exits[id] = exit return exit, nil } func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) { s.mu.Lock() if existing, ok := s.bindings[env.Alias]; ok { if existing.AccountStatus == "" { existing.AccountStatus, existing.AuthorizationStatus = "active", "authorized" } s.mu.Unlock() return existing, false, nil } s.mu.Unlock() if err := s.CreateEnv(ctx, env); err != nil { return hub.EnvironmentContext{}, false, err } s.mu.Lock() defer s.mu.Unlock() bound := hub.EnvironmentContext{Env: env, AccountID: accountID, AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: accountID, BindingVersion: 1, Exit: s.exits[exitID]} s.bindings[env.Alias] = bound return bound, true, nil } func (s *memoryStore) GetEnvironmentContext(_ context.Context, alias string) (hub.EnvironmentContext, error) { s.mu.Lock() defer s.mu.Unlock() if bound, ok := s.bindings[alias]; ok { if bound.AccountStatus == "" { bound.AccountStatus, bound.AuthorizationStatus = "active", "authorized" } return bound, nil } env, ok := s.envs[alias] if !ok { return hub.EnvironmentContext{}, hub.ErrNotFound } return hub.EnvironmentContext{Env: env, AccountID: alias, AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}, nil } func (s *memoryStore) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) { s.mu.Lock() for alias, bound := range s.bindings { if bound.AccountID == accountID { s.mu.Unlock() return s.GetEnvironmentContext(ctx, alias) } } s.mu.Unlock() return hub.EnvironmentContext{}, hub.ErrNotFound } func (s *memoryStore) ValidateEnvironmentRebind(_ context.Context, alias, exitID string, expectedBindingVersion int64) error { s.mu.Lock() defer s.mu.Unlock() bound, ok := s.bindings[alias] if !ok { bound.BindingVersion = 1 } if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending || s.exits[exitID].HealthStatus != "healthy" { return hub.ErrConflict } return nil } func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkIDs ...string) (hub.EnvironmentContext, error) { s.mu.Lock() defer s.mu.Unlock() bound, ok := s.bindings[alias] if !ok { bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} } if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending { return hub.EnvironmentContext{}, hub.ErrConflict } bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", "" if runtimeID != "" { bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID if len(networkIDs) == 1 { bound.RuntimeNetworkID = networkIDs[0] } } bound.Exit, bound.BindingVersion = s.exits[exitID], bound.BindingVersion+1 s.bindings[alias] = bound return bound, nil } func (s *memoryStore) ActivateRuntime(_ context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkIDs ...string) (hub.EnvironmentContext, error) { s.mu.Lock() defer s.mu.Unlock() if len(networkIDs) != 1 || networkIDs[0] == "" { return hub.EnvironmentContext{}, hub.ErrInvalid } bound, ok := s.bindings[alias] if !ok { bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} } if bound.AccountStatus == "" { bound.AccountStatus, bound.AuthorizationStatus = "active", "authorized" } if bound.AccountStatus != "active" || bound.AuthorizationStatus != "authorized" || bound.BindingVersion != bindingVersion || bound.Exit.ID != exitID || bound.RuntimeCleanupPending { return hub.EnvironmentContext{}, hub.ErrConflict } bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID bound.RuntimeNetworkID = networkIDs[0] s.bindings[alias] = bound return bound, nil } func (s *memoryStore) ReleaseRuntime(_ context.Context, environment hub.EnvironmentContext) error { s.mu.Lock() defer s.mu.Unlock() if s.releaseErr != nil { return s.releaseErr } bound, ok := s.bindings[environment.Alias] if !ok { bound = hub.EnvironmentContext{Env: s.envs[environment.Alias], AccountID: environment.Alias, BindingID: environment.Alias, BindingVersion: 1, Exit: s.exits["exit-1"]} } if environment.RuntimeInstanceID == "" { return nil } if bound.BindingID != environment.BindingID || bound.BindingVersion != environment.BindingVersion || bound.RuntimeInstanceID != environment.RuntimeInstanceID { return hub.ErrConflict } bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", "" s.bindings[environment.Alias] = bound return nil } func (s *memoryStore) SetRuntimeCleanupPending(_ context.Context, environment hub.EnvironmentContext, pending bool) error { s.mu.Lock() defer s.mu.Unlock() if pending && environment.RuntimeCleanupRuntimeID == "" { return hub.ErrInvalid } if s.cleanupPendingErr != nil && !s.cleanupPendingErrAfterMutation { return s.cleanupPendingErr } if pending && s.releaseErr != nil { return s.releaseErr } bound, ok := s.bindings[environment.Alias] if !ok { bound = hub.EnvironmentContext{Env: s.envs[environment.Alias], AccountID: environment.Alias, BindingID: environment.Alias, BindingVersion: 1, Exit: s.exits["exit-1"]} } if bound.BindingID != environment.BindingID || bound.BindingVersion != environment.BindingVersion { return hub.ErrConflict } if bound.RuntimeCleanupPending { if bound.RuntimeCleanupBindingVersion != environment.RuntimeCleanupBindingVersion || bound.RuntimeCleanupInstanceID != environment.RuntimeCleanupInstanceID || bound.RuntimeCleanupRuntimeID != environment.RuntimeCleanupRuntimeID || bound.RuntimeCleanupNetworkID != environment.RuntimeCleanupNetworkID { return hub.ErrConflict } } else if pending && bound.RuntimeInstanceID != environment.RuntimeCleanupInstanceID { return hub.ErrConflict } bound.RuntimeCleanupPending = pending if pending { bound.RuntimeCleanupBindingVersion = environment.RuntimeCleanupBindingVersion bound.RuntimeCleanupInstanceID = environment.RuntimeCleanupInstanceID bound.RuntimeCleanupRuntimeID = environment.RuntimeCleanupRuntimeID bound.RuntimeCleanupNetworkID = environment.RuntimeCleanupNetworkID bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", "" } else { bound.RuntimeCleanupBindingVersion = 0 bound.RuntimeCleanupInstanceID, bound.RuntimeCleanupRuntimeID, bound.RuntimeCleanupNetworkID = "", "", "" } s.bindings[environment.Alias] = bound return s.cleanupPendingErr } func (s *memoryStore) AppendEnvironmentAction(_ context.Context, _ string, action hub.EnvironmentAction) error { s.mu.Lock() defer s.mu.Unlock() s.actions = append(s.actions, action) return nil } func TestPhaseARuntimeLeaseRoutesAreInternal(t *testing.T) { app := fiber.New() registerPhaseA(app, nil, nil, nil) for _, request := range []struct { method, path string }{ {http.MethodPost, "/api/phase-a/runtimes"}, {http.MethodDelete, "/api/phase-a/runtimes/runtime-a"}, } { if response := do(app, request.method, request.path, `{}`); response.Code != http.StatusNotFound { t.Fatalf("%s %s remained public: %d", request.method, request.path, response.Code) } } } func TestPauseSerializesResumeAndRetainsClaimGateOnUnknownStop(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) accountStore, err := accountdomain.Open(ctx, fixture.databaseURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = accountStore.Close() }) if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil { t.Fatal(err) } if _, err := fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666"); err != nil { t.Fatal(err) } blocking := &blockingRuntimeStopStore{Store: fixture.store, entered: make(chan struct{}, 1), release: make(chan struct{})} app := fiber.New() registerPhaseA(app, accountStore, blocking, nil) pauseDone := make(chan *httptest.ResponseRecorder, 1) resumeDone := make(chan *httptest.ResponseRecorder, 1) go func() { pauseDone <- do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", "") }() select { case <-blocking.entered: case <-time.After(5 * time.Second): t.Fatal("pause did not enter runtime stop") } go func() { resumeDone <- do(app, http.MethodPost, "/api/phase-a/accounts/account-a/resume", "") }() select { case response := <-resumeDone: t.Fatalf("resume bypassed the runtime operation lock: %d", response.Code) case <-time.After(time.Second): } close(blocking.release) if response := <-pauseDone; response.Code != http.StatusInternalServerError { t.Fatalf("unknown stop returned %d: %s", response.Code, response.Body.String()) } if response := <-resumeDone; response.Code != http.StatusConflict { t.Fatalf("resume reopened an unconverged lease: %d: %s", response.Code, response.Body.String()) } account, err := accountStore.GetAccount(ctx, "account-a") if err != nil || account.RuntimeStatus != "paused" { t.Fatalf("resume reopened the claim gate: account=%#v err=%v", account, err) } } type recordedRequest struct { method string path string body map[string]any } // fakeGateway 模拟 native browser gateway:按路由表应答并记录请求。 type fakeGateway struct { mu sync.Mutex createOnce sync.Once deleteOnce sync.Once token string requests []recordedRequest runtimes []runtimeStatus failCreate int // 前 N 次 create 返回失败 failCreateStatus int failDelete int // 前 N 次 delete 返回 500 且保留容器 deleteNotFound int failProxy bool createStarted chan struct{} releaseCreate <-chan struct{} deleteDone chan struct{} releaseDelete <-chan struct{} cleanupPending int disconnectDelete int disconnectList int failList int invalidList int invalidListBody string readErrorList int disconnectListAfterDelete bool } type fakeExitProbe struct { observation hub.ExitObservation failure string } type blockingExitProbe struct { started chan struct{} release <-chan struct{} once sync.Once } func (probe *blockingExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) { probe.once.Do(func() { close(probe.started) }) <-probe.release return hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "" } func (probe fakeExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) { if probe.observation.PublicIP == "" && probe.failure == "" { probe.observation = hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"} } return probe.observation, probe.failure } type sequenceExitProbe struct { calls int failures []string } func (probe *sequenceExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) { failure := "" if probe.calls < len(probe.failures) { failure = probe.failures[probe.calls] } probe.calls++ return hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, failure } func (g *fakeGateway) handler(t *testing.T) http.Handler { return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet && request.URL.Path == "/healthz" { response.WriteHeader(http.StatusNoContent) return } if request.Header.Get("Authorization") != "Bearer "+g.token { response.WriteHeader(http.StatusUnauthorized) _, _ = response.Write([]byte(`{"error":"gateway token rejected"}`)) return } var body map[string]any if request.Body != nil { raw, _ := io.ReadAll(request.Body) if len(raw) > 0 { _ = json.Unmarshal(raw, &body) } } g.mu.Lock() g.requests = append(g.requests, recordedRequest{method: request.Method, path: request.URL.Path, body: body}) g.mu.Unlock() switch { case request.Method == http.MethodPost && request.URL.Path == "/v1/browsers": g.mu.Lock() if g.failCreate > 0 { g.failCreate-- status := http.StatusConflict if g.failCreateStatus != 0 { status = g.failCreateStatus } g.mu.Unlock() response.WriteHeader(status) _, _ = response.Write([]byte(`{"error":"alias already in use"}`)) return } g.mu.Unlock() if g.createStarted != nil { g.createOnce.Do(func() { close(g.createStarted) }) } if g.releaseCreate != nil { <-g.releaseCreate } state, proxyReady := "running", true networkID := "native-cccccccccccccccccccccccccccccccc" if stopped, _ := body["stopped"].(bool); stopped { state, proxyReady, networkID = "exited", false, "" } g.mu.Lock() g.runtimes = []runtimeStatus{{ ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady, BindingVersion: int64(body["binding_version"].(float64)), NetworkExitID: body["network_exit_id"].(string), NetworkID: networkID, }} g.mu.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a","network_id":"` + networkID + `"}`)) case request.Method == http.MethodGet && request.URL.Path == "/v1/browsers": g.mu.Lock() if g.failList > 0 { g.failList-- g.mu.Unlock() response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(`{"error":"docker unavailable"}`)) return } if g.invalidList > 0 { g.invalidList-- body := g.invalidListBody if body == "" { body = `{"not":"a browser list"}` } g.mu.Unlock() _, _ = response.Write([]byte(body)) return } if g.readErrorList > 0 { g.readErrorList-- g.mu.Unlock() connection, _, _ := response.(http.Hijacker).Hijack() _, _ = connection.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n[]")) _ = connection.Close() return } if g.disconnectList > 0 { g.disconnectList-- g.mu.Unlock() connection, _, _ := response.(http.Hijacker).Hijack() _ = connection.Close() return } runtimes := append([]runtimeStatus{}, g.runtimes...) g.mu.Unlock() _ = json.NewEncoder(response).Encode(runtimes) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/v1/browsers/"): g.mu.Lock() if len(g.runtimes) > 0 && (body["runtime_id"] != g.runtimes[0].ID || int64(body["binding_version"].(float64)) != g.runtimes[0].BindingVersion) { g.mu.Unlock() response.WriteHeader(http.StatusConflict) return } if g.failDelete > 0 { g.failDelete-- g.mu.Unlock() response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(`{"error":"docker delete failed"}`)) return } g.runtimes = nil if g.deleteNotFound > 0 { g.deleteNotFound-- g.mu.Unlock() response.WriteHeader(http.StatusNotFound) return } g.mu.Unlock() if g.deleteDone != nil { g.deleteOnce.Do(func() { close(g.deleteDone) }) } if g.releaseDelete != nil { <-g.releaseDelete } g.mu.Lock() if g.cleanupPending > 0 { g.cleanupPending-- g.mu.Unlock() response.WriteHeader(http.StatusAccepted) return } if g.disconnectDelete > 0 { g.disconnectDelete-- if g.disconnectListAfterDelete { g.disconnectList++ } g.mu.Unlock() connection, _, _ := response.(http.Hijacker).Hijack() _ = connection.Close() return } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"): g.mu.Lock() if len(g.runtimes) > 0 { g.runtimes[0].State, g.runtimes[0].Status = "running", "Up" } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"): g.mu.Lock() if len(g.runtimes) > 0 && (body["runtime_id"] != g.runtimes[0].ID || int64(body["binding_version"].(float64)) != g.runtimes[0].BindingVersion) { g.mu.Unlock() response.WriteHeader(http.StatusConflict) return } if len(g.runtimes) > 0 { g.runtimes[0].State, g.runtimes[0].Status = "exited", "Exited" } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/proxy"): g.mu.Lock() if len(g.runtimes) > 0 && body["runtime_id"] != g.runtimes[0].ID { g.mu.Unlock() response.WriteHeader(http.StatusConflict) return } if g.failProxy { g.mu.Unlock() response.WriteHeader(http.StatusBadGateway) return } if len(g.runtimes) > 0 { g.runtimes[0].ProxyReady = true } g.mu.Unlock() response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path) } }) } func (g *fakeGateway) recorded() []recordedRequest { g.mu.Lock() defer g.mu.Unlock() return append([]recordedRequest{}, g.requests...) } func newTestApp(t *testing.T, store *memoryStore, gateway *fakeGateway) *fiber.App { return newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) } func newTestAppWithNetwork(t *testing.T, store *memoryStore, gateway *fakeGateway, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) *fiber.App { t.Helper() server := httptest.NewServer(gateway.handler(t)) t.Cleanup(server.Close) if store.gateways == nil { store.gateways = map[string]hub.Gateway{} } store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} app := fiber.New() registerHubWithNetwork(app, store, probe, resolve) return app } func do(app *fiber.App, method, path, body string, credentials ...string) *httptest.ResponseRecorder { response := httptest.NewRecorder() var reader io.Reader if body != "" { reader = strings.NewReader(body) } request := httptest.NewRequest(method, path, reader) if len(credentials) == 2 { request.SetBasicAuth(credentials[0], credentials[1]) } adaptor.FiberApp(app).ServeHTTP(response, request) return response } const createEnvBody = `{"alias":"account-a","name":"店铺一号","gateway":"gw-1",` + `"fingerprint":{"seed":2024,"platform":"windows","timezone":"Asia/Shanghai"},"account_id":"account-a","network_exit_id":"exit-1"}` func TestUpdateGatewayRenamesAndPreservesReferences(t *testing.T) { store := newMemoryStore() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: "http://gw-1:8081", Token: "unit-test-gateway-token"} store.gateways["gw-existing"] = hub.Gateway{Name: "gw-existing", Endpoint: "http://gw-existing:8081", Token: "existing-gateway-token"} store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPut, "/api/gateways/gw-1", `{"name":"gw-main","endpoint":"http://gw-main:8081","token":""}`) if response.Code != http.StatusOK { t.Fatalf("update gateway returned %d: %s", response.Code, response.Body.String()) } var updated hub.Gateway if err := json.Unmarshal(response.Body.Bytes(), &updated); err != nil { t.Fatal(err) } if updated.Name != "gw-main" || updated.Endpoint != "http://gw-main:8081" || updated.Token != "unit-test-gateway-token" { t.Fatalf("gateway update lost fields: %#v", updated) } if store.envs["account-a"].Gateway != "gw-main" { t.Fatalf("environment gateway reference was not renamed: %#v", store.envs["account-a"]) } response = do(app, http.MethodPut, "/api/gateways/gw-main", `{"name":"gw-existing","endpoint":"http://gw-main:8081","token":""}`) if response.Code != http.StatusConflict { t.Fatalf("rename conflict returned %d: %s", response.Code, response.Body.String()) } response = do(app, http.MethodPut, "/api/gateways/missing", `{"name":"gw-new","endpoint":"http://gw-new:8081","token":""}`) if response.Code != http.StatusNotFound { t.Fatalf("missing gateway update returned %d: %s", response.Code, response.Body.String()) } } func TestListGatewaysReturnsStoredRecordsWithoutProbing(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token"} server := httptest.NewServer(gateway.handler(t)) t.Cleanup(server.Close) stored := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} store.gateways[stored.Name] = stored app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodGet, "/api/gateways", "") if response.Code != http.StatusOK { t.Fatalf("list gateways returned %d: %s", response.Code, response.Body.String()) } var gateways []hub.Gateway if err := json.Unmarshal(response.Body.Bytes(), &gateways); err != nil || len(gateways) != 1 || gateways[0] != stored { t.Fatalf("stored gateway was not returned unchanged: gateways=%#v err=%v", gateways, err) } if strings.Contains(response.Body.String(), "connectivity") || strings.Contains(response.Body.String(), "health") { t.Fatalf("gateway list exposed live status fields: %s", response.Body.String()) } if requests := gateway.recorded(); len(requests) != 0 { t.Fatalf("gateway list performed live probes: %#v", requests) } } func TestParseGatewayBrowserListStrict(t *testing.T) { for _, test := range []struct { name string body string ok bool }{ {name: "empty", body: `[]`, ok: true}, {name: "minimal browser", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a","state":"running"}]`, ok: true}, {name: "top-level null", body: `null`}, {name: "null element", body: `[null]`}, {name: "missing id", body: `[{"alias":"account-a","state":"running"}]`}, {name: "missing alias", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"running"}]`}, {name: "missing state", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a"}]`}, {name: "duplicate alias", body: `[{"id":"one","alias":"account-a","state":"running"},{"id":"two","alias":"account-a","state":"exited"}]`}, } { t.Run(test.name, func(t *testing.T) { browsers, err := parseGatewayBrowserList([]byte(test.body)) if (err == nil) != test.ok { t.Fatalf("parse result browsers=%#v err=%v", browsers, err) } }) } } func TestGatewayCallPropagatesBodyReadError(t *testing.T) { gateway := &fakeGateway{token: "unit-test-gateway-token", readErrorList: 1} server := httptest.NewServer(gateway.handler(t)) defer server.Close() status, body, err := gatewayCall(context.Background(), hub.Gateway{Endpoint: server.URL, Token: gateway.token}, http.MethodGet, "/v1/browsers", nil, time.Second) if status != http.StatusOK || string(body) != "[]" || err == nil { t.Fatalf("partial response was accepted: status=%d body=%q err=%v", status, body, err) } } func TestGatewayCreatePayloadStripsLegacyProxyFingerprint(t *testing.T) { payload := gatewayCreatePayload(hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Name: "甲", Fingerprint: hub.Fingerprint{ Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true, }}, BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1"}}, "/opt/creatorhub/browsers/1", gatewayNetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}) encoded, _ := json.Marshal(payload) if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") { t.Fatalf("legacy proxy URI entered the gateway contract: %s", encoded) } } func TestCreateBrowserSupportsDirectMachineExit(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token"} server := httptest.NewServer(gateway.handler(t)) defer server.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{failure: "must_not_probe"}, func(hub.NetworkExitAccess) (string, error) { return "", errors.New("must not resolve direct exit credentials") }) body := `{"alias":"direct-env","name":"直连环境","gateway":"gw-1",` + `"fingerprint":{"seed":2024},"account_id":"account-a","network_exit_id":""}` response := do(app, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusCreated { t.Fatalf("expected direct create 201, got %d: %s", response.Code, response.Body.String()) } bound := store.bindings["direct-env"] if bound.Exit.ID != "" || bound.RuntimeID == "" { t.Fatalf("direct binding was not activated: %#v", bound) } requests := gateway.recorded() if len(requests) != 1 || requests[0].body["network_exit_id"] != "" { t.Fatalf("direct create sent an unexpected gateway payload: %#v", requests) } exit, ok := requests[0].body["network_exit"].(map[string]any) if !ok || exit["protocol"] != "" || exit["host"] != "" || exit["port"] != float64(0) { t.Fatalf("direct create must not send proxy settings: %#v", requests[0].body) } } func TestExitFailuresStopCreateBeforeGateway(t *testing.T) { for _, test := range []struct { name string probe fakeExitProbe reason string }{ {name: "identity drift", probe: fakeExitProbe{observation: hub.ExitObservation{PublicIP: "203.0.113.11", Region: "test"}}, reason: "exit_ip_drift"}, {name: "authentication rejected", probe: fakeExitProbe{failure: "proxy_auth_failed"}, reason: "proxy_auth_failed"}, {name: "proxy disconnected", probe: fakeExitProbe{failure: "proxy_check_failed"}, reason: "proxy_check_failed"}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.exits["exit-1"] = hub.NetworkExit{ ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, ExpectedPublicIP: "203.0.113.10", HealthStatus: "healthy", Version: 1, } gateway := &fakeGateway{token: "unit-test-gateway-token"} server := httptest.NewServer(gateway.handler(t)) defer server.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} app := fiber.New() registerHubWithNetwork(app, store, test.probe, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusConflict { t.Fatalf("expected fail-close 409, got %d: %s", response.Code, response.Body.String()) } if len(gateway.recorded()) != 0 { t.Fatalf("exit failure must stop before any gateway start/create call: %#v", gateway.recorded()) } bound, err := store.GetEnvironmentContext(context.Background(), "account-a") if err != nil || bound.RuntimeInstanceID != "" || store.exits["exit-1"].HealthStatus != "unhealthy" { t.Fatalf("failed create must retain only the stable inactive binding: %#v err=%v", bound, err) } if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != test.reason { t.Fatalf("exit failure must be auditable without secrets: %#v", store.actions) } }) } } func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusConflict { t.Fatalf("expected gateway conflict to pass through as 409, got %d: %s", response.Code, response.Body.String()) } if _, kept := store.envs["account-a"]; !kept { t.Fatal("stable environment and Profile anchor must remain retryable") } } func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadGateway} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusBadGateway { t.Fatalf("expected failed gateway create, got %d: %s", response.Code, response.Body.String()) } requests := gateway.recorded() if len(requests) != 1 || requests[0].method != http.MethodPost { t.Fatalf("unknown network generation was discovered or cleaned by mutable alias: %#v", requests) } environment, err := store.GetEnvironmentContext(context.Background(), "account-a") if err != nil || !environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "" { t.Fatalf("unknown network generation was not retained for manual reconciliation: environment=%#v err=%v", environment, err) } } func TestCreateBrowserDeterministicRejectionDoesNotWedgeBinding(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadRequest} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "alias already in use") { t.Fatalf("deterministic gateway rejection must surface its reason, got %d: %s", response.Code, response.Body.String()) } environment, err := store.GetEnvironmentContext(context.Background(), "account-a") if err != nil || environment.RuntimeCleanupPending { t.Fatalf("deterministic rejection wedged the binding for manual reconciliation: environment=%#v err=%v", environment, err) } if requests := gateway.recorded(); len(requests) != 1 || requests[0].method != http.MethodPost { t.Fatalf("deterministic rejection must not reconcile or mutate the gateway: %#v", requests) } } func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"} store.bindings[environment.Alias] = environment gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-99999999999999999999999999999999", ProxyReady: true, }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} err := reconcileGatewayCreate(context.Background(), store, target, environment, http.StatusBadGateway, nil, []byte(`{"error":"result unknown"}`)) after := store.bindings[environment.Alias] if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.runtimes) != 1 || !after.RuntimeCleanupPending || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" { t.Fatalf("unknown create reused the old generation: err=%v requests=%#v runtimes=%#v environment=%#v", err, gateway.recorded(), gateway.runtimes, after) } } func TestValidCreatedRuntimeGeneration(t *testing.T) { environment := hub.EnvironmentContext{RuntimeNetworkID: "native-99999999999999999999999999999999"} for _, test := range []struct { name string created runtimeStatus environment hub.EnvironmentContext running bool want bool }{ {name: "active matching generation", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-99999999999999999999999999999999"}, running: true, want: true}, {name: "active empty runtime", created: runtimeStatus{NetworkID: "native-99999999999999999999999999999999"}, running: true}, {name: "active invalid runtime", created: runtimeStatus{ID: "container c1", NetworkID: "native-99999999999999999999999999999999"}, running: true}, {name: "active empty network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, running: true}, {name: "active invalid network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "network n1"}, running: true}, {name: "active replacement network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-88888888888888888888888888888888"}, running: true}, {name: "active existing matching generation", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-99999999999999999999999999999999"}, running: true, want: true, environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}}, {name: "active existing successor container", created: runtimeStatus{ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", NetworkID: "native-99999999999999999999999999999999"}, running: true, environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}}, {name: "active legacy lease without network", created: runtimeStatus{ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", NetworkID: "native-88888888888888888888888888888888"}, running: true, environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}}, {name: "stopped empty network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, want: true}, {name: "stopped invalid network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "network n1"}}, } { t.Run(test.name, func(t *testing.T) { current := environment if test.environment.RuntimeInstanceID != "" { current = test.environment } if got := validCreatedRuntime(test.created, current, test.running); got != test.want { t.Fatalf("validCreatedRuntime(%#v, running=%v)=%v, want %v", test.created, test.running, got, test.want) } }) } } type activationConflictStore struct { *memoryStore current hub.EnvironmentContext } func (s *activationConflictStore) GetEnvironmentContext(_ context.Context, _ string) (hub.EnvironmentContext, error) { return s.current, nil } func (s *activationConflictStore) ActivateRuntime(context.Context, string, string, int64, string, ...string) (hub.EnvironmentContext, error) { return hub.EnvironmentContext{}, hub.ErrConflict } func TestActivationConflictKeepsTheGenerationWonByTheHeartbeat(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} candidateID := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" candidateNetwork := "native-88888888888888888888888888888888" current := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, RuntimeInstanceID: "runtime-instance-c1", RuntimeID: candidateID, RuntimeNetworkID: candidateNetwork, Exit: store.exits["exit-1"], } store.bindings["account-a"] = current conflict := &activationConflictStore{memoryStore: store, current: current} got, err := activateGatewayRuntime(context.Background(), conflict, hub.Gateway{}, current, candidateID, candidateNetwork) if err != nil { t.Fatalf("activation race was reported as failure: %v", err) } if got.RuntimeID != candidateID || got.RuntimeNetworkID != candidateNetwork || got.RuntimeInstanceID == "" { t.Fatalf("heartbeat winner was not returned: %#v", got) } } func TestLegacyActiveRuntimeWithoutNetworkGenerationDoesNotTouchSuccessor(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} store.bindings[environment.Alias] = environment gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-88888888888888888888888888888888", ProxyReady: true, }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} if runtimeMatchesBinding(gateway.runtimes[0], environment) { t.Fatal("legacy active lease accepted a successor network generation") } if _, err := activateGatewayRuntime(context.Background(), store, target, environment, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "native-88888888888888888888888888888888"); !errors.Is(err, hub.ErrConflict) { t.Fatalf("successor activation did not fail closed: %v", err) } if _, err := removeGatewayRuntime(context.Background(), store, target, environment); !errors.Is(err, hub.ErrConflict) { t.Fatalf("legacy cleanup did not fence the successor: %v", err) } after := store.bindings[environment.Alias] requests := gateway.recorded() if len(requests) != 1 || requests[0].method != http.MethodGet || len(gateway.runtimes) != 1 || after.RuntimeID != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || after.RuntimeNetworkID != "" || after.RuntimeCleanupPending { t.Fatalf("legacy C1 cleanup touched C2/N2: requests=%#v runtimes=%#v environment=%#v", requests, gateway.runtimes, after) } } func TestCreateBrowserInvalid201TracksNetworkCleanupGeneration(t *testing.T) { for _, networkID := range []string{"native-99999999999999999999999999999999", ""} { name := "known network" if networkID == "" { name = "unknown network" } t.Run(name, func(t *testing.T) { var deleteBody map[string]any gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" { response.WriteHeader(http.StatusUnauthorized) return } switch request.Method { case http.MethodPost: response.WriteHeader(http.StatusCreated) _ = json.NewEncoder(response).Encode(map[string]string{"network_id": networkID}) case http.MethodGet: _, _ = response.Write([]byte(`[]`)) case http.MethodDelete: _ = json.NewDecoder(request.Body).Decode(&deleteBody) if networkID == "" { response.WriteHeader(http.StatusConflict) return } response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path) } })) defer gatewayServer.Close() store := newMemoryStore() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusBadGateway || (networkID != "" && (deleteBody["runtime_id"] != missingRuntimeID || deleteBody["network_id"] != networkID)) { t.Fatalf("invalid 201 did not reconcile its generation: status=%d delete=%v body=%s", response.Code, deleteBody, response.Body.String()) } environment := store.bindings["account-a"] if networkID != "" && environment.RuntimeCleanupPending { t.Fatalf("known network cleanup did not converge: %#v", environment) } if networkID == "" && (!environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "") { t.Fatalf("unknown generation was not retained for manual reconciliation: %#v", environment) } }) } } func TestCreateBrowserRejectsActiveRuntimeWithoutNetworkGeneration(t *testing.T) { requests := 0 gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { requests++ if request.Method != http.MethodPost { t.Fatalf("active NULL network response triggered alias reconciliation: %s %s", request.Method, request.URL.Path) } response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","network_id":""}`)) })) defer gatewayServer.Close() store := newMemoryStore() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) environment := store.bindings["account-a"] if response.Code != http.StatusBadGateway || requests != 1 || !environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "" || environment.RuntimeID != "" { t.Fatalf("active NULL network generation was activated: status=%d requests=%d environment=%#v body=%s", response.Code, requests, environment, response.Body.String()) } } func TestCreateBrowserDoesNotDiscoverDisconnectedCreateByAlias(t *testing.T) { for _, test := range []struct { name string reconcile bool wantStatus int }{ {name: "completed create", reconcile: true, wantStatus: http.StatusBadGateway}, {name: "unknown result", wantStatus: http.StatusBadGateway}, } { t.Run(test.name, func(t *testing.T) { var created bool var mu sync.Mutex gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" { response.WriteHeader(http.StatusUnauthorized) return } if request.Method == http.MethodGet && test.reconcile { mu.Lock() exists := created mu.Unlock() if exists { _ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) return } _ = json.NewEncoder(response).Encode([]runtimeStatus{}) return } if request.Method == http.MethodPost { mu.Lock() created = true mu.Unlock() } hijacker, ok := response.(http.Hijacker) if !ok { t.Error("test server does not support hijacking") return } connection, _, err := hijacker.Hijack() if err != nil { t.Errorf("hijack gateway response: %v", err) return } _ = connection.Close() })) defer gatewayServer.Close() store := newMemoryStore() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != test.wantStatus { t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) } if _, err := store.GetEnv(context.Background(), "account-a"); err != nil { t.Fatalf("unknown gateway result must retain the environment for reconciliation: %v", err) } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" { t.Fatalf("gateway result without an immutable generation must be unknown: %#v", store.actions) } mu.Lock() wasCreated := created mu.Unlock() if !wasCreated || store.bindings["account-a"].RuntimeCleanupNetworkID != "" { t.Fatalf("disconnected create lost its manual reconcile marker: %#v", store.bindings["account-a"]) } }) } } func TestCreateBrowserDoesNotDiscoverBadGatewayCreateByAlias(t *testing.T) { for _, test := range []struct { name string missingReads int }{ {name: "Docker create disconnect returned 502"}, {name: "container becomes visible after query window", missingReads: 2}, } { t.Run(test.name, func(t *testing.T) { var mu sync.Mutex listCalls := 0 gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" { response.WriteHeader(http.StatusUnauthorized) return } if request.Method == http.MethodPost { response.WriteHeader(http.StatusBadGateway) _, _ = response.Write([]byte(`{"error":"create container: context deadline exceeded"}`)) return } mu.Lock() listCalls++ missing := listCalls <= test.missingReads mu.Unlock() if missing { _ = json.NewEncoder(response).Encode([]runtimeStatus{}) return } _ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) })) defer gatewayServer.Close() store := newMemoryStore() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusBadGateway { t.Fatalf("expected unknown create, status=%d body=%s", response.Code, response.Body.String()) } if _, err := store.GetEnv(context.Background(), "account-a"); err != nil { t.Fatalf("reconciled create must retain DB state: err=%v", err) } mu.Lock() gotCalls := listCalls mu.Unlock() if gotCalls != 0 { t.Fatalf("unknown network generation was inspected by alias %d times", gotCalls) } }) } } func TestCreateBrowserRejectsInvalidFingerprintBeforeSideEffects(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers", `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":0}}`) if response.Code != http.StatusBadRequest { t.Fatalf("expected 400 for invalid fingerprint, got %d: %s", response.Code, response.Body.String()) } if len(gateway.recorded()) != 0 || len(store.envs) != 0 { t.Fatal("invalid input must not reach the gateway or the store") } } func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 3, Exit: store.exits["exit-1"], } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", NetworkID: "native-cccccccccccccccccccccccccccccccc", ProxyReady: false, }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("gateway restart recovery failed: %v", err) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("runtime was not activated after proxy recovery: %q", runtime) } requests := gateway.recorded() if len(requests) != 2 || requests[0].path != "/v1/browsers" || requests[1].path != "/v1/browsers/account-a/proxy" { t.Fatalf("expected list then proxy recovery without rebuild: %#v", requests) } if requests[1].body["binding_version"] != float64(3) || requests[1].body["runtime_id"] != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || requests[1].body["network_exit_id"] != "exit-1" { t.Fatalf("proxy recovery did not use the current binding: %#v", requests[1].body) } } func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"]} gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666", }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); !errors.Is(err, hub.ErrConflict) { t.Fatalf("lease-free running orphan was not fenced: %v", err) } requests := gateway.recorded() if len(requests) != 3 || requests[1].path != "/v1/browsers/account-a/proxy" { t.Fatalf("unproven orphan generation reached rebuild side effects: %#v", requests) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { t.Fatalf("unproven orphan generation was activated: %q", runtime) } } type postgresRebindFixture struct { store *hub.Store db *sql.DB databaseURL string gateway *fakeGateway bound hub.EnvironmentContext exit hub.NetworkExit } type cleanupCommitUnknownStore struct { hubStore } type failingRuntimeReleaseStore struct { *hub.Store err error } func (s failingRuntimeReleaseStore) ReleaseRuntime(context.Context, hub.EnvironmentContext) error { return s.err } func setFixtureAccountStatus(t *testing.T, databaseURL, status string) { t.Helper() db, err := sql.Open("pgx", databaseURL) if err != nil { t.Fatal(err) } defer db.Close() if _, err := db.Exec(`UPDATE social_account SET status = $1 WHERE id = 'account-a'`, status); err != nil { t.Fatal(err) } } func (s cleanupCommitUnknownStore) SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error { if err := s.hubStore.SetRuntimeCleanupPending(ctx, environment, pending); err != nil { return err } return errors.New("cleanup commit result unknown") } type failContextRefreshStore struct { hubStore reads int } func (s *failContextRefreshStore) GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) { s.reads++ if s.reads >= 5 { return hub.EnvironmentContext{}, errors.New("context refresh unavailable") } return s.hubStore.GetEnvironmentContext(ctx, alias) } func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFixture { t.Helper() ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) accountStore, err := accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } if err := accountStore.Close(); err != nil { t.Fatal(err) } store, err := hub.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = store.Close() }) accountStore, err = accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } if err := accountStore.CreateAccount(ctx, accountdomain.Account{ ID: "account-a", Name: "账号 A", Platform: "douyin", PlatformAccountKey: "account-a", Tags: []string{}, Cookies: "sessionid=value", CredentialReference: accountdomain.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-a/cookies", }, &testCredentialBridge{values: map[string]string{}}); err != nil { t.Fatal(err) } if err := accountStore.Close(); err != nil { t.Fatal(err) } gateway := &fakeGateway{token: "unit-test-gateway-token"} gatewayServer := httptest.NewServer(gateway.handler(t)) t.Cleanup(gatewayServer.Close) if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil { t.Fatal(err) } exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080}) if err != nil { t.Fatal(err) } exit, _, err = store.RecordNetworkExitCheck(ctx, exit.ID, hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "") if err != nil { t.Fatal(err) } bound, created, err := store.CreateBoundEnv(ctx, hub.Env{ Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}, }, "account-a", exit.ID) if err != nil || !created { t.Fatalf("create bound environment: created=%v err=%v", created, err) } db, err := sql.Open("pgx", databaseURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = db.Close() }) return postgresRebindFixture{store: store, db: db, databaseURL: databaseURL, gateway: gateway, bound: bound, exit: exit} } func TestPauseClosesClaimGateBeforeStoppingRuntime(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } for _, test := range []struct { name string gatewayEndpoint string wantStatus int wantOutcome string wantReason string }{ {name: "confirmed stop", wantStatus: http.StatusNoContent, wantOutcome: "succeeded", wantReason: "environment_stopped"}, {name: "gateway unreachable", gatewayEndpoint: "http://127.0.0.1:1", wantStatus: http.StatusBadGateway, wantOutcome: "unknown", wantReason: "gateway_result_unknown"}, } { t.Run(test.name, func(t *testing.T) { ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) accountStore, err := accountdomain.Open(ctx, fixture.databaseURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = accountStore.Close() }) if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil { t.Fatal(err) } account, err := accountStore.GetAccount(ctx, "account-a") if err != nil { t.Fatal(err) } fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "active-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-33333333333333333333333333333333") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "active-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} if test.gatewayEndpoint != "" { if _, err := fixture.db.ExecContext(ctx, `UPDATE gateway SET endpoint = $1 WHERE name = 'gw-1'`, test.gatewayEndpoint); err != nil { t.Fatal(err) } } if err := accountStore.CreateDraft(ctx, accountdomain.Draft{ID: "pause-draft", AccountID: "account-a", Version: 1, Content: "pause"}); err != nil { t.Fatal(err) } if err := accountStore.Confirm(ctx, accountdomain.Confirmation{ ID: "pause-confirmation", AccountID: "account-a", AccountVersion: account.Version, DraftID: "pause-draft", DraftVersion: 1, Version: 1, }); err != nil { t.Fatal(err) } if _, inserted, err := accountStore.Enqueue(ctx, accountdomain.Task{ ID: "pause-task", IdempotencyKey: "pause-idempotency", AccountID: "account-a", AccountVersion: account.Version, DraftID: "pause-draft", DraftVersion: 1, ConfirmationID: "pause-confirmation", ConfirmationVersion: 1, }); err != nil || !inserted { t.Fatalf("enqueue pause task: inserted=%v err=%v", inserted, err) } if _, err := fixture.db.ExecContext(ctx, ` INSERT INTO execution_attempt (id, task_id) VALUES ('pause-attempt', 'pause-task'); UPDATE operation_task SET state = 'executing', current_attempt_id = 'pause-attempt', lease_owner = 'worker-pause', lease_until = now() + interval '1 minute' WHERE id = 'pause-task'`); err != nil { t.Fatal(err) } app := fiber.New() registerPhaseA(app, accountStore, fixture.store, nil) response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", "") if response.Code != test.wantStatus { t.Fatalf("pause returned %d, want %d: %s", response.Code, test.wantStatus, response.Body.String()) } paused, err := accountStore.GetAccount(ctx, "account-a") if err != nil || paused.RuntimeStatus != "paused" { t.Fatalf("claim gate remained open: account=%#v err=%v", paused, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM operation_task WHERE id = 'pause-task' AND state = 'needs_confirmation'`, 1) assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM execution_attempt WHERE id = 'pause-attempt' AND outcome = 'uncertain'`, 1) assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM audit_event WHERE task_id = 'pause-task' AND attempt_id = 'pause-attempt' AND browser_env_alias = 'account-a' AND network_exit_id IS NOT NULL AND runtime_instance_id IS NOT NULL AND binding_version = 1 AND reason_code = 'account_paused'`, 1) var outcome, reason string if err := fixture.db.QueryRowContext(ctx, `SELECT outcome, reason_code FROM audit_event WHERE account_id = 'account-a' AND action = 'stop' AND event_type = 'environment_action_finished' ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil || outcome != test.wantOutcome || reason != test.wantReason { t.Fatalf("stop audit: outcome=%q reason=%q err=%v", outcome, reason, err) } }) } } func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) accountStore, err := accountdomain.Open(ctx, fixture.databaseURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = accountStore.Close() }) fixture.gateway.runtimes = []runtimeStatus{{ ID: "stopped-container", Alias: fixture.bound.Alias, State: "exited", BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} app := fiber.New() registerPhaseA(app, accountStore, fixture.store, nil) if err := accountStore.ResumeAccount(ctx, fixture.bound.AccountID); err != nil { t.Fatal(err) } if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", ""); response.Code != http.StatusNoContent { t.Fatalf("pause stopped container without lease: %d %s", response.Code, response.Body.String()) } if err := accountStore.ResumeAccount(ctx, fixture.bound.AccountID); err != nil { t.Fatal(err) } fixture.gateway.mu.Lock() fixture.gateway.runtimes[0].State = "running" fixture.gateway.mu.Unlock() if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/revoke", ""); response.Code != http.StatusNoContent { t.Fatalf("revoke running orphan without lease: %d %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, fixture.bound.Alias) if err != nil || after.RuntimeID != "" || after.RuntimeCleanupPending { t.Fatalf("lease-free stop did not converge: %#v err=%v", after, err) } account, err := accountStore.GetAccount(ctx, fixture.bound.AccountID) if err != nil || account.AuthorizationStatus != "revoked" || account.RuntimeStatus != "paused" { t.Fatalf("revoke gate did not remain closed: %#v err=%v", account, err) } fixture.gateway.mu.Lock() container := fixture.gateway.runtimes[0] fixture.gateway.mu.Unlock() if container.State != "exited" { t.Fatalf("running orphan was not stopped: %#v", container) } requests := fixture.gateway.recorded() stopCalls := 0 for _, request := range requests { if request.method == http.MethodPost && request.path == "/v1/browsers/account-a/stop" { stopCalls++ if request.body["runtime_id"] != "stopped-container" || request.body["binding_version"] != float64(fixture.bound.BindingVersion) { t.Fatalf("stop used the wrong discovered generation: %#v", request) } } } if stopCalls != 2 { t.Fatalf("pause/revoke did not reconcile both lease-free runtimes: %#v", requests) } } func installCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB, condition string) { t.Helper() if _, err := db.ExecContext(ctx, ` CREATE FUNCTION fail_cleanup_transition() RETURNS trigger AS $$ BEGIN IF `+condition+` THEN RAISE EXCEPTION 'injected cleanup state failure'; END IF; RETURN NEW; END $$ LANGUAGE plpgsql`); err != nil { t.Fatal(err) } if _, err := db.ExecContext(ctx, ` CREATE TRIGGER fail_cleanup_transition BEFORE UPDATE OF runtime_cleanup_pending ON environment_binding FOR EACH ROW EXECUTE FUNCTION fail_cleanup_transition()`); err != nil { t.Fatal(err) } } func dropCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB) { t.Helper() if _, err := db.ExecContext(ctx, `DROP TRIGGER fail_cleanup_transition ON environment_binding`); err != nil { t.Fatal(err) } if _, err := db.ExecContext(ctx, `DROP FUNCTION fail_cleanup_transition()`); err != nil { t.Fatal(err) } } func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() for _, test := range []struct { name string cleanupPending int disconnectDelete int disconnectListAfterDelete bool }{ {name: "accepted cleanup pending", cleanupPending: 2}, {name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true}, } { t.Run(test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} fixture.gateway.cleanupPending = test.cleanupPending fixture.gateway.disconnectDelete = test.disconnectDelete fixture.gateway.disconnectListAfterDelete = test.disconnectListAfterDelete app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) target, err := fixture.store.GetGateway(ctx, "gw-1") if err != nil { t.Fatal(err) } removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) if !removed || cleanupErr == nil { t.Fatalf("non-final cleanup lost the removed fact: removed=%v err=%v", removed, cleanupErr) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || !after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion || after.Exit.ID != fixture.bound.Exit.ID || after.RuntimeID != "" { t.Fatalf("PostgreSQL did not persist cleanup pending atomically: %#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM environment_binding WHERE browser_env_alias = 'account-a' AND runtime_cleanup_pending`, 1) assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) if err := reconcileRuntimeLeases(ctx, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile did not confirm cleanup: %v", err) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending { t.Fatalf("confirmed cleanup remained pending: %#v err=%v", after, err) } setFixtureAccountStatus(t, fixture.databaseURL, "paused") response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+fixture.exit.ID+`"}`) if response.Code != http.StatusOK { t.Fatalf("rebind retry failed after cleanup confirmation: %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion+1 || after.RuntimeID != "" { t.Fatalf("retry did not commit a consistent stopped generation: %#v err=%v", after, err) } }) } } func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { for _, test := range []struct { name string commitKnown bool outcome string reason string }{ {name: "rollback", commitKnown: true, outcome: "failed", reason: "runtime_persistence_failed"}, {name: "commit unknown", outcome: "unknown", reason: "cleanup_result_unknown"}, } { t.Run("restore "+test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } store.cleanupPendingErr = errors.New("cleanup state unavailable") store.cleanupPendingErrAfterMutation = !test.commitKnown gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }}} app := newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil { t.Fatal("failed cleanup mark unexpectedly succeeded") } after := store.bindings["account-a"] if test.commitKnown { if after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("rollback changed the old generation: %#v", after) } } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("commit-unknown lost its retry marker: %#v", after) } if len(gateway.runtimes) != 1 { t.Fatalf("failed cleanup mark touched the old container: runtimes=%#v requests=%#v", gateway.runtimes, gateway.recorded()) } for _, request := range gateway.recorded() { if request.method != http.MethodGet { t.Fatalf("failed cleanup mark reached a gateway side effect: %#v", gateway.recorded()) } } if len(store.actions) != 2 || store.actions[1].Outcome != test.outcome || store.actions[1].ReasonCode != test.reason { t.Fatalf("restore audit mismatch: %#v", store.actions) } store.cleanupPendingErr = nil store.cleanupPendingErrAfterMutation = false if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile retry failed: %v", err) } if test.commitKnown { after = store.bindings["account-a"] } else { if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { t.Fatalf("start retry failed: %d: %s", response.Code, response.Body.String()) } after = store.bindings["account-a"] } if after.RuntimeCleanupPending || after.RuntimeID == "" || len(gateway.runtimes) != 1 || !runtimeMatchesBinding(gateway.runtimes[0], after) { t.Fatalf("retry ended inconsistently: after=%#v runtimes=%#v", after, gateway.runtimes) } }) t.Run("discard "+test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } store.cleanupPendingErr = errors.New("cleanup state unavailable") store.cleanupPendingErrAfterMutation = !test.commitKnown gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", }}} app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers/account-a/start", "") if response.Code != http.StatusInternalServerError { t.Fatalf("failed discard mark returned %d: %s", response.Code, response.Body.String()) } after := store.bindings["account-a"] if test.commitKnown { if after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("discard rollback changed the old generation: %#v", after) } } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("discard commit-unknown lost its retry marker: %#v", after) } if len(gateway.runtimes) != 1 { t.Fatalf("failed discard mark touched the gateway: runtimes=%#v requests=%#v", gateway.runtimes, gateway.recorded()) } for _, request := range gateway.recorded() { if request.method != http.MethodGet { t.Fatalf("failed discard mark reached a gateway side effect: %#v", gateway.recorded()) } } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" { t.Fatalf("discard audit mismatch: %#v", store.actions) } store.cleanupPendingErr = nil store.cleanupPendingErrAfterMutation = false response = do(app, http.MethodPost, "/api/browsers/account-a/start", "") if response.Code != http.StatusConflict { t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String()) } after = store.bindings["account-a"] if after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.runtimes) != 0 { t.Fatalf("discard retry ended inconsistently: after=%#v runtimes=%#v", after, gateway.runtimes) } if len(store.actions) != 4 || store.actions[3].Outcome != "failed" || store.actions[3].ReasonCode != "proxy_auth_failed" { t.Fatalf("discard retry audit mismatch: %#v", store.actions) } }) } } func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666", }}} app := newTestApp(t, store, gateway) body := `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}` response := do(app, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusBadGateway { t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String()) } if after := store.bindings["account-a"]; after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || after.RuntimeCleanupPending { t.Fatalf("gateway unknown changed the existing lease: %#v", after) } for _, request := range gateway.recorded() { if request.method == http.MethodPost { t.Fatalf("gateway unknown created a duplicate container: %#v", gateway.recorded()) } } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("gateway unknown create audit mismatch: %#v", store.actions) } response = do(app, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusOK { t.Fatalf("create retry did not reuse the runtime: %d: %s", response.Code, response.Body.String()) } if after := store.bindings["account-a"]; after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || after.RuntimeCleanupPending { t.Fatalf("create retry changed the matching generation: %#v", after) } if len(gateway.runtimes) != 1 || len(store.actions) != 4 || store.actions[3].Outcome != "succeeded" || store.actions[3].ReasonCode != "environment_reused" { t.Fatalf("create retry ended inconsistently: runtimes=%#v actions=%#v", gateway.runtimes, store.actions) } } func TestRemoveGatewayRuntimePreservesKnownNetworkGeneration(t *testing.T) { for _, test := range []struct { name string observedNetwork string wantConflict bool }{ {name: "legacy gateway omits network", observedNetwork: ""}, {name: "replacement network conflicts", observedNetwork: "native-88888888888888888888888888888888", wantConflict: true}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"} environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"} store.bindings["account-a"] = environment gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: test.observedNetwork, ProxyReady: true}}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} _, err := removeGatewayRuntime(context.Background(), store, target, environment) if test.wantConflict { if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 1 || store.bindings["account-a"].RuntimeID != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { t.Fatalf("replacement network was not fenced: err=%v requests=%#v environment=%#v", err, gateway.recorded(), store.bindings["account-a"]) } return } requests := gateway.recorded() if err != nil || len(requests) != 2 || requests[1].body["network_id"] != "native-99999999999999999999999999999999" || store.bindings["account-a"].RuntimeCleanupPending { t.Fatalf("known N1 was not preserved through cleanup: err=%v requests=%#v environment=%#v", err, requests, store.bindings["account-a"]) } }) } } func TestGatewayLookupFailurePreservesLease(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } store.bindings[environment.Alias] = environment store.gatewayFn = func(string) (hub.Gateway, error) { return hub.Gateway{}, errors.New("gateway lookup unavailable") } if _, err := restoreOrRebuildRuntime(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }, environment, runtimeStatus{ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: environment.Alias, State: "running"}); err == nil { t.Fatal("restore accepted an unknown gateway") } if err := discardRuntime(context.Background(), store, environment); err == nil { t.Fatal("discard accepted an unknown gateway") } if after := store.bindings[environment.Alias]; after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("gateway lookup failure changed the old generation: %#v", after) } } func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() t.Run("pre-mark rollback skips DELETE", func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) target, err := fixture.store.GetGateway(ctx, "gw-1") if err != nil { t.Fatal(err) } removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) if cleanupErr == nil || removed || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet { t.Fatalf("rolled-back pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, cleanupErr, fixture.gateway.recorded()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { t.Fatalf("PostgreSQL rollback did not preserve the active generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) dropCleanupTransitionFailure(t, ctx, fixture.db) fixture.gateway.cleanupPending = 2 removed, cleanupErr = removeGatewayRuntime(ctx, fixture.store, target, after) if cleanupErr == nil || !removed { t.Fatalf("202 retry did not retain pending cleanup: removed=%v err=%v", removed, cleanupErr) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.runtimes) != 0 { t.Fatalf("202 retry was not durable: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) }) t.Run("clear rollback remains pending and lifecycle retry recovers", func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} fixture.gateway.deleteNotFound = 1 installCleanupTransitionFailure(t, ctx, fixture.db, `OLD.runtime_cleanup_pending AND NOT NEW.runtime_cleanup_pending`) target, err := fixture.store.GetGateway(ctx, "gw-1") if err != nil { t.Fatal(err) } removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) if cleanupErr == nil || !removed { t.Fatalf("404 clear rollback lost the delete fact: removed=%v err=%v", removed, cleanupErr) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.runtimes) != 0 { t.Fatalf("clear rollback was not durably pending: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) dropCleanupTransitionFailure(t, ctx, fixture.db) setFixtureAccountStatus(t, fixture.databaseURL, "active") app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) if err := reconcileRuntimeLeases(context.Background(), fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile retry did not confirm cleanup: %v", err) } if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { t.Fatalf("lifecycle retry did not rebuild runtime: %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) { t.Fatalf("lifecycle retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } }) } func TestPostgresLegacyCleanupPendingRequiresManualReconcile(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) if _, err := fixture.db.ExecContext(ctx, ` UPDATE environment_binding SET runtime_cleanup_pending = true, runtime_cleanup_binding_version = NULL, runtime_cleanup_instance_id = NULL, runtime_cleanup_runtime_id = NULL WHERE id = $1`, fixture.bound.BindingID); err != nil { t.Fatalf("write legacy pending state: %v", err) } environment, err := fixture.store.GetEnvironmentContext(ctx, fixture.bound.Alias) if err != nil || !environment.RuntimeCleanupPending || environment.RuntimeCleanupBindingVersion != 0 || environment.RuntimeCleanupRuntimeID != "" { t.Fatalf("read legacy pending state: %#v err=%v", environment, err) } target, err := fixture.store.GetGateway(ctx, environment.Gateway) if err != nil { t.Fatal(err) } removed, err := removeGatewayRuntime(ctx, fixture.store, target, environment) if !errors.Is(err, hub.ErrReconcileRequired) || removed || len(fixture.gateway.recorded()) != 0 { t.Fatalf("legacy pending reached gateway mutation: removed=%v err=%v requests=%#v", removed, err, fixture.gateway.recorded()) } } func TestPostgresCleanupMissingContainerReleasesKnownLease(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") environment, err := fixture.store.ActivateRuntime(ctx, fixture.bound.Alias, "known-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-44444444444444444444444444444444") if err != nil { t.Fatal(err) } target, err := fixture.store.GetGateway(ctx, environment.Gateway) if err != nil { t.Fatal(err) } removed, err := removeGatewayRuntime(ctx, fixture.store, target, environment) if err != nil || !removed { t.Fatalf("cleanup missing gateway container: removed=%v err=%v", removed, err) } after, err := fixture.store.GetEnvironmentContext(ctx, environment.Alias) if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("known PostgreSQL lease did not converge: %#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) requests := fixture.gateway.recorded() if len(requests) == 0 || requests[len(requests)-1].method != http.MethodDelete || requests[len(requests)-1].body["runtime_id"] != "known-container" { t.Fatalf("cleanup did not preserve the known runtime generation: %#v", requests) } } func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() for _, test := range []struct { name string commitUnknown bool outcome string reason string }{ {name: "rollback", outcome: "failed", reason: "runtime_persistence_failed"}, {name: "commit unknown", commitUnknown: true, outcome: "unknown", reason: "cleanup_result_unknown"}, } { t.Run("restore "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} var store HubStore = fixture.store if test.commitUnknown { store = cleanupCommitUnknownStore{hubStore: fixture.store} } else { installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) } if err := reconcileRuntimeLeases(ctx, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil { t.Fatal("restore cleanup failure unexpectedly succeeded") } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if test.commitUnknown { if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("restore commit-unknown lost pending state: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) } else { if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("restore rollback changed the old generation: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } if len(fixture.gateway.runtimes) != 1 { t.Fatalf("restore cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded()) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { t.Fatalf("restore cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded()) } } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'reconcile' ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != test.outcome || reason != test.reason { t.Fatalf("restore audit mismatch: outcome=%s reason=%s", outcome, reason) } if !test.commitUnknown { dropCleanupTransitionFailure(t, ctx, fixture.db) } if err := reconcileRuntimeLeases(ctx, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("restore reconcile retry failed: %v", err) } if test.commitUnknown { retryApp := fiber.New() registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) if response := do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { t.Fatalf("restore start retry failed: %d: %s", response.Code, response.Body.String()) } } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) { t.Fatalf("restore retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) }) t.Run("create restore "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} var store HubStore = fixture.store if test.commitUnknown { store = cleanupCommitUnknownStore{hubStore: fixture.store} } else { installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) } app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) body := `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` response := do(app, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusInternalServerError { t.Fatalf("create restore cleanup failure returned %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if test.commitUnknown { if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("create restore commit-unknown lost pending state: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) } else { if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("create restore rollback changed the old generation: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } if len(fixture.gateway.runtimes) != 1 { t.Fatalf("create restore cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded()) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { t.Fatalf("create restore cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded()) } } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'create' ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != test.outcome || reason != test.reason { t.Fatalf("create restore audit mismatch: outcome=%s reason=%s", outcome, reason) } if !test.commitUnknown { dropCleanupTransitionFailure(t, ctx, fixture.db) } retryApp := fiber.New() registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response = do(retryApp, http.MethodPost, "/api/browsers", body) if response.Code != http.StatusOK && response.Code != http.StatusCreated { t.Fatalf("create restore retry failed: %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) { t.Fatalf("create restore retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) }) t.Run("discard "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} var store HubStore = fixture.store if test.commitUnknown { store = cleanupCommitUnknownStore{hubStore: fixture.store} } else { installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) } app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers/account-a/start", "") if response.Code != http.StatusInternalServerError { t.Fatalf("discard cleanup failure returned %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if test.commitUnknown { if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("discard commit-unknown lost pending state: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) } else { if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("discard rollback changed the old generation: after=%#v err=%v", after, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) } if len(fixture.gateway.runtimes) != 1 { t.Fatalf("discard cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded()) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { t.Fatalf("discard cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded()) } } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'start' ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != "unknown" || reason != "cleanup_result_unknown" { t.Fatalf("discard audit mismatch: outcome=%s reason=%s", outcome, reason) } if !test.commitUnknown { dropCleanupTransitionFailure(t, ctx, fixture.db) } if err := reconcileRuntimeLeases(ctx, fixture.store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("discard reconcile retry failed: %v", err) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("discard retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } for _, container := range fixture.gateway.runtimes { if container.State == "running" { t.Fatalf("discard retry retained a running container: %#v", fixture.gateway.runtimes) } } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) }) } } func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() for _, test := range []struct { name string set func(*fakeGateway, int) }{ {name: "disconnect", set: func(gateway *fakeGateway, count int) { gateway.disconnectList = count }}, {name: "status 500", set: func(gateway *fakeGateway, count int) { gateway.failList = count }}, {name: "invalid JSON", set: func(gateway *fakeGateway, count int) { gateway.invalidList = count }}, } { t.Run("list "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} test.set(fixture.gateway, 1) app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { t.Fatalf("gateway failure affected browser list: %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { t.Fatalf("browser list changed the generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) requests := fixture.gateway.recorded() if len(requests) != 1 || requests[0].method != http.MethodGet || requests[0].path != "/v1/browsers" { t.Fatalf("browser list did not perform the expected live gateway status read: %#v", requests) } }) t.Run("create "+test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} test.set(fixture.gateway, gatewayReconcileAttempts) app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) body := `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusBadGateway { t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { t.Fatalf("gateway unknown create changed the generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } for _, request := range fixture.gateway.recorded() { if request.method == http.MethodPost { t.Fatalf("gateway unknown create sent a duplicate create: %#v", fixture.gateway.recorded()) } } var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'create' ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != "unknown" || reason != "gateway_result_unknown" { t.Fatalf("gateway unknown create audit mismatch: outcome=%s reason=%s", outcome, reason) } if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusOK { t.Fatalf("gateway create retry failed: %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 { t.Fatalf("gateway create retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err) } if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = 'create' ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != "succeeded" || reason != "environment_reused" { t.Fatalf("gateway create retry audit mismatch: outcome=%s reason=%s", outcome, reason) } }) } } func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() for _, test := range []struct { name, method, path, action, successReason string configure func(*fakeGateway, int) active bool wantStatus int }{ {name: "create rejects null element", method: http.MethodPost, path: "/api/browsers", action: "create", active: true, wantStatus: http.StatusOK, successReason: "environment_reused", configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `[null]` }}, {name: "start rejects missing fields", method: http.MethodPost, path: "/api/browsers/account-a/start", action: "start", active: true, wantStatus: http.StatusNoContent, successReason: "gateway_reconciled", configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `[{"alias":"account-a","state":"running"}]` }}, {name: "rebind rejects partial body", method: http.MethodPost, path: "/api/browsers/account-a/rebind", action: "rebind", wantStatus: http.StatusOK, successReason: "environment_rebound", configure: func(gateway *fakeGateway, count int) { gateway.readErrorList = count }}, } { t.Run(test.name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) if test.action == "start" || test.action == "" { setFixtureAccountStatus(t, fixture.databaseURL, "active") } if test.active { setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } } state, networkExitID, networkID := "running", fixture.bound.Exit.ID, "native-66666666666666666666666666666666" if test.action == "rebind" { state, networkExitID, networkID = "exited", "", "" } fixture.gateway.runtimes = []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: state, ProxyReady: state == "running", BindingVersion: fixture.bound.BindingVersion, NetworkExitID: networkExitID, NetworkID: networkID, }} attempts := gatewayReconcileAttempts if test.action == "" { attempts = 1 } test.configure(fixture.gateway, attempts) app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) body := "" if test.action == "create" { body = `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` } else if test.action == "rebind" { body = `{"network_exit_id":"` + fixture.exit.ID + `"}` } response := do(app, test.method, test.path, body) if response.Code != http.StatusBadGateway { t.Fatalf("strict gateway validation returned %d: %s", response.Code, response.Body.String()) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || after.BindingVersion != fixture.bound.BindingVersion || after.Exit.ID != fixture.bound.Exit.ID || after.RuntimeCleanupPending || after.RuntimeID != fixture.bound.RuntimeID || len(fixture.gateway.runtimes) != 1 { t.Fatalf("strict gateway validation changed the generation: before=%#v after=%#v runtimes=%#v err=%v", fixture.bound, after, fixture.gateway.runtimes, err) } for _, request := range fixture.gateway.recorded() { if request.method != http.MethodGet { t.Fatalf("strict gateway validation advanced the lifecycle: %#v", fixture.gateway.recorded()) } } activeCount := 0 if test.active { activeCount = 1 } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeCount) if test.action != "" { var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = $1 ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != "unknown" || reason != "gateway_result_unknown" { t.Fatalf("strict gateway audit mismatch: outcome=%s reason=%s", outcome, reason) } } response = do(app, test.method, test.path, body) if response.Code != test.wantStatus { t.Fatalf("strict gateway retry returned %d: %s", response.Code, response.Body.String()) } after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") fixture.gateway.mu.Lock() runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...) fixture.gateway.mu.Unlock() wantVersion := fixture.bound.BindingVersion if test.action == "rebind" { wantVersion++ } validRuntime := after.RuntimeID != "" && len(runtimes) == 1 && runtimeMatchesBinding(runtimes[0], after) if test.action == "rebind" { validRuntime = after.RuntimeID == "" && len(runtimes) == 0 } if err != nil || after.BindingVersion != wantVersion || !validRuntime { t.Fatalf("strict gateway retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err) } activeAfter := 1 if test.action == "rebind" { activeAfter = 0 } assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeAfter) if test.action != "" { var outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT outcome, reason_code FROM audit_event WHERE action = $1 ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil { t.Fatal(err) } if outcome != "succeeded" || reason != test.successReason { t.Fatalf("strict gateway retry audit mismatch: outcome=%s reason=%s", outcome, reason) } } }) } } func isolatedControlPlaneDatabaseURL(t *testing.T, databaseURL string) string { t.Helper() admin, err := sql.Open("pgx", databaseURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = admin.Close() }) schema := fmt.Sprintf("creatorhub_hh803_%d", time.Now().UnixNano()) if _, err := admin.Exec("CREATE SCHEMA " + schema); err != nil { t.Fatal(err) } t.Cleanup(func() { if _, err := admin.Exec("DROP SCHEMA " + schema + " CASCADE"); err != nil { t.Errorf("drop test schema: %v", err) } }) parsed, err := url.Parse(databaseURL) if err != nil { t.Fatal(err) } query := parsed.Query() query.Set("search_path", schema) parsed.RawQuery = query.Encode() return parsed.String() } func assertControlPlaneDatabaseCount(t *testing.T, db *sql.DB, query string, want int) { t.Helper() var got int if err := db.QueryRow(query).Scan(&got); err != nil { t.Fatal(err) } if got != want { t.Fatalf("query %q returned %d, want %d", query, got, want) } } func TestPostgresRejectsInvalidLifecycleTargetsWithSanitizedAuditPairs(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } const secret = "http://operator:secret@proxy.example" for _, test := range []struct { action string body string reason string }{ {action: "rebind", body: `{"network_exit_id":"` + secret + `"}`, reason: "rebind_input_rejected"}, } { t.Run(test.action, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) app := fiber.New() registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers/account-a/"+test.action, test.body) if response.Code != http.StatusBadRequest { t.Fatalf("invalid %s returned %d: %s", test.action, response.Code, response.Body.String()) } var events, operations int var reason string if err := fixture.db.QueryRow(` SELECT count(*), count(DISTINCT operation_id), coalesce(max(reason_code) FILTER (WHERE event_type = 'environment_action_finished'), '') FROM audit_event WHERE action = $1`, test.action).Scan(&events, &operations, &reason); err != nil { t.Fatal(err) } if events != 2 || operations != 1 || reason != test.reason { t.Fatalf("invalid %s audit mismatch: events=%d operations=%d reason=%q", test.action, events, operations, reason) } var leaked int if err := fixture.db.QueryRow(`SELECT count(*) FROM audit_event WHERE row_to_json(audit_event)::text LIKE '%' || $1 || '%'`, secret).Scan(&leaked); err != nil { t.Fatal(err) } if leaked != 0 { t.Fatalf("invalid %s target leaked into audit", test.action) } }) } } func TestPostgresReconcileFinishedAuditUsesActivatedRuntime(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(context.Background(), "account-a", "stale-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666") if err != nil { t.Fatal(err) } fixture.gateway.runtimes = []runtimeStatus{{ ID: "stale-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }} if err := reconcileRuntimeLeases(context.Background(), fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile failed: %v", err) } after, err := fixture.store.GetEnvironmentContext(context.Background(), "account-a") if err != nil || after.RuntimeInstanceID == "" || after.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("reconcile did not activate replacement runtime: %#v err=%v", after, err) } var auditedRuntime string var auditedBinding int64 if err := fixture.db.QueryRow(` SELECT runtime_instance_id, binding_version FROM audit_event WHERE event_type = 'environment_action_finished' AND action = 'reconcile' ORDER BY id DESC LIMIT 1`).Scan(&auditedRuntime, &auditedBinding); err != nil { t.Fatal(err) } if auditedRuntime != after.RuntimeInstanceID || auditedBinding != after.BindingVersion { t.Fatalf("finished audit retained stale runtime: runtime=%q binding=%d current=%#v", auditedRuntime, auditedBinding, after) } } func TestPostgresNonRunnableReconcileAuditsRuntimeRelease(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() for _, authorization := range []string{"authorized", "revoked"} { for _, observation := range []string{"stopped", "missing"} { for _, releaseFailure := range []bool{false, true} { name := authorization + "/" + observation + "/success" if releaseFailure { name = authorization + "/" + observation + "/release failure" } t.Run(name, func(t *testing.T) { fixture := newPostgresRebindFixture(t, databaseURL) setFixtureAccountStatus(t, fixture.databaseURL, "active") var err error fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "stopped-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "stopped-network") if err != nil { t.Fatal(err) } if _, err := fixture.db.ExecContext(ctx, ` UPDATE social_account SET status = 'paused', authorization_status = $1 WHERE id = 'account-a'`, authorization); err != nil { t.Fatal(err) } if observation == "stopped" { fixture.gateway.runtimes = []runtimeStatus{{ ID: "stopped-container", Alias: "account-a", State: "exited", BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, }} } var store HubStore = fixture.store if releaseFailure { store = failingRuntimeReleaseStore{Store: fixture.store, err: errors.New("release unavailable")} } err = reconcileRuntimeLeases(ctx, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) wantOutcome, wantReason := "succeeded", "runtime_released" if releaseFailure { wantOutcome, wantReason = "failed", "runtime_release_failed" } if (err != nil) != releaseFailure { t.Fatalf("reconcile error=%v, releaseFailure=%v", err, releaseFailure) } after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") if err != nil || (after.RuntimeInstanceID != "") == !releaseFailure { t.Fatalf("runtime lease after reconcile: %#v err=%v", after, err) } var events, operations, missingOperations int var eventTypes, outcome, reason string if err := fixture.db.QueryRowContext(ctx, ` SELECT count(*), count(DISTINCT operation_id), count(*) FILTER (WHERE coalesce(operation_id, '') = ''), string_agg(event_type, ',' ORDER BY id), coalesce(max(outcome) FILTER (WHERE event_type = 'environment_action_finished'), ''), coalesce(max(reason_code) FILTER (WHERE event_type = 'environment_action_finished'), '') FROM audit_event WHERE action = 'reconcile'`).Scan( &events, &operations, &missingOperations, &eventTypes, &outcome, &reason); err != nil { t.Fatal(err) } if events != 2 || operations != 1 || missingOperations != 0 || eventTypes != "environment_action_requested,environment_action_finished" || outcome != wantOutcome || reason != wantReason { t.Fatalf("reconcile audit mismatch: events=%d operations=%d missing=%d types=%q outcome=%q reason=%q", events, operations, missingOperations, eventTypes, outcome, reason) } }) } } } } func TestBrowserActionRoutesStartStopAndRejectsUnknown(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { t.Fatalf("expected 204 for start, got %d: %s", response.Code, response.Body.String()) } if response := do(app, http.MethodPost, "/api/browsers/account-a/stop", ""); response.Code != http.StatusNoContent { t.Fatalf("expected 204 for stop, got %d: %s", response.Code, response.Body.String()) } if response := do(app, http.MethodPost, "/api/browsers/account-a/pause", ""); response.Code != http.StatusBadRequest { t.Fatalf("expected 400 for unknown action, got %d", response.Code) } if len(store.actions) != 4 || store.actions[0].Action != "start" || store.actions[1].Outcome != "succeeded" || store.actions[2].Action != "stop" || store.actions[3].Outcome != "succeeded" { t.Fatalf("start and stop must each emit an audit pair: %#v", store.actions) } } func TestStartRejectsPausedOrRevokedAccountBeforeGatewayCall(t *testing.T) { for _, test := range []struct { name, status, authorization string }{ {name: "paused", status: "paused", authorization: "authorized"}, {name: "revoked", status: "paused", authorization: "revoked"}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: test.status, AuthorizationStatus: test.authorization, BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], } gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers/account-a/start", "") if response.Code != http.StatusConflict { t.Fatalf("start returned %d: %s", response.Code, response.Body.String()) } if len(gateway.recorded()) != 0 || len(store.actions) != 0 { t.Fatalf("rejected start reached side effects: requests=%#v actions=%#v", gateway.recorded(), store.actions) } }) } } func TestReconcileStopsRuntimeForPausedOrRevokedAccount(t *testing.T) { for _, authorization := range []string{"authorized", "revoked"} { t.Run(authorization, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: authorization, BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile failed: %v", err) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { t.Fatalf("non-runnable account retained runtime %q", runtime) } requests := gateway.recorded() if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost || requests[1].path != "/v1/browsers/account-a/stop" { t.Fatalf("reconcile did not stop the runtime: %#v", requests) } }) } } func TestActivationConflictCleanupIsGenerationSafe(t *testing.T) { for _, test := range []struct { name string containerID string wantDeleted bool }{ {name: "candidate remains current", containerID: "candidate-container", wantDeleted: true}, {name: "alias points to newer container", containerID: "new-container"}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} environment := hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], } store.bindings[environment.Alias] = environment gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: test.containerID, Alias: environment.Alias, State: "running", ProxyReady: true, BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, NetworkID: "native-11111111111111111111111111111111", }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} _, err := activateGatewayRuntime(context.Background(), store, target, environment, "candidate-container", "native-11111111111111111111111111111111") if err == nil { t.Fatal("paused account unexpectedly activated the candidate") } deleted := len(gateway.runtimes) == 0 if deleted != test.wantDeleted { t.Fatalf("generation-safe cleanup mismatch: deleted=%v requests=%#v", deleted, gateway.recorded()) } if !test.wantDeleted { for _, request := range gateway.recorded() { if request.method == http.MethodDelete { t.Fatalf("stale candidate deleted a newer alias generation: %#v", gateway.recorded()) } } } }) } } func TestExplicitStopRejectsStaleBindingBeforeGatewaySideEffect(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "new-instance", RuntimeID: "new-container", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "new-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} stale := store.bindings["account-a"] stale.BindingVersion, stale.RuntimeInstanceID, stale.RuntimeID = 1, "old-instance", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" if err := stopEnvironmentRuntime(context.Background(), store, stale); !errors.Is(err, hub.ErrConflict) { t.Fatalf("stale explicit stop was not rejected: %v", err) } if len(gateway.recorded()) != 0 || len(gateway.runtimes) != 1 || gateway.runtimes[0].ID != "new-container" { t.Fatalf("stale explicit stop reached the gateway: requests=%#v runtimes=%#v", gateway.recorded(), gateway.runtimes) } } func TestReconcileFinishedAuditUsesRebuiltRuntime(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-2"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666", }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("reconcile failed: %v", err) } if len(store.actions) != 2 || store.actions[1].Outcome != "succeeded" || store.actions[1].RuntimeInstanceID != "runtime-instance" || store.actions[1].BindingVersion != 2 || store.actions[1].NetworkExitID != "exit-2" { t.Fatalf("finished audit used stale runtime context: %#v", store.actions) } } func TestReconcileContextRefreshFailureClearsAllAuditCorrelation(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", }}} server := httptest.NewServer(gateway.handler(t)) defer server.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} failing := &failContextRefreshStore{hubStore: store} if err := reconcileRuntimeLeases(context.Background(), failing, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil { t.Fatal("context refresh failure unexpectedly succeeded") } finished := store.actions[len(store.actions)-1] if finished.RuntimeInstanceID != "" || finished.BindingVersion != 0 || finished.NetworkExitID != "" { t.Fatalf("failed context refresh retained partial stale correlation: %#v", finished) } } func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) { gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet { _ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "exited"}}) return } connection, _, err := response.(http.Hijacker).Hijack() if err == nil { _ = connection.Close() } })) defer gatewayServer.Close() store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} store.releaseErr = errors.New("database unavailable") store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers/account-a/stop", "") if response.Code != http.StatusInternalServerError { t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String()) } if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_release_failed" { t.Fatalf("stopped reconcile release failure must be audited: %#v", store.actions) } } func TestReconcileAuditsRuntimeReleaseFailure(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } store.releaseErr = errors.New("database unavailable") gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "exited", BindingVersion: 1, NetworkExitID: "exit-1", }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil { t.Fatal("expected release failure") } if len(store.actions) != 2 || store.actions[0].Action != "reconcile" || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_release_failed" { t.Fatalf("reconcile release failure must have its own audit pair: %#v", store.actions) } } func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) if response.Code != http.StatusOK { t.Fatalf("running runtime rebind failed: %d: %s", response.Code, response.Body.String()) } bound := store.bindings["account-a"] if bound.Exit.ID != "exit-2" || bound.BindingVersion != 2 || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("running runtime was not rebuilt on the latest binding: %#v", bound) } requests := gateway.recorded() if len(requests) != 4 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost { t.Fatalf("running rebind must inspect, delete and recreate: %#v", requests) } if requests[3].body["binding_version"] != float64(2) || requests[3].body["network_exit_id"] != "exit-2" { t.Fatalf("recreated runtime did not carry the latest binding CAS: %#v", requests[3].body) } } func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "expired-runtime", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-1"}`) if response.Code != http.StatusOK { t.Fatalf("same-exit rebind failed: %d: %s", response.Code, response.Body.String()) } bound := store.bindings["account-a"] if bound.BindingVersion != 2 || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("same-exit rebind did not rotate the binding generation and lease: %#v", bound) } requests := gateway.recorded() if len(requests) != 4 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost { t.Fatalf("same-exit rebind bypassed gateway reconciliation: %#v", requests) } if requests[3].body["binding_version"] != float64(2) || requests[3].body["network_exit_id"] != "exit-1" { t.Fatalf("same-exit rebuild used stale metadata: %#v", requests[3].body) } } func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} probe := fakeExitProbe{failure: "exit_auth_failed"} _ = newTestAppWithNetwork(t, store, gateway, probe, func(hub.NetworkExitAccess) (string, error) { return "", nil }) if err := reconcileRuntimeLeases(context.Background(), store, probe, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil { t.Fatalf("unhealthy runtime reconciliation failed: %v", err) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { t.Fatalf("failed exit remained active: %q", runtime) } requests := gateway.recorded() if len(requests) != 3 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete { t.Fatalf("failed exit must be rechecked and removed instead of renewed: %#v", requests) } } func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", RuntimeNetworkID: "native-66666666666666666666666666666666", } gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, runtimes: []runtimeStatus{{ ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", ProxyReady: true, }}} _ = newTestApp(t, store, gateway) if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil { t.Fatal("delete failure unexpectedly succeeded") } if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { t.Fatalf("delete failure retained DB runtime lease: %q", runtime) } if len(store.actions) != 2 || store.actions[0].Action != "reconcile" || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("delete failure was not audited as retryable unknown: %#v", store.actions) } if len(gateway.runtimes) != 1 { t.Fatal("failed delete unexpectedly removed the gateway container") } } func TestDisableExitImmediatelyDiscardsRuntime(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "") if response.Code != http.StatusOK { t.Fatalf("disable cleanup failed: %d: %s", response.Code, response.Body.String()) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { t.Fatalf("disabled exit remained active: %q", runtime) } requests := gateway.recorded() if len(requests) != 3 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete { t.Fatalf("disable must reconcile and remove the runtime: %#v", requests) } } func TestDisableExitWaitsForInFlightBrowserLifecycle(t *testing.T) { releaseCreate := make(chan struct{}) gateway := &fakeGateway{ token: "unit-test-gateway-token", createStarted: make(chan struct{}), releaseCreate: releaseCreate, } store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], } gatewayServer := httptest.NewServer(gateway.handler(t)) defer gatewayServer.Close() store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) server := httptest.NewServer(adaptor.FiberApp(app)) defer server.Close() statuses := make(chan int, 2) go func() { response, err := server.Client().Post(server.URL+"/api/browsers/account-a/start", "application/json", nil) if err != nil { statuses <- 0 return } defer response.Body.Close() statuses <- response.StatusCode }() select { case <-gateway.createStarted: case <-time.After(time.Second): t.Fatal("start did not reach gateway create") } go func() { response, err := server.Client().Post(server.URL+"/api/network-exits/exit-1/disable", "application/json", nil) if err != nil { statuses <- 0 return } defer response.Body.Close() statuses <- response.StatusCode }() select { case status := <-statuses: t.Fatalf("lifecycle operation completed before the running create: %d", status) case <-time.After(50 * time.Millisecond): } close(releaseCreate) for range 2 { if status := <-statuses; status != http.StatusNoContent && status != http.StatusOK { t.Fatalf("unexpected lifecycle status: %d", status) } } if store.exits["exit-1"].HealthStatus != "disabled" || store.bindings["account-a"].RuntimeID != "" { t.Fatalf("disable did not clean the completed runtime: exit=%#v binding=%#v", store.exits["exit-1"], store.bindings["account-a"]) } } func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) { for _, test := range []struct { name string handler http.HandlerFunc }{ {name: "disconnect", handler: func(response http.ResponseWriter, _ *http.Request) { connection, _, err := response.(http.Hijacker).Hijack() if err == nil { _ = connection.Close() } }}, {name: "status 500", handler: func(response http.ResponseWriter, _ *http.Request) { response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(`{"error":"docker unavailable"}`)) }}, {name: "invalid JSON", handler: func(response http.ResponseWriter, _ *http.Request) { _, _ = response.Write([]byte(`{"not":"a browser list"}`)) }}, } { t.Run(test.name, func(t *testing.T) { gatewayServer := httptest.NewServer(test.handler) defer gatewayServer.Close() store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "") if response.Code != http.StatusBadGateway { t.Fatalf("gateway unknown must not return 200: %d: %s", response.Code, response.Body.String()) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("gateway unknown changed the unconfirmed runtime lease: %q", runtime) } if store.exits["exit-1"].HealthStatus != "disabled" { t.Fatal("disable state was not retained for retryable reconciliation") } }) } } func TestDirectBindingIsListable(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{ Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true, }} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, RuntimeInstanceID: "4444444444444444444444444444444444444444444444444444444444444444", RuntimeID: "direct-container", RuntimeNetworkID: "direct-network", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "direct-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkID: "direct-network", ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodGet, "/api/browsers", "") if response.Code != http.StatusOK { t.Fatalf("direct binding broke browser listing: %d: %s", response.Code, response.Body.String()) } var views []envView if err := json.Unmarshal(response.Body.Bytes(), &views); err != nil || len(views) != 1 || views[0].RecoveryRequired || views[0].NetworkExitID != "" || views[0].ScheduleStatus != "ready" { t.Fatalf("direct runtime state was not visible: %#v err=%v", views, err) } } func TestRebindDeleteFailureKeepsOriginalBinding(t *testing.T) { for _, test := range []struct { name string binding hub.EnvironmentContext targetID string }{ {name: "running binding", binding: hub.EnvironmentContext{ AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy"}, RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", }, targetID: "exit-2"}, {name: "legacy NULL binding", binding: hub.EnvironmentContext{ AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, RuntimeInstanceID: "5555555555555555555555555555555555555555555555555555555555555555", RuntimeID: "legacy-container", }, targetID: "exit-1"}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} before := test.binding before.Env = store.envs["account-a"] store.bindings["account-a"] = before gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, runtimes: []runtimeStatus{{ ID: before.RuntimeID, Alias: "account-a", State: "running", BindingVersion: before.BindingVersion, NetworkExitID: before.Exit.ID, ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+test.targetID+`"}`) if response.Code != http.StatusBadGateway { t.Fatalf("delete failure must remain unknown: %d: %s", response.Code, response.Body.String()) } after := store.bindings["account-a"] if after.BindingVersion != before.BindingVersion || after.Exit.ID != before.Exit.ID || !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("binding committed before old container deletion: before=%#v after=%#v", before, after) } if len(gateway.runtimes) != 1 { t.Fatalf("non-final delete result lost the existing container: %#v", gateway.runtimes) } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("delete failure was not audited as unknown: %#v", store.actions) } }) } } func TestRebindNetworkCleanupPendingBlocksUntilConfirmed(t *testing.T) { for _, test := range []struct { name string cleanupPending int disconnectDelete int disconnectListAfterDelete bool }{ {name: "cleanup pending", cleanupPending: 2}, {name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", } gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: test.cleanupPending, disconnectDelete: test.disconnectDelete, disconnectListAfterDelete: test.disconnectListAfterDelete, runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) if response.Code != http.StatusBadGateway { t.Fatalf("network cleanup uncertainty returned %d: %s", response.Code, response.Body.String()) } after := store.bindings["account-a"] if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || !after.RuntimeCleanupPending || after.RuntimeID != "" { t.Fatalf("network cleanup uncertainty changed the binding: %#v", after) } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("network cleanup uncertainty audit mismatch: %#v", store.actions) } if len(gateway.runtimes) != 0 { t.Fatalf("container-removed fact was lost: %#v", gateway.runtimes) } response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) if response.Code != http.StatusOK { t.Fatalf("confirmed cleanup did not allow retry: %d: %s", response.Code, response.Body.String()) } after = store.bindings["account-a"] if after.RuntimeCleanupPending || after.BindingVersion != 2 || after.Exit.ID != "exit-2" || after.RuntimeID != "" { t.Fatalf("retry did not commit a clean stopped generation: %#v", after) } }) } } func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) { for _, test := range []struct { name, method, path, body string }{ {name: "create reuse", method: http.MethodPost, path: "/api/browsers", body: `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`}, {name: "start", method: http.MethodPost, path: "/api/browsers/account-a/start"}, {name: "rebind", method: http.MethodPost, path: "/api/browsers/account-a/rebind", body: `{"network_exit_id":"exit-1"}`}, {name: "recycle", method: http.MethodDelete, path: "/api/browsers/account-a"}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, RuntimeCleanupPending: true, RuntimeCleanupBindingVersion: 1, RuntimeCleanupRuntimeID: missingRuntimeID, RuntimeCleanupNetworkID: "native-66666666666666666666666666666666", Exit: store.exits["exit-1"], } gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: 2} app := newTestApp(t, store, gateway) response := do(app, test.method, test.path, test.body) if response.Code != http.StatusBadGateway { t.Fatalf("pending cleanup returned %d: %s", response.Code, response.Body.String()) } after := store.bindings["account-a"] if !after.RuntimeCleanupPending || after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" { t.Fatalf("lifecycle path advanced a pending generation: %#v", after) } for _, request := range gateway.recorded() { if request.method == http.MethodPost && request.path == "/v1/browsers" { t.Fatalf("lifecycle path created before cleanup confirmation: %#v", gateway.recorded()) } } }) } } func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) { t.Skip("browser path lookup is gateway-owned") store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"} gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, nil) response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) if response.Code < 400 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { t.Fatalf("runtime preparation failure touched the old container: status=%d requests=%#v", response.Code, gateway.recorded()) } if after := store.bindings["account-a"]; after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" { t.Fatalf("runtime preparation failure changed state: %#v", after) } } func TestRebindCandidateUnknownCreateLeavesGenerationPending(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"} gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, runtimes: []runtimeStatus{{ ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, }}} app := newTestApp(t, store, gateway) response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) if response.Code != http.StatusConflict { t.Fatalf("candidate create failure returned %d: %s", response.Code, response.Body.String()) } after := store.bindings["account-a"] if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" || !after.RuntimeCleanupPending || after.RuntimeCleanupBindingVersion != 2 || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" || len(gateway.runtimes) != 0 { t.Fatalf("candidate create unknown result was not retained for manual reconcile: after=%#v runtimes=%#v", after, gateway.runtimes) } if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { t.Fatalf("candidate create failure audit mismatch: %#v", store.actions) } } func TestExistingEnvironmentCleanupNeverReturnsReusedSuccess(t *testing.T) { for _, test := range []struct { name string probe *sequenceExitProbe resolve func(hub.NetworkExitAccess) (string, error) }{ {name: "second probe fails", probe: &sequenceExitProbe{failures: []string{"", "exit_auth_failed"}}, resolve: func(hub.NetworkExitAccess) (string, error) { return "", nil }}, } { t.Run(test.name, func(t *testing.T) { store := newMemoryStore() store.exits["exit-1"] = hub.NetworkExit{ ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1, Username: "username", Password: "password", } store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 2024, Platform: "windows", Timezone: "Asia/Shanghai"}} store.bindings["account-a"] = hub.EnvironmentContext{ Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", }}} app := newTestAppWithNetwork(t, store, gateway, test.probe, test.resolve) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusConflict { t.Fatalf("cleaned runtime must not return reused success: %d: %s", response.Code, response.Body.String()) } if !strings.Contains(response.Body.String(), "not running") { t.Fatalf("cleaned runtime response did not state the stopped result: %s", response.Body.String()) } if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { t.Fatalf("failed recovery retained runtime: %q probe_calls=%d actions=%#v requests=%#v", runtime, test.probe.calls, store.actions, gateway.recorded()) } if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_unavailable" { t.Fatalf("cleaned runtime was audited as reused success: %#v", store.actions) } }) } } func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) { store := newMemoryStore() store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1} store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"]} gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "exited", BindingVersion: 1}}} app := newTestApp(t, store, gateway) if response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`); response.Code != http.StatusOK { t.Fatalf("rebind failed: %d %s", response.Code, response.Body.String()) } if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { t.Fatalf("start failed: %d %s", response.Code, response.Body.String()) } requests := gateway.recorded() var createRequest recordedRequest for _, request := range requests { if request.method == http.MethodPost && request.path == "/v1/browsers" { createRequest = request } } if createRequest.body == nil { t.Fatalf("stopped container was not recreated: %#v", requests) } exit := createRequest.body["network_exit"].(map[string]any) if exit["host"] != "proxy-2.example" { t.Fatalf("recreated container did not use rebound exit: %#v", createRequest.body) } } func TestRecycleBrowserKeepsStableEnvironment(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}} gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) if response := do(app, http.MethodDelete, "/api/browsers/account-a", ""); response.Code != http.StatusNoContent { t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String()) } _, kept := store.envs["account-a"] if !kept { t.Fatal("recycle must preserve the stable environment and Profile anchor") } if len(store.actions) != 2 || store.actions[0].Action != "recycle" || store.actions[1].Outcome != "succeeded" { t.Fatalf("recycle must emit an audit pair: %#v", store.actions) } } func TestNetworkExitRoutesStoreAndExposePlainCredentials(t *testing.T) { store := newMemoryStore() gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) invalid := do(app, http.MethodPost, "/api/network-exits", `{"protocol":"socks5","host":"proxy.example","port":1080,"password":"password-only"}`) if invalid.Code != http.StatusBadRequest || len(store.exits) != 1 { t.Fatalf("password without username must be rejected: status=%d exits=%#v", invalid.Code, store.exits) } created := do(app, http.MethodPost, "/api/network-exits", `{"protocol":"socks5","host":"proxy.example","port":1080,"username":"proxy-user","password":"plain-password","expected_public_ip":"203.0.113.1","expected_region":"test"}`) if created.Code != http.StatusCreated || !strings.Contains(created.Body.String(), `"username":"proxy-user"`) || !strings.Contains(created.Body.String(), `"password":"plain-password"`) { t.Fatalf("network exit response must expose stored credentials: status=%d body=%s", created.Code, created.Body.String()) } detail := do(app, http.MethodGet, "/api/network-exits/exit-created", "") if detail.Code != http.StatusOK || !strings.Contains(detail.Body.String(), `"username":"proxy-user"`) || !strings.Contains(detail.Body.String(), `"password":"plain-password"`) { t.Fatalf("network exit detail must expose stored credentials: status=%d body=%s", detail.Code, detail.Body.String()) } checked := do(app, http.MethodPost, "/api/network-exits/exit-created/check", "") if checked.Code != http.StatusOK || !strings.Contains(checked.Body.String(), `"health_status":"healthy"`) { t.Fatalf("network exit check did not record health: status=%d body=%s", checked.Code, checked.Body.String()) } disabled := do(app, http.MethodPost, "/api/network-exits/exit-created/disable", "") if disabled.Code != http.StatusOK || !strings.Contains(disabled.Body.String(), `"health_status":"disabled"`) { t.Fatalf("network exit disable failed: status=%d body=%s", disabled.Code, disabled.Body.String()) } }