@@ -926,28 +926,13 @@ func TestUpdateGatewayRenamesAndPreservesReferences(t *testing.T) {
}
}
func TestListGatewaysExposesConnectivityAndHealth ( t * testing . T ) {
func TestListGatewaysReturnsStoredRecordsWithoutProbing ( t * testing . T ) {
store := newMemoryStore ( )
// gw-1:正常网关,/healthz 可达且 /v1/browsers 可认证。
gateway := & fakeGateway { token : "unit-test-gateway-token" }
server := httptest . NewServer ( gateway . handler ( t ) )
t . Cleanup ( server . Close )
store . gateways [ "gw-1" ] = hub . Gateway { Name : "gw-1" , Endpoint : server . URL , Token : gateway . token }
// gw-auth: /healthz 可达但令牌被拒,应区分连通与 API 健康。
authServer := httptest . NewServer ( http . HandlerFunc ( func ( response http . ResponseWriter , request * http . Request ) {
if request . Method == http . MethodGet && request . URL . Path == "/healthz" {
response . WriteHeader ( http . StatusNoContent )
return
}
response . WriteHeader ( http . StatusUnauthorized )
} ) )
t . Cleanup ( authServer . Close )
store . gateways [ "gw-auth" ] = hub . Gateway { Name : "gw-auth" , Endpoint : authServer . URL , Token : "unit-test-gateway-token" }
// gw-offline:端口已关闭,两个探测都应失败并携带原因。
closed := httptest . NewServer ( http . HandlerFunc ( func ( http . ResponseWriter , * http . Request ) { } ) )
closedURL := closed . URL
closed . Close ( )
store . gateways [ "gw-offline" ] = hub . Gateway { Name : "gw-offline" , Endpoint : closedURL , Token : "offline-token-123456789" }
stored := hub . Gateway { Name : "gw-1" , Endpoint : server . URL , Token : gateway . token }
store . gateways [ stored . Name ] = stored
app := fiber . New ( )
registerHubWithNetwork ( app , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
@@ -955,22 +940,15 @@ func TestListGatewaysExposesConnectivityAndHealth(t *testing.T) {
if response . Code != http . StatusOK {
t . Fatalf ( "list gateways returned %d: %s" , response . Code , response . Body . String ( ) )
}
var statuses [ ] gatewayStatus
if err := json . Unmarshal ( response . Body . Bytes ( ) , & statuses ) ; err != nil || len ( statuses ) != 3 {
t . Fatalf ( "decode gateway statuses failed: statuse s=%#v err=%v" , statuses , err )
var gateways [ ] hub . Gateway
if err := json . Unmarshal ( response . Body . Bytes ( ) , & gateways ) ; err != nil || len ( gateways ) != 1 || gateways [ 0 ] != stored {
t . Fatalf ( "stored gateway was not returned unchanged: gateway s=%#v err=%v" , gateways , err )
}
byName := map [ string ] gatewayStatus { }
for _ , status := range statuses {
byName [ status . Name ] = status
if strings . Contains ( response . Body . String ( ) , "connectivity" ) || strings . Contains ( response . Body . String ( ) , "health" ) {
t . Fatalf ( "gateway list exposed live status fields: %s" , response . Body . String ( ) )
}
if status := byName [ "gw-1" ] ; status . Connectivity != "online" || status . Health != "healthy" || status . ConnectivityReason != "" || status . HealthReason != "" {
t . Fatalf ( "healthy gateway reported %#v" , status )
}
if status := byName [ "gw-auth" ] ; status . Connectivity != "online" || status . Health != "unhealthy" || status . HealthReason == "" {
t . Fatalf ( "auth-rejected gateway reported %#v" , status )
}
if status := byName [ "gw-offline" ] ; status . Connectivity != "offline" || status . Health != "unhealthy" || status . ConnectivityReason == "" || status . HealthReason == "" {
t . Fatalf ( "offline gateway reported %#v" , status )
if requests := gateway . recorded ( ) ; len ( requests ) != 0 {
t . Fatalf ( "gateway list performed live probes: %#v" , requests )
}
}
@@ -1534,14 +1512,18 @@ func TestCreateBrowserRejectsInvalidFingerprintBeforeSideEffects(t *testing.T) {
}
}
func TestListBrowsersMergesLiveGatewayState ( t * testing . T ) {
func TestListBrowsersUsesPersistedStateWithoutProbing ( t * testing . T ) {
store := newMemoryStore ( )
store . envs [ "account-a" ] = hub . Env { Alias : "account-a" , Name : "店铺一号" , Gateway : "gw-1" , ImageVersion : "148" }
store . envs [ "account-b" ] = hub . Env { Alias : "account-b" , Name : "店铺二号" , Gateway : "gw-1" , ImageVersion : "148" }
store . bindings [ "account-a" ] = hub . EnvironmentContext {
Env : store . envs [ "account-a" ] , AccountID : "account-a" , AccountStatus : "active" , AuthorizationStatus : "authorized" ,
BindingID : "account-a" , BindingVersion : 1 , Exit : store . exits [ "exit-1" ] , RuntimeInstanceID : "runtime-1" , RuntimeID : "persisted-container" ,
}
gateway := & fakeGateway {
token : "unit-test-gateway-token" ,
containers : [ ] containerStatus {
{ ID : "id-1 " , Alias : "account-a" , State : "running" , Status : "Up" , Endpoint : "http://creatorhub-browser-account-a:9222" , BindingVersion : 1 , NetworkExitID : "exit-1" , NetworkID : "network-id" , ProxyReady : true } ,
{ ID : "live-container " , Alias : "account-a" , State : "running" , Status : "Up" , Endpoint : "http://creatorhub-browser-account-a:9222" , BindingVersion : 1 , NetworkExitID : "exit-1" , NetworkID : "network-id" , ProxyReady : true } ,
} ,
}
app := newTestApp ( t , store , gateway )
@@ -1558,17 +1540,20 @@ func TestListBrowsersMergesLiveGatewayState(t *testing.T) {
for _ , view := range views {
byAlias [ view . Alias ] = view
}
if byAlias [ "account-a" ] . State != "running" || byAlias [ "ac cou nt-a" ] . ContainerID != "id-1 " {
t . Fatalf ( "running container state must b e m erged: %#v" , byAlias [ "account-a" ] )
if view := byAlias [ "account-a" ] ; view . State != "running" || view . Status != "已记录运行实例" || view . ContainerID != "persisted- container" || view . Endpoint != "" {
t . Fatalf ( "list must expos e p ersisted runtime data only: %#v" , view )
}
if view := byAlias [ "account-a" ] ; view . AccountID != "account-a" || view . NetworkExitHealth != "healthy" || view . ScheduleStatus != "ready" || view . ScheduleBlockReason != "" {
t . Fatalf ( "binding readiness must be exposed without credential s: %#v" , view )
t . Fatalf ( "binding readiness must be exposed without gateway acces s: %#v" , view )
}
if byAlias [ "account-b" ] . State != "missing" {
t . Fatalf ( "env without container must report missing: %#v" , byAlias [ "account-b" ] )
t . Fatalf ( "env without a persisted runtime must report missing: %#v" , byAlias [ "account-b" ] )
}
if runtime := store . bindings [ "account-a" ] . RuntimeID ; runtime != "id-1" {
t . Fatalf ( "list reconciliation must heartbeat the running runtime, got %q " , runtime )
if response = do ( app , http . MethodGet , "/api/browsers/account-a" , "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "browser detail returned %d: %s " , response . Code , response . Body . String ( ) )
}
if requests := gateway . recorded ( ) ; len ( requests ) != 0 {
t . Fatalf ( "browser list/detail performed live probes: %#v" , requests )
}
}
@@ -1581,14 +1566,13 @@ func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) {
gateway := & fakeGateway { token : "unit-test-gateway-token" , containers : [ ] containerStatus { {
ID : "container-id" , Alias : "account-a" , State : "running" , BindingVersion : 3 , NetworkExitID : "exit-1" , NetworkID : "network-id" , ProxyReady : false ,
} } }
app := newTestApp ( t , store , gateway )
_ = newTestApp ( t , store , gateway )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "gateway restart recovery failed: %d %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "gateway restart recovery failed: %v" , err )
}
if runtime := store . bindings [ "account-a" ] . RuntimeID ; runtime != "container-id" {
t . Fatalf ( "runtime was activated before proxy recovery completed : %q" , runtime )
t . Fatalf ( "runtime was not activated after proxy recovery: %q" , runtime )
}
requests := gateway . recorded ( )
if len ( requests ) != 2 || requests [ 0 ] . path != "/v1/browsers" || requests [ 1 ] . path != "/v1/browsers/account-a/proxy" {
@@ -1608,10 +1592,10 @@ func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing.
gateway := & fakeGateway { token : "unit-test-gateway-token" , failProxy : true , containers : [ ] containerStatus { {
ID : "old-container" , Alias : "account-a" , State : "running" , BindingVersion : 2 , NetworkExitID : "exit-1" , NetworkID : "network-old" ,
} } }
app := newTestApp ( t , store , gateway )
_ = newTestApp ( t , store , gateway )
if response := do ( app , http . MethodGet , "/api/browsers ", "" ) ; response . Code != http . StatusConflict {
t . Fatalf ( "lease-free running orphan was not fenced: %d %s " , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return " ", nil } ) ; ! errors . Is ( err , hub . ErrConflict ) {
t . Fatalf ( "lease-free running orphan was not fenced: %v " , err )
}
requests := gateway . recorded ( )
if len ( requests ) != 3 || requests [ 1 ] . path != "/v1/browsers/account-a/proxy" {
@@ -1755,17 +1739,15 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) {
NetworkExitID : "stale-exit" , NetworkID : "network-old" , ProxyReady : true ,
} }
gateway . mu . Unlock ( )
response = do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusBadGateway {
t . Fatalf ( "PostgreSQL-backed reconcile delete failure returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( ctx , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err == nil {
t . Fatal ( "PostgreSQL-backed reconcile delete failure unexpectedly succeeded" )
}
released , err := store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || released . RuntimeID != "" || released . BindingVersion != after . BindingVersion || ! released . RuntimeCleanupPending {
t . Fatalf ( "delete failure must release the real Store lease without changing binding: %#v err=%v" , released , err )
}
response = do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "PostgreSQL-backed cleanup retry returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( ctx , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "PostgreSQL-backed cleanup retry failed: %v" , err )
}
released , err = store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || released . RuntimeCleanupPending {
@@ -2326,9 +2308,8 @@ func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) {
assertControlPlaneDatabaseCount ( t , fixture . db , ` SELECT count(*) FROM environment_binding WHERE browser_env_alias = 'account-a' AND runtime_cleanup_pending ` , 1 )
assertControlPlaneDatabaseCount ( t , fixture . db , ` SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL ` , 0 )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "list reconcile did not confirm cleanup: %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( ctx , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "reconcile did not confirm cleanup: %v" , err )
}
after , err = fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || after . RuntimeCleanupPending {
@@ -2336,7 +2317,7 @@ func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) {
}
setFixtureAccountStatus ( t , fixture . databaseURL , "paused" )
response = do ( app , http . MethodPost , "/api/browsers/account-a/rebind" , ` { "network_exit_id":" ` + fixture . exit . ID + ` "} ` )
response := do ( app , http . MethodPost , "/api/browsers/account-a/rebind" , ` { "network_exit_id":" ` + fixture . exit . ID + ` "} ` )
if response . Code != http . StatusOK {
t . Fatalf ( "rebind retry failed after cleanup confirmation: %d: %s" , response . Code , response . Body . String ( ) )
}
@@ -2500,7 +2481,7 @@ func TestRemoveGatewayRuntimeConvergesUnknownGeneration(t *testing.T) {
store . envs [ "account-a" ] = hub . Env { Alias : "account-a" , Gateway : "gw-1" , ImageVersion : "148" }
environment := hub . EnvironmentContext {
Env : store . envs [ "account-a" ] , AccountID : "account-a" , BindingID : "account-a" , BindingVersion : 1 ,
Exit : store . exits [ "exit-1" ] ,
Exit : store . exits [ "exit-1" ] ,
RuntimeCleanupPending : true , RuntimeCleanupBindingVersion : 1 ,
RuntimeCleanupRuntimeID : missingRuntimeID ,
}
@@ -2595,9 +2576,8 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) {
} } }
app := newTestApp ( t , store , gateway )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusInternalServerError {
t . Fatalf ( "failed cleanup mark returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err == nil {
t . Fatal ( "failed cleanup mark unexpectedly succeeded" )
}
after := store . bindings [ "account-a" ]
if test . commitKnown {
@@ -2621,13 +2601,13 @@ func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) {
store . cleanupPendingErr = nil
store . cleanupPendingErrAfterMutation = false
if response = do ( app , http . MethodGet , "/api/browsers ", "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "list retry failed: %d: %s " , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return " ", nil } ) ; err != nil {
t . Fatalf ( "reconcile retry failed: %v " , err )
}
if test . commitKnown {
after = store . bindings [ "account-a" ]
} else {
if response = do ( app , http . MethodPost , "/api/browsers/account-a/start" , "" ) ; response . Code != http . StatusNoContent {
if response := do ( app , http . MethodPost , "/api/browsers/account-a/start" , "" ) ; response . Code != http . StatusNoContent {
t . Fatalf ( "start retry failed: %d: %s" , response . Code , response . Body . String ( ) )
}
after = store . bindings [ "account-a" ]
@@ -2874,8 +2854,8 @@ func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) {
setFixtureAccountStatus ( t , fixture . databaseURL , "active" )
app := fiber . New ( )
registerHubWithNetwork ( app , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
if response := do ( app , http . MethodGet , "/api/browsers ", "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "list retry did not confirm cleanup: %d: %s " , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return " ", nil } ) ; err != nil {
t . Fatalf ( "reconcile retry did not confirm cleanup: %v " , err )
}
if response := do ( app , http . MethodPost , "/api/browsers/account-a/start" , "" ) ; response . Code != http . StatusNoContent {
t . Fatalf ( "lifecycle retry did not rebuild runtime: %d: %s" , response . Code , response . Body . String ( ) )
@@ -2983,12 +2963,8 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) {
} else {
installCleanupTransitionFailure ( t , ctx , fixture . db , ` NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending ` )
}
app := fiber . New ( )
registerHubWithNetwork ( app , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusInternalServerError {
t . Fatalf ( "restore cleanup failure returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( ctx , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err == nil {
t . Fatal ( "restore cleanup failure unexpectedly succeeded" )
}
after , err := fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if test . commitUnknown {
@@ -3023,13 +2999,13 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) {
if ! test . commitUnknown {
dropCleanupTransitionFailure ( t , ctx , fixture . db )
}
retryApp := fiber . New ( )
registerHubWithNetwork ( retryApp , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return " ", nil } )
if response = do ( retryApp , http . MethodGet , "/api/browsers" , "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "restore list retry failed: %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( ctx , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "restore reconcile retry failed: %v ", err )
}
if test . commitUnknown {
if response = do ( retryApp , http . MethodPost , "/api/browsers/account-a/start" , "" ) ; response . Code ! = http . StatusNoContent {
retryApp : = fiber . New ( )
registerHubWithNetwork ( retryApp , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
if response := do ( retryApp , http . MethodPost , "/api/browsers/account-a/start" , "" ) ; response . Code != http . StatusNoContent {
t . Fatalf ( "restore start retry failed: %d: %s" , response . Code , response . Body . String ( ) )
}
}
@@ -3176,10 +3152,8 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) {
if ! test . commitUnknown {
dropCleanupTransitionFailure ( t , ctx , fixture . db )
}
retryApp := fiber . New ( )
registerHubWithNetwork ( retryApp , fixture . store , fakeExitProbe { failure : "proxy_auth_failed" } , func ( hub . NetworkExitAccess ) ( string , error ) { return " ", nil } )
if response = do ( retryApp , http . MethodGet , "/api/browsers" , "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "discard retry returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( ctx , fixture . store , fakeExitProbe { failure : "proxy_auth_failed" } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "discard reconcile retry failed: %v ", err )
}
after , err = fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || after . RuntimeCleanupPending || after . RuntimeID != "" {
@@ -3195,7 +3169,7 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) {
}
}
func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry ( t * testing . T ) {
func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate ( t * testing . T ) {
databaseURL := os . Getenv ( "CREATORHUB_POSTGRES_TEST_URL" )
if databaseURL == "" {
t . Skip ( "set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage" )
@@ -3225,20 +3199,16 @@ func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry(t *testing.T) {
app := fiber . New ( )
registerHubWithNetwork ( app , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
if response := do ( app , http . MethodGet , "/api/browsers" , "" ) ; response . Code != http . StatusBadGateway {
t . Fatalf ( "gateway unknown list returned %d: %s" , response . Code , response . Body . String ( ) )
if response := do ( app , http . MethodGet , "/api/browsers" , "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "gateway failure affected browser list: %d: %s" , response . Code , response . Body . String ( ) )
}
after , err := fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || after . RuntimeCleanupPending || after . RuntimeID != "old-container" || len ( fixture . gateway . containers ) != 1 {
t . Fatalf ( "gateway unknown list changed the generation: after=%#v containers=%#v err=%v" , after , fixture . gateway . containers , err )
t . Fatalf ( "browser list changed the generation: after=%#v containers=%#v err=%v" , after , fixture . gateway . containers , err )
}
assertControlPlaneDatabaseCount ( t , fixture . db , ` SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL ` , 1 )
if response := do ( app , http . MethodGet , "/api/browsers" , "" ) ; response . Code != http . StatusOK {
t . Fatalf ( "gateway list retry failed: %d : %s " , response . Code , response . Body . String ( ) )
}
after , err = fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || after . RuntimeID != "old-container" || len ( fixture . gateway . containers ) != 1 {
t . Fatalf ( "gateway list retry ended inconsistently: after=%#v containers=%#v err=%v" , after , fixture . gateway . containers , err )
if requests := fixture . gateway . recorded ( ) ; len ( requests ) != 0 {
t . Fatalf ( "browser list performed live gateway detection : %#v " , requests )
}
} )
@@ -3300,7 +3270,7 @@ func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry(t *testing.T) {
}
}
func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry ( t * testing . T ) {
func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry ( t * testing . T ) {
databaseURL := os . Getenv ( "CREATORHUB_POSTGRES_TEST_URL" )
if databaseURL == "" {
t . Skip ( "set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage" )
@@ -3312,8 +3282,6 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) {
active bool
wantStatus int
} {
{ name : "list rejects top-level null" , method : http . MethodGet , path : "/api/browsers" , active : true , wantStatus : http . StatusOK ,
configure : func ( gateway * fakeGateway , count int ) { gateway . invalidList , gateway . invalidListBody = count , ` null ` } } ,
{ name : "create rejects null element" , method : http . MethodPost , path : "/api/browsers" , action : "create" , active : true ,
wantStatus : http . StatusOK , successReason : "environment_reused" ,
configure : func ( gateway * fakeGateway , count int ) { gateway . invalidList , gateway . invalidListBody = count , ` [null] ` } } ,
@@ -3363,17 +3331,17 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) {
response := do ( app , test . method , test . path , body )
if response . Code != http . StatusBadGateway {
t . Fatalf ( "strict list failure returned %d: %s" , response . Code , response . Body . String ( ) )
t . Fatalf ( "strict gateway validation returned %d: %s" , response . Code , response . Body . String ( ) )
}
after , err := fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || after . BindingVersion != fixture . bound . BindingVersion || after . Exit . ID != fixture . bound . Exit . ID ||
after . RuntimeCleanupPending || after . RuntimeID != fixture . bound . RuntimeID || len ( fixture . gateway . containers ) != 1 {
t . Fatalf ( "strict list failure changed the generation: before=%#v after=%#v containers=%#v err=%v" ,
t . Fatalf ( "strict gateway validation changed the generation: before=%#v after=%#v containers=%#v err=%v" ,
fixture . bound , after , fixture . gateway . containers , err )
}
for _ , request := range fixture . gateway . recorded ( ) {
if request . method != http . MethodGet {
t . Fatalf ( "strict list failure advanced the lifecycle: %#v" , fixture . gateway . recorded ( ) )
t . Fatalf ( "strict gateway validation advanced the lifecycle: %#v" , fixture . gateway . recorded ( ) )
}
}
activeCount := 0
@@ -3389,13 +3357,13 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) {
t . Fatal ( err )
}
if outcome != "unknown" || reason != "gateway_result_unknown" {
t . Fatalf ( "strict list audit mismatch: outcome=%s reason=%s" , outcome , reason )
t . Fatalf ( "strict gateway audit mismatch: outcome=%s reason=%s" , outcome , reason )
}
}
response = do ( app , test . method , test . path , body )
if response . Code != test . wantStatus {
t . Fatalf ( "strict list retry returned %d: %s" , response . Code , response . Body . String ( ) )
t . Fatalf ( "strict gateway retry returned %d: %s" , response . Code , response . Body . String ( ) )
}
after , err = fixture . store . GetEnvironmentContext ( ctx , "account-a" )
fixture . gateway . mu . Lock ( )
@@ -3410,7 +3378,7 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) {
validRuntime = after . RuntimeID == "" && len ( containers ) == 0
}
if err != nil || after . BindingVersion != wantVersion || ! validRuntime {
t . Fatalf ( "strict list retry ended inconsistently: after=%#v containers=%#v err=%v" , after , containers , err )
t . Fatalf ( "strict gateway retry ended inconsistently: after=%#v containers=%#v err=%v" , after , containers , err )
}
activeAfter := 1
if test . action == "rebind" {
@@ -3425,7 +3393,7 @@ func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) {
t . Fatal ( err )
}
if outcome != "succeeded" || reason != test . successReason {
t . Fatalf ( "strict list retry audit mismatch: outcome=%s reason=%s" , outcome , reason )
t . Fatalf ( "strict gateway retry audit mismatch: outcome=%s reason=%s" , outcome , reason )
}
}
} )
@@ -3595,11 +3563,8 @@ func TestPostgresReconcileFinishedAuditUsesActivatedRuntime(t *testing.T) {
ID : "stale-container" , Alias : "account-a" , State : "running" , ProxyReady : true ,
BindingVersion : fixture . bound . BindingVersion , NetworkExitID : "stale-exit" , NetworkID : "network-old" ,
} }
app := fiber . New ( )
registerHubWithNetwork ( app , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return " ", nil } )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "reconcile returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , fixture . store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "reconcile failed: %v ", err )
}
after , err := fixture . store . GetEnvironmentContext ( context . Background ( ) , "account-a" )
if err != nil || after . RuntimeInstanceID == "" || after . RuntimeID != "container-id" {
@@ -3655,16 +3620,13 @@ func TestPostgresNonRunnableReconcileAuditsRuntimeRelease(t *testing.T) {
if releaseFailure {
store = failingRuntimeReleaseStore { Store : fixture . store , err : errors . New ( "release unavailable" ) }
}
app := fiber . New ( )
registerHubWithNetwork ( app , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
wantStatus := http . StatusOK
err = reconcileRuntimeLeases ( ctx , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
wantOutcome , wantReason := "succeeded" , "runtime_released"
if releaseFailure {
wantStatus , wantOutcome , wantReason = http . StatusInternalServerError , "failed" , "runtime_release_failed"
wantOutcome , wantReason = "failed" , "runtime_release_failed"
}
if response . Code != wantStatus {
t . Fatalf ( "reconcile returned %d, want %d: %s" , response . Code , wantStatus , response . Body . String ( ) )
if ( err != nil ) != releaseFailure {
t . Fatalf ( "reconcile error=%v, releaseFailure=%v" , err , releaseFailure )
}
after , err := fixture . store . GetEnvironmentContext ( ctx , "account-a" )
if err != nil || ( after . RuntimeInstanceID != "" ) == ! releaseFailure {
@@ -3767,20 +3729,31 @@ func TestPostgresImageDisableWaitsForEveryImageConsumer(t *testing.T) {
err error
}
lifecycleDone := make ( chan result , 1 )
go func ( ) {
request , err := http . NewRequest ( method , server . URL + path , strings . NewReader ( body ) )
if err = = nil {
request . Header . Set ( "Content-Type" , "application/json" )
var response * http . Response
response , err = server . Client ( ) . Do ( request )
if err == nil {
defer response . Body . Close ( )
lifecycleDone <- result { status : response . StatusCode }
if action == "reconcile" {
go func ( ) {
err : = reconcileRuntimeLeases ( context . Background ( ) , fixture . store , probe , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
if err != nil {
lifecycleDone <- result { err : err }
return
}
}
lifecycleDone <- result { err : err }
} ( )
lifecycleDone <- result { status : expected }
} ( )
} else {
go func ( ) {
request , err := http . NewRequest ( method , server . URL + path , strings . NewReader ( body ) )
if err == nil {
request . Header . Set ( "Content-Type" , "application/json" )
var response * http . Response
response , err = server . Client ( ) . Do ( request )
if err == nil {
defer response . Body . Close ( )
lifecycleDone <- result { status : response . StatusCode }
return
}
}
lifecycleDone <- result { err : err }
} ( )
}
select {
case <- lifecycleStarted :
case result := <- lifecycleDone :
@@ -3909,7 +3882,7 @@ func TestImageDisableWaitsForUpgradeCommit(t *testing.T) {
}
}
func TestListAndHeartbeatWaitForUpgradeCoordination ( t * testing . T ) {
func TestListDoesNotWaitForUpgradeWhileHeartbeatDoes ( t * testing . T ) {
releaseCreate := make ( chan struct { } )
gateway := & fakeGateway {
token : "unit-test-gateway-token" ,
@@ -3975,9 +3948,17 @@ func TestListAndHeartbeatWaitForUpgradeCoordination(t *testing.T) {
<- heartbeatStarted
select {
case result := <- listDone :
t . Fatalf ( "list reconciled a stale snapshot during upgrade: %#v" , result )
if result . err != nil || result . status != http . StatusOK {
t . Fatalf ( "list failed while upgrade was in progress: %#v" , result )
}
case err := <- heartbeatDone :
t . Fatalf ( "heartbeat reconciled a stale snapshot during upgrade : %v" , err )
t . Fatalf ( "heartbeat ignored upgrade coordination : %v" , err )
case <- time . After ( time . Second ) :
t . Fatal ( "list did not return while upgrade was in progress" )
}
select {
case err := <- heartbeatDone :
t . Fatalf ( "heartbeat finished before the upgrade released its resources: %v" , err )
case <- time . After ( time . Second ) :
}
@@ -3985,9 +3966,6 @@ func TestListAndHeartbeatWaitForUpgradeCoordination(t *testing.T) {
if result := <- upgradeDone ; result . err != nil || result . status != http . StatusNoContent {
t . Fatalf ( "upgrade failed: %#v" , result )
}
if result := <- listDone ; result . err != nil || result . status != http . StatusOK {
t . Fatalf ( "coordinated list failed: %#v" , result )
}
if err := <- heartbeatDone ; err != nil {
t . Fatalf ( "coordinated heartbeat failed: %v" , err )
}
@@ -4056,11 +4034,10 @@ func TestReconcileStopsRuntimeForPausedOrRevokedAccount(t *testing.T) {
gateway := & fakeGateway { token : "unit-test-gateway-token" , containers : [ ] containerStatus { {
ID : "old-container" , Alias : "account-a" , State : "running" , ProxyReady : true , BindingVersion : 1 , NetworkExitID : "exit-1" ,
} } }
app := newTestApp ( t , store , gateway )
_ = newTestApp ( t , store , gateway )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "reconcile returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "reconcile failed: %v" , err )
}
if runtime := store . bindings [ "account-a" ] . RuntimeID ; runtime != "" {
t . Fatalf ( "non-runnable account retained runtime %q" , runtime )
@@ -4183,11 +4160,10 @@ func TestReconcileFinishedAuditUsesRebuiltRuntime(t *testing.T) {
gateway := & fakeGateway { token : "unit-test-gateway-token" , containers : [ ] containerStatus { {
ID : "old-container" , Alias : "account-a" , State : "running" , ProxyReady : true , BindingVersion : 2 , NetworkExitID : "exit-1" , NetworkID : "network-old" ,
} } }
app := newTestApp ( t , store , gateway )
_ = newTestApp ( t , store , gateway )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "reconcile returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "reconcile failed: %v" , err )
}
if len ( store . actions ) != 2 || store . actions [ 1 ] . Outcome != "succeeded" ||
store . actions [ 1 ] . RuntimeInstanceID != "runtime-instance" || store . actions [ 1 ] . BindingVersion != 2 || store . actions [ 1 ] . NetworkExitID != "exit-2" {
@@ -4210,12 +4186,9 @@ func TestReconcileContextRefreshFailureClearsAllAuditCorrelation(t *testing.T) {
defer server . Close ( )
store . gateways [ "gw-1" ] = hub . Gateway { Name : "gw-1" , Endpoint : server . URL , Token : gateway . token }
failing := & failContextRefreshStore { hubStore : store }
app := fiber . New ( )
registerHubWithNetwork ( app , failing , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusInternalServerError {
t . Fatalf ( "context refresh failure returned %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , failing , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err == nil {
t . Fatal ( "context refresh failure unexpectedly succeeded" )
}
finished := store . actions [ len ( store . actions ) - 1 ]
if finished . RuntimeInstanceID != "" || finished . BindingVersion != 0 || finished . NetworkExitID != "" {
@@ -4252,7 +4225,7 @@ func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) {
}
}
func TestListReconcileAuditsRuntimeReleaseFailure ( t * testing . T ) {
func TestReconcileAuditsRuntimeReleaseFailure ( t * testing . T ) {
store := newMemoryStore ( )
store . envs [ "account-a" ] = hub . Env { Alias : "account-a" , Name : "甲" , Gateway : "gw-1" , ImageVersion : "148" , Fingerprint : hub . Fingerprint { Seed : 1 } }
store . bindings [ "account-a" ] = hub . EnvironmentContext {
@@ -4263,11 +4236,10 @@ func TestListReconcileAuditsRuntimeReleaseFailure(t *testing.T) {
gateway := & fakeGateway { token : "unit-test-gateway-token" , containers : [ ] containerStatus { {
ID : "container-id" , Alias : "account-a" , State : "exited" , BindingVersion : 1 , NetworkExitID : "exit-1" ,
} } }
app := newTestApp ( t , store , gateway )
_ = newTestApp ( t , store , gateway )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusInternalServerError {
t . Fatalf ( "expected release failure, got %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err == nil {
t . Fatal ( "expected release failure" )
}
if len ( store . actions ) != 2 || store . actions [ 0 ] . Action != "reconcile" || store . actions [ 1 ] . Outcome != "failed" ||
store . actions [ 1 ] . ReasonCode != "runtime_release_failed" {
@@ -4346,12 +4318,12 @@ func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) {
gateway := & fakeGateway { token : "unit-test-gateway-token" , containers : [ ] containerStatus { {
ID : "container-id" , Alias : "account-a" , State : "running" , BindingVersion : 1 , NetworkExitID : "exit-1" , ProxyReady : true ,
} } }
app := newTestAppWithNetwork ( t , store , gateway , fakeExitProbe { failure : "exit_auth_failed" } ,
probe := fakeExitProbe { failure : "exit_auth_failed" }
_ = newTestAppWithNetwork ( t , store , gateway , probe ,
func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusOK {
t . Fatalf ( "unhealthy runtime reconciliation failed: %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , probe , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err != nil {
t . Fatalf ( "unhealthy runtime reconciliation failed: %v" , err )
}
if runtime := store . bindings [ "account-a" ] . RuntimeID ; runtime != "" {
t . Fatalf ( "failed exit remained active: %q" , runtime )
@@ -4372,11 +4344,10 @@ func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) {
gateway := & fakeGateway { token : "unit-test-gateway-token" , failDelete : 1 , containers : [ ] containerStatus { {
ID : "stale-container" , Alias : "account-a" , State : "running" , BindingVersion : 2 , NetworkExitID : "stale-exit" , NetworkID : "network-old" , ProxyReady : true ,
} } }
app := newTestApp ( t , store , gateway )
_ = newTestApp ( t , store , gateway )
response := do ( app , http . MethodGet , "/api/browsers" , "" )
if response . Code != http . StatusBadGateway {
t . Fatalf ( "delete failure must remain unknown, got %d: %s" , response . Code , response . Body . String ( ) )
if err := reconcileRuntimeLeases ( context . Background ( ) , store , fakeExitProbe { } , func ( hub . NetworkExitAccess ) ( string , error ) { return "" , nil } ) ; err == nil {
t . Fatal ( "delete failure unexpectedly succeeded" )
}
if runtime := store . bindings [ "account-a" ] . RuntimeID ; runtime != "" {
t . Fatalf ( "delete failure retained DB runtime lease: %q" , runtime )
@@ -4653,7 +4624,6 @@ func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) {
for _ , test := range [ ] struct {
name , method , path , body string
} {
{ name : "list reconcile" , method : http . MethodGet , path : "/api/browsers" } ,
{ name : "create reuse" , method : http . MethodPost , path : "/api/browsers" , body : ` { "alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint": { "seed":1},"account_id":"account-a","network_exit_id":"exit-1"} ` } ,
{ name : "start" , method : http . MethodPost , path : "/api/browsers/account-a/start" } ,
{ name : "upgrade" , method : http . MethodPost , path : "/api/browsers/account-a/upgrade" , body : ` { "version":"149"} ` } ,