3686 lines
174 KiB
Go
3686 lines
174 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
accountdomain "git.ipao.vip/rogee/creator-hub/internal/account"
|
|
hub "git.ipao.vip/rogee/creator-hub/internal/environment"
|
|
"github.com/gofiber/fiber/v3"
|
|
"github.com/gofiber/fiber/v3/middleware/adaptor"
|
|
)
|
|
|
|
// memoryStore 是 hubStore 的内存桩,记录写入以便断言编排副作用。
|
|
type memoryStore struct {
|
|
mu sync.Mutex
|
|
locksMu sync.Mutex
|
|
locks map[string]*sync.Mutex
|
|
gateways map[string]hub.Gateway
|
|
envs map[string]hub.Env
|
|
exits map[string]hub.NetworkExit
|
|
bindings map[string]hub.EnvironmentContext
|
|
actions []hub.EnvironmentAction
|
|
upgraded map[string]string
|
|
upgradeErr error
|
|
releaseErr error
|
|
cleanupPendingErr error
|
|
cleanupPendingErrAfterMutation bool
|
|
gatewayFn func(name string) (hub.Gateway, error)
|
|
}
|
|
|
|
type blockingRuntimeStopStore struct {
|
|
*hub.Store
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
func (s *blockingRuntimeStopStore) GetGateway(context.Context, string) (hub.Gateway, error) {
|
|
s.entered <- struct{}{}
|
|
<-s.release
|
|
return hub.Gateway{}, errors.New("gateway unavailable")
|
|
}
|
|
|
|
func newMemoryStore() *memoryStore {
|
|
return &memoryStore{
|
|
locks: map[string]*sync.Mutex{},
|
|
gateways: map[string]hub.Gateway{},
|
|
envs: map[string]hub.Env{},
|
|
exits: map[string]hub.NetworkExit{
|
|
"exit-1": {ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1},
|
|
},
|
|
bindings: map[string]hub.EnvironmentContext{},
|
|
upgraded: map[string]string{},
|
|
}
|
|
}
|
|
|
|
func TestResumeBlockReasonIsStable(t *testing.T) {
|
|
account := accountdomain.Account{AuthorizationStatus: "authorized"}
|
|
healthy := hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "healthy"}}
|
|
for name, test := range map[string]struct {
|
|
account accountdomain.Account
|
|
env hub.EnvironmentContext
|
|
found bool
|
|
want string
|
|
}{
|
|
"revoked": {accountdomain.Account{AuthorizationStatus: "revoked"}, healthy, true, "account_revoked"},
|
|
"missing binding": {account, hub.EnvironmentContext{}, false, "binding_missing"},
|
|
"direct exit": {account, hub.EnvironmentContext{}, true, "account_conflict"},
|
|
"unhealthy exit": {account, hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "unhealthy"}}, true, "network_exit_unhealthy"},
|
|
"cleanup pending": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeCleanupPending: true}, true, "runtime_stop_pending"},
|
|
"runtime active": {account, hub.EnvironmentContext{Exit: healthy.Exit, RuntimeInstanceID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, true, "runtime_active"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if got := resumeBlockReason(test.account, test.env, test.found); got != test.want {
|
|
t.Fatalf("got %q, want %q", got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPhaseAReadinessErrorsAreStructured(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name, reason string
|
|
unavailable bool
|
|
status int
|
|
}{
|
|
{name: "version conflict", reason: "draft_version_changed", status: http.StatusConflict},
|
|
{name: "resource unavailable", reason: "network_exit_unhealthy", unavailable: true, status: http.StatusServiceUnavailable},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
app := fiber.New()
|
|
app.Get("/", func(c fiber.Ctx) error {
|
|
return phaseAError(c, &accountdomain.ReadinessError{Reason: test.reason, Unavailable: test.unavailable})
|
|
})
|
|
response := do(app, http.MethodGet, "/", "")
|
|
if response.Code != test.status || !strings.Contains(response.Body.String(), `"reason_code":"`+test.reason+`"`) {
|
|
t.Fatalf("unexpected response: %d %s", response.Code, response.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func (s *memoryStore) LockResources(_ context.Context, aliases, exitIDs []string, imageVersionSets ...[]string) (func(), error) {
|
|
imageVersions := []string(nil)
|
|
if len(imageVersionSets) > 0 {
|
|
imageVersions = imageVersionSets[0]
|
|
}
|
|
keys := make([]string, 0, len(aliases)+len(exitIDs)+len(imageVersions))
|
|
for _, alias := range aliases {
|
|
keys = append(keys, "environment:"+alias)
|
|
}
|
|
for _, id := range exitIDs {
|
|
keys = append(keys, "network-exit:"+id)
|
|
}
|
|
for _, version := range imageVersions {
|
|
keys = append(keys, "image:"+version)
|
|
}
|
|
sort.Strings(keys)
|
|
unlocks := make([]func(), 0, len(keys))
|
|
seen := map[string]bool{}
|
|
for _, key := range keys {
|
|
if !seen[key] {
|
|
seen[key] = true
|
|
unlocks = append(unlocks, s.lock(key))
|
|
}
|
|
}
|
|
return func() {
|
|
for index := len(unlocks) - 1; index >= 0; index-- {
|
|
unlocks[index]()
|
|
}
|
|
}, nil
|
|
}
|
|
|
|
func (s *memoryStore) lock(key string) func() {
|
|
s.locksMu.Lock()
|
|
lock := s.locks[key]
|
|
if lock == nil {
|
|
lock = &sync.Mutex{}
|
|
s.locks[key] = lock
|
|
}
|
|
s.locksMu.Unlock()
|
|
lock.Lock()
|
|
return lock.Unlock
|
|
}
|
|
|
|
func (s *memoryStore) CreateGateway(_ context.Context, _, _, _ string) (hub.Gateway, error) {
|
|
return hub.Gateway{}, nil
|
|
}
|
|
func (s *memoryStore) UpdateGateway(_ context.Context, currentName, name, endpoint, token string) (hub.Gateway, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
gateway, exists := s.gateways[currentName]
|
|
if !exists {
|
|
return hub.Gateway{}, hub.ErrNotFound
|
|
}
|
|
if currentName != name {
|
|
if _, exists := s.gateways[name]; exists {
|
|
return hub.Gateway{}, hub.ErrConflict
|
|
}
|
|
delete(s.gateways, currentName)
|
|
for alias, env := range s.envs {
|
|
if env.Gateway == currentName {
|
|
env.Gateway = name
|
|
s.envs[alias] = env
|
|
}
|
|
}
|
|
}
|
|
gateway.Name, gateway.Endpoint = name, endpoint
|
|
if token != "" {
|
|
gateway.Token = token
|
|
}
|
|
s.gateways[name] = gateway
|
|
return gateway, nil
|
|
}
|
|
func (s *memoryStore) ListGateways(context.Context) ([]hub.Gateway, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
gateways := make([]hub.Gateway, 0, len(s.gateways))
|
|
for _, gateway := range s.gateways {
|
|
gateways = append(gateways, gateway)
|
|
}
|
|
return gateways, nil
|
|
}
|
|
func (s *memoryStore) GetGateway(_ context.Context, name string) (hub.Gateway, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.gatewayFn != nil {
|
|
return s.gatewayFn(name)
|
|
}
|
|
gateway, ok := s.gateways[name]
|
|
if !ok {
|
|
return hub.Gateway{}, hub.ErrNotFound
|
|
}
|
|
return gateway, nil
|
|
}
|
|
func (s *memoryStore) DeleteGateway(context.Context, string) error { return nil }
|
|
func (s *memoryStore) CreateEnv(_ context.Context, env hub.Env) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if _, exists := s.envs[env.Alias]; exists {
|
|
return hub.ErrConflict
|
|
}
|
|
s.envs[env.Alias] = env
|
|
return nil
|
|
}
|
|
func (s *memoryStore) ListEnvs(context.Context) ([]hub.Env, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
envs := make([]hub.Env, 0, len(s.envs))
|
|
for _, env := range s.envs {
|
|
envs = append(envs, env)
|
|
}
|
|
return envs, nil
|
|
}
|
|
func (s *memoryStore) GetEnv(_ context.Context, alias string) (hub.Env, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
env, ok := s.envs[alias]
|
|
if !ok {
|
|
return hub.Env{}, hub.ErrNotFound
|
|
}
|
|
return env, nil
|
|
}
|
|
func (s *memoryStore) CreateNetworkExit(_ context.Context, exit hub.NetworkExit) (hub.NetworkExit, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if len(exit.Username) > 255 || len(exit.Password) > 255 || (exit.Username == "" && exit.Password != "") {
|
|
return hub.NetworkExit{}, hub.ErrInvalid
|
|
}
|
|
exit.ID, exit.HealthStatus, exit.Version = "exit-created", "unchecked", 1
|
|
s.exits[exit.ID] = exit
|
|
return exit, nil
|
|
}
|
|
func (s *memoryStore) ListNetworkExits(context.Context) ([]hub.NetworkExit, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
result := make([]hub.NetworkExit, 0, len(s.exits))
|
|
for _, exit := range s.exits {
|
|
result = append(result, exit)
|
|
}
|
|
return result, nil
|
|
}
|
|
func (s *memoryStore) GetNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
exit, ok := s.exits[id]
|
|
if !ok {
|
|
return hub.NetworkExit{}, hub.ErrNotFound
|
|
}
|
|
return exit, nil
|
|
}
|
|
func (s *memoryStore) GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error) {
|
|
exit, err := s.GetNetworkExit(ctx, id)
|
|
return hub.NetworkExitAccess{NetworkExit: exit}, err
|
|
}
|
|
func (s *memoryStore) RecordNetworkExitCheck(_ context.Context, id string, observation hub.ExitObservation, failure string) (hub.NetworkExit, string, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
exit, ok := s.exits[id]
|
|
if !ok {
|
|
return hub.NetworkExit{}, "exit_unavailable", hub.ErrNotFound
|
|
}
|
|
reason := failure
|
|
if reason == "" && exit.ExpectedPublicIP != "" && exit.ExpectedPublicIP != observation.PublicIP {
|
|
reason = "exit_ip_drift"
|
|
}
|
|
if reason == "" && exit.ExpectedRegion != "" && exit.ExpectedRegion != observation.Region {
|
|
reason = "exit_region_drift"
|
|
}
|
|
exit.ObservedPublicIP, exit.ObservedRegion = observation.PublicIP, observation.Region
|
|
if reason == "" {
|
|
exit.HealthStatus, reason = "healthy", "exit_healthy"
|
|
} else {
|
|
exit.HealthStatus = "unhealthy"
|
|
}
|
|
s.exits[id] = exit
|
|
return exit, reason, nil
|
|
}
|
|
func (s *memoryStore) DisableNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
exit, ok := s.exits[id]
|
|
if !ok {
|
|
return hub.NetworkExit{}, hub.ErrNotFound
|
|
}
|
|
exit.HealthStatus = "disabled"
|
|
s.exits[id] = exit
|
|
return exit, nil
|
|
}
|
|
func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) {
|
|
s.mu.Lock()
|
|
if existing, ok := s.bindings[env.Alias]; ok {
|
|
if existing.AccountStatus == "" {
|
|
existing.AccountStatus, existing.AuthorizationStatus = "active", "authorized"
|
|
}
|
|
s.mu.Unlock()
|
|
return existing, false, nil
|
|
}
|
|
s.mu.Unlock()
|
|
if err := s.CreateEnv(ctx, env); err != nil {
|
|
return hub.EnvironmentContext{}, false, err
|
|
}
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
bound := hub.EnvironmentContext{Env: env, AccountID: accountID, AccountStatus: "active", AuthorizationStatus: "authorized",
|
|
BindingID: accountID, BindingVersion: 1, Exit: s.exits[exitID]}
|
|
s.bindings[env.Alias] = bound
|
|
return bound, true, nil
|
|
}
|
|
func (s *memoryStore) GetEnvironmentContext(_ context.Context, alias string) (hub.EnvironmentContext, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if bound, ok := s.bindings[alias]; ok {
|
|
if bound.AccountStatus == "" {
|
|
bound.AccountStatus, bound.AuthorizationStatus = "active", "authorized"
|
|
}
|
|
return bound, nil
|
|
}
|
|
env, ok := s.envs[alias]
|
|
if !ok {
|
|
return hub.EnvironmentContext{}, hub.ErrNotFound
|
|
}
|
|
return hub.EnvironmentContext{Env: env, AccountID: alias, AccountStatus: "active", AuthorizationStatus: "authorized",
|
|
BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}, nil
|
|
}
|
|
func (s *memoryStore) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) {
|
|
s.mu.Lock()
|
|
for alias, bound := range s.bindings {
|
|
if bound.AccountID == accountID {
|
|
s.mu.Unlock()
|
|
return s.GetEnvironmentContext(ctx, alias)
|
|
}
|
|
}
|
|
s.mu.Unlock()
|
|
return hub.EnvironmentContext{}, hub.ErrNotFound
|
|
}
|
|
func (s *memoryStore) ValidateEnvironmentRebind(_ context.Context, alias, exitID string, expectedBindingVersion int64) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
bound, ok := s.bindings[alias]
|
|
if !ok {
|
|
bound.BindingVersion = 1
|
|
}
|
|
if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending || s.exits[exitID].HealthStatus != "healthy" {
|
|
return hub.ErrConflict
|
|
}
|
|
return nil
|
|
}
|
|
func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkIDs ...string) (hub.EnvironmentContext, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
bound, ok := s.bindings[alias]
|
|
if !ok {
|
|
bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
|
|
}
|
|
if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending {
|
|
return hub.EnvironmentContext{}, hub.ErrConflict
|
|
}
|
|
bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", ""
|
|
if runtimeID != "" {
|
|
bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID
|
|
if len(networkIDs) == 1 {
|
|
bound.RuntimeNetworkID = networkIDs[0]
|
|
}
|
|
}
|
|
bound.Exit, bound.BindingVersion = s.exits[exitID], bound.BindingVersion+1
|
|
s.bindings[alias] = bound
|
|
return bound, nil
|
|
}
|
|
func (s *memoryStore) ActivateRuntime(_ context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkIDs ...string) (hub.EnvironmentContext, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if len(networkIDs) != 1 || networkIDs[0] == "" {
|
|
return hub.EnvironmentContext{}, hub.ErrInvalid
|
|
}
|
|
bound, ok := s.bindings[alias]
|
|
if !ok {
|
|
bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
|
|
}
|
|
if bound.AccountStatus == "" {
|
|
bound.AccountStatus, bound.AuthorizationStatus = "active", "authorized"
|
|
}
|
|
if bound.AccountStatus != "active" || bound.AuthorizationStatus != "authorized" ||
|
|
bound.BindingVersion != bindingVersion || bound.Exit.ID != exitID || bound.RuntimeCleanupPending {
|
|
return hub.EnvironmentContext{}, hub.ErrConflict
|
|
}
|
|
bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID
|
|
bound.RuntimeNetworkID = networkIDs[0]
|
|
s.bindings[alias] = bound
|
|
return bound, nil
|
|
}
|
|
func (s *memoryStore) ReleaseRuntime(_ context.Context, environment hub.EnvironmentContext) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.releaseErr != nil {
|
|
return s.releaseErr
|
|
}
|
|
bound, ok := s.bindings[environment.Alias]
|
|
if !ok {
|
|
bound = hub.EnvironmentContext{Env: s.envs[environment.Alias], AccountID: environment.Alias,
|
|
BindingID: environment.Alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
|
|
}
|
|
if environment.RuntimeInstanceID == "" {
|
|
return nil
|
|
}
|
|
if bound.BindingID != environment.BindingID || bound.BindingVersion != environment.BindingVersion ||
|
|
bound.RuntimeInstanceID != environment.RuntimeInstanceID {
|
|
return hub.ErrConflict
|
|
}
|
|
bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", ""
|
|
s.bindings[environment.Alias] = bound
|
|
return nil
|
|
}
|
|
func (s *memoryStore) SetRuntimeCleanupPending(_ context.Context, environment hub.EnvironmentContext, pending bool) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if pending && environment.RuntimeCleanupRuntimeID == "" {
|
|
return hub.ErrInvalid
|
|
}
|
|
if s.cleanupPendingErr != nil && !s.cleanupPendingErrAfterMutation {
|
|
return s.cleanupPendingErr
|
|
}
|
|
if pending && s.releaseErr != nil {
|
|
return s.releaseErr
|
|
}
|
|
bound, ok := s.bindings[environment.Alias]
|
|
if !ok {
|
|
bound = hub.EnvironmentContext{Env: s.envs[environment.Alias], AccountID: environment.Alias,
|
|
BindingID: environment.Alias, BindingVersion: 1, Exit: s.exits["exit-1"]}
|
|
}
|
|
if bound.BindingID != environment.BindingID || bound.BindingVersion != environment.BindingVersion {
|
|
return hub.ErrConflict
|
|
}
|
|
if bound.RuntimeCleanupPending {
|
|
if bound.RuntimeCleanupBindingVersion != environment.RuntimeCleanupBindingVersion ||
|
|
bound.RuntimeCleanupInstanceID != environment.RuntimeCleanupInstanceID ||
|
|
bound.RuntimeCleanupRuntimeID != environment.RuntimeCleanupRuntimeID ||
|
|
bound.RuntimeCleanupNetworkID != environment.RuntimeCleanupNetworkID {
|
|
return hub.ErrConflict
|
|
}
|
|
} else if pending && bound.RuntimeInstanceID != environment.RuntimeCleanupInstanceID {
|
|
return hub.ErrConflict
|
|
}
|
|
bound.RuntimeCleanupPending = pending
|
|
if pending {
|
|
bound.RuntimeCleanupBindingVersion = environment.RuntimeCleanupBindingVersion
|
|
bound.RuntimeCleanupInstanceID = environment.RuntimeCleanupInstanceID
|
|
bound.RuntimeCleanupRuntimeID = environment.RuntimeCleanupRuntimeID
|
|
bound.RuntimeCleanupNetworkID = environment.RuntimeCleanupNetworkID
|
|
bound.RuntimeInstanceID, bound.RuntimeID, bound.RuntimeNetworkID = "", "", ""
|
|
} else {
|
|
bound.RuntimeCleanupBindingVersion = 0
|
|
bound.RuntimeCleanupInstanceID, bound.RuntimeCleanupRuntimeID, bound.RuntimeCleanupNetworkID = "", "", ""
|
|
}
|
|
s.bindings[environment.Alias] = bound
|
|
return s.cleanupPendingErr
|
|
}
|
|
func (s *memoryStore) AppendEnvironmentAction(_ context.Context, _ string, action hub.EnvironmentAction) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.actions = append(s.actions, action)
|
|
return nil
|
|
}
|
|
|
|
func TestPhaseARuntimeLeaseRoutesAreInternal(t *testing.T) {
|
|
app := fiber.New()
|
|
registerPhaseA(app, nil, nil, nil)
|
|
for _, request := range []struct {
|
|
method, path string
|
|
}{
|
|
{http.MethodPost, "/api/phase-a/runtimes"},
|
|
{http.MethodDelete, "/api/phase-a/runtimes/runtime-a"},
|
|
} {
|
|
if response := do(app, request.method, request.path, `{}`); response.Code != http.StatusNotFound {
|
|
t.Fatalf("%s %s remained public: %d", request.method, request.path, response.Code)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPauseSerializesResumeAndRetainsClaimGateOnUnknownStop(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
accountStore, err := accountdomain.Open(ctx, fixture.databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = accountStore.Close() })
|
|
if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
blocking := &blockingRuntimeStopStore{Store: fixture.store, entered: make(chan struct{}, 1), release: make(chan struct{})}
|
|
app := fiber.New()
|
|
registerPhaseA(app, accountStore, blocking, nil)
|
|
pauseDone := make(chan *httptest.ResponseRecorder, 1)
|
|
resumeDone := make(chan *httptest.ResponseRecorder, 1)
|
|
go func() { pauseDone <- do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", "") }()
|
|
select {
|
|
case <-blocking.entered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("pause did not enter runtime stop")
|
|
}
|
|
go func() { resumeDone <- do(app, http.MethodPost, "/api/phase-a/accounts/account-a/resume", "") }()
|
|
select {
|
|
case response := <-resumeDone:
|
|
t.Fatalf("resume bypassed the runtime operation lock: %d", response.Code)
|
|
case <-time.After(time.Second):
|
|
}
|
|
close(blocking.release)
|
|
if response := <-pauseDone; response.Code != http.StatusInternalServerError {
|
|
t.Fatalf("unknown stop returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if response := <-resumeDone; response.Code != http.StatusConflict {
|
|
t.Fatalf("resume reopened an unconverged lease: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
account, err := accountStore.GetAccount(ctx, "account-a")
|
|
if err != nil || account.RuntimeStatus != "paused" {
|
|
t.Fatalf("resume reopened the claim gate: account=%#v err=%v", account, err)
|
|
}
|
|
}
|
|
|
|
type recordedRequest struct {
|
|
method string
|
|
path string
|
|
body map[string]any
|
|
}
|
|
|
|
// fakeGateway 模拟 native browser gateway:按路由表应答并记录请求。
|
|
type fakeGateway struct {
|
|
mu sync.Mutex
|
|
createOnce sync.Once
|
|
deleteOnce sync.Once
|
|
token string
|
|
requests []recordedRequest
|
|
runtimes []runtimeStatus
|
|
failCreate int // 前 N 次 create 返回失败
|
|
failCreateStatus int
|
|
failDelete int // 前 N 次 delete 返回 500 且保留容器
|
|
deleteNotFound int
|
|
failProxy bool
|
|
createStarted chan struct{}
|
|
releaseCreate <-chan struct{}
|
|
deleteDone chan struct{}
|
|
releaseDelete <-chan struct{}
|
|
cleanupPending int
|
|
disconnectDelete int
|
|
disconnectList int
|
|
failList int
|
|
invalidList int
|
|
invalidListBody string
|
|
readErrorList int
|
|
disconnectListAfterDelete bool
|
|
}
|
|
|
|
type fakeExitProbe struct {
|
|
observation hub.ExitObservation
|
|
failure string
|
|
}
|
|
|
|
type blockingExitProbe struct {
|
|
started chan struct{}
|
|
release <-chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func (probe *blockingExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) {
|
|
probe.once.Do(func() { close(probe.started) })
|
|
<-probe.release
|
|
return hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, ""
|
|
}
|
|
|
|
func (probe fakeExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) {
|
|
if probe.observation.PublicIP == "" && probe.failure == "" {
|
|
probe.observation = hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}
|
|
}
|
|
return probe.observation, probe.failure
|
|
}
|
|
|
|
type sequenceExitProbe struct {
|
|
calls int
|
|
failures []string
|
|
}
|
|
|
|
func (probe *sequenceExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) {
|
|
failure := ""
|
|
if probe.calls < len(probe.failures) {
|
|
failure = probe.failures[probe.calls]
|
|
}
|
|
probe.calls++
|
|
return hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, failure
|
|
}
|
|
|
|
func (g *fakeGateway) handler(t *testing.T) http.Handler {
|
|
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.Method == http.MethodGet && request.URL.Path == "/healthz" {
|
|
response.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if request.Header.Get("Authorization") != "Bearer "+g.token {
|
|
response.WriteHeader(http.StatusUnauthorized)
|
|
_, _ = response.Write([]byte(`{"error":"gateway token rejected"}`))
|
|
return
|
|
}
|
|
var body map[string]any
|
|
if request.Body != nil {
|
|
raw, _ := io.ReadAll(request.Body)
|
|
if len(raw) > 0 {
|
|
_ = json.Unmarshal(raw, &body)
|
|
}
|
|
}
|
|
g.mu.Lock()
|
|
g.requests = append(g.requests, recordedRequest{method: request.Method, path: request.URL.Path, body: body})
|
|
g.mu.Unlock()
|
|
|
|
switch {
|
|
case request.Method == http.MethodPost && request.URL.Path == "/v1/browsers":
|
|
g.mu.Lock()
|
|
if g.failCreate > 0 {
|
|
g.failCreate--
|
|
status := http.StatusConflict
|
|
if g.failCreateStatus != 0 {
|
|
status = g.failCreateStatus
|
|
}
|
|
g.mu.Unlock()
|
|
response.WriteHeader(status)
|
|
_, _ = response.Write([]byte(`{"error":"alias already in use"}`))
|
|
return
|
|
}
|
|
g.mu.Unlock()
|
|
if g.createStarted != nil {
|
|
g.createOnce.Do(func() { close(g.createStarted) })
|
|
}
|
|
if g.releaseCreate != nil {
|
|
<-g.releaseCreate
|
|
}
|
|
state, proxyReady := "running", true
|
|
networkID := "native-cccccccccccccccccccccccccccccccc"
|
|
if stopped, _ := body["stopped"].(bool); stopped {
|
|
state, proxyReady, networkID = "exited", false, ""
|
|
}
|
|
g.mu.Lock()
|
|
g.runtimes = []runtimeStatus{{
|
|
ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady,
|
|
BindingVersion: int64(body["binding_version"].(float64)), NetworkExitID: body["network_exit_id"].(string),
|
|
NetworkID: networkID,
|
|
}}
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusCreated)
|
|
_, _ = response.Write([]byte(`{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a","network_id":"` + networkID + `"}`))
|
|
case request.Method == http.MethodGet && request.URL.Path == "/v1/browsers":
|
|
g.mu.Lock()
|
|
if g.failList > 0 {
|
|
g.failList--
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = response.Write([]byte(`{"error":"docker unavailable"}`))
|
|
return
|
|
}
|
|
if g.invalidList > 0 {
|
|
g.invalidList--
|
|
body := g.invalidListBody
|
|
if body == "" {
|
|
body = `{"not":"a browser list"}`
|
|
}
|
|
g.mu.Unlock()
|
|
_, _ = response.Write([]byte(body))
|
|
return
|
|
}
|
|
if g.readErrorList > 0 {
|
|
g.readErrorList--
|
|
g.mu.Unlock()
|
|
connection, _, _ := response.(http.Hijacker).Hijack()
|
|
_, _ = connection.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n[]"))
|
|
_ = connection.Close()
|
|
return
|
|
}
|
|
if g.disconnectList > 0 {
|
|
g.disconnectList--
|
|
g.mu.Unlock()
|
|
connection, _, _ := response.(http.Hijacker).Hijack()
|
|
_ = connection.Close()
|
|
return
|
|
}
|
|
runtimes := append([]runtimeStatus{}, g.runtimes...)
|
|
g.mu.Unlock()
|
|
_ = json.NewEncoder(response).Encode(runtimes)
|
|
case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/v1/browsers/"):
|
|
g.mu.Lock()
|
|
if len(g.runtimes) > 0 && (body["runtime_id"] != g.runtimes[0].ID ||
|
|
int64(body["binding_version"].(float64)) != g.runtimes[0].BindingVersion) {
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusConflict)
|
|
return
|
|
}
|
|
if g.failDelete > 0 {
|
|
g.failDelete--
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = response.Write([]byte(`{"error":"docker delete failed"}`))
|
|
return
|
|
}
|
|
g.runtimes = nil
|
|
if g.deleteNotFound > 0 {
|
|
g.deleteNotFound--
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
g.mu.Unlock()
|
|
if g.deleteDone != nil {
|
|
g.deleteOnce.Do(func() { close(g.deleteDone) })
|
|
}
|
|
if g.releaseDelete != nil {
|
|
<-g.releaseDelete
|
|
}
|
|
g.mu.Lock()
|
|
if g.cleanupPending > 0 {
|
|
g.cleanupPending--
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusAccepted)
|
|
return
|
|
}
|
|
if g.disconnectDelete > 0 {
|
|
g.disconnectDelete--
|
|
if g.disconnectListAfterDelete {
|
|
g.disconnectList++
|
|
}
|
|
g.mu.Unlock()
|
|
connection, _, _ := response.(http.Hijacker).Hijack()
|
|
_ = connection.Close()
|
|
return
|
|
}
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusNoContent)
|
|
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"):
|
|
g.mu.Lock()
|
|
if len(g.runtimes) > 0 {
|
|
g.runtimes[0].State, g.runtimes[0].Status = "running", "Up"
|
|
}
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusNoContent)
|
|
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"):
|
|
g.mu.Lock()
|
|
if len(g.runtimes) > 0 && (body["runtime_id"] != g.runtimes[0].ID ||
|
|
int64(body["binding_version"].(float64)) != g.runtimes[0].BindingVersion) {
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusConflict)
|
|
return
|
|
}
|
|
if len(g.runtimes) > 0 {
|
|
g.runtimes[0].State, g.runtimes[0].Status = "exited", "Exited"
|
|
}
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusNoContent)
|
|
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/proxy"):
|
|
g.mu.Lock()
|
|
if len(g.runtimes) > 0 && body["runtime_id"] != g.runtimes[0].ID {
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusConflict)
|
|
return
|
|
}
|
|
if g.failProxy {
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusBadGateway)
|
|
return
|
|
}
|
|
if len(g.runtimes) > 0 {
|
|
g.runtimes[0].ProxyReady = true
|
|
}
|
|
g.mu.Unlock()
|
|
response.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path)
|
|
}
|
|
})
|
|
}
|
|
|
|
func (g *fakeGateway) recorded() []recordedRequest {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
return append([]recordedRequest{}, g.requests...)
|
|
}
|
|
|
|
func newTestApp(t *testing.T, store *memoryStore, gateway *fakeGateway) *fiber.App {
|
|
return newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
}
|
|
|
|
func newTestAppWithNetwork(t *testing.T, store *memoryStore, gateway *fakeGateway, probe NetworkExitProbe,
|
|
resolve func(hub.NetworkExitAccess) (string, error)) *fiber.App {
|
|
t.Helper()
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
t.Cleanup(server.Close)
|
|
if store.gateways == nil {
|
|
store.gateways = map[string]hub.Gateway{}
|
|
}
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, probe, resolve)
|
|
return app
|
|
}
|
|
|
|
func do(app *fiber.App, method, path, body string, credentials ...string) *httptest.ResponseRecorder {
|
|
response := httptest.NewRecorder()
|
|
var reader io.Reader
|
|
if body != "" {
|
|
reader = strings.NewReader(body)
|
|
}
|
|
request := httptest.NewRequest(method, path, reader)
|
|
if len(credentials) == 2 {
|
|
request.SetBasicAuth(credentials[0], credentials[1])
|
|
}
|
|
adaptor.FiberApp(app).ServeHTTP(response, request)
|
|
return response
|
|
}
|
|
|
|
const createEnvBody = `{"alias":"account-a","name":"店铺一号","gateway":"gw-1",` +
|
|
`"fingerprint":{"seed":2024,"platform":"windows","timezone":"Asia/Shanghai"},"account_id":"account-a","network_exit_id":"exit-1"}`
|
|
|
|
func TestUpdateGatewayRenamesAndPreservesReferences(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: "http://gw-1:8081", Token: "unit-test-gateway-token"}
|
|
store.gateways["gw-existing"] = hub.Gateway{Name: "gw-existing", Endpoint: "http://gw-existing:8081", Token: "existing-gateway-token"}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPut, "/api/gateways/gw-1", `{"name":"gw-main","endpoint":"http://gw-main:8081","token":""}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("update gateway returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
var updated hub.Gateway
|
|
if err := json.Unmarshal(response.Body.Bytes(), &updated); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if updated.Name != "gw-main" || updated.Endpoint != "http://gw-main:8081" || updated.Token != "unit-test-gateway-token" {
|
|
t.Fatalf("gateway update lost fields: %#v", updated)
|
|
}
|
|
if store.envs["account-a"].Gateway != "gw-main" {
|
|
t.Fatalf("environment gateway reference was not renamed: %#v", store.envs["account-a"])
|
|
}
|
|
|
|
response = do(app, http.MethodPut, "/api/gateways/gw-main", `{"name":"gw-existing","endpoint":"http://gw-main:8081","token":""}`)
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("rename conflict returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
response = do(app, http.MethodPut, "/api/gateways/missing", `{"name":"gw-new","endpoint":"http://gw-new:8081","token":""}`)
|
|
if response.Code != http.StatusNotFound {
|
|
t.Fatalf("missing gateway update returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestListGatewaysReturnsStoredRecordsWithoutProbing(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
t.Cleanup(server.Close)
|
|
stored := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
store.gateways[stored.Name] = stored
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodGet, "/api/gateways", "")
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("list gateways returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
var gateways []hub.Gateway
|
|
if err := json.Unmarshal(response.Body.Bytes(), &gateways); err != nil || len(gateways) != 1 || gateways[0] != stored {
|
|
t.Fatalf("stored gateway was not returned unchanged: gateways=%#v err=%v", gateways, err)
|
|
}
|
|
if strings.Contains(response.Body.String(), "connectivity") || strings.Contains(response.Body.String(), "health") {
|
|
t.Fatalf("gateway list exposed live status fields: %s", response.Body.String())
|
|
}
|
|
if requests := gateway.recorded(); len(requests) != 0 {
|
|
t.Fatalf("gateway list performed live probes: %#v", requests)
|
|
}
|
|
}
|
|
|
|
func TestParseGatewayBrowserListStrict(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
body string
|
|
ok bool
|
|
}{
|
|
{name: "empty", body: `[]`, ok: true},
|
|
{name: "minimal browser", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a","state":"running"}]`, ok: true},
|
|
{name: "top-level null", body: `null`},
|
|
{name: "null element", body: `[null]`},
|
|
{name: "missing id", body: `[{"alias":"account-a","state":"running"}]`},
|
|
{name: "missing alias", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"running"}]`},
|
|
{name: "missing state", body: `[{"id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","alias":"account-a"}]`},
|
|
{name: "duplicate alias", body: `[{"id":"one","alias":"account-a","state":"running"},{"id":"two","alias":"account-a","state":"exited"}]`},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
browsers, err := parseGatewayBrowserList([]byte(test.body))
|
|
if (err == nil) != test.ok {
|
|
t.Fatalf("parse result browsers=%#v err=%v", browsers, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGatewayCallPropagatesBodyReadError(t *testing.T) {
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", readErrorList: 1}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
|
|
status, body, err := gatewayCall(context.Background(), hub.Gateway{Endpoint: server.URL, Token: gateway.token},
|
|
http.MethodGet, "/v1/browsers", nil, time.Second)
|
|
if status != http.StatusOK || string(body) != "[]" || err == nil {
|
|
t.Fatalf("partial response was accepted: status=%d body=%q err=%v", status, body, err)
|
|
}
|
|
}
|
|
|
|
func TestGatewayCreatePayloadStripsLegacyProxyFingerprint(t *testing.T) {
|
|
payload := gatewayCreatePayload(hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Name: "甲", Fingerprint: hub.Fingerprint{
|
|
Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true,
|
|
}}, BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1"}}, "/opt/creatorhub/browsers/1", gatewayNetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080})
|
|
encoded, _ := json.Marshal(payload)
|
|
if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") {
|
|
t.Fatalf("legacy proxy URI entered the gateway contract: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserSupportsDirectMachineExit(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{failure: "must_not_probe"}, func(hub.NetworkExitAccess) (string, error) {
|
|
return "", errors.New("must not resolve direct exit credentials")
|
|
})
|
|
body := `{"alias":"direct-env","name":"直连环境","gateway":"gw-1",` +
|
|
`"fingerprint":{"seed":2024},"account_id":"account-a","network_exit_id":""}`
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", body)
|
|
if response.Code != http.StatusCreated {
|
|
t.Fatalf("expected direct create 201, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
bound := store.bindings["direct-env"]
|
|
if bound.Exit.ID != "" || bound.RuntimeID == "" {
|
|
t.Fatalf("direct binding was not activated: %#v", bound)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 1 || requests[0].body["network_exit_id"] != "" {
|
|
t.Fatalf("direct create sent an unexpected gateway payload: %#v", requests)
|
|
}
|
|
exit, ok := requests[0].body["network_exit"].(map[string]any)
|
|
if !ok || exit["protocol"] != "" || exit["host"] != "" || exit["port"] != float64(0) {
|
|
t.Fatalf("direct create must not send proxy settings: %#v", requests[0].body)
|
|
}
|
|
}
|
|
|
|
func TestExitFailuresStopCreateBeforeGateway(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
probe fakeExitProbe
|
|
reason string
|
|
}{
|
|
{name: "identity drift", probe: fakeExitProbe{observation: hub.ExitObservation{PublicIP: "203.0.113.11", Region: "test"}}, reason: "exit_ip_drift"},
|
|
{name: "authentication rejected", probe: fakeExitProbe{failure: "proxy_auth_failed"}, reason: "proxy_auth_failed"},
|
|
{name: "proxy disconnected", probe: fakeExitProbe{failure: "proxy_check_failed"}, reason: "proxy_check_failed"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-1"] = hub.NetworkExit{
|
|
ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080,
|
|
ExpectedPublicIP: "203.0.113.10", HealthStatus: "healthy", Version: 1,
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, test.probe, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("expected fail-close 409, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if len(gateway.recorded()) != 0 {
|
|
t.Fatalf("exit failure must stop before any gateway start/create call: %#v", gateway.recorded())
|
|
}
|
|
bound, err := store.GetEnvironmentContext(context.Background(), "account-a")
|
|
if err != nil || bound.RuntimeInstanceID != "" || store.exits["exit-1"].HealthStatus != "unhealthy" {
|
|
t.Fatalf("failed create must retain only the stable inactive binding: %#v err=%v", bound, err)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != test.reason {
|
|
t.Fatalf("exit failure must be auditable without secrets: %#v", store.actions)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("expected gateway conflict to pass through as 409, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if _, kept := store.envs["account-a"]; !kept {
|
|
t.Fatal("stable environment and Profile anchor must remain retryable")
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserTracksUnknownNetworkGenerationWithoutAliasCleanup(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadGateway}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("expected failed gateway create, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 1 || requests[0].method != http.MethodPost {
|
|
t.Fatalf("unknown network generation was discovered or cleaned by mutable alias: %#v", requests)
|
|
}
|
|
environment, err := store.GetEnvironmentContext(context.Background(), "account-a")
|
|
if err != nil || !environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "" {
|
|
t.Fatalf("unknown network generation was not retained for manual reconciliation: environment=%#v err=%v", environment, err)
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserDeterministicRejectionDoesNotWedgeBinding(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, failCreateStatus: http.StatusBadRequest}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "alias already in use") {
|
|
t.Fatalf("deterministic gateway rejection must surface its reason, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
environment, err := store.GetEnvironmentContext(context.Background(), "account-a")
|
|
if err != nil || environment.RuntimeCleanupPending {
|
|
t.Fatalf("deterministic rejection wedged the binding for manual reconciliation: environment=%#v err=%v", environment, err)
|
|
}
|
|
if requests := gateway.recorded(); len(requests) != 1 || requests[0].method != http.MethodPost {
|
|
t.Fatalf("deterministic rejection must not reconcile or mutate the gateway: %#v", requests)
|
|
}
|
|
}
|
|
|
|
func TestReconcileGatewayCreateDoesNotReuseOldNetworkGeneration(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"}
|
|
environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active",
|
|
AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"],
|
|
RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}
|
|
store.bindings[environment.Alias] = environment
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-99999999999999999999999999999999", ProxyReady: true,
|
|
}}}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
|
|
err := reconcileGatewayCreate(context.Background(), store, target, environment, http.StatusBadGateway, nil, []byte(`{"error":"result unknown"}`))
|
|
after := store.bindings[environment.Alias]
|
|
if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 0 || len(gateway.runtimes) != 1 ||
|
|
!after.RuntimeCleanupPending || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" {
|
|
t.Fatalf("unknown create reused the old generation: err=%v requests=%#v runtimes=%#v environment=%#v",
|
|
err, gateway.recorded(), gateway.runtimes, after)
|
|
}
|
|
}
|
|
|
|
func TestValidCreatedRuntimeGeneration(t *testing.T) {
|
|
environment := hub.EnvironmentContext{RuntimeNetworkID: "native-99999999999999999999999999999999"}
|
|
for _, test := range []struct {
|
|
name string
|
|
created runtimeStatus
|
|
environment hub.EnvironmentContext
|
|
running bool
|
|
want bool
|
|
}{
|
|
{name: "active matching generation", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-99999999999999999999999999999999"}, running: true, want: true},
|
|
{name: "active empty runtime", created: runtimeStatus{NetworkID: "native-99999999999999999999999999999999"}, running: true},
|
|
{name: "active invalid runtime", created: runtimeStatus{ID: "container c1", NetworkID: "native-99999999999999999999999999999999"}, running: true},
|
|
{name: "active empty network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, running: true},
|
|
{name: "active invalid network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "network n1"}, running: true},
|
|
{name: "active replacement network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-88888888888888888888888888888888"}, running: true},
|
|
{name: "active existing matching generation", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "native-99999999999999999999999999999999"}, running: true, want: true,
|
|
environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}},
|
|
{name: "active existing successor container", created: runtimeStatus{ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", NetworkID: "native-99999999999999999999999999999999"}, running: true,
|
|
environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}},
|
|
{name: "active legacy lease without network", created: runtimeStatus{ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", NetworkID: "native-88888888888888888888888888888888"}, running: true,
|
|
environment: hub.EnvironmentContext{RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}},
|
|
{name: "stopped empty network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, want: true},
|
|
{name: "stopped invalid network", created: runtimeStatus{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", NetworkID: "network n1"}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
current := environment
|
|
if test.environment.RuntimeInstanceID != "" {
|
|
current = test.environment
|
|
}
|
|
if got := validCreatedRuntime(test.created, current, test.running); got != test.want {
|
|
t.Fatalf("validCreatedRuntime(%#v, running=%v)=%v, want %v", test.created, test.running, got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type activationConflictStore struct {
|
|
*memoryStore
|
|
current hub.EnvironmentContext
|
|
}
|
|
|
|
func (s *activationConflictStore) GetEnvironmentContext(_ context.Context, _ string) (hub.EnvironmentContext, error) {
|
|
return s.current, nil
|
|
}
|
|
|
|
func (s *activationConflictStore) ActivateRuntime(context.Context, string, string, int64, string, ...string) (hub.EnvironmentContext, error) {
|
|
return hub.EnvironmentContext{}, hub.ErrConflict
|
|
}
|
|
|
|
func TestActivationConflictKeepsTheGenerationWonByTheHeartbeat(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"}
|
|
candidateID := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
|
|
candidateNetwork := "native-88888888888888888888888888888888"
|
|
current := hub.EnvironmentContext{
|
|
Env: store.envs["account-a"],
|
|
AccountID: "account-a",
|
|
AccountStatus: "active",
|
|
AuthorizationStatus: "authorized",
|
|
BindingID: "account-a",
|
|
BindingVersion: 1,
|
|
RuntimeInstanceID: "runtime-instance-c1",
|
|
RuntimeID: candidateID,
|
|
RuntimeNetworkID: candidateNetwork,
|
|
Exit: store.exits["exit-1"],
|
|
}
|
|
store.bindings["account-a"] = current
|
|
conflict := &activationConflictStore{memoryStore: store, current: current}
|
|
|
|
got, err := activateGatewayRuntime(context.Background(), conflict, hub.Gateway{}, current, candidateID, candidateNetwork)
|
|
if err != nil {
|
|
t.Fatalf("activation race was reported as failure: %v", err)
|
|
}
|
|
if got.RuntimeID != candidateID || got.RuntimeNetworkID != candidateNetwork || got.RuntimeInstanceID == "" {
|
|
t.Fatalf("heartbeat winner was not returned: %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestLegacyActiveRuntimeWithoutNetworkGenerationDoesNotTouchSuccessor(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"}
|
|
environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active",
|
|
AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"],
|
|
RuntimeInstanceID: "runtime-instance-c1", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}
|
|
store.bindings[environment.Alias] = environment
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-88888888888888888888888888888888", ProxyReady: true,
|
|
}}}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
|
|
if runtimeMatchesBinding(gateway.runtimes[0], environment) {
|
|
t.Fatal("legacy active lease accepted a successor network generation")
|
|
}
|
|
if _, err := activateGatewayRuntime(context.Background(), store, target, environment, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "native-88888888888888888888888888888888"); !errors.Is(err, hub.ErrConflict) {
|
|
t.Fatalf("successor activation did not fail closed: %v", err)
|
|
}
|
|
if _, err := removeGatewayRuntime(context.Background(), store, target, environment); !errors.Is(err, hub.ErrConflict) {
|
|
t.Fatalf("legacy cleanup did not fence the successor: %v", err)
|
|
}
|
|
after := store.bindings[environment.Alias]
|
|
requests := gateway.recorded()
|
|
if len(requests) != 1 || requests[0].method != http.MethodGet || len(gateway.runtimes) != 1 ||
|
|
after.RuntimeID != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || after.RuntimeNetworkID != "" || after.RuntimeCleanupPending {
|
|
t.Fatalf("legacy C1 cleanup touched C2/N2: requests=%#v runtimes=%#v environment=%#v", requests, gateway.runtimes, after)
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserInvalid201TracksNetworkCleanupGeneration(t *testing.T) {
|
|
for _, networkID := range []string{"native-99999999999999999999999999999999", ""} {
|
|
name := "known network"
|
|
if networkID == "" {
|
|
name = "unknown network"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
var deleteBody map[string]any
|
|
gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" {
|
|
response.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
switch request.Method {
|
|
case http.MethodPost:
|
|
response.WriteHeader(http.StatusCreated)
|
|
_ = json.NewEncoder(response).Encode(map[string]string{"network_id": networkID})
|
|
case http.MethodGet:
|
|
_, _ = response.Write([]byte(`[]`))
|
|
case http.MethodDelete:
|
|
_ = json.NewDecoder(request.Body).Decode(&deleteBody)
|
|
if networkID == "" {
|
|
response.WriteHeader(http.StatusConflict)
|
|
return
|
|
}
|
|
response.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path)
|
|
}
|
|
}))
|
|
defer gatewayServer.Close()
|
|
store := newMemoryStore()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusBadGateway || (networkID != "" &&
|
|
(deleteBody["runtime_id"] != missingRuntimeID || deleteBody["network_id"] != networkID)) {
|
|
t.Fatalf("invalid 201 did not reconcile its generation: status=%d delete=%v body=%s", response.Code, deleteBody, response.Body.String())
|
|
}
|
|
environment := store.bindings["account-a"]
|
|
if networkID != "" && environment.RuntimeCleanupPending {
|
|
t.Fatalf("known network cleanup did not converge: %#v", environment)
|
|
}
|
|
if networkID == "" && (!environment.RuntimeCleanupPending || environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "") {
|
|
t.Fatalf("unknown generation was not retained for manual reconciliation: %#v", environment)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserRejectsActiveRuntimeWithoutNetworkGeneration(t *testing.T) {
|
|
requests := 0
|
|
gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
requests++
|
|
if request.Method != http.MethodPost {
|
|
t.Fatalf("active NULL network response triggered alias reconciliation: %s %s", request.Method, request.URL.Path)
|
|
}
|
|
response.WriteHeader(http.StatusCreated)
|
|
_, _ = response.Write([]byte(`{"id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","network_id":""}`))
|
|
}))
|
|
defer gatewayServer.Close()
|
|
store := newMemoryStore()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
environment := store.bindings["account-a"]
|
|
if response.Code != http.StatusBadGateway || requests != 1 || !environment.RuntimeCleanupPending ||
|
|
environment.RuntimeCleanupRuntimeID != missingRuntimeID || environment.RuntimeCleanupNetworkID != "" || environment.RuntimeID != "" {
|
|
t.Fatalf("active NULL network generation was activated: status=%d requests=%d environment=%#v body=%s",
|
|
response.Code, requests, environment, response.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserDoesNotDiscoverDisconnectedCreateByAlias(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
reconcile bool
|
|
wantStatus int
|
|
}{
|
|
{name: "completed create", reconcile: true, wantStatus: http.StatusBadGateway},
|
|
{name: "unknown result", wantStatus: http.StatusBadGateway},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var created bool
|
|
var mu sync.Mutex
|
|
gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" {
|
|
response.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if request.Method == http.MethodGet && test.reconcile {
|
|
mu.Lock()
|
|
exists := created
|
|
mu.Unlock()
|
|
if exists {
|
|
_ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}})
|
|
return
|
|
}
|
|
_ = json.NewEncoder(response).Encode([]runtimeStatus{})
|
|
return
|
|
}
|
|
if request.Method == http.MethodPost {
|
|
mu.Lock()
|
|
created = true
|
|
mu.Unlock()
|
|
}
|
|
hijacker, ok := response.(http.Hijacker)
|
|
if !ok {
|
|
t.Error("test server does not support hijacking")
|
|
return
|
|
}
|
|
connection, _, err := hijacker.Hijack()
|
|
if err != nil {
|
|
t.Errorf("hijack gateway response: %v", err)
|
|
return
|
|
}
|
|
_ = connection.Close()
|
|
}))
|
|
defer gatewayServer.Close()
|
|
|
|
store := newMemoryStore()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != test.wantStatus {
|
|
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
|
|
}
|
|
if _, err := store.GetEnv(context.Background(), "account-a"); err != nil {
|
|
t.Fatalf("unknown gateway result must retain the environment for reconciliation: %v", err)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" {
|
|
t.Fatalf("gateway result without an immutable generation must be unknown: %#v", store.actions)
|
|
}
|
|
mu.Lock()
|
|
wasCreated := created
|
|
mu.Unlock()
|
|
if !wasCreated || store.bindings["account-a"].RuntimeCleanupNetworkID != "" {
|
|
t.Fatalf("disconnected create lost its manual reconcile marker: %#v", store.bindings["account-a"])
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserDoesNotDiscoverBadGatewayCreateByAlias(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
missingReads int
|
|
}{
|
|
{name: "Docker create disconnect returned 502"},
|
|
{name: "container becomes visible after query window", missingReads: 2},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var mu sync.Mutex
|
|
listCalls := 0
|
|
gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.Header.Get("Authorization") != "Bearer unit-test-gateway-token" {
|
|
response.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if request.Method == http.MethodPost {
|
|
response.WriteHeader(http.StatusBadGateway)
|
|
_, _ = response.Write([]byte(`{"error":"create container: context deadline exceeded"}`))
|
|
return
|
|
}
|
|
mu.Lock()
|
|
listCalls++
|
|
missing := listCalls <= test.missingReads
|
|
mu.Unlock()
|
|
if missing {
|
|
_ = json.NewEncoder(response).Encode([]runtimeStatus{})
|
|
return
|
|
}
|
|
_ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}})
|
|
}))
|
|
defer gatewayServer.Close()
|
|
|
|
store := newMemoryStore()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("expected unknown create, status=%d body=%s", response.Code, response.Body.String())
|
|
}
|
|
if _, err := store.GetEnv(context.Background(), "account-a"); err != nil {
|
|
t.Fatalf("reconciled create must retain DB state: err=%v", err)
|
|
}
|
|
mu.Lock()
|
|
gotCalls := listCalls
|
|
mu.Unlock()
|
|
if gotCalls != 0 {
|
|
t.Fatalf("unknown network generation was inspected by alias %d times", gotCalls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCreateBrowserRejectsInvalidFingerprintBeforeSideEffects(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers",
|
|
`{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":0}}`)
|
|
if response.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 for invalid fingerprint, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if len(gateway.recorded()) != 0 || len(store.envs) != 0 {
|
|
t.Fatal("invalid input must not reach the gateway or the store")
|
|
}
|
|
}
|
|
|
|
func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 3, Exit: store.exits["exit-1"],
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", NetworkID: "native-cccccccccccccccccccccccccccccccc", ProxyReady: false,
|
|
}}}
|
|
_ = newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("gateway restart recovery failed: %v", err)
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
|
t.Fatalf("runtime was not activated after proxy recovery: %q", runtime)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 2 || requests[0].path != "/v1/browsers" || requests[1].path != "/v1/browsers/account-a/proxy" {
|
|
t.Fatalf("expected list then proxy recovery without rebuild: %#v", requests)
|
|
}
|
|
if requests[1].body["binding_version"] != float64(3) || requests[1].body["runtime_id"] != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ||
|
|
requests[1].body["network_exit_id"] != "exit-1" {
|
|
t.Fatalf("proxy recovery did not use the current binding: %#v", requests[1].body)
|
|
}
|
|
}
|
|
|
|
func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"]}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666",
|
|
}}}
|
|
_ = newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); !errors.Is(err, hub.ErrConflict) {
|
|
t.Fatalf("lease-free running orphan was not fenced: %v", err)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 3 || requests[1].path != "/v1/browsers/account-a/proxy" {
|
|
t.Fatalf("unproven orphan generation reached rebuild side effects: %#v", requests)
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
|
|
t.Fatalf("unproven orphan generation was activated: %q", runtime)
|
|
}
|
|
}
|
|
|
|
type postgresRebindFixture struct {
|
|
store *hub.Store
|
|
db *sql.DB
|
|
databaseURL string
|
|
gateway *fakeGateway
|
|
bound hub.EnvironmentContext
|
|
exit hub.NetworkExit
|
|
}
|
|
|
|
type cleanupCommitUnknownStore struct {
|
|
hubStore
|
|
}
|
|
|
|
type failingRuntimeReleaseStore struct {
|
|
*hub.Store
|
|
err error
|
|
}
|
|
|
|
func (s failingRuntimeReleaseStore) ReleaseRuntime(context.Context, hub.EnvironmentContext) error {
|
|
return s.err
|
|
}
|
|
|
|
func setFixtureAccountStatus(t *testing.T, databaseURL, status string) {
|
|
t.Helper()
|
|
db, err := sql.Open("pgx", databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(`UPDATE social_account SET status = $1 WHERE id = 'account-a'`, status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (s cleanupCommitUnknownStore) SetRuntimeCleanupPending(ctx context.Context, environment hub.EnvironmentContext, pending bool) error {
|
|
if err := s.hubStore.SetRuntimeCleanupPending(ctx, environment, pending); err != nil {
|
|
return err
|
|
}
|
|
return errors.New("cleanup commit result unknown")
|
|
}
|
|
|
|
type failContextRefreshStore struct {
|
|
hubStore
|
|
reads int
|
|
}
|
|
|
|
func (s *failContextRefreshStore) GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) {
|
|
s.reads++
|
|
if s.reads >= 5 {
|
|
return hub.EnvironmentContext{}, errors.New("context refresh unavailable")
|
|
}
|
|
return s.hubStore.GetEnvironmentContext(ctx, alias)
|
|
}
|
|
|
|
func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFixture {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL)
|
|
accountStore, err := accountdomain.Open(ctx, databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := accountStore.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err := hub.Open(ctx, databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = store.Close() })
|
|
accountStore, err = accountdomain.Open(ctx, databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := accountStore.CreateAccount(ctx, accountdomain.Account{
|
|
ID: "account-a", Name: "账号 A", Platform: "douyin", PlatformAccountKey: "account-a",
|
|
Tags: []string{}, Cookies: "sessionid=value",
|
|
CredentialReference: accountdomain.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"},
|
|
CredentialKey: "creatorhub/account-a/cookies",
|
|
}, &testCredentialBridge{values: map[string]string{}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := accountStore.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
gatewayServer := httptest.NewServer(gateway.handler(t))
|
|
t.Cleanup(gatewayServer.Close)
|
|
if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
exit, _, err = store.RecordNetworkExitCheck(ctx, exit.ID, hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bound, created, err := store.CreateBoundEnv(ctx, hub.Env{
|
|
Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1},
|
|
}, "account-a", exit.ID)
|
|
if err != nil || !created {
|
|
t.Fatalf("create bound environment: created=%v err=%v", created, err)
|
|
}
|
|
db, err := sql.Open("pgx", databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
return postgresRebindFixture{store: store, db: db, databaseURL: databaseURL, gateway: gateway, bound: bound, exit: exit}
|
|
}
|
|
|
|
func TestPauseClosesClaimGateBeforeStoppingRuntime(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
for _, test := range []struct {
|
|
name string
|
|
gatewayEndpoint string
|
|
wantStatus int
|
|
wantOutcome string
|
|
wantReason string
|
|
}{
|
|
{name: "confirmed stop", wantStatus: http.StatusNoContent, wantOutcome: "succeeded", wantReason: "environment_stopped"},
|
|
{name: "gateway unreachable", gatewayEndpoint: "http://127.0.0.1:1", wantStatus: http.StatusBadGateway, wantOutcome: "unknown", wantReason: "gateway_result_unknown"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx := context.Background()
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
accountStore, err := accountdomain.Open(ctx, fixture.databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = accountStore.Close() })
|
|
if err := accountStore.ResumeAccount(ctx, "account-a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
account, err := accountStore.GetAccount(ctx, "account-a")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "active-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-33333333333333333333333333333333")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "active-container", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
|
|
}}
|
|
if test.gatewayEndpoint != "" {
|
|
if _, err := fixture.db.ExecContext(ctx, `UPDATE gateway SET endpoint = $1 WHERE name = 'gw-1'`, test.gatewayEndpoint); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := accountStore.CreateDraft(ctx, accountdomain.Draft{ID: "pause-draft", AccountID: "account-a", Version: 1, Content: "pause"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := accountStore.Confirm(ctx, accountdomain.Confirmation{
|
|
ID: "pause-confirmation", AccountID: "account-a", AccountVersion: account.Version,
|
|
DraftID: "pause-draft", DraftVersion: 1, Version: 1,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, inserted, err := accountStore.Enqueue(ctx, accountdomain.Task{
|
|
ID: "pause-task", IdempotencyKey: "pause-idempotency", AccountID: "account-a", AccountVersion: account.Version,
|
|
DraftID: "pause-draft", DraftVersion: 1, ConfirmationID: "pause-confirmation", ConfirmationVersion: 1,
|
|
}); err != nil || !inserted {
|
|
t.Fatalf("enqueue pause task: inserted=%v err=%v", inserted, err)
|
|
}
|
|
if _, err := fixture.db.ExecContext(ctx, `
|
|
INSERT INTO execution_attempt (id, task_id) VALUES ('pause-attempt', 'pause-task');
|
|
UPDATE operation_task SET state = 'executing', current_attempt_id = 'pause-attempt',
|
|
lease_owner = 'worker-pause', lease_until = now() + interval '1 minute'
|
|
WHERE id = 'pause-task'`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
app := fiber.New()
|
|
registerPhaseA(app, accountStore, fixture.store, nil)
|
|
response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", "")
|
|
if response.Code != test.wantStatus {
|
|
t.Fatalf("pause returned %d, want %d: %s", response.Code, test.wantStatus, response.Body.String())
|
|
}
|
|
paused, err := accountStore.GetAccount(ctx, "account-a")
|
|
if err != nil || paused.RuntimeStatus != "paused" {
|
|
t.Fatalf("claim gate remained open: account=%#v err=%v", paused, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM operation_task WHERE id = 'pause-task' AND state = 'needs_confirmation'`, 1)
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM execution_attempt WHERE id = 'pause-attempt' AND outcome = 'uncertain'`, 1)
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM audit_event WHERE task_id = 'pause-task' AND attempt_id = 'pause-attempt'
|
|
AND browser_env_alias = 'account-a' AND network_exit_id IS NOT NULL AND runtime_instance_id IS NOT NULL
|
|
AND binding_version = 1 AND reason_code = 'account_paused'`, 1)
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `SELECT outcome, reason_code FROM audit_event
|
|
WHERE account_id = 'account-a' AND action = 'stop' AND event_type = 'environment_action_finished'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil || outcome != test.wantOutcome || reason != test.wantReason {
|
|
t.Fatalf("stop audit: outcome=%q reason=%q err=%v", outcome, reason, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
accountStore, err := accountdomain.Open(ctx, fixture.databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = accountStore.Close() })
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "stopped-container", Alias: fixture.bound.Alias, State: "exited",
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
|
|
}}
|
|
app := fiber.New()
|
|
registerPhaseA(app, accountStore, fixture.store, nil)
|
|
|
|
if err := accountStore.ResumeAccount(ctx, fixture.bound.AccountID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/pause", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("pause stopped container without lease: %d %s", response.Code, response.Body.String())
|
|
}
|
|
if err := accountStore.ResumeAccount(ctx, fixture.bound.AccountID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.mu.Lock()
|
|
fixture.gateway.runtimes[0].State = "running"
|
|
fixture.gateway.mu.Unlock()
|
|
if response := do(app, http.MethodPost, "/api/phase-a/accounts/account-a/revoke", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("revoke running orphan without lease: %d %s", response.Code, response.Body.String())
|
|
}
|
|
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, fixture.bound.Alias)
|
|
if err != nil || after.RuntimeID != "" || after.RuntimeCleanupPending {
|
|
t.Fatalf("lease-free stop did not converge: %#v err=%v", after, err)
|
|
}
|
|
account, err := accountStore.GetAccount(ctx, fixture.bound.AccountID)
|
|
if err != nil || account.AuthorizationStatus != "revoked" || account.RuntimeStatus != "paused" {
|
|
t.Fatalf("revoke gate did not remain closed: %#v err=%v", account, err)
|
|
}
|
|
fixture.gateway.mu.Lock()
|
|
container := fixture.gateway.runtimes[0]
|
|
fixture.gateway.mu.Unlock()
|
|
if container.State != "exited" {
|
|
t.Fatalf("running orphan was not stopped: %#v", container)
|
|
}
|
|
requests := fixture.gateway.recorded()
|
|
stopCalls := 0
|
|
for _, request := range requests {
|
|
if request.method == http.MethodPost && request.path == "/v1/browsers/account-a/stop" {
|
|
stopCalls++
|
|
if request.body["runtime_id"] != "stopped-container" || request.body["binding_version"] != float64(fixture.bound.BindingVersion) {
|
|
t.Fatalf("stop used the wrong discovered generation: %#v", request)
|
|
}
|
|
}
|
|
}
|
|
if stopCalls != 2 {
|
|
t.Fatalf("pause/revoke did not reconcile both lease-free runtimes: %#v", requests)
|
|
}
|
|
}
|
|
|
|
func installCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB, condition string) {
|
|
t.Helper()
|
|
if _, err := db.ExecContext(ctx, `
|
|
CREATE FUNCTION fail_cleanup_transition() RETURNS trigger AS $$
|
|
BEGIN
|
|
IF `+condition+` THEN
|
|
RAISE EXCEPTION 'injected cleanup state failure';
|
|
END IF;
|
|
RETURN NEW;
|
|
END
|
|
$$ LANGUAGE plpgsql`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `
|
|
CREATE TRIGGER fail_cleanup_transition BEFORE UPDATE OF runtime_cleanup_pending ON environment_binding
|
|
FOR EACH ROW EXECUTE FUNCTION fail_cleanup_transition()`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func dropCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB) {
|
|
t.Helper()
|
|
if _, err := db.ExecContext(ctx, `DROP TRIGGER fail_cleanup_transition ON environment_binding`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `DROP FUNCTION fail_cleanup_transition()`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
for _, test := range []struct {
|
|
name string
|
|
cleanupPending int
|
|
disconnectDelete int
|
|
disconnectListAfterDelete bool
|
|
}{
|
|
{name: "accepted cleanup pending", cleanupPending: 2},
|
|
{name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666",
|
|
}}
|
|
fixture.gateway.cleanupPending = test.cleanupPending
|
|
fixture.gateway.disconnectDelete = test.disconnectDelete
|
|
fixture.gateway.disconnectListAfterDelete = test.disconnectListAfterDelete
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
target, err := fixture.store.GetGateway(ctx, "gw-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound)
|
|
if !removed || cleanupErr == nil {
|
|
t.Fatalf("non-final cleanup lost the removed fact: removed=%v err=%v", removed, cleanupErr)
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || !after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion ||
|
|
after.Exit.ID != fixture.bound.Exit.ID || after.RuntimeID != "" {
|
|
t.Fatalf("PostgreSQL did not persist cleanup pending atomically: %#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM environment_binding WHERE browser_env_alias = 'account-a' AND runtime_cleanup_pending`, 1)
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
|
|
if err := reconcileRuntimeLeases(ctx, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("reconcile did not confirm cleanup: %v", err)
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeCleanupPending {
|
|
t.Fatalf("confirmed cleanup remained pending: %#v err=%v", after, err)
|
|
}
|
|
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "paused")
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+fixture.exit.ID+`"}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("rebind retry failed after cleanup confirmation: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion+1 || after.RuntimeID != "" {
|
|
t.Fatalf("retry did not commit a consistent stopped generation: %#v err=%v", after, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
commitKnown bool
|
|
outcome string
|
|
reason string
|
|
}{
|
|
{name: "rollback", commitKnown: true, outcome: "failed", reason: "runtime_persistence_failed"},
|
|
{name: "commit unknown", outcome: "unknown", reason: "cleanup_result_unknown"},
|
|
} {
|
|
t.Run("restore "+test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666",
|
|
}
|
|
store.cleanupPendingErr = errors.New("cleanup state unavailable")
|
|
store.cleanupPendingErrAfterMutation = !test.commitKnown
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666",
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil {
|
|
t.Fatal("failed cleanup mark unexpectedly succeeded")
|
|
}
|
|
after := store.bindings["account-a"]
|
|
if test.commitKnown {
|
|
if after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("rollback changed the old generation: %#v", after)
|
|
}
|
|
} else if !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("commit-unknown lost its retry marker: %#v", after)
|
|
}
|
|
if len(gateway.runtimes) != 1 {
|
|
t.Fatalf("failed cleanup mark touched the old container: runtimes=%#v requests=%#v", gateway.runtimes, gateway.recorded())
|
|
}
|
|
for _, request := range gateway.recorded() {
|
|
if request.method != http.MethodGet {
|
|
t.Fatalf("failed cleanup mark reached a gateway side effect: %#v", gateway.recorded())
|
|
}
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != test.outcome || store.actions[1].ReasonCode != test.reason {
|
|
t.Fatalf("restore audit mismatch: %#v", store.actions)
|
|
}
|
|
|
|
store.cleanupPendingErr = nil
|
|
store.cleanupPendingErrAfterMutation = false
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("reconcile retry failed: %v", err)
|
|
}
|
|
if test.commitKnown {
|
|
after = store.bindings["account-a"]
|
|
} else {
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("start retry failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after = store.bindings["account-a"]
|
|
}
|
|
if after.RuntimeCleanupPending || after.RuntimeID == "" || len(gateway.runtimes) != 1 || !runtimeMatchesBinding(gateway.runtimes[0], after) {
|
|
t.Fatalf("retry ended inconsistently: after=%#v runtimes=%#v", after, gateway.runtimes)
|
|
}
|
|
})
|
|
|
|
t.Run("discard "+test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}
|
|
store.cleanupPendingErr = errors.New("cleanup state unavailable")
|
|
store.cleanupPendingErrAfterMutation = !test.commitKnown
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1",
|
|
}}}
|
|
app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "proxy_auth_failed"},
|
|
func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/start", "")
|
|
if response.Code != http.StatusInternalServerError {
|
|
t.Fatalf("failed discard mark returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after := store.bindings["account-a"]
|
|
if test.commitKnown {
|
|
if after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("discard rollback changed the old generation: %#v", after)
|
|
}
|
|
} else if !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("discard commit-unknown lost its retry marker: %#v", after)
|
|
}
|
|
if len(gateway.runtimes) != 1 {
|
|
t.Fatalf("failed discard mark touched the gateway: runtimes=%#v requests=%#v", gateway.runtimes, gateway.recorded())
|
|
}
|
|
for _, request := range gateway.recorded() {
|
|
if request.method != http.MethodGet {
|
|
t.Fatalf("failed discard mark reached a gateway side effect: %#v", gateway.recorded())
|
|
}
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" {
|
|
t.Fatalf("discard audit mismatch: %#v", store.actions)
|
|
}
|
|
|
|
store.cleanupPendingErr = nil
|
|
store.cleanupPendingErrAfterMutation = false
|
|
response = do(app, http.MethodPost, "/api/browsers/account-a/start", "")
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after = store.bindings["account-a"]
|
|
if after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.runtimes) != 0 {
|
|
t.Fatalf("discard retry ended inconsistently: after=%#v runtimes=%#v", after, gateway.runtimes)
|
|
}
|
|
if len(store.actions) != 4 || store.actions[3].Outcome != "failed" || store.actions[3].ReasonCode != "proxy_auth_failed" {
|
|
t.Fatalf("discard retry audit mismatch: %#v", store.actions)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666",
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
body := `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", body)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if after := store.bindings["account-a"]; after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || after.RuntimeCleanupPending {
|
|
t.Fatalf("gateway unknown changed the existing lease: %#v", after)
|
|
}
|
|
for _, request := range gateway.recorded() {
|
|
if request.method == http.MethodPost {
|
|
t.Fatalf("gateway unknown created a duplicate container: %#v", gateway.recorded())
|
|
}
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
|
|
t.Fatalf("gateway unknown create audit mismatch: %#v", store.actions)
|
|
}
|
|
|
|
response = do(app, http.MethodPost, "/api/browsers", body)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("create retry did not reuse the runtime: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if after := store.bindings["account-a"]; after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || after.RuntimeCleanupPending {
|
|
t.Fatalf("create retry changed the matching generation: %#v", after)
|
|
}
|
|
if len(gateway.runtimes) != 1 || len(store.actions) != 4 || store.actions[3].Outcome != "succeeded" || store.actions[3].ReasonCode != "environment_reused" {
|
|
t.Fatalf("create retry ended inconsistently: runtimes=%#v actions=%#v", gateway.runtimes, store.actions)
|
|
}
|
|
}
|
|
|
|
func TestRemoveGatewayRuntimePreservesKnownNetworkGeneration(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
observedNetwork string
|
|
wantConflict bool
|
|
}{
|
|
{name: "legacy gateway omits network", observedNetwork: ""},
|
|
{name: "replacement network conflicts", observedNetwork: "native-88888888888888888888888888888888", wantConflict: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1"}
|
|
environment := hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active",
|
|
AuthorizationStatus: "authorized", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"],
|
|
RuntimeInstanceID: "runtime-instance", RuntimeID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RuntimeNetworkID: "native-99999999999999999999999999999999"}
|
|
store.bindings["account-a"] = environment
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Alias: "account-a",
|
|
State: "running", BindingVersion: 1, NetworkExitID: "exit-1", NetworkID: test.observedNetwork, ProxyReady: true}}}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
_, err := removeGatewayRuntime(context.Background(), store, target, environment)
|
|
if test.wantConflict {
|
|
if !errors.Is(err, hub.ErrConflict) || len(gateway.recorded()) != 1 || store.bindings["account-a"].RuntimeID != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" {
|
|
t.Fatalf("replacement network was not fenced: err=%v requests=%#v environment=%#v", err, gateway.recorded(), store.bindings["account-a"])
|
|
}
|
|
return
|
|
}
|
|
requests := gateway.recorded()
|
|
if err != nil || len(requests) != 2 || requests[1].body["network_id"] != "native-99999999999999999999999999999999" || store.bindings["account-a"].RuntimeCleanupPending {
|
|
t.Fatalf("known N1 was not preserved through cleanup: err=%v requests=%#v environment=%#v", err, requests, store.bindings["account-a"])
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGatewayLookupFailurePreservesLease(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
environment := hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}
|
|
store.bindings[environment.Alias] = environment
|
|
store.gatewayFn = func(string) (hub.Gateway, error) { return hub.Gateway{}, errors.New("gateway lookup unavailable") }
|
|
|
|
if _, err := restoreOrRebuildRuntime(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil },
|
|
environment, runtimeStatus{ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: environment.Alias, State: "running"}); err == nil {
|
|
t.Fatal("restore accepted an unknown gateway")
|
|
}
|
|
if err := discardRuntime(context.Background(), store, environment); err == nil {
|
|
t.Fatal("discard accepted an unknown gateway")
|
|
}
|
|
if after := store.bindings[environment.Alias]; after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("gateway lookup failure changed the old generation: %#v", after)
|
|
}
|
|
}
|
|
|
|
func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
t.Run("pre-mark rollback skips DELETE", func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
|
|
}}
|
|
installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
|
|
target, err := fixture.store.GetGateway(ctx, "gw-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound)
|
|
if cleanupErr == nil || removed || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet {
|
|
t.Fatalf("rolled-back pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, cleanupErr, fixture.gateway.recorded())
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("PostgreSQL rollback did not preserve the active generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
|
|
dropCleanupTransitionFailure(t, ctx, fixture.db)
|
|
fixture.gateway.cleanupPending = 2
|
|
removed, cleanupErr = removeGatewayRuntime(ctx, fixture.store, target, after)
|
|
if cleanupErr == nil || !removed {
|
|
t.Fatalf("202 retry did not retain pending cleanup: removed=%v err=%v", removed, cleanupErr)
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.runtimes) != 0 {
|
|
t.Fatalf("202 retry was not durable: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
})
|
|
|
|
t.Run("clear rollback remains pending and lifecycle retry recovers", func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
|
|
}}
|
|
fixture.gateway.deleteNotFound = 1
|
|
installCleanupTransitionFailure(t, ctx, fixture.db, `OLD.runtime_cleanup_pending AND NOT NEW.runtime_cleanup_pending`)
|
|
target, err := fixture.store.GetGateway(ctx, "gw-1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound)
|
|
if cleanupErr == nil || !removed {
|
|
t.Fatalf("404 clear rollback lost the delete fact: removed=%v err=%v", removed, cleanupErr)
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.runtimes) != 0 {
|
|
t.Fatalf("clear rollback was not durably pending: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
|
|
dropCleanupTransitionFailure(t, ctx, fixture.db)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
if err := reconcileRuntimeLeases(context.Background(), fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("reconcile retry did not confirm cleanup: %v", err)
|
|
}
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("lifecycle retry did not rebuild runtime: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
fixture.gateway.mu.Lock()
|
|
runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...)
|
|
fixture.gateway.mu.Unlock()
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) {
|
|
t.Fatalf("lifecycle retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPostgresLegacyCleanupPendingRequiresManualReconcile(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
if _, err := fixture.db.ExecContext(ctx, `
|
|
UPDATE environment_binding SET runtime_cleanup_pending = true,
|
|
runtime_cleanup_binding_version = NULL, runtime_cleanup_instance_id = NULL, runtime_cleanup_runtime_id = NULL
|
|
WHERE id = $1`, fixture.bound.BindingID); err != nil {
|
|
t.Fatalf("write legacy pending state: %v", err)
|
|
}
|
|
environment, err := fixture.store.GetEnvironmentContext(ctx, fixture.bound.Alias)
|
|
if err != nil || !environment.RuntimeCleanupPending || environment.RuntimeCleanupBindingVersion != 0 || environment.RuntimeCleanupRuntimeID != "" {
|
|
t.Fatalf("read legacy pending state: %#v err=%v", environment, err)
|
|
}
|
|
target, err := fixture.store.GetGateway(ctx, environment.Gateway)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
removed, err := removeGatewayRuntime(ctx, fixture.store, target, environment)
|
|
if !errors.Is(err, hub.ErrReconcileRequired) || removed || len(fixture.gateway.recorded()) != 0 {
|
|
t.Fatalf("legacy pending reached gateway mutation: removed=%v err=%v requests=%#v", removed, err, fixture.gateway.recorded())
|
|
}
|
|
}
|
|
|
|
func TestPostgresCleanupMissingContainerReleasesKnownLease(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
environment, err := fixture.store.ActivateRuntime(ctx, fixture.bound.Alias, "known-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-44444444444444444444444444444444")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
target, err := fixture.store.GetGateway(ctx, environment.Gateway)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
removed, err := removeGatewayRuntime(ctx, fixture.store, target, environment)
|
|
if err != nil || !removed {
|
|
t.Fatalf("cleanup missing gateway container: removed=%v err=%v", removed, err)
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, environment.Alias)
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("known PostgreSQL lease did not converge: %#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
requests := fixture.gateway.recorded()
|
|
if len(requests) == 0 || requests[len(requests)-1].method != http.MethodDelete ||
|
|
requests[len(requests)-1].body["runtime_id"] != "known-container" {
|
|
t.Fatalf("cleanup did not preserve the known runtime generation: %#v", requests)
|
|
}
|
|
}
|
|
|
|
func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
for _, test := range []struct {
|
|
name string
|
|
commitUnknown bool
|
|
outcome string
|
|
reason string
|
|
}{
|
|
{name: "rollback", outcome: "failed", reason: "runtime_persistence_failed"},
|
|
{name: "commit unknown", commitUnknown: true, outcome: "unknown", reason: "cleanup_result_unknown"},
|
|
} {
|
|
t.Run("restore "+test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit",
|
|
}}
|
|
var store HubStore = fixture.store
|
|
if test.commitUnknown {
|
|
store = cleanupCommitUnknownStore{hubStore: fixture.store}
|
|
} else {
|
|
installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
|
|
}
|
|
if err := reconcileRuntimeLeases(ctx, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil {
|
|
t.Fatal("restore cleanup failure unexpectedly succeeded")
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if test.commitUnknown {
|
|
if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("restore commit-unknown lost pending state: after=%#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
} else {
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("restore rollback changed the old generation: after=%#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
}
|
|
if len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("restore cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded())
|
|
}
|
|
for _, request := range fixture.gateway.recorded() {
|
|
if request.method != http.MethodGet {
|
|
t.Fatalf("restore cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded())
|
|
}
|
|
}
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = 'reconcile'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != test.outcome || reason != test.reason {
|
|
t.Fatalf("restore audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
|
|
if !test.commitUnknown {
|
|
dropCleanupTransitionFailure(t, ctx, fixture.db)
|
|
}
|
|
if err := reconcileRuntimeLeases(ctx, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("restore reconcile retry failed: %v", err)
|
|
}
|
|
if test.commitUnknown {
|
|
retryApp := fiber.New()
|
|
registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
if response := do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("restore start retry failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
fixture.gateway.mu.Lock()
|
|
runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...)
|
|
fixture.gateway.mu.Unlock()
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) {
|
|
t.Fatalf("restore retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
})
|
|
|
|
t.Run("create restore "+test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit",
|
|
}}
|
|
var store HubStore = fixture.store
|
|
if test.commitUnknown {
|
|
store = cleanupCommitUnknownStore{hubStore: fixture.store}
|
|
} else {
|
|
installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
|
|
}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
body := `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}`
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", body)
|
|
if response.Code != http.StatusInternalServerError {
|
|
t.Fatalf("create restore cleanup failure returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if test.commitUnknown {
|
|
if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("create restore commit-unknown lost pending state: after=%#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
} else {
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("create restore rollback changed the old generation: after=%#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
}
|
|
if len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("create restore cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded())
|
|
}
|
|
for _, request := range fixture.gateway.recorded() {
|
|
if request.method != http.MethodGet {
|
|
t.Fatalf("create restore cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded())
|
|
}
|
|
}
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = 'create'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != test.outcome || reason != test.reason {
|
|
t.Fatalf("create restore audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
|
|
if !test.commitUnknown {
|
|
dropCleanupTransitionFailure(t, ctx, fixture.db)
|
|
}
|
|
retryApp := fiber.New()
|
|
registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
response = do(retryApp, http.MethodPost, "/api/browsers", body)
|
|
if response.Code != http.StatusOK && response.Code != http.StatusCreated {
|
|
t.Fatalf("create restore retry failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
fixture.gateway.mu.Lock()
|
|
runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...)
|
|
fixture.gateway.mu.Unlock()
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(runtimes) != 1 || !runtimeMatchesBinding(runtimes[0], after) {
|
|
t.Fatalf("create restore retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
})
|
|
|
|
t.Run("discard "+test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666",
|
|
}}
|
|
var store HubStore = fixture.store
|
|
if test.commitUnknown {
|
|
store = cleanupCommitUnknownStore{hubStore: fixture.store}
|
|
} else {
|
|
installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`)
|
|
}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/start", "")
|
|
if response.Code != http.StatusInternalServerError {
|
|
t.Fatalf("discard cleanup failure returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if test.commitUnknown {
|
|
if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("discard commit-unknown lost pending state: after=%#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
} else {
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("discard rollback changed the old generation: after=%#v err=%v", after, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
}
|
|
if len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("discard cleanup failure touched the container: runtimes=%#v requests=%#v", fixture.gateway.runtimes, fixture.gateway.recorded())
|
|
}
|
|
for _, request := range fixture.gateway.recorded() {
|
|
if request.method != http.MethodGet {
|
|
t.Fatalf("discard cleanup failure reached a gateway side effect: %#v", fixture.gateway.recorded())
|
|
}
|
|
}
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = 'start'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != "unknown" || reason != "cleanup_result_unknown" {
|
|
t.Fatalf("discard audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
|
|
if !test.commitUnknown {
|
|
dropCleanupTransitionFailure(t, ctx, fixture.db)
|
|
}
|
|
if err := reconcileRuntimeLeases(ctx, fixture.store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("discard reconcile retry failed: %v", err)
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("discard retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
for _, container := range fixture.gateway.runtimes {
|
|
if container.State == "running" {
|
|
t.Fatalf("discard retry retained a running container: %#v", fixture.gateway.runtimes)
|
|
}
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPostgresGatewayUnknownDoesNotAffectListAndStillBlocksCreate(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
for _, test := range []struct {
|
|
name string
|
|
set func(*fakeGateway, int)
|
|
}{
|
|
{name: "disconnect", set: func(gateway *fakeGateway, count int) { gateway.disconnectList = count }},
|
|
{name: "status 500", set: func(gateway *fakeGateway, count int) { gateway.failList = count }},
|
|
{name: "invalid JSON", set: func(gateway *fakeGateway, count int) { gateway.invalidList = count }},
|
|
} {
|
|
t.Run("list "+test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666",
|
|
}}
|
|
test.set(fixture.gateway, 1)
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK {
|
|
t.Fatalf("gateway failure affected browser list: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("browser list changed the generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1)
|
|
requests := fixture.gateway.recorded()
|
|
if len(requests) != 1 || requests[0].method != http.MethodGet || requests[0].path != "/v1/browsers" {
|
|
t.Fatalf("browser list did not perform the expected live gateway status read: %#v", requests)
|
|
}
|
|
})
|
|
|
|
t.Run("create "+test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666",
|
|
}}
|
|
test.set(fixture.gateway, gatewayReconcileAttempts)
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
body := `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}`
|
|
|
|
if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusBadGateway {
|
|
t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("gateway unknown create changed the generation: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
for _, request := range fixture.gateway.recorded() {
|
|
if request.method == http.MethodPost {
|
|
t.Fatalf("gateway unknown create sent a duplicate create: %#v", fixture.gateway.recorded())
|
|
}
|
|
}
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = 'create'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != "unknown" || reason != "gateway_result_unknown" {
|
|
t.Fatalf("gateway unknown create audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
|
|
if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusOK {
|
|
t.Fatalf("gateway create retry failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" || len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("gateway create retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, fixture.gateway.runtimes, err)
|
|
}
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = 'create'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != "succeeded" || reason != "environment_reused" {
|
|
t.Fatalf("gateway create retry audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPostgresStrictGatewayValidationBlocksLifecycleUntilRetry(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
for _, test := range []struct {
|
|
name, method, path, action, successReason string
|
|
configure func(*fakeGateway, int)
|
|
active bool
|
|
wantStatus int
|
|
}{
|
|
{name: "create rejects null element", method: http.MethodPost, path: "/api/browsers", action: "create", active: true,
|
|
wantStatus: http.StatusOK, successReason: "environment_reused",
|
|
configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `[null]` }},
|
|
{name: "start rejects missing fields", method: http.MethodPost, path: "/api/browsers/account-a/start", action: "start", active: true,
|
|
wantStatus: http.StatusNoContent, successReason: "gateway_reconciled",
|
|
configure: func(gateway *fakeGateway, count int) {
|
|
gateway.invalidList, gateway.invalidListBody = count, `[{"alias":"account-a","state":"running"}]`
|
|
}},
|
|
{name: "rebind rejects partial body", method: http.MethodPost, path: "/api/browsers/account-a/rebind", action: "rebind",
|
|
wantStatus: http.StatusOK, successReason: "environment_rebound",
|
|
configure: func(gateway *fakeGateway, count int) { gateway.readErrorList = count }},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
if test.action == "start" || test.action == "" {
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
}
|
|
if test.active {
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
state, networkExitID, networkID := "running", fixture.bound.Exit.ID, "native-66666666666666666666666666666666"
|
|
if test.action == "rebind" {
|
|
state, networkExitID, networkID = "exited", "", ""
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: state, ProxyReady: state == "running",
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: networkExitID, NetworkID: networkID,
|
|
}}
|
|
attempts := gatewayReconcileAttempts
|
|
if test.action == "" {
|
|
attempts = 1
|
|
}
|
|
test.configure(fixture.gateway, attempts)
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
body := ""
|
|
if test.action == "create" {
|
|
body = `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}`
|
|
} else if test.action == "rebind" {
|
|
body = `{"network_exit_id":"` + fixture.exit.ID + `"}`
|
|
}
|
|
|
|
response := do(app, test.method, test.path, body)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("strict gateway validation returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || after.BindingVersion != fixture.bound.BindingVersion || after.Exit.ID != fixture.bound.Exit.ID ||
|
|
after.RuntimeCleanupPending || after.RuntimeID != fixture.bound.RuntimeID || len(fixture.gateway.runtimes) != 1 {
|
|
t.Fatalf("strict gateway validation changed the generation: before=%#v after=%#v runtimes=%#v err=%v",
|
|
fixture.bound, after, fixture.gateway.runtimes, err)
|
|
}
|
|
for _, request := range fixture.gateway.recorded() {
|
|
if request.method != http.MethodGet {
|
|
t.Fatalf("strict gateway validation advanced the lifecycle: %#v", fixture.gateway.recorded())
|
|
}
|
|
}
|
|
activeCount := 0
|
|
if test.active {
|
|
activeCount = 1
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeCount)
|
|
if test.action != "" {
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = $1
|
|
ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != "unknown" || reason != "gateway_result_unknown" {
|
|
t.Fatalf("strict gateway audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
}
|
|
|
|
response = do(app, test.method, test.path, body)
|
|
if response.Code != test.wantStatus {
|
|
t.Fatalf("strict gateway retry returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after, err = fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
fixture.gateway.mu.Lock()
|
|
runtimes := append([]runtimeStatus{}, fixture.gateway.runtimes...)
|
|
fixture.gateway.mu.Unlock()
|
|
wantVersion := fixture.bound.BindingVersion
|
|
if test.action == "rebind" {
|
|
wantVersion++
|
|
}
|
|
validRuntime := after.RuntimeID != "" && len(runtimes) == 1 && runtimeMatchesBinding(runtimes[0], after)
|
|
if test.action == "rebind" {
|
|
validRuntime = after.RuntimeID == "" && len(runtimes) == 0
|
|
}
|
|
if err != nil || after.BindingVersion != wantVersion || !validRuntime {
|
|
t.Fatalf("strict gateway retry ended inconsistently: after=%#v runtimes=%#v err=%v", after, runtimes, err)
|
|
}
|
|
activeAfter := 1
|
|
if test.action == "rebind" {
|
|
activeAfter = 0
|
|
}
|
|
assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeAfter)
|
|
if test.action != "" {
|
|
var outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT outcome, reason_code FROM audit_event WHERE action = $1
|
|
ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if outcome != "succeeded" || reason != test.successReason {
|
|
t.Fatalf("strict gateway retry audit mismatch: outcome=%s reason=%s", outcome, reason)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func isolatedControlPlaneDatabaseURL(t *testing.T, databaseURL string) string {
|
|
t.Helper()
|
|
admin, err := sql.Open("pgx", databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = admin.Close() })
|
|
schema := fmt.Sprintf("creatorhub_hh803_%d", time.Now().UnixNano())
|
|
if _, err := admin.Exec("CREATE SCHEMA " + schema); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
if _, err := admin.Exec("DROP SCHEMA " + schema + " CASCADE"); err != nil {
|
|
t.Errorf("drop test schema: %v", err)
|
|
}
|
|
})
|
|
parsed, err := url.Parse(databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
query := parsed.Query()
|
|
query.Set("search_path", schema)
|
|
parsed.RawQuery = query.Encode()
|
|
return parsed.String()
|
|
}
|
|
|
|
func assertControlPlaneDatabaseCount(t *testing.T, db *sql.DB, query string, want int) {
|
|
t.Helper()
|
|
var got int
|
|
if err := db.QueryRow(query).Scan(&got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got != want {
|
|
t.Fatalf("query %q returned %d, want %d", query, got, want)
|
|
}
|
|
}
|
|
|
|
func TestPostgresRejectsInvalidLifecycleTargetsWithSanitizedAuditPairs(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
const secret = "http://operator:secret@proxy.example"
|
|
for _, test := range []struct {
|
|
action string
|
|
body string
|
|
reason string
|
|
}{
|
|
{action: "upgrade", body: `{"version":"` + secret + `"}`, reason: "upgrade_input_rejected"},
|
|
{action: "rebind", body: `{"network_exit_id":"` + secret + `"}`, reason: "rebind_input_rejected"},
|
|
} {
|
|
t.Run(test.action, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/"+test.action, test.body)
|
|
if response.Code != http.StatusBadRequest {
|
|
t.Fatalf("invalid %s returned %d: %s", test.action, response.Code, response.Body.String())
|
|
}
|
|
var events, operations int
|
|
var reason string
|
|
if err := fixture.db.QueryRow(`
|
|
SELECT count(*), count(DISTINCT operation_id),
|
|
coalesce(max(reason_code) FILTER (WHERE event_type = 'environment_action_finished'), '')
|
|
FROM audit_event WHERE action = $1`, test.action).Scan(&events, &operations, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if events != 2 || operations != 1 || reason != test.reason {
|
|
t.Fatalf("invalid %s audit mismatch: events=%d operations=%d reason=%q", test.action, events, operations, reason)
|
|
}
|
|
var leaked int
|
|
if err := fixture.db.QueryRow(`SELECT count(*) FROM audit_event WHERE row_to_json(audit_event)::text LIKE '%' || $1 || '%'`, secret).Scan(&leaked); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if leaked != 0 {
|
|
t.Fatalf("invalid %s target leaked into audit", test.action)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPostgresReconcileFinishedAuditUsesActivatedRuntime(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(context.Background(), "account-a", "stale-container",
|
|
fixture.bound.BindingVersion, fixture.bound.Exit.ID, "native-66666666666666666666666666666666")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "stale-container", Alias: "account-a", State: "running", ProxyReady: true,
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666",
|
|
}}
|
|
if err := reconcileRuntimeLeases(context.Background(), fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("reconcile failed: %v", err)
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(context.Background(), "account-a")
|
|
if err != nil || after.RuntimeInstanceID == "" || after.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
|
t.Fatalf("reconcile did not activate replacement runtime: %#v err=%v", after, err)
|
|
}
|
|
var auditedRuntime string
|
|
var auditedBinding int64
|
|
if err := fixture.db.QueryRow(`
|
|
SELECT runtime_instance_id, binding_version FROM audit_event
|
|
WHERE event_type = 'environment_action_finished' AND action = 'reconcile'
|
|
ORDER BY id DESC LIMIT 1`).Scan(&auditedRuntime, &auditedBinding); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if auditedRuntime != after.RuntimeInstanceID || auditedBinding != after.BindingVersion {
|
|
t.Fatalf("finished audit retained stale runtime: runtime=%q binding=%d current=%#v", auditedRuntime, auditedBinding, after)
|
|
}
|
|
}
|
|
|
|
func TestPostgresNonRunnableReconcileAuditsRuntimeRelease(t *testing.T) {
|
|
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
|
}
|
|
ctx := context.Background()
|
|
for _, authorization := range []string{"authorized", "revoked"} {
|
|
for _, observation := range []string{"stopped", "missing"} {
|
|
for _, releaseFailure := range []bool{false, true} {
|
|
name := authorization + "/" + observation + "/success"
|
|
if releaseFailure {
|
|
name = authorization + "/" + observation + "/release failure"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
fixture := newPostgresRebindFixture(t, databaseURL)
|
|
setFixtureAccountStatus(t, fixture.databaseURL, "active")
|
|
var err error
|
|
fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "stopped-container",
|
|
fixture.bound.BindingVersion, fixture.bound.Exit.ID, "stopped-network")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := fixture.db.ExecContext(ctx, `
|
|
UPDATE social_account SET status = 'paused', authorization_status = $1 WHERE id = 'account-a'`, authorization); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if observation == "stopped" {
|
|
fixture.gateway.runtimes = []runtimeStatus{{
|
|
ID: "stopped-container", Alias: "account-a", State: "exited",
|
|
BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID,
|
|
}}
|
|
}
|
|
|
|
var store HubStore = fixture.store
|
|
if releaseFailure {
|
|
store = failingRuntimeReleaseStore{Store: fixture.store, err: errors.New("release unavailable")}
|
|
}
|
|
err = reconcileRuntimeLeases(ctx, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
wantOutcome, wantReason := "succeeded", "runtime_released"
|
|
if releaseFailure {
|
|
wantOutcome, wantReason = "failed", "runtime_release_failed"
|
|
}
|
|
if (err != nil) != releaseFailure {
|
|
t.Fatalf("reconcile error=%v, releaseFailure=%v", err, releaseFailure)
|
|
}
|
|
after, err := fixture.store.GetEnvironmentContext(ctx, "account-a")
|
|
if err != nil || (after.RuntimeInstanceID != "") == !releaseFailure {
|
|
t.Fatalf("runtime lease after reconcile: %#v err=%v", after, err)
|
|
}
|
|
|
|
var events, operations, missingOperations int
|
|
var eventTypes, outcome, reason string
|
|
if err := fixture.db.QueryRowContext(ctx, `
|
|
SELECT count(*), count(DISTINCT operation_id),
|
|
count(*) FILTER (WHERE coalesce(operation_id, '') = ''),
|
|
string_agg(event_type, ',' ORDER BY id),
|
|
coalesce(max(outcome) FILTER (WHERE event_type = 'environment_action_finished'), ''),
|
|
coalesce(max(reason_code) FILTER (WHERE event_type = 'environment_action_finished'), '')
|
|
FROM audit_event WHERE action = 'reconcile'`).Scan(
|
|
&events, &operations, &missingOperations, &eventTypes, &outcome, &reason); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if events != 2 || operations != 1 || missingOperations != 0 ||
|
|
eventTypes != "environment_action_requested,environment_action_finished" ||
|
|
outcome != wantOutcome || reason != wantReason {
|
|
t.Fatalf("reconcile audit mismatch: events=%d operations=%d missing=%d types=%q outcome=%q reason=%q",
|
|
events, operations, missingOperations, eventTypes, outcome, reason)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBrowserActionRoutesStartStopAndRejectsUnknown(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("expected 204 for start, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/stop", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("expected 204 for stop, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/pause", ""); response.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 for unknown action, got %d", response.Code)
|
|
}
|
|
if len(store.actions) != 4 || store.actions[0].Action != "start" || store.actions[1].Outcome != "succeeded" ||
|
|
store.actions[2].Action != "stop" || store.actions[3].Outcome != "succeeded" {
|
|
t.Fatalf("start and stop must each emit an audit pair: %#v", store.actions)
|
|
}
|
|
}
|
|
|
|
func TestStartRejectsPausedOrRevokedAccountBeforeGatewayCall(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name, status, authorization string
|
|
}{
|
|
{name: "paused", status: "paused", authorization: "authorized"},
|
|
{name: "revoked", status: "paused", authorization: "revoked"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: test.status,
|
|
AuthorizationStatus: test.authorization, BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"],
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/start", "")
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("start returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if len(gateway.recorded()) != 0 || len(store.actions) != 0 {
|
|
t.Fatalf("rejected start reached side effects: requests=%#v actions=%#v", gateway.recorded(), store.actions)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcileStopsRuntimeForPausedOrRevokedAccount(t *testing.T) {
|
|
for _, authorization := range []string{"authorized", "revoked"} {
|
|
t.Run(authorization, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: authorization,
|
|
BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"],
|
|
RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1",
|
|
}}}
|
|
_ = newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("reconcile failed: %v", err)
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
|
|
t.Fatalf("non-runnable account retained runtime %q", runtime)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodPost ||
|
|
requests[1].path != "/v1/browsers/account-a/stop" {
|
|
t.Fatalf("reconcile did not stop the runtime: %#v", requests)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestActivationConflictCleanupIsGenerationSafe(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
containerID string
|
|
wantDeleted bool
|
|
}{
|
|
{name: "candidate remains current", containerID: "candidate-container", wantDeleted: true},
|
|
{name: "alias points to newer container", containerID: "new-container"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
environment := hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "paused", AuthorizationStatus: "authorized",
|
|
BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"],
|
|
}
|
|
store.bindings[environment.Alias] = environment
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: test.containerID, Alias: environment.Alias, State: "running", ProxyReady: true,
|
|
BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, NetworkID: "native-11111111111111111111111111111111",
|
|
}}}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
target := hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
|
|
_, err := activateGatewayRuntime(context.Background(), store, target, environment, "candidate-container", "native-11111111111111111111111111111111")
|
|
if err == nil {
|
|
t.Fatal("paused account unexpectedly activated the candidate")
|
|
}
|
|
deleted := len(gateway.runtimes) == 0
|
|
if deleted != test.wantDeleted {
|
|
t.Fatalf("generation-safe cleanup mismatch: deleted=%v requests=%#v", deleted, gateway.recorded())
|
|
}
|
|
if !test.wantDeleted {
|
|
for _, request := range gateway.recorded() {
|
|
if request.method == http.MethodDelete {
|
|
t.Fatalf("stale candidate deleted a newer alias generation: %#v", gateway.recorded())
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExplicitStopRejectsStaleBindingBeforeGatewaySideEffect(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized",
|
|
BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "new-instance", RuntimeID: "new-container",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "new-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1",
|
|
}}}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
stale := store.bindings["account-a"]
|
|
stale.BindingVersion, stale.RuntimeInstanceID, stale.RuntimeID = 1, "old-instance", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
|
|
|
if err := stopEnvironmentRuntime(context.Background(), store, stale); !errors.Is(err, hub.ErrConflict) {
|
|
t.Fatalf("stale explicit stop was not rejected: %v", err)
|
|
}
|
|
if len(gateway.recorded()) != 0 || len(gateway.runtimes) != 1 || gateway.runtimes[0].ID != "new-container" {
|
|
t.Fatalf("stale explicit stop reached the gateway: requests=%#v runtimes=%#v", gateway.recorded(), gateway.runtimes)
|
|
}
|
|
}
|
|
|
|
func TestReconcileFinishedAuditUsesRebuiltRuntime(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized",
|
|
BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-2"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "exit-1", NetworkID: "native-66666666666666666666666666666666",
|
|
}}}
|
|
_ = newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("reconcile failed: %v", err)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "succeeded" ||
|
|
store.actions[1].RuntimeInstanceID != "runtime-instance" || store.actions[1].BindingVersion != 2 || store.actions[1].NetworkExitID != "exit-2" {
|
|
t.Fatalf("finished audit used stale runtime context: %#v", store.actions)
|
|
}
|
|
}
|
|
|
|
func TestReconcileContextRefreshFailureClearsAllAuditCorrelation(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", AccountStatus: "active", AuthorizationStatus: "authorized",
|
|
BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"], RuntimeInstanceID: "old-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", RuntimeNetworkID: "native-66666666666666666666666666666666",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666",
|
|
}}}
|
|
server := httptest.NewServer(gateway.handler(t))
|
|
defer server.Close()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token}
|
|
failing := &failContextRefreshStore{hubStore: store}
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), failing, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil {
|
|
t.Fatal("context refresh failure unexpectedly succeeded")
|
|
}
|
|
finished := store.actions[len(store.actions)-1]
|
|
if finished.RuntimeInstanceID != "" || finished.BindingVersion != 0 || finished.NetworkExitID != "" {
|
|
t.Fatalf("failed context refresh retained partial stale correlation: %#v", finished)
|
|
}
|
|
}
|
|
|
|
func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) {
|
|
gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.Method == http.MethodGet {
|
|
_ = json.NewEncoder(response).Encode([]runtimeStatus{{ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "exited"}})
|
|
return
|
|
}
|
|
connection, _, err := response.(http.Hijacker).Hijack()
|
|
if err == nil {
|
|
_ = connection.Close()
|
|
}
|
|
}))
|
|
defer gatewayServer.Close()
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
|
|
store.releaseErr = errors.New("database unavailable")
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/stop", "")
|
|
if response.Code != http.StatusInternalServerError {
|
|
t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_release_failed" {
|
|
t.Fatalf("stopped reconcile release failure must be audited: %#v", store.actions)
|
|
}
|
|
}
|
|
|
|
func TestReconcileAuditsRuntimeReleaseFailure(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
}
|
|
store.releaseErr = errors.New("database unavailable")
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "exited", BindingVersion: 1, NetworkExitID: "exit-1",
|
|
}}}
|
|
_ = newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil {
|
|
t.Fatal("expected release failure")
|
|
}
|
|
if len(store.actions) != 2 || store.actions[0].Action != "reconcile" || store.actions[1].Outcome != "failed" ||
|
|
store.actions[1].ReasonCode != "runtime_release_failed" {
|
|
t.Fatalf("reconcile release failure must have its own audit pair: %#v", store.actions)
|
|
}
|
|
}
|
|
|
|
func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("running runtime rebind failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
bound := store.bindings["account-a"]
|
|
if bound.Exit.ID != "exit-2" || bound.BindingVersion != 2 || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
|
t.Fatalf("running runtime was not rebuilt on the latest binding: %#v", bound)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 4 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost {
|
|
t.Fatalf("running rebind must inspect, delete and recreate: %#v", requests)
|
|
}
|
|
if requests[3].body["binding_version"] != float64(2) || requests[3].body["network_exit_id"] != "exit-2" {
|
|
t.Fatalf("recreated runtime did not carry the latest binding CAS: %#v", requests[3].body)
|
|
}
|
|
}
|
|
|
|
func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "expired-runtime", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-1"}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("same-exit rebind failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
bound := store.bindings["account-a"]
|
|
if bound.BindingVersion != 2 || bound.RuntimeID != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
|
t.Fatalf("same-exit rebind did not rotate the binding generation and lease: %#v", bound)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 4 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost {
|
|
t.Fatalf("same-exit rebind bypassed gateway reconciliation: %#v", requests)
|
|
}
|
|
if requests[3].body["binding_version"] != float64(2) || requests[3].body["network_exit_id"] != "exit-1" {
|
|
t.Fatalf("same-exit rebuild used stale metadata: %#v", requests[3].body)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
probe := fakeExitProbe{failure: "exit_auth_failed"}
|
|
_ = newTestAppWithNetwork(t, store, gateway, probe,
|
|
func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, probe, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err != nil {
|
|
t.Fatalf("unhealthy runtime reconciliation failed: %v", err)
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
|
|
t.Fatalf("failed exit remained active: %q", runtime)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 3 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete {
|
|
t.Fatalf("failed exit must be rechecked and removed instead of renewed: %#v", requests)
|
|
}
|
|
}
|
|
|
|
func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", RuntimeNetworkID: "native-66666666666666666666666666666666",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, runtimes: []runtimeStatus{{
|
|
ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "stale-exit", NetworkID: "native-66666666666666666666666666666666", ProxyReady: true,
|
|
}}}
|
|
_ = newTestApp(t, store, gateway)
|
|
|
|
if err := reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }); err == nil {
|
|
t.Fatal("delete failure unexpectedly succeeded")
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
|
|
t.Fatalf("delete failure retained DB runtime lease: %q", runtime)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[0].Action != "reconcile" || store.actions[1].Outcome != "unknown" ||
|
|
store.actions[1].ReasonCode != "gateway_result_unknown" {
|
|
t.Fatalf("delete failure was not audited as retryable unknown: %#v", store.actions)
|
|
}
|
|
if len(gateway.runtimes) != 1 {
|
|
t.Fatal("failed delete unexpectedly removed the gateway container")
|
|
}
|
|
}
|
|
|
|
func TestDisableExitImmediatelyDiscardsRuntime(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "")
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("disable cleanup failed: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
|
|
t.Fatalf("disabled exit remained active: %q", runtime)
|
|
}
|
|
requests := gateway.recorded()
|
|
if len(requests) != 3 || requests[0].method != http.MethodGet || requests[2].method != http.MethodDelete {
|
|
t.Fatalf("disable must reconcile and remove the runtime: %#v", requests)
|
|
}
|
|
}
|
|
|
|
func TestDisableExitWaitsForInFlightBrowserLifecycle(t *testing.T) {
|
|
releaseCreate := make(chan struct{})
|
|
gateway := &fakeGateway{
|
|
token: "unit-test-gateway-token",
|
|
createStarted: make(chan struct{}),
|
|
releaseCreate: releaseCreate,
|
|
}
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"],
|
|
}
|
|
gatewayServer := httptest.NewServer(gateway.handler(t))
|
|
defer gatewayServer.Close()
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
server := httptest.NewServer(adaptor.FiberApp(app))
|
|
defer server.Close()
|
|
|
|
statuses := make(chan int, 2)
|
|
go func() {
|
|
response, err := server.Client().Post(server.URL+"/api/browsers/account-a/start", "application/json", nil)
|
|
if err != nil {
|
|
statuses <- 0
|
|
return
|
|
}
|
|
defer response.Body.Close()
|
|
statuses <- response.StatusCode
|
|
}()
|
|
select {
|
|
case <-gateway.createStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("start did not reach gateway create")
|
|
}
|
|
|
|
go func() {
|
|
response, err := server.Client().Post(server.URL+"/api/network-exits/exit-1/disable", "application/json", nil)
|
|
if err != nil {
|
|
statuses <- 0
|
|
return
|
|
}
|
|
defer response.Body.Close()
|
|
statuses <- response.StatusCode
|
|
}()
|
|
select {
|
|
case status := <-statuses:
|
|
t.Fatalf("lifecycle operation completed before the running create: %d", status)
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
close(releaseCreate)
|
|
for range 2 {
|
|
if status := <-statuses; status != http.StatusNoContent && status != http.StatusOK {
|
|
t.Fatalf("unexpected lifecycle status: %d", status)
|
|
}
|
|
}
|
|
if store.exits["exit-1"].HealthStatus != "disabled" || store.bindings["account-a"].RuntimeID != "" {
|
|
t.Fatalf("disable did not clean the completed runtime: exit=%#v binding=%#v", store.exits["exit-1"], store.bindings["account-a"])
|
|
}
|
|
}
|
|
|
|
func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
handler http.HandlerFunc
|
|
}{
|
|
{name: "disconnect", handler: func(response http.ResponseWriter, _ *http.Request) {
|
|
connection, _, err := response.(http.Hijacker).Hijack()
|
|
if err == nil {
|
|
_ = connection.Close()
|
|
}
|
|
}},
|
|
{name: "status 500", handler: func(response http.ResponseWriter, _ *http.Request) {
|
|
response.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = response.Write([]byte(`{"error":"docker unavailable"}`))
|
|
}},
|
|
{name: "invalid JSON", handler: func(response http.ResponseWriter, _ *http.Request) {
|
|
_, _ = response.Write([]byte(`{"not":"a browser list"}`))
|
|
}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
gatewayServer := httptest.NewServer(test.handler)
|
|
defer gatewayServer.Close()
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
}
|
|
store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"}
|
|
app := fiber.New()
|
|
registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil })
|
|
|
|
response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "")
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("gateway unknown must not return 200: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
|
t.Fatalf("gateway unknown changed the unconfirmed runtime lease: %q", runtime)
|
|
}
|
|
if store.exits["exit-1"].HealthStatus != "disabled" {
|
|
t.Fatal("disable state was not retained for retryable reconciliation")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDirectBindingIsListable(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{
|
|
Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true,
|
|
}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
RuntimeInstanceID: "4444444444444444444444444444444444444444444444444444444444444444", RuntimeID: "direct-container", RuntimeNetworkID: "direct-network",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "direct-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkID: "direct-network", ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodGet, "/api/browsers", "")
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("direct binding broke browser listing: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
var views []envView
|
|
if err := json.Unmarshal(response.Body.Bytes(), &views); err != nil || len(views) != 1 || views[0].RecoveryRequired || views[0].NetworkExitID != "" || views[0].ScheduleStatus != "ready" {
|
|
t.Fatalf("direct runtime state was not visible: %#v err=%v", views, err)
|
|
}
|
|
}
|
|
|
|
func TestRebindDeleteFailureKeepsOriginalBinding(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
binding hub.EnvironmentContext
|
|
targetID string
|
|
}{
|
|
{name: "running binding", binding: hub.EnvironmentContext{
|
|
AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: hub.NetworkExit{ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy"},
|
|
RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}, targetID: "exit-2"},
|
|
{name: "legacy NULL binding", binding: hub.EnvironmentContext{
|
|
AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
RuntimeInstanceID: "5555555555555555555555555555555555555555555555555555555555555555", RuntimeID: "legacy-container",
|
|
}, targetID: "exit-1"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
before := test.binding
|
|
before.Env = store.envs["account-a"]
|
|
store.bindings["account-a"] = before
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, runtimes: []runtimeStatus{{
|
|
ID: before.RuntimeID, Alias: "account-a", State: "running", BindingVersion: before.BindingVersion, NetworkExitID: before.Exit.ID, ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+test.targetID+`"}`)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("delete failure must remain unknown: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after := store.bindings["account-a"]
|
|
if after.BindingVersion != before.BindingVersion || after.Exit.ID != before.Exit.ID || !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("binding committed before old container deletion: before=%#v after=%#v", before, after)
|
|
}
|
|
if len(gateway.runtimes) != 1 {
|
|
t.Fatalf("non-final delete result lost the existing container: %#v", gateway.runtimes)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
|
|
t.Fatalf("delete failure was not audited as unknown: %#v", store.actions)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRebindNetworkCleanupPendingBlocksUntilConfirmed(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
cleanupPending int
|
|
disconnectDelete int
|
|
disconnectListAfterDelete bool
|
|
}{
|
|
{name: "cleanup pending", cleanupPending: 2},
|
|
{name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: test.cleanupPending,
|
|
disconnectDelete: test.disconnectDelete, disconnectListAfterDelete: test.disconnectListAfterDelete,
|
|
runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("network cleanup uncertainty returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after := store.bindings["account-a"]
|
|
if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || !after.RuntimeCleanupPending || after.RuntimeID != "" {
|
|
t.Fatalf("network cleanup uncertainty changed the binding: %#v", after)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
|
|
t.Fatalf("network cleanup uncertainty audit mismatch: %#v", store.actions)
|
|
}
|
|
if len(gateway.runtimes) != 0 {
|
|
t.Fatalf("container-removed fact was lost: %#v", gateway.runtimes)
|
|
}
|
|
|
|
response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
|
|
if response.Code != http.StatusOK {
|
|
t.Fatalf("confirmed cleanup did not allow retry: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after = store.bindings["account-a"]
|
|
if after.RuntimeCleanupPending || after.BindingVersion != 2 || after.Exit.ID != "exit-2" || after.RuntimeID != "" {
|
|
t.Fatalf("retry did not commit a clean stopped generation: %#v", after)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name, method, path, body string
|
|
}{
|
|
{name: "create reuse", method: http.MethodPost, path: "/api/browsers", body: `{"alias":"account-a","name":"甲","gateway":"gw-1","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`},
|
|
{name: "start", method: http.MethodPost, path: "/api/browsers/account-a/start"},
|
|
{name: "rebind", method: http.MethodPost, path: "/api/browsers/account-a/rebind", body: `{"network_exit_id":"exit-1"}`},
|
|
{name: "recycle", method: http.MethodDelete, path: "/api/browsers/account-a"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
RuntimeCleanupPending: true, RuntimeCleanupBindingVersion: 1,
|
|
RuntimeCleanupRuntimeID: missingRuntimeID, RuntimeCleanupNetworkID: "native-66666666666666666666666666666666", Exit: store.exits["exit-1"],
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: 2}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, test.method, test.path, test.body)
|
|
if response.Code != http.StatusBadGateway {
|
|
t.Fatalf("pending cleanup returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after := store.bindings["account-a"]
|
|
if !after.RuntimeCleanupPending || after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" {
|
|
t.Fatalf("lifecycle path advanced a pending generation: %#v", after)
|
|
}
|
|
for _, request := range gateway.recorded() {
|
|
if request.method == http.MethodPost && request.path == "/v1/browsers" {
|
|
t.Fatalf("lifecycle path created before cleanup confirmation: %#v", gateway.recorded())
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) {
|
|
t.Skip("browser path lookup is gateway-owned")
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a",
|
|
BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, nil)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
|
|
if response.Code < 400 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet {
|
|
t.Fatalf("runtime preparation failure touched the old container: status=%d requests=%#v", response.Code, gateway.recorded())
|
|
}
|
|
if after := store.bindings["account-a"]; after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" {
|
|
t.Fatalf("runtime preparation failure changed state: %#v", after)
|
|
}
|
|
}
|
|
|
|
func TestRebindCandidateUnknownCreateLeavesGenerationPending(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a",
|
|
BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, runtimes: []runtimeStatus{{
|
|
ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true,
|
|
}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`)
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("candidate create failure returned %d: %s", response.Code, response.Body.String())
|
|
}
|
|
after := store.bindings["account-a"]
|
|
if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" || !after.RuntimeCleanupPending ||
|
|
after.RuntimeCleanupBindingVersion != 2 || after.RuntimeCleanupRuntimeID != missingRuntimeID || after.RuntimeCleanupNetworkID != "" ||
|
|
len(gateway.runtimes) != 0 {
|
|
t.Fatalf("candidate create unknown result was not retained for manual reconcile: after=%#v runtimes=%#v", after, gateway.runtimes)
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" {
|
|
t.Fatalf("candidate create failure audit mismatch: %#v", store.actions)
|
|
}
|
|
}
|
|
|
|
func TestExistingEnvironmentCleanupNeverReturnsReusedSuccess(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
probe *sequenceExitProbe
|
|
resolve func(hub.NetworkExitAccess) (string, error)
|
|
}{
|
|
{name: "second probe fails", probe: &sequenceExitProbe{failures: []string{"", "exit_auth_failed"}},
|
|
resolve: func(hub.NetworkExitAccess) (string, error) { return "", nil }},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-1"] = hub.NetworkExit{
|
|
ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1,
|
|
Username: "username", Password: "password",
|
|
}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 2024, Platform: "windows", Timezone: "Asia/Shanghai"}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{
|
|
Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1,
|
|
Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{
|
|
ID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1",
|
|
}}}
|
|
app := newTestAppWithNetwork(t, store, gateway, test.probe, test.resolve)
|
|
|
|
response := do(app, http.MethodPost, "/api/browsers", createEnvBody)
|
|
if response.Code != http.StatusConflict {
|
|
t.Fatalf("cleaned runtime must not return reused success: %d: %s", response.Code, response.Body.String())
|
|
}
|
|
if !strings.Contains(response.Body.String(), "not running") {
|
|
t.Fatalf("cleaned runtime response did not state the stopped result: %s", response.Body.String())
|
|
}
|
|
if runtime := store.bindings["account-a"].RuntimeID; runtime != "" {
|
|
t.Fatalf("failed recovery retained runtime: %q probe_calls=%d actions=%#v requests=%#v", runtime, test.probe.calls, store.actions, gateway.recorded())
|
|
}
|
|
if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_unavailable" {
|
|
t.Fatalf("cleaned runtime was audited as reused success: %#v", store.actions)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1}
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"]}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token", runtimes: []runtimeStatus{{ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "exited", BindingVersion: 1}}}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`); response.Code != http.StatusOK {
|
|
t.Fatalf("rebind failed: %d %s", response.Code, response.Body.String())
|
|
}
|
|
if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("start failed: %d %s", response.Code, response.Body.String())
|
|
}
|
|
requests := gateway.recorded()
|
|
var createRequest recordedRequest
|
|
for _, request := range requests {
|
|
if request.method == http.MethodPost && request.path == "/v1/browsers" {
|
|
createRequest = request
|
|
}
|
|
}
|
|
if createRequest.body == nil {
|
|
t.Fatalf("stopped container was not recreated: %#v", requests)
|
|
}
|
|
exit := createRequest.body["network_exit"].(map[string]any)
|
|
if exit["host"] != "proxy-2.example" {
|
|
t.Fatalf("recreated container did not use rebound exit: %#v", createRequest.body)
|
|
}
|
|
}
|
|
|
|
func TestRecycleBrowserKeepsStableEnvironment(t *testing.T) {
|
|
store := newMemoryStore()
|
|
store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", Fingerprint: hub.Fingerprint{Seed: 1}}
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
if response := do(app, http.MethodDelete, "/api/browsers/account-a", ""); response.Code != http.StatusNoContent {
|
|
t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String())
|
|
}
|
|
_, kept := store.envs["account-a"]
|
|
if !kept {
|
|
t.Fatal("recycle must preserve the stable environment and Profile anchor")
|
|
}
|
|
if len(store.actions) != 2 || store.actions[0].Action != "recycle" || store.actions[1].Outcome != "succeeded" {
|
|
t.Fatalf("recycle must emit an audit pair: %#v", store.actions)
|
|
}
|
|
}
|
|
|
|
func TestNetworkExitRoutesStoreAndExposePlainCredentials(t *testing.T) {
|
|
store := newMemoryStore()
|
|
gateway := &fakeGateway{token: "unit-test-gateway-token"}
|
|
app := newTestApp(t, store, gateway)
|
|
|
|
invalid := do(app, http.MethodPost, "/api/network-exits",
|
|
`{"protocol":"socks5","host":"proxy.example","port":1080,"password":"password-only"}`)
|
|
if invalid.Code != http.StatusBadRequest || len(store.exits) != 1 {
|
|
t.Fatalf("password without username must be rejected: status=%d exits=%#v", invalid.Code, store.exits)
|
|
}
|
|
created := do(app, http.MethodPost, "/api/network-exits",
|
|
`{"protocol":"socks5","host":"proxy.example","port":1080,"username":"proxy-user","password":"plain-password","expected_public_ip":"203.0.113.1","expected_region":"test"}`)
|
|
if created.Code != http.StatusCreated || !strings.Contains(created.Body.String(), `"username":"proxy-user"`) ||
|
|
!strings.Contains(created.Body.String(), `"password":"plain-password"`) {
|
|
t.Fatalf("network exit response must expose stored credentials: status=%d body=%s", created.Code, created.Body.String())
|
|
}
|
|
detail := do(app, http.MethodGet, "/api/network-exits/exit-created", "")
|
|
if detail.Code != http.StatusOK || !strings.Contains(detail.Body.String(), `"username":"proxy-user"`) ||
|
|
!strings.Contains(detail.Body.String(), `"password":"plain-password"`) {
|
|
t.Fatalf("network exit detail must expose stored credentials: status=%d body=%s", detail.Code, detail.Body.String())
|
|
}
|
|
checked := do(app, http.MethodPost, "/api/network-exits/exit-created/check", "")
|
|
if checked.Code != http.StatusOK || !strings.Contains(checked.Body.String(), `"health_status":"healthy"`) {
|
|
t.Fatalf("network exit check did not record health: status=%d body=%s", checked.Code, checked.Body.String())
|
|
}
|
|
disabled := do(app, http.MethodPost, "/api/network-exits/exit-created/disable", "")
|
|
if disabled.Code != http.StatusOK || !strings.Contains(disabled.Body.String(), `"health_status":"disabled"`) {
|
|
t.Fatalf("network exit disable failed: status=%d body=%s", disabled.Code, disabled.Body.String())
|
|
}
|
|
}
|