package environment import ( "context" "encoding/json" "errors" "fmt" "os" "reflect" "slices" "strings" "sync/atomic" "testing" "time" "git.ipao.vip/rogee/creator-hub/internal/taskstate" ) func TestNetworkExitCredentialValidation(t *testing.T) { valid := NetworkExit{Protocol: "socks5", Host: "proxy.example", Port: 1080, Username: "operator", Password: "plain-password"} if !validNetworkExit(valid) { t.Fatal("valid stored credentials were rejected") } for name, mutate := range map[string]func(*NetworkExit){ "password without username": func(exit *NetworkExit) { exit.Username = "" }, "username too long": func(exit *NetworkExit) { exit.Username = strings.Repeat("u", 256) }, "password too long": func(exit *NetworkExit) { exit.Password = strings.Repeat("p", 256) }, "username control character": func(exit *NetworkExit) { exit.Username = "operator\n" }, "password control character": func(exit *NetworkExit) { exit.Password = "plain\x7fpassword" }, } { t.Run(name, func(t *testing.T) { exit := valid mutate(&exit) if validNetworkExit(exit) { t.Fatalf("invalid credentials were accepted: %#v", exit) } }) } } func TestEnvironmentLocksCoordinateAcrossStoreInstances(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() testURL := isolatedDatabaseURL(t, databaseURL) first := openFullyMigratedHub(t, ctx, testURL) t.Cleanup(func() { _ = first.Close() }) second, err := Open(ctx, testURL) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = second.Close() }) unlockFirst, err := first.LockResources(ctx, []string{"account-a"}, nil, nil) if err != nil { t.Fatal(err) } firstReleased := false defer func() { if !firstReleased { unlockFirst() } }() differentAlias, err := second.LockResources(ctx, []string{"account-b"}, nil, nil) if err != nil { t.Fatalf("different aliases must not share a lock: %v", err) } differentAlias() acquired := make(chan func(), 1) errors := make(chan error, 1) started := make(chan struct{}) go func() { close(started) unlock, lockErr := second.LockResources(ctx, []string{"account-a"}, nil, nil) if lockErr != nil { errors <- lockErr return } acquired <- unlock }() <-started select { case unlock := <-acquired: unlock() t.Fatal("same alias lock did not block across Store instances") case err := <-errors: t.Fatal(err) case <-time.After(50 * time.Millisecond): } unlockFirst() firstReleased = true select { case unlock := <-acquired: unlock() case err := <-errors: t.Fatal(err) case <-time.After(time.Second): t.Fatal("same alias lock was not released") } } func TestResourceLocksReserveConnectionsForLifecycleQueries(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) t.Cleanup(func() { _ = store.Close() }) const workers = 10 var acquired atomic.Int32 startQueries := make(chan struct{}) done := make(chan error, workers) for worker := 0; worker < workers; worker++ { go func(worker int) { unlock, err := store.LockResources(ctx, []string{fmt.Sprintf("account-%d", worker)}, nil, nil) if err != nil { done <- err return } defer unlock() acquired.Add(1) <-startQueries _, err = store.ListEnvs(ctx) done <- err }(worker) } deadline := time.NewTimer(100 * time.Millisecond) ticker := time.NewTicker(time.Millisecond) for acquired.Load() < workers { select { case <-ticker.C: case <-deadline.C: goto release } } release: ticker.Stop() if !deadline.Stop() { select { case <-deadline.C: default: } } close(startQueries) for worker := 0; worker < workers; worker++ { if err := <-done; err != nil { t.Fatalf("locked lifecycle query %d did not complete: %v", worker, err) } } } func TestFingerprintArgsFollowUpstreamCommandLineContract(t *testing.T) { full := Fingerprint{ Seed: 2024, Platform: "windows", PlatformVersion: "11.0.0", Brand: "Edge", BrandVersion: "132.0.6834.159", HardwareConcurrency: 8, Lang: "zh-CN", AcceptLang: "zh-CN,en-US", Timezone: "Asia/Shanghai", ProxyServer: "socks5://127.0.0.1:1080", DisableNonProxiedUDP: true, DisableSpoofing: "font,gpu", } if err := full.Validate(); err != nil { t.Fatalf("expected full fingerprint to be valid: %v", err) } want := []string{ "--fingerprint=2024", "--fingerprint-platform=windows", "--fingerprint-platform-version=11.0.0", "--fingerprint-brand=Edge", "--fingerprint-brand-version=132.0.6834.159", "--fingerprint-hardware-concurrency=8", "--lang=zh-CN", "--accept-lang=zh-CN,en-US", "--timezone=Asia/Shanghai", "--proxy-server=socks5://127.0.0.1:1080", "--disable-non-proxied-udp", "--disable-spoofing=font,gpu", } if !reflect.DeepEqual(full.Args(), want) { t.Fatalf("unexpected args:\n got %v\nwant %v", full.Args(), want) } minimal := Fingerprint{Seed: 1} if err := minimal.Validate(); err != nil { t.Fatalf("minimal fingerprint must be valid: %v", err) } if args := minimal.Args(); len(args) != 1 || args[0] != "--fingerprint=1" { t.Fatalf("zero-value fields must be omitted: %v", args) } encoded, err := json.Marshal(minimal) if err != nil { t.Fatal(err) } var decoded Fingerprint if err := json.Unmarshal(encoded, &decoded); err != nil || decoded.Seed != 1 || decoded.Args()[0] != "--fingerprint=1" { t.Fatalf("fingerprint must survive JSON round trip: %#v %v", decoded, err) } } func TestFingerprintValidateRejectsUnsupportedValues(t *testing.T) { invalid := map[string]Fingerprint{ "seed zero": {Seed: 0}, "seed overflow": {Seed: 2147483648}, "platform": {Seed: 1, Platform: "android"}, "brand": {Seed: 1, Brand: "Firefox"}, "platform version": {Seed: 1, PlatformVersion: "bad value"}, "brand version": {Seed: 1, BrandVersion: strings.Repeat("x", 33)}, "concurrency": {Seed: 1, HardwareConcurrency: 129}, "lang": {Seed: 1, Lang: "zh CN"}, "accept lang": {Seed: 1, AcceptLang: "zh-CN;drop"}, "timezone": {Seed: 1, Timezone: "Asia/Shanghai\n"}, "proxy scheme": {Seed: 1, ProxyServer: "ftp://proxy:21"}, "proxy host": {Seed: 1, ProxyServer: "http://"}, "proxy userinfo": {Seed: 1, ProxyServer: "socks5://user:password@proxy:1080"}, "spoofing unknown": {Seed: 1, DisableSpoofing: "webrtc"}, "spoofing repeated": {Seed: 1, DisableSpoofing: "font,font"}, } for name, fingerprint := range invalid { t.Run(name, func(t *testing.T) { if err := fingerprint.Validate(); err == nil { t.Fatalf("expected rejection for %#v", fingerprint) } }) } } func TestStoreValidationRejectsInvalidInputsBeforePersistence(t *testing.T) { store := &Store{} ctx := context.Background() if _, err := store.CreateGateway(ctx, "bad name!", "http://gw:8081", ""); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid gateway name, got %v", err) } if _, err := store.CreateGateway(ctx, "gw-1", "ftp://gw:8081", ""); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid gateway endpoint, got %v", err) } if _, err := store.CreateGateway(ctx, "gw-1", "http://gw:8081", "short-token"); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid gateway token, got %v", err) } for _, test := range []struct { name string currentName string newName string endpoint string token string }{ {name: "current name", currentName: "bad name!", newName: "gw-2", endpoint: "http://gw:8081"}, {name: "new name", currentName: "gw-1", newName: "bad name!", endpoint: "http://gw:8081"}, {name: "endpoint", currentName: "gw-1", newName: "gw-2", endpoint: "ftp://gw:8081"}, {name: "token", currentName: "gw-1", newName: "gw-2", endpoint: "http://gw:8081", token: "short-token"}, } { if _, err := store.UpdateGateway(ctx, test.currentName, test.newName, test.endpoint, test.token); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid gateway update %s, got %v", test.name, err) } } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "v1", BrowserPath: "registry/img:1"}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid image version, got %v", err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.0.1", BrowserPath: "has space"}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid image ref, got %v", err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.0.1", BrowserPath: "registry/img:1", Note: strings.Repeat("长", 201)}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected overlong note to be rejected, got %v", err) } if err := store.CreateEnv(ctx, Env{Alias: "UP", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid alias, got %v", err) } if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: strings.Repeat("名", 65), Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected overlong name, got %v", err) } if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 0}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid fingerprint, got %v", err) } for name, exit := range map[string]NetworkExit{ "protocol": {Protocol: "direct", Host: "proxy.example", Port: 1080}, "userinfo": {Protocol: "socks5", Host: "user@proxy.example", Port: 1080}, "URL host": {Protocol: "socks5", Host: "socks5://proxy.example", Port: 1080}, "port": {Protocol: "socks5", Host: "proxy.example", Port: 0}, "ip": {Protocol: "socks5", Host: "proxy.example", Port: 1080, ExpectedPublicIP: "not-an-ip"}, } { t.Run("network exit "+name, func(t *testing.T) { if _, err := store.CreateNetworkExit(ctx, exit); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid network exit, got %v", err) } }) } if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1, ProxyServer: "socks5://proxy.example:1080"}}); !errors.Is(err, ErrInvalid) { t.Fatalf("stored fingerprint proxy must be rejected, got %v", err) } } func TestFingerprintSeedIsGloballyUnique(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) t.Cleanup(func() { _ = store.Close() }) if _, err := store.db.ExecContext(ctx, `TRUNCATE environment_binding, browser_env, browser_version, social_account, credential_reference, gateway CASCADE`); err != nil { t.Fatal(err) } if _, err := store.CreateGateway(ctx, "gw-seed", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil { t.Fatal(err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-seed-a', 'os_keyring', 'creatorhub/seed-a'), ('credential-seed-b', 'os_keyring', 'creatorhub/seed-b'); INSERT INTO social_account (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) VALUES ('seed-account-a', 'credential-seed-a', 'mock', 'seed-account-a', 'owned', 'authorized'), ('seed-account-b', 'credential-seed-b', 'mock', 'seed-account-b', 'owned', 'authorized')`); err != nil { t.Fatal(err) } env := Env{Alias: "seed-environment-a", Name: "Seed A", Gateway: "gw-seed", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 77}} if _, created, err := store.CreateBoundEnv(ctx, env, "seed-account-a", ""); err != nil || !created { t.Fatalf("create first seeded environment: created=%v err=%v", created, err) } env.Alias, env.Name = "seed-environment-b", "Seed B" if _, _, err := store.CreateBoundEnv(ctx, env, "seed-account-b", ""); !errors.Is(err, ErrConflict) { t.Fatalf("duplicate fingerprint seed was accepted: %v", err) } } func TestHubWorkflow(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } databaseURL = isolatedDatabaseURL(t, databaseURL) ctx := context.Background() store := openFullyMigratedHub(t, ctx, databaseURL) t.Cleanup(func() { _ = store.Close() }) if _, err := store.db.ExecContext(ctx, `TRUNCATE browser_env, browser_version, gateway CASCADE`); err != nil { t.Fatal(err) } gateway, err := store.CreateGateway(ctx, "gw-main", "http://127.0.0.1:8081", "") if err != nil { t.Fatal(err) } if len(gateway.Token) != 48 { t.Fatalf("platform must assign a 48-hex-char token: %q", gateway.Token) } custom, err := store.CreateGateway(ctx, "gw-custom", "http://127.0.0.3:8081", "operator-provided-token-1234") if err != nil || custom.Token != "operator-provided-token-1234" { t.Fatalf("explicit token must be honored: %#v %v", custom, err) } if _, err := store.CreateGateway(ctx, "gw-main", "http://127.0.0.2:8081", ""); !errors.Is(err, ErrConflict) { t.Fatalf("expected duplicate gateway conflict, got %v", err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148", Note: "主力版本", Enabled: true}); err != nil { t.Fatal(err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: false}); err != nil { t.Fatal(err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148.0.7778.215", BrowserPath: "/opt/creatorhub/browsers/148b"}); !errors.Is(err, ErrConflict) { t.Fatalf("expected duplicate version conflict, got %v", err) } env := Env{ Alias: "shop-01", Name: "店铺一号", Gateway: "gw-main", BrowserVersion: "148.0.7778.215", Fingerprint: Fingerprint{Seed: 1000, Timezone: "Asia/Shanghai", Lang: "zh-CN"}, } if err := store.CreateEnv(ctx, env); err != nil { t.Fatal(err) } if err := store.CreateEnv(ctx, env); !errors.Is(err, ErrConflict) { t.Fatalf("expected duplicate alias conflict, got %v", err) } if err := store.CreateEnv(ctx, Env{Alias: "shop-02", Name: "店铺二号", Gateway: "missing", BrowserVersion: "148.0.7778.215", Fingerprint: Fingerprint{Seed: 1}}); !errors.Is(err, ErrConflict) { t.Fatalf("expected unknown gateway conflict, got %v", err) } listed, err := store.ListEnvs(ctx) if err != nil || len(listed) != 1 { t.Fatalf("expected one env, err=%v list=%#v", err, listed) } if listed[0].Name != "店铺一号" || listed[0].Fingerprint.Seed != 1000 || listed[0].Fingerprint.Timezone != "Asia/Shanghai" { t.Fatalf("fingerprint must round trip through jsonb: %#v", listed[0]) } updatedGateway, err := store.UpdateGateway(ctx, "gw-main", "gw-renamed", "http://127.0.0.4:8081", "") if err != nil || updatedGateway.Name != "gw-renamed" || updatedGateway.Endpoint != "http://127.0.0.4:8081" || updatedGateway.Token != gateway.Token { t.Fatalf("gateway update did not preserve the token: %#v err=%v", updatedGateway, err) } renamedEnv, err := store.GetEnv(ctx, "shop-01") if err != nil || renamedEnv.Gateway != "gw-renamed" { t.Fatalf("gateway rename did not cascade to environment: %#v err=%v", renamedEnv, err) } if _, err := store.UpdateGateway(ctx, "gw-renamed", "gw-custom", "http://127.0.0.4:8081", ""); !errors.Is(err, ErrConflict) { t.Fatalf("expected gateway rename conflict, got %v", err) } if _, err := store.UpdateGateway(ctx, "missing", "gw-missing", "http://127.0.0.5:8081", ""); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing gateway on update, got %v", err) } if _, err := store.UpdateGateway(ctx, "gw-renamed", "gw-main", "http://127.0.0.1:8081", ""); err != nil { t.Fatalf("restore gateway name after cascade check: %v", err) } if _, err := store.GetEnv(ctx, "ghost"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing env, got %v", err) } if _, err := store.BrowserPath(ctx, "144.0.7559.132"); !errors.Is(err, ErrNotFound) { t.Fatalf("disabled version must not resolve, got %v", err) } if err := store.UpdateBrowserVersion(ctx, BrowserVersion{Version: "144.0.7559.132", BrowserPath: "/opt/creatorhub/browsers/144", Enabled: true}); err != nil { t.Fatal(err) } if ref, err := store.BrowserPath(ctx, "144.0.7559.132"); err != nil || !strings.HasSuffix(ref, "/144") { t.Fatalf("enabled version must resolve: %v %q", err, ref) } if _, err := store.BrowserPath(ctx, "999"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing version, got %v", err) } if err := store.UpgradeEnv(ctx, "shop-01", "144.0.7559.132"); err != nil { t.Fatal(err) } upgraded, err := store.GetEnv(ctx, "shop-01") if err != nil || upgraded.BrowserVersion != "144.0.7559.132" || upgraded.Fingerprint.Seed != 1000 { t.Fatalf("upgrade must only change image version: %#v %v", upgraded, err) } if err := store.UpgradeEnv(ctx, "ghost", "144.0.7559.132"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing env on upgrade, got %v", err) } if err := store.DeleteBrowserVersion(ctx, "144.0.7559.132"); !errors.Is(err, ErrConflict) { t.Fatalf("referenced version must not be deletable, got %v", err) } if err := store.DeleteGateway(ctx, "gw-main"); !errors.Is(err, ErrConflict) { t.Fatalf("referenced gateway must not be deletable, got %v", err) } if err := store.DeleteEnv(ctx, "shop-01"); err != nil { t.Fatal(err) } if err := store.DeleteEnv(ctx, "shop-01"); !errors.Is(err, ErrNotFound) { t.Fatalf("expected missing env on double delete, got %v", err) } if err := store.DeleteBrowserVersion(ctx, "144.0.7559.132"); err != nil { t.Fatal(err) } if err := store.DeleteGateway(ctx, "gw-main"); err != nil { t.Fatal(err) } if err := store.DeleteGateway(ctx, "gw-custom"); err != nil { t.Fatal(err) } } func TestRuntimeUseLeaseLifecycle(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) t.Cleanup(func() { _ = store.Close() }) if _, err := store.db.ExecContext(ctx, `TRUNCATE runtime_use_lease, runtime_instance, environment_binding, browser_env, browser_version, gateway, social_account, credential_reference CASCADE`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` INSERT INTO credential_reference (id, provider, reference_key) VALUES ('lease-credential', 'os_keyring', 'creatorhub/lease-account'); INSERT INTO social_account (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) VALUES ('lease-account', 'lease-credential', 'mock', 'lease-account', 'owned', 'authorized')`); err != nil { t.Fatal(err) } if _, err := store.CreateGateway(ctx, "lease-gateway", "http://127.0.0.1:8081", "lease-gateway-token"); err != nil { t.Fatal(err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148-lease", BrowserPath: "/opt/creatorhub/browsers/lease", Enabled: true}); err != nil { t.Fatal(err) } environment, created, err := store.CreateBoundEnv(ctx, Env{ Alias: "lease-env", Name: "Lease environment", Gateway: "lease-gateway", BrowserVersion: "148-lease", Fingerprint: Fingerprint{Seed: 1}, }, "lease-account", "") if err != nil || !created { t.Fatalf("create lease environment: %#v created=%v err=%v", environment, created, err) } if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'lease-account'`); err != nil { t.Fatal(err) } active, err := store.ActivateRuntime(ctx, environment.Alias, "runtime-lease-generation", environment.BindingVersion, "", "native-lease-network") if err != nil { t.Fatal(err) } lease, err := store.AcquireRuntimeUse(ctx, environment.Alias, "task", "lease-owner", "") if err != nil || lease.RuntimeInstanceID != active.RuntimeInstanceID || lease.Purpose != "task" { t.Fatalf("acquire runtime-use lease: %#v err=%v", lease, err) } if renewed, renewErr := store.RenewRuntimeUse(ctx, lease.Token); renewErr != nil || !renewed.LeaseUntil.After(lease.LeaseUntil) { t.Fatalf("renew runtime-use lease: %#v err=%v", renewed, renewErr) } if err := store.ReleaseRuntimeUse(ctx, lease.Token); err != nil { t.Fatal(err) } if err := store.ReleaseRuntimeUse(ctx, lease.Token); err != nil { t.Fatalf("releasing an already released lease must be idempotent: %v", err) } listener, err := store.AcquireRuntimeUse(ctx, environment.Alias, "listener", "listener-owner", "") if err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_use_lease SET lease_until = now() - interval '1 second' WHERE token = $1`, listener.Token); err != nil { t.Fatal(err) } if _, err := store.RenewRuntimeUse(ctx, listener.Token); !errors.Is(err, ErrConflict) { t.Fatalf("expired runtime-use lease renewed: %v", err) } if err := store.ReleaseRuntime(ctx, active); err != nil { t.Fatal(err) } var released, runtimeReleased bool if err := store.db.QueryRowContext(ctx, `SELECT released_at IS NOT NULL FROM runtime_use_lease WHERE token = $1`, listener.Token).Scan(&released); err != nil { t.Fatal(err) } if err := store.db.QueryRowContext(ctx, `SELECT released_at IS NOT NULL FROM runtime_instance WHERE id = $1`, active.RuntimeInstanceID).Scan(&runtimeReleased); err != nil { t.Fatal(err) } if !released || !runtimeReleased { t.Fatalf("runtime cleanup did not revoke use leases: lease_released=%v runtime_released=%v", released, runtimeReleased) } if _, err := store.RenewRuntimeUse(ctx, listener.Token); !errors.Is(err, ErrConflict) { t.Fatalf("runtime cleanup left an old lease renewable: %v", err) } } func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") } ctx := context.Background() store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) t.Cleanup(func() { _ = store.Close() }) var notifications []taskstate.Transition store.SetTaskNotifier(func(transition taskstate.Transition) { notifications = append(notifications, transition) }) if _, err := store.db.ExecContext(ctx, `TRUNCATE audit_event, runtime_instance, environment_binding, network_exit, social_account, credential_reference, browser_env, browser_version, gateway CASCADE`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, ` INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-account', 'os_keyring', 'creatorhub/account-a'); INSERT INTO social_account (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) VALUES ('account-a', 'credential-account', 'mock', 'account-a', 'owned', 'authorized')`); err != nil { t.Fatal(err) } if _, err := store.CreateGateway(ctx, "gw-main", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil { t.Fatal(err) } if err := store.CreateBrowserVersion(ctx, BrowserVersion{Version: "148", BrowserPath: "/opt/creatorhub/browsers/148", Enabled: true}); err != nil { t.Fatal(err) } exit, err := store.CreateNetworkExit(ctx, NetworkExit{ Protocol: "socks5", Host: "proxy.example", Port: 1080, Username: "proxy-user", Password: "plain-password", ExpectedPublicIP: "203.0.113.10", ExpectedRegion: "test-region", }) if err != nil || exit.HealthStatus != "unchecked" || exit.Username != "proxy-user" || exit.Password != "plain-password" { t.Fatalf("unexpected network exit: %#v err=%v", exit, err) } exported, _ := json.Marshal(exit) if !strings.Contains(string(exported), `"username":"proxy-user"`) || !strings.Contains(string(exported), `"password":"plain-password"`) { t.Fatalf("network exit response must include stored credentials: %s", exported) } access, err := store.GetNetworkExitAccess(ctx, exit.ID) if err != nil || access.Username != "proxy-user" || access.Password != "plain-password" { t.Fatalf("runtime network exit credentials unavailable: %#v err=%v", access, err) } exit, reason, err := store.RecordNetworkExitCheck(ctx, exit.ID, ExitObservation{PublicIP: "203.0.113.11", Region: "test-region"}, "") if err != nil || exit.HealthStatus != "unhealthy" || reason != "exit_ip_drift" { t.Fatalf("identity drift must make the exit unhealthy: %#v reason=%s err=%v", exit, reason, err) } exit, reason, err = store.RecordNetworkExitCheck(ctx, exit.ID, ExitObservation{PublicIP: "203.0.113.10", Region: "test-region"}, "") if err != nil || exit.HealthStatus != "healthy" || reason != "exit_healthy" { t.Fatalf("matching identity must make the exit healthy: %#v reason=%s err=%v", exit, reason, err) } env := Env{Alias: "environment-a", Name: "环境 A", Gateway: "gw-main", BrowserVersion: "148", Fingerprint: Fingerprint{Seed: 1}} bound, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID) if err != nil || !created || bound.AccountID != "account-a" || bound.Exit.ID != exit.ID { t.Fatalf("create stable binding: %#v created=%v err=%v", bound, created, err) } reused, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID) if err != nil || created || reused.Alias != bound.Alias || reused.BindingID != bound.BindingID { t.Fatalf("same account must reuse its environment: %#v created=%v err=%v", reused, created, err) } if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } active, err := store.ActivateRuntime(ctx, env.Alias, "runtime-a", bound.BindingVersion, bound.Exit.ID, "network-a") if err != nil || active.RuntimeInstanceID == "" { t.Fatalf("activate runtime: %#v err=%v", active, err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE event_type = 'runtime_bound' AND reason_code = 'runtime_bound' AND account_id = 'account-a' AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3 AND details = '{}'::jsonb`, 1, active.Exit.ID, active.RuntimeInstanceID, active.BindingVersion) second, err := store.CreateNetworkExit(ctx, NetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}) if err != nil { t.Fatal(err) } second, _, err = store.RecordNetworkExitCheck(ctx, second.ID, ExitObservation{PublicIP: "198.51.100.2", Region: "other"}, "") if err != nil || second.HealthStatus != "healthy" { t.Fatalf("prepare second exit: %#v err=%v", second, err) } if _, err := store.db.ExecContext(ctx, ` INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-account-b', 'os_keyring', 'creatorhub/account-b'); INSERT INTO social_account (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) VALUES ('account-b', 'credential-account-b', 'mock', 'account-b', 'owned', 'authorized'); INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES ('environment-b', '环境 B', 'gw-main', '148', '{"seed":2}'); INSERT INTO environment_binding (id, account_id, browser_env_alias) VALUES ('binding-b', 'account-b', 'environment-b')`); err != nil { t.Fatal(err) } legacyRebound, err := store.RebindEnvironment(ctx, "environment-b", second.ID, "", 1) if err != nil || legacyRebound.Exit.ID != second.ID { t.Fatalf("legacy binding without an exit must support explicit rebind: %#v err=%v", legacyRebound, err) } if _, err := store.ActivateRuntime(ctx, env.Alias, "runtime-a", bound.BindingVersion+1, second.ID, "network-a"); !errors.Is(err, ErrConflict) { t.Fatalf("stale binding metadata must not activate a runtime: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '10 seconds' WHERE id = $1`, active.RuntimeInstanceID); err != nil { t.Fatal(err) } if _, err := store.ActivateRuntime(ctx, env.Alias, "runtime-a", bound.BindingVersion, bound.Exit.ID, "network-a"); err != nil { t.Fatalf("runtime heartbeat failed: %v", err) } var renewed bool if err := store.db.QueryRowContext(ctx, `SELECT lease_until > now() + interval '30 seconds' FROM runtime_instance WHERE id = $1`, active.RuntimeInstanceID).Scan(&renewed); err != nil || !renewed { t.Fatalf("runtime lease was not renewed: renewed=%v err=%v", renewed, err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil { t.Fatal(err) } expired, err := store.GetEnvironmentContext(ctx, env.Alias) if err != nil || expired.RuntimeInstanceID != active.RuntimeInstanceID || expired.RuntimeID != active.RuntimeID { t.Fatalf("context read discarded expired cleanup generation: %#v err=%v", expired, err) } if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } rebound, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", bound.BindingVersion) if err != nil || rebound.Exit.ID != second.ID || rebound.BindingVersion != 2 { t.Fatalf("expired runtime must be transactionally released before rebind: %#v err=%v", rebound, err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE event_type = 'runtime_released' AND reason_code = 'runtime_released' AND account_id = 'account-a' AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3 AND details = '{}'::jsonb`, 1, active.Exit.ID, active.RuntimeInstanceID, active.BindingVersion) if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } expiredBeforeActivation, err := store.ActivateRuntime(ctx, env.Alias, "expired-runtime", rebound.BindingVersion, rebound.Exit.ID, "network-expired") if err != nil { t.Fatalf("activate runtime to expire: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, expiredBeforeActivation.RuntimeInstanceID); err != nil { t.Fatal(err) } if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-runtime", rebound.BindingVersion, rebound.Exit.ID, "network-same-exit"); err != nil { t.Fatalf("replace expired runtime before same-exit rebind: %v", err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE event_type = 'runtime_released' AND reason_code = 'runtime_released' AND account_id = 'account-a' AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3 AND details = '{}'::jsonb`, 1, expiredBeforeActivation.Exit.ID, expiredBeforeActivation.RuntimeInstanceID, expiredBeforeActivation.BindingVersion) if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } if _, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", rebound.BindingVersion); !errors.Is(err, ErrConflict) { t.Fatalf("active runtime must block same-exit rebind: %v", err) } active, err = store.GetEnvironmentContext(ctx, env.Alias) if err != nil { t.Fatal(err) } if err := store.ReleaseRuntime(ctx, active); err != nil { t.Fatal(err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE event_type = 'runtime_released' AND reason_code = 'runtime_released' AND account_id = 'account-a' AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3 AND details = '{}'::jsonb`, 1, active.Exit.ID, active.RuntimeInstanceID, active.BindingVersion) rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-runtime", rebound.BindingVersion) if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-runtime" { t.Fatalf("same-exit rebind must atomically CAS the binding and runtime: %#v err=%v", rebound, err) } if err := store.ReleaseRuntime(ctx, active); !errors.Is(err, ErrConflict) { t.Fatalf("stale generation release must conflict: %v", err) } current, err := store.GetEnvironmentContext(ctx, env.Alias) if err != nil || current.RuntimeInstanceID != rebound.RuntimeInstanceID || current.RuntimeID != "rebound-runtime" { t.Fatalf("stale release changed the current runtime: %#v err=%v", current, err) } cleanup := current cleanup.RuntimeCleanupBindingVersion = current.BindingVersion cleanup.RuntimeCleanupInstanceID = current.RuntimeInstanceID cleanup.RuntimeCleanupRuntimeID = current.RuntimeID if err := store.SetRuntimeCleanupPending(ctx, cleanup, true); err != nil { t.Fatalf("set generation cleanup pending: %v", err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE event_type = 'runtime_released' AND browser_env_alias = 'environment-a' AND network_exit_id = $1 AND runtime_instance_id = $2 AND binding_version = $3`, 1, current.Exit.ID, current.RuntimeInstanceID, current.BindingVersion) wrongCleanup := cleanup wrongCleanup.RuntimeCleanupRuntimeID = "other-runtime" if err := store.SetRuntimeCleanupPending(ctx, wrongCleanup, false); !errors.Is(err, ErrConflict) { t.Fatalf("wrong cleanup generation cleared pending state: %v", err) } pending, err := store.GetEnvironmentContext(ctx, env.Alias) if err != nil || !pending.RuntimeCleanupPending || pending.RuntimeCleanupRuntimeID != current.RuntimeID { t.Fatalf("cleanup generation was not persisted: %#v err=%v", pending, err) } if _, err := store.ActivateRuntime(ctx, env.Alias, "candidate-runtime", current.BindingVersion, current.Exit.ID, "network-candidate"); !errors.Is(err, ErrConflict) { t.Fatalf("paused account activated a stale request: %v", err) } if err := store.SetRuntimeCleanupPending(ctx, pending, false); err != nil { t.Fatalf("clear matching cleanup generation: %v", err) } newGeneration, err := store.RebindEnvironment(ctx, env.Alias, current.Exit.ID, "", current.BindingVersion) if err != nil { t.Fatalf("advance binding generation: %v", err) } if err := store.SetRuntimeCleanupPending(ctx, cleanup, true); !errors.Is(err, ErrConflict) { t.Fatalf("stale binding set cleanup pending on version %d: %v", newGeneration.BindingVersion, err) } if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status = 'active' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } pauseTx, err := store.db.BeginTx(ctx, nil) if err != nil { t.Fatal(err) } var locked string if err := pauseTx.QueryRowContext(ctx, `SELECT id FROM social_account WHERE id = 'account-a' FOR UPDATE`).Scan(&locked); err != nil { t.Fatal(err) } activation := make(chan error, 1) go func() { _, err := store.ActivateRuntime(ctx, env.Alias, "racing-runtime", newGeneration.BindingVersion, newGeneration.Exit.ID, "network-racing") activation <- err }() select { case err := <-activation: t.Fatalf("activation bypassed the locked account row: %v", err) case <-time.After(time.Second): } if _, err := pauseTx.ExecContext(ctx, `UPDATE social_account SET status = 'paused' WHERE id = 'account-a'`); err != nil { t.Fatal(err) } if err := pauseTx.Commit(); err != nil { t.Fatal(err) } if err := <-activation; !errors.Is(err, ErrConflict) { t.Fatalf("activation won the pause race: %v", err) } latest, err := store.GetEnvironmentContext(ctx, env.Alias) if err != nil || latest.RuntimeID != "" { t.Fatalf("pause race left an active runtime: %#v err=%v", latest, err) } action := EnvironmentAction{ OperationID: NewOperationID(), Action: "start", AccountID: rebound.AccountID, BrowserEnvAlias: rebound.Alias, NetworkExitID: rebound.Exit.ID, BindingVersion: rebound.BindingVersion, ReasonCode: "action_requested", } if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { t.Fatal(err) } action.Outcome, action.ReasonCode = "succeeded", "environment_started" if err := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); err != nil { t.Fatal(err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+action.OperationID+`'`, 2) invalidAction := action invalidAction.OperationID = NewOperationID() invalidAction.NewBrowserVersion = "http://operator:secret@proxy.example" if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", invalidAction); !errors.Is(err, ErrInvalid) { t.Fatalf("invalid image version must not reach audit persistence: %v", err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+invalidAction.OperationID+`'`, 0) var accountVersion int64 if err := store.db.QueryRowContext(ctx, `SELECT version FROM social_account WHERE id = 'account-a'`).Scan(&accountVersion); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO content_draft (id, account_id, version, content) VALUES ('exit-hold-draft', 'account-a', 1, 'test')`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO confirmation (id, account_id, account_version, draft_id, draft_version, version) VALUES ('exit-hold-confirmation', 'account-a', $1, 'exit-hold-draft', 1, 1)`, accountVersion); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO operation_task (id, idempotency_key, account_id, account_version, draft_id, draft_version, confirmation_id, confirmation_version, state, lease_owner, lease_until) VALUES ('exit-hold-queued', 'exit-hold-queued-key', 'account-a', $1, 'exit-hold-draft', 1, 'exit-hold-confirmation', 1, 'queued', NULL, NULL), ('exit-hold-executing', 'exit-hold-executing-key', 'account-a', $1, 'exit-hold-draft', 1, 'exit-hold-confirmation', 1, 'executing', 'worker-old', now() + interval '1 minute')`, accountVersion); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO execution_attempt (id, task_id) VALUES ('exit-hold-attempt', 'exit-hold-executing')`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `UPDATE operation_task SET current_attempt_id = 'exit-hold-attempt' WHERE id = 'exit-hold-executing'`); err != nil { t.Fatal(err) } if _, _, err := store.RecordNetworkExitCheck(ctx, newGeneration.Exit.ID, ExitObservation{}, "proxy_check_failed"); err != nil { t.Fatal(err) } assertDatabaseCount(t, store.db, `SELECT count(*) FROM operation_task WHERE id = 'exit-hold-queued' AND state = 'policy_hold' AND hold_reason = 'exit_unhealthy'`, 1) assertDatabaseCount(t, store.db, `SELECT count(*) FROM operation_task WHERE id = 'exit-hold-executing' AND state = 'needs_confirmation' AND hold_reason = 'exit_unhealthy' AND lease_owner IS NULL`, 1) assertDatabaseCount(t, store.db, `SELECT count(*) FROM execution_attempt WHERE id = 'exit-hold-attempt' AND outcome = 'uncertain' AND finished_at IS NOT NULL`, 1) for _, want := range []taskstate.Transition{ {State: "policy_hold", ReasonCode: "exit_unhealthy", AccountID: "account-a", TaskID: "exit-hold-queued"}, {State: "needs_confirmation", ReasonCode: "exit_unhealthy", AccountID: "account-a", TaskID: "exit-hold-executing"}, } { if !slices.Contains(notifications, want) { t.Fatalf("missing network exit transition %+v in %+v", want, notifications) } } if err := store.db.QueryRowContext(ctx, `SELECT version FROM social_account WHERE id = 'account-a'`).Scan(&accountVersion); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO content_draft (id, account_id, version, content) VALUES ('exit-disable-draft', 'account-a', 1, 'test')`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO confirmation (id, account_id, account_version, draft_id, draft_version, version) VALUES ('exit-disable-confirmation', 'account-a', $1, 'exit-disable-draft', 1, 1)`, accountVersion); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `INSERT INTO operation_task (id, idempotency_key, account_id, account_version, draft_id, draft_version, confirmation_id, confirmation_version, state, lease_owner, lease_until) VALUES ('exit-disable-queued', 'exit-disable-queued-key', 'account-a', $1, 'exit-disable-draft', 1, 'exit-disable-confirmation', 1, 'queued', NULL, NULL), ('exit-disable-executing', 'exit-disable-executing-key', 'account-a', $1, 'exit-disable-draft', 1, 'exit-disable-confirmation', 1, 'executing', 'worker-disabled', now() + interval '1 minute')`, accountVersion); err != nil { t.Fatal(err) } if _, err := store.DisableNetworkExit(ctx, newGeneration.Exit.ID); !errors.Is(err, ErrConflict) { t.Fatalf("running exit must not be disabled: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE operation_task SET state='cancelled', lease_owner=NULL, lease_until=NULL WHERE id IN ('exit-disable-queued','exit-disable-executing')`); err != nil { t.Fatal(err) } if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET released_at=now() WHERE binding_id IN (SELECT id FROM environment_binding WHERE network_exit_id=$1)`, newGeneration.Exit.ID); err != nil { t.Fatal(err) } if _, err := store.DisableNetworkExit(ctx, newGeneration.Exit.ID); err != nil { t.Fatalf("stopped exit should be disableable: %v", err) } var auditText string if err := store.db.QueryRowContext(ctx, `SELECT string_agg(row_to_json(event)::text, '') FROM audit_event event`).Scan(&auditText); err != nil { t.Fatal(err) } for _, forbidden := range []string{"creatorhub/proxy-main", "credential-exit", "username", "password"} { if strings.Contains(auditText, forbidden) { t.Fatalf("audit leaked sensitive value %q: %s", forbidden, auditText) } } }