Files
creator-hub/internal/account/store_test.go
T

998 lines
48 KiB
Go

package account
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"git.ipao.vip/rogee/creator-hub/internal/taskstate"
"github.com/jackc/pgx/v5"
)
type testCredentialBridge struct {
values map[string]string
storeErr error
}
func (bridge *testCredentialBridge) Store(ctx context.Context, _ CredentialReference, key, value string) error {
if err := ctx.Err(); err != nil {
return err
}
bridge.values[key] = value
return bridge.storeErr
}
func TestCreateAccountCompensatesPartialCredentialStore(t *testing.T) {
credentials := &testCredentialBridge{values: map[string]string{}, storeErr: errors.New("provider failed after write")}
account := Account{
ID: "account-partial-store", Name: "Partial Store", Platform: "douyin", PlatformAccountKey: "partial-store",
Tags: []string{}, Cookies: "sessionid=value",
CredentialReference: CredentialReference{ID: "account-partial-store-cookies", Provider: "os_keyring"},
CredentialKey: "creatorhub/account-partial-store/cookies",
}
if err := (&Store{}).CreateAccount(context.Background(), account, credentials); err == nil {
t.Fatal("partial credential store returned success")
}
if _, ok := credentials.values[account.CredentialKey]; ok {
t.Fatal("partial credential store left an orphan")
}
}
func (bridge *testCredentialBridge) Delete(ctx context.Context, _ CredentialReference, key string) error {
if err := ctx.Err(); err != nil {
return err
}
delete(bridge.values, key)
return nil
}
func TestNewAccountIDIsStoreValidAndUnique(t *testing.T) {
first, second := NewAccountID(), NewAccountID()
if first == second || !idPattern.MatchString(first) || !idPattern.MatchString(second) {
t.Fatalf("invalid generated account ids: %q %q", first, second)
}
}
func TestValidationRejectsInvalidInputsBeforePersistence(t *testing.T) {
store := &Store{}
valid := Account{
ID: "account-a", Name: "账号 A", Platform: "douyin", PlatformAccountKey: "platform-a",
Tags: []string{"主账号"}, Cookies: "sessionid=value; token=second",
CredentialReference: CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"},
CredentialKey: "creatorhub/account-a/cookies",
}
credentials := &testCredentialBridge{values: map[string]string{}}
for _, platform := range []string{"douyin", "xiaohongshu", "wechat-official", "kuaishou"} {
account := valid
account.Platform = platform
if !validAccount(account) {
t.Fatalf("supported platform rejected: %s", platform)
}
}
empty := valid
empty.Cookies = ""
if !validAccount(empty) {
t.Fatal("empty cookies must stay valid (scan-to-login account)")
}
for name, mutate := range map[string]func(*Account){
"id": func(account *Account) { account.ID = "INVALID" },
"name": func(account *Account) { account.Name = " " },
"platform": func(account *Account) { account.Platform = "mock" },
"platform account key": func(account *Account) { account.PlatformAccountKey = "secret value" },
"tag": func(account *Account) { account.Tags = []string{""} },
"cookie header": func(account *Account) { account.Cookies = "sessionid" },
} {
t.Run(name, func(t *testing.T) {
account := valid
mutate(&account)
if err := store.CreateAccount(context.Background(), account, credentials); !errors.Is(err, ErrInvalid) {
t.Fatalf("expected invalid account, got %v", err)
}
})
}
if _, _, err := store.Enqueue(context.Background(), Task{ID: "task-a"}); !errors.Is(err, ErrInvalid) {
t.Fatalf("expected invalid task, got %v", err)
}
if _, _, err := store.EnqueueConfirmation(context.Background(), ""); !errors.Is(err, ErrInvalid) {
t.Fatalf("expected missing confirmation to be rejected, got %v", err)
}
if _, err := store.ExecuteMock(context.Background(), "worker-a", "retry"); !errors.Is(err, ErrInvalid) {
t.Fatalf("expected unsupported outcome to be rejected, got %v", err)
}
}
func TestTaskRecoveryActionsAndEvidenceAreFailClosed(t *testing.T) {
for _, test := range []struct {
task Task
want string
}{
{task: Task{State: "needs_confirmation", HoldReason: "execution_lease_expired"}, want: "verify"},
{task: Task{State: "policy_hold", HoldReason: "account_version_changed"}, want: "reconfirm"},
{task: Task{State: "needs_confirmation", HoldReason: "confirmation_missing"}, want: ""},
{task: Task{State: "policy_hold", HoldReason: "future_reason"}, want: ""},
{task: Task{State: "needs_confirmation", HoldReason: "task_result_uncertain", VerificationResult: "not_executed"}, want: "resume"},
{task: Task{State: "needs_confirmation", HoldReason: "task_result_uncertain", VerificationResult: "succeeded"}, want: "finish"},
} {
if got := taskAllowedAction(test.task); got != test.want {
t.Fatalf("allowed action: got=%q want=%q task=%+v", got, test.want, test.task)
}
}
redacted := safeDetails(json.RawMessage(`{"state":{"state":"safe","api_key":"a","private_key":"b","credential_key":"c","authorization_header":"d","proxy_url":"e"}}`))
if string(redacted) != `{"state":{"state":"safe"}}` {
t.Fatalf("unexpected redaction: %s", redacted)
}
if evidence := safeEvidence(json.RawMessage(`{"mock_outcome":"uncertain","cookie":"secret"}`)); len(evidence) != 1 || evidence["mock_outcome"] != "uncertain" {
t.Fatalf("unexpected evidence allowlist: %#v", evidence)
}
}
func TestPhaseAOfflineWorkflow(t *testing.T) {
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
if databaseURL == "" {
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
}
ctx := context.Background()
store, err := Open(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
var notificationMu sync.Mutex
var notifications []taskstate.Transition
store.SetTaskNotifier(func(transition taskstate.Transition) {
notificationMu.Lock()
defer notificationMu.Unlock()
notifications = append(notifications, transition)
})
assertNotified := func(state, reason, accountID, taskID string) {
t.Helper()
notificationMu.Lock()
defer notificationMu.Unlock()
want := taskstate.Transition{State: state, ReasonCode: reason, AccountID: accountID, TaskID: taskID}
for _, notification := range notifications {
if notification == want {
return
}
}
t.Fatalf("missing task notification %+v in %+v", want, notifications)
}
applyHubMigrationsForPhaseATest(t, store)
if _, err := store.db.ExecContext(ctx, `
TRUNCATE audit_event, execution_attempt, operation_task, confirmation, content_draft,
runtime_instance, environment_binding, network_exit, social_account, credential_reference,
browser_env, browser_version, gateway RESTART IDENTITY CASCADE`); err != nil {
t.Fatal(err)
}
accounts := []Account{
{ID: "account-a", Name: "账号 A", Platform: "douyin", PlatformAccountKey: "platform-a", Tags: []string{"主账号"}, Cookies: "sessionid=a",
CredentialReference: CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-a/cookies"},
{ID: "account-b", Name: "账号 B", Platform: "douyin", PlatformAccountKey: "platform-b", Tags: []string{}, Cookies: "sessionid=b",
CredentialReference: CredentialReference{ID: "account-b-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-b/cookies"},
}
credentials := &testCredentialBridge{values: map[string]string{}}
for _, account := range accounts {
if err := store.CreateAccount(ctx, account, credentials); err != nil {
t.Fatal(err)
}
}
createdAccount, err := store.GetAccount(ctx, "account-a")
if err != nil || createdAccount.RuntimeStatus != "paused" || createdAccount.Version != 1 ||
len(createdAccount.Tags) != 1 || createdAccount.Tags[0] != "主账号" {
t.Fatalf("new account must start paused: %#v %v", createdAccount, err)
}
if credentials.values["creatorhub/account-a/cookies"] != "sessionid=a" ||
credentials.values["creatorhub/account-b/cookies"] != "sessionid=b" {
t.Fatalf("cookies were not stored in the credential bridge: %#v", credentials.values)
}
assertCount(t, store, `SELECT count(*) FROM credential_reference WHERE id IN ('account-a-cookies', 'account-b-cookies')`, 2)
assertCount(t, store, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema()
AND table_name = 'social_account' AND column_name = 'cookies'`, 0)
if err := store.ResumeAccount(ctx, "account-a"); !errors.Is(err, ErrConflict) {
t.Fatalf("account without a healthy binding must not resume: %v", err)
}
if err := store.CreateAccount(ctx, Account{
ID: "account-c", Name: "账号 C", Platform: "douyin", PlatformAccountKey: "platform-a", Tags: []string{}, Cookies: "sessionid=c",
CredentialReference: CredentialReference{ID: "account-c-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-c/cookies",
}, credentials); !errors.Is(err, ErrConflict) {
t.Fatalf("expected platform accounts to be unique, got %v", err)
}
if _, ok := credentials.values["creatorhub/account-c/cookies"]; ok {
t.Fatal("credential bridge retained a secret after database rejection")
}
if _, err := store.db.ExecContext(ctx, `
INSERT INTO gateway (name, endpoint, token) VALUES ('test-gateway', 'http://127.0.0.1:8081', 'phase-a-test-gateway-token')`); err != nil {
t.Fatal(err)
}
if _, err := store.db.ExecContext(ctx, `
INSERT INTO browser_version (version, browser_path) VALUES ('1', '/opt/creatorhub/browsers/1')`); err != nil {
t.Fatal(err)
}
if _, err := store.db.ExecContext(ctx, `
INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint) VALUES
('account-a', 'Account A', 'test-gateway', '1', '{"seed":1}'),
('account-b', 'Account B', 'test-gateway', '1', '{"seed":2}');
INSERT INTO network_exit (id, protocol, host, port, health_status)
VALUES ('exit-shared', 'socks5', '127.0.0.1', 1080, 'healthy');
INSERT INTO environment_binding (id, account_id, browser_env_alias, network_exit_id) VALUES
('binding-a', 'account-a', 'account-a', 'exit-shared'),
('binding-b', 'account-b', 'account-b', 'exit-shared')`); err != nil {
t.Fatal(err)
}
for _, id := range []string{"account-a", "account-b"} {
if err := store.ResumeAccount(ctx, id); err != nil {
t.Fatalf("resume %s: %v", id, err)
}
}
accountA, err := store.GetAccount(ctx, "account-a")
if err != nil || accountA.RuntimeStatus != "active" || accountA.Version != 2 {
t.Fatalf("unexpected resumed account: %#v %v", accountA, err)
}
exportedAccount, _ := json.Marshal(accountA)
if strings.Contains(string(exportedAccount), "sessionid=a") || strings.Contains(string(exportedAccount), "cookies") {
t.Fatalf("account response leaked cookies: %s", exportedAccount)
}
listedAccounts, err := store.ListAccounts(ctx)
if err != nil || len(listedAccounts) != 2 || len(listedAccounts[0].Tags) != 1 || len(listedAccounts[1].Tags) != 0 {
t.Fatalf("unexpected account list: %#v %v", listedAccounts, err)
}
accountB, err := store.GetAccount(ctx, "account-b")
if err != nil || len(accountB.Tags) != 0 {
t.Fatalf("empty tags did not round-trip: %#v err=%v", accountB, err)
}
if _, err := store.db.ExecContext(ctx, `
INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until) VALUES
('runtime-instance-a', 'account-a', 'binding-a', 1, 'runtime-a', now() + interval '1 minute'),
('runtime-instance-b', 'account-b', 'binding-b', 1, 'runtime-b', now() + interval '1 minute')`); err != nil {
t.Fatal(err)
}
draftV1, err := store.CreateDraftVersion(ctx, "account-a", "first system-generated draft")
if err != nil || draftV1.ID == "" || draftV1.Version != 1 {
t.Fatalf("create generated draft: %#v %v", draftV1, err)
}
confirmation, inserted, err := store.ConfirmDraft(ctx, draftV1.ID, accountA.Version, draftV1.Version)
if err != nil || !inserted || confirmation.ID == "" {
t.Fatalf("confirm generated draft: %#v inserted=%v err=%v", confirmation, inserted, err)
}
if repeated, inserted, err := store.ConfirmDraft(ctx, draftV1.ID, accountA.Version, draftV1.Version); err != nil || inserted || repeated.ID != confirmation.ID {
t.Fatalf("confirmation was not idempotent: %#v inserted=%v err=%v", repeated, inserted, err)
}
queued, inserted, err := store.EnqueueConfirmation(ctx, confirmation.ID)
if err != nil || !inserted || queued.ID == "" {
t.Fatalf("enqueue confirmed draft: %#v inserted=%v err=%v", queued, inserted, err)
}
if repeated, inserted, err := store.EnqueueConfirmation(ctx, confirmation.ID); err != nil || inserted || repeated.ID != queued.ID {
t.Fatalf("enqueue was not idempotent: %#v inserted=%v err=%v", repeated, inserted, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE confirmation_id = $1`, 1, confirmation.ID)
detail, err := store.GetDraftDetail(ctx, draftV1.ID)
if err != nil || len(detail.Confirmations) != 1 || detail.Confirmations[0].BrowserEnvAlias != "account-a" ||
detail.Confirmations[0].NetworkExitID != "exit-shared" || len(detail.Tasks) != 1 {
t.Fatalf("confirmation snapshot is not traceable: %#v err=%v", detail, err)
}
if err := store.CancelTask(ctx, queued.ID); err != nil {
t.Fatal(err)
}
draftV2, err := store.CreateDraftVersion(ctx, "account-a", "newer draft snapshot")
if err != nil || draftV2.Version != 2 {
t.Fatalf("create second draft version: %#v %v", draftV2, err)
}
if _, _, err := store.ConfirmDraft(ctx, draftV1.ID, accountA.Version, draftV1.Version); readinessReason(err) != "draft_version_changed" {
t.Fatalf("stale draft was confirmed: %v", err)
}
confirmationV2, _, err := store.ConfirmDraft(ctx, draftV2.ID, accountA.Version, draftV2.Version)
if err != nil {
t.Fatal(err)
}
if _, err := store.db.ExecContext(ctx, `UPDATE network_exit SET health_status = 'unhealthy' WHERE id = 'exit-shared'`); err != nil {
t.Fatal(err)
}
if _, _, err := store.EnqueueConfirmation(ctx, confirmationV2.ID); readinessReason(err) != "network_exit_unhealthy" {
t.Fatalf("unhealthy exit accepted for enqueue: %v", err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE confirmation_id = $1`, 0, confirmationV2.ID)
assertCount(t, store, `SELECT count(*) FROM execution_attempt attempt JOIN operation_task task ON task.id = attempt.task_id WHERE task.confirmation_id = $1`, 0, confirmationV2.ID)
if _, err := store.db.ExecContext(ctx, `UPDATE network_exit SET health_status = 'healthy' WHERE id = 'exit-shared'`); err != nil {
t.Fatal(err)
}
if _, err := store.CreateDraftVersion(ctx, "account-a", "third draft makes confirmation stale"); err != nil {
t.Fatal(err)
}
if _, _, err := store.EnqueueConfirmation(ctx, confirmationV2.ID); readinessReason(err) != "draft_version_changed" {
t.Fatalf("stale confirmation accepted for enqueue: %v", err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE confirmation_id = $1`, 0, confirmationV2.ID)
assertCount(t, store, `SELECT count(*) FROM execution_attempt attempt JOIN operation_task task ON task.id = attempt.task_id WHERE task.confirmation_id = $1`, 0, confirmationV2.ID)
createApprovedDraft(t, store, "account-a", accountA.Version, "draft-a", "confirmation-a")
createApprovedDraft(t, store, "account-b", accountB.Version, "draft-b", "confirmation-b")
for index := range 20 {
accountID, accountVersion, draftID, confirmationID := "account-a", accountA.Version, "draft-a", "confirmation-a"
if index%2 == 1 {
accountID, accountVersion, draftID, confirmationID = "account-b", accountB.Version, "draft-b", "confirmation-b"
}
task := approvedTask(index, accountID, accountVersion, draftID, confirmationID)
if _, inserted, err := store.Enqueue(ctx, task); err != nil || !inserted {
t.Fatalf("enqueue task %d: inserted=%v err=%v", index, inserted, err)
}
task.ID = fmt.Sprintf("duplicate-%02d", index)
if existing, inserted, err := store.Enqueue(ctx, task); err != nil || inserted || existing.ID == task.ID {
t.Fatalf("redeliver task %d: existing=%q inserted=%v err=%v", index, existing.ID, inserted, err)
}
}
var executed atomic.Int64
var workers sync.WaitGroup
errorsFromWorkers := make(chan error, 4)
for worker := range 4 {
workers.Add(1)
go func() {
defer workers.Done()
for {
execution, err := store.ExecuteMock(ctx, fmt.Sprintf("worker-%d", worker), "succeeded")
if err != nil {
errorsFromWorkers <- err
return
}
if !execution.WasClaimed {
var queued int
if err := store.db.QueryRowContext(ctx, `SELECT count(*) FROM operation_task WHERE state = 'queued'`).Scan(&queued); err != nil {
errorsFromWorkers <- err
return
}
if queued == 0 {
return
}
time.Sleep(5 * time.Millisecond)
continue
}
executed.Add(1)
}
}()
}
workers.Wait()
close(errorsFromWorkers)
for err := range errorsFromWorkers {
t.Fatal(err)
}
if executed.Load() != 20 {
t.Fatalf("expected exactly 20 executions, got %d", executed.Load())
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE state = 'succeeded'`, 20)
assertCount(t, store, `SELECT count(*) FROM execution_attempt`, 20)
for _, test := range []struct {
name, accountStatus, authorizationStatus, exitStatus, runtimeLease, wantState, wantReason string
binding, versionChanged, confirmationChanged bool
}{
{name: "binding-missing", accountStatus: "active", authorizationStatus: "authorized", wantState: "policy_hold", wantReason: "binding_missing"},
{name: "exit-unhealthy", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "unhealthy", wantState: "policy_hold", wantReason: "exit_unhealthy"},
{name: "runtime-missing", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", wantState: "policy_hold", wantReason: "runtime_missing"},
{name: "runtime-expired", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "expired", wantState: "policy_hold", wantReason: "runtime_lease_expired"},
{name: "binding-version", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", versionChanged: true, wantState: "needs_confirmation", wantReason: "binding_version_changed"},
{name: "confirmation-version", accountStatus: "active", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", confirmationChanged: true, wantState: "needs_confirmation", wantReason: "confirmation_version_changed"},
{name: "paused", accountStatus: "paused", authorizationStatus: "authorized", binding: true, exitStatus: "healthy", runtimeLease: "active", wantState: "policy_hold", wantReason: "account_paused"},
{name: "revoked", accountStatus: "paused", authorizationStatus: "revoked", binding: true, exitStatus: "healthy", runtimeLease: "active", wantState: "policy_hold", wantReason: "account_revoked"},
} {
t.Run("gate "+test.name, func(t *testing.T) {
taskID := seedGateTask(t, store, test.name, test.accountStatus, test.authorizationStatus, test.exitStatus,
test.runtimeLease, test.binding, test.versionChanged, test.confirmationChanged)
if execution, err := store.ExecuteMock(ctx, "worker-gate-"+test.name, "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("invalid task was claimed: execution=%+v err=%v", execution, err)
}
assertTaskGate(t, store, taskID, test.wantState, test.wantReason)
assertNotified(test.wantState, test.wantReason, "gate-"+test.name, taskID)
})
}
if err := store.VerifyTask(ctx, "task-gate-exit-unhealthy", "not_executed"); err != nil {
t.Fatal(err)
}
blockedDetail, err := store.GetTaskDetail(ctx, "task-gate-exit-unhealthy")
if err != nil || blockedDetail.AllowedAction != "" || blockedDetail.ReadinessReason != "exit_unhealthy" {
t.Fatalf("unhealthy exit exposed resume after verification: detail=%+v err=%v", blockedDetail, err)
}
// A held task is never revived in place: only a newly confirmed task with a new idempotency key may run.
if _, err := store.db.ExecContext(ctx, `
INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until)
VALUES ('runtime-recovered', 'gate-runtime-missing', 'binding-runtime-missing', 1, 'runtime-recovered', now() + interval '1 minute')`); err != nil {
t.Fatal(err)
}
if execution, err := store.ExecuteMock(ctx, "worker-old-held", "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("held task was revived after health recovery: execution=%+v err=%v", execution, err)
}
createApprovedDraft(t, store, "gate-runtime-missing", 1, "draft-recovered", "confirmation-recovered")
recovered := Task{
ID: "task-recovered", IdempotencyKey: "idempotency-recovered", AccountID: "gate-runtime-missing", AccountVersion: 1,
DraftID: "draft-recovered", DraftVersion: 1, ConfirmationID: "confirmation-recovered", ConfirmationVersion: 1,
}
if _, inserted, err := store.Enqueue(ctx, recovered); err != nil || !inserted {
t.Fatalf("enqueue recovered task: inserted=%v err=%v", inserted, err)
}
if execution, err := store.ExecuteMock(ctx, "worker-recovered", "succeeded"); err != nil || !execution.WasClaimed || execution.State != "succeeded" {
t.Fatalf("newly confirmed task did not run after recovery: execution=%+v err=%v", execution, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-gate-runtime-missing' AND state = 'policy_hold'`, 1)
unconfirmed := approvedTask(30, "account-b", accountB.Version, "draft-b", "")
unconfirmed.ConfirmationVersion = 0
if _, _, err := store.Enqueue(ctx, unconfirmed); err != nil {
t.Fatal(err)
}
mismatch := approvedTask(31, "account-b", accountB.Version, "draft-b", "confirmation-b")
mismatch.AccountVersion++
if _, _, err := store.Enqueue(ctx, mismatch); err != nil {
t.Fatal(err)
}
if execution, err := store.ExecuteMock(ctx, "worker-validation", "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("invalid tasks must not execute: execution=%+v err=%v", execution, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id IN ('task-30', 'task-31') AND state = 'needs_confirmation'`, 2)
assertNotified("needs_confirmation", "confirmation_missing", "account-b", "task-30")
assertNotified("needs_confirmation", "account_version_changed", "account-b", "task-31")
assertCount(t, store, `SELECT count(*) FROM execution_attempt WHERE task_id IN ('task-30', 'task-31')`, 0)
nullConfirmation, err := store.GetTaskDetail(ctx, "task-30")
if err != nil || nullConfirmation.Confirmation.ID != "" || nullConfirmation.AllowedAction != "" {
t.Fatalf("NULL confirmation detail was not readable and fail-closed: detail=%+v err=%v", nullConfirmation, err)
}
uncertain := approvedTask(32, "account-b", accountB.Version, "draft-b", "confirmation-b")
if _, _, err := store.Enqueue(ctx, uncertain); err != nil {
t.Fatal(err)
}
if execution, err := store.ExecuteMock(ctx, "worker-uncertain", "uncertain"); err != nil || execution.State != "needs_confirmation" {
t.Fatalf("uncertain outcome was not held: execution=%+v err=%v", execution, err)
}
assertNotified("needs_confirmation", "task_result_uncertain", "account-b", "task-32")
if execution, err := store.ExecuteMock(ctx, "worker-uncertain", "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("uncertain task was retried: execution=%+v err=%v", execution, err)
}
assertCount(t, store, `SELECT count(*) FROM execution_attempt WHERE task_id = 'task-32'`, 1)
policy := approvedTask(33, "account-b", accountB.Version, "draft-b", "confirmation-b")
if _, _, err := store.Enqueue(ctx, policy); err != nil {
t.Fatal(err)
}
if execution, err := store.ExecuteMock(ctx, "worker-policy", "policy_hold"); err != nil || execution.State != "policy_hold" {
t.Fatalf("policy outcome was not held: execution=%+v err=%v", execution, err)
}
assertNotified("policy_hold", "task_policy_hold", "account-b", "task-33")
if execution, err := store.ExecuteMock(ctx, "worker-policy", "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("policy task was retried: execution=%+v err=%v", execution, err)
}
paused := approvedTask(34, "account-b", accountB.Version, "draft-b", "confirmation-b")
if _, _, err := store.Enqueue(ctx, paused); err != nil {
t.Fatal(err)
}
pausedQueued := approvedTask(340, "account-b", accountB.Version, "draft-b", "confirmation-b")
pausedQueued.ID, pausedQueued.IdempotencyKey = "task-34-queued", "idempotency-34-queued"
if _, _, err := store.Enqueue(ctx, pausedQueued); err != nil {
t.Fatal(err)
}
if claimed, err := store.claim(ctx, "worker-account-pause"); err != nil || claimed.TaskID != paused.ID {
t.Fatalf("claim executing pause fixture: execution=%+v err=%v", claimed, err)
}
pauseStarted := time.Now()
if err := store.PauseAccount(ctx, "account-b"); err != nil {
t.Fatal(err)
}
if err := store.PauseAccount(ctx, "account-b"); err != nil {
t.Fatalf("pause should be idempotent: %v", err)
}
if execution, err := store.ExecuteMock(ctx, "worker-paused", "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("paused account task was claimed: execution=%+v err=%v", execution, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-34' AND state = 'needs_confirmation'`, 1)
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-34-queued' AND state = 'policy_hold'`, 1)
assertNotified("needs_confirmation", "account_paused", "account-b", "task-34")
assertNotified("policy_hold", "account_paused", "account-b", "task-34-queued")
pausedAccount, err := store.GetAccount(ctx, "account-b")
if err != nil || pausedAccount.Version != accountB.Version+1 {
t.Fatalf("idempotent pause must increment once: %#v %v", pausedAccount, err)
}
var pausedAt time.Time
if err := store.db.QueryRowContext(ctx, `SELECT paused_at FROM social_account WHERE id = 'account-b'`).Scan(&pausedAt); err != nil || pausedAt.Sub(pauseStarted) >= time.Minute {
t.Fatalf("account was not paused within one minute: paused_at=%v err=%v", pausedAt, err)
}
revokeQueued := approvedTask(341, "account-b", pausedAccount.Version, "draft-b", "confirmation-b")
revokeQueued.ID, revokeQueued.IdempotencyKey = "task-revoke-queued", "idempotency-revoke-queued"
revokeExecuting := approvedTask(342, "account-b", pausedAccount.Version, "draft-b", "confirmation-b")
revokeExecuting.ID, revokeExecuting.IdempotencyKey = "task-revoke-executing", "idempotency-revoke-executing"
for _, task := range []Task{revokeQueued, revokeExecuting} {
if _, _, err := store.Enqueue(ctx, task); err != nil {
t.Fatal(err)
}
}
if _, err := store.db.ExecContext(ctx, `UPDATE operation_task SET state = 'executing', lease_owner = 'worker-revoke', lease_until = now() + interval '1 minute' WHERE id = $1`, revokeExecuting.ID); err != nil {
t.Fatal(err)
}
if err := store.RevokeAccount(ctx, "account-b"); err != nil {
t.Fatal(err)
}
if err := store.RevokeAccount(ctx, "account-b"); err != nil {
t.Fatalf("revoke should be idempotent: %v", err)
}
revokedAccount, err := store.GetAccount(ctx, "account-b")
if err != nil || revokedAccount.AuthorizationStatus != "revoked" || revokedAccount.Version != accountB.Version+2 {
t.Fatalf("unexpected revoked account: %#v %v", revokedAccount, err)
}
assertNotified("policy_hold", "account_revoked", "account-b", revokeQueued.ID)
assertNotified("needs_confirmation", "account_revoked", "account-b", revokeExecuting.ID)
if err := store.ResumeAccount(ctx, "account-b"); !errors.Is(err, ErrConflict) {
t.Fatalf("revoked account must not resume: %v", err)
}
cancelled := approvedTask(35, "account-a", accountA.Version, "draft-a", "confirmation-a")
if _, _, err := store.Enqueue(ctx, cancelled); err != nil {
t.Fatal(err)
}
if err := store.CancelTask(ctx, cancelled.ID); err != nil {
t.Fatal(err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-35' AND state = 'cancelled'`, 1)
expired := approvedTask(36, "account-a", accountA.Version, "draft-a", "confirmation-a")
if _, _, err := store.Enqueue(ctx, expired); err != nil {
t.Fatal(err)
}
claimed, err := store.claim(ctx, "worker-crashed")
if err != nil || !claimed.WasClaimed {
t.Fatalf("claim task for lease test: execution=%+v err=%v", claimed, err)
}
if _, err := store.db.ExecContext(ctx, `UPDATE operation_task SET lease_until = now() - interval '1 second' WHERE id = $1`, claimed.TaskID); err != nil {
t.Fatal(err)
}
if execution, err := store.ExecuteMock(ctx, "worker-after-crash", "succeeded"); err != nil || execution.WasClaimed {
t.Fatalf("expired unknown result was retried: execution=%+v err=%v", execution, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-36' AND state = 'needs_confirmation'`, 1)
assertNotified("needs_confirmation", "execution_lease_expired", "account-a", "task-36")
assertCount(t, store, `SELECT count(*) FROM execution_attempt WHERE task_id = 'task-36' AND outcome = 'uncertain'`, 1)
if err := store.ResumeTask(ctx, "task-36"); !errors.Is(err, ErrConflict) {
t.Fatalf("unknown result resumed without verification: %v", err)
}
if err := store.VerifyTask(ctx, "task-36", "not_executed"); err != nil {
t.Fatal(err)
}
if err := store.VerifyTask(ctx, "task-36", "succeeded"); !errors.Is(err, ErrConflict) {
t.Fatalf("repeated verification changed the recorded conclusion: %v", err)
}
verifiedDetail, err := store.GetTaskDetail(ctx, "task-36")
if err != nil || verifiedDetail.AllowedAction != "resume" || verifiedDetail.ReadinessReason != "" {
t.Fatalf("ready verified task did not expose one resume action: detail=%+v err=%v", verifiedDetail, err)
}
if err := store.ResumeTask(ctx, "task-36"); err != nil {
t.Fatal(err)
}
if execution, err := store.ExecuteMock(ctx, "worker-after-verification", "succeeded"); err != nil || !execution.WasClaimed || execution.TaskID != "task-36" {
t.Fatalf("verified task did not resume: execution=%+v err=%v", execution, err)
}
assertCount(t, store, `SELECT count(*) FROM execution_attempt WHERE task_id = 'task-36'`, 2)
attempts, err := store.GetTaskDetail(ctx, "task-36")
if err != nil || len(attempts.Attempts) != 2 {
t.Fatalf("task attempts are not traceable: detail=%+v err=%v", attempts, err)
}
attemptDetail, err := store.GetTaskAttemptDetail(ctx, attempts.Attempts[1].ID)
if err != nil || attemptDetail.TaskID != "task-36" || attemptDetail.BrowserEnvAlias == "" || attemptDetail.NetworkExitID == "" {
t.Fatalf("attempt deep link is incomplete: detail=%+v err=%v", attemptDetail, err)
}
stale := approvedTask(37, "account-a", accountA.Version, "draft-a", "confirmation-a")
if _, _, err := store.Enqueue(ctx, stale); err != nil {
t.Fatal(err)
}
staleExecution, err := store.claim(ctx, "worker-stale")
if err != nil || staleExecution.TaskID != stale.ID {
t.Fatalf("claim stale completion fixture: execution=%+v err=%v", staleExecution, err)
}
if _, err := store.db.ExecContext(ctx, `UPDATE network_exit SET health_status = 'unhealthy' WHERE id = 'exit-shared'`); err != nil {
t.Fatal(err)
}
completed, err := store.complete(ctx, "worker-stale", staleExecution, "succeeded")
if err != nil || completed.State != "needs_confirmation" {
t.Fatalf("stale worker persisted a successful result: execution=%+v err=%v", completed, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task WHERE id = 'task-37' AND state = 'needs_confirmation' AND hold_reason = 'exit_unhealthy'`, 1)
assertNotified("needs_confirmation", "exit_unhealthy", "account-a", "task-37")
if _, err := store.db.ExecContext(ctx, `UPDATE network_exit SET health_status = 'healthy' WHERE id = 'exit-shared'`); err != nil {
t.Fatal(err)
}
concurrent := approvedTask(38, "account-a", accountA.Version, "draft-a", "confirmation-a")
if _, _, err := store.Enqueue(ctx, concurrent); err != nil {
t.Fatal(err)
}
concurrentExecution, err := store.claim(ctx, "worker-concurrent-release")
if err != nil || concurrentExecution.TaskID != concurrent.ID {
t.Fatalf("claim concurrent completion fixture: execution=%+v err=%v", concurrentExecution, err)
}
releaseTx, err := store.db.BeginTx(ctx, nil)
if err != nil {
t.Fatal(err)
}
if _, err := releaseTx.ExecContext(ctx, `UPDATE runtime_instance SET released_at = now() WHERE id = 'runtime-instance-a'`); err != nil {
t.Fatal(err)
}
type completionResult struct {
execution Execution
err error
}
completionStarted := make(chan struct{})
completionDone := make(chan completionResult, 1)
go func() {
close(completionStarted)
execution, err := store.complete(ctx, "worker-concurrent-release", concurrentExecution, "succeeded")
completionDone <- completionResult{execution: execution, err: err}
}()
<-completionStarted
select {
case result := <-completionDone:
releaseTx.Rollback()
t.Fatalf("completion bypassed an in-flight runtime release: execution=%+v err=%v", result.execution, result.err)
case <-time.After(100 * time.Millisecond):
}
if err := releaseTx.Commit(); err != nil {
t.Fatal(err)
}
select {
case result := <-completionDone:
if result.err != nil || result.execution.State != "needs_confirmation" {
t.Fatalf("completion after runtime release was not quarantined: execution=%+v err=%v", result.execution, result.err)
}
case <-time.After(5 * time.Second):
t.Fatal("completion remained blocked after runtime release committed")
}
assertCount(t, store, `SELECT count(*) FROM operation_task
WHERE id = 'task-38' AND state = 'needs_confirmation' AND hold_reason = 'runtime_missing'`, 1)
assertNotified("needs_confirmation", "runtime_missing", "account-a", "task-38")
concurrentDetail, err := store.GetTaskDetail(ctx, concurrent.ID)
if err != nil || concurrentDetail.RuntimeInstanceID != "runtime-instance-a" {
t.Fatalf("task detail lost its immutable claim runtime: detail=%+v err=%v", concurrentDetail, err)
}
if _, err := store.db.ExecContext(ctx, `
INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until)
VALUES ('runtime-instance-a2', 'account-a', 'binding-a', 1, 'runtime-a2', now() + interval '1 minute')`); err != nil {
t.Fatal(err)
}
replaced := approvedTask(39, "account-a", accountA.Version, "draft-a", "confirmation-a")
if _, _, err := store.Enqueue(ctx, replaced); err != nil {
t.Fatal(err)
}
replacedExecution, err := store.claim(ctx, "worker-replaced-runtime")
if err != nil || replacedExecution.TaskID != replaced.ID {
t.Fatalf("claim replaced runtime fixture: execution=%+v err=%v", replacedExecution, err)
}
if _, err := store.db.ExecContext(ctx, `
UPDATE runtime_instance SET released_at = now() WHERE id = 'runtime-instance-a2';
INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until)
VALUES ('runtime-instance-a3', 'account-a', 'binding-a', 1, 'runtime-a3', now() + interval '1 minute')`); err != nil {
t.Fatal(err)
}
replacedCompletion, err := store.complete(ctx, "worker-replaced-runtime", replacedExecution, "succeeded")
if err != nil || replacedCompletion.State != "needs_confirmation" {
t.Fatalf("old worker completed against a replacement runtime: execution=%+v err=%v", replacedCompletion, err)
}
assertCount(t, store, `SELECT count(*) FROM operation_task
WHERE id = 'task-39' AND state = 'needs_confirmation' AND hold_reason = 'task_result_uncertain'`, 1)
assertNotified("needs_confirmation", "task_result_uncertain", "account-a", "task-39")
replacedDetail, err := store.GetTaskDetail(ctx, replaced.ID)
if err != nil || replacedDetail.AllowedAction != "verify" || replacedDetail.BrowserEnvAlias != "account-a" ||
replacedDetail.NetworkExitID != "exit-shared" || replacedDetail.RuntimeInstanceID != "runtime-instance-a2" || replacedDetail.BindingVersion != 1 {
t.Fatalf("replacement task detail did not preserve the claim snapshot: detail=%+v err=%v", replacedDetail, err)
}
replacedAttempt, err := store.GetTaskAttemptDetail(ctx, replacedExecution.AttemptID)
if err != nil || replacedAttempt.BrowserEnvAlias != "account-a" || replacedAttempt.NetworkExitID != "exit-shared" ||
replacedAttempt.RuntimeInstanceID != "runtime-instance-a2" || replacedAttempt.BindingVersion != 1 {
t.Fatalf("replacement attempt detail did not preserve the claim snapshot: detail=%+v err=%v", replacedAttempt, err)
}
if _, err := store.db.ExecContext(ctx, `INSERT INTO audit_event (event_type, details) VALUES ('redaction_test',
'{"state":"safe","api_key":"api-secret","private_key":"private-secret","credential_key":"credential-secret","authorization_header":"auth-secret","proxy_url":"proxy-secret","nested":{"state":"nested-safe","token":"nested-secret"}}')`); err != nil {
t.Fatal(err)
}
events, err := store.Audit(ctx)
if err != nil {
t.Fatal(err)
}
traceable := false
for _, event := range events {
if event.EventType == "task_finished" && event.ConfirmationID != "" && event.ConfirmationVersion == 1 && event.AttemptID != "" && event.TaskID != "" {
traceable = true
break
}
}
if !traceable {
t.Fatal("audit does not trace confirmation version, task, and attempt")
}
exported, _ := json.Marshal(events)
for _, forbidden := range []string{"password", "cookie", "token", "credential-a", "creatorhub/account-a", "api-secret", "private-secret", "credential-secret", "auth-secret", "proxy-secret", "nested-secret"} {
if strings.Contains(strings.ToLower(string(exported)), forbidden) {
t.Fatalf("audit export contains sensitive field or credential reference %q", forbidden)
}
}
firstPage, err := store.ListAudit(ctx, AuditFilter{Page: 1, PageSize: 1})
if err != nil || firstPage.Total < 2 || len(firstPage.Data) != 1 {
t.Fatalf("first audit page: page=%+v err=%v", firstPage, err)
}
secondPage, err := store.ListAudit(ctx, AuditFilter{Page: 2, PageSize: 1})
if err != nil || len(secondPage.Data) != 1 || secondPage.Data[0].ID == firstPage.Data[0].ID {
t.Fatalf("second audit page: page=%+v err=%v", secondPage, err)
}
if _, err := store.db.ExecContext(ctx, `UPDATE audit_event SET event_type = 'rewritten' WHERE id = 1`); err == nil {
t.Fatal("audit events must be append-only")
}
}
func TestCreateAccountWithoutCookiesSkipsCredentialStore(t *testing.T) {
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
if databaseURL == "" {
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
}
ctx := context.Background()
store, err := Open(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
applyHubMigrationsForPhaseATest(t, store)
if _, err := store.db.ExecContext(ctx, `
TRUNCATE audit_event, execution_attempt, operation_task, confirmation, content_draft,
runtime_instance, environment_binding, network_exit, social_account, credential_reference,
browser_env, browser_version, gateway RESTART IDENTITY CASCADE`); err != nil {
t.Fatal(err)
}
credentials := &testCredentialBridge{values: map[string]string{}}
account := Account{ID: "account-no-cookies", Name: "扫码账号", Platform: "douyin", PlatformAccountKey: "qr-login",
Tags: []string{}, Cookies: "",
CredentialReference: CredentialReference{ID: "account-no-cookies-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-no-cookies/cookies"}
if err := store.CreateAccount(ctx, account, credentials); err != nil {
t.Fatalf("creating an account without cookies failed: %v", err)
}
if _, stored := credentials.values[account.CredentialKey]; stored {
t.Fatal("empty cookies must not be written to the credential provider")
}
assertCount(t, store, `SELECT count(*) FROM social_account WHERE id = $1`, 1, account.ID)
assertCount(t, store, `SELECT count(*) FROM credential_reference WHERE id = $1`, 1, account.CredentialReference.ID)
}
func TestAccountCredentialCommitResult(t *testing.T) {
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
if databaseURL == "" {
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
}
ctx := context.Background()
store, err := Open(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
applyHubMigrationsForPhaseATest(t, store)
if _, err := store.db.ExecContext(ctx, `
TRUNCATE audit_event, execution_attempt, operation_task, confirmation, content_draft,
runtime_instance, environment_binding, network_exit, social_account, credential_reference,
browser_env, browser_version, gateway RESTART IDENTITY CASCADE`); err != nil {
t.Fatal(err)
}
credentials := &testCredentialBridge{values: map[string]string{}}
account := func(id, platformKey string) Account {
return Account{ID: id, Name: id, Platform: "douyin", PlatformAccountKey: platformKey, Tags: []string{}, Cookies: "sessionid=" + id,
CredentialReference: CredentialReference{ID: id + "-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/" + id + "/cookies"}
}
store.accountCommit = func(tx *sql.Tx) error {
if err := tx.Commit(); err != nil {
return err
}
return io.ErrUnexpectedEOF
}
committed := account("account-committed", "platform-committed")
if err := store.CreateAccount(ctx, committed, credentials); !errors.Is(err, ErrAccountCreationUnknown) {
t.Fatalf("ambiguous commit did not return unknown: %v", err)
}
if credentials.values[committed.CredentialKey] == "" {
t.Fatal("committed unknown result deleted its credential")
}
assertCount(t, store, `SELECT count(*) FROM social_account WHERE id = $1`, 1, committed.ID)
store.accountCommit = func(tx *sql.Tx) error {
_ = tx.Rollback()
return pgx.ErrTxCommitRollback
}
rolledBack := account("account-rolled-back", "platform-rolled-back")
if err := store.CreateAccount(ctx, rolledBack, credentials); err == nil || errors.Is(err, ErrAccountCreationUnknown) {
t.Fatalf("known rollback did not return a known failure: %v", err)
}
if _, ok := credentials.values[rolledBack.CredentialKey]; ok {
t.Fatal("known rollback retained its credential")
}
store.accountCommit = func(tx *sql.Tx) error {
_ = tx.Rollback()
return io.ErrUnexpectedEOF
}
ambiguousRollback := account("account-ambiguous", "platform-ambiguous")
if err := store.CreateAccount(ctx, ambiguousRollback, credentials); !errors.Is(err, ErrAccountCreationUnknown) {
t.Fatalf("transport error did not preserve an unknown result: %v", err)
}
if credentials.values[ambiguousRollback.CredentialKey] == "" {
t.Fatal("commit-unknown deleted the credential")
}
}
func createApprovedDraft(t *testing.T, store *Store, accountID string, accountVersion int64, draftID, confirmationID string) {
t.Helper()
ctx := context.Background()
if err := store.CreateDraft(ctx, Draft{ID: draftID, AccountID: accountID, Version: 1, Content: "offline mock content"}); err != nil {
t.Fatal(err)
}
if err := store.Confirm(ctx, Confirmation{
ID: confirmationID, AccountID: accountID, AccountVersion: accountVersion, DraftID: draftID, DraftVersion: 1, Version: 1,
}); err != nil {
t.Fatal(err)
}
}
func approvedTask(index int, accountID string, accountVersion int64, draftID, confirmationID string) Task {
return Task{
ID: fmt.Sprintf("task-%02d", index), IdempotencyKey: fmt.Sprintf("idempotency-%02d", index),
AccountID: accountID, AccountVersion: accountVersion, DraftID: draftID, DraftVersion: 1,
ConfirmationID: confirmationID, ConfirmationVersion: 1,
}
}
func seedGateTask(t *testing.T, store *Store, suffix, accountStatus, authorizationStatus, exitStatus, runtimeLease string,
binding, versionChanged, confirmationChanged bool) string {
t.Helper()
ctx := context.Background()
accountID, taskID := "gate-"+suffix, "task-gate-"+suffix
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `INSERT INTO credential_reference (id, provider, reference_key) VALUES ($1, 'os_keyring', $2)`,
"credential-"+suffix, "creatorhub/gate-"+suffix); err != nil {
t.Fatal(err)
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO social_account
(id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status, status, version)
VALUES ($1, $2, 'mock', $1, 'owned', $3, $4, 1)`, accountID, "credential-"+suffix, authorizationStatus, accountStatus); err != nil {
t.Fatal(err)
}
if _, err := tx.ExecContext(ctx, `INSERT INTO content_draft (id, account_id, version, content) VALUES ($1, $2, 1, 'gate test')`,
"draft-gate-"+suffix, accountID); err != nil {
t.Fatal(err)
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO confirmation (id, account_id, account_version, draft_id, draft_version, version)
VALUES ($1, $2, 1, $3, 1, 1)`, "confirmation-gate-"+suffix, accountID, "draft-gate-"+suffix); err != nil {
t.Fatal(err)
}
if binding {
exitID := "exit-gate-" + suffix
if _, err := tx.ExecContext(ctx, `INSERT INTO browser_env (alias, name, gateway_name, browser_version, fingerprint)
VALUES ($1, $1, 'test-gateway', '1', '{"seed":3}')`, accountID); err != nil {
t.Fatal(err)
}
if _, err := tx.ExecContext(ctx, `INSERT INTO network_exit (id, protocol, host, port, health_status)
VALUES ($1, 'socks5', '127.0.0.1', 1080, $2)`, exitID, exitStatus); err != nil {
t.Fatal(err)
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO environment_binding (id, account_id, browser_env_alias, network_exit_id, version)
VALUES ($1, $2, $2, $3, 1)`, "binding-"+suffix, accountID, exitID); err != nil {
t.Fatal(err)
}
if runtimeLease != "" {
interval := "1 minute"
if runtimeLease == "expired" {
interval = "-1 second"
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO runtime_instance (id, account_id, binding_id, binding_version, runtime_id, lease_until)
VALUES ($1, $2, $3, 1, $4, now() + $5::interval)`, "runtime-gate-"+suffix, accountID,
"binding-"+suffix, "runtime-gate-"+suffix, interval); err != nil {
t.Fatal(err)
}
}
}
if versionChanged {
if _, err := tx.ExecContext(ctx, `UPDATE environment_binding SET version = 2 WHERE account_id = $1`, accountID); err != nil {
t.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
confirmationVersion := int64(1)
if confirmationChanged {
confirmationVersion = 2
}
if _, _, err := store.Enqueue(ctx, Task{
ID: taskID, IdempotencyKey: "idempotency-gate-" + suffix, AccountID: accountID, AccountVersion: 1,
DraftID: "draft-gate-" + suffix, DraftVersion: 1, ConfirmationID: "confirmation-gate-" + suffix,
ConfirmationVersion: confirmationVersion,
}); err != nil {
t.Fatal(err)
}
return taskID
}
func assertTaskGate(t *testing.T, store *Store, taskID, wantState, wantReason string) {
t.Helper()
var state string
if err := store.db.QueryRow(`SELECT state FROM operation_task WHERE id = $1`, taskID).Scan(&state); err != nil || state != wantState {
t.Fatalf("task gate state: got=%q want=%q err=%v", state, wantState, err)
}
var attempts int
if err := store.db.QueryRow(`SELECT count(*) FROM execution_attempt WHERE task_id = $1`, taskID).Scan(&attempts); err != nil || attempts != 0 {
t.Fatalf("gated task executed: attempts=%d err=%v", attempts, err)
}
var reason string
if err := store.db.QueryRow(`SELECT reason_code FROM audit_event WHERE task_id = $1 AND event_type = 'task_quarantined' ORDER BY id DESC LIMIT 1`, taskID).Scan(&reason); err != nil || reason != wantReason {
t.Fatalf("task gate reason: got=%q want=%q err=%v", reason, wantReason, err)
}
}
func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) {
t.Helper()
for _, migrationFile := range []struct {
version int
name string
}{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"},
{6, "006_runtime_cleanup.sql"}, {7, "007_runtime_binding_version.sql"}, {8, "008_runtime_cleanup_generation.sql"},
{9, "009_runtime_cleanup_compatibility.sql"}, {10, "010_runtime_network_generation.sql"}, {11, "011_task_recovery.sql"},
{12, "012_task_recovery_compatibility.sql"}, {13, "013_account_creation.sql"}, {14, "014_account_creation_compatibility.sql"},
{15, "015_gateway_rename_cascade.sql"}, {16, "016_network_exit_plain_credentials.sql"}, {17, "017_native_browser_versions.sql"}} {
var applied bool
if err := store.db.QueryRow(`SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migrationFile.version).Scan(&applied); err != nil {
t.Fatal(err)
}
if applied {
continue
}
migration, err := os.ReadFile(filepath.Join("..", "environment", "migrations", migrationFile.name))
if err != nil {
t.Fatal(err)
}
if _, err := store.db.Exec(string(migration)); err != nil {
t.Fatalf("apply test migration %d: %v", migrationFile.version, err)
}
if _, err := store.db.Exec(`INSERT INTO schema_migration (version) VALUES ($1)`, migrationFile.version); err != nil {
t.Fatal(err)
}
}
}
func readinessReason(err error) string {
var readiness *ReadinessError
if errors.As(err, &readiness) {
return readiness.Reason
}
return ""
}
func assertCount(t *testing.T, store *Store, query string, expected int, args ...any) {
t.Helper()
var actual int
if err := store.db.QueryRowContext(context.Background(), query, args...).Scan(&actual); err != nil || actual != expected {
t.Fatalf("count mismatch: expected=%d actual=%d err=%v query=%s", expected, actual, err, query)
}
}