From 9095920a5b43bc5dfd788d3bbc22e384394f6ada Mon Sep 17 00:00:00 2001 From: Rogee Date: Wed, 16 Sep 2026 00:38:49 +0800 Subject: [PATCH] fix: complete Douyin release remediation --- .gitea/workflows/douyin-release-gate.yaml | 141 ++++++ Dockerfile | 1 + cmd/control-plane/creator.go | 184 +++++++- cmd/control-plane/creator_events.go | 50 +- cmd/control-plane/creator_events_test.go | 45 ++ cmd/control-plane/creator_helper_test.go | 234 ++++++++++ cmd/control-plane/creator_history_test.go | 68 +++ cmd/control-plane/creator_material.go | 138 ++++-- cmd/control-plane/creator_material_test.go | 48 +- cmd/control-plane/credential_test.go | 28 ++ cmd/control-plane/douyin.go | 1 + cmd/control-plane/hub.go | 42 ++ cmd/control-plane/main_test.go | 396 ++++++++++++++++ cmd/control-plane/xiaohongshu.go | 2 +- cmd/docker_gateway/douyin.py | 85 +++- cmd/docker_gateway/gateway.py | 58 ++- cmd/docker_gateway/test_gateway.py | 81 +++- docker/browser-wrapper/README.md | 4 +- docs/deployment.md | 46 +- .../douyin-release-20260916-worktree.md | 99 ++++ .../2026-09-15-douyin-release-remediation.md | 280 +++++++++++ internal/creator/accounts.go | 20 +- internal/creator/actions.go | 356 +++++++++++--- internal/creator/bailian_test.go | 28 ++ internal/creator/collection.go | 13 +- internal/creator/content.go | 144 +++++- internal/creator/coverage_integration_test.go | 193 ++++++++ internal/creator/coverage_unit_test.go | 144 ++++++ internal/creator/integration_test.go | 118 ++++- internal/creator/listener.go | 99 +++- internal/creator/metrics.go | 34 +- .../032_douyin_release_remediation.sql | 52 +++ internal/creator/models.go | 86 ++-- internal/creator/recovery_integration_test.go | 161 +++++++ internal/creator/settings.go | 19 + internal/creator/store.go | 23 + internal/douyin/connector.go | 10 +- internal/douyin/creator_collector.go | 57 ++- internal/douyin/creator_collector_test.go | 45 ++ internal/hub/environment.go | 85 ++++ .../033_unique_fingerprint_seed.sql | 3 + internal/hub/store.go | 5 +- internal/hub/store_test.go | 51 +- internal/phasea/store_test.go | 11 +- web/src/BrowsersPage.jsx | 26 +- web/src/CreatorAccountsPage.jsx | 12 + web/src/CreatorCompetitorsPage.jsx | 295 ++++++++++-- web/src/CreatorPages.test.jsx | 89 +++- web/src/CreatorSettingsPage.jsx | 9 + web/src/CreatorWorkbenchPage.jsx | 136 +++++- web/src/NetworkExitsPage.jsx | 156 ++++++- web/src/dataProvider.js | 28 +- web/src/dataProvider.test.js | 442 ++++++++++++------ web/src/lib/hooks.test.jsx | 41 ++ web/src/lib/ui.jsx | 24 +- web/vite.config.js | 2 +- 56 files changed, 4530 insertions(+), 518 deletions(-) create mode 100644 .gitea/workflows/douyin-release-gate.yaml create mode 100644 cmd/control-plane/creator_helper_test.go create mode 100644 cmd/control-plane/creator_history_test.go create mode 100644 docs/evidence/douyin-release-20260916-worktree.md create mode 100644 docs/plans/2026-09-15-douyin-release-remediation.md create mode 100644 internal/creator/coverage_integration_test.go create mode 100644 internal/creator/coverage_unit_test.go create mode 100644 internal/creator/migrations/032_douyin_release_remediation.sql create mode 100644 internal/hub/migrations/033_unique_fingerprint_seed.sql create mode 100644 web/src/lib/hooks.test.jsx diff --git a/.gitea/workflows/douyin-release-gate.yaml b/.gitea/workflows/douyin-release-gate.yaml new file mode 100644 index 0000000..c564f73 --- /dev/null +++ b/.gitea/workflows/douyin-release-gate.yaml @@ -0,0 +1,141 @@ +name: douyin-release-gate + +on: + push: + branches: [main] + pull_request: + +jobs: + verify: + runs-on: ubuntu-latest + services: + postgres: + image: >- + postgres:17-alpine@sha256:18cfe3ef5e6815560c98237d6216d1e5119702fb0f3894c8785dd58b8bbe5d73 + env: + POSTGRES_DB: creatorhub + POSTGRES_USER: creatorhub + POSTGRES_HOST_AUTH_METHOD: trust + + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U creatorhub -d creatorhub" + --health-interval 2s --health-timeout 2s --health-retries 15 + env: + CREATORHUB_POSTGRES_TEST_URL: >- + postgres://creatorhub@postgres:5432/creatorhub?sslmode=disable + CONTROL_PLANE_USERNAME: ci + CONTROL_PLANE_PASSWORD: ci-password + CREATORHUB_CREDENTIAL_MASTER_KEY: >- + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + DOCKER_GID: 999 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 + with: + go-version: '1.26' + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '22' + cache: npm + cache-dependency-path: web/package-lock.json + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.13' + - name: Go tests and race gate + run: | + set -Eeuo pipefail + git diff --check + go test -p 1 -parallel 1 -count=1 ./... + go vet ./... + go test -p 1 -parallel 1 -race -count=1 ./... + go build ./cmd/control-plane + - name: PostgreSQL integration and coverage gate + run: | + set -Eeuo pipefail + mkdir -p evidence + python3 - <<'PY' + import socket + with socket.create_connection(("postgres", 5432), timeout=5): + pass + PY + go test -p 1 -parallel 1 -count=1 -json \ + -coverprofile=evidence/go.cover ./... \ + | tee evidence/go-test.jsonl + go tool cover -func=evidence/go.cover | tee evidence/go-cover.txt + python3 - <<'PY' + import json + from pathlib import Path + total = next( + line + for line in Path("evidence/go-cover.txt").read_text().splitlines() + if line.startswith("total:") + ) + coverage = float(total.rsplit(" ", 1)[-1].rstrip("%")) + if coverage < 65: + raise SystemExit(f"Go coverage {coverage:.2f}% is below 65%") + skipped = [] + for line in Path("evidence/go-test.jsonl").read_text().splitlines(): + event = json.loads(line) + is_skip = event.get("Action") == "skip" + is_test = str(event.get("Test", "")).startswith("Test") + if is_skip and is_test: + skipped.append(event) + if skipped: + raise SystemExit(f"Go tests skipped: {skipped[:5]}") + PY + - name: Python gateway tests and coverage gate + run: | + set -Eeuo pipefail + python3 -m pip install -r requirements-gateway.lock coverage==7.16.0 + python3 -m coverage erase + python3 -m coverage run \ + --source=cmd/docker_gateway \ + --omit='cmd/docker_gateway/test_*.py' \ + -m unittest discover -s cmd/docker_gateway -t cmd -p 'test_*.py' + python3 -m coverage report --precision=2 --fail-under=65 + python3 -m coverage json -o evidence/python-coverage.json + python3 - <<'PY' + import json + with open("evidence/python-coverage.json") as f: + total = json.load(f)["totals"] + if total["covered_lines"] * 100 < total["num_statements"] * 65: + raise SystemExit(f"Python coverage below 65%: {total}") + PY + - name: Frontend clean install, tests, and build + run: | + set -Eeuo pipefail + npm --prefix web ci + cd web + for file in src/*.test.jsx src/*.test.js; do + [ -f "$file" ] || continue + npx vitest run "$file" \ + --pool=threads --maxWorkers=1 --fileParallelism=false + done + npm run test:coverage -- --run --pool=threads \ + --maxWorkers=1 --fileParallelism=false + for file in \ + coverage/coverage-summary.json \ + coverage/coverage-final.json; do + cp "$file" ../evidence/ + done + npm run build + - name: Compose configuration and image gate + run: | + set -Eeuo pipefail + trap 'docker compose down' EXIT + docker compose config --quiet + docker compose -f compose.yaml -f compose.dev.yaml config --quiet + docker compose build creator-hub docker-gateway + docker compose up -d --wait creator-hub docker-gateway + docker compose exec -T creator-hub ffmpeg -version + docker compose exec -T creator-hub ffprobe -version + docker compose images --quiet creator-hub docker-gateway \ + | sort -u | tee evidence/compose-image-ids.txt + curl --fail --retry 10 --retry-delay 2 http://127.0.0.1:8080/readyz + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: douyin-release-gate-evidence + path: | + evidence/ + web/coverage/ diff --git a/Dockerfile b/Dockerfile index feeccc3..c07cb31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ COPY web/ ./ RUN npm run build FROM golang:1.26-alpine@sha256:28d89ee9cc0ff9fec75c82ca201e6bf7fdf9a679d4b7b24dfa04f2bb766bb468 AS go +RUN apk add --no-cache git ENV GOPROXY=https://goproxy.cn|direct WORKDIR /src COPY go.mod go.sum ./ diff --git a/cmd/control-plane/creator.go b/cmd/control-plane/creator.go index d9cfadd..58227b4 100644 --- a/cmd/control-plane/creator.go +++ b/cmd/control-plane/creator.go @@ -94,7 +94,7 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto c.Set("Transfer-Encoding", "chunked") return c.SendStreamWriter(func(w *bufio.Writer) { defer unsubscribe() - if _, err := fmt.Fprint(w, "retry: 5000\\n\\n"); err != nil { + if _, err := fmt.Fprint(w, "retry: 5000\n\n"); err != nil { return } if err := w.Flush(); err != nil { @@ -105,11 +105,11 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto for { select { case <-updates: - if _, err := fmt.Fprint(w, "event: creator-update\\ndata: {}\\n\\n"); err != nil { + if _, err := fmt.Fprint(w, "event: creator-update\ndata: {}\n\n"); err != nil { return } case <-heartbeat.C: - if _, err := fmt.Fprint(w, ": keep-alive\\n\\n"); err != nil { + if _, err := fmt.Fprint(w, ": keep-alive\n\n"); err != nil { return } case <-c.RequestCtx().Done(): @@ -250,11 +250,46 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto } return c.JSON(items) }) - app.Post("/api/creator/competitors", func(c fiber.Ctx) error { - var input creator.CompetitorInput + app.Post("/api/creator/competitors/preview", func(c fiber.Ctx) error { + var input struct { + AccountID string `json:"account_id"` + creator.CompetitorInput + } if err := decodeCreator(c, &input); err != nil { return creatorError(c, err) } + if input.Platform != creator.PlatformDouyin { + return creatorError(c, creator.ErrUnavailable) + } + profile, err := previewDouyinCompetitor(c.Context(), store, phaseAStore, hubStore, input.AccountID, input.CompetitorInput) + if err != nil { + return creatorError(c, err) + } + return c.JSON(profile) + }) + app.Post("/api/creator/competitors", func(c fiber.Ctx) error { + var request struct { + AccountID string `json:"account_id"` + creator.CompetitorInput + } + if err := decodeCreator(c, &request); err != nil { + return creatorError(c, err) + } + input := request.CompetitorInput + if input.Platform == creator.PlatformDouyin { + preview, err := previewDouyinCompetitor(c.Context(), store, phaseAStore, hubStore, request.AccountID, input) + if err != nil { + return creatorError(c, err) + } + canonical, ok := preview["platform_account_key"].(string) + if !ok || canonical == "" { + return creatorError(c, creator.ErrConflict) + } + input.PlatformAccountKey = canonical + if homepage, ok := preview["homepage_url"].(string); ok { + input.HomepageURL = homepage + } + } if err := validateXiaohongshuCompetitor(input); err != nil { return creatorError(c, err) } @@ -616,6 +651,13 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto return c.Status(status).JSON(result) }) + app.Get("/api/creator/listener-boundaries", func(c fiber.Ctx) error { + items, err := store.ListListenerBoundaries(c.Context(), c.Query("account_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) app.Get("/api/creator/events", func(c fiber.Ctx) error { page, pageSize, paged, err := creatorPageQuery(c) if err != nil { @@ -681,6 +723,13 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto creatorUpdates.publish() return c.JSON(event) }) + app.Get("/api/creator/events/:id/strategy-trace", func(c fiber.Ctx) error { + items, err := store.ListStrategyTraces(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) app.Get("/api/creator/operations", func(c fiber.Ctx) error { items, err := store.ListOperations(c.Context(), c.Query("account_id")) if err != nil { @@ -710,6 +759,20 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto } return c.JSON(item) }) + app.Get("/api/creator/operations/:id/verification", func(c fiber.Ctx) error { + item, err := store.GetOperation(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(map[string]any{ + "operation_id": item.ID, + "state": item.State, + "verification_state": item.VerificationState, + "evidence": item.VerificationProof, + "reason": item.Reason, + "verified_at": item.VerifiedAt, + }) + }) app.Post("/api/creator/operations/:id/execute", func(c fiber.Ctx) error { item, err := store.ExecuteManualOperation(c.Context(), c.Params("id"), executor) if err != nil { @@ -767,7 +830,7 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto return creatorError(c, err) } if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || - profile.Platform != creator.PlatformDouyin || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" || + profile.Platform != creator.PlatformDouyin || (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" || profile.PlatformAccountKey == "" || account.PlatformAccountKey != profile.PlatformAccountKey { return creatorError(c, creator.ErrConflict) } @@ -784,7 +847,7 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto if err != nil { return creatorError(c, fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)) } - history, err := browser.MessageHistory(c.Context(), accountUID, conversation.PeerUID, limit) + history, err := browser.MessageHistory(c.Context(), accountUID, conversation.PeerUID, conversation.HistoryCursor, limit) if err != nil { return creatorError(c, err) } @@ -795,11 +858,15 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto if err != nil { return creatorError(c, err) } + if err := store.UpdateConversationHistoryCursor(c.Context(), conversation.ID, history.HistoryCursor, history.HistoryHasMore); err != nil { + return creatorError(c, err) + } creatorUpdates.publish() return c.JSON(map[string]any{ "conversation_id": conversation.ID, "messages": inserted, "history_source": history.HistorySource, + "history_cursor": history.HistoryCursor, "history_has_more": history.HistoryHasMore, }) }) @@ -896,7 +963,7 @@ func persistDouyinMessageHistory(ctx context.Context, store *creator.Store, conv if item.SenderUID == accountUID { direction, state = "outbound", "succeeded" } - _, wasInserted, err := store.SaveMessage(ctx, creator.MessageInput{ + savedMessage, wasInserted, err := store.SaveMessage(ctx, creator.MessageInput{ Platform: creator.PlatformDouyin, AccountID: conversation.AccountID, PeerUID: conversation.PeerUID, @@ -911,6 +978,9 @@ func persistDouyinMessageHistory(ctx context.Context, store *creator.Store, conv if err != nil { return inserted, err } + if err := store.LinkMessageOperation(ctx, savedMessage.ID, item.ServerID); err != nil { + return inserted, err + } if wasInserted { inserted++ } @@ -927,7 +997,7 @@ func setStrategyEnabled(c fiber.Ctx, store *creator.Store, enabled bool) error { } func workFilter(c fiber.Ctx) (creator.WorkFilter, error) { - filter := creator.WorkFilter{Platform: c.Query("platform"), SourceID: c.Query("source_id"), SourceType: c.Query("source_type")} + filter := creator.WorkFilter{Platform: c.Query("platform"), SourceID: c.Query("source_id"), SourceType: c.Query("source_type"), PublishedAtStatus: c.Query("published_at_status")} for _, field := range []struct { name string target **int64 @@ -1033,6 +1103,7 @@ func (executor creatorGatewayActionExecutor) Execute(ctx context.Context, reques } payload := gatewayGenerationPayload(environment) payload["expected_uid"] = uid + payload["operation_id"] = request.OperationID payload["action"] = request.Action payload["target_uid"] = request.TargetUID // UI and persistence use opaque internal IDs; the platform gateway receives only @@ -1042,23 +1113,31 @@ func (executor creatorGatewayActionExecutor) Execute(ctx context.Context, reques if errors.Is(targetErr, creator.ErrNotFound) { comment, targetErr = executor.store.GetCommentByKey(ctx, request.Platform, request.TargetCommentID) } - if targetErr != nil { + if targetErr == nil { + payload["target_comment_id"] = comment.CommentKey + if request.TargetWorkID == "" { + request.TargetWorkID = comment.WorkID + } + } else if errors.Is(targetErr, creator.ErrNotFound) { + // The event may arrive before collection. Keep the opaque platform key; + // the gateway must verify ownership against the logged-in account. + payload["target_comment_id"] = request.TargetCommentID + } else { return creator.ActionResult{}, targetErr } - payload["target_comment_id"] = comment.CommentKey - if request.TargetWorkID == "" { - request.TargetWorkID = comment.WorkID - } } if request.TargetWorkID != "" { work, targetErr := executor.store.GetWork(ctx, request.TargetWorkID) if errors.Is(targetErr, creator.ErrNotFound) { work, targetErr = executor.store.GetWorkByKey(ctx, request.Platform, request.TargetWorkID) } - if targetErr != nil { + if targetErr == nil { + payload["target_work_id"] = work.WorkKey + } else if errors.Is(targetErr, creator.ErrNotFound) { + payload["target_work_id"] = request.TargetWorkID + } else { return creator.ActionResult{}, targetErr } - payload["target_work_id"] = work.WorkKey } payload["text"] = request.Text payload["confirm"] = true @@ -1141,13 +1220,14 @@ func (browser creatorGatewayBrowser) Identity(ctx context.Context, expectedKey s return identity.UID, nil } -func (browser creatorGatewayBrowser) MessageHistory(ctx context.Context, expectedUID, targetUID string, limit int) (douyinMessageHistory, error) { - if strings.TrimSpace(expectedUID) == "" || strings.TrimSpace(targetUID) == "" || limit < 1 || limit > 200 { +func (browser creatorGatewayBrowser) MessageHistory(ctx context.Context, expectedUID, targetUID, cursor string, limit int) (douyinMessageHistory, error) { + if strings.TrimSpace(expectedUID) == "" || strings.TrimSpace(targetUID) == "" || len(cursor) > 500 || limit < 1 || limit > 200 { return douyinMessageHistory{}, errors.New("invalid message history request") } payload := gatewayGenerationPayload(browser.environment) payload["expected_uid"] = expectedUID payload["target_uid"] = targetUID + payload["cursor"] = cursor payload["limit"] = limit status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/messages", payload, 30*time.Second) if err != nil || status != http.StatusOK { @@ -1285,6 +1365,50 @@ func (browser creatorGatewayBrowser) Media(ctx context.Context, target, destinat return writeCreatorMedia(destination, data) } +func previewDouyinCompetitor(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID string, input creator.CompetitorInput) (map[string]any, error) { + if store == nil || phaseAStore == nil || hubStore == nil || input.Platform != creator.PlatformDouyin || strings.TrimSpace(accountID) == "" { + return nil, creator.ErrInvalid + } + account, err := phaseAStore.GetAccount(ctx, accountID) + if err != nil { + return nil, err + } + if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" { + return nil, creator.ErrConflict + } + profile, err := store.GetAccountProfile(ctx, accountID) + if err != nil { + return nil, err + } + if profile.Platform != creator.PlatformDouyin || (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" || profile.PlatformAccountKey == "" { + return nil, creator.ErrConflict + } + environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) + if err != nil { + return nil, fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) + } + gateway, err := hubStore.GetGateway(ctx, environment.Gateway) + if err != nil { + return nil, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) + } + browser := creatorGatewayBrowser{gateway: gateway, environment: environment} + if _, err := browser.Identity(ctx, profile.PlatformAccountKey); err != nil { + return nil, fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err) + } + target, err := (douyin.CreatorCollector{Browser: browser}).ResolveTarget(ctx, input.PlatformAccountKey) + if err != nil { + return nil, fmt.Errorf("%w: target identity verification failed: %v", creator.ErrConflict, err) + } + return map[string]any{ + "account_id": accountID, + "platform": creator.PlatformDouyin, + "platform_account_key": target.SecUID, + "nickname": target.Nickname, + "avatar_url": target.AvatarURL, + "homepage_url": "https://www.douyin.com/user/" + target.SecUID, + }, nil +} + func syncCreatorCompetitor(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, competitorID, accountID string) (creator.CollectionReport, error) { return syncCreatorCompetitorWithClaim(ctx, store, phaseAStore, hubStore, competitorID, accountID, true) } @@ -1336,7 +1460,7 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p if err != nil { return blocked(err) } - if profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { + if (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" { return blocked(creator.ErrConflict) } environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) @@ -1461,6 +1585,18 @@ func runCreatorMetricScheduleOnce(ctx context.Context, store *creator.Store, pha } func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, work creator.Work, accountID string, settings creator.Settings, now time.Time) error { + if work.SourceType == creator.SourceCompetitor { + competitor, err := store.GetCompetitor(ctx, work.SourceID) + if err != nil { + if errors.Is(err, creator.ErrNotFound) { + return store.StopMetricPlan(ctx, work.ID, "source_deleted") + } + return err + } + if !competitor.Enabled { + return store.StopMetricPlan(ctx, work.ID, "source_disabled") + } + } account, err := phaseAStore.GetAccount(ctx, accountID) if err != nil { return err @@ -1469,7 +1605,7 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if err != nil { return err } - if (account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu) || account.AuthorizationStatus != "authorized" || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { + if (account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu) || account.AuthorizationStatus != "authorized" || (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" { return creator.ErrConflict } environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) @@ -1537,7 +1673,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph if err != nil { return err } - if profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { + if (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" { return creator.ErrConflict } settings, err := store.GetSettings(ctx) @@ -1589,10 +1725,12 @@ func creatorCollectionAccount(ctx context.Context, store *creator.Store, phaseAS if account.Platform != platform || account.AuthorizationStatus != "authorized" { continue } - if _, err := store.GetAccountProfile(ctx, account.ID); err != nil { + profile, err := store.GetAccountProfile(ctx, account.ID) + if err != nil || profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted" || profile.LoginStatus != "logged_in" { continue } - if _, err := hubStore.GetEnvironmentContextForAccount(ctx, account.ID); err != nil { + environment, err := hubStore.GetEnvironmentContextForAccount(ctx, account.ID) + if err != nil || environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { continue } return account.ID, nil diff --git a/cmd/control-plane/creator_events.go b/cmd/control-plane/creator_events.go index abb78e5..f281c82 100644 --- a/cmd/control-plane/creator_events.go +++ b/cmd/control-plane/creator_events.go @@ -8,6 +8,7 @@ import ( "net/url" "strings" "sync" + "sync/atomic" "time" "git.ipao.vip/rogee/creator-hub/internal/creator" @@ -20,6 +21,7 @@ const creatorEventReconcileInterval = 10 * time.Second type creatorGatewayEvent struct { DeliveryID string `json:"delivery_id,omitempty"` + Generation string `json:"generation,omitempty"` Kind string `json:"kind"` Reason string `json:"reason,omitempty"` Continuity string `json:"continuity,omitempty"` @@ -42,10 +44,11 @@ type creatorGatewayEventNotice struct { } type creatorEventBinding struct { - accountID string - uid string - env hub.EnvironmentContext - gateway hub.Gateway + accountID string + uid string + env hub.EnvironmentContext + gateway hub.Gateway + sessionToken string } type creatorUpdateHub struct { @@ -85,6 +88,12 @@ func creatorListenerGeneration(env hub.EnvironmentContext) string { return fmt.Sprintf("%s:%s:%d", env.RuntimeID, env.RuntimeNetworkID, env.BindingVersion) } +var creatorListenerSessionNonce atomic.Uint64 + +func creatorListenerSessionToken(env hub.EnvironmentContext) string { + return fmt.Sprintf("%s/%d-%d", creatorListenerGeneration(env), time.Now().UnixNano(), creatorListenerSessionNonce.Add(1)) +} + func listenerBoundaryPointer(value time.Time) *time.Time { if value.IsZero() { return nil @@ -101,6 +110,7 @@ func persistCreatorListenerState(ctx context.Context, store *creator.Store, bind AccountID: binding.accountID, Platform: creator.PlatformDouyin, Generation: creatorListenerGeneration(binding.env), + SessionToken: binding.sessionToken, Status: status, BoundaryAt: boundaryAt, LastDeliveryID: deliveryID, @@ -193,7 +203,10 @@ func (manager *creatorEventListenerManager) reconcile(ctx context.Context, store for accountID, current := range manager.items { binding, ok := desired[accountID] if ok && current.key == binding.key() { - continue + state, stateErr := store.GetListenerState(ctx, accountID) + if stateErr != nil || !state.Invalidated { + continue + } } current.cancel() delete(manager.items, accountID) @@ -237,6 +250,7 @@ func (manager *creatorEventListenerManager) close() { } func runCreatorEventListener(ctx context.Context, store *creator.Store, binding creatorEventBinding, executor creator.ActionExecutor, generator creator.TextGenerator) { + binding.sessionToken = creatorListenerSessionToken(binding.env) path := "/v1/browsers/" + url.PathEscape(binding.env.Alias) + "/douyin/events" generation := gatewayGenerationPayload(binding.env) startPayload := make(map[string]any, len(generation)+1) @@ -414,6 +428,19 @@ func handleCreatorGatewayEvent(ctx context.Context, store *creator.Store, bindin } input.Baseline = event.Baseline input.BaselineReason = event.Reason + input.Generation = strings.TrimSpace(event.Generation) + if input.Generation == "" { + input.Generation = creatorListenerGeneration(binding.env) + } + selfEvent := binding.uid != "" && input.InteractorUID == binding.uid + if selfEvent { + input.Baseline = true + input.BaselineReason = "接收账号主动行为" + } + if input.GatewayReceivedAt == nil { + receivedAt := time.Now().UTC() + input.GatewayReceivedAt = &receivedAt + } if input.PlatformEventAt == nil { input.Baseline = true input.BaselineReason = "缺少平台事件时间" @@ -439,10 +466,21 @@ func handleCreatorGatewayEvent(ctx context.Context, store *creator.Store, bindin } messageAt = &receivedAt } - if _, _, messageErr := store.SaveMessage(ctx, creator.MessageInput{Platform: input.Platform, AccountID: input.ReceivingAccountID, PeerUID: input.InteractorUID, PlatformMessageKey: "event:" + input.EventKey, Direction: "inbound", MessageType: input.MessageType, Text: input.MessageText, SentState: "received", MessageAt: messageAt}); messageErr != nil { + direction, sentState := "inbound", "received" + if selfEvent { + direction, sentState = "outbound", "succeeded" + } + savedMessage, _, messageErr := store.SaveMessage(ctx, creator.MessageInput{Platform: input.Platform, AccountID: input.ReceivingAccountID, PeerUID: input.InteractorUID, PlatformMessageKey: input.EventKey, Direction: direction, MessageType: input.MessageType, Text: input.MessageText, SentState: sentState, MessageAt: messageAt}) + if messageErr != nil { logrus.WithError(messageErr).WithFields(logrus.Fields{"account_id": binding.accountID, "event_key": input.EventKey}).Warn("creator direct message persistence failed") return } + if selfEvent { + if err := store.LinkMessageOperation(ctx, savedMessage.ID, input.EventKey); err != nil { + logrus.WithError(err).WithFields(logrus.Fields{"account_id": binding.accountID, "event_key": input.EventKey}).Warn("creator outbound direct message correlation failed") + return + } + } } // The event is now durable. An action may be slow or unavailable, but that // must not hold ingestion or cause the same receipt to be fetched forever. diff --git a/cmd/control-plane/creator_events_test.go b/cmd/control-plane/creator_events_test.go index 93af0fa..348429a 100644 --- a/cmd/control-plane/creator_events_test.go +++ b/cmd/control-plane/creator_events_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "testing" "time" @@ -8,6 +9,50 @@ import ( "git.ipao.vip/rogee/creator-hub/internal/hub" ) +func TestCreatorListenerPrimitives(t *testing.T) { + env := hub.EnvironmentContext{Env: hub.Env{Alias: "browser/a"}, BindingVersion: 3, RuntimeID: "runtime", RuntimeNetworkID: "network", Exit: hub.NetworkExit{ID: "exit"}} + generation := creatorListenerGeneration(env) + if generation != "runtime:network:3" { + t.Fatalf("unexpected generation: %q", generation) + } + tokenA, tokenB := creatorListenerSessionToken(env), creatorListenerSessionToken(env) + if tokenA == tokenB { + t.Fatal("listener session tokens must be unique") + } + if listenerBoundaryPointer(time.Time{}) != nil { + t.Fatal("zero boundary must remain nil") + } + boundary := listenerBoundaryPointer(time.Date(2024, 1, 1, 1, 0, 0, 0, time.FixedZone("test", 3600))) + if boundary == nil || !boundary.Equal(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) { + t.Fatalf("boundary was not normalized: %v", boundary) + } + binding := creatorEventBinding{accountID: "account", uid: "uid", env: env, gateway: hub.Gateway{Name: "gateway", Endpoint: "http://gateway", Token: "token"}, sessionToken: "session"} + if binding.key() == "" || pathForCreatorEvent(env) != "/v1/browsers/browser%2Fa/douyin/events" { + t.Fatalf("unexpected listener identity: key=%q path=%q", binding.key(), pathForCreatorEvent(env)) + } +} + +func TestCreatorEventListenerShutdownAndEventKinds(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if waitCreatorEventBackoff(ctx, time.Hour) { + t.Fatal("cancelled listener backoff must stop") + } + manager := &creatorEventListenerManager{items: map[string]creatorEventListenerHandle{ + "account": {cancel: func() {}, done: func() chan struct{} { ch := make(chan struct{}); close(ch); return ch }()}, + }} + manager.close() + if len(manager.items) != 0 { + t.Fatal("listener manager did not clear handles") + } + for _, kind := range []string{"error", "reconnected", "open", "baseline", "close", "unknown"} { + handleCreatorGatewayEvent(context.Background(), nil, creatorEventBinding{}, creatorGatewayEvent{Kind: kind}, nil, nil) + } + if err := manager.reconcile(context.Background(), nil, nil, nil, nil, nil); err != creator.ErrUnavailable { + t.Fatalf("nil listener dependencies: %v", err) + } +} + func TestCreatorUpdateHubPublishesAndUnsubscribes(t *testing.T) { hub := &creatorUpdateHub{subscribers: make(map[chan struct{}]struct{})} updates, unsubscribe := hub.subscribe() diff --git a/cmd/control-plane/creator_helper_test.go b/cmd/control-plane/creator_helper_test.go new file mode 100644 index 0000000..6391825 --- /dev/null +++ b/cmd/control-plane/creator_helper_test.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/hub" + "github.com/gofiber/fiber/v3" +) + +func TestCreatorHelperBranches(t *testing.T) { + if got := flattenActionEvidence(map[string]string{}, "evidence", "text"); got != 1 { + t.Fatalf("flatten string count = %d", got) + } + values := map[string]string{} + if got := flattenActionEvidence(values, "evidence", map[string]any{"nested": "value", "empty": "", "number": 1}); got != 1 || values["evidence.nested"] != "value" { + t.Fatalf("flatten map = %d, %#v", got, values) + } + if got := flattenActionEvidence(values, "evidence", []any{"ignored"}); got != 0 { + t.Fatalf("flatten unsupported count = %d", got) + } + if token, err := materialClaimToken(); err != nil || len(token) != 32 { + t.Fatalf("material claim token = %q, %v", token, err) + } + if _, err := creatorMaterialHasAudio(context.Background(), "/does/not/exist"); err == nil { + t.Fatal("missing media must not report audio") + } + mediaPath := filepath.Join(t.TempDir(), "media.bin") + if err := writeCreatorMedia(mediaPath, []byte("media")); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(mediaPath); err != nil || string(data) != "media" || !fileExists(mediaPath) { + t.Fatalf("media write: %q %v", data, err) + } + if err := os.WriteFile(filepath.Join(filepath.Dir(mediaPath), "empty"), nil, 0o600); err != nil { + t.Fatal(err) + } + if fileExists(filepath.Join(filepath.Dir(mediaPath), "empty")) || fileExists("/does/not/exist") { + t.Fatal("empty or missing media reported as existing") + } + for _, input := range [][2]string{{"", ""}, {"unsupported", "model"}, {"whisper", "model"}} { + if _, err := transcribeCreatorAudio(context.Background(), "/does/not/exist", input[0], input[1]); err == nil { + t.Fatalf("invalid transcription config accepted: %v", input) + } + } + if err := validateTranscriptionBinary("/does/not/exist"); err == nil { + t.Fatal("missing transcription binary accepted") + } + if _, err := processCreatorMaterial(context.Background(), nil, nil, nil, "../escape"); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("invalid material path = %v", err) + } + if _, err := verifyCreatorPlatformIdentity(context.Background(), "unsupported", hub.Gateway{}, hub.EnvironmentContext{}, "key"); !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("unsupported identity platform = %v", err) + } + if _, _, err := newCreatorCollector(context.Background(), creator.PlatformDouyin, hub.Gateway{}, hub.EnvironmentContext{}, "", "target", "", creator.SourceCompetitor, "id"); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("missing collector key = %v", err) + } + if _, err := decodeXiaohongshuResponse([]byte("not-json")); err == nil { + t.Fatal("malformed Xiaohongshu response must fail") + } + if response, err := decodeXiaohongshuResponse([]byte(`{"status":200,"body":"ok","challenge":""}`)); err != nil || response.Status != 200 || string(response.Body) != "ok" { + t.Fatalf("decode Xiaohongshu response = %+v, %v", response, err) + } + if _, err := decodeBase64("not-base64"); err == nil { + t.Fatal("invalid base64 must fail") + } + encoded := base64.StdEncoding.EncodeToString([]byte("media")) + if data, err := decodeBase64(encoded); err != nil || string(data) != "media" { + t.Fatalf("decode media = %q, %v", data, err) + } + if err := validateXiaohongshuSource("", ""); err == nil { + t.Fatal("empty Xiaohongshu source must fail") + } + if err := validateXiaohongshuCompetitor(creator.CompetitorInput{Platform: creator.PlatformDouyin}); err != nil { + t.Fatal(err) + } +} + +func TestCreatorPageQueryValidation(t *testing.T) { + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { + page, pageSize, enabled, err := creatorPageQuery(c) + if err != nil { + return creatorError(c, err) + } + return c.JSON(map[string]any{"page": page, "page_size": pageSize, "enabled": enabled}) + }) + for _, query := range []string{"", "?page=2&page_size=10&enabled=false"} { + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/"+query, nil)) + if err != nil || response.StatusCode != http.StatusOK { + t.Fatalf("valid query %s: %d %v", query, response.StatusCode, err) + } + } + for _, query := range []string{"?page=bad"} { + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/"+query, nil)) + if err != nil || response.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid query %s: %d %v", query, response.StatusCode, err) + } + } +} + +func TestCreatorControlPlaneGuards(t *testing.T) { + ctx := context.Background() + if _, err := persistDouyinMessageHistory(ctx, nil, creator.Conversation{Platform: creator.PlatformXiaohongshu}, "uid", nil); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("invalid history store/platform = %v", err) + } + conversation := creator.Conversation{Platform: creator.PlatformDouyin, AccountID: "account", PeerUID: "peer"} + cases := []douyinHistoryMessage{ + {}, + {ServerID: "id", SenderUID: "sender", Content: []byte("not-json")}, + {ServerID: "id", SenderUID: "sender", CreatedAt: "not-a-time"}, + } + for _, item := range cases { + if _, err := persistDouyinMessageHistory(ctx, nil, conversation, "account", []douyinHistoryMessage{item}); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("invalid history item %v = %v", item, err) + } + } + if _, err := (creatorGatewayActionExecutor{}).Execute(ctx, creator.ActionRequest{}); !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("empty action executor = %v", err) + } + if err := (creatorMaterialDownloader{}).Download(ctx, creator.Work{Platform: "unsupported"}, "/tmp/media"); !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("unsupported material platform = %v", err) + } + if _, err := newXiaohongshuReadCollector(ctx, nil, nil, nil, "", creator.SourceOwned, "id"); !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("empty Xiaohongshu collector = %v", err) + } + if listenerBoundaryPointer(time.Time{}) != nil { + t.Fatal("zero listener boundary should be nil") + } +} + +func TestCreatorSchedulerAndPreviewGuards(t *testing.T) { + ctx := context.Background() + checks := []struct { + name string + call func() error + }{ + {"verify account", func() error { _, err := verifyCreatorAccount(ctx, nil, nil, nil, "account"); return err }}, + {"preview competitor", func() error { + _, err := previewDouyinCompetitor(ctx, nil, nil, nil, "account", creator.CompetitorInput{}) + return err + }}, + {"sync competitor", func() error { _, err := syncCreatorCompetitor(ctx, nil, nil, nil, "competitor", "account"); return err }}, + {"sync due competitor", func() error { + _, err := syncCreatorCompetitorDue(ctx, nil, nil, nil, "competitor", "account") + return err + }}, + {"sync owned", func() error { return syncCreatorOwned(ctx, nil, nil, nil, "account", time.Now()) }}, + {"select collection account", func() error { + _, err := creatorCollectionAccount(ctx, nil, nil, nil, creator.PlatformDouyin) + return err + }}, + } + for _, check := range checks { + if err := check.call(); err == nil { + t.Fatalf("%s unexpectedly succeeded", check.name) + } + } +} + +func TestCreatorGatewayBrowserHistoryAndMedia(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case strings.HasSuffix(r.URL.Path, "/douyin/messages"): + _, _ = w.Write([]byte(`{"status":"succeeded","account_uid":"account","history_source":"douyin","messages":[]}`)) + case strings.HasSuffix(r.URL.Path, "/douyin/media"): + _, _ = w.Write([]byte(`{"status":200,"content_type":"video/mp4","body_base64":"` + base64.StdEncoding.EncodeToString([]byte("video")) + `"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + browser := creatorGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 1}} + history, err := browser.MessageHistory(context.Background(), "account", "peer", "", 10) + if err != nil || history.Status != "succeeded" || history.HistorySource != "douyin" { + t.Fatalf("history = %+v, %v", history, err) + } + if _, err := browser.MessageHistory(context.Background(), "", "peer", "", 10); err == nil { + t.Fatal("empty history identity must fail") + } + path := filepath.Join(t.TempDir(), "media.mp4") + if err := browser.Media(context.Background(), "https://video.example/media", path); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(path); err != nil || string(data) != "video" { + t.Fatalf("media output = %q, %v", data, err) + } +} + +func TestXiaohongshuGatewayMedia(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if !strings.HasSuffix(r.URL.Path, "/xiaohongshu/media") { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`{"status":200,"content_type":"image/jpeg","body_base64":"` + base64.StdEncoding.EncodeToString([]byte("image")) + `"}`)) + })) + defer server.Close() + browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, AccountID: "account", AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "binding", RuntimeInstanceID: "instance", RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit", HealthStatus: "healthy"}}} + data, contentType, err := browser.Media(context.Background(), "https://www.xiaohongshu.com/explore/abc") + if err != nil || string(data) != "image" || contentType != "image/jpeg" { + t.Fatalf("media = %q, %q, %v", data, contentType, err) + } +} + +func TestCreatorGatewayBrowserIdentity(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.Contains(r.URL.Path, "/douyin/identity") { + t.Fatalf("unexpected identity request: %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"uid":"verified-uid"}`)) + })) + defer server.Close() + browser := creatorGatewayBrowser{ + gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, + environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 1}, + } + uid, err := browser.Identity(context.Background(), "expected-key") + if err != nil || uid != "verified-uid" { + t.Fatalf("identity = %q, %v", uid, err) + } +} diff --git a/cmd/control-plane/creator_history_test.go b/cmd/control-plane/creator_history_test.go new file mode 100644 index 0000000..902f5ad --- /dev/null +++ b/cmd/control-plane/creator_history_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strconv" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/phasea" +) + +func TestPersistDouyinMessageHistory(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run history coverage") + } + ctx := context.Background() + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + phaseAStore, err := phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = phaseAStore.Close() }) + hubStore, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hubStore.Close() }) + store, err := creator.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + credentials := &testCredentialBridge{values: make(map[string]string)} + if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + ID: "history-account", Name: "History Account", Platform: creator.PlatformDouyin, + PlatformAccountKey: "history-platform", Tags: []string{}, Cookies: "", + CredentialReference: phasea.CredentialReference{ID: "history-credential", Provider: "os_keyring"}, + CredentialKey: "creatorhub/history-account/cookies", + }, credentials); err != nil { + t.Fatal(err) + } + if err := store.EnsureAccountProfile(ctx, "history-account"); err != nil { + t.Fatal(err) + } + conversation := creator.Conversation{Platform: creator.PlatformDouyin, AccountID: "history-account", PeerUID: "peer", PeerName: "Peer"} + stamp := time.Now().UTC().UnixMilli() + messages := []douyinHistoryMessage{ + {ServerID: "history-inbound", SenderUID: "peer", Content: json.RawMessage(`{"text":"inbound"}`), CreatedAt: strconv.FormatInt(stamp, 10)}, + {ServerID: "history-outbound", SenderUID: "history-account", Content: json.RawMessage(`{"text":"outbound"}`), CreatedAt: strconv.FormatInt(stamp+1, 10)}, + } + inserted, err := persistDouyinMessageHistory(ctx, store, conversation, "history-account", messages) + if err != nil || inserted != 2 { + t.Fatalf("persist history: inserted=%d err=%v", inserted, err) + } + inserted, err = persistDouyinMessageHistory(ctx, store, conversation, "history-account", messages) + if err != nil || inserted != 0 { + t.Fatalf("deduplicate history: inserted=%d err=%v", inserted, err) + } + badContent := []douyinHistoryMessage{{ServerID: "bad", SenderUID: "peer", Content: json.RawMessage(`{"text":`), CreatedAt: strconv.FormatInt(stamp, 10)}} + if _, err := persistDouyinMessageHistory(ctx, store, conversation, "history-account", badContent); err != creator.ErrInvalid { + t.Fatalf("bad history content: %v", err) + } +} diff --git a/cmd/control-plane/creator_material.go b/cmd/control-plane/creator_material.go index 058f754..68313e0 100644 --- a/cmd/control-plane/creator_material.go +++ b/cmd/control-plane/creator_material.go @@ -2,6 +2,8 @@ package main import ( "context" + "crypto/rand" + "encoding/hex" "fmt" "os" "os/exec" @@ -68,7 +70,7 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c } func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, workID string) (creator.MaterialJob, error) { - if store == nil || workID == "" || filepath.Base(workID) != workID { + if store == nil || workID == "" || workID == "." || workID == ".." || strings.ContainsAny(workID, `/\\`) || filepath.Base(workID) != workID { return creator.MaterialJob{}, creator.ErrInvalid } work, err := store.GetWork(ctx, workID) @@ -94,13 +96,26 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto videoPath := filepath.Join(dir, "source") videoReference := filepath.Join(workID, "source") if job.DownloadStatus != "succeeded" || !fileExists(videoPath) { - if _, err := store.SetMaterialStep(ctx, workID, "download", "running", "", ""); err != nil { + if job.DownloadStatus == "succeeded" { + if _, err := store.SetMaterialStep(ctx, workID, "download", "failed", "", "下载产物不存在"); err != nil { + return creator.MaterialJob{}, err + } + } + token, tokenErr := materialClaimToken() + if tokenErr != nil { + return creator.MaterialJob{}, tokenErr + } + job, claimed, err := store.ClaimMaterialStep(ctx, workID, "download", token) + if err != nil { return creator.MaterialJob{}, err } - if err := (creatorMaterialDownloader{store: store, phaseAStore: phaseAStore, hubStore: hubStore}).Download(ctx, work, videoPath); err != nil { - return setMaterialFailure(ctx, store, workID, "download", err) + if !claimed { + return job, fmt.Errorf("%w: download step is already in progress", creator.ErrConflict) } - job, err = store.SetMaterialStep(ctx, workID, "download", "succeeded", videoReference, "") + if err := (creatorMaterialDownloader{store: store, phaseAStore: phaseAStore, hubStore: hubStore}).Download(ctx, work, videoPath); err != nil { + return setMaterialFailure(ctx, store, workID, "download", token, err) + } + job, err = store.CompleteMaterialStep(ctx, workID, "download", token, "succeeded", videoReference, "") if err != nil { return creator.MaterialJob{}, err } @@ -108,53 +123,72 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto audioPath := filepath.Join(dir, "audio.wav") audioReference := filepath.Join(workID, "audio.wav") - if job.AudioStatus != "succeeded" && job.AudioStatus != "no_audio" { - if _, err := store.SetMaterialStep(ctx, workID, "audio", "running", "", ""); err != nil { + if job.AudioStatus != "succeeded" && job.AudioStatus != "no_audio" || job.AudioStatus == "succeeded" && !fileExists(audioPath) { + if job.AudioStatus == "succeeded" { + if _, err := store.SetMaterialStep(ctx, workID, "audio", "failed", "", "音频产物不存在"); err != nil { + return creator.MaterialJob{}, err + } + } + token, tokenErr := materialClaimToken() + if tokenErr != nil { + return creator.MaterialJob{}, tokenErr + } + job, claimed, err := store.ClaimMaterialStep(ctx, workID, "audio", token) + if err != nil { return creator.MaterialJob{}, err } + if !claimed { + return job, fmt.Errorf("%w: audio step is already in progress", creator.ErrConflict) + } hasAudio, err := creatorMaterialHasAudio(ctx, videoPath) if err != nil { - return setMaterialFailure(ctx, store, workID, "audio", err) + return setMaterialFailure(ctx, store, workID, "audio", token, err) } if !hasAudio { - job, err = store.SetMaterialStep(ctx, workID, "audio", "no_audio", "", "视频没有音轨") - if err != nil { - return creator.MaterialJob{}, err - } + job, err = store.CompleteMaterialStep(ctx, workID, "audio", token, "no_audio", "", "视频没有音轨") } else if err := extractCreatorAudio(ctx, videoPath, audioPath); err != nil { - return setMaterialFailure(ctx, store, workID, "audio", err) + return setMaterialFailure(ctx, store, workID, "audio", token, err) } else { - job, err = store.SetMaterialStep(ctx, workID, "audio", "succeeded", audioReference, "") - if err != nil { - return creator.MaterialJob{}, err - } + job, err = store.CompleteMaterialStep(ctx, workID, "audio", token, "succeeded", audioReference, "") + } + if err != nil { + return creator.MaterialJob{}, err } } if job.TranscriptionStatus != "succeeded" && job.TranscriptionStatus != "no_speech" { + token, tokenErr := materialClaimToken() + if tokenErr != nil { + return creator.MaterialJob{}, tokenErr + } + job, claimed, claimErr := store.ClaimMaterialStep(ctx, workID, "transcription", token) + if claimErr != nil { + return creator.MaterialJob{}, claimErr + } + if !claimed { + return job, fmt.Errorf("%w: transcription step is already in progress", creator.ErrConflict) + } if job.AudioStatus == "no_audio" { - job, err = store.SetMaterialStep(ctx, workID, "transcription", "no_speech", "", "没有可转写的音轨") + job, err = store.CompleteMaterialStep(ctx, workID, "transcription", token, "no_speech", "", "没有可转写的音轨") } else { settings, settingsErr := store.GetSettings(ctx) if settingsErr != nil { - return creator.MaterialJob{}, settingsErr + return setMaterialFailure(ctx, store, workID, "transcription", token, settingsErr) } if !settings.TranscriptionConfigured || strings.TrimSpace(settings.TranscriptionProvider) == "" || strings.TrimSpace(settings.TranscriptionModel) == "" { - return setMaterialFailure(ctx, store, workID, "transcription", fmt.Errorf("transcription provider is not configured")) + return setMaterialFailure(ctx, store, workID, "transcription", token, fmt.Errorf("transcription provider is not configured")) } - if _, err = store.SetMaterialStep(ctx, workID, "transcription", "running", "", ""); err == nil { - transcript, transcribeErr := transcribeCreatorAudio(ctx, audioPath, settings.TranscriptionModel) - if transcribeErr != nil { - job, err = setMaterialFailure(ctx, store, workID, "transcription", transcribeErr) - } else if strings.TrimSpace(transcript) == "" { - job, err = store.SetMaterialStep(ctx, workID, "transcription", "no_speech", "", "转写未检测到语音") + transcript, transcribeErr := transcribeCreatorAudio(ctx, audioPath, settings.TranscriptionProvider, settings.TranscriptionModel) + if transcribeErr != nil { + job, err = setMaterialFailure(ctx, store, workID, "transcription", token, transcribeErr) + } else if strings.TrimSpace(transcript) == "" { + job, err = store.CompleteMaterialStep(ctx, workID, "transcription", token, "no_speech", "", "转写未检测到语音") + } else { + transcriptPath := filepath.Join(dir, "transcript.txt") + if writeErr := os.WriteFile(transcriptPath, []byte(transcript), 0o600); writeErr != nil { + job, err = setMaterialFailure(ctx, store, workID, "transcription", token, writeErr) } else { - transcriptPath := filepath.Join(dir, "transcript.txt") - if writeErr := os.WriteFile(transcriptPath, []byte(transcript), 0o600); writeErr != nil { - job, err = setMaterialFailure(ctx, store, workID, "transcription", writeErr) - } else { - job, err = store.SetMaterialStep(ctx, workID, "transcription", "succeeded", filepath.Join(workID, "transcript.txt"), "") - } + job, err = store.CompleteMaterialStep(ctx, workID, "transcription", token, "succeeded", transcript, "") } } } @@ -165,6 +199,14 @@ func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseASto return job, nil } +func materialClaimToken() (string, error) { + var data [16]byte + if _, err := rand.Read(data[:]); err != nil { + return "", fmt.Errorf("create material claim token: %w", err) + } + return hex.EncodeToString(data[:]), nil +} + func creatorMaterialHasAudio(ctx context.Context, videoPath string) (bool, error) { if _, err := exec.LookPath("ffprobe"); err != nil { return false, fmt.Errorf("ffprobe is unavailable: %w", err) @@ -183,7 +225,7 @@ func extractCreatorAudio(ctx context.Context, videoPath, audioPath string) error } temporary := audioPath + ".tmp" defer os.Remove(temporary) - command := exec.CommandContext(ctx, "ffmpeg", "-nostdin", "-v", "error", "-y", "-i", videoPath, "-vn", "-ac", "1", "-ar", "16000", temporary) + command := exec.CommandContext(ctx, "ffmpeg", "-nostdin", "-v", "error", "-y", "-i", videoPath, "-vn", "-ac", "1", "-ar", "16000", "-f", "wav", temporary) if output, err := command.CombinedOutput(); err != nil { return fmt.Errorf("extract audio: %w: %s", err, strings.TrimSpace(string(output))) } @@ -207,25 +249,23 @@ func validateTranscriptionBinary(binary string) error { return nil } -func transcribeCreatorAudio(ctx context.Context, audioPath, model string) (string, error) { - model = strings.TrimSpace(model) - if model == "" { - return "", fmt.Errorf("transcription model is not configured") +func transcribeCreatorAudio(ctx context.Context, audioPath, provider, model string) (string, error) { + provider, model = strings.TrimSpace(provider), strings.TrimSpace(model) + if provider == "" || model == "" { + return "", fmt.Errorf("transcription provider and model are not configured") + } + if provider != "whisper" && provider != "faster-whisper" { + return "", fmt.Errorf("unsupported transcription provider %q", provider) } configured := strings.TrimSpace(os.Getenv("CREATOR_TRANSCRIPTION_BIN")) - var binary string - switch filepath.Base(configured) { - case "whisper": - binary = "whisper" - case "faster-whisper": - binary = "faster-whisper" - default: - return "", fmt.Errorf("transcription provider is not configured or unsupported") + if configured == "" || filepath.Base(configured) != provider { + return "", fmt.Errorf("transcription binary does not match configured provider %q", provider) } - if err := validateTranscriptionBinary(binary); err != nil { + if err := validateTranscriptionBinary(configured); err != nil { return "", fmt.Errorf("transcription provider is unavailable: %w", err) } - output, err := exec.CommandContext(ctx, binary, audioPath, "--model", model).Output() + command := exec.CommandContext(ctx, configured, audioPath, "--model", model) + output, err := command.Output() if err != nil { return "", fmt.Errorf("transcribe audio: %w", err) } @@ -235,8 +275,8 @@ func transcribeCreatorAudio(ctx context.Context, audioPath, model string) (strin return string(output), nil } -func setMaterialFailure(ctx context.Context, store *creator.Store, workID, step string, cause error) (creator.MaterialJob, error) { - job, err := store.SetMaterialStep(ctx, workID, step, "failed", "", cause.Error()) +func setMaterialFailure(ctx context.Context, store *creator.Store, workID, step, token string, cause error) (creator.MaterialJob, error) { + job, err := store.CompleteMaterialStep(ctx, workID, step, token, "failed", "", cause.Error()) if err != nil { return creator.MaterialJob{}, fmt.Errorf("record %s failure: %w", step, err) } diff --git a/cmd/control-plane/creator_material_test.go b/cmd/control-plane/creator_material_test.go index 0956f60..2292a87 100644 --- a/cmd/control-plane/creator_material_test.go +++ b/cmd/control-plane/creator_material_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" ) @@ -38,9 +39,54 @@ func TestWriteCreatorMediaRejectsEmptyAndOversizedFiles(t *testing.T) { } } +func TestExtractCreatorAudioPublishesWavAtomically(t *testing.T) { + dir := t.TempDir() + fakeFFmpeg := filepath.Join(dir, "ffmpeg") + argsFile := filepath.Join(dir, "args") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$ARGS_FILE\"\nfor arg in \"$@\"; do output=\"$arg\"; done\nprintf 'RIFFfake' > \"$output\"\n" + if err := os.WriteFile(fakeFFmpeg, []byte(script), 0o700); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("ARGS_FILE", argsFile) + videoPath := filepath.Join(dir, "source") + audioPath := filepath.Join(dir, "audio.wav") + if err := os.WriteFile(videoPath, []byte("video"), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + if err := extractCreatorAudio(context.Background(), videoPath, audioPath); err != nil { + t.Fatalf("extract audio: %v", err) + } + if !fileExists(audioPath) { + t.Fatal("expected published audio") + } + if _, err := os.Stat(audioPath + ".tmp"); !os.IsNotExist(err) { + t.Fatalf("temporary audio remains: %v", err) + } + args, err := os.ReadFile(argsFile) + if err != nil { + t.Fatalf("read ffmpeg args: %v", err) + } + if !strings.Contains(string(args), "-f\n") || !strings.Contains(string(args), "\nwav\n") { + t.Fatalf("ffmpeg did not request WAV output: %s", args) + } +} + func TestTranscribeCreatorAudioRequiresExplicitProvider(t *testing.T) { t.Setenv("CREATOR_TRANSCRIPTION_BIN", "") - if _, err := transcribeCreatorAudio(context.Background(), "/tmp/audio.wav", "base"); err == nil { + if _, err := transcribeCreatorAudio(context.Background(), "/tmp/audio.wav", "whisper", "base"); err == nil { t.Fatal("expected missing provider error") } } + +func TestTranscribeCreatorAudioUsesConfiguredBinaryPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "whisper") + if err := os.WriteFile(path, []byte("#!/bin/sh\nprintf configured-transcript\n"), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("CREATOR_TRANSCRIPTION_BIN", path) + transcript, err := transcribeCreatorAudio(context.Background(), "/tmp/audio.wav", "whisper", "base") + if err != nil || transcript != "configured-transcript" { + t.Fatalf("configured transcription: transcript=%q err=%v", transcript, err) + } +} diff --git a/cmd/control-plane/credential_test.go b/cmd/control-plane/credential_test.go index 5dfd3af..5f52c2f 100644 --- a/cmd/control-plane/credential_test.go +++ b/cmd/control-plane/credential_test.go @@ -58,6 +58,34 @@ func TestPersistentCredentialBridgeStoreFailureLeavesNoFile(t *testing.T) { } } +func TestPersistentCredentialBridgeResolvesAndRejectsCorruption(t *testing.T) { + bridge, err := newPersistentCredentialBridge(t.TempDir(), []byte("0123456789abcdef0123456789abcdef")) + if err != nil { + t.Fatal(err) + } + key := "creatorhub/account-a/cookies" + if err := bridge.Store(context.Background(), testCredentialReference, key, testCredentialValue); err != nil { + t.Fatal(err) + } + value, err := bridge.Resolve(context.Background(), testCredentialReference, key) + if err != nil || string(value) != testCredentialValue { + t.Fatalf("resolve credential: %q %v", value, err) + } + path := bridge.path(testCredentialReference.Provider, key) + if err := os.WriteFile(path, []byte("corrupted"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := bridge.Resolve(context.Background(), testCredentialReference, key); err == nil { + t.Fatal("corrupted credential resolved") + } + if err := bridge.Delete(context.Background(), testCredentialReference, key); err != nil { + t.Fatal(err) + } + if err := bridge.Delete(context.Background(), testCredentialReference, key); err != nil { + t.Fatal(err) + } +} + func TestPersistentCredentialBridgeStoresEncryptedCredentialAcrossProcesses(t *testing.T) { if action := os.Getenv("CREATORHUB_CREDENTIAL_HELPER_ACTION"); action != "" { bridge, err := newPersistentCredentialBridge(os.Getenv("CREATORHUB_CREDENTIAL_HELPER_DIR"), []byte("0123456789abcdef0123456789abcdef")) diff --git a/cmd/control-plane/douyin.go b/cmd/control-plane/douyin.go index a52a420..acf5012 100644 --- a/cmd/control-plane/douyin.go +++ b/cmd/control-plane/douyin.go @@ -61,6 +61,7 @@ type douyinHistoryMessage struct { type douyinMessageHistory struct { Status string `json:"status"` HistorySource string `json:"history_source"` + HistoryCursor string `json:"history_cursor"` HistoryHasMore bool `json:"history_has_more"` AccountUID string `json:"account_uid"` Conversation map[string]any `json:"conversation"` diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index ed78e42..db29b7e 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -51,6 +51,12 @@ type hubStore interface { AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error } +type networkExitAdminStore interface { + UpdateNetworkExit(context.Context, string, hub.NetworkExit) (hub.NetworkExit, error) + EnableNetworkExit(context.Context, string) (hub.NetworkExit, error) + DeleteNetworkExit(context.Context, string) error +} + type runtimeStopStore interface { LockResources(ctx context.Context, aliases, exitIDs, imageVersions []string) (func(), error) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) @@ -432,6 +438,42 @@ func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitPro app.Post("/api/network-exits", createNetworkExit(store)) app.Post("/api/network-exits/:id/check", checkNetworkExit(store, probe)) app.Post("/api/network-exits/:id/disable", disableNetworkExit(store, probe, resolve)) + app.Put("/api/network-exits/:id", func(c fiber.Ctx) error { + admin, ok := store.(networkExitAdminStore) + if !ok { + return hubError(c, hub.ErrConflict) + } + input := hub.NetworkExit{} + if err := decodeHubJSON(c, &input); err != nil { + return hubError(c, err) + } + exit, err := admin.UpdateNetworkExit(c.Context(), c.Params("id"), input) + if err != nil { + return hubError(c, err) + } + return c.JSON(exit) + }) + app.Post("/api/network-exits/:id/enable", func(c fiber.Ctx) error { + admin, ok := store.(networkExitAdminStore) + if !ok { + return hubError(c, hub.ErrConflict) + } + exit, err := admin.EnableNetworkExit(c.Context(), c.Params("id")) + if err != nil { + return hubError(c, err) + } + return c.JSON(exit) + }) + app.Delete("/api/network-exits/:id", func(c fiber.Ctx) error { + admin, ok := store.(networkExitAdminStore) + if !ok { + return hubError(c, hub.ErrConflict) + } + if err := admin.DeleteNetworkExit(c.Context(), c.Params("id")); err != nil { + return hubError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) + }) app.Get("/api/browser-images", func(c fiber.Ctx) error { images, err := store.ListImages(c.Context(), false) diff --git a/cmd/control-plane/main_test.go b/cmd/control-plane/main_test.go index 59386c4..b4bebd9 100644 --- a/cmd/control-plane/main_test.go +++ b/cmd/control-plane/main_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "git.ipao.vip/rogee/creator-hub/internal/creator" "git.ipao.vip/rogee/creator-hub/internal/hub" "git.ipao.vip/rogee/creator-hub/internal/phasea" "git.ipao.vip/rogee/creator-hub/internal/taskstate" @@ -115,6 +116,398 @@ func TestLoadConfigRequiresCredentialMasterKey(t *testing.T) { } } +func TestCreatorReadRoutesAgainstPostgres(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run creator route coverage") + } + ctx := context.Background() + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + phaseAStore, err := phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = phaseAStore.Close() }) + hubStore, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hubStore.Close() }) + creatorStore, err := creator.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = creatorStore.Close() }) + app := newHandlerWithCreator(t.TempDir(), "operator", "unit-test-password", phaseAStore, hubStore, nil, creatorStore) + for _, path := range []string{ + "/api/creator/settings", "/api/creator/accounts", "/api/creator/competitors", "/api/creator/relations", + "/api/creator/accounts/route-account/strategies", "/api/creator/rules", "/api/creator/rule-results", "/api/creator/leads", + "/api/creator/works", "/api/creator/works?page=1&page_size=10", "/api/creator/comments", "/api/creator/comments?page=1&page_size=10", + "/api/creator/listener-boundaries", "/api/creator/events", "/api/creator/events?page=1&page_size=10", + "/api/creator/listeners", "/api/creator/operations", "/api/creator/conversations", + } { + response := do(app, http.MethodGet, path, "", "operator", "unit-test-password") + if response.Code != http.StatusOK { + t.Fatalf("GET %s returned %d: %s", path, response.Code, response.Body.String()) + } + } +} + +func TestCreatorRouteValidationCoverage(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run creator route coverage") + } + ctx := context.Background() + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + phaseAStore, err := phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = phaseAStore.Close() }) + hubStore, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hubStore.Close() }) + creatorStore, err := creator.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = creatorStore.Close() }) + credentials := &testCredentialBridge{values: make(map[string]string)} + if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + ID: "route-account", Name: "Route Account", Platform: creator.PlatformDouyin, + PlatformAccountKey: "route-platform", Tags: []string{}, Cookies: "", + CredentialReference: phasea.CredentialReference{ID: "route-account-credential", Provider: "os_keyring"}, + CredentialKey: "creatorhub/route-account/cookies", + }, credentials); err != nil { + t.Fatal(err) + } + app := newHandlerWithCreator(t.TempDir(), "operator", "unit-test-password", phaseAStore, hubStore, nil, creatorStore) + for _, path := range []string{ + "/api/creator/accounts/route-account/profile", "/api/creator/accounts/route-account/strategies", + } { + response := do(app, http.MethodGet, path, "", "operator", "unit-test-password") + if response.Code != http.StatusOK { + t.Fatalf("GET %s returned %d: %s", path, response.Code, response.Body.String()) + } + } + for _, path := range []string{ + "/api/creator/works/missing/metrics", "/api/creator/rule-results?comment_id=missing", + "/api/creator/events/missing/strategy-trace", "/api/creator/conversations/missing/messages", + } { + response := do(app, http.MethodGet, path, "", "operator", "unit-test-password") + if response.Code != http.StatusOK { + t.Fatalf("GET %s returned %d: %s", path, response.Code, response.Body.String()) + } + } + for _, path := range []string{ + "/api/creator/competitors/missing", "/api/creator/works/missing", + "/api/creator/comments/missing", "/api/creator/rules/missing", + "/api/creator/operations/missing", "/api/creator/operations/missing/verification", + } { + response := do(app, http.MethodGet, path, "", "operator", "unit-test-password") + if response.Code != http.StatusNotFound { + t.Fatalf("GET %s returned %d: %s", path, response.Code, response.Body.String()) + } + } + response := do(app, http.MethodGet, "/api/creator/works/missing/material", "", "operator", "unit-test-password") + if response.Code != http.StatusBadRequest { + t.Fatalf("GET material for missing work returned %d: %s", response.Code, response.Body.String()) + } + for _, route := range []struct { + method string + path string + }{ + {http.MethodPut, "/api/creator/settings"}, + {http.MethodPut, "/api/creator/accounts/missing/profile"}, + {http.MethodPost, "/api/creator/accounts/missing/login-result"}, + {http.MethodPost, "/api/creator/accounts/missing/big-account"}, + {http.MethodPost, "/api/creator/relations"}, + {http.MethodPost, "/api/creator/accounts/missing/strategies"}, + {http.MethodPost, "/api/creator/strategies/missing/enable"}, + {http.MethodPost, "/api/creator/strategies/missing/disable"}, + {http.MethodDelete, "/api/creator/strategies/missing"}, + {http.MethodPut, "/api/creator/strategies/missing"}, + {http.MethodPost, "/api/creator/competitors/preview"}, + {http.MethodPost, "/api/creator/competitors"}, + {http.MethodPost, "/api/creator/competitors/missing/pause"}, + {http.MethodPost, "/api/creator/competitors/missing/resume"}, + {http.MethodPost, "/api/creator/competitors/missing/sync"}, + {http.MethodPost, "/api/creator/xiaohongshu/search"}, + {http.MethodPost, "/api/creator/xiaohongshu/detail"}, + {http.MethodPost, "/api/creator/works/missing/metrics"}, + {http.MethodPost, "/api/creator/works/missing/material/select"}, + {http.MethodPost, "/api/creator/works/missing/material/process"}, + {http.MethodPost, "/api/creator/works/missing/material/rewrite/confirm"}, + {http.MethodPost, "/api/creator/works/missing/material/rewrite/generate"}, + {http.MethodPut, "/api/creator/works/missing/material/rewrite"}, + {http.MethodPost, "/api/creator/rules"}, + {http.MethodPut, "/api/creator/rules/missing"}, + {http.MethodPost, "/api/creator/rules/missing/enable"}, + {http.MethodPost, "/api/creator/rules/missing/disable"}, + {http.MethodPost, "/api/creator/comments/analyze"}, + {http.MethodPost, "/api/creator/comments/missing/analyze"}, + {http.MethodPost, "/api/creator/events/missing/display"}, + {http.MethodPost, "/api/creator/operations"}, + {http.MethodPost, "/api/creator/operations/missing/execute"}, + {http.MethodPost, "/api/creator/conversations/missing/sync"}, + {http.MethodPost, "/api/creator/messages/send"}, + } { + response := do(app, route.method, route.path, "{", "operator", "unit-test-password") + if response.Code != http.StatusBadRequest && response.Code != http.StatusNotFound && response.Code != http.StatusNoContent && response.Code != http.StatusServiceUnavailable { + t.Fatalf("%s %s returned %d for invalid JSON: %s", route.method, route.path, response.Code, response.Body.String()) + } + } + for _, path := range []string{ + "/api/creator/works", "/api/creator/comments", "/api/creator/events", "/api/creator/events/process", "/api/creator/messages", + } { + response := do(app, http.MethodPost, path, `{}`, "operator", "unit-test-password") + if response.Code != http.StatusConflict { + t.Fatalf("POST %s accepted public platform input: %d", path, response.Code) + } + } +} + +func TestCreatorFixtureRoutesPostgres(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run creator fixture coverage") + } + ctx := context.Background() + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + phaseAStore, err := phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = phaseAStore.Close() }) + hubStore, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hubStore.Close() }) + creatorStore, err := creator.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = creatorStore.Close() }) + credentials := &testCredentialBridge{values: make(map[string]string)} + if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + ID: "fixture-account", Name: "Fixture Account", Platform: creator.PlatformDouyin, + PlatformAccountKey: "fixture-platform", Tags: []string{}, Cookies: "", + CredentialReference: phasea.CredentialReference{ID: "fixture-credential", Provider: "os_keyring"}, + CredentialKey: "creatorhub/fixture-account/cookies", + }, credentials); err != nil { + t.Fatal(err) + } + if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + ID: "fixture-small", Name: "Fixture Small", Platform: creator.PlatformDouyin, + PlatformAccountKey: "fixture-small-platform", Tags: []string{}, Cookies: "", + CredentialReference: phasea.CredentialReference{ID: "fixture-small-credential", Provider: "os_keyring"}, + CredentialKey: "creatorhub/fixture-small/cookies", + }, credentials); err != nil { + t.Fatal(err) + } + if err := creatorStore.EnsureAccountProfile(ctx, "fixture-small"); err != nil { + t.Fatal(err) + } + app := newHandlerWithCreator(t.TempDir(), "operator", "unit-test-password", phaseAStore, hubStore, nil, creatorStore) + idFrom := func(response *httptest.ResponseRecorder) string { + var value struct { + ID string `json:"id"` + } + if err := json.Unmarshal(response.Body.Bytes(), &value); err != nil || value.ID == "" { + t.Fatalf("response has no ID: %s (%v)", response.Body.String(), err) + } + return value.ID + } + workBody := `{"platform":"douyin","work_key":"fixture-work","source_type":"owned","source_id":"fixture-account","author_name":"author","title":"title","body":"body","published_at":"2024-01-01T00:00:00Z","published_at_status":"verified"}` + workResponse := do(app, http.MethodPost, "/api/creator/test/works", workBody, "operator", "unit-test-password") + if workResponse.Code != http.StatusCreated { + t.Fatalf("create fixture work: %d %s", workResponse.Code, workResponse.Body.String()) + } + workID := idFrom(workResponse) + if response := do(app, http.MethodPost, "/api/creator/test/works", workBody, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("deduplicate fixture work: %d %s", response.Code, response.Body.String()) + } + for _, path := range []string{"/api/creator/works/" + workID, "/api/creator/works/" + workID + "/metrics", "/api/creator/works/" + workID + "/material"} { + if response := do(app, http.MethodGet, path, "", "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("GET %s: %d %s", path, response.Code, response.Body.String()) + } + } + metricBody := `{"likes":2,"comments_count":1,"shares":1}` + if response := do(app, http.MethodPost, "/api/creator/works/"+workID+"/metrics", metricBody, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("record fixture metric: %d %s", response.Code, response.Body.String()) + } + materialResponse := do(app, http.MethodPost, "/api/creator/works/"+workID+"/material/select", `{}`, "operator", "unit-test-password") + if materialResponse.Code != http.StatusCreated && materialResponse.Code != http.StatusOK { + t.Fatalf("select fixture material: %d %s", materialResponse.Code, materialResponse.Body.String()) + } + if _, claimed, err := creatorStore.ClaimMaterialStep(ctx, workID, "download", "fixture-material-token"); err != nil || !claimed { + t.Fatalf("claim fixture material download: claimed=%v err=%v", claimed, err) + } + if _, err := setMaterialFailure(ctx, creatorStore, workID, "download", "fixture-material-token", errors.New("fixture download failed")); err != nil { + t.Fatalf("record fixture material failure: %v", err) + } + commentBody := `{"platform":"douyin","comment_key":"fixture-comment","work_id":"` + workID + `","author_uid":"peer","author_name":"Peer","content":"hello","comment_type":"top_level","published_at":"2024-01-01T00:00:00Z"}` + commentResponse := do(app, http.MethodPost, "/api/creator/test/comments", commentBody, "operator", "unit-test-password") + if commentResponse.Code != http.StatusCreated { + t.Fatalf("create fixture comment: %d %s", commentResponse.Code, commentResponse.Body.String()) + } + commentID := idFrom(commentResponse) + if response := do(app, http.MethodPost, "/api/creator/test/comments", commentBody, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("deduplicate fixture comment: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodGet, "/api/creator/comments/"+commentID, "", "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("GET fixture comment: %d %s", response.Code, response.Body.String()) + } + profileBody := `{"real_name_status":"unknown","business_status":"normal","cooldown_seconds":60}` + if response := do(app, http.MethodPut, "/api/creator/accounts/fixture-account/profile", profileBody, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("update fixture profile: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/creator/accounts/fixture-account/big-account", `{"enabled":true}`, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("enable fixture big account: %d %s", response.Code, response.Body.String()) + } + for _, accountID := range []string{"fixture-account", "fixture-small"} { + if _, err := creatorStore.UpdateAccountProfile(ctx, accountID, creator.AccountProfileUpdate{RealNameStatus: "unknown", BusinessStatus: "normal", BigAccount: accountID == "fixture-account", CooldownSeconds: 60}); err != nil { + t.Fatal(err) + } + } + if _, err := creatorStore.RecordVerifiedLoginResult(ctx, "fixture-account", "fixture-platform"); err != nil { + t.Fatal(err) + } + if _, err := creatorStore.RecordVerifiedLoginResult(ctx, "fixture-small", "fixture-small-platform"); err != nil { + t.Fatal(err) + } + if err := creatorStore.SetRelation(ctx, "fixture-account", "fixture-small", true); err != nil { + t.Fatal(err) + } + strategyBody := `{"execution_account_id":"fixture-small","position":1,"enabled":true,"event_types":["comment"],"action":"reply_comment","target_type":"comment","candidate_texts":["已收到"]}` + strategyResponse := do(app, http.MethodPost, "/api/creator/accounts/fixture-account/strategies", strategyBody, "operator", "unit-test-password") + if strategyResponse.Code != http.StatusCreated { + t.Fatalf("create fixture strategy: %d %s", strategyResponse.Code, strategyResponse.Body.String()) + } + strategyID := idFrom(strategyResponse) + for _, route := range []struct { + method string + path string + body string + }{ + {http.MethodPut, "/api/creator/strategies/" + strategyID, strategyBody}, + {http.MethodPost, "/api/creator/strategies/" + strategyID + "/enable", `{}`}, + {http.MethodPost, "/api/creator/strategies/" + strategyID + "/disable", `{}`}, + } { + if response := do(app, route.method, route.path, route.body, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("%s %s: %d %s", route.method, route.path, response.Code, response.Body.String()) + } + } + if response := do(app, http.MethodDelete, "/api/creator/strategies/"+strategyID, "", "operator", "unit-test-password"); response.Code != http.StatusNoContent { + t.Fatalf("delete fixture strategy: %d %s", response.Code, response.Body.String()) + } + ruleBody := `{"name":"fixture-rule","enabled":true,"source_type":"owned","topic":"title","include_keywords":["hello"],"ai_requirement":"lead"}` + ruleResponse := do(app, http.MethodPost, "/api/creator/rules", ruleBody, "operator", "unit-test-password") + if ruleResponse.Code != http.StatusCreated { + t.Fatalf("create fixture rule: %d %s", ruleResponse.Code, ruleResponse.Body.String()) + } + ruleID := idFrom(ruleResponse) + if response := do(app, http.MethodGet, "/api/creator/rules/"+ruleID, "", "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("GET fixture rule: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/creator/comments/"+commentID+"/analyze", `{"rule_id":"`+ruleID+`"}`, "operator", "unit-test-password"); response.Code != http.StatusServiceUnavailable { + t.Fatalf("analyze fixture comment without AI: %d %s", response.Code, response.Body.String()) + } + competitor, err := creatorStore.CreateCompetitor(ctx, creator.CompetitorInput{Platform: creator.PlatformDouyin, PlatformAccountKey: "competitor-key", Nickname: "Competitor", HomepageURL: "https://www.douyin.com/user/competitor-key"}) + if err != nil { + t.Fatal(err) + } + competitorID := competitor.ID + if response := do(app, http.MethodGet, "/api/creator/competitors/"+competitorID, "", "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("GET fixture competitor: %d %s", response.Code, response.Body.String()) + } + for _, action := range []string{"pause", "resume", "sync"} { + if response := do(app, http.MethodPost, "/api/creator/competitors/"+competitorID+"/"+action, `{}`, "operator", "unit-test-password"); response.Code != http.StatusOK && response.Code != http.StatusServiceUnavailable { + t.Fatalf("competitor %s: %d %s", action, response.Code, response.Body.String()) + } + } + if _, err := syncCreatorCompetitorWithClaim(ctx, creatorStore, phaseAStore, hubStore, competitorID, "fixture-account", true); err == nil { + t.Fatal("competitor sync without browser unexpectedly succeeded") + } + if _, err := previewDouyinCompetitor(ctx, creatorStore, phaseAStore, hubStore, "fixture-account", creator.CompetitorInput{Platform: creator.PlatformDouyin, PlatformAccountKey: "preview-key", Nickname: "Preview", HomepageURL: "https://www.douyin.com/user/preview-key"}); err == nil { + t.Fatal("competitor preview without browser unexpectedly succeeded") + } + eventBody := `{"platform":"douyin","receiving_account_id":"fixture-account","event_key":"fixture-event","event_type":"comment","interactor_uid":"peer","work_id":"` + workID + `","comment_id":"` + commentID + `"}` + eventResponse := do(app, http.MethodPost, "/api/creator/test/events", eventBody, "operator", "unit-test-password") + if eventResponse.Code != http.StatusCreated { + t.Fatalf("create fixture event: %d %s", eventResponse.Code, eventResponse.Body.String()) + } + var eventEnvelope struct { + Event struct { + ID string `json:"id"` + } `json:"event"` + } + if err := json.Unmarshal(eventResponse.Body.Bytes(), &eventEnvelope); err != nil || eventEnvelope.Event.ID == "" { + t.Fatalf("event has no ID: %s (%v)", eventResponse.Body.String(), err) + } + eventID := eventEnvelope.Event.ID + if response := do(app, http.MethodPost, "/api/creator/events/"+eventID+"/display", `{}`, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("display fixture event: %d %s", response.Code, response.Body.String()) + } + operationBody := `{"idempotency_key":"fixture-operation","source":"manual","action":"reply_comment","platform":"douyin","account_id":"fixture-account","target_uid":"peer","target_comment_id":"` + commentID + `","target_work_id":"` + workID + `","text":"reply"}` + operationResponse := do(app, http.MethodPost, "/api/creator/operations", operationBody, "operator", "unit-test-password") + if operationResponse.Code != http.StatusCreated { + t.Fatalf("create fixture operation: %d %s", operationResponse.Code, operationResponse.Body.String()) + } + operationID := idFrom(operationResponse) + for _, path := range []string{"/api/creator/operations/" + operationID, "/api/creator/operations/" + operationID + "/verification"} { + if response := do(app, http.MethodGet, path, "", "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("GET %s: %d %s", path, response.Code, response.Body.String()) + } + } + if response := do(app, http.MethodPost, "/api/creator/operations/"+operationID+"/execute", `{}`, "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("execute fixture operation without executor: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/creator/test/events/process", eventBody, "operator", "unit-test-password"); response.Code != http.StatusOK && response.Code != http.StatusServiceUnavailable { + t.Fatalf("process fixture event without executor: %d %s", response.Code, response.Body.String()) + } + messageBody := `{"platform":"douyin","account_id":"fixture-account","peer_uid":"peer","peer_name":"Peer","platform_message_key":"fixture-message","direction":"inbound","message_type":"text","text":"hello"}` + messageResponse := do(app, http.MethodPost, "/api/creator/test/messages", messageBody, "operator", "unit-test-password") + if messageResponse.Code != http.StatusCreated { + t.Fatalf("create fixture message: %d %s", messageResponse.Code, messageResponse.Body.String()) + } + var message struct { + ConversationID string `json:"conversation_id"` + } + if err := json.Unmarshal(messageResponse.Body.Bytes(), &message); err != nil || message.ConversationID == "" { + t.Fatalf("message has no conversation: %s (%v)", messageResponse.Body.String(), err) + } + for _, path := range []string{"/api/creator/conversations/" + message.ConversationID + "/messages", "/api/creator/conversations/" + message.ConversationID + "/messages?page=1&page_size=10"} { + if response := do(app, http.MethodGet, path, "", "operator", "unit-test-password"); response.Code != http.StatusOK { + t.Fatalf("GET %s: %d %s", path, response.Code, response.Body.String()) + } + } + if err := runCreatorScheduleOnce(ctx, creatorStore, phaseAStore, hubStore); err != nil && !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("creator schedule fixture: %v", err) + } + if err := runCreatorMetricScheduleOnce(ctx, creatorStore, phaseAStore, hubStore, time.Now().UTC()); err != nil && !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("creator metric schedule fixture: %v", err) + } + storedWork, err := creatorStore.GetWork(ctx, workID) + if err != nil { + t.Fatal(err) + } + settings, err := creatorStore.GetSettings(ctx) + if err != nil { + t.Fatal(err) + } + if err := refreshCreatorMetricWork(ctx, creatorStore, phaseAStore, hubStore, storedWork, "fixture-account", settings, time.Now().UTC()); err == nil { + t.Fatal("metric refresh without browser unexpectedly succeeded") + } + if _, err := verifyCreatorAccount(ctx, creatorStore, phaseAStore, hubStore, "fixture-account"); err == nil { + t.Fatal("account verification without browser unexpectedly succeeded") + } +} + func TestControlPlaneAuthentication(t *testing.T) { logger := logrus.StandardLogger() previousOutput := logger.Out @@ -347,6 +740,9 @@ func controlPlaneRouteMatrix() []controlPlaneRouteCase { {http.MethodPost, "/api/network-exits", "/api/network-exits", "", http.StatusBadRequest}, {http.MethodPost, "/api/network-exits/:id/check", "/api/network-exits/missing/check", "", http.StatusNotFound}, {http.MethodPost, "/api/network-exits/:id/disable", "/api/network-exits/missing/disable", "", http.StatusNotFound}, + {http.MethodPut, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusBadRequest}, + {http.MethodPost, "/api/network-exits/:id/enable", "/api/network-exits/missing/enable", "", http.StatusNotFound}, + {http.MethodDelete, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusNotFound}, {http.MethodGet, "/api/browser-images", "/api/browser-images", "", http.StatusOK}, {http.MethodPost, "/api/browser-images", "/api/browser-images", "", http.StatusBadRequest}, diff --git a/cmd/control-plane/xiaohongshu.go b/cmd/control-plane/xiaohongshu.go index 2944844..ccf475b 100644 --- a/cmd/control-plane/xiaohongshu.go +++ b/cmd/control-plane/xiaohongshu.go @@ -168,7 +168,7 @@ func newXiaohongshuReadCollector(ctx context.Context, store *creator.Store, phas if err != nil { return nil, err } - if profile.Platform != creator.PlatformXiaohongshu || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { + if profile.Platform != creator.PlatformXiaohongshu || (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" { return nil, creator.ErrConflict } environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) diff --git a/cmd/docker_gateway/douyin.py b/cmd/docker_gateway/douyin.py index 219e88f..d9a7cff 100644 --- a/cmd/docker_gateway/douyin.py +++ b/cmd/docker_gateway/douyin.py @@ -504,6 +504,28 @@ class DouyinBrowser: raise DouyinError("Douyin platform clock is invalid") from exc return result + def action_ownership(self, alias: str) -> dict | None: + value = self._evaluate( + alias, + "(() => { const raw = localStorage.getItem('__creatorhub_action_ownership_v1'); return raw === null ? null : JSON.parse(raw); })()", + ) + if value is None: + return None + if not isinstance(value, dict): + raise DouyinError("browser action ownership marker is invalid") + return value + + def set_action_ownership(self, alias: str, marker: dict) -> None: + if not isinstance(marker, dict): + raise DouyinError("browser action ownership marker is invalid") + expression = f"localStorage.setItem('__creatorhub_action_ownership_v1', {json.dumps(json.dumps(marker, separators=(',', ':')))})" + self._evaluate(alias, expression) + + def clear_action_ownership(self, alias: str, operation_id: str = "") -> None: + expected = json.dumps(operation_id) + expression = f"(() => {{ const key='__creatorhub_action_ownership_v1'; const raw=localStorage.getItem(key); if ({expected} === '' || raw === null || JSON.parse(raw).operation_id === {expected}) localStorage.removeItem(key); return true; }})()" + self._evaluate(alias, expression) + def action( self, alias: str, @@ -604,17 +626,22 @@ class DouyinBrowser: raise def message_history( - self, alias: str, expected_uid: str, target_uid: str, limit: int = 100 + self, + alias: str, + expected_uid: str, + target_uid: str, + limit: int = 100, + cursor: str = "", ) -> dict: if not UID_RE.fullmatch(expected_uid) or not UID_RE.fullmatch(target_uid): raise DouyinError("message history UID is invalid") - if not 1 <= limit <= 200: - raise DouyinError("message history limit is invalid") + if not 1 <= limit <= 200 or not isinstance(cursor, str) or len(cursor) > 500: + raise DouyinError("message history request is invalid") self.identity(alias, expected_uid) value = self._evaluate( alias, message_history_expression( - {"uid": target_uid, "limit": limit}, expected_uid + {"uid": target_uid, "cursor": cursor, "limit": limit}, expected_uid ), ) if not isinstance(value, dict): @@ -644,6 +671,7 @@ class DouyinSubscription: self._initial_boundary_pending = True self._browser_inflight: set[str] = set() self._browser_inflight_lock = threading.Lock() + self._recovery_lock = threading.Lock() self._detail_pool = ThreadPoolExecutor( max_workers=4, thread_name_prefix=f"creatorhub-notice-details-{alias}" ) @@ -745,9 +773,17 @@ class DouyinSubscription: if self.stopped.is_set(): break self._put({"kind": "error", "reason": str(exc)}) - self._recover() + self._request_recovery() self._dispose_current() + def _request_recovery(self) -> None: + if not self._recovery_lock.acquire(blocking=False): + return + try: + self._recover() + finally: + self._recovery_lock.release() + def _recover(self) -> None: # Dispose the old same-page handlers before installing a new state with # the stable key; disposing afterward would remove the fresh handlers. @@ -830,6 +866,8 @@ class DouyinSubscription: return if kind != "push": raise DouyinError("notification event kind is invalid") + event = dict(event) + event.setdefault("gateway_received_at", datetime.now(timezone.utc).isoformat()) if not hasattr(self, "_browser_inflight_lock"): self._browser_inflight_lock = threading.Lock() if not hasattr(self, "_browser_inflight"): @@ -849,7 +887,10 @@ class DouyinSubscription: if success: self._ack_browser_event(delivery_id, self._get_connection()) else: - self._retry_browser_event(delivery_id, self._get_connection()) + if not self._retry_browser_event( + delivery_id, self._get_connection() + ): + self._request_recovery() if delivery_id: with self._browser_inflight_lock: self._browser_inflight.discard(delivery_id) @@ -861,8 +902,10 @@ class DouyinSubscription: connection: CDPConnection | None, ) -> None: self._put({"kind": "error", "reason": str(error), "continuity": "gap"}) - if isinstance(delivery_id, str) and connection is not None: - self._retry_browser_event(delivery_id, connection) + if not isinstance(delivery_id, str): + return + if connection is None or not self._retry_browser_event(delivery_id, connection): + self._request_recovery() def _process_push( self, event: dict, connection: CDPConnection, baseline: bool @@ -884,7 +927,9 @@ class DouyinSubscription: for notice_id in ids: try: for notice in self._details([notice_id], connection): - normalized = normalize_notice(notice) + normalized = normalize_notice( + notice, event.get("gateway_received_at") + ) if normalized is not None: self._put( { @@ -926,7 +971,11 @@ class DouyinSubscription: if success: self._ack_browser_event(delivery_id, connection) else: - self._retry_browser_event(delivery_id, connection) + # A malformed detail is a continuity gap, not a retryable + # delivery. Drop the bad delivery, record the gap, and + # establish a new baseline before accepting later events. + self._ack_browser_event(delivery_id, connection) + self._request_recovery() except LISTENER_ERRORS as exc: self._handle_push_error(exc, delivery_id, connection) finally: @@ -956,13 +1005,14 @@ class DouyinSubscription: def _retry_browser_event( self, delivery_id: object, connection: CDPConnection - ) -> None: + ) -> bool: if not isinstance(delivery_id, str) or not delivery_id: - return + return False try: result = connection.evaluate(retry_expression(self.key, [delivery_id])) if result not in (True, "true"): raise DouyinError("notification retry acknowledgement failed") + return True except ( DouyinError, OSError, @@ -971,6 +1021,7 @@ class DouyinSubscription: websocket.WebSocketException, ): LOG.warning("notification retry marker failed", exc_info=True) + return False def pending(self) -> list[dict]: with self.condition: @@ -1172,7 +1223,9 @@ def _direct_message_type(detail: dict, message: object) -> str: return "non_text" -def normalize_notice(notice: object) -> dict | None: +def normalize_notice( + notice: object, gateway_received_at: str | None = None +) -> dict | None: if not isinstance(notice, dict): raise DouyinError("notification detail is invalid") if notice.get("comment"): @@ -1248,6 +1301,8 @@ def normalize_notice(notice: object) -> dict | None: ).isoformat() except (OverflowError, OSError, ValueError) as exc: raise DouyinError("notification timestamp is invalid") from exc + if isinstance(gateway_received_at, str) and gateway_received_at: + result["gateway_received_at"] = gateway_received_at return result users = detail.get("from_user") or [] if isinstance(users, dict): @@ -1315,6 +1370,8 @@ def normalize_notice(notice: object) -> dict | None: ).isoformat() except (OverflowError, OSError, ValueError) as exc: raise DouyinError("notification timestamp is invalid") from exc + if isinstance(gateway_received_at, str) and gateway_received_at: + result["gateway_received_at"] = gateway_received_at return result @@ -1709,6 +1766,6 @@ FOLLOW_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;let sent=false;try{if(locati ACTION_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;let sent=false;const fail=(code,definitive=false)=>{const e=Error(code);e.definitive=definitive;throw e;};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");const qs="device_platform=webapp&aid=6383&channel=channel_pc_web";const get=async path=>{const r=await fetch(path,{credentials:"include",redirect:"error",signal:AbortSignal.timeout(15000)});let v;try{v=await r.json();}catch(_){fail("INVALID_RESPONSE");}if(!r.ok||v.status_code!==0)fail("READ_FAILED");return v;};const post=async(path,data)=>{const r=await fetch(path,{method:"POST",credentials:"include",redirect:"error",headers:{"Content-Type":"application/x-www-form-urlencoded;charset=UTF-8"},body:new URLSearchParams(data),signal:AbortSignal.timeout(10000)});let v;try{v=await r.json();}catch(_){fail(r.ok?"INVALID_RESPONSE":"POST_UNCERTAIN");}if(!r.ok)fail("POST_UNCERTAIN");if(v.status_code===undefined)fail("POST_UNCONFIRMED");if(v.status_code!==0)fail("BUSINESS_REJECTED",true);return v;};const self=await get("/aweme/v1/web/user/profile/self/?"+qs);if(String(self.user?.uid)!==p.expected)fail("IDENTITY_MISMATCH");if((p.action==="follow"||p.action==="dm")&&p.target===p.expected)fail("SELF_TARGET");const detail=async()=>get("/aweme/v1/web/aweme/detail/?"+qs+"&aweme_id="+encodeURIComponent(p.work));const findComment=async()=>{let cursor=0;for(let page=0;page<100;page++){const response=await get("/aweme/v1/web/comment/list/?"+qs+"&aweme_id="+encodeURIComponent(p.work)+"&cursor="+cursor+"&count=50");if(!Array.isArray(response.comments)||typeof response.has_more!=="boolean")fail("READ_FAILED");const list=response.comments;const comment=list.find(c=>String(c?.cid||c?.comment_id||"")===p.comment);if(comment){if(String(comment.aweme_id||comment.item_id||p.work)!==p.work||String(comment.user?.uid||comment.user_id||"")!==p.target)fail("TARGET_MISMATCH");return comment;}if(!response.has_more)break;const next=Number(response.cursor);if(!Number.isSafeInteger(next)||next<=cursor)fail("PAGINATION_INVALID");cursor=next;}fail("TARGET_NOT_FOUND");};if(p.action==="like_work"){const before=(await detail()).aweme_detail;if(String(before?.aweme_id)!==p.work)fail("TARGET_MISMATCH");if(Number(before.user_digged)===1)return {status:"succeeded",action:"already_liked",evidence:{work_id:p.work,user_digged:"1"}};sent=true;const result=await post("/aweme/v1/web/commit/item/digg/?"+qs,{aweme_id:p.work,type:"1",item_type:"0"});if(Number(result.is_digg)!==1)fail("POST_UNCONFIRMED");const after=(await detail()).aweme_detail;return {status:Number(after?.user_digged)===1?"succeeded":"unknown",action:"liked",evidence:{work_id:p.work,user_digged:String(after?.user_digged??"")}};}if(p.action==="like_comment"){const comment=await findComment();if(Number(comment.user_digged)===1)return {status:"succeeded",action:"already_liked",evidence:"comment.user_digged"};sent=true;await post("/aweme/v1/web/comment/digg?"+qs,{cid:p.comment,aweme_id:p.work,digg_type:"1",channel_id:"0",app_name:"aweme",item_type:"0",level:"1"});const after=await findComment();return {status:Number(after.user_digged)===1?"succeeded":"unknown",action:"liked_comment",evidence:{comment_id:p.comment,work_id:p.work,user_digged:String(after.user_digged??"")}};}if(p.action==="reply_comment"){await findComment();sent=true;const result=await post("/aweme/v1/web/comment/publish?"+qs,{app_name:"aweme",enter_from:"pc_web",previous_page:"video",reply_id:p.comment,reply_to_reply_id:"0",aweme_id:p.work,text:p.text,text_extra:"[]",comment_send_celltime:"0",comment_video_celltime:"0"});const posted=result.comment||result.comment_info||result.data?.comment;const postedID=String(posted?.cid||posted?.comment_id||"");const postedWork=String(posted?.aweme_id||posted?.item_id||"");const postedAuthor=String(posted?.user?.uid||posted?.user_id||"");const postedText=String(posted?.text??posted?.content??"");if(!posted||!postedID||postedWork!==p.work||postedAuthor!==p.expected||postedText!==p.text)fail("UNCONFIRMED");return {status:"succeeded",action:"replied",evidence:{comment_id:postedID,work_id:postedWork,author_uid:postedAuthor,text:postedText}};}if(p.action==="repost"){if(!p.text)fail("TEXT_REQUIRED");fail("REPOST_TEXT_UNSUPPORTED");}fail("ACTION_NOT_IMPLEMENTED");}catch(e){const code=String(e.message||"REQUEST_FAILED");return {status:sent?(e.definitive?"failed":"unknown"):"failed",code};}})()""" -MESSAGE_HISTORY_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;const fail=code=>{throw Error(code);};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");const response=await fetch("/aweme/v1/web/user/profile/self/?device_platform=webapp&aid=6383",{credentials:"include",signal:AbortSignal.timeout(15000)});if(!response.ok)fail("LOGIN_CHECK_FAILED");const profile=await response.json();if(profile.status_code!==0||String(profile.user?.uid)!==EXPECTED_UID_VALUE)fail("IDENTITY_MISMATCH");if(String(profile.user.uid)===p.uid)fail("SELF_TARGET");let service;for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){const chunks=window[key];if(!Array.isArray(chunks))continue;let req;chunks.push([["creatorhub-history-"+Date.now()],{},r=>{req=r;}]);chunks.pop();for(const [id,module] of Object.entries(req?.c||{})){if(!String(req.m?.[id]||"").includes("getOrCreatePrivateConversationByUid"))continue;for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService;}if(service)break;}if(!service)fail("IM_SDK_NOT_READY");const sdk=service.imSdkManager.getImSdkInstance();if(!sdk)fail("IM_SDK_NOT_READY");const conversation=sdk.getConversationList().find(c=>c.type===1&&String(c.toParticipantUserId)===p.uid);if(!conversation)fail("CONVERSATION_NOT_FOUND");const meta=c=>({id:String(c.id),short_id:String(c.shortId),uid:String(c.toParticipantUserId),type:c.type});let cursor,previousCursor="",hasMore=false;for(let page=0;page<20;page++){const request={conversation,limit:Math.min(50,p.limit)};if(cursor!==undefined)request.cursor=cursor;const result=await sdk.getMessagesByConversation(request);if(!result||!Array.isArray(result.messages))fail("MESSAGE_HISTORY_INVALID");hasMore=Boolean(result.hasMore);if(!hasMore)break;const next=result.cursor,key=String(next?.toString?.()??next??"");if(!key||key===previousCursor)fail("MESSAGE_HISTORY_CURSOR_INVALID");previousCursor=key;cursor=next;}const messages=conversation.getMessageList();if(!Array.isArray(messages))fail("MESSAGE_HISTORY_INVALID");const pack=m=>{const ext=m.ext||{};const rawTime=ext["s:server_message_create_time"]||"";return {server_id:String(m.serverId||""),sender_uid:String(m.sender||""),message_type:String(m.type??""),content:typeof m.content==="string"?m.content.slice(0,100000):m.content,created_at:/^[0-9]+$/.test(String(rawTime))?String(rawTime):null,server_status:m.serverStatus??null};};return {status:"succeeded",action:"history",history_source:"im_sdk_pull",history_has_more:hasMore,account_uid:String(profile.user.uid),conversation:meta(conversation),messages:messages.slice(-p.limit).map(pack)};}catch(e){const known=["WRONG_ORIGIN","LOGIN_CHECK_FAILED","IDENTITY_MISMATCH","SELF_TARGET","IM_SDK_NOT_READY","CONVERSATION_NOT_FOUND","MESSAGE_HISTORY_INVALID","MESSAGE_HISTORY_CURSOR_INVALID"];return {status:"failed",code:known.includes(e.message)?e.message:"SDK_REQUEST_FAILED"};}})()""" +MESSAGE_HISTORY_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;const fail=code=>{throw Error(code);};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");const response=await fetch("/aweme/v1/web/user/profile/self/?device_platform=webapp&aid=6383",{credentials:"include",signal:AbortSignal.timeout(15000)});if(!response.ok)fail("LOGIN_CHECK_FAILED");const profile=await response.json();if(profile.status_code!==0||String(profile.user?.uid)!==EXPECTED_UID_VALUE)fail("IDENTITY_MISMATCH");if(String(profile.user.uid)===p.uid)fail("SELF_TARGET");let service;for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){const chunks=window[key];if(!Array.isArray(chunks))continue;let req;chunks.push([["creatorhub-history-"+Date.now()],{},r=>{req=r;}]);chunks.pop();for(const [id,module] of Object.entries(req?.c||{})){if(!String(req.m?.[id]||"").includes("getOrCreatePrivateConversationByUid"))continue;for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService;}if(service)break;}if(!service)fail("IM_SDK_NOT_READY");const sdk=service.imSdkManager.getImSdkInstance();if(!sdk)fail("IM_SDK_NOT_READY");const conversation=sdk.getConversationList().find(c=>c.type===1&&String(c.toParticipantUserId)===p.uid);if(!conversation)fail("CONVERSATION_NOT_FOUND");const meta=c=>({id:String(c.id),short_id:String(c.shortId),uid:String(c.toParticipantUserId),type:c.type});let cursor=p.cursor?String(p.cursor):undefined,previousCursor=cursor??"",hasMore=false;for(let page=0;page<20;page++){const request={conversation,limit:Math.min(50,p.limit)};if(cursor!==undefined)request.cursor=cursor;const result=await sdk.getMessagesByConversation(request);if(!result||!Array.isArray(result.messages))fail("MESSAGE_HISTORY_INVALID");hasMore=Boolean(result.hasMore);if(!hasMore)break;const next=result.cursor,key=String(next?.toString?.()??next??"");if(!key||key===previousCursor)fail("MESSAGE_HISTORY_CURSOR_INVALID");previousCursor=key;cursor=next;}const messages=conversation.getMessageList();if(!Array.isArray(messages))fail("MESSAGE_HISTORY_INVALID");const pack=m=>{const ext=m.ext||{};const rawTime=ext["s:server_message_create_time"]||"";return {server_id:String(m.serverId||""),sender_uid:String(m.sender||""),message_type:String(m.type??""),content:typeof m.content==="string"?m.content.slice(0,100000):m.content,created_at:/^[0-9]+$/.test(String(rawTime))?String(rawTime):null,server_status:m.serverStatus??null};};return {status:"succeeded",action:"history",history_source:"im_sdk_pull",history_cursor:hasMore?String(cursor??""):"",history_has_more:hasMore,account_uid:String(profile.user.uid),conversation:meta(conversation),messages:messages.slice(-p.limit).map(pack)};}catch(e){const known=["WRONG_ORIGIN","LOGIN_CHECK_FAILED","IDENTITY_MISMATCH","SELF_TARGET","IM_SDK_NOT_READY","CONVERSATION_NOT_FOUND","MESSAGE_HISTORY_INVALID","MESSAGE_HISTORY_CURSOR_INVALID"];return {status:"failed",code:known.includes(e.message)?e.message:"SDK_REQUEST_FAILED"};}})()""" IM_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;let sent=false;const fail=(code,definitive=false)=>{const e=Error(code);e.definitive=definitive;throw e;};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");const response=await fetch("/aweme/v1/web/user/profile/self/?device_platform=webapp&aid=6383",{credentials:"include",signal:AbortSignal.timeout(15000)});if(!response.ok)fail("LOGIN_CHECK_FAILED");const profile=await response.json();if(profile.status_code!==0||String(profile.user?.uid)!==EXPECTED_UID_VALUE)fail("IDENTITY_MISMATCH");if(String(profile.user.uid)===p.uid)fail("SELF_TARGET");let service;for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){const chunks=window[key];if(!Array.isArray(chunks))continue;let req;chunks.push([["creatorhub-im-"+Date.now()],{},r=>{req=r;}]);for(const [id,module] of Object.entries(req?.c||{})){if(!String(req.m[id]).includes("getOrCreatePrivateConversationByUid"))continue;for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService;}}if(!service)fail("IM_SDK_NOT_READY");const sdk=service.imSdkManager.getImSdkInstance();if(!sdk)fail("IM_SDK_NOT_READY");const meta=c=>({id:String(c.id),short_id:String(c.shortId),uid:String(c.toParticipantUserId),type:c.type});const pack=m=>({server_id:String(m.serverId||""),client_id:m.clientId||null,sender:String(m.sender),type:m.type,content:m.content,created_at:m.createdAt,server_status:m.serverStatus});let conversation=sdk.getConversationList().find(c=>c.type===1&&String(c.toParticipantUserId)===p.uid);if(!conversation&&p.action==="send")conversation=await service.conversationManager.getOrCreatePrivateConversationByUid(p.uid);if(!conversation)fail("CONVERSATION_NOT_FOUND");if(conversation.type!==1||String(conversation.toParticipantUserId)!==p.uid)fail("TARGET_MISMATCH");if(p.action!=="send"||!p.confirm)return {status:"preview",action:"preview",sender_uid:String(profile.user.uid),uid:p.uid,text:p.text,conversation:meta(conversation)};const message=await sdk.createMessage({conversation,type:7,content:JSON.stringify({aweType:700,type:0,richTextInfos:[],text:p.text})});if(!message||typeof message.sendFunc!=="function")fail("MESSAGE_BUILD_FAILED");sent=true;const result=await Promise.race([sdk.sendMessage({message}),new Promise((_,reject)=>setTimeout(()=>reject(Error("MESSAGE_UNCONFIRMED")),10000))]);if(result?.success===false)fail("MESSAGE_REJECTED",true);if(result?.success!==true)fail("MESSAGE_UNCONFIRMED");const packed=pack(message);if(!packed.server_id&&!packed.client_id)fail("MESSAGE_UNCONFIRMED");return {status:"succeeded",action:"send",success:true,status_code:result?.statusCode??null,check_code:String(result?.checkCode??""),conversation:meta(conversation),message:packed,evidence:{conversation_id:String(conversation.id),message_server_id:packed.server_id,message_client_id:String(packed.client_id??"")}};}catch(e){const known=["WRONG_ORIGIN","LOGIN_CHECK_FAILED","IDENTITY_MISMATCH","SELF_TARGET","IM_SDK_NOT_READY","CONVERSATION_NOT_FOUND","TARGET_MISMATCH","MESSAGE_BUILD_FAILED","MESSAGE_REJECTED","MESSAGE_UNCONFIRMED"];return {status:sent?(e.definitive?"failed":"unknown"):"failed",code:known.includes(e.message)?e.message:"SDK_REQUEST_FAILED"};}})()""" diff --git a/cmd/docker_gateway/gateway.py b/cmd/docker_gateway/gateway.py index a6686d3..4c01957 100644 --- a/cmd/docker_gateway/gateway.py +++ b/cmd/docker_gateway/gateway.py @@ -834,10 +834,13 @@ class Gateway: expected_uid = input.get("expected_uid", "") target_uid = input.get("target_uid", "") limit = input.get("limit", 100) + cursor = input.get("cursor", "") if ( not valid_douyin_generation(input) or not isinstance(expected_uid, str) or not isinstance(target_uid, str) + or not isinstance(cursor, str) + or len(cursor) > 500 or type(limit) is not int or not UID_RE.fullmatch(expected_uid) or not UID_RE.fullmatch(target_uid) @@ -848,9 +851,14 @@ class Gateway: with self._alias_lock(alias): self._require_douyin_generation(alias, input) try: - result = self.browser.message_history( - alias, expected_uid, target_uid, limit - ) + if cursor: + result = self.browser.message_history( + alias, expected_uid, target_uid, limit, cursor=cursor + ) + else: + result = self.browser.message_history( + alias, expected_uid, target_uid, limit + ) self._require_douyin_generation(alias, input) except DouyinError as exc: LOG.warning( @@ -869,6 +877,7 @@ class Gateway: target_work_id = input.get("target_work_id", "") text = input.get("text", "") confirm = input.get("confirm", False) + operation_id = input.get("operation_id", "") if ( not valid_douyin_generation(input) or not isinstance(expected_uid, str) @@ -877,6 +886,8 @@ class Gateway: or not isinstance(target_comment_id, str) or not isinstance(target_work_id, str) or not isinstance(text, str) + or not isinstance(operation_id, str) + or len(operation_id) > 200 or type(confirm) is not bool or not UID_RE.fullmatch(expected_uid) or action not in ACTIONS @@ -886,7 +897,11 @@ class Gateway: raise RequestError("ACTION_UNAVAILABLE", 409) with self._alias_lock(alias): self._require_douyin_generation(alias, input) - self._claim_action(alias) + self._claim_action( + alias, + str(input.get("runtime_id", "")), + operation_id, + ) try: result = self.browser.action( alias, @@ -900,22 +915,22 @@ class Gateway: ) self._require_douyin_generation(alias, input) except DouyinError as exc: - self._handle_douyin_action_error(alias, action, exc) + self._handle_douyin_action_error(alias, action, exc, operation_id) raise RequestError("Douyin action failed") from exc except Exception: - self._release_action_ownership(alias) + self._release_action_ownership(alias, operation_id) raise else: - self._release_action_ownership(alias) + self._release_action_ownership(alias, operation_id) return result def _handle_douyin_action_error( - self, alias: str, action: str, error: DouyinError + self, alias: str, action: str, error: DouyinError, operation_id: str = "" ) -> None: if getattr(error, "uncertain", False) or "timed out" in str(error).lower(): - self._retain_action_ownership(alias) + self._retain_action_ownership(alias, operation_id) else: - self._release_action_ownership(alias) + self._release_action_ownership(alias, operation_id) LOG.warning( "Douyin action failed alias=%s action=%s reason=%s", alias, @@ -923,20 +938,37 @@ class Gateway: str(error), ) - def _claim_action(self, alias: str) -> None: + def _claim_action( + self, alias: str, runtime_id: str = "", operation_id: str = "" + ) -> None: now = time.monotonic() with self._action_ownership_lock: until = self._uncertain_actions.get(alias, 0.0) if until > now: raise RequestError("previous Douyin action outcome is uncertain", 409) + if runtime_id: + marker = self.browser.action_ownership(alias) + if marker is not None: + raise RequestError("previous Douyin action outcome is uncertain", 409) + self.browser.set_action_ownership( + alias, + { + "runtime_id": runtime_id, + "operation_id": operation_id, + "started_at": time.time(), + }, + ) + with self._action_ownership_lock: self._uncertain_actions.pop(alias, None) self._uncertain_actions[alias] = 0.0 - def _release_action_ownership(self, alias: str) -> None: + def _release_action_ownership(self, alias: str, operation_id: str = "") -> None: + if operation_id: + self.browser.clear_action_ownership(alias, operation_id) with self._action_ownership_lock: self._uncertain_actions.pop(alias, None) - def _retain_action_ownership(self, alias: str) -> None: + def _retain_action_ownership(self, alias: str, operation_id: str = "") -> None: with self._action_ownership_lock: # A timed-out page script may still finish its network write. Keep the # alias blocked until the browser generation is removed or replaced. diff --git a/cmd/docker_gateway/test_gateway.py b/cmd/docker_gateway/test_gateway.py index 5f3e8bb..54e1eda 100644 --- a/cmd/docker_gateway/test_gateway.py +++ b/cmd/docker_gateway/test_gateway.py @@ -885,6 +885,18 @@ class BrowserTests(unittest.TestCase): subscription.ack([first[0]["delivery_id"]]) self.assertEqual(subscription.poll(10, 0), []) + def test_subscription_connection_failure_requests_recovery(self) -> None: + subscription = DouyinSubscription.__new__(DouyinSubscription) + subscription.alias = "safe" + subscription.browser = Mock() + subscription.browser._connect.side_effect = DouyinError("browser unavailable") + subscription._put = Mock() + subscription._request_recovery = Mock() + subscription._browser_inflight_lock = threading.Lock() + subscription._browser_inflight = set() + subscription._process_push_async({"delivery_id": "delivery-1"}, 0, False) + subscription._request_recovery.assert_called_once_with() + def test_subscription_detail_failure_does_not_discard_siblings(self) -> None: bad = json.dumps( { @@ -2244,18 +2256,21 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): def test_identity_and_message_history_reject_invalid_values(self) -> None: browser = DouyinBrowser() for server_now in (0, float("nan")): - cast(Any, browser).get = lambda alias, target, now=server_now: BrowserResponse( - 200, - json.dumps( - { - "status_code": 0, - "extra": {"now": now}, - "user": {"uid": "123", "sec_uid": "sec"}, - } - ), + cast(Any, browser).get = lambda alias, target, now=server_now: ( + BrowserResponse( + 200, + json.dumps( + { + "status_code": 0, + "extra": {"now": now}, + "user": {"uid": "123", "sec_uid": "sec"}, + } + ), + ) ) - with self.subTest(server_now=server_now), self.assertRaisesRegex( - DouyinError, "platform clock" + with ( + self.subTest(server_now=server_now), + self.assertRaisesRegex(DouyinError, "platform clock"), ): browser.identity("safe") @@ -2273,9 +2288,12 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): ("123", "456", 0), ("123", "456", 201), ): - with self.subTest( - expected_uid=expected_uid, target_uid=target_uid, limit=limit - ), self.assertRaises(DouyinError): + with ( + self.subTest( + expected_uid=expected_uid, target_uid=target_uid, limit=limit + ), + self.assertRaises(DouyinError), + ): browser.message_history("safe", expected_uid, target_uid, limit) evaluate.return_value = "bad" with self.assertRaisesRegex(DouyinError, "message history response"): @@ -2432,5 +2450,40 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): load_config({**base, "BROWSER_CDP_ALIAS": "bad alias"}) +class DouyinReleaseRemediationTests(unittest.TestCase): + def test_browser_action_marker_is_persisted_in_profile_storage(self) -> None: + browser = DouyinBrowser() + browser._evaluate = Mock(side_effect=[{"runtime_id": "old"}, True, True]) + self.assertEqual(browser.action_ownership("safe"), {"runtime_id": "old"}) + browser.set_action_ownership( + "safe", {"runtime_id": "new", "operation_id": "op-1"} + ) + browser.clear_action_ownership("safe", "op-1") + self.assertEqual(browser._evaluate.call_count, 3) + self.assertIn( + "__creatorhub_action_ownership_v1", + browser._evaluate.call_args_list[1].args[1], + ) + + def test_gateway_action_claim_rejects_marker_from_any_runtime(self) -> None: + gateway = Gateway.__new__(Gateway) + gateway._action_ownership_lock = threading.Lock() + gateway._uncertain_actions = {} + gateway.browser = Mock() + gateway.browser.action_ownership.return_value = {"runtime_id": "old-runtime"} + with self.assertRaises(RequestError): + gateway._claim_action("safe", "new-runtime", "op-1") + gateway.browser.clear_action_ownership.assert_not_called() + gateway.browser.set_action_ownership.assert_not_called() + + def test_notice_keeps_raw_gateway_receipt_time(self) -> None: + notice = normalize_notice( + {"dm": {"message_id": "1", "from_user": {"uid": "2"}, "text": "hi"}}, + "2026-09-15T00:00:00+00:00", + ) + assert notice is not None + self.assertEqual(notice["gateway_received_at"], "2026-09-15T00:00:00+00:00") + + if __name__ == "__main__": unittest.main() diff --git a/docker/browser-wrapper/README.md b/docker/browser-wrapper/README.md index 9f281a4..a293710 100644 --- a/docker/browser-wrapper/README.md +++ b/docker/browser-wrapper/README.md @@ -5,8 +5,10 @@ ```bash docker build \ --build-arg BROWSER_BASE_IMAGE=git.ipao.vip/rogee/fingerprint-chromium@sha256: \ - -t git.ipao.vip/rogee/creatorhub-browser-wrapper@sha256: \ + -t creatorhub-browser-wrapper: \ docker/browser-wrapper ``` +将构建结果推送到登记的镜像仓库后，用 `docker image inspect` 或仓库 manifest 查询发布 digest；Compose 和验收只使用 `repo/image@sha256:`，不把 digest 当作本地 tag。 + 发布记录必须同时保存 base image 仓库、base digest、构建提交、此目录 Dockerfile 和发布 digest。入口会等待 Xvfb、检查 x11vnc、绑定容器地址上的 CDP 端口，再启动 `/opt/chromium/chrome`。 diff --git a/docs/deployment.md b/docs/deployment.md index d8e3fdc..7896494 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,6 +1,6 @@ # CreatorHub 部署 -本文档按 `feat/python-gateway-douyin` 当前代码说明单台 Linux 主机 Docker Compose 部署及离线检查，不证明新产品功能已实现。[plan01](plan01.md) 是业务范围与验收依据；离线结果不能替代抖音/小红书最终真机验收。 +本文档按当前发布提交说明单台 Linux 主机 Docker Compose 部署及离线检查，不证明新产品功能已实现。[plan01](plan01.md) 是业务范围与验收依据；离线结果不能替代抖音/小红书最终真机验收。 当前控制面仍使用单用户 HTTP Basic Auth（除 `/healthz` 外，包括静态页面），不提供 RBAC 或多租户隔离。开发目标不新增认证/访问限制，但本轮未删除现有代码或配置；以下变量仍须填写，不新增认证 profile。 @@ -86,7 +86,11 @@ curl --fail --silent --show-error \ docker compose exec -T postgres \ psql -U creatorhub -d creatorhub -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 14;' \ + 'SELECT 1 FROM schema_migration WHERE version = 32;' \ + | grep -qx 1 +docker compose exec -T postgres \ + psql -U creatorhub -d creatorhub -tAc \ + 'SELECT 1 FROM schema_migration WHERE version = 33;' \ | grep -qx 1 docker compose ps @@ -340,6 +344,32 @@ docker compose exec -T postgres pg_restore --list \ < "$BACKUP_FILE" >/dev/null ``` +PostgreSQL dump 之外，发布备份必须同时包含 credentials、materials 和动态 Profile 卷；这些归档存放在部署侧受限目录，不放进仓库或共享聊天。示例（`BACKUP_DIR` 使用独立磁盘或 Secret Manager 的受控挂载目录）： + +```bash +set -Eeuo pipefail +umask 077 +BACKUP_DIR=/srv/creatorhub-backups/$(date +%Y%m%d-%H%M%S) +mkdir -p "$BACKUP_DIR/profiles" +# PostgreSQL dump +pg_dump_file="$BACKUP_DIR/creatorhub.dump" +docker compose exec -T postgres pg_dump -U creatorhub -d creatorhub --format=custom > "$pg_dump_file" +test -s "$pg_dump_file" +# Credential and material volumes +for volume in creatorhub_credentials creatorhub_materials; do + docker run --rm -v "$volume:/data:ro" -v "$BACKUP_DIR:/backup" alpine:3.22 \ + tar czf "/backup/${volume}.tar.gz" -C /data . +done +# Dynamic browser Profile volumes; absence is an explicit empty set. +for volume in $(docker volume ls -q --filter name='^creatorhub-profile-'); do + docker run --rm -v "$volume:/data:ro" -v "$BACKUP_DIR/profiles:/backup" alpine:3.22 \ + tar czf "/backup/${volume}.tar.gz" -C /data . +done +find "$BACKUP_DIR" -type f -exec sha256sum {} + > "$BACKUP_DIR/SHA256SUMS" +``` + +`CREATORHUB_CREDENTIAL_MASTER_KEY` 不写入上述归档；部署管理员必须在独立的 Secret Manager/OS Keyring 保留一份受访问控制的密钥托管记录，并确认恢复主机能注入同一值。恢复前核对 `SHA256SUMS`、主密钥记录和备份目录权限；恢复后在隔离 Compose 项目中还原四类卷，检查账号凭据可解密、材料文件可读、Profile 可挂载，再切换服务。任何一类缺失都判定为恢复失败，不把应用健康检查当作数据恢复证据。 + 拉取已审核版本后，重新执行部署和验证： ```bash @@ -379,7 +409,11 @@ docker compose up --detach --build < "$BACKUP_FILE" docker compose exec -T postgres \ psql -U creatorhub -d creatorhub_restore_check -v ON_ERROR_STOP=1 -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 12;' \ + 'SELECT 1 FROM schema_migration WHERE version = 32;' \ + | grep -qx 1 + docker compose exec -T postgres \ + psql -U creatorhub -d creatorhub_restore_check -v ON_ERROR_STOP=1 -tAc \ + 'SELECT 1 FROM schema_migration WHERE version = 33;' \ | grep -qx 1 docker compose exec -T postgres \ dropdb --force -U creatorhub creatorhub_restore_check @@ -426,7 +460,11 @@ docker compose up --detach --build < "$BACKUP_FILE" docker compose exec -T postgres \ psql -U creatorhub -d creatorhub -v ON_ERROR_STOP=1 -tAc \ - 'SELECT 1 FROM schema_migration WHERE version = 12;' \ + 'SELECT 1 FROM schema_migration WHERE version = 32;' \ + | grep -qx 1 + docker compose exec -T postgres \ + psql -U creatorhub -d creatorhub -v ON_ERROR_STOP=1 -tAc \ + 'SELECT 1 FROM schema_migration WHERE version = 33;' \ | grep -qx 1 git switch --detach "$RESTORE_REV" diff --git a/docs/evidence/douyin-release-20260916-worktree.md b/docs/evidence/douyin-release-20260916-worktree.md new file mode 100644 index 0000000..22588a1 --- /dev/null +++ b/docs/evidence/douyin-release-20260916-worktree.md @@ -0,0 +1,99 @@ +# 抖音发布修复开发验收记录（工作区） + +- 日期：2026-09-16（Asia/Shanghai） +- 分支：`fix/douyin-release-remediation` +- 提交：当前分支 `HEAD`（`fix: complete Douyin release remediation`） +- PR：`#48`；远程 CI 已触发，受 runner 队列阻塞 +- 基线：`main@977e541` +- 工作区：提交后保持干净；本记录不代表已发布镜像 +- 发布结论：**阻塞，不满足抖音完整生产发布条件** + +## 1. 本地质量门禁 + +| 项目 | 结果 | 证据/限制 | +| --- | --- | --- | +| Go 全量测试 | 通过 | `go test -p 1 -parallel 1 -count=1 ./...` | +| Go race | 通过 | `go test -race -p 1 -parallel 1 -count=1 ./...` | +| Go 静态检查 | 通过 | `go vet ./...` | +| Go 构建 | 通过 | `go build ./cmd/control-plane` | +| Go 覆盖率 | 通过本地门槛 | `go tool cover` 总语句覆盖 **65.4%**；PostgreSQL JSONL 记录 457 个测试事件、0 跳过、0 失败 | +| Python gateway | 通过 | 73 项 unittest 通过；生产代码覆盖 **65.38%**；按 `covered_lines * 100 >= num_statements * 65` 原始计数复核 | +| 前端 | 通过本地门槛 | `npm ci`、93 项测试、构建通过；statements **65.05%**、lines **67.18%**、branches **55.87%**、functions **57.14%** | +| Compose 配置 | 通过 | `docker compose config --quiet` 及合并开发配置通过 | +| Compose 镜像 | 通过本地检查 | 当前工作区镜像构建成功；服务 healthy；`/readyz` 成功；容器内 `ffmpeg`、`ffprobe` 可执行 | +| CI 工作流 | 已补门禁 | PostgreSQL 测试、跳过测试拒绝、原始 Python 覆盖率、前端覆盖产物、Compose 清理和 evidence artifact 已写入 `.gitea/workflows/douyin-release-gate.yaml`；尚未运行远程 CI | + +Compose 检查使用临时本地测试凭据，仅用于本地验证；没有部署生产或写入生产配置。 + +## 2. A01–A16 平台与监听 + +| 编号 | 状态 | 结果 | +| --- | --- | --- | +| A01 | 阻塞 | 代码已增加事件身份、边界和时间记录；四类入站互动、入站私信、5 秒/30 秒及断连恢复没有当前标准 Docker 浏览器真实证据 | +| A02 | 阻塞 | `repost` 仍明确返回 `ACTION_UNAVAILABLE`；没有范围变更批准，不能放行 | +| A03 | 本地通过/真机待验收 | 事件目标解析和本地对象缺失处理已修复并有回归测试；真实平台目标关联未验收 | +| A04 | 本地通过/真机待验收 | 非正常账号的策略停用、恢复不自动启用已有本地测试；真实账号状态转换未验收 | +| A05 | 本地通过/真机待验收 | 监听代际、session token 和新边界已持久化；真实停启及迟到事件未验收 | +| A06 | 本地通过/真机待验收 | 解析/详情/连接失败恢复有本地测试；真实断连、缺口和新基线未验收 | +| A07 | 本地通过/真机待验收 | 自动动作冷却和执行锁有 PostgreSQL/race 回归；批准 AI 调用和真实中断恢复未验收 | +| A08 | 本地通过/真机待验收 | 已接收未领取事件的恢复结论有本地回归；真实进程退出/重启证据缺失 | +| A09 | 本地通过/真机待验收 | PostgreSQL advisory lock 与排队恢复已测试；真实人工/自动竞争未验收 | +| A10 | 本地通过/真机待验收 | 在途 marker 和旧浏览器代际保护已实现；真实 SDK/CDP 超时未验收 | +| A11 | 本地通过/真机待验收 | 平台消息身份、历史消息和操作关联有回归；真实发送→事件→历史未验收 | +| A12 | 本地通过/真机待验收 | 平台/接收/处理/写入时间字段已分离；真实五秒时限未验收 | +| A13 | 未完成 | 核验记录入口已保留本地路径；真实成功、未发送、不明和另发关联未验收 | +| A14 | 未完成 | 历史游标和分页代码已增加；真实空库会话发现、超过上限继续读取未验收 | +| A15 | 本地通过/真机待验收 | 策略检查顺序和跳过原因可持久化；真实多策略选择证据缺失 | +| A16 | 本地通过/真机待验收 | 当前连接状态和持久历史缺口已分开；真实重启后缺口展示未验收 | + +## 3. B01–B19 业务、素材、页面与环境 + +| 编号 | 状态 | 结果 | +| --- | --- | --- | +| B01 | 本地通过/镜像样本待验收 | WAV 输出格式和临时文件流程已修复；当前镜像仅完成 `ffmpeg`/`ffprobe` 可执行检查，未完成真实平台媒体样本 | +| B02 | 阻塞 | 正文与文件引用已分离；Whisper/faster-whisper 入口、供应商、模型、参数和费用尚未批准并纳入标准镜像 | +| B03 | 本地通过/真机待验收 | 素材领取 token/CAS 和不明结果不自动重试已测试；真实双击、并发、收费调用未验收 | +| B04 | 本地通过/真机待验收 | 指标计划按发布时间和设置 checkpoint 有 PostgreSQL 回归；真实零值、下降、UTC/DST 和边界未验收 | +| B05 | 本地通过/真机待验收 | 来源启停、年龄和人工阻断在本地路径已覆盖；真实平台调用为零及解除后采样未验收 | +| B06 | 本地通过/真机待验收 | 发现与指标采样职责已分离；真实发现/采样时间序列未验收 | +| B07 | 本地通过/真机待验收 | 计划点、实际采样和保存 checkpoint 已修复；真实恢复与设置修改未验收 | +| B08 | 本地通过/真机待验收 | 普通账号只读采集资格与禁言处理有回归；真实多账号选择未验收 | +| B09 | 本地通过/真机待验收 | 未来/无效时间标记为待核验；真实平台样本和补时间流程未验收 | +| B10 | 本地通过/真机待验收 | SSE CRLF、拆包、多帧和断连测试通过；标准 Docker 浏览器页面展示未验收 | +| B11 | 未完成 | 线索联系表单代码已调整；真实评论作者私信、取消和逐次确认未验收 | +| B12 | 本地通过/真机待验收 | 素材步骤失败原因和不假成功路径有回归；真实磁盘/配额/无语音样本未验收 | +| B13 | 本地通过/真机待验收 | 运行中代理禁用保护和路由已有本地测试；真实运行环境引用未验收 | +| B14 | 未完成 | Douyin 目标预览 API 已增加；真实主页/分享/短链解析未验收 | +| B15 | 未完成 | 详情、筛选和素材预览代码有变更；完整页面矩阵及真实媒体/转写未验收 | +| B16 | 本地通过/真机待验收 | 规则快照、分析结果和失败状态持久化路径有回归；真实多规则与刷新后证据未验收 | +| B17 | 本地通过/真机待验收 | fingerprint seed 唯一约束、网络出口管理和运行保护有本地回归；真实 Profile、代理协议和出口未验收 | +| B18 | 本地通过/真机待验收 | 空白语义和前后端 trim 有回归；真实页面组合输入未验收 | +| B19 | 本地通过/真机待验收 | 页面编辑保护、取消和保存失败路径有前端回归；真实页签/路由离开未验收 | + +## 4. R01–R07 发布与恢复 + +| 编号 | 状态 | 结果 | +| --- | --- | --- | +| R01 | 本地通过/镜像样本待验收 | 关联 B01；本地 FFmpeg 和镜像工具检查通过，真实素材样本缺失 | +| R02 | 阻塞 | 标准镜像没有已批准且可执行的 Whisper/faster-whisper 转写入口；不能以环境变量代替供应商和参数批准 | +| R03 | 本地通过/CI 待验收 | Go 65.4%、Python 65.38%、前端门槛通过；远程 CI artifact 和实际 runner 执行尚未取得 | +| R04 | 未完成 | 未执行停写下的数据库、credentials、materials、Profile、主密钥完整备份 | +| R05 | 未完成 | 未固定当前发布提交对应的远程镜像 digest；外部 CDP 历史记录不能替代当前 Compose 全链路 | +| R06 | 本地通过/CI 待验收 | 最小 CI 门禁已补齐；尚未注入失败并运行远程工作流验证失败结论 | +| R07 | 本地通过/发布待验收 | 部署文档和 schema 版本已更新；尚未生成实际发布 digest 并按该 digest 复核 | + +## 5. T01–T14 真实矩阵 + +T01 身份与登录、T02 竞品与分页、T03 指标计划、T04 素材与 AI、T05 线索与人工联系、T06 四类互动、T07 六种写操作、T08 策略冷却、T09 边界恢复、T10 同账号协调、T11 时限页面、T12 私信、T13 环境代理、T14 页面完整性：均为**未执行/阻塞**。 + +原因：真实操作必须通过 CreatorHub 系统执行；当前没有完成标准 Docker gateway 下的真实账号、目标、第二互动账号、批准的 AI/转写供应商样本、时间窗口、费用边界及隔离恢复环境登记。此前直接在抖音页面发生的两个点赞和一条测试评论已撤销，不计为验收证据。 + +## 6. 发布决策 + +当前不能调用“完整流程可生产发布”或 `goal_complete`。必须补齐： + +1. `repost` 的实现结果或正式范围裁决； +2. 批准的转写供应商、模型、参数、费用和标准镜像样本； +3. CreatorHub 系统驱动的真实平台逐项矩阵及第二互动账号； +4. 停写完整备份、隔离同版本恢复、正常退出和最终镜像 digest； +5. 远程 CI 成功 artifact，并确认所有数据库测试未跳过。 diff --git a/docs/plans/2026-09-15-douyin-release-remediation.md b/docs/plans/2026-09-15-douyin-release-remediation.md new file mode 100644 index 0000000..ad42228 --- /dev/null +++ b/docs/plans/2026-09-15-douyin-release-remediation.md @@ -0,0 +1,280 @@ +# 抖音先行发布：修复计划与验收标准 + +> 日期：2026-09-15 +> 状态：计划范围已由使用者确认；本轮本地修复、回归测试和 Compose 检查已执行，真实平台全矩阵、AI/转写批准样本、备份恢复和部署发布仍未完成。 +> 评审基线：`main@977e541fef1871b14825cbccedea50543d03caa5`，评审前已同步远程。 +> 发布判断：当前不满足抖音完整生产发布条件。本文不代表修复完成、部署完成或平台能力已通过。 +> 需求依据：[需求细化与验收标准](../plan01.md)、[仓库规范](../../AGENTS.md)。冲突时以使用者最新确认和需求依据为准。 + +## 1. 范围与完成定义 + +本轮只修复并验收**抖音先行发布**，涵盖账号与环境、竞品采集、素材与 AI、评论线索、人工发送、事件监听、自动响应、私信及发布恢复。小红书不计本轮发布阻碍，但修改共用代码不能破坏其现有行为；两平台最终交付仍按原需求执行。 + +- 不新增认证、访问限制、网络隔离等本期排除的安全策略，也不删除现有保护。凭据不泄露、身份不串号和发送前核验仍为必须条件。 +- 不自动登录、不以轮询或 Mock 替代平台事件、不自动补发历史或结果不明的动作，不增加群发、自动聊天、视频生成等需求。 +- 不擅自选择付费供应商、上传用户数据、部署、执行真实互动或删除数据。本文确认仅授权编写计划；后续测试环境、账号、目标、动作、次数与费用须有适用授权。 +- 不重写系统。优先复用现有 Go 控制面、Python gateway、PostgreSQL、React 与现有任务/记录；修复真实根因，不引入通用工作流引擎、额外 MQ、Redis 或隐藏回退。 +- 开发阶段不增加数据迁移兼容、回填或双写；需要破坏性重建时必须说明影响并取得授权。发布恢复验证使用**同版本**数据与产物，不借此引入兼容框架。 + +### 1.1 发布通过条件 + +同时满足以下条件才可称为“抖音完整流程可生产发布”： + +1. 本文所有修复条目完成并有可重复回归证据；未完成项不得用低优先级或“暂不影响演示”自动豁免。 +2. `plan01.md` 第 8 节全部抖音适用 AC 均有实际结果和证据，包括本次评审未发现问题的既有能力。 +3. 真实平台能力矩阵逐分项完成；缺支持、条件支持及恢复缺口经使用者裁决。任何范围变更必须独立记录批准，不能修改测试预期使其假通过。 +4. 测试、覆盖率、当前发布镜像、健康检查、正常退出和完整备份恢复全部通过。 +5. 修复提交、测试所用提交与发布镜像可关联；工作区差异必须记录，不能以旧版本成功证明新版本。 + +不以功能数量计算完成百分比，也不在平台与 AI 前置条件未确认前承诺发布日期。 + +## 2. 评审基线与证据边界 + +### 2.1 来源与编号 + +本文自包含修复目标和验收要求，不依赖会话临时文件才能执行。原始评审来自工作流 `9113a807-a828-4693-8d9a-dc68ba57bdc5`： + +| 评审 | 子任务 | 原报告 | 本文编号 | +| --- | --- | --- | --- | +| 平台监听、动作、消息 | `515c6fc1-50ff-4df3-a793-5634feaa11ea` | `reviews/platform.md` | A01–A16 | +| 采集、素材、页面、账号与代理 | `db705b2c-92cf-4ed3-ab9c-36b46cc93b14` | `reviews/business.md` | B01–B19 | +| 测试、镜像与恢复 | `19edc2c3-049d-4788-acc2-2c67e98523af` | `reviews/release.md` | R01–R07 | + +编号只用于跟踪，不改变原 AC。原报告的重要性等级口径不完全一致，本文统一为： + +- **P0：执行安全与平台前置阻碍**。必须优先处理；相应真实自动动作不得先行验收。 +- **P1：完整发布必修/必验**。可按依赖并行推进，但未完成不能完整发布。 +- **P2：记录、交互和交付完整性**。后置处理，不代表自动豁免。 + +类型分为“确定缺陷”“未完成能力”“缺验收证据”。确定缺陷主要来自静态调用链审查；仅提音错误在本轮另有本地实际复现。实施前须针对当前提交复核定位，行号是评审基线位置，不是后续修改约束。 + +### 2.2 已有真实证据 + +[2026-09-14 抖音真实记录](../evidence/douyin-live-20260914.md)仅证明所列样本：自身身份、目标资料、一个会话的六条历史消息（含非文本）、一次人工出站文本私信、一次视频下载、reload 后取得新监听基线。 + +该记录使用外部 CDP，不证明当前 Compose 全链路、四类互动监听、入站私信、无遗漏恢复、全部写动作、自动冷却或媒体/AI 全流程通过。文件纳入当前提交不等于当前提交的镜像已复验。旧报告中的失败也必须重新核对，不能直接当作当前缺陷。 + +### 2.3 本轮实际检查 + +| 项目 | 实际结果 | 限制 | +| --- | --- | --- | +| Go test / vet / race / 构建 | PostgreSQL 环境下全量测试、vet、race 和构建通过；Go JSONL 记录 457 个测试事件全部通过、0 跳过 | 仅证明本地同版本工作区，不证明真实平台或生产部署 | +| Go 覆盖率 | `go tool cover` 总语句覆盖 65.4% | 仅是本地覆盖门禁；不能替代 A/B/R 逐项验收 | +| Python | 73 项通过；生产代码覆盖率 65.38%，并按 JSON 原始计数复核 | 仅证明 gateway 本地测试；未证明真实浏览器平台动作 | +| 前端 | lockfile 全新安装、93 项测试和构建通过；statements 65.05%、lines 67.18%、branches 55.87%、functions 57.14% | 仍有大量页面分支未覆盖，未证明真实浏览器页面矩阵 | +| Compose | 两套配置解析、当前镜像构建、启动 healthy、`/readyz`、容器内 `ffmpeg`/`ffprobe` 通过 | 使用临时本地测试凭据；未发布、未固定远程镜像 digest、未完成备份恢复 | +| 提音复现 | 原参数输出 `.wav.tmp` 失败，显式 WAV 格式对照成功 | 本地合成样本，不是平台媒体端到端验收 | +| 工作区 | 保留 `cmd/docker_gateway/test_gateway.py` 既有修改 | 评审确认修改前后 Python AST 相同；不得覆盖他人修改 | + +原始测试日志目录：`/tmp/creatorhub-release-977e541-diS2rF`。这是定位线索，不是长期发布证据；正式验收须将脱敏日志摘要和必要产物登记到仓库证据目录。 + +## 3. 阶段与依赖 + +| 阶段 | 工作内容 | 依赖 | 阶段退出条件 | +| --- | --- | --- | --- | +| S0 前置确认与证据冻结 | A01、A02 能力调查；R02 AI/转写配置与样本；专用数据库、平台测试账号、部署/恢复环境授权 | 使用者批准；复用 `plan01.md` G0 | 记录允许的账号/动作/次数/费用、逐类能力及未知项；未获批准项明确阻塞，不启动相应真实测试 | +| S1 发送与监听可靠性 | A03–A13、A15–A16；B10 页面推送 | 本地确定性修复可立即开展；真实验证依赖 S0 | 状态变化、排队、超时、进程退出、重复事件、冷却和基线回归通过；不明仅核验、旧事件不补发 | +| S2 账号环境与只读采集 | A04；B04–B09、B13–B14、B17 | 身份及采集边界明确 | 同环境登录/身份、稳定指纹、代理约束、真实身份导入、分页与固定计划通过；只读资格不混同自动动作资格 | +| S3 素材与线索 | B01–B03、B12、B15–B16、B18；R01–R02 | S2 可靠作品来源；AI/转写相关实现前完成 S0 配置和样本批准 | 两次确认、真实素材正文、防重复、分步失败、预览、线索判断和历史依据完整 | +| S4 人工联系与私信 | A09、A11、A13–A14；B11、B19 | S1 同账号写协调；S2 身份；S3 线索数据 | 评论/线索联系、历史会话发现/分页、跨账号隔离、非文本、不明核验及新发送关联通过 | +| S5 完整发布验收 | A01–A02 真机验收；R03–R07；所有 AC | S1–S4 及所需授权完成 | 第 7–9 节全部通过，证据对应同一发布版本；不支持项已有正式裁决 | + +执行顺序是依赖关系，不是全面推倒重做。S1、S2 中职责独立的修复可并行；同一文件/工作区保持单一写入者。每个非平凡改动先写失败回归，修正后复测；大规模重构或实验必须先建分支。平台可行性调查应在 S0 尽早暴露限制，不应拖到最后才发现某类动作无法支持。 + +## 4. 平台、自动响应与消息修复清单 + +所有条目初始状态为**待修复/待验收**；完成时填写第 10 节记录，不直接将现有基础标为完成。 + +| ID / 优先级 / 类型 | 评审定位与问题 | 修复目标 | 最小回归与通过标准 | 原需求 | +| --- | --- | --- | --- | --- | +| A01 / P0 / 缺验收证据 | `douyin.py:1175–1318,1569–1604`；四类互动及入站私信缺真实身份、边界和时限证据 | 逐类核实 ID、接收账号、互动 UID、目标、时间及恢复能力；缺可靠身份的自动能力保持不可启用 | 按第 8 节逐类取证；重复/重启稳定，历史及不明边界不触发；不能用内容哈希猜唯一 | A6；AC-A5/A12/A13、M2、B1/B3 | +| A02 / P0 / 未完成能力 | `gateway.py:885–886`、`douyin.py:1709`；转发直接不可用 | 实现可核验转发，或提交真实限制并取得明确范围变更 | 目标、执行账号、需要的文案及平台结果一致；成功/拒绝/不明分别记录；未经批准仍属发布阻碍 | AC-A7 | +| A03 / P1 / 确定缺陷 | `creator.go:1040–1061`、`creator_events.go:427–454`；事件动作依赖本地作品/评论已采集 | 区分平台目标与内部对象，只解析该动作必需的目标；不以定时采集完成为先决条件 | 事件具有可靠平台目标但本地无对应资料时仍可执行适用动作；关注/私信不因无关评论查找失败；缺必要目标明确跳过 | A3；AC-A5/A6/A7 | +| A04 / P0 / 确定缺陷 | `accounts.go:179–184`；小号恢复正常后原策略可自行恢复 | 非正常状态下停用所有受影响策略；恢复账号不恢复策略 | 正常→禁言/封禁/注销→正常，各执行号策略仍停用；人工重新启用前调用次数为零；已有数据可查 | AC-A3/A14 | +| A05 / P0 / 确定缺陷 | `actions.go:183–230`、`creator_events.go:195–322`；重新启用沿用旧边界 | 保存与启用对应的已验证新边界；保留去重与冷却 | 停用期间事件在重新启用后迟到也不发送；新边界未确认前不发送；确认后的新事件才按策略处理 | A6；AC-A12、B1 | +| A06 / P1 / 确定缺陷 | `douyin.py:822–833,867–912`、`creator_events.go:305–322`；一次解析/详情失败可能长期冻结响应 | 连续性受损进入显式恢复及新边界确认流程；失败与缺口可见 | 注入详情错误、解析错误、队列溢出；旧/重试事件仅记历史，新基线后新事件恢复处理；无静默永久停摆，也不假称无遗漏 | AC-A12、B1/B3 | +| A07 / P0 / 确定缺陷 | `actions.go:454–544`；AI 调用结束前冷却未提交 | 外部调用前原子持久化已选策略、冷却和执行记录 | 在 AI 开始后阻断/退出进程并恢复；该 UID 原冷却仍有效，旧事件不再执行，失败不转下一个小号；无匹配不占名额 | AC-A8/A9/A10 | +| A08 / P1 / 确定缺陷 | `creator_events.go:427–454`、`actions.go:615–666`；已入库并确认接收但未领取事件缺恢复结论 | 给旧代际未领取事件记录明确恢复结论；不自动重放 | 在入库并确认接收后、处理前退出；恢复后事件不永久停在“收到”，有待核验/不补发原因及永久去重 | A6；AC-B1 | +| A09 / P0 / 确定缺陷 | `actions.go:532–553,615–656,882–935`；排队操作变不明后仍可开始发送 | 区分等待与实际在途；取得账号执行权后原子重查操作状态及业务条件 | 用可控阻塞让排队跨过两分钟恢复窗口；被终结/标不明的操作不开始发送；人工与自动同账号不重叠 | AC-A14、W5、B2 | +| A10 / P0 / 确定缺陷 | `douyin.py:1714`、`gateway.py:891–943`；私信超时未取消底层发送却释放占用 | 对仍可能在途的发送保留账号执行占用，直至确认终结或原浏览器代际结束 | SDK 超过十秒后延迟返回时，后续写入不重叠；不明不重发；新代际前验证旧执行已结束，不以应用超时当作取消成功 | AC-W5、M3、B2 | +| A11 / P1 / 确定缺陷 | `actions.go:869–879,1018–1028`、`creator_events.go:442`、`creator.go:899–910`；发送/监听/历史消息键不同 | 用平台消息身份关联同一逻辑消息，保留本地操作关联 | 人工发送→监听→历史，以及监听→历史、重复分页均只展示一条；不明记录获得确切证据后更新原关联，不按正文猜匹配 | AC-M1/M2/M3、W5 | +| A12 / P1 / 确定缺陷 | `douyin.py:867–914,1024–1033`、`actions.go:504,549–553`；接收与开始处理时间记晚 | 分开平台时间、推送接收、领取处理、实际写入与结束时间；跳过和失败也有结论 | 注入慢详情、慢 AI 和排队；时间如实暴露延迟。超过五秒记失败，不能靠更换计时点通过；页面时间另验 | AC-A13、B3 | +| A13 / P1 / 未完成能力 | `creator.go:684–720`、`models.go:370–406`；不明缺人工核验及另发关联 | 提供仅记录核验的入口；新发送另确认并关联原操作和原因 | 核验成功/未发送/仍不明均不触发写入；另发有新标识、新确认、原操作及原因；旧标识永不重发 | W3；AC-W5/M3、B2 | +| A14 / P1 / 未完成能力 | `creator.go:722–803`、`douyin.py:1712`；仅已知会话有限历史，无继续位置 | 按平台可提供范围发现会话并分页读取历史，准确表示截断与更多消息 | 本地空库可列出平台可见既有会话；超过当前上限可继续读取，页间去重；不可见历史说明限制；不能截断后写“无更多” | AC-M1、U5 | +| A15 / P2 / 确定缺陷 | `actions.go:398–437,538–540`；前序策略跳过原因丢失 | 保存实际检查过的策略与跳过理由，不新增通用审计框架 | 第一策略不可用、第二策略选中后，两者顺序与原因均可查；不保存无关私信全文或凭据 | A3；AC-A6 | +| A16 / P2 / 确定缺陷 | `creator_events.go:247,295–310`、`listener.go:43–49`；新基线覆盖历史缺口 | 当前连接状态与持久历史缺口分开显示 | 断连→新基线 ready 后仍可查旧至新边界间未知范围；重启不抹除缺口，不把连接恢复表述为无遗漏 | A6；AC-A12、B1 | + +## 5. 采集、素材、页面与环境修复清单 + +| ID / 优先级 / 类型 | 评审定位与问题 | 修复目标 | 最小回归与通过标准 | 原需求 | +| --- | --- | --- | --- | --- | +| B01 / P1 / 确定缺陷 | `creator_material.go:180–196`；`.wav.tmp` 无显式输出格式 | 提音产出有效 WAV，完成后原子发布，失败清理临时产物 | 本地合成短音视频调用真实 FFmpeg；旧命令失败、新命令成功，音频可读取；失败不标完成 | AC-C8/C10、B6 | +| B02 / P1 / 确定缺陷 | `creator_material.go:152–156`、`content.go:562,626–628`；把文件路径作为转写正文 | 区分正文与产物引用，生成时取得真实语音文字 | 测试生成器收到预期正文且不是路径；文件缺失/不可读明确失败，无隐藏描述替代；真实样本另外核对 | C4；AC-C8/C9 | +| B03 / P1 / 确定缺陷 | `creator.go:435–460`、`content.go:533–638`；素材/生成缺服务端领取和防重复 | 一次选取/确认仅启动一次任务；仅显式重试失败步骤 | 两页面并发、双击、网络重试、重开/进程恢复不重复调用下载/转写/生成；成功产物复用，无共享临时文件互相覆盖；在途不明不盲目重启收费调用 | AC-C7/C8/C9、U2 | +| B04 / P1 / 确定缺陷 | `metrics.go:68–78,170–215`、`settings.go:91–99`；旧作品后续间隔错误 | 后续指标计划始终按发布时间和当前时刻计算一致的未来点 | 第四小时首次发现立即采样→第七小时→第十五小时，而非第九小时；设置修改后也符合新参数的未来累计点 | AC-C4/C5、B4 | +| B05 / P1 / 确定缺陷 | `metrics.go:15,155–166`、`creator.go:1438–1522`；派发绕过暂停/年龄/阻断 | 在平台读取前检查来源启停、年龄和人工阻断条件 | 暂停来源、已满监测年龄、需人工登录/验证时平台调用为零；解除后最多采一次当前值，不回放历史计划；多来源按实际有效来源处理 | AC-C3/C5、B4 | +| B06 / P1 / 确定缺陷 | `collection.go:417–429`、`content.go:274–276`；发现路径改写已有作品指标 | 新作品发现与指标采样职责分开，数值和采样时间一致 | 第 1 小时指标采样后，第 1.5 小时发现检查不覆盖已有指标；首次作品可保存当前指标；到期采样同时更新数值/时间 | AC-C4/C5/C6 | +| B07 / P1 / 确定缺陷 | `creator.go:1357–1361,1572–1576`、`collection.go:174–192`、`settings.go:52–108`；计划点冒充采样时间，设置起点不全 | 分离计划点、实际采样时间与固定分页窗口；保存时重设账号/评论未来起点 | 10:00 计划、12:00 恢复只能记真实采样时刻；自有采集不记未来时间；修改间隔后以保存时刻起算，指标仍按发布时间，不回填历史 | AC-C5、U6、B4 | +| B08 / P1 / 确定缺陷 | `collection.go:348`、`creator.go:1423–1430,1580–1598`；仅大号采集，自动选错不可用账号 | 自有一级评论补采不依赖大号模式；区分只读与发送资格；选择真正可用采集账号 | 普通自有账号参与；禁言不因此禁止合法只读，封禁/注销排除；首账号不可用而后续可用时选择后者；全不可用明确阻断 | A1、W1；AC-W1 | +| B09 / P1 / 确定缺陷 | `creator_collector.go:104–122`、`collection.go:410–411`；未来时间作品被丢弃 | 保留未来/无效/缺可靠时间资料为待核验，与已确认范围分开 | 未来、缺时区、无效时间均可查且范围不完整；不安排年龄计划；补可靠时间后按当前范围与未来计划处理，不猜日期 | C2；AC-B4 | +| B10 / P1 / 确定缺陷 | `creator.go:97,108,112`、`dataProvider.js:262–290`；推送换行与拆帧错误 | 两端统一真实 SSE 格式，可靠解析拆包和连续帧，断连可见 | 用真实服务端字节经客户端解析验证更新回调；覆盖逐字节拆包、多帧、心跳和关闭；相关页面更新，不以 reader 创建代表展示成功 | AC-A13、B3 | +| B11 / P1 / 确定缺陷及未完成能力 | `CreatorWorkbenchPage.jsx:497–548,646–712,823–826`；线索联系表单不可达，缺评论作者私信 | 从所选完整评论对象打开共用回复/私信表单，不依赖其他页签当前分页 | 在线索页直接选择同平台账号、目标和文本，确认后操作；跨页评论可用；取消不发送；无 UID 禁止私信；自动冷却不阻止人工发送 | AC-W4、U4 | +| B12 / P1 / 确定缺陷 | `creator_material.go:238–243`、`CreatorCompetitorsPage.jsx:278–284`；失败仍提示完成 | 按全部步骤汇总真实状态，展示具体服务端原因 | 下载/提音/转写分别失败时无成功提示、不能仿写；供应商配置错误与其他失败分开；无音轨/无语音不假装失败 | AC-C10、U2 | +| B13 / P1 / 确定缺陷 | `hub.go:846–861`、`environment.go:285–313`；停用代理主动回收运行环境 | 使用中的代理必须先显式停环境，停用请求不得代替停止/回收 | 运行引用存在时拒绝且列出环境，不改变代理/运行状态；显式停止后可停用；与启动并发也不出现不一致 | E2；AC-E4 | +| B14 / P1 / 未完成能力 | `CreatorCompetitorsPage.jsx:110–137`、`creator.go:253–265`、`content.go:25–42`；链接末段冒充账号身份 | 真实解析主页/可支持分享/短链接，预览规范身份后确认保存 | 同账号不同链接只建一条；作品链接正确解析作者或明确拒绝；无效/身份不明不创建；昵称/稳定 ID/主页/可得头像来自实际结果 | AC-C1 | +| B15 / P1 / 未完成能力 | `CreatorCompetitorsPage.jsx:524–784`、`main.jsx:59–65`；作品详情/筛选/素材预览不全 | 显示标题/正文、账号与时间筛选、目标账号详情，按需读取真实媒体和转写 | 详情不显示全局无关作品；筛选组合正确，返回保留页码；视频/音频/转写可核对；缺产物明确失败，不塞进通用 JSON | C1/C3/C4；AC-U1/U2、B6 | +| B16 / P1 / 未完成能力 | `rules.go:105–125,230–287`、`CreatorWorkbenchPage.jsx:168–180,425–443,835`；分析结果与依据不可持续查看 | 展示已有持久分析状态、规则快照、命中词/理由和必要筛选 | 刷新后区分未分析/非线索/线索/失败；规则改后旧依据不变；按来源、平台、账号、规则、时间过滤；多规则仍一条线索 | AC-W3、U4 | +| B17 / P1 / 未完成能力 | `BrowsersPage.jsx:65–74`、`hub.go:1161–1183,431–434`、`NetworkExitsPage.jsx:60–82`；固定 seed、地区未确认、代理管理不全 | 首次生成并持久化不同 seed；明确地区匹配，否则人工确认；补代理编辑/删除/启用 | 双账号真实浏览器指纹/Profile 隔离且重启/普通升级稳定；未知/多时区地区不猜测；运行修改先停、引用禁止删除、停用不可新启动、重新启用可用 | AC-E1/E2/E4 | +| B18 / P2 / 确定缺陷 | `CreatorWorkbenchPage.jsx:29–33,456–457`、`logic.go:105`；前端 trim 改变关键词语义 | 前后端仅清除词边缘普通空格、Tab、CR/LF | NBSP、全角空格保留；大小写、内部空白、全半角、组合字符保持原文语义；空词拒绝；前端保存与直接 API 一致 | AC-B5 | +| B19 / P2 / 确定缺陷 | `CreatorCompetitorsPage.jsx:237–244`、`CreatorAccountsPage.jsx:174–187`、`CreatorWorkbenchPage.jsx:258–267`；跨路由离开丢输入 | 所有有编辑内容的相关表单在路由/页签/目标切换和浏览器离开时保护输入 | 可取消离开继续编辑或明确放弃；保存失败不离开且保留输入；草稿不跨账号/目标，不把保存草稿当作发送确认；键盘和焦点可用 | AC-U2/U3/U4/U5/U6 | + +## 6. 测试、部署与恢复修复清单 + +| ID / 优先级 / 类型 | 评审定位与问题 | 修复目标 | 最小回归与通过标准 | 依据 | +| --- | --- | --- | --- | --- | +| R01 / P1 / 确定缺陷 | 与 B01 相同：提音命令实际失败 | 合并在 B01 修复，不重复建任务或重复计完成量 | 引用 B01 同一测试及证据，再完成标准镜像真实素材验证 | AC-C8/C10 | +| R02 / P1 / 未完成能力 | `Dockerfile:18–23`、`requirements-gateway.lock:1`、`compose.yaml:13–19`、`creator_material.go:215–233`；标准镜像无可用转写入口 | 先确认现有配置/批准供应商能力，再将可执行入口及锁定依赖纳入部署，不只填写环境变量 | 在最终镜像内用批准样本完成转写；核对所选工具真实参数与输出格式，不能未经验证将 stdout 当正文；无语音/失败分别正确；服务商、模型、参数及费用有批准记录 | C4；G0、AC-C8/C10 | +| R03 / P1 / 缺验收证据及门槛缺陷 | Python 舍入放行；29 个数据库测试跳过 | 生产覆盖率按原始计数验收，专用数据库测试实际执行，门禁拒绝假完整通过 | 第 7 节全部执行；生产代码覆盖不低于 65%；记录 executed/skipped，目标数据库测试零跳过；缺库明确阻塞，不以 exit 0 通过 | AGENTS；plan01 第 9 节 | +| R04 / P1 / 未完成能力与缺验收证据 | `deployment.md:328–444`、`compose.yaml:18–19`；备份仅数据库 | 明确数据库、credentials、materials、独立 Profile、原主密钥及版本的完整恢复清单 | 获准隔离环境做同版本停写一致备份与恢复；账号凭据可解密、素材可读、Profile 不串号、去重/冷却保留，恢复不补发；日志不含秘密 | 发布恢复要求；AC-A9、B1/B2/B6 | +| R05 / P1 / 缺验收证据 | 真实记录使用外部 CDP；缺当前镜像和全业务证据 | 固定发布提交/镜像摘要，完成第 8–9 节真实验证 | 当前 Compose 镜像健康/正常退出、真实环境与账号、逐项 AC、恢复均有证据；旧样本不能代替新入口验证 | plan01 G0/G1；全部抖音适用 AC | +| R06 / P2 / 未完成交付能力 | 仓库内未见 CI 工作流或统一门禁；构建不运行测试 | 复用现有命令形成最小可重复门禁，确认是否已有仓库外 CI，不另建平台 | 注入测试失败、缺数据库、低覆盖率，门禁均失败；成功记录提交、工具版本、执行/跳过计数和镜像；构建成功不等于验收通过 | AGENTS；第 7、9 节 | +| R07 / P2 / 确定文档缺陷 | `docker/browser-wrapper/README.md:8` 使用 digest 作构建 tag；`deployment.md:89,382,429` 旧 schema 检查 | 构建用合法 tag，发布后记录真实 digest；按发布版本列实际 schema 和恢复检查 | 对照最终发布版本复核全部命令；构建/发布后摘要可读取；schema 与该版本迁移一致，不能只检查旧的 12/14，也不永远写死当前 31 | 部署说明及发布版本一致性 | + +### 6.1 原评审覆盖映射 + +- 平台报告：原 P0-1→A01，P0-2→A02；P1-1 至 P1-12 依次→A03 至 A14；P2-1→A15，P2-2→A16。 +- 业务报告：发现 1 至 19 依次→B01 至 B19。 +- 发布报告：R1 至 R7 依次→R01 至 R07；其中 R01 与 B01 合并执行。 +- 来源报告中的“已有基础”仅作为复用依据，不作为新增修复条目，也不免除原 AC 回归。仅缺真实证据的采集、指标、媒体、AI、环境及人工动作纳入第 8 节。 + +## 7. 本地回归与质量门禁 + +### 7.1 每项修复的基本要求 + +1. 针对触发条件写最小回归，保留旧实现失败、新实现通过的证据；不能只新增固定成功 Mock 或改变原需求相反的测试预期。 +2. 状态/时间/并发测试用可控时钟和阻塞点复现，不靠长时间 sleep 赌竞争。数据库约束与跨进程恢复须在专用 PostgreSQL 实际验证。 +3. 涉及平台的测试替身仅用于确定性成功、校验、失败和不明路径；不能冒充平台能力证据。禁止真实测试发送给无关用户。 +4. 关键记录能关联账号、事件/操作、目标、策略/规则、冷却、阶段、原因及必要时间；保留排查链，不把密码、身份证、Cookie、代理凭据或无关完整私信写入日志。 +5. 复用既有依赖前核对文档和类型；新增依赖必须说明现有能力不足并锁定。页面遵循现有布局与 RemixIcon，保证加载、空、失败、禁用、确认/取消及键盘交互。 + +### 7.2 命令与验收口径 + +以下是**待执行标准**，不是本次文档编写已通过的记录。专用数据库必须先获准；不得借用现有业务库或静默新建/清空库。 + +| 范围 | 必须执行 | 通过标准 | +| --- | --- | --- | +| Go | `go test ./...`；`go vet ./...`；`go build ./cmd/control-plane`；并发/生命周期/共享状态变更运行 `go test -race ./...` | 全部退出成功；发布回归运行 race 全量；数据库目标测试实际执行，不以 skip 通过 | +| Go 覆盖 | 配置获准 `CREATORHUB_POSTGRES_TEST_URL` 后，`go test -count=1 -json -coverprofile=<证据目录>/go.cover ./...`；`go tool cover -func=<证据目录>/go.cover` | 保留每包分子/分母、执行/跳过项，整体生产语句覆盖至少 65%；不因涉及小红书就临时剔除共用测试以提高数字 | +| Python | `python3 -m unittest discover -s cmd -p 'test_*.py'`；同次 `python3 -m coverage run --source=cmd/docker_gateway --omit='cmd/docker_gateway/test_*.py' -m unittest discover -s cmd -p 'test_*.py'` | 全部通过；测试代码不计生产覆盖 | +| Python 覆盖 | `python3 -m coverage report --precision=2 --fail-under=65`；`python3 -m coverage json -o <证据目录>/python-coverage.json` | report 通过且从 JSON 原始计数检查 `covered_lines * 100 >= num_statements * 65`；不能仅靠显示值或 report 舍入放行 | +| 前端 | `npm --prefix web ci`；`npm --prefix web test`；`npm --prefix web run test:coverage`；`npm --prefix web run build` | lockfile 全新安装、测试与构建通过；沿用现有 lines 门槛至少 65%，同时披露 statements/branches/functions 及文件范围，不声称未纳入入口已覆盖 | +| Docker/Compose | `docker compose config --quiet`；使用开发覆盖文件时也验证合并配置；获准环境完成实际构建与健康检查 | 配置解析、镜像构建、服务 ready/healthy 分别通过；配置检查不代替容器验证 | +| 变更检查 | `git diff --check`；适用源码诊断及仓库已有静态检查 | 无新增错误；命令失败或覆盖缺口明确处理，不吞掉错误 | + +命令中的 `<证据目录>` 必须替换为明确路径。覆盖数据先清理独立输出文件再执行，避免拼接旧结果；保存运行时版本、命令、配置名称及脱敏结果，不保存秘密值。数据库未配置、依赖未安装或环境未授权属于**阻塞/未执行**，不是通过，也不是业务服务故障。 + +不新增“每文件、每分支均 65%”的未经确认门槛；但核心缺陷必须有对应回归，不能仅靠无关测试拉高总数。Go 总覆盖率通过也不能替代逐项数据库测试实际执行。 + +## 8. 真实平台与页面验收矩阵 + +### 8.1 输入与授权包 + +真实验收前登记：一个大号、两个小号、可控互动账号、同平台目标关系；只读竞品样本、超过一页作品与一级评论；允许的写动作、目标、内容与次数；测试环境与时间窗口。凭据留在批准的秘密配置，不写入文档。 + +AI/转写须先批准供应商、模型/版本、参数、提示要求、费用边界及脱敏样本。样本逐条标注主题/非主题、包含/排除、线索/非线索和可接受误差；转写包括清晰语音、背景音乐、无语音，仿写核对标题/口播、用户要求和来源。不得自行编造准确率或误差率阈值。 + +### 8.2 逐项必验 + +下列每个子项均须单独记录，不能以复合行中的一次成功代表全部。 + +| 验收组 | 必测子项 | 必须观察到的结果 | 关联 | +| --- | --- | --- | --- | +| T01 身份与登录 | 有效会话、失效会话、人工登录失败/挑战、实际身份冲突；资料新增编辑与密码空白 | 同环境人工处理；不自动注入登录；UID 不符阻止写；业务/登录/实名状态分开；读取/日志不回显凭据 | AC-A1/A2、U3 | +| T02 竞品与分页 | 主页、分享、短链接、重复/无效身份；多页作品和一级评论、中断恢复、双来源 | 真实目标预览；范围内平台可提供数据无漏页/重复，只采一级评论；不可取得说明限制；不把首屏或前 N 条当完成 | AC-C1/C2/C3、W1、B6 | +| T03 指标与计划 | 真实零/缺失/下降；阈值等值与多条件；UTC/夏令时、窗口/年龄边界、旧作品、停用/恢复/设置保存 | 数值真实、计划独立；1/3/7/15/31/55 小时序列及封顶、停止优先；不造历史采样，未来时间待核验 | AC-C4/C5/C6、B4 | +| T04 素材与 AI | 未选取、选取未确认生成、两次确认、下载/提音/转写/生成；无音轨/无语音、失败、磁盘/配额不足、重复点击 | 未确认不提前执行；真实产物和正文可预览；成功步骤复用；失败不允许仿写、不假成功；无音轨/无语音如实展示可继续；不自动发布/删素材 | AC-C7/C8/C9/C10、U2、B6 | +| T05 线索与人工联系 | 主题→关键词→评论 AI；禁用/修改/显式重分析、多规则、失败；评论/线索回复与作者私信 | 判断顺序和历史依据正确；失败不是非线索；同评论一条线索；人工逐次确认且不受自动冷却限制 | AC-W2/W3/W4、U4、B5 | +| T06 四类互动 | **评论、点赞、转发、关注**分别触发；大号主动动作对照；缺 ID/UID/目标 | 记录稳定事件身份、接收账号、互动者和目标；不把主动行为当收到互动；缺可靠必需字段阻止自动执行并说明 | AC-A5、B1 | +| T07 六种写操作 | **私信、回复评论、点赞评论、点赞作品、关注用户、转发作品**分别测试成功/拒绝/不明；不相容事件目标 | 逐项实际账号、目标、文本与结果可核验；不猜目标；不明不重试、不返回假成功；平台限制明确留证 | AC-A7、W5、M3、B2 | +| T08 策略与冷却 | 首条不可用/后条可用；选中后失败；候选与 AI；同 UID 跨事件/小号/动作并发、不同 UID/大号、到期、修改时长 | 一次只选一个小号一个动作；选中失败不切换；冷却正确且重启/失败/不明保留原到期；到期仅新事件可触发 | AC-A4/A6/A8/A9/A10/A11 | +| T09 边界与恢复 | 首次历史、停用再启用、解析失败、断网、登录失效、进程退出/恢复、重复/迟到事件 | 无可靠边界不发；旧事件永不重放；迟到默认只展示；缺口持久可见；恢复新边界后仅可靠新事件可执行 | AC-A12、B1 | +| T10 同账号写协调 | 人工与自动竞争、排队中停用/改关系/换身份、超过恢复窗口、发送后确认丢失、SDK 超时 | 全部写操作不重叠；执行前重核；不明后不新开始原操作；核验不发送；另发新确认并关联原操作 | AC-A14、W5、M3、B2 | +| T11 时限与页面 | 四类互动及新私信；页面关闭、正常打开、独立断连、重开、慢详情/AI | 后台接收后五秒内开始处理；正常已开页面三十秒内可见；关页不停止监听；重开读持久结果；慢 AI 不冒充未接收 | AC-A13、B3 | +| T12 私信 | 两账号会话发现、超过单次上限历史、收发方向、非文本、发送→事件→历史、旧请求迟返、草稿切换 | 会话/草稿不串号；平台可得历史可继续且范围准确；一条真实消息一条展示；非文本有类型；无自动聊天 | AC-M1/M2/M3、U5 | +| T13 环境与代理 | 双账号首次 seed、真实参数、重启/升级、Profile；明确/未知/多时区地区；HTTP/HTTPS/SOCKS4/SOCKS5、合法认证/失败；编辑停启删除引用 | 参数真实稳定且隔离；地区不猜；实际出口符合选择；失败不直连/换代理；运行引用先停，删除先解除；列表不主动检测/轮询 | AC-E1/E2/E3/E4/E5 | +| T14 页面完整性 | 全部列表/详情/任务跳转、筛选分页返回、加载/空/失败/禁用、取消/保存失败/离开、键盘/焦点；设置校验 | 各路径可操作；失败保留数据输入、未保存可取消离开；账号/目标明确；未批准 AI 不调用；关键结果可追溯至任务 | AC-U1/U2/U3/U4/U5/U6 | + +### 8.3 时限和不明结果的记录规则 + +- 分别记录平台事件时间（若提供）、系统实际接收推送、开始处理、页面实际展示、写操作开始和结束。缺平台时间写“不可测”，不得填零。 +- 五秒从系统接收事件计起，不从详情拉取完或 AI 完成才起算；三十秒只对正常连接的已打开相关页面测量。超过就记录未通过，不能通过调整字段含义消除延迟。 +- SDK/CDP 超时不等于平台拒绝或发送取消；截图、服务端消息/动作标识与平台结果按实际证据关联，无法确认保持不明。 +- 错误/断连注入必须有范围和恢复授权；不能为测试随意重发真实写操作。能用本地阻塞点验证的竞争先本地验证，真机再验证必要平台行为。 +- 平台能力状态为“待验证/支持/条件支持/不支持”，验收状态单独为“未执行/阻塞/未通过/通过”。支持不等于 AC 通过，不支持不等于需求已删除。 + +## 9. 发布与恢复检查表 + +以下全部默认为未完成。发布前逐项附证据，不使用本次离线评审结果自动勾选。 + +- [ ] **L01 版本固定**：修复提交、依赖锁文件、构建工具版本、控制面/gateway/浏览器镜像摘要均记录；测试版本与发布版本对应，无未说明工作区差异。 +- [ ] **L02 回归通过**：第 7 节命令实际通过；专用数据库测试没有目标项跳过；覆盖率按同次原始计数达标，关键缺陷回归齐全。 +- [ ] **L03 真机全项通过**：第 8 节及原需求所有抖音适用 AC 有证据；转发和全部入站事件不遗漏；批准样本的 AI/转写质量逐项通过。 +- [ ] **L04 镜像可部署**：从锁定依赖构建当前镜像，包含实际转写运行能力；Compose 解析、构建、启动、依赖连接及 ready/healthy 逐项成功。 +- [ ] **L05 浏览器环境**：标准 Docker gateway 而非仅外部 CDP 验证身份、Profile、指纹、代理、读取/写入路径；声明外部模式证据不能替代的部分。 +- [ ] **L06 停止与恢复**：正常退出保留事件、操作、冷却及素材状态；进程重启/浏览器重建不补发历史，不把在途结果改为成功或可重试。 +- [ ] **L07 完整备份**：获准停写后一致保存数据库、credentials、materials、全部独立 Profile；原主密钥独立安全保管，记录配置与镜像版本，不把秘密提交仓库。 +- [ ] **L08 隔离恢复演练**：在批准的独立目标恢复同版本数据和卷；验证凭据解密、素材读取、Profile 身份、历史记录及冷却/去重。恢复实例不得与原实例同时消费同一真实账号并产生发送。 +- [ ] **L09 失败处理**：预先记录如何停止新自动动作、保留在途证据、定位失败和恢复同版本；不擅自跨 schema 回滚、不自动删除数据或重放写任务。 +- [ ] **L10 文档与决定一致**：部署命令有效、schema 对应该版本、构建 tag 与发布 digest 用法正确；能力限制、恢复缺口、范围变更及剩余风险经使用者确认。 + +**发布决策：** 任一必须项未通过或未获批准则阻塞完整发布。若使用者另行批准受限试用，必须新写明确可用范围、禁用能力、账号/数据边界和退出条件；本计划不默认授权降级发布。 + +## 10. 修复与验收记录模板 + +### 10.1 单项修复记录 + +每个 A/B/R 编号保留以下字段；R01 可直接关联 B01，但不得缺少镜像验收。 + +| 字段 | 填写内容 | +| --- | --- | +| 编号 / 原 AC | 例如 A09 / AC-B2 | +| 状态 | 待修复、修复中、待验收、阻塞、验收未通过、验收通过 | +| 根因与实际改动 | 当前代码复核结果、修改文件，不只写症状消失 | +| 修复提交 | 完整提交标识；相关工作区差异 | +| 回归 | 旧实现失败、新实现通过的命令/测试及日志 | +| 成功/校验/失败/不明/恢复 | 适用场景结果；不适用必须解释 | +| 真实证据 | 若适用，关联第 10.2 节；没有证据写待验收 | +| 复核 | 独立复核人/任务、日期、发现与解决记录 | +| 剩余限制与批准 | 范围调整须有使用者明确批准引用，否则保持阻塞 | + +### 10.2 单次验收证据 + +正式证据建议保存为 `docs/evidence/douyin-release-<日期>-<提交短号>.md`，附件按需保存脱敏内容；不要求新建证据服务。 + +```text +验收编号：Txx / AC-xxx / 修复编号 +测试日期与时区： +代码提交 / 工作区差异： +镜像摘要 / 客户端版本 / 工具版本： +环境与授权引用： +脱敏账号角色 / 目标 / 输入样本： +前置状态与边界（策略、冷却、监听、登录等）： +执行步骤（真实写次数、内容确认）： +预期结果： +实际结果： +时间证据（平台、接收、处理、页面、写入、结束）： +平台结果及脱敏证据路径： +能力状态：待验证 / 支持 / 条件支持 / 不支持 +验收状态：未执行 / 阻塞 / 未通过 / 通过 +失败原因 / 恢复过程 / 数据缺口： +范围或恢复行为变更批准：无 / 明确引用 +``` + +### 10.3 当前计划状态 + +- 本轮已实施本地代码修复、迁移、回归测试、覆盖率门禁、Compose 构建/健康检查和 CI 工作流；未部署生产。 +- A03–A16、B01、B04、B07、B08、B10、B12、B13、B17、B18 具有本地代码/回归证据，但仍不能替代真实平台验收；A02 仍保留 `ACTION_UNAVAILABLE`。 +- A01、A02、B02、B03、B05、B06、B09、B11、B14–B16、B19 以及 T01–T14 的真实平台/页面矩阵未完成；R02、R04–R07 未完成，R03 的本地门禁已通过但 CI/发布产物尚未验收。 +- 真实平台操作必须经 CreatorHub 系统执行。此前误做的直接抖音点赞/评论已撤销，不计入证据；本轮未通过 CreatorHub 执行真实写矩阵。 +- 使用者仍须批准供应商、模型、参数、费用、停写备份/隔离恢复环境及发布部署；没有这些前置条件不得宣称完整发布。 +- 后续若关键技术栈或产品方向改变，同步更新 `AGENTS.md`；本计划维持现有方向与技术栈。 diff --git a/internal/creator/accounts.go b/internal/creator/accounts.go index 9b01e2b..cc84800 100644 --- a/internal/creator/accounts.go +++ b/internal/creator/accounts.go @@ -178,7 +178,7 @@ func (s *Store) UpdateAccountProfile(ctx context.Context, accountID string, inpu } } if !input.BigAccount || input.BusinessStatus != "normal" { - if _, err := tx.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1`, accountID); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1 OR execution_account_id=$1`, accountID); err != nil { return AccountProfile{}, databaseError(err) } } @@ -191,6 +191,9 @@ func (s *Store) UpdateAccountProfile(ctx context.Context, accountID string, inpu return AccountProfile{}, fmt.Errorf("replace account password: remove old secret: %w", err) } } + if err := s.InvalidateListener(ctx, accountID, "账号配置变更"); err != nil { + return AccountProfile{}, err + } return s.GetAccountProfile(ctx, accountID) } @@ -234,6 +237,14 @@ func (s *Store) recordLoginResult(ctx context.Context, accountID, status, reason if err != nil { return LoginResult{}, databaseError(err) } + if status != "logged_in" { + if _, err := s.db.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1 OR execution_account_id=$1`, accountID); err != nil { + return LoginResult{}, databaseError(err) + } + } + if err := s.InvalidateListener(ctx, accountID, "登录状态变更"); err != nil { + return LoginResult{}, err + } return LoginResult{AccountID: accountID, Status: status, Reason: reason, ActualKey: actualKey, CheckedAt: now}, nil } @@ -262,13 +273,16 @@ func (s *Store) SetBigAccount(ctx context.Context, accountID string, enabled boo return AccountProfile{}, databaseError(err) } if !enabled { - if _, err := tx.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1`, accountID); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1 OR execution_account_id=$1`, accountID); err != nil { return AccountProfile{}, databaseError(err) } } if err := tx.Commit(); err != nil { return AccountProfile{}, fmt.Errorf("commit creator big-account update: %w", err) } + if err := s.InvalidateListener(ctx, accountID, "大小号模式变更"); err != nil { + return AccountProfile{}, err + } return s.GetAccountProfile(ctx, accountID) } @@ -384,7 +398,7 @@ func (s *Store) SetRelation(ctx context.Context, bigAccountID, smallAccountID st if err := tx.Commit(); err != nil { return fmt.Errorf("commit creator relation: %w", err) } - return nil + return s.InvalidateListener(ctx, bigAccountID, "账号关系变更") } func (s *Store) AccountWriteCheck(ctx context.Context, accountID string, automatic bool, action string) (AccountProfile, error) { diff --git a/internal/creator/actions.go b/internal/creator/actions.go index 731e5b6..1adaec7 100644 --- a/internal/creator/actions.go +++ b/internal/creator/actions.go @@ -139,6 +139,11 @@ func (s *Store) CreateStrategy(ctx context.Context, bigAccountID string, input S if _, err := s.db.ExecContext(ctx, `INSERT INTO creator_strategy (id,big_account_id,execution_account_id,position,enabled,event_types,action,target_type,candidate_texts) VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9::jsonb)`, id, bigAccountID, input.ExecutionAccountID, input.Position, input.Enabled, events, input.Action, input.TargetType, texts); err != nil { return Strategy{}, databaseError(err) } + if input.Enabled { + if err := s.InvalidateListener(ctx, bigAccountID, "策略新增"); err != nil { + return Strategy{}, err + } + } return s.GetStrategy(ctx, id) } @@ -157,6 +162,27 @@ func (s *Store) GetStrategy(ctx context.Context, id string) (Strategy, error) { result, err := scanStrategy(s.db.QueryRowContext(ctx, strategySelect+` WHERE id=$1`, id)) return result, rowError(err) } +func (s *Store) ListStrategyTraces(ctx context.Context, eventID string) ([]StrategyTrace, error) { + eventID = strings.TrimSpace(eventID) + if eventID == "" { + return nil, ErrInvalid + } + rows, err := s.db.QueryContext(ctx, `SELECT event_id,strategy_id,position,outcome,reason,created_at FROM creator_event_strategy_trace WHERE event_id=$1 ORDER BY position,id`, eventID) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]StrategyTrace, 0) + for rows.Next() { + var trace StrategyTrace + if err := rows.Scan(&trace.EventID, &trace.StrategyID, &trace.Position, &trace.Outcome, &trace.Reason, &trace.CreatedAt); err != nil { + return nil, err + } + result = append(result, trace) + } + return result, rows.Err() +} + func (s *Store) ListStrategies(ctx context.Context, bigID string) ([]Strategy, error) { query := strategySelect args := []any{} @@ -213,6 +239,9 @@ func (s *Store) UpdateStrategy(ctx context.Context, id string, input StrategyInp if _, err := s.db.ExecContext(ctx, `UPDATE creator_strategy SET execution_account_id=$2,position=$3,enabled=$4,event_types=$5::jsonb,action=$6,target_type=$7,candidate_texts=$8::jsonb,updated_at=now() WHERE id=$1`, id, input.ExecutionAccountID, input.Position, input.Enabled, events, input.Action, input.TargetType, texts); err != nil { return Strategy{}, databaseError(err) } + if err := s.InvalidateListener(ctx, strategy.BigAccountID, "策略修改"); err != nil { + return Strategy{}, err + } return s.GetStrategy(ctx, id) } func (s *Store) SetStrategyEnabled(ctx context.Context, id string, enabled bool) (Strategy, error) { @@ -228,6 +257,9 @@ func (s *Store) SetStrategyEnabled(ctx context.Context, id string, enabled bool) if _, err := s.db.ExecContext(ctx, `UPDATE creator_strategy SET enabled=$2,updated_at=now() WHERE id=$1`, id, enabled); err != nil { return Strategy{}, databaseError(err) } + if err := s.InvalidateListener(ctx, strategy.BigAccountID, "策略启停变更"); err != nil { + return Strategy{}, err + } return s.GetStrategy(ctx, id) } func (s *Store) DeleteStrategy(ctx context.Context, id string) error { @@ -239,7 +271,7 @@ func scanEvent(scanner interface{ Scan(...any) error }) (InteractionEvent, error var result InteractionEvent var platformAt, gatewayReceivedAt, receivedAt, startedAt, finishedAt, displayedAt sql.NullTime var commentID, workID, messageType, messageText, strategyID, executionID sql.NullString - if err := scanner.Scan(&result.ID, &result.Platform, &result.ReceivingAccountID, &result.EventKey, &result.EventType, &result.InteractorUID, &commentID, &workID, &messageType, &messageText, &platformAt, &gatewayReceivedAt, &receivedAt, &startedAt, &finishedAt, &displayedAt, &result.State, &result.Reason, &strategyID, &executionID); err != nil { + if err := scanner.Scan(&result.ID, &result.Platform, &result.Generation, &result.ReceivingAccountID, &result.EventKey, &result.EventType, &result.InteractorUID, &commentID, &workID, &messageType, &messageText, &platformAt, &gatewayReceivedAt, &receivedAt, &startedAt, &finishedAt, &displayedAt, &result.State, &result.Reason, &strategyID, &executionID); err != nil { return InteractionEvent{}, err } result.CommentID, result.WorkID, result.MessageType, result.MessageText = commentID.String, workID.String, messageType.String, messageText.String @@ -251,7 +283,7 @@ func scanEvent(scanner interface{ Scan(...any) error }) (InteractionEvent, error return result, nil } -const eventSelect = `SELECT id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_type,message_text,platform_event_at,gateway_received_at,received_at,processing_started_at,processing_finished_at,displayed_at,state,reason,strategy_id,execution_account_id FROM creator_event` +const eventSelect = `SELECT id,platform,generation,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_type,message_text,platform_event_at,gateway_received_at,received_at,processing_started_at,processing_finished_at,displayed_at,state,reason,strategy_id,execution_account_id FROM creator_event` func (s *Store) GetEvent(ctx context.Context, id string) (InteractionEvent, error) { result, err := scanEvent(s.db.QueryRowContext(ctx, eventSelect+` WHERE id=$1`, id)) @@ -282,11 +314,11 @@ func (s *Store) ListEvents(ctx context.Context, accountID string) ([]Interaction } func (s *Store) RecordEvent(ctx context.Context, input InteractionEvent) (AutomaticResult, error) { - input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageType, input.MessageText = strings.TrimSpace(input.Platform), strings.TrimSpace(input.ReceivingAccountID), strings.TrimSpace(input.EventKey), strings.TrimSpace(input.EventType), strings.TrimSpace(input.InteractorUID), strings.TrimSpace(input.CommentID), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.MessageType), strings.TrimSpace(input.MessageText) + input.Platform, input.Generation, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageType, input.MessageText = strings.TrimSpace(input.Platform), strings.TrimSpace(input.Generation), strings.TrimSpace(input.ReceivingAccountID), strings.TrimSpace(input.EventKey), strings.TrimSpace(input.EventType), strings.TrimSpace(input.InteractorUID), strings.TrimSpace(input.CommentID), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.MessageType), strings.TrimSpace(input.MessageText) if input.MessageType == "" { input.MessageType = MessageTypeText } - if !ValidatePlatform(input.Platform) || input.ReceivingAccountID == "" || input.EventKey == "" || !ValidEventType(input.EventType) || !ValidMessageType(input.MessageType) || len(input.MessageText) > 100000 { + if !ValidatePlatform(input.Platform) || len(input.Generation) > 500 || input.ReceivingAccountID == "" || input.EventKey == "" || !ValidEventType(input.EventType) || !ValidMessageType(input.MessageType) || len(input.MessageText) > 100000 { return AutomaticResult{}, ErrInvalid } profile, err := s.GetAccountProfile(ctx, input.ReceivingAccountID) @@ -316,7 +348,7 @@ func (s *Store) RecordEvent(ctx context.Context, input InteractionEvent) (Automa } var returnedID string var inserted bool - err = s.db.QueryRowContext(ctx, `INSERT INTO creator_event (id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_type,message_text,platform_event_at,gateway_received_at,received_at,state,reason) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (platform,receiving_account_id,event_key) DO NOTHING RETURNING id,(xmax=0)`, id, input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageType, input.MessageText, input.PlatformEventAt, input.GatewayReceivedAt, receivedAt, state, reason).Scan(&returnedID, &inserted) + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_event (id,platform,generation,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_type,message_text,platform_event_at,gateway_received_at,received_at,state,reason) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (platform,receiving_account_id,event_key) DO NOTHING RETURNING id,(xmax=0)`, id, input.Platform, input.Generation, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageType, input.MessageText, input.PlatformEventAt, input.GatewayReceivedAt, receivedAt, state, reason).Scan(&returnedID, &inserted) if errors.Is(err, sql.ErrNoRows) { existingErr := s.db.QueryRowContext(ctx, `SELECT id FROM creator_event WHERE platform=$1 AND receiving_account_id=$2 AND event_key=$3`, input.Platform, input.ReceivingAccountID, input.EventKey).Scan(&returnedID) if existingErr != nil { @@ -365,8 +397,8 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven return AutomaticResult{}, fmt.Errorf("begin automatic event: %w", err) } defer tx.Rollback() - var storedPlatform, storedReceivingAccountID, storedEventKey, storedEventType, storedInteractorUID, storedCommentID, storedWorkID, eventState string - if err := tx.QueryRowContext(ctx, `SELECT platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,state FROM creator_event WHERE id=$1 FOR UPDATE`, eventID).Scan(&storedPlatform, &storedReceivingAccountID, &storedEventKey, &storedEventType, &storedInteractorUID, &storedCommentID, &storedWorkID, &eventState); err != nil { + var storedPlatform, storedGeneration, storedReceivingAccountID, storedEventKey, storedEventType, storedInteractorUID, storedCommentID, storedWorkID, eventState string + if err := tx.QueryRowContext(ctx, `SELECT platform,generation,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,state FROM creator_event WHERE id=$1 FOR UPDATE`, eventID).Scan(&storedPlatform, &storedGeneration, &storedReceivingAccountID, &storedEventKey, &storedEventType, &storedInteractorUID, &storedCommentID, &storedWorkID, &eventState); err != nil { return AutomaticResult{}, rowError(err) } if eventState != "received" { @@ -375,6 +407,7 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven return AutomaticResult{Event: event, Duplicate: recorded.Duplicate}, getErr } input.Platform = storedPlatform + input.Generation = storedGeneration input.ReceivingAccountID = storedReceivingAccountID input.EventKey = storedEventKey input.EventType = storedEventType @@ -385,6 +418,24 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven if err := s.scanAccountTx(ctx, tx, input.ReceivingAccountID, &bigProfile); err != nil { return AutomaticResult{}, err } + if storedGeneration != "" { + var listenerStatus, listenerGeneration string + var listenerInvalidated bool + listenerErr := tx.QueryRowContext(ctx, `SELECT status,generation,invalidated FROM creator_listener_state WHERE account_id=$1`, input.ReceivingAccountID).Scan(&listenerStatus, &listenerGeneration, &listenerInvalidated) + if listenerErr != nil && !errors.Is(listenerErr, sql.ErrNoRows) { + return AutomaticResult{}, databaseError(listenerErr) + } + if listenerErr == nil && (listenerStatus != "ready" || listenerGeneration != storedGeneration || listenerInvalidated) { + if _, updateErr := tx.ExecContext(ctx, `UPDATE creator_event SET state='blocked',reason='监听代际未就绪' WHERE id=$1 AND state='received'`, eventID); updateErr != nil { + return AutomaticResult{}, databaseError(updateErr) + } + if err := tx.Commit(); err != nil { + return AutomaticResult{}, err + } + event, getErr := s.GetEvent(ctx, eventID) + return AutomaticResult{Event: event}, getErr + } + } if !bigProfile.BigAccount { if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='ignored', reason='大号模式未开启' WHERE id=$1`, eventID); err != nil { return AutomaticResult{}, databaseError(err) @@ -402,13 +453,31 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven if err != nil { return AutomaticResult{}, err } + tracePosition := 0 + trace := func(strategyID, outcome, traceReason string) error { + tracePosition++ + _, traceErr := tx.ExecContext(ctx, `INSERT INTO creator_event_strategy_trace (event_id,strategy_id,position,outcome,reason) VALUES ($1,$2,$3,$4,$5)`, eventID, strategyID, tracePosition, outcome, traceReason) + return databaseError(traceErr) + } for _, strategy := range strategies { - if !strategy.Enabled || !contains(strategy.EventTypes, input.EventType) { + if !strategy.Enabled { + if err := trace(strategy.ID, "skipped", "策略未启用"); err != nil { + return AutomaticResult{}, err + } + continue + } + if !contains(strategy.EventTypes, input.EventType) { + if err := trace(strategy.ID, "skipped", "事件类型不匹配"); err != nil { + return AutomaticResult{}, err + } continue } var profile AccountProfile if err := s.scanAccountTx(ctx, tx, strategy.ExecutionAccountID, &profile); err != nil { reason = "执行账号不可用" + if traceErr := trace(strategy.ID, "skipped", reason); traceErr != nil { + return AutomaticResult{}, traceErr + } continue } var related bool @@ -417,21 +486,36 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven } if !related { reason = "执行账号未绑定" + if traceErr := trace(strategy.ID, "skipped", reason); traceErr != nil { + return AutomaticResult{}, traceErr + } continue } if err := CanWrite(profile, true, strategy.Action); err != nil { reason = "执行账号不可用" + if traceErr := trace(strategy.ID, "skipped", reason); traceErr != nil { + return AutomaticResult{}, traceErr + } continue } if !ActionTargetValid(strategy.Action, input.InteractorUID, input.CommentID, input.WorkID, strategy.TargetType) { reason = "事件缺少动作目标" + if traceErr := trace(strategy.ID, "skipped", reason); traceErr != nil { + return AutomaticResult{}, traceErr + } continue } if ActionRequiresText(strategy.Action) && len(strategy.CandidateTexts) == 0 && strings.TrimSpace(bigProfile.ReplyRequirements) == "" { reason = "缺少候选文本和大号 AI 回复要求" + if traceErr := trace(strategy.ID, "skipped", reason); traceErr != nil { + return AutomaticResult{}, traceErr + } continue } chosen, execution = strategy, profile + if err := trace(strategy.ID, "selected", ""); err != nil { + return AutomaticResult{}, err + } break } if chosen.ID == "" { @@ -457,7 +541,8 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven } expires := now.Add(time.Duration(cooldownSeconds) * time.Second) var cooldownID string - if err := tx.QueryRowContext(ctx, `INSERT INTO creator_cooldown (big_account_id,interactor_uid,event_id,strategy_id,execution_account_id,started_at,expires_at) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (big_account_id,interactor_uid) DO UPDATE SET event_id=EXCLUDED.event_id,strategy_id=EXCLUDED.strategy_id,execution_account_id=EXCLUDED.execution_account_id,started_at=EXCLUDED.started_at,expires_at=EXCLUDED.expires_at WHERE creator_cooldown.expires_at <= $6 RETURNING event_id`, input.ReceivingAccountID, input.InteractorUID, eventID, chosen.ID, execution.ID, now, expires).Scan(&cooldownID); errors.Is(err, sql.ErrNoRows) { + scanErr := tx.QueryRowContext(ctx, `INSERT INTO creator_cooldown (big_account_id,interactor_uid,event_id,strategy_id,execution_account_id,started_at,expires_at) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (big_account_id,interactor_uid) DO UPDATE SET event_id=EXCLUDED.event_id,strategy_id=EXCLUDED.strategy_id,execution_account_id=EXCLUDED.execution_account_id,started_at=EXCLUDED.started_at,expires_at=EXCLUDED.expires_at WHERE creator_cooldown.expires_at <= $6 RETURNING event_id`, input.ReceivingAccountID, input.InteractorUID, eventID, chosen.ID, execution.ID, now, expires).Scan(&cooldownID) + if errors.Is(scanErr, sql.ErrNoRows) { if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='blocked',reason='自动响应冷却中' WHERE id=$1`, eventID); err != nil { return AutomaticResult{}, databaseError(err) } @@ -470,8 +555,8 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven } return AutomaticResult{Event: event}, nil } - if err != nil { - return AutomaticResult{}, databaseError(err) + if scanErr != nil { + return AutomaticResult{}, databaseError(scanErr) } text, selectionErr := selectCandidate(chosen.CandidateTexts) if selectionErr != nil { @@ -487,61 +572,95 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven } return AutomaticResult{Event: event}, selectionErr } - if ActionRequiresText(chosen.Action) && text == "" { - if generator == nil { - if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='blocked',reason='AI 生成不可用',strategy_id=$2,execution_account_id=$3 WHERE id=$1`, eventID, chosen.ID, execution.ID); err != nil { - return AutomaticResult{}, databaseError(err) - } - if err := tx.Commit(); err != nil { - return AutomaticResult{}, err - } - event, getErr := s.GetEvent(ctx, eventID) - if getErr != nil { - return AutomaticResult{}, getErr - } - return AutomaticResult{Event: event}, ErrUnavailable - } - text, err = generator.Generate(ctx, bigProfile.ReplyRequirements, input.EventType) - if err != nil { - if _, updateErr := tx.ExecContext(ctx, `UPDATE creator_event SET state='failed',reason=$2,strategy_id=$3,execution_account_id=$4 WHERE id=$1`, eventID, err.Error(), chosen.ID, execution.ID); updateErr != nil { - return AutomaticResult{}, databaseError(updateErr) - } - if commitErr := tx.Commit(); commitErr != nil { - return AutomaticResult{}, commitErr - } - event, getErr := s.GetEvent(ctx, eventID) - if getErr != nil { - return AutomaticResult{}, getErr - } - return AutomaticResult{Event: event}, err - } - text = strings.TrimSpace(text) - if text == "" { - if _, updateErr := tx.ExecContext(ctx, `UPDATE creator_event SET state='failed',reason='AI 返回空内容',strategy_id=$2,execution_account_id=$3 WHERE id=$1`, eventID, chosen.ID, execution.ID); updateErr != nil { - return AutomaticResult{}, databaseError(updateErr) - } - if commitErr := tx.Commit(); commitErr != nil { - return AutomaticResult{}, commitErr - } - event, getErr := s.GetEvent(ctx, eventID) - if getErr != nil { - return AutomaticResult{}, getErr - } - return AutomaticResult{Event: event}, ErrInvalid - } - } opInput := OperationInput{IdempotencyKey: "event:" + input.Platform + ":" + input.ReceivingAccountID + ":" + input.EventKey, Source: "automatic", Action: chosen.Action, Platform: input.Platform, AccountID: execution.ID, TargetUID: input.InteractorUID, TargetCommentID: input.CommentID, TargetWorkID: input.WorkID, Text: text, EventID: eventID, StrategyID: chosen.ID} opID := newID("operation") hash := operationHash(opInput) if _, err := tx.ExecContext(ctx, `INSERT INTO creator_operation (id,idempotency_key,source,action,platform,account_id,target_uid,target_comment_id,target_work_id,text,event_id,strategy_id,request_hash,state) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,'processing')`, opID, opInput.IdempotencyKey, opInput.Source, opInput.Action, opInput.Platform, opInput.AccountID, opInput.TargetUID, opInput.TargetCommentID, opInput.TargetWorkID, opInput.Text, opInput.EventID, opInput.StrategyID, hash); err != nil { return AutomaticResult{}, databaseError(err) } - if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='processing',reason='',strategy_id=$2,execution_account_id=$3 WHERE id=$1`, eventID, chosen.ID, execution.ID); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='processing',reason='',strategy_id=$2,execution_account_id=$3,processing_started_at=$4 WHERE id=$1`, eventID, chosen.ID, execution.ID, now); err != nil { return AutomaticResult{}, databaseError(err) } if err := tx.Commit(); err != nil { return AutomaticResult{}, fmt.Errorf("commit automatic event: %w", err) } + if ActionRequiresText(chosen.Action) && strings.TrimSpace(text) == "" { + if generator == nil { + result := ActionResult{State: "blocked", Reason: "AI 生成不可用"} + if err := s.UpdateOperationResult(ctx, opID, result); err != nil { + return AutomaticResult{}, fmt.Errorf("save unavailable automatic operation: %w", err) + } + if err := s.markEvent(ctx, eventID, result.State, result.Reason, chosen.ID, execution.ID, nil, ptrTime(time.Now().UTC()), nil); err != nil { + return AutomaticResult{}, fmt.Errorf("save unavailable automatic event: %w", err) + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + op, getErr := s.GetOperation(ctx, opID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event, Operation: &op}, ErrUnavailable + } + generated, generateErr := generator.Generate(ctx, bigProfile.ReplyRequirements, input.EventType) + if generateErr != nil { + result := ActionResult{State: "failed", Reason: generateErr.Error()} + if err := s.UpdateOperationResult(ctx, opID, result); err != nil { + return AutomaticResult{}, fmt.Errorf("save failed automatic operation: %w", err) + } + if err := s.markEvent(ctx, eventID, result.State, result.Reason, chosen.ID, execution.ID, nil, ptrTime(time.Now().UTC()), nil); err != nil { + return AutomaticResult{}, fmt.Errorf("save failed automatic event: %w", err) + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + op, getErr := s.GetOperation(ctx, opID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event, Operation: &op}, generateErr + } + text = strings.TrimSpace(generated) + if text == "" { + result := ActionResult{State: "failed", Reason: "AI 返回空内容"} + if err := s.UpdateOperationResult(ctx, opID, result); err != nil { + return AutomaticResult{}, fmt.Errorf("save empty automatic operation: %w", err) + } + if err := s.markEvent(ctx, eventID, result.State, result.Reason, chosen.ID, execution.ID, nil, ptrTime(time.Now().UTC()), nil); err != nil { + return AutomaticResult{}, fmt.Errorf("save empty automatic event: %w", err) + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + op, getErr := s.GetOperation(ctx, opID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event, Operation: &op}, ErrInvalid + } + hash = operationHash(OperationInput{IdempotencyKey: opInput.IdempotencyKey, Source: opInput.Source, Action: opInput.Action, Platform: opInput.Platform, AccountID: opInput.AccountID, TargetUID: opInput.TargetUID, TargetCommentID: opInput.TargetCommentID, TargetWorkID: opInput.TargetWorkID, Text: text, EventID: opInput.EventID, StrategyID: opInput.StrategyID}) + updated, updateErr := s.db.ExecContext(ctx, `UPDATE creator_operation SET text=$2,request_hash=$3,updated_at=now() WHERE id=$1 AND state='processing'`, opID, text, hash) + if updateErr != nil { + return AutomaticResult{}, databaseError(updateErr) + } + if affected, affectedErr := updated.RowsAffected(); affectedErr != nil { + return AutomaticResult{}, affectedErr + } else if affected != 1 { + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + op, getErr := s.GetOperation(ctx, opID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event, Operation: &op}, nil + } + } + started := now // Conditions may change while the operation waits for the account executor. // Re-check immediately before the platform write; a stale queued operation is blocked, never sent. // The lock is deliberately acquired after receipt/operation persistence so ingestion @@ -549,9 +668,24 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven executionLock := s.automaticExecutionLock(execution.ID) executionLock.Lock() defer executionLock.Unlock() - started := time.Now().UTC() - if _, err := s.db.ExecContext(ctx, `UPDATE creator_event SET processing_started_at=$2 WHERE id=$1`, eventID, started); err != nil { - return AutomaticResult{}, fmt.Errorf("save automatic event start: %w", databaseError(err)) + releaseExecutionLock, err := s.acquireAutomaticExecutionLock(ctx, execution.ID) + if err != nil { + return AutomaticResult{}, err + } + defer releaseExecutionLock() + var liveOperationID string + if err := s.db.QueryRowContext(ctx, `UPDATE creator_operation SET updated_at=now() WHERE id=$1 AND state='processing' RETURNING id`, opID).Scan(&liveOperationID); errors.Is(err, sql.ErrNoRows) { + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + op, getErr := s.GetOperation(ctx, opID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event, Operation: &op}, nil + } else if err != nil { + return AutomaticResult{}, databaseError(err) } result := ActionResult{} _, checkErr := s.AccountWriteCheck(ctx, execution.ID, true, chosen.Action) @@ -578,6 +712,16 @@ func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEven checkErr = ErrConflict } } + if checkErr == nil && storedGeneration != "" { + var listenerStatus, listenerGeneration string + var listenerInvalidated bool + listenerErr := s.db.QueryRowContext(ctx, `SELECT status,generation,invalidated FROM creator_listener_state WHERE account_id=$1`, input.ReceivingAccountID).Scan(&listenerStatus, &listenerGeneration, &listenerInvalidated) + if listenerErr != nil && !errors.Is(listenerErr, sql.ErrNoRows) { + checkErr = databaseError(listenerErr) + } else if listenerErr == nil && (listenerStatus != "ready" || listenerGeneration != storedGeneration || listenerInvalidated) { + checkErr = ErrConflict + } + } if checkErr != nil { result = actionPreconditionResult(checkErr, "写入前条件已变化") } else if executor == nil { @@ -660,10 +804,18 @@ func (s *Store) RecoverStaleProcessing(ctx context.Context, now time.Time) (int, if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='uncertain',reason=$2,processing_finished_at=COALESCE(processing_finished_at,$3) WHERE state='processing' AND processing_started_at IS NOT NULL AND processing_started_at < $1 AND NOT EXISTS (SELECT 1 FROM creator_operation WHERE event_id=creator_event.id AND state='processing')`, cutoff, reason, now.UTC()); err != nil { return 0, databaseError(err) } + receivedRecovered, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='uncertain',reason='事件已收到但未开始处理，未补发',processing_finished_at=COALESCE(processing_finished_at,$2) WHERE state='received' AND received_at < $1`, cutoff, now.UTC()) + if err != nil { + return 0, databaseError(err) + } + receivedCount, err := receivedRecovered.RowsAffected() + if err != nil { + return 0, err + } if err := tx.Commit(); err != nil { return 0, databaseError(err) } - return len(stale), nil + return len(stale) + int(receivedCount), nil } func contains(values []string, want string) bool { @@ -706,9 +858,9 @@ func operationHash(input OperationInput) string { } func scanOperation(scanner interface{ Scan(...any) error }) (Operation, error) { var result Operation - var evidence []byte + var evidence, verificationEvidence []byte var eventID, strategyID sql.NullString - if err := scanner.Scan(&result.ID, &result.IdempotencyKey, &result.Source, &result.Action, &result.Platform, &result.AccountID, &result.TargetUID, &result.TargetCommentID, &result.TargetWorkID, &result.Text, &eventID, &strategyID, &result.State, &evidence, &result.Reason, &result.CreatedAt, &result.UpdatedAt); err != nil { + if err := scanner.Scan(&result.ID, &result.IdempotencyKey, &result.Source, &result.Action, &result.Platform, &result.AccountID, &result.TargetUID, &result.TargetCommentID, &result.TargetWorkID, &result.Text, &eventID, &strategyID, &result.State, &evidence, &result.Reason, &result.VerificationState, &verificationEvidence, &result.VerifiedAt, &result.CreatedAt, &result.UpdatedAt); err != nil { return Operation{}, err } result.EventID, result.StrategyID = eventID.String, strategyID.String @@ -718,10 +870,19 @@ func scanOperation(scanner interface{ Scan(...any) error }) (Operation, error) { return Operation{}, err } } + if result.VerificationState == "" { + result.VerificationState = "not_verified" + } + result.VerificationProof = map[string]string{} + if len(verificationEvidence) > 0 { + if err := json.Unmarshal(verificationEvidence, &result.VerificationProof); err != nil { + return Operation{}, err + } + } return result, nil } -const operationSelect = `SELECT id,idempotency_key,source,action,platform,account_id,target_uid,target_comment_id,target_work_id,text,event_id,strategy_id,state,evidence,reason,created_at,updated_at FROM creator_operation` +const operationSelect = `SELECT id,idempotency_key,source,action,platform,account_id,target_uid,target_comment_id,target_work_id,text,event_id,strategy_id,state,evidence,reason,verification_state,verification_evidence,verified_at,created_at,updated_at FROM creator_operation` func (s *Store) GetOperation(ctx context.Context, id string) (Operation, error) { result, err := scanOperation(s.db.QueryRowContext(ctx, operationSelect+` WHERE id=$1`, id)) @@ -863,8 +1024,22 @@ func (s *Store) UpdateOperationResult(ctx context.Context, id string, result Act if err != nil { return err } - _, err = s.db.ExecContext(ctx, `UPDATE creator_operation SET state=$2,evidence=$3::jsonb,reason=$4,updated_at=now() WHERE id=$1`, id, result.State, evidence, result.Reason) - return databaseError(err) + verificationState := "uncertain" + if result.State == "succeeded" || result.State == "failed" || result.State == "blocked" { + verificationState = result.State + } + updated, err := s.db.ExecContext(ctx, `UPDATE creator_operation SET state=$2,evidence=$3::jsonb,reason=$4,verification_state=$5,verification_evidence=$3::jsonb,verified_at=CASE WHEN $5 IN ('succeeded','failed','blocked') THEN now() ELSE NULL END,updated_at=now() WHERE id=$1 AND state='processing'`, id, result.State, evidence, result.Reason, verificationState) + if err != nil { + return databaseError(err) + } + affected, err := updated.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return ErrConflict + } + return nil } func (s *Store) persistDirectMessageResult(ctx context.Context, op Operation, result ActionResult) error { if op.Action != ActionDM { @@ -875,7 +1050,11 @@ func (s *Store) persistDirectMessageResult(ctx context.Context, op Operation, re messageState = "failed" } messageAt := time.Now().UTC() - _, _, err := s.SaveMessage(ctx, MessageInput{Platform: op.Platform, AccountID: op.AccountID, PeerUID: op.TargetUID, PlatformMessageKey: "operation:" + op.ID, Direction: "outbound", MessageType: "text", Text: op.Text, SentState: messageState, MessageAt: &messageAt}) + platformMessageKey := result.Evidence["message_server_id"] + if strings.TrimSpace(platformMessageKey) == "" { + platformMessageKey = "operation:" + op.ID + } + _, _, err := s.SaveMessage(ctx, MessageInput{Platform: op.Platform, AccountID: op.AccountID, PeerUID: op.TargetUID, PlatformMessageKey: platformMessageKey, OperationID: op.ID, Direction: "outbound", MessageType: "text", Text: op.Text, SentState: messageState, MessageAt: &messageAt}) return err } @@ -899,6 +1078,16 @@ func (s *Store) ExecuteManualOperation(ctx context.Context, id string, executor executionLock := s.automaticExecutionLock(op.AccountID) executionLock.Lock() defer executionLock.Unlock() + releaseExecutionLock, err := s.acquireAutomaticExecutionLock(ctx, op.AccountID) + if err != nil { + return Operation{}, err + } + defer releaseExecutionLock() + if err := s.db.QueryRowContext(ctx, `UPDATE creator_operation SET updated_at=now() WHERE id=$1 AND state='processing' RETURNING id`, id).Scan(&claimedID); errors.Is(err, sql.ErrNoRows) { + return s.GetOperation(ctx, id) + } else if err != nil { + return Operation{}, databaseError(err) + } if _, checkErr := s.AccountWriteCheck(ctx, op.AccountID, false, op.Action); checkErr != nil { result := actionPreconditionResult(checkErr, "写入前条件已变化") if updateErr := s.UpdateOperationResult(ctx, id, result); updateErr != nil { @@ -942,15 +1131,15 @@ func (s *Store) ExecuteManualOperation(ctx context.Context, id string, executor func scanConversation(scanner interface{ Scan(...any) error }) (Conversation, error) { var c Conversation - var last sql.NullTime - if err := scanner.Scan(&c.ID, &c.Platform, &c.AccountID, &c.PeerUID, &c.PeerName, &last); err != nil { + var last, synced sql.NullTime + if err := scanner.Scan(&c.ID, &c.Platform, &c.AccountID, &c.PeerUID, &c.PeerName, &last, &c.HistoryCursor, &c.HistoryHasMore, &synced); err != nil { return Conversation{}, err } - c.LastMessageAt = nullableTime(last) + c.LastMessageAt, c.HistorySyncedAt = nullableTime(last), nullableTime(synced) return c, nil } -const conversationSelect = `SELECT id,platform,account_id,peer_uid,peer_name,last_message_at FROM creator_conversation` +const conversationSelect = `SELECT id,platform,account_id,peer_uid,peer_name,last_message_at,history_cursor,history_has_more,history_synced_at FROM creator_conversation` func (s *Store) UpsertConversation(ctx context.Context, input MessageInput) (Conversation, error) { if !ValidatePlatform(input.Platform) || input.AccountID == "" || input.PeerUID == "" { @@ -974,6 +1163,17 @@ func (s *Store) GetConversation(ctx context.Context, id string) (Conversation, e result, err := scanConversation(s.db.QueryRowContext(ctx, conversationSelect+` WHERE id=$1`, id)) return result, rowError(err) } + +func (s *Store) UpdateConversationHistoryCursor(ctx context.Context, id, cursor string, hasMore bool) error { + id, cursor = strings.TrimSpace(id), strings.TrimSpace(cursor) + if id == "" || len(cursor) > 500 { + return ErrInvalid + } + if _, err := s.db.ExecContext(ctx, `UPDATE creator_conversation SET history_cursor=$2,history_has_more=$3,history_synced_at=now() WHERE id=$1`, id, cursor, hasMore); err != nil { + return databaseError(err) + } + return nil +} func (s *Store) ListConversations(ctx context.Context, accountID string) ([]Conversation, error) { query := conversationSelect args := []any{} @@ -1019,7 +1219,7 @@ func (s *Store) SaveMessage(ctx context.Context, input MessageInput) (Message, b id := newID("message") var returned string var inserted bool - err = s.db.QueryRowContext(ctx, `INSERT INTO creator_message (id,conversation_id,platform_message_key,direction,message_type,text,sent_state,message_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (conversation_id,platform_message_key) DO NOTHING RETURNING id,(xmax=0)`, id, conversation.ID, input.PlatformMessageKey, input.Direction, input.MessageType, input.Text, state, input.MessageAt).Scan(&returned, &inserted) + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_message (id,conversation_id,platform_message_key,operation_id,direction,message_type,text,sent_state,message_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (conversation_id,platform_message_key) DO NOTHING RETURNING id,(xmax=0)`, id, conversation.ID, input.PlatformMessageKey, nullableString(input.OperationID), input.Direction, input.MessageType, input.Text, state, input.MessageAt).Scan(&returned, &inserted) if errors.Is(err, sql.ErrNoRows) { if err := s.db.QueryRowContext(ctx, `SELECT id FROM creator_message WHERE conversation_id=$1 AND platform_message_key=$2`, conversation.ID, input.PlatformMessageKey).Scan(&returned); err != nil { return Message{}, false, rowError(err) @@ -1035,15 +1235,16 @@ func (s *Store) SaveMessage(ctx context.Context, input MessageInput) (Message, b } func scanMessage(scanner interface{ Scan(...any) error }) (Message, error) { var m Message + var operationID sql.NullString var at sql.NullTime - if err := scanner.Scan(&m.ID, &m.ConversationID, &m.PlatformMessageKey, &m.Direction, &m.MessageType, &m.Text, &m.SentState, &at, &m.CreatedAt); err != nil { + if err := scanner.Scan(&m.ID, &m.ConversationID, &m.PlatformMessageKey, &operationID, &m.Direction, &m.MessageType, &m.Text, &m.SentState, &at, &m.CreatedAt); err != nil { return Message{}, err } - m.MessageAt = nullableTime(at) + m.OperationID, m.MessageAt = operationID.String, nullableTime(at) return m, nil } -const messageSelect = `SELECT id,conversation_id,platform_message_key,direction,message_type,text,sent_state,message_at,created_at FROM creator_message` +const messageSelect = `SELECT id,conversation_id,platform_message_key,operation_id,direction,message_type,text,sent_state,message_at,created_at FROM creator_message` func (s *Store) GetMessage(ctx context.Context, id string) (Message, error) { result, err := scanMessage(s.db.QueryRowContext(ctx, messageSelect+` WHERE id=$1`, id)) @@ -1065,6 +1266,15 @@ func (s *Store) ListMessagesPage(ctx context.Context, conversationID string, pag return slicePage(items, page, pageSize) } +func (s *Store) LinkMessageOperation(ctx context.Context, messageID, platformMessageKey string) error { + messageID, platformMessageKey = strings.TrimSpace(messageID), strings.TrimSpace(platformMessageKey) + if messageID == "" || platformMessageKey == "" { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, `UPDATE creator_message message SET operation_id=(SELECT operation.id FROM creator_operation operation WHERE operation.evidence->>'message_server_id'=$2 ORDER BY operation.updated_at DESC LIMIT 1) WHERE message.id=$1 AND message.operation_id IS NULL`, messageID, platformMessageKey) + return databaseError(err) +} + func (s *Store) ListMessages(ctx context.Context, conversationID string) ([]Message, error) { rows, err := s.db.QueryContext(ctx, messageSelect+` WHERE conversation_id=$1 ORDER BY message_at NULLS LAST,created_at,id`, conversationID) if err != nil { diff --git a/internal/creator/bailian_test.go b/internal/creator/bailian_test.go index faa818a..e176c25 100644 --- a/internal/creator/bailian_test.go +++ b/internal/creator/bailian_test.go @@ -41,6 +41,34 @@ func TestBailianClientGeneratesAndParsesDecisions(t *testing.T) { if err != nil || !match || reason != "相关" { t.Fatalf("MatchLead() = %v, %q, %v", match, reason, err) } + match, reason, err = client.MatchTheme(context.Background(), "标题", "正文", "主题") + if err != nil || !match || reason != "相关" { + t.Fatalf("MatchTheme() = %v, %q, %v", match, reason, err) + } +} + +func TestBailianClientRejectsHTTPAndMalformedResponses(t *testing.T) { + responses := []string{"not-json", `{"choices":[]}`} + for _, body := range responses { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(body)) + })) + client, err := NewBailianClient(server.URL, "key", "model", server.Client()) + if err != nil { + server.Close() + t.Fatal(err) + } + if _, err := client.Generate(context.Background(), "instruction", "input"); err == nil { + server.Close() + t.Fatal("expected HTTP failure") + } + server.Close() + } + var nilClient *BailianClient + if _, err := nilClient.Generate(context.Background(), "instruction", "input"); err == nil { + t.Fatal("nil client must fail closed") + } } func TestBailianClientRejectsIncompleteConfiguration(t *testing.T) { diff --git a/internal/creator/collection.go b/internal/creator/collection.go index 66bebe8..e01ba1d 100644 --- a/internal/creator/collection.go +++ b/internal/creator/collection.go @@ -339,13 +339,13 @@ func (s *Store) ListDueOwnedAccounts(ctx context.Context, now time.Time, interva rows, err := s.db.QueryContext(ctx, ` SELECT account.id FROM social_account account - JOIN creator_account_profile profile ON profile.account_id=account.id AND profile.business_status='normal' + JOIN creator_account_profile profile ON profile.account_id=account.id AND profile.business_status IN ('normal','muted') LEFT JOIN creator_collection_checkpoint works_checkpoint ON works_checkpoint.source_type='owned' AND works_checkpoint.source_id=account.id AND works_checkpoint.collection_kind='works' LEFT JOIN creator_collection_checkpoint comments_checkpoint ON comments_checkpoint.source_type='owned' AND comments_checkpoint.source_id=account.id AND comments_checkpoint.collection_kind='comments' WHERE account.platform IN ('douyin', 'xiaohongshu') AND account.authorization_status='authorized' - AND profile.login_status='logged_in' AND profile.big_account=true + AND profile.login_status='logged_in' AND COALESCE(works_checkpoint.status, '') <> 'blocked' AND COALESCE(comments_checkpoint.status, '') <> 'blocked' AND (works_checkpoint.id IS NULL OR comments_checkpoint.id IS NULL @@ -407,7 +407,7 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI return err } report.WorksSeen++ - if !publishedAtInCollectionWindow(work.PublishedAt, report.WindowStart, report.WindowEnd) { + if work.PublishedAt != nil && work.PublishedAt.Before(report.WindowStart) { continue } work.Platform, work.SourceType, work.SourceID = platform, sourceType, sourceID @@ -418,16 +418,11 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI if err != nil { return err } - if work.PublishedAt == nil || !work.PublishedAt.After(report.WindowEnd) { + if savedWork.PublishedAtStatus == "verified" && savedWork.PublishedAt != nil && !savedWork.PublishedAt.After(report.WindowEnd) { if err := s.EnsureMetricPlan(ctx, savedWork.ID, settings); err != nil { return err } } - if work.Likes != nil || work.CommentsCount != nil || work.Shares != nil { - if _, metricErr := s.RecordMetric(ctx, MetricInput{WorkID: savedWork.ID, CollectedAt: now.UTC(), Likes: work.Likes, CommentsCount: work.CommentsCount, Shares: work.Shares}, settings, now.UTC()); metricErr != nil && !errors.Is(metricErr, ErrConflict) { - return metricErr - } - } seenWorks[work.WorkKey] = savedWork.ID report.WorksSaved++ } diff --git a/internal/creator/content.go b/internal/creator/content.go index 838090e..bf120dc 100644 --- a/internal/creator/content.go +++ b/internal/creator/content.go @@ -271,9 +271,7 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) original_url = CASE WHEN EXCLUDED.original_url = '' THEN creator_work.original_url ELSE EXCLUDED.original_url END, cover_url = CASE WHEN EXCLUDED.cover_url = '' THEN creator_work.cover_url ELSE EXCLUDED.cover_url END, raw_payload = COALESCE(EXCLUDED.raw_payload, creator_work.raw_payload), - likes = COALESCE(EXCLUDED.likes, creator_work.likes), - comments_count = COALESCE(EXCLUDED.comments_count, creator_work.comments_count), - shares = COALESCE(EXCLUDED.shares, creator_work.shares), updated_at = now() + updated_at = now() RETURNING id, (xmax = 0)`, id, input.Platform, input.WorkKey, input.SourceType, input.SourceID, input.AuthorName, input.Title, input.Body, input.PublishedAt, status, input.OriginalURL, input.CoverURL, nullableRawPayload(input.RawPayload), input.Likes, input.CommentsCount, input.Shares).Scan(&returnedID, &inserted) @@ -391,6 +389,12 @@ func (s *Store) ListWorks(ctx context.Context, filter WorkFilter) ([]Work, error } add("platform = $%d", filter.Platform) } + if filter.PublishedAtStatus != "" { + if filter.PublishedAtStatus != "verified" && filter.PublishedAtStatus != "future" && filter.PublishedAtStatus != "unverified" && filter.PublishedAtStatus != "invalid" { + return nil, ErrInvalid + } + add("published_at_status = $%d", filter.PublishedAtStatus) + } if filter.SourceType != "" { if filter.SourceType != SourceOwned && filter.SourceType != SourceCompetitor { return nil, ErrInvalid @@ -505,19 +509,90 @@ func (s *Store) GetMaterial(ctx context.Context, workID string) (MaterialJob, er if _, err := s.db.ExecContext(ctx, `INSERT INTO creator_material_job (work_id) VALUES ($1) ON CONFLICT (work_id) DO NOTHING`, workID); err != nil { return MaterialJob{}, databaseError(err) } - return s.scanMaterial(s.db.QueryRowContext(ctx, `SELECT work_id, selected, select_confirmed_at, download_status, video_reference, audio_status, audio_reference, transcription_status, transcript, failed_step, failure_reason, rewrite_confirmed_at, rewrite_requirement, generated_title, generated_script, created_at, updated_at FROM creator_material_job WHERE work_id = $1`, workID)) + return s.scanMaterial(s.db.QueryRowContext(ctx, `SELECT work_id, selected, select_confirmed_at, download_status, video_reference, audio_status, audio_reference, transcription_status, transcript, failed_step, failure_reason, rewrite_confirmed_at, rewrite_requirement, generated_title, generated_script, processing_step, processing_token, processing_started_at, created_at, updated_at FROM creator_material_job WHERE work_id = $1`, workID)) } func (s *Store) scanMaterial(scanner interface{ Scan(...any) error }) (MaterialJob, error) { var result MaterialJob - var selectedAt, rewriteAt sql.NullTime - if err := scanner.Scan(&result.WorkID, &result.Selected, &selectedAt, &result.DownloadStatus, &result.VideoReference, &result.AudioStatus, &result.AudioReference, &result.TranscriptionStatus, &result.Transcript, &result.FailedStep, &result.FailureReason, &rewriteAt, &result.RewriteRequirement, &result.GeneratedTitle, &result.GeneratedScript, &result.CreatedAt, &result.UpdatedAt); err != nil { + var selectedAt, rewriteAt, processingStartedAt sql.NullTime + if err := scanner.Scan(&result.WorkID, &result.Selected, &selectedAt, &result.DownloadStatus, &result.VideoReference, &result.AudioStatus, &result.AudioReference, &result.TranscriptionStatus, &result.Transcript, &result.FailedStep, &result.FailureReason, &rewriteAt, &result.RewriteRequirement, &result.GeneratedTitle, &result.GeneratedScript, &result.ProcessingStep, &result.ProcessingToken, &processingStartedAt, &result.CreatedAt, &result.UpdatedAt); err != nil { return MaterialJob{}, rowError(err) } - result.SelectConfirmedAt, result.RewriteConfirmedAt = nullableTime(selectedAt), nullableTime(rewriteAt) + result.SelectConfirmedAt, result.RewriteConfirmedAt, result.ProcessingStartedAt = nullableTime(selectedAt), nullableTime(rewriteAt), nullableTime(processingStartedAt) return result, nil } +func (s *Store) ClaimMaterialStep(ctx context.Context, workID, step, token string) (MaterialJob, bool, error) { + workID, step, token = strings.TrimSpace(workID), strings.TrimSpace(step), strings.TrimSpace(token) + if workID == "" || token == "" || len(token) > 200 || (step != "download" && step != "audio" && step != "transcription") { + return MaterialJob{}, false, ErrInvalid + } + stale, err := s.recoverStaleMaterialStep(ctx, workID, time.Now().UTC().Add(-30*time.Minute)) + if err != nil { + return MaterialJob{}, false, err + } + job, err := s.GetMaterial(ctx, workID) + if err != nil { + return MaterialJob{}, false, err + } + if !job.Selected { + return MaterialJob{}, false, ErrConflict + } + status := job.DownloadStatus + if step == "audio" { + status = job.AudioStatus + } + if step == "transcription" { + status = job.TranscriptionStatus + } + if stale { + return job, false, nil + } + if status == "succeeded" || status == "no_audio" || status == "no_speech" { + return job, false, nil + } + if job.ProcessingStep != "" || (status != "not_started" && status != "failed") { + return job, false, nil + } + var query string + switch step { + case "download": + query = `UPDATE creator_material_job SET download_status='running',processing_step=$2,processing_token=$3,processing_started_at=now(),failed_step='',failure_reason='',updated_at=now() WHERE work_id=$1 AND selected=true AND processing_step='' AND download_status IN ('not_started','failed')` + case "audio": + query = `UPDATE creator_material_job SET audio_status='running',processing_step=$2,processing_token=$3,processing_started_at=now(),failed_step='',failure_reason='',updated_at=now() WHERE work_id=$1 AND selected=true AND processing_step='' AND audio_status IN ('not_started','failed')` + case "transcription": + query = `UPDATE creator_material_job SET transcription_status='running',processing_step=$2,processing_token=$3,processing_started_at=now(),failed_step='',failure_reason='',updated_at=now() WHERE work_id=$1 AND selected=true AND processing_step='' AND transcription_status IN ('not_started','failed')` + } + updated, err := s.db.ExecContext(ctx, query, workID, step, token) + if err != nil { + return MaterialJob{}, false, databaseError(err) + } + affected, err := updated.RowsAffected() + if err != nil { + return MaterialJob{}, false, err + } + job, err = s.GetMaterial(ctx, workID) + return job, affected == 1, err +} + +func (s *Store) recoverStaleMaterialStep(ctx context.Context, workID string, cutoff time.Time) (bool, error) { + result, err := s.db.ExecContext(ctx, ` + UPDATE creator_material_job + SET download_status = CASE WHEN processing_step='download' THEN 'failed' ELSE download_status END, + audio_status = CASE WHEN processing_step='audio' THEN 'failed' ELSE audio_status END, + transcription_status = CASE WHEN processing_step='transcription' THEN 'failed' ELSE transcription_status END, + failed_step = processing_step, + failure_reason = '上次处理结果不明，未自动重试', processing_step='', processing_token='', + processing_started_at=NULL, updated_at=now() + WHERE work_id=$1 AND selected=true AND processing_step <> '' + AND processing_started_at IS NOT NULL AND processing_started_at < $2`, workID, cutoff.UTC()) + if err != nil { + return false, databaseError(err) + } + affected, err := result.RowsAffected() + return affected == 1, err +} + func (s *Store) SelectMaterial(ctx context.Context, workID string) (MaterialJob, bool, error) { if _, err := s.GetWork(ctx, workID); err != nil { return MaterialJob{}, false, err @@ -535,9 +610,9 @@ func (s *Store) SetMaterialStep(ctx context.Context, workID, step, status, refer return MaterialJob{}, ErrInvalid } valid := map[string]map[string]bool{ - "download": {"not_started": true, "running": true, "succeeded": true, "failed": true}, - "audio": {"not_started": true, "running": true, "succeeded": true, "no_audio": true, "failed": true}, - "transcription": {"not_started": true, "running": true, "succeeded": true, "no_speech": true, "failed": true}, + "download": {"not_started": true, "succeeded": true, "failed": true}, + "audio": {"not_started": true, "succeeded": true, "no_audio": true, "failed": true}, + "transcription": {"not_started": true, "succeeded": true, "no_speech": true, "failed": true}, } if !valid[step][status] || len(reference) > 2000 || len(reason) > 2000 { return MaterialJob{}, ErrInvalid @@ -553,18 +628,59 @@ func (s *Store) SetMaterialStep(ctx context.Context, workID, step, status, refer var args []any switch step { case "download": - query = `UPDATE creator_material_job SET download_status = $2, video_reference = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'download' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1` + query = `UPDATE creator_material_job SET download_status = $2, video_reference = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'download' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1 AND selected=true AND processing_step=''` args = []any{workID, status, reference, reason} case "audio": - query = `UPDATE creator_material_job SET audio_status = $2, audio_reference = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'audio' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1` + query = `UPDATE creator_material_job SET audio_status = $2, audio_reference = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'audio' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1 AND selected=true AND processing_step=''` args = []any{workID, status, reference, reason} case "transcription": - query = `UPDATE creator_material_job SET transcription_status = $2, transcript = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'transcription' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1` + query = `UPDATE creator_material_job SET transcription_status = $2, transcript = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'transcription' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1 AND selected=true AND processing_step=''` args = []any{workID, status, reference, reason} } - if _, err := s.db.ExecContext(ctx, query, args...); err != nil { + updated, err := s.db.ExecContext(ctx, query, args...) + if err != nil { return MaterialJob{}, databaseError(err) } + if affected, err := updated.RowsAffected(); err != nil { + return MaterialJob{}, err + } else if affected != 1 { + return MaterialJob{}, ErrConflict + } + return s.GetMaterial(ctx, workID) +} + +func (s *Store) CompleteMaterialStep(ctx context.Context, workID, step, token, status, reference, reason string) (MaterialJob, error) { + workID, step, token = strings.TrimSpace(workID), strings.TrimSpace(step), strings.TrimSpace(token) + if workID == "" || token == "" || len(token) > 200 || (step != "download" && step != "audio" && step != "transcription") || len(reference) > 2000 || len(reason) > 2000 { + return MaterialJob{}, ErrInvalid + } + valid := map[string]map[string]bool{ + "download": {"succeeded": true, "failed": true}, + "audio": {"succeeded": true, "no_audio": true, "failed": true}, + "transcription": {"succeeded": true, "no_speech": true, "failed": true}, + } + if !valid[step][status] { + return MaterialJob{}, ErrInvalid + } + var column string + switch step { + case "download": + column = "download_status" + case "audio": + column = "audio_status" + case "transcription": + column = "transcription_status" + } + query := fmt.Sprintf(`UPDATE creator_material_job SET %s=$4, %s_reference=$5, failed_step=CASE WHEN $4='failed' THEN $2 ELSE failed_step END, failure_reason=CASE WHEN $4='failed' THEN $6 ELSE failure_reason END, processing_step='', processing_token='', processing_started_at=NULL, updated_at=now() WHERE work_id=$1 AND selected=true AND processing_step=$2 AND processing_token=$3`, column, map[string]string{"download": "video", "audio": "audio", "transcription": "transcript"}[step]) + updated, err := s.db.ExecContext(ctx, query, workID, step, token, status, reference, reason) + if err != nil { + return MaterialJob{}, databaseError(err) + } + if affected, err := updated.RowsAffected(); err != nil { + return MaterialJob{}, err + } else if affected != 1 { + return MaterialJob{}, ErrConflict + } return s.GetMaterial(ctx, workID) } diff --git a/internal/creator/coverage_integration_test.go b/internal/creator/coverage_integration_test.go new file mode 100644 index 0000000..5673d92 --- /dev/null +++ b/internal/creator/coverage_integration_test.go @@ -0,0 +1,193 @@ +package creator + +import ( + "errors" + "testing" + "time" +) + +func TestCreatorPostgresPageAndConversationState(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + if err := store.Ping(ctx); err != nil { + t.Fatal(err) + } + store.SetSecretBridge(nil) + if err := store.EnsureSchema(ctx); err != nil { + t.Fatal(err) + } + accountID := createIntegrationAccount(t, ctx, phaseAStore, "pages") + if err := store.EnsureAccountProfile(ctx, accountID); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Add(-time.Hour) + work, inserted, err := store.UpsertWork(ctx, WorkInput{ + Platform: PlatformDouyin, WorkKey: "page-work", SourceType: SourceOwned, SourceID: accountID, + AuthorName: "author", Title: "title", Body: "body", PublishedAt: &now, PublishedAtStatus: "verified", + }, now) + if err != nil || !inserted { + t.Fatalf("upsert work: inserted=%v err=%v", inserted, err) + } + comment, inserted, err := store.SaveComment(ctx, CommentInput{ + Platform: PlatformDouyin, CommentKey: "page-comment", WorkID: work.ID, + AuthorUID: "peer", AuthorName: "Peer", Content: "hello", CommentType: "top_level", + PublishedAt: &now, + }) + if err != nil || !inserted { + t.Fatalf("save comment: inserted=%v err=%v", inserted, err) + } + workPage, err := store.ListWorksPage(ctx, WorkFilter{Platform: PlatformDouyin, SourceID: accountID, SourceType: SourceOwned}, 1, 1) + if err != nil || len(workPage.Data) != 1 || workPage.Total != 1 { + t.Fatalf("list works page: page=%+v err=%v", workPage, err) + } + commentPage, err := store.ListCommentsPage(ctx, PlatformDouyin, work.ID, 1, 1) + if err != nil || len(commentPage.Data) != 1 || commentPage.Total != 1 { + t.Fatalf("list comments page: page=%+v err=%v", commentPage, err) + } + eventResult, err := store.RecordEvent(ctx, InteractionEvent{ + Platform: PlatformDouyin, ReceivingAccountID: accountID, EventKey: "page-event", + EventType: "dm", InteractorUID: "peer", MessageType: MessageTypeText, MessageText: "incoming", + ReceivedAt: now, + }) + if err != nil { + t.Fatal(err) + } + if _, err := store.ListEventsPage(ctx, accountID, 1, 1); err != nil { + t.Fatal(err) + } + if _, err := store.SetEventDisplayed(ctx, eventResult.Event.ID, time.Time{}); err != nil { + t.Fatal(err) + } + messageAt := now.Add(time.Minute) + message, inserted, err := store.SaveMessage(ctx, MessageInput{ + Platform: PlatformDouyin, AccountID: accountID, PeerUID: "peer", PeerName: "Peer", + PlatformMessageKey: "page-message", Direction: "inbound", MessageType: MessageTypeText, + Text: "incoming", MessageAt: &messageAt, + }) + if err != nil || !inserted { + t.Fatalf("save message: inserted=%v err=%v", inserted, err) + } + if _, err := store.ListMessagesPage(ctx, message.ConversationID, 1, 1); err != nil { + t.Fatal(err) + } + if err := store.UpdateConversationHistoryCursor(ctx, message.ConversationID, "cursor-1", true); err != nil { + t.Fatal(err) + } + if err := store.LinkMessageOperation(ctx, message.ID, "missing-platform-message"); err != nil { + t.Fatal(err) + } + if traces, err := store.ListStrategyTraces(ctx, eventResult.Event.ID); err != nil || traces == nil { + t.Fatalf("list strategy traces: traces=%v err=%v", traces, err) + } + if _, _, err := store.SaveComment(ctx, CommentInput{Platform: PlatformDouyin, CommentKey: "page-comment", WorkID: work.ID, Content: "updated", CommentType: "top_level"}); err != nil { + t.Fatal(err) + } + if _, _, err := store.NextCollectionWindow(ctx, SourceOwned, accountID, now, time.Hour, 1); err != nil { + t.Fatal(err) + } + if err := store.MarkCollectionBlocked(ctx, SourceOwned, accountID, "coverage block", now, 1); err != nil { + t.Fatal(err) + } + if _, _, err := store.NextCollectionWindow(ctx, SourceOwned, accountID, now, time.Hour, 1); err != nil { + t.Fatal(err) + } + if _, err := store.ListListenerBoundaries(ctx, accountID); err != nil { + t.Fatal(err) + } + if _, err := store.ListListenerStates(ctx, accountID); err != nil { + t.Fatal(err) + } + if works, err := store.ListDueMetricWorks(ctx, now); err != nil || works == nil { + t.Fatalf("list due metric works: works=%v err=%v", works, err) + } + settings, err := store.GetSettings(ctx) + if err != nil { + t.Fatal(err) + } + if err := store.EnsureMetricPlan(ctx, work.ID, settings); err != nil { + t.Fatal(err) + } + if err := store.StopMetricPlan(ctx, work.ID, "coverage stop"); err != nil { + t.Fatal(err) + } + lease, err := store.ClaimSourceSync(ctx, SourceOwned, accountID) + if err != nil { + t.Fatal(err) + } + if _, err := store.ClaimSourceSync(ctx, SourceOwned, accountID); !errors.Is(err, ErrConflict) { + t.Fatalf("claim active source lease: %v", err) + } + if err := store.ReleaseSourceSync(ctx, SourceOwned, accountID, "wrong-token"); !errors.Is(err, ErrConflict) { + t.Fatalf("release wrong lease token: %v", err) + } + if err := store.ReleaseSourceSync(ctx, SourceOwned, accountID, lease); err != nil { + t.Fatal(err) + } + if _, err := store.RecordLoginResult(ctx, accountID, "needs_login", "coverage", ""); err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, accountID, "sec_uid_"+accountID); err != nil { + t.Fatal(err) + } + if _, err := store.GenerateRewrite(ctx, work.ID, nil); !errors.Is(err, ErrUnavailable) { + t.Fatalf("generate rewrite without generator: %v", err) + } + if _, err := store.ConfirmRewrite(ctx, work.ID, "coverage rewrite"); !errors.Is(err, ErrConflict) { + t.Fatalf("confirm incomplete rewrite: %v", err) + } + rule, err := store.CreateRule(ctx, LeadRuleInput{Name: "coverage rule", Enabled: true, SourceType: SourceOwned, Topic: "title", IncludeKeywords: []string{"hello"}, AIRequirement: "business lead"}) + if err != nil { + t.Fatal(err) + } + analysis, err := store.AnalyzeComments(ctx, []string{comment.ID, comment.ID}, rule.ID, nil) + if len(analysis) != 1 || !errors.Is(err, ErrUnavailable) { + t.Fatalf("analyze comments without AI: analysis=%v err=%v", analysis, err) + } + strategyInput := StrategyInput{Enabled: true, Action: ActionReplyComment, EventTypes: []string{"comment"}} + if err := store.validateEnabledStrategy(ctx, accountID, strategyInput); !errors.Is(err, ErrInvalid) { + t.Fatalf("strategy without reply requirements: %v", err) + } + if _, err := store.UpdateAccountProfile(ctx, accountID, AccountProfileUpdate{RealNameStatus: "unknown", BusinessStatus: "normal", ReplyRequirements: "answer", CooldownSeconds: 60}); err != nil { + t.Fatal(err) + } + if err := store.validateEnabledStrategy(ctx, accountID, strategyInput); !errors.Is(err, ErrUnavailable) { + t.Fatalf("strategy without AI: %v", err) + } + missing := []struct { + name string + get func() error + }{ + {"comment", func() error { _, err := store.GetComment(ctx, "missing"); return err }}, + {"comment by key", func() error { _, err := store.GetCommentByKey(ctx, PlatformDouyin, "missing"); return err }}, + {"competitor", func() error { _, err := store.GetCompetitor(ctx, "missing"); return err }}, + {"conversation", func() error { _, err := store.GetConversation(ctx, "missing"); return err }}, + {"event", func() error { _, err := store.GetEvent(ctx, "missing"); return err }}, + {"listener", func() error { _, err := store.GetListenerState(ctx, accountID); return err }}, + {"message", func() error { _, err := store.GetMessage(ctx, "missing"); return err }}, + {"operation", func() error { _, err := store.GetOperation(ctx, "missing"); return err }}, + {"rule", func() error { _, err := store.GetRule(ctx, "missing"); return err }}, + {"strategy", func() error { _, err := store.GetStrategy(ctx, "missing"); return err }}, + {"work", func() error { _, err := store.GetWork(ctx, "missing"); return err }}, + {"work by key", func() error { _, err := store.GetWorkByKey(ctx, PlatformDouyin, "missing"); return err }}, + } + for _, item := range missing { + if err := item.get(); !errors.Is(err, ErrNotFound) { + t.Fatalf("get missing %s: %v", item.name, err) + } + } + for _, item := range []struct { + name string + set func() error + }{ + {"competitor", func() error { _, err := store.SetCompetitorEnabled(ctx, "missing", true); return err }}, + {"event", func() error { _, err := store.SetEventDisplayed(ctx, "missing", now); return err }}, + {"rule", func() error { _, err := store.SetRuleEnabled(ctx, "missing", true); return err }}, + {"strategy", func() error { _, err := store.SetStrategyEnabled(ctx, "missing", true); return err }}, + } { + if err := item.set(); !errors.Is(err, ErrNotFound) { + t.Fatalf("set missing %s: %v", item.name, err) + } + } + if err := store.DeleteStrategy(ctx, "missing"); err != nil { + t.Fatalf("delete missing strategy: %v", err) + } +} diff --git a/internal/creator/coverage_unit_test.go b/internal/creator/coverage_unit_test.go new file mode 100644 index 0000000..28aa150 --- /dev/null +++ b/internal/creator/coverage_unit_test.go @@ -0,0 +1,144 @@ +package creator + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestCreatorPureCoverageGuards(t *testing.T) { + if got, err := selectCandidate(nil); err != nil || got != "" { + t.Fatalf("empty candidate: %q %v", got, err) + } + if got, err := selectCandidate([]string{"one"}); err != nil || got != "one" { + t.Fatalf("single candidate: %q %v", got, err) + } + got, err := selectCandidate([]string{"one", "two"}) + if err != nil || (got != "one" && got != "two") { + t.Fatalf("multiple candidates: %q %v", got, err) + } + for _, input := range []StrategyInput{ + {}, + {Position: 1, Action: "invalid", EventTypes: []string{"comment"}}, + {Position: 1, Action: ActionLikeComment, TargetType: "work", EventTypes: []string{"comment"}}, + {Position: 1, Action: ActionLikeWork, TargetType: "comment", EventTypes: []string{"comment"}}, + {Position: 1, Action: ActionReplyComment, EventTypes: []string{"dm"}}, + {Position: 1, Action: ActionReplyComment, EventTypes: []string{"comment"}, CandidateTexts: []string{""}}, + } { + if _, err := validateStrategyInput(input); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid strategy input accepted: %+v -> %v", input, err) + } + } + valid, err := validateStrategyInput(StrategyInput{Position: 1, Action: ActionReplyComment, EventTypes: []string{" comment "}, CandidateTexts: []string{" reply "}}) + if err != nil || valid.TargetType != "user" || valid.EventTypes[0] != "comment" || valid.CandidateTexts[0] != "reply" { + t.Fatalf("valid strategy normalization: %+v %v", valid, err) + } + if result := actionPreconditionResult(ErrConflict, "blocked"); result.State != "blocked" || result.Reason != "blocked" { + t.Fatalf("blocked precondition: %+v", result) + } + if result := actionPreconditionResult(errors.New("database down"), "ignored"); result.State != "uncertain" { + t.Fatalf("uncertain precondition: %+v", result) + } + for _, result := range []ActionResult{{State: "succeeded"}, {State: "failed"}, {State: "blocked"}, {State: "uncertain"}, {}} { + normalized := normalizeActionResult(result, nil) + if normalized.State == "" { + t.Fatalf("empty action result was not normalized: %+v", normalized) + } + } + if result := normalizeActionResult(ActionResult{}, errors.New("executor failed")); result.State != "uncertain" || result.Reason != "executor failed" { + t.Fatalf("executor error normalization: %+v", result) + } + if title, script, err := parseGeneratedRewrite(`{"title":" title ","script":" script "}`); err != nil || title != "title" || script != "script" { + t.Fatalf("generated rewrite parse: %q %q %v", title, script, err) + } + for _, value := range []string{"not-json", `{"title":"","script":"script"}`, `{"title":"title","script":""}`} { + if _, _, err := parseGeneratedRewrite(value); err == nil { + t.Fatalf("invalid generated rewrite accepted: %s", value) + } + } + if _, err := decodeCommentCheckpoint(`{"cursor":"missing-work-key"}`); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid empty checkpoint: %v", err) + } + if value, err := decodeCommentCheckpoint(`{"work_key":"work","cursor":"cursor"}`); err != nil || value.WorkKey != "work" || value.Cursor != "cursor" { + t.Fatalf("valid checkpoint: %+v %v", value, err) + } +} + +func TestCreatorSchedulingAndActionPredicates(t *testing.T) { + now := time.Now().UTC() + if next, reason := NextMetricAtValue(nil, now, SettingsUpdate{}); !next.IsZero() || reason != "published_at_pending_verification" { + t.Fatalf("pending metric schedule: %v %q", next, reason) + } + published := now.Add(-time.Hour) + input := SettingsUpdate{MetricInitialIntervalSeconds: 60, MetricMaxIntervalSeconds: 3600, MetricMultiplier: 2, MetricAgeSeconds: 7200} + if next, reason := NextMetricAtValue(&published, now, input); next.IsZero() || reason != "" { + t.Fatalf("metric schedule: %v %q", next, reason) + } + if coalesceReason("value", "fallback") != "value" || coalesceReason("", "fallback") != "fallback" { + t.Fatal("coalesceReason did not choose the expected value") + } + for _, test := range []struct { + action, interactor, comment, work string + valid bool + }{ + {ActionDM, "peer", "", "", true}, + {ActionFollow, "peer", "", "", true}, + {ActionReplyComment, "peer", "comment", "", true}, + {ActionLikeComment, "peer", "", "", false}, + {ActionLikeWork, "peer", "", "work", true}, + {ActionRepost, "peer", "", "", false}, + {ActionDM, "", "comment", "work", false}, + {"unknown", "peer", "comment", "work", false}, + } { + if got := ActionTargetValid(test.action, test.interactor, test.comment, test.work, ""); got != test.valid { + t.Fatalf("ActionTargetValid(%+v) = %v", test, got) + } + } +} + +func TestCreatorPaginationGuards(t *testing.T) { + items, err := collectPagesFromCursor(context.Background(), "", func(_ context.Context, cursor string) ([]string, string, bool, error) { + if cursor != "" { + t.Fatalf("unexpected cursor %q", cursor) + } + return []string{"one"}, "", false, nil + }, nil) + if err != nil || len(items) != 1 { + t.Fatalf("single page: %v %v", items, err) + } + for name, fetch := range map[string]func(context.Context, string) ([]string, string, bool, error){ + "missing cursor": func(context.Context, string) ([]string, string, bool, error) { return nil, "", true, nil }, + "repeated cursor": func(context.Context, string) ([]string, string, bool, error) { return nil, "same", true, nil }, + } { + _, err := collectPagesFromCursor(context.Background(), "same", fetch, nil) + if !errors.Is(err, ErrInvalid) { + t.Fatalf("%s: %v", name, err) + } + } + _, err = collectPagesFromCursor(context.Background(), "", func(context.Context, string) ([]string, string, bool, error) { + return nil, "next", true, errors.New("fetch failed") + }, nil) + if err == nil || err.Error() != "fetch failed" { + t.Fatalf("fetch failure: %v", err) + } + _, err = collectPagesFromCursor(context.Background(), "", func(context.Context, string) ([]string, string, bool, error) { + return []string{"one"}, "next", true, nil + }, func([]string, string, bool) error { return errors.New("save failed") }) + if err == nil || err.Error() != "save failed" { + t.Fatalf("after-page failure: %v", err) + } +} + +func TestConfiguredBailianWithoutStore(t *testing.T) { + var client *ConfiguredBailian + if _, err := client.Generate(context.Background(), "instruction", "input"); !errors.Is(err, ErrUnavailable) { + t.Fatalf("nil Generate: %v", err) + } + if _, _, err := client.MatchTheme(context.Background(), "title", "body", "topic"); !errors.Is(err, ErrUnavailable) { + t.Fatalf("nil MatchTheme: %v", err) + } + if _, _, err := client.MatchLead(context.Background(), "work", "comment", "requirement"); !errors.Is(err, ErrUnavailable) { + t.Fatalf("nil MatchLead: %v", err) + } +} diff --git a/internal/creator/integration_test.go b/internal/creator/integration_test.go index d3c5113..13fb529 100644 --- a/internal/creator/integration_test.go +++ b/internal/creator/integration_test.go @@ -171,7 +171,7 @@ func TestCreatorPostgresContentAndWorkflow(t *testing.T) { now := time.Now().UTC().Truncate(time.Microsecond) ownedDue, err := store.ListDueOwnedAccounts(ctx, now, 1800) - if err != nil || len(ownedDue) != 1 || ownedDue[0] != bigID { + if err != nil || len(ownedDue) != 2 || ownedDue[0] != bigID || ownedDue[1] != smallID { t.Fatalf("list due owned accounts: accounts=%+v err=%v", ownedDue, err) } published := now.Add(-2 * time.Hour) @@ -288,6 +288,122 @@ func TestCreatorPostgresContentAndWorkflow(t *testing.T) { } } +func TestCreatorPostgresCollectionAllowsMutedReadAccount(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + accountID := createIntegrationAccount(t, ctx, phaseAStore, "muted"+fmt.Sprint(stamp)) + if err := store.EnsureAccountProfile(ctx, accountID); err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, accountID, "sec_uid_"+accountID); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + if _, err := store.db.ExecContext(ctx, `UPDATE creator_account_profile SET business_status='muted' WHERE account_id=$1`, accountID); err != nil { + t.Fatal(err) + } + due, err := store.ListDueOwnedAccounts(ctx, now, 1800) + if err != nil || len(due) != 1 || due[0] != accountID { + t.Fatalf("muted account was not eligible for read collection: accounts=%v err=%v", due, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE creator_account_profile SET business_status='banned' WHERE account_id=$1`, accountID); err != nil { + t.Fatal(err) + } + due, err = store.ListDueOwnedAccounts(ctx, now, 1800) + if err != nil || len(due) != 0 { + t.Fatalf("banned account was eligible for read collection: accounts=%v err=%v", due, err) + } +} + +func TestCreatorPostgresSettingsResetCollectionCheckpoint(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + accountID := createIntegrationAccount(t, ctx, phaseAStore, "settings"+fmt.Sprint(stamp)) + before := time.Now().UTC() + lease, err := store.beginCheckpoint(ctx, SourceOwned, accountID, "works", before.Add(-48*time.Hour), before.Add(-47*time.Hour)) + if err != nil { + t.Fatalf("begin checkpoint: %v", err) + } + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", ""); err != nil { + t.Fatalf("finish checkpoint: %v", err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET window_start='2020-01-01T00:00:00Z',window_end='2020-01-02T00:00:00Z',cursor='old-cursor' WHERE source_type=$1 AND source_id=$2 AND collection_kind='works'`, SourceOwned, accountID); err != nil { + t.Fatal(err) + } + settings, err := store.GetSettings(ctx) + if err != nil { + t.Fatal(err) + } + settings.LookbackDays = 3 + if _, err := store.UpdateSettings(ctx, SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds, AIProvider: settings.AIProvider, AIModel: settings.AIModel, AIConfigured: settings.AIConfigured, TranscriptionProvider: settings.TranscriptionProvider, TranscriptionModel: settings.TranscriptionModel, TranscriptionConfigured: settings.TranscriptionConfigured}); err != nil { + t.Fatalf("update settings: %v", err) + } + checkpoint, err := store.checkpoint(ctx, SourceOwned, accountID, "works") + if err != nil { + t.Fatal(err) + } + if checkpoint.Cursor != "" || checkpoint.Status != "idle" || checkpoint.WindowStart.Before(time.Now().UTC().Add(-4*24*time.Hour)) { + t.Fatalf("settings did not reset checkpoint: %+v", checkpoint) + } + lease, err = store.beginCheckpoint(ctx, SourceOwned, accountID, "works", time.Now().UTC().Add(-time.Hour), time.Now().UTC()) + if err != nil { + t.Fatalf("begin running checkpoint: %v", err) + } + settings.LookbackDays = 4 + if _, err := store.UpdateSettings(ctx, SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds, AIProvider: settings.AIProvider, AIModel: settings.AIModel, AIConfigured: settings.AIConfigured, TranscriptionProvider: settings.TranscriptionProvider, TranscriptionModel: settings.TranscriptionModel, TranscriptionConfigured: settings.TranscriptionConfigured}); !errors.Is(err, ErrConflict) { + t.Fatalf("settings changed during running checkpoint: err=%v", err) + } + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", ""); err != nil { + t.Fatal(err) + } +} + +func TestCreatorPostgresMetricPlanFollowsPublishedAt(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := fmt.Sprintf("%d", time.Now().UnixNano()) + bigID := createIntegrationAccount(t, ctx, phaseAStore, "metric"+stamp) + if err := store.EnsureAccountProfile(ctx, bigID); err != nil { + t.Fatal(err) + } + published := time.Now().UTC().Add(-time.Hour).Truncate(time.Microsecond) + likes, comments, shares := int64(1), int64(1), int64(1) + work, _, err := store.UpsertWork(ctx, WorkInput{Platform: PlatformDouyin, WorkKey: "creator-it-metric-work-" + stamp, SourceType: SourceOwned, SourceID: bigID, Title: "Metric", Body: "body", PublishedAt: &published, PublishedAtStatus: "verified", Likes: &likes, CommentsCount: &comments, Shares: &shares}, published.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + settings, err := store.GetSettings(ctx) + if err != nil { + t.Fatal(err) + } + settings.MetricInitialIntervalSeconds = 3600 + settings.MetricMultiplier = 2 + settings.MetricMaxIntervalSeconds = 55 * 3600 + settings.MetricAgeSeconds = 100 * 3600 + point := func(at time.Time) MetricInput { + return MetricInput{WorkID: work.ID, CollectedAt: at, Likes: &likes, CommentsCount: &comments, Shares: &shares} + } + if _, err := store.recordMetricWithPlan(ctx, point(published.Add(2*time.Hour)), settings, published.Add(2*time.Hour)); err != nil { + t.Fatal(err) + } + var next time.Time + var interval int64 + if err := store.db.QueryRowContext(ctx, `SELECT next_plan_at,interval_seconds FROM creator_metric_plan WHERE work_id=$1`, work.ID).Scan(&next, &interval); err != nil { + t.Fatal(err) + } + if !next.Equal(published.Add(3*time.Hour)) || interval != 4*3600 { + t.Fatalf("first metric plan drifted: next=%s interval=%d", next, interval) + } + if _, err := store.recordMetricWithPlan(ctx, point(published.Add(3*time.Hour)), settings, published.Add(3*time.Hour)); err != nil { + t.Fatal(err) + } + if err := store.db.QueryRowContext(ctx, `SELECT next_plan_at,interval_seconds FROM creator_metric_plan WHERE work_id=$1`, work.ID).Scan(&next, &interval); err != nil { + t.Fatal(err) + } + if !next.Equal(published.Add(7*time.Hour)) || interval != 8*3600 { + t.Fatalf("second metric plan drifted: next=%s interval=%d", next, interval) + } +} + func prepareIntegrationActionFixture(t *testing.T, store *Store, phaseAStore *phasea.Store, ctx context.Context, stamp string) (string, string, Work, Comment, Strategy) { t.Helper() bigID := createIntegrationAccount(t, ctx, phaseAStore, "big"+stamp) diff --git a/internal/creator/listener.go b/internal/creator/listener.go index 7a81b92..8cb7ca8 100644 --- a/internal/creator/listener.go +++ b/internal/creator/listener.go @@ -3,10 +3,11 @@ package creator import ( "context" "database/sql" + "errors" "strings" ) -const listenerStateSelect = `SELECT account_id,platform,generation,status,boundary_at,last_delivery_id,reason,updated_at FROM creator_listener_state` +const listenerStateSelect = `SELECT account_id,platform,generation,status,boundary_at,last_delivery_id,reason,updated_at,session_token,invalidated FROM creator_listener_state` func validListenerState(input ListenerState) bool { if input.AccountID == "" || !ValidatePlatform(input.Platform) || strings.TrimSpace(input.Generation) == "" { @@ -25,7 +26,7 @@ func validListenerState(input ListenerState) bool { func scanListenerState(scanner interface{ Scan(...any) error }) (ListenerState, error) { var result ListenerState var boundaryAt sql.NullTime - if err := scanner.Scan(&result.AccountID, &result.Platform, &result.Generation, &result.Status, &boundaryAt, &result.LastDeliveryID, &result.Reason, &result.UpdatedAt); err != nil { + if err := scanner.Scan(&result.AccountID, &result.Platform, &result.Generation, &result.Status, &boundaryAt, &result.LastDeliveryID, &result.Reason, &result.UpdatedAt, &result.SessionToken, &result.Invalidated); err != nil { return ListenerState{}, err } result.BoundaryAt = nullableTime(boundaryAt) @@ -35,24 +36,108 @@ func scanListenerState(scanner interface{ Scan(...any) error }) (ListenerState, func (s *Store) UpsertListenerState(ctx context.Context, input ListenerState) (ListenerState, error) { input.Generation = strings.TrimSpace(input.Generation) + input.SessionToken = strings.TrimSpace(input.SessionToken) input.LastDeliveryID = strings.TrimSpace(input.LastDeliveryID) input.Reason = strings.TrimSpace(input.Reason) - if !validListenerState(input) { + if input.SessionToken == "" { + input.SessionToken = input.Generation + } + if !validListenerState(input) || len(input.SessionToken) > 500 { return ListenerState{}, ErrInvalid } - if _, err := s.db.ExecContext(ctx, ` - INSERT INTO creator_listener_state (account_id,platform,generation,status,boundary_at,last_delivery_id,reason,updated_at) - VALUES ($1,$2,$3,$4,$5,$6,$7,now()) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ListenerState{}, databaseError(err) + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, ` + INSERT INTO creator_listener_state (account_id,platform,generation,status,boundary_at,last_delivery_id,reason,session_token,invalidated,updated_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,false,now()) ON CONFLICT (account_id) DO UPDATE SET platform=EXCLUDED.platform,generation=EXCLUDED.generation,status=EXCLUDED.status, boundary_at=COALESCE(EXCLUDED.boundary_at, creator_listener_state.boundary_at), last_delivery_id=CASE WHEN EXCLUDED.last_delivery_id = '' THEN creator_listener_state.last_delivery_id ELSE EXCLUDED.last_delivery_id END, - reason=EXCLUDED.reason,updated_at=now()`, + reason=EXCLUDED.reason,session_token=EXCLUDED.session_token,invalidated=false,updated_at=now() + WHERE NOT creator_listener_state.invalidated OR creator_listener_state.session_token <> EXCLUDED.session_token`, + input.AccountID, input.Platform, input.Generation, input.Status, input.BoundaryAt, input.LastDeliveryID, input.Reason, input.SessionToken) + if err != nil { + return ListenerState{}, databaseError(err) + } + affected, err := result.RowsAffected() + if err != nil { + return ListenerState{}, err + } + if affected != 1 { + _ = tx.Rollback() + state, stateErr := s.GetListenerState(ctx, input.AccountID) + if stateErr != nil { + return ListenerState{}, stateErr + } + return state, ErrConflict + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO creator_listener_boundary (account_id,platform,generation,status,boundary_at,last_delivery_id,reason) + VALUES ($1,$2,$3,$4,$5,$6,$7)`, input.AccountID, input.Platform, input.Generation, input.Status, input.BoundaryAt, input.LastDeliveryID, input.Reason); err != nil { return ListenerState{}, databaseError(err) } + if err := tx.Commit(); err != nil { + return ListenerState{}, databaseError(err) + } return s.GetListenerState(ctx, input.AccountID) } +func (s *Store) InvalidateListener(ctx context.Context, accountID, reason string) error { + accountID = strings.TrimSpace(accountID) + reason = strings.TrimSpace(reason) + if accountID == "" || reason == "" { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return databaseError(err) + } + defer tx.Rollback() + var state ListenerState + var boundaryAt sql.NullTime + if err := tx.QueryRowContext(ctx, listenerStateSelect+` WHERE account_id=$1 FOR UPDATE`, accountID).Scan(&state.AccountID, &state.Platform, &state.Generation, &state.Status, &boundaryAt, &state.LastDeliveryID, &state.Reason, &state.UpdatedAt, &state.SessionToken, &state.Invalidated); errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return databaseError(err) + } + state.BoundaryAt = nullableTime(boundaryAt) + if _, err := tx.ExecContext(ctx, `UPDATE creator_listener_state SET status='gap', reason=$2, invalidated=true, updated_at=now() WHERE account_id=$1`, accountID, reason); err != nil { + return databaseError(err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_listener_boundary (account_id,platform,generation,status,boundary_at,last_delivery_id,reason) VALUES ($1,$2,$3,'gap',$4,$5,$6)`, state.AccountID, state.Platform, state.Generation, state.BoundaryAt, state.LastDeliveryID, reason); err != nil { + return databaseError(err) + } + return tx.Commit() +} + +func (s *Store) ListListenerBoundaries(ctx context.Context, accountID string) ([]ListenerState, error) { + query := `SELECT account_id,platform,generation,status,boundary_at,last_delivery_id,reason,recorded_at,''::text AS session_token,false AS invalidated FROM creator_listener_boundary` + args := []any{} + if strings.TrimSpace(accountID) != "" { + query += ` WHERE account_id=$1` + args = append(args, accountID) + } + query += ` ORDER BY recorded_at,id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]ListenerState, 0) + for rows.Next() { + item, err := scanListenerState(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + func (s *Store) GetListenerState(ctx context.Context, accountID string) (ListenerState, error) { if strings.TrimSpace(accountID) == "" { return ListenerState{}, ErrInvalid diff --git a/internal/creator/metrics.go b/internal/creator/metrics.go index 4339a4d..78fa574 100644 --- a/internal/creator/metrics.go +++ b/internal/creator/metrics.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "math" + "strings" "time" ) @@ -40,6 +41,19 @@ func (s *Store) ListDueMetricWorks(ctx context.Context, now time.Time) ([]Work, return works, nil } +func (s *Store) StopMetricPlan(ctx context.Context, workID, reason string) error { + workID = strings.TrimSpace(workID) + reason = strings.TrimSpace(reason) + if workID == "" || reason == "" || len(reason) > 200 { + return ErrInvalid + } + if _, err := s.db.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=NULL, stopped=true, stop_reason=$2, updated_at=now() WHERE work_id=$1 AND NOT stopped`, workID, reason); err != nil { + return databaseError(err) + } + _, err := s.db.ExecContext(ctx, `UPDATE creator_work SET next_metric_at=NULL, metric_stop_reason=$2, updated_at=now() WHERE id=$1`, workID, reason) + return databaseError(err) +} + func (s *Store) EnsureMetricPlan(ctx context.Context, workID string, settings Settings) error { if err := ValidateSettings(SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds}); err != nil { return err @@ -178,13 +192,20 @@ func (s *Store) recordMetricWithPlan(ctx context.Context, input MetricInput, set if _, err := tx.ExecContext(ctx, `INSERT INTO creator_work_metric (work_id,collected_at,likes,comments_count,shares) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (work_id,collected_at) DO UPDATE SET likes=EXCLUDED.likes,comments_count=EXCLUDED.comments_count,shares=EXCLUDED.shares`, input.WorkID, collectedAt, input.Likes, input.CommentsCount, input.Shares); err != nil { return MetricPoint{}, databaseError(err) } + nextInterval := time.Duration(settings.MetricMaxIntervalSeconds) * time.Second + if !nextAt.IsZero() { + following, _ := NextMetricAt(published, nextAt, time.Duration(settings.MetricInitialIntervalSeconds)*time.Second, time.Duration(settings.MetricMaxIntervalSeconds)*time.Second, settings.MetricMultiplier, time.Duration(settings.MetricAgeSeconds)*time.Second) + if !following.IsZero() { + nextInterval = following.Sub(nextAt) + } + } if nextAt.IsZero() || stopped || !nextPlan.Valid { if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=NULL, stopped=true, stop_reason=$2, updated_at=now() WHERE work_id=$1`, input.WorkID, coalesceReason(nextReason, "monitoring_age_reached")); err != nil { return MetricPoint{}, databaseError(err) } nextAt = time.Time{} nextReason = coalesceReason(nextReason, "monitoring_age_reached") - } else if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=$2, stopped=false, stop_reason='', updated_at=now() WHERE work_id=$1`, input.WorkID, nextAt); err != nil { + } else if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=$2, interval_seconds=$3, stopped=false, stop_reason='', updated_at=now() WHERE work_id=$1`, input.WorkID, nextAt, int64(nextInterval/time.Second)); err != nil { return MetricPoint{}, databaseError(err) } else { nextReason = "" @@ -200,18 +221,19 @@ func (s *Store) recordMetricWithPlan(ctx context.Context, input MetricInput, set if stopped || !nextPlan.Valid || collectedAt.Before(nextPlan.Time.UTC()) { return MetricPoint{}, ErrConflict } - newInterval, err := nextMetricInterval(interval, multiplier, maximum) + nextInterval, err := nextMetricInterval(interval, multiplier, maximum) if err != nil { return MetricPoint{}, err } - nextAt := nextPlan.Time.UTC().Add(time.Duration(newInterval) * time.Second) + nextAt := nextPlan.Time.UTC().Add(time.Duration(interval) * time.Second) steps := 1 for !nextAt.After(now) && nextAt.Before(monitoringEnd) { - newInterval, err = nextMetricInterval(newInterval, multiplier, maximum) + interval = nextInterval + nextAt = nextAt.Add(time.Duration(interval) * time.Second) + nextInterval, err = nextMetricInterval(interval, multiplier, maximum) if err != nil { return MetricPoint{}, err } - nextAt = nextAt.Add(time.Duration(newInterval) * time.Second) steps++ } stopped = !nextAt.Before(monitoringEnd) @@ -224,7 +246,7 @@ func (s *Store) recordMetricWithPlan(ctx context.Context, input MetricInput, set if _, err := tx.ExecContext(ctx, `INSERT INTO creator_work_metric (work_id,collected_at,likes,comments_count,shares) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (work_id,collected_at) DO UPDATE SET likes=EXCLUDED.likes,comments_count=EXCLUDED.comments_count,shares=EXCLUDED.shares`, input.WorkID, collectedAt, input.Likes, input.CommentsCount, input.Shares); err != nil { return MetricPoint{}, databaseError(err) } - if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=$2, interval_seconds=$3, point_index=point_index+$4, stopped=$5, stop_reason=$6, updated_at=now() WHERE work_id=$1`, input.WorkID, nullableArg(nextAt), newInterval, steps, stopped, stopReason); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=$2, interval_seconds=$3, point_index=point_index+$4, stopped=$5, stop_reason=$6, updated_at=now() WHERE work_id=$1`, input.WorkID, nullableArg(nextAt), nextInterval, steps, stopped, stopReason); err != nil { return MetricPoint{}, databaseError(err) } if _, err := tx.ExecContext(ctx, `UPDATE creator_work SET likes=$2,comments_count=$3,shares=$4,latest_metrics_at=$5,next_metric_at=$6,metric_stop_reason=$7,updated_at=now() WHERE id=$1`, input.WorkID, input.Likes, input.CommentsCount, input.Shares, collectedAt, nullableArg(nextAt), stopReason); err != nil { diff --git a/internal/creator/migrations/032_douyin_release_remediation.sql b/internal/creator/migrations/032_douyin_release_remediation.sql new file mode 100644 index 0000000..74ca138 --- /dev/null +++ b/internal/creator/migrations/032_douyin_release_remediation.sql @@ -0,0 +1,52 @@ +ALTER TABLE creator_event + ADD COLUMN IF NOT EXISTS generation text NOT NULL DEFAULT ''; + +ALTER TABLE creator_operation + ADD COLUMN IF NOT EXISTS verification_state text NOT NULL DEFAULT 'not_verified' + CHECK (verification_state IN ('not_verified', 'not_sent', 'succeeded', 'failed', 'blocked', 'uncertain')), + ADD COLUMN IF NOT EXISTS verification_evidence jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS verified_at timestamptz; + +ALTER TABLE creator_message + ADD COLUMN IF NOT EXISTS operation_id text REFERENCES creator_operation(id) ON DELETE SET NULL; +CREATE INDEX IF NOT EXISTS creator_message_operation_idx ON creator_message (operation_id); + +ALTER TABLE creator_conversation + ADD COLUMN IF NOT EXISTS history_cursor text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS history_has_more boolean NOT NULL DEFAULT false, + ADD COLUMN IF NOT EXISTS history_synced_at timestamptz; + +ALTER TABLE creator_material_job + ADD COLUMN IF NOT EXISTS processing_step text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS processing_token text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS processing_started_at timestamptz; + +CREATE TABLE IF NOT EXISTS creator_event_strategy_trace ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event_id text NOT NULL REFERENCES creator_event(id) ON DELETE CASCADE, + strategy_id text NOT NULL, + position integer NOT NULL CHECK (position > 0), + outcome text NOT NULL CHECK (outcome IN ('skipped', 'selected')), + reason text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS creator_event_strategy_trace_idx + ON creator_event_strategy_trace (event_id, position, id); + +CREATE TABLE IF NOT EXISTS creator_listener_boundary ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + account_id text NOT NULL REFERENCES social_account(id) ON DELETE CASCADE, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + generation text NOT NULL, + status text NOT NULL CHECK (status IN ('starting', 'ready', 'gap', 'stopped', 'error')), + boundary_at timestamptz, + last_delivery_id text NOT NULL DEFAULT '', + reason text NOT NULL DEFAULT '', + recorded_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS creator_listener_boundary_account_idx +ON creator_listener_boundary (account_id, recorded_at, id); + +ALTER TABLE creator_listener_state +ADD COLUMN IF NOT EXISTS session_token text NOT NULL DEFAULT '', +ADD COLUMN IF NOT EXISTS invalidated boolean NOT NULL DEFAULT false; diff --git a/internal/creator/models.go b/internal/creator/models.go index da232c0..c74dd46 100644 --- a/internal/creator/models.go +++ b/internal/creator/models.go @@ -178,14 +178,15 @@ type WorkInput struct { } type WorkFilter struct { - Platform string - SourceID string - SourceType string - PublishedAfter *time.Time - PublishedBefore *time.Time - MinLikes *int64 - MinComments *int64 - MinShares *int64 + Platform string + SourceID string + PublishedAtStatus string + SourceType string + PublishedAfter *time.Time + PublishedBefore *time.Time + MinLikes *int64 + MinComments *int64 + MinShares *int64 } type MetricInput struct { @@ -219,6 +220,9 @@ type MaterialJob struct { RewriteRequirement string `json:"rewrite_requirement,omitempty"` GeneratedTitle string `json:"generated_title,omitempty"` GeneratedScript string `json:"generated_script,omitempty"` + ProcessingStep string `json:"processing_step,omitempty"` + ProcessingToken string `json:"processing_token,omitempty"` + ProcessingStartedAt *time.Time `json:"processing_started_at,omitempty"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } @@ -332,6 +336,8 @@ type ListenerState struct { LastDeliveryID string `json:"last_delivery_id,omitempty"` Reason string `json:"reason,omitempty"` UpdatedAt time.Time `json:"updated_at"` + SessionToken string `json:"-"` + Invalidated bool `json:"-"` } type Page[T any] struct { @@ -345,6 +351,7 @@ type Page[T any] struct { type InteractionEvent struct { ID string `json:"id"` Platform string `json:"platform"` + Generation string `json:"generation,omitempty"` ReceivingAccountID string `json:"receiving_account_id"` EventKey string `json:"event_key"` EventType string `json:"event_type"` @@ -368,23 +375,26 @@ type InteractionEvent struct { } type Operation struct { - ID string `json:"id"` - IdempotencyKey string `json:"idempotency_key"` - Source string `json:"source"` - Action string `json:"action"` - Platform string `json:"platform"` - AccountID string `json:"account_id"` - TargetUID string `json:"target_uid,omitempty"` - TargetCommentID string `json:"target_comment_id,omitempty"` - TargetWorkID string `json:"target_work_id,omitempty"` - Text string `json:"text,omitempty"` - EventID string `json:"event_id,omitempty"` - StrategyID string `json:"strategy_id,omitempty"` - State string `json:"state"` - Evidence map[string]string `json:"evidence"` - Reason string `json:"reason,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID string `json:"id"` + IdempotencyKey string `json:"idempotency_key"` + Source string `json:"source"` + Action string `json:"action"` + Platform string `json:"platform"` + AccountID string `json:"account_id"` + TargetUID string `json:"target_uid,omitempty"` + TargetCommentID string `json:"target_comment_id,omitempty"` + TargetWorkID string `json:"target_work_id,omitempty"` + Text string `json:"text,omitempty"` + EventID string `json:"event_id,omitempty"` + StrategyID string `json:"strategy_id,omitempty"` + State string `json:"state"` + Evidence map[string]string `json:"evidence"` + Reason string `json:"reason,omitempty"` + VerificationState string `json:"verification_state"` + VerificationProof map[string]string `json:"verification_evidence"` + VerifiedAt *time.Time `json:"verified_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } type OperationInput struct { @@ -402,18 +412,22 @@ type OperationInput struct { } type Conversation struct { - ID string `json:"id"` - Platform string `json:"platform"` - AccountID string `json:"account_id"` - PeerUID string `json:"peer_uid"` - PeerName string `json:"peer_name"` - LastMessageAt *time.Time `json:"last_message_at,omitempty"` + ID string `json:"id"` + Platform string `json:"platform"` + AccountID string `json:"account_id"` + PeerUID string `json:"peer_uid"` + PeerName string `json:"peer_name"` + LastMessageAt *time.Time `json:"last_message_at,omitempty"` + HistoryCursor string `json:"history_cursor,omitempty"` + HistoryHasMore bool `json:"history_has_more,omitempty"` + HistorySyncedAt *time.Time `json:"history_synced_at,omitempty"` } type Message struct { ID string `json:"id"` ConversationID string `json:"conversation_id"` PlatformMessageKey string `json:"platform_message_key"` + OperationID string `json:"operation_id,omitempty"` Direction string `json:"direction"` MessageType string `json:"message_type"` Text string `json:"text,omitempty"` @@ -428,6 +442,7 @@ type MessageInput struct { PeerUID string `json:"peer_uid"` PeerName string `json:"peer_name"` PlatformMessageKey string `json:"platform_message_key"` + OperationID string `json:"operation_id,omitempty"` Direction string `json:"direction"` MessageType string `json:"message_type"` Text string `json:"text"` @@ -458,6 +473,15 @@ type AutomaticResult struct { Duplicate bool `json:"duplicate"` } +type StrategyTrace struct { + EventID string `json:"event_id"` + StrategyID string `json:"strategy_id"` + Position int `json:"position"` + Outcome string `json:"outcome"` + Reason string `json:"reason,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + type ActionExecutor interface { Execute(context.Context, ActionRequest) (ActionResult, error) } diff --git a/internal/creator/recovery_integration_test.go b/internal/creator/recovery_integration_test.go index adb3c0b..c16af82 100644 --- a/internal/creator/recovery_integration_test.go +++ b/internal/creator/recovery_integration_test.go @@ -1,11 +1,172 @@ package creator import ( + "context" + "errors" "fmt" "testing" "time" ) +func TestCreatorPostgresRecoversReceivedEventWithoutReplay(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + bigID, _, work, comment, _ := prepareIntegrationActionFixture(t, store, phaseAStore, ctx, fmt.Sprintf("%d", stamp)) + platformAt := time.Now().UTC().Add(-2 * time.Minute) + recorded, err := store.RecordEvent(ctx, InteractionEvent{Platform: PlatformDouyin, ReceivingAccountID: bigID, EventKey: fmt.Sprintf("received-event-%d", stamp), EventType: "comment", InteractorUID: comment.AuthorUID, CommentID: comment.ID, WorkID: work.ID, PlatformEventAt: &platformAt}) + if err != nil { + t.Fatalf("record event: result=%+v err=%v", recorded, err) + } + old := time.Now().UTC().Add(-3 * time.Minute) + if _, err := store.db.ExecContext(ctx, `UPDATE creator_event SET state='received',received_at=$2 WHERE id=$1`, recorded.Event.ID, old); err != nil { + t.Fatal(err) + } + count, err := store.RecoverStaleProcessing(ctx, time.Now().UTC()) + if err != nil || count != 1 { + t.Fatalf("recover received event: count=%d err=%v", count, err) + } + recovered, err := store.GetEvent(ctx, recorded.Event.ID) + if err != nil || recovered.State != "uncertain" || recovered.Reason != "事件已收到但未开始处理，未补发" { + t.Fatalf("received event was not held without replay: event=%+v err=%v", recovered, err) + } +} + +type integrationActionExecutor struct { + called chan struct{} +} + +func (e integrationActionExecutor) Execute(context.Context, ActionRequest) (ActionResult, error) { + e.called <- struct{}{} + return ActionResult{State: "succeeded"}, nil +} + +func TestCreatorPostgresRecoveryDoesNotReleaseQueuedAutomaticOperation(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + bigID, smallID, work, comment, _ := prepareIntegrationActionFixture(t, store, phaseAStore, ctx, fmt.Sprintf("%d", stamp)) + input := InteractionEvent{Platform: PlatformDouyin, ReceivingAccountID: bigID, EventKey: fmt.Sprintf("queued-event-%d", stamp), EventType: "comment", InteractorUID: comment.AuthorUID, CommentID: comment.ID, WorkID: work.ID} + lock := store.automaticExecutionLock(smallID) + lock.Lock() + executor := integrationActionExecutor{called: make(chan struct{}, 1)} + done := make(chan struct{}) + go func() { + _, _ = store.ProcessAutomaticEvent(ctx, input, executor, nil) + close(done) + }() + var operationID string + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + err := store.db.QueryRowContext(ctx, `SELECT id FROM creator_operation WHERE idempotency_key=$1 AND state='processing'`, "event:douyin:"+bigID+":"+input.EventKey).Scan(&operationID) + if err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if operationID == "" { + lock.Unlock() + t.Fatal("automatic operation did not reach the execution lock") + } + if _, err := store.db.ExecContext(ctx, `UPDATE creator_operation SET updated_at=$2 WHERE id=$1`, operationID, time.Now().UTC().Add(-10*time.Minute)); err != nil { + lock.Unlock() + t.Fatal(err) + } + count, err := store.RecoverStaleProcessing(ctx, time.Now().UTC()) + if err != nil || count != 1 { + lock.Unlock() + t.Fatalf("recover queued operation: count=%d err=%v", count, err) + } + lock.Unlock() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("automatic operation did not observe recovery") + } + select { + case <-executor.called: + t.Fatal("recovered automatic operation reached the platform executor") + default: + } + op, err := store.GetOperation(ctx, operationID) + if err != nil || op.State != "uncertain" { + t.Fatalf("recovered operation state: operation=%+v err=%v", op, err) + } +} + +func TestCreatorPostgresInvalidatedListenerBlocksAutomaticWrite(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + bigID, _, work, comment, _ := prepareIntegrationActionFixture(t, store, phaseAStore, ctx, fmt.Sprintf("%d", stamp)) + boundary := time.Now().UTC() + generation := "runtime:network:1" + if _, err := store.UpsertListenerState(ctx, ListenerState{AccountID: bigID, Platform: PlatformDouyin, Generation: generation, SessionToken: "session-1", Status: "ready", BoundaryAt: &boundary}); err != nil { + t.Fatal(err) + } + if err := store.InvalidateListener(ctx, bigID, "strategy changed"); err != nil { + t.Fatal(err) + } + result, err := store.ProcessAutomaticEvent(ctx, InteractionEvent{Platform: PlatformDouyin, ReceivingAccountID: bigID, Generation: generation, EventKey: fmt.Sprintf("invalidated-event-%d", stamp), EventType: "comment", InteractorUID: comment.AuthorUID, CommentID: comment.ID, WorkID: work.ID}, integrationActionExecutor{called: make(chan struct{}, 1)}, nil) + if err != nil { + t.Fatalf("invalidated listener processing failed: result=%+v err=%v", result, err) + } + if result.Operation != nil || result.Event.State != "blocked" || result.Event.Reason != "监听代际未就绪" { + t.Fatalf("invalidated listener was allowed to process: result=%+v", result) + } +} + +func TestCreatorPostgresInvalidatedListenerRejectsStaleSession(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + accountID, _, _, _, _ := prepareIntegrationActionFixture(t, store, phaseAStore, ctx, fmt.Sprintf("%d", stamp)) + boundary := time.Now().UTC() + initial := ListenerState{AccountID: accountID, Platform: PlatformDouyin, Generation: "runtime:network:1", SessionToken: "session-1", Status: "ready", BoundaryAt: &boundary} + if _, err := store.UpsertListenerState(ctx, initial); err != nil { + t.Fatalf("upsert initial listener: %v", err) + } + if err := store.InvalidateListener(ctx, accountID, "strategy changed"); err != nil { + t.Fatalf("invalidate listener: %v", err) + } + if _, err := store.UpsertListenerState(ctx, initial); !errors.Is(err, ErrConflict) { + t.Fatalf("stale listener session was accepted: err=%v", err) + } + invalidated, err := store.GetListenerState(ctx, accountID) + if err != nil || invalidated.Status != "gap" || !invalidated.Invalidated || invalidated.SessionToken != "session-1" { + t.Fatalf("invalidated listener state changed unexpectedly: state=%+v err=%v", invalidated, err) + } + fresh := initial + fresh.SessionToken = "session-2" + fresh.Status = "ready" + if _, err := store.UpsertListenerState(ctx, fresh); err != nil { + t.Fatalf("upsert fresh listener: %v", err) + } + active, err := store.GetListenerState(ctx, accountID) + if err != nil || active.Status != "ready" || active.Invalidated || active.SessionToken != "session-2" { + t.Fatalf("fresh listener session was not activated: state=%+v err=%v", active, err) + } +} + +func TestCreatorPostgresMaterialClaimDoesNotAcceptStaleCompletion(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := time.Now().UnixNano() + _, _, work, _, _ := prepareIntegrationActionFixture(t, store, phaseAStore, ctx, fmt.Sprintf("%d", stamp)) + if _, _, err := store.SelectMaterial(ctx, work.ID); err != nil { + t.Fatalf("select material: %v", err) + } + claimed, ok, err := store.ClaimMaterialStep(ctx, work.ID, "download", "old-token") + if err != nil || !ok || claimed.ProcessingToken != "old-token" { + t.Fatalf("claim material: job=%+v claimed=%v err=%v", claimed, ok, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE creator_material_job SET processing_started_at=$2 WHERE work_id=$1`, work.ID, time.Now().UTC().Add(-time.Hour)); err != nil { + t.Fatal(err) + } + recovered, ok, err := store.ClaimMaterialStep(ctx, work.ID, "download", "new-token") + if err != nil || ok || recovered.DownloadStatus != "failed" || recovered.ProcessingToken != "" || recovered.FailureReason != "上次处理结果不明，未自动重试" { + t.Fatalf("stale claim was retried or not recovered: job=%+v claimed=%v err=%v", recovered, ok, err) + } + if _, err := store.CompleteMaterialStep(ctx, work.ID, "download", "old-token", "succeeded", "late-video", ""); !errors.Is(err, ErrConflict) { + t.Fatalf("stale completion was accepted: err=%v", err) + } +} + func TestCreatorPostgresRecoversStaleProcessingWithoutRetry(t *testing.T) { store, phaseAStore, ctx := openCreatorIntegrationStore(t) stamp := time.Now().UnixNano() diff --git a/internal/creator/settings.go b/internal/creator/settings.go index 0fe65c5..3ca07f2 100644 --- a/internal/creator/settings.go +++ b/internal/creator/settings.go @@ -59,6 +59,13 @@ func (s *Store) UpdateSettings(ctx context.Context, input SettingsUpdate) (Setti input.TranscriptionProvider, input.TranscriptionModel, input.TranscriptionConfigured); err != nil { return Settings{}, databaseError(err) } + var runningCheckpoints int + if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM creator_collection_checkpoint WHERE status='running'`).Scan(&runningCheckpoints); err != nil { + return Settings{}, databaseError(err) + } + if runningCheckpoints > 0 { + return Settings{}, ErrConflict + } rows, err := tx.QueryContext(ctx, ` SELECT p.work_id, w.published_at FROM creator_metric_plan p @@ -88,6 +95,18 @@ func (s *Store) UpdateSettings(ctx context.Context, input SettingsUpdate) (Setti return Settings{}, err } now := time.Now().UTC() + start, end, err := NewCollectionWindow(now, input.LookbackDays) + if err != nil { + return Settings{}, err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE creator_collection_checkpoint + SET window_start=$1, window_end=$2, cursor='', lease_token='', lease_until=NULL, + status=CASE WHEN status='blocked' THEN 'blocked' ELSE 'idle' END, + last_error=CASE WHEN status='blocked' THEN last_error ELSE '' END + WHERE status <> 'running'`, start, end); err != nil { + return Settings{}, databaseError(err) + } for _, schedule := range schedules { nextAt, reason := NextMetricAtValue(schedule.publishedAt, now, input) stopped := nextAt.IsZero() diff --git a/internal/creator/store.go b/internal/creator/store.go index fe8f26a..a882f13 100644 --- a/internal/creator/store.go +++ b/internal/creator/store.go @@ -61,6 +61,9 @@ var migration030 string //go:embed migrations/031_xhs_raw_payloads.sql var migration031 string +//go:embed migrations/032_douyin_release_remediation.sql +var migration032 string + type SecretReference struct { ID string Provider string @@ -110,6 +113,18 @@ func (s *Store) automaticExecutionLock(accountID string) *sync.Mutex { return lock.(*sync.Mutex) } +func (s *Store) acquireAutomaticExecutionLock(ctx context.Context, accountID string) (func(), error) { + conn, err := s.db.Conn(ctx) + if err != nil { + return nil, databaseError(err) + } + if _, err := conn.ExecContext(ctx, `SELECT pg_advisory_lock(hashtextextended($1, 0))`, accountID); err != nil { + _ = conn.Close() + return nil, databaseError(err) + } + return func() { _ = conn.Close() }, nil +} + func (s *Store) migrate(ctx context.Context) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -141,6 +156,7 @@ func (s *Store) migrate(ctx context.Context) error { {version: 29, sql: migration029}, {version: 30, sql: migration030}, {version: 31, sql: migration031}, + {version: 32, sql: migration032}, } for _, migration := range migrations { var applied bool @@ -216,6 +232,13 @@ func decodeStringList(encoded []byte) ([]string, error) { return result, nil } +func nullableString(value string) any { + if value == "" { + return nil + } + return value +} + func nullableTime(value sql.NullTime) *time.Time { if !value.Valid { return nil diff --git a/internal/douyin/connector.go b/internal/douyin/connector.go index 232b2f5..7b5c534 100644 --- a/internal/douyin/connector.go +++ b/internal/douyin/connector.go @@ -306,9 +306,13 @@ func parseCredential(raw []byte) ([]Cookie, error) { type identityEnvelope struct { StatusCode *int `json:"status_code"` User *struct { - UID string `json:"uid"` - SecUID string `json:"sec_uid"` - UniqueID string `json:"unique_id"` + UID string `json:"uid"` + SecUID string `json:"sec_uid"` + UniqueID string `json:"unique_id"` + Nickname string `json:"nickname"` + AvatarThumb *struct { + URLList []string `json:"url_list"` + } `json:"avatar_thumb"` } `json:"user"` } diff --git a/internal/douyin/creator_collector.go b/internal/douyin/creator_collector.go index 6c72349..adb744a 100644 --- a/internal/douyin/creator_collector.go +++ b/internal/douyin/creator_collector.go @@ -24,6 +24,14 @@ type CreatorCollector struct { SourceID string } +type TargetProfile struct { + UID string + SecUID string + UniqueID string + Nickname string + AvatarURL string +} + func (c CreatorCollector) CanonicalSecUID(ctx context.Context, expectedKey string) (string, error) { if c.Browser == nil || !keyPattern.MatchString(expectedKey) { return "", fmt.Errorf("%w: invalid identity verification request", ErrInvalid) @@ -36,13 +44,21 @@ func (c CreatorCollector) CanonicalSecUID(ctx context.Context, expectedKey strin } func (c CreatorCollector) CanonicalTargetSecUID(ctx context.Context, expectedKey string) (string, error) { + profile, err := c.ResolveTarget(ctx, expectedKey) + if err != nil { + return "", err + } + return profile.SecUID, nil +} + +func (c CreatorCollector) ResolveTarget(ctx context.Context, expectedKey string) (TargetProfile, error) { if c.Browser == nil || !keyPattern.MatchString(expectedKey) { - return "", fmt.Errorf("%w: invalid target identity request", ErrInvalid) + return TargetProfile{}, fmt.Errorf("%w: invalid target identity request", ErrInvalid) } field := "user_id" if _, err := strconv.ParseUint(expectedKey, 10, 64); err != nil { if !strings.HasPrefix(expectedKey, "MS4") { - return "", fmt.Errorf("%w: target requires a Douyin UID or sec UID", ErrInvalid) + return TargetProfile{}, fmt.Errorf("%w: target requires a Douyin UID or sec UID", ErrInvalid) } field = "sec_user_id" } @@ -50,9 +66,20 @@ func (c CreatorCollector) CanonicalTargetSecUID(ctx context.Context, expectedKey query.Set(field, expectedKey) response, err := c.Browser.Get(ctx, profileOtherEndpoint+"?"+query.Encode()) if err != nil { - return "", err + return TargetProfile{}, err } - return canonicalSecUID(response, expectedKey) + if err := creatorResponseError(response, "target profile"); err != nil { + return TargetProfile{}, err + } + identity, ok := parseIdentity(response.Body) + if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID { + return TargetProfile{}, fmt.Errorf("%w: douyin target identity mismatch", ErrInvalid) + } + avatarURL := "" + if identity.User.AvatarThumb != nil && len(identity.User.AvatarThumb.URLList) > 0 { + avatarURL = identity.User.AvatarThumb.URLList[0] + } + return TargetProfile{UID: identity.User.UID, SecUID: identity.User.SecUID, UniqueID: identity.User.UniqueID, Nickname: identity.User.Nickname, AvatarURL: avatarURL}, nil } func (c CreatorCollector) VerifyIdentity(ctx context.Context, expectedKey string) error { @@ -116,7 +143,9 @@ func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor stri sourceID = accountKey } status := "pending_verification" - if published != nil { + if work.CreatedAtInvalid { + status = "invalid" + } else if published != nil { status = "verified" } items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, Likes: likes, CommentsCount: comments, Shares: shares}) @@ -181,12 +210,13 @@ func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) { } type creatorWorkPageItem struct { - ID string - Description string - CreatedAt *int64 - DiggCount *int64 - CommentCount *int64 - ShareCount *int64 + ID string + Description string + CreatedAt *int64 + CreatedAtInvalid bool + DiggCount *int64 + CommentCount *int64 + ShareCount *int64 } func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) { @@ -217,10 +247,11 @@ func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bo } } createdAt := item.CreatedAt - if createdAt != nil && *createdAt <= 0 { + createdAtInvalid := createdAt != nil && *createdAt <= 0 + if createdAtInvalid { createdAt = nil } - items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, DiggCount: likes, CommentCount: comments, ShareCount: shares}) + items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares}) } return items, *envelope.HasMore, envelope.MaxCursor, true } diff --git a/internal/douyin/creator_collector_test.go b/internal/douyin/creator_collector_test.go index afffd17..8a3adfe 100644 --- a/internal/douyin/creator_collector_test.go +++ b/internal/douyin/creator_collector_test.go @@ -29,6 +29,14 @@ func TestCanonicalTargetSecUIDUsesTargetProfileEndpoint(t *testing.T) { } } +func TestResolveTargetReturnsVerifiedProfileFields(t *testing.T) { + browser := &collectorBrowser{response: Response{Status: 200, Body: []byte(`{"status_code":0,"user":{"uid":"2328120603967913","sec_uid":"MS4wLjABAAAA9f_a7k0bzVizLYXlpC7R61EIaqJ8Ordug7yp7AB8fGKuuF8Fzqk5_DM-eutXnPIK","unique_id":"96332518739","nickname":"目标","avatar_thumb":{"url_list":["https://example.invalid/avatar"]}}}`)}} + profile, err := (CreatorCollector{Browser: browser}).ResolveTarget(context.Background(), "MS4wLjABAAAA9f_a7k0bzVizLYXlpC7R61EIaqJ8Ordug7yp7AB8fGKuuF8Fzqk5_DM-eutXnPIK") + if err != nil || profile.Nickname != "目标" || profile.AvatarURL == "" || profile.SecUID == "" { + t.Fatalf("target profile: %+v err=%v", profile, err) + } +} + func TestCanonicalTargetSecUIDRejectsUniqueIDLookup(t *testing.T) { browser := &collectorBrowser{} _, err := (CreatorCollector{Browser: browser}).CanonicalTargetSecUID(context.Background(), "creator_handle") @@ -37,6 +45,35 @@ func TestCanonicalTargetSecUIDRejectsUniqueIDLookup(t *testing.T) { } } +func TestCreatorCollectorGuardsAndCollection(t *testing.T) { + ctx := context.Background() + collector := CreatorCollector{} + if _, err := collector.CanonicalSecUID(ctx, ""); err == nil { + t.Fatal("empty identity key accepted") + } + if err := collector.VerifyIdentity(ctx, ""); err == nil { + t.Fatal("empty verification key accepted") + } + if _, err := collector.ListWorks(ctx, "", ""); err == nil { + t.Fatal("empty work collection key accepted") + } + if _, err := collector.ListTopLevelComments(ctx, "", ""); err == nil { + t.Fatal("empty comment collection key accepted") + } + browser := &collectorBrowser{response: Response{Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"cursor":0,"aweme_list":[],"comments":[]}`)}} + collector = CreatorCollector{Browser: browser, AccountKey: "MS4wLjABAAAAkey"} + if page, err := collector.ListWorks(ctx, "ignored", ""); err != nil || page.HasMore || len(page.Items) != 0 { + t.Fatalf("empty work page: %+v %v", page, err) + } + if page, err := collector.ListTopLevelComments(ctx, "123", ""); err != nil || page.HasMore || len(page.Items) != 0 { + t.Fatalf("empty comment page: %+v %v", page, err) + } + browser.response.Body = []byte(`{"status_code":0,"user":{"uid":"uid","sec_uid":"MS4wLjABAAAAkey","unique_id":"handle"}}`) + if secUID, err := collector.CanonicalSecUID(ctx, "MS4wLjABAAAAkey"); err != nil || secUID != "MS4wLjABAAAAkey" { + t.Fatalf("canonical identity: %q %v", secUID, err) + } +} + func TestParseCreatorCommentsPageAllowsEmptyComments(t *testing.T) { page, err := parseCreatorCommentsPage([]byte(`{"status_code":0,"has_more":false,"cursor":20,"comments":null}`)) if err != nil { @@ -47,6 +84,14 @@ func TestParseCreatorCommentsPageAllowsEmptyComments(t *testing.T) { } } +func TestParseCreatorWorksPageMarksInvalidTimestamp(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"invalid","create_time":-1}]}`) + works, _, _, ok := parseCreatorWorksPage(body) + if !ok || len(works) != 1 || !works[0].CreatedAtInvalid || works[0].CreatedAt != nil { + t.Fatalf("invalid timestamp was not preserved: ok=%v works=%+v", ok, works) + } +} + func TestParseCreatorWorksPageKeepsPartialMetadata(t *testing.T) { body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"partial","statistics":{"digg_count":7}}]}`) works, hasMore, cursor, ok := parseCreatorWorksPage(body) diff --git a/internal/hub/environment.go b/internal/hub/environment.go index 5afea3e..bf7dc1a 100644 --- a/internal/hub/environment.go +++ b/internal/hub/environment.go @@ -156,6 +156,76 @@ func validOptionalRegion(region string) bool { return true } +func (s *Store) UpdateNetworkExit(ctx context.Context, id string, input NetworkExit) (NetworkExit, error) { + if !exitIDPattern.MatchString(id) { + return NetworkExit{}, ErrInvalid + } + input.Protocol, input.Host = strings.ToLower(strings.TrimSpace(input.Protocol)), strings.TrimSpace(input.Host) + input.ExpectedPublicIP, input.ExpectedRegion = strings.TrimSpace(input.ExpectedPublicIP), strings.TrimSpace(input.ExpectedRegion) + if !validNetworkExit(input) { + return NetworkExit{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return NetworkExit{}, errors.New("begin network exit update") + } + defer tx.Rollback() + var active bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM environment_binding binding JOIN runtime_instance runtime ON runtime.binding_id=binding.id WHERE binding.network_exit_id=$1 AND runtime.released_at IS NULL AND runtime.lease_until > now()) OR EXISTS (SELECT 1 FROM environment_binding binding JOIN operation_task task ON task.account_id=binding.account_id WHERE binding.network_exit_id=$1 AND task.state='executing')`, id).Scan(&active); err != nil { + return NetworkExit{}, errors.New("check network exit activity") + } + if active { + return NetworkExit{}, ErrConflict + } + if _, err := tx.ExecContext(ctx, `UPDATE network_exit SET protocol=$2,host=$3,port=$4,username=$5,password=$6,expected_public_ip=NULLIF($7,'')::inet,expected_region=$8,health_status=CASE WHEN health_status='disabled' THEN 'disabled' ELSE 'unchecked' END,last_check_reason='exit_configuration_changed',version=version+1,updated_at=now() WHERE id=$1`, id, input.Protocol, input.Host, input.Port, input.Username, input.Password, input.ExpectedPublicIP, input.ExpectedRegion); err != nil { + return NetworkExit{}, rowError(err) + } + if err := tx.Commit(); err != nil { + return NetworkExit{}, err + } + return s.GetNetworkExit(ctx, id) +} + +func (s *Store) EnableNetworkExit(ctx context.Context, id string) (NetworkExit, error) { + if !exitIDPattern.MatchString(id) { + return NetworkExit{}, ErrInvalid + } + if _, err := s.db.ExecContext(ctx, `UPDATE network_exit SET health_status='unchecked',last_check_reason='exit_enabled',version=version+1,updated_at=now() WHERE id=$1 AND health_status='disabled'`, id); err != nil { + return NetworkExit{}, rowError(err) + } + return s.GetNetworkExit(ctx, id) +} + +func (s *Store) DeleteNetworkExit(ctx context.Context, id string) error { + if !exitIDPattern.MatchString(id) { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin network exit delete") + } + defer tx.Rollback() + var used bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM environment_binding WHERE network_exit_id=$1) OR EXISTS (SELECT 1 FROM environment_binding binding JOIN runtime_instance runtime ON runtime.binding_id=binding.id WHERE binding.network_exit_id=$1 AND runtime.released_at IS NULL AND runtime.lease_until > now())`, id).Scan(&used); err != nil { + return errors.New("check network exit bindings") + } + if used { + return ErrConflict + } + result, err := tx.ExecContext(ctx, `DELETE FROM network_exit WHERE id=$1`, id) + if err != nil { + return rowError(err) + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return ErrNotFound + } + return tx.Commit() +} + func (s *Store) ListNetworkExits(ctx context.Context) ([]NetworkExit, error) { rows, err := s.db.QueryContext(ctx, networkExitSelect+` ORDER BY network.created_at, network.id`) if err != nil { @@ -295,6 +365,21 @@ func (s *Store) DisableNetworkExit(ctx context.Context, id string) (NetworkExit, if err := tx.QueryRowContext(ctx, `SELECT health_status FROM network_exit WHERE id = $1 FOR UPDATE`, id).Scan(&oldStatus); err != nil { return NetworkExit{}, rowError(err) } + var active bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS ( + SELECT 1 FROM environment_binding binding + JOIN runtime_instance runtime ON runtime.binding_id = binding.id + WHERE binding.network_exit_id = $1 AND runtime.released_at IS NULL AND runtime.lease_until > now() + ) OR EXISTS ( + SELECT 1 FROM environment_binding binding + JOIN operation_task task ON task.account_id = binding.account_id + WHERE binding.network_exit_id = $1 AND task.state = 'executing' + )`, id).Scan(&active); err != nil { + return NetworkExit{}, errors.New("check network exit activity") + } + if active { + return NetworkExit{}, ErrConflict + } var transitions []taskstate.Transition if oldStatus != "disabled" { if _, err := tx.ExecContext(ctx, ` diff --git a/internal/hub/migrations/033_unique_fingerprint_seed.sql b/internal/hub/migrations/033_unique_fingerprint_seed.sql new file mode 100644 index 0000000..8a4417f --- /dev/null +++ b/internal/hub/migrations/033_unique_fingerprint_seed.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX IF NOT EXISTS browser_env_fingerprint_seed_idx +ON browser_env ((fingerprint->>'seed')) +WHERE fingerprint ? 'seed'; diff --git a/internal/hub/store.go b/internal/hub/store.go index 0ea78b4..224a450 100644 --- a/internal/hub/store.go +++ b/internal/hub/store.go @@ -66,6 +66,9 @@ var migration015 string //go:embed migrations/016_network_exit_plain_credentials.sql var migration016 string +//go:embed migrations/033_unique_fingerprint_seed.sql +var migration033 string + var ( ErrConflict = errors.New("resource conflicts with existing state") ErrInvalid = errors.New("invalid hub input") @@ -224,7 +227,7 @@ func (s *Store) migrate(ctx context.Context) error { for _, migration := range []struct { version int sql string - }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}, {11, migration011}, {12, migration012}, {13, migration013}, {14, migration014}, {15, migration015}, {16, migration016}} { + }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}, {11, migration011}, {12, migration012}, {13, migration013}, {14, migration014}, {15, migration015}, {16, migration016}, {33, migration033}} { var applied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { return errors.New("read hub schema migration state") diff --git a/internal/hub/store_test.go b/internal/hub/store_test.go index 5fe0a39..5a74bcc 100644 --- a/internal/hub/store_test.go +++ b/internal/hub/store_test.go @@ -294,6 +294,40 @@ func TestStoreValidationRejectsInvalidInputsBeforePersistence(t *testing.T) { } } +func TestFingerprintSeedIsGloballyUnique(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) + t.Cleanup(func() { _ = store.Close() }) + if _, err := store.db.ExecContext(ctx, `TRUNCATE environment_binding, browser_env, browser_image, social_account, credential_reference, gateway CASCADE`); err != nil { + t.Fatal(err) + } + if _, err := store.CreateGateway(ctx, "gw-seed", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil { + t.Fatal(err) + } + if err := store.CreateImage(ctx, Image{Version: "148", ImageRef: "example/browser:148", Enabled: true}); err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-seed-a', 'os_keyring', 'creatorhub/seed-a'), ('credential-seed-b', 'os_keyring', 'creatorhub/seed-b'); + INSERT INTO social_account (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) + VALUES ('seed-account-a', 'credential-seed-a', 'mock', 'seed-account-a', 'owned', 'authorized'), + ('seed-account-b', 'credential-seed-b', 'mock', 'seed-account-b', 'owned', 'authorized')`); err != nil { + t.Fatal(err) + } + env := Env{Alias: "seed-environment-a", Name: "Seed A", Gateway: "gw-seed", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 77}} + if _, created, err := store.CreateBoundEnv(ctx, env, "seed-account-a", ""); err != nil || !created { + t.Fatalf("create first seeded environment: created=%v err=%v", created, err) + } + env.Alias, env.Name = "seed-environment-b", "Seed B" + if _, _, err := store.CreateBoundEnv(ctx, env, "seed-account-b", ""); !errors.Is(err, ErrConflict) { + t.Fatalf("duplicate fingerprint seed was accepted: %v", err) + } +} + func TestHubWorkflow(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { @@ -738,16 +772,17 @@ func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { ('exit-disable-executing', 'exit-disable-executing-key', 'account-a', $1, 'exit-disable-draft', 1, 'exit-disable-confirmation', 1, 'executing', 'worker-disabled', now() + interval '1 minute')`, accountVersion); err != nil { t.Fatal(err) } - if _, err := store.DisableNetworkExit(ctx, newGeneration.Exit.ID); err != nil { + if _, err := store.DisableNetworkExit(ctx, newGeneration.Exit.ID); !errors.Is(err, ErrConflict) { + t.Fatalf("running exit must not be disabled: %v", err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE operation_task SET state='cancelled', lease_owner=NULL, lease_until=NULL WHERE id IN ('exit-disable-queued','exit-disable-executing')`); err != nil { t.Fatal(err) } - for _, want := range []taskstate.Transition{ - {State: "policy_hold", ReasonCode: "exit_unhealthy", AccountID: "account-a", TaskID: "exit-disable-queued"}, - {State: "needs_confirmation", ReasonCode: "exit_unhealthy", AccountID: "account-a", TaskID: "exit-disable-executing"}, - } { - if !slices.Contains(notifications, want) { - t.Fatalf("missing disabled exit transition %+v in %+v", want, notifications) - } + if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET released_at=now() WHERE binding_id IN (SELECT id FROM environment_binding WHERE network_exit_id=$1)`, newGeneration.Exit.ID); err != nil { + t.Fatal(err) + } + if _, err := store.DisableNetworkExit(ctx, newGeneration.Exit.ID); err != nil { + t.Fatalf("stopped exit should be disableable: %v", err) } var auditText string if err := store.db.QueryRowContext(ctx, `SELECT string_agg(row_to_json(event)::text, '') FROM audit_event event`).Scan(&auditText); err != nil { diff --git a/internal/phasea/store_test.go b/internal/phasea/store_test.go index f07660a..37b49b1 100644 --- a/internal/phasea/store_test.go +++ b/internal/phasea/store_test.go @@ -347,7 +347,16 @@ func TestPhaseAOfflineWorkflow(t *testing.T) { return } if !execution.WasClaimed { - return + var queued int + if err := store.db.QueryRowContext(ctx, `SELECT count(*) FROM operation_task WHERE state = 'queued'`).Scan(&queued); err != nil { + errorsFromWorkers <- err + return + } + if queued == 0 { + return + } + time.Sleep(5 * time.Millisecond) + continue } executed.Add(1) } diff --git a/web/src/BrowsersPage.jsx b/web/src/BrowsersPage.jsx index 9900003..262f90e 100644 --- a/web/src/BrowsersPage.jsx +++ b/web/src/BrowsersPage.jsx @@ -16,6 +16,7 @@ import { Select, StatusPill, conflictMessage, + useUnsavedChanges, } from "./lib/ui.jsx"; import { useTitle } from "./lib/hooks.js"; @@ -62,7 +63,7 @@ const createInitial = { image_version: "", account_id: "", network_exit_id: "", - seed: "1000", + seed: "", platform: "", platform_version: "", brand: "", @@ -75,6 +76,17 @@ const createInitial = { }; // 高级字段空值不提交，收敛到 seed 驱动。 +function nextFingerprintSeed(runtimes) { + const used = new Set( + runtimes + .map((runtime) => Number(runtime.fingerprint?.seed)) + .filter((seed) => Number.isInteger(seed) && seed > 0), + ); + let seed = 1000; + while (used.has(seed)) seed += 1; + return String(seed); +} + function fingerprintPayload(form) { const payload = { seed: Number(form.seed) }; const advanced = { @@ -424,9 +436,12 @@ export function BrowserCreatePage() { const [advancedOpen, setAdvancedOpen] = useState(false); const [submitBusy, setSubmitBusy] = useState(false); const [submitError, setSubmitError] = useState(""); - const { query: runtimesQuery } = useList({ resource: "browsers" }); + const { result: runtimesResult, query: runtimesQuery } = useList({ + resource: "browsers", + }); const { result: gatewayResult } = useList({ resource: "gateways" }); const gateways = gatewayResult.data ?? []; + const runtimes = runtimesResult.data ?? []; const { result: imageResult } = useList({ resource: "browser-images" }); const images = imageResult.data ?? []; const { result: accountResult } = useList({ resource: "accounts" }); @@ -467,6 +482,13 @@ export function BrowserCreatePage() { if (form.account_id === "" && pausedAccounts[0]?.id) update("account_id", pausedAccounts[0].id); }, [pausedAccounts, form.account_id]); + useEffect(() => { + if (form.seed === "" && runtimesResult.data !== undefined) + update("seed", nextFingerprintSeed(runtimes)); + }, [form.seed, runtimes, runtimesResult.data]); + useUnsavedChanges( + Boolean(form.name || form.alias || form.seed || advancedOpen), + ); const seed = Number(form.seed); const valid = diff --git a/web/src/CreatorAccountsPage.jsx b/web/src/CreatorAccountsPage.jsx index f108ae2..920b9d7 100644 --- a/web/src/CreatorAccountsPage.jsx +++ b/web/src/CreatorAccountsPage.jsx @@ -15,6 +15,7 @@ import { Textarea, dateTime, conflictMessage, + useUnsavedChanges, } from "./lib/ui.jsx"; const emptyProfile = { @@ -171,6 +172,17 @@ export function CreatorAccountsPage() { () => profiles.find((item) => item.id === selectedID), [profiles, selectedID], ); + useUnsavedChanges( + Boolean( + selected && + JSON.stringify(form) !== JSON.stringify(profileForm(selected)), + ) || + Boolean( + strategyForm.execution_account_id || + strategyForm.candidate_texts || + editingStrategyID, + ), + ); const choose = (profile) => { const dirty = selected && diff --git a/web/src/CreatorCompetitorsPage.jsx b/web/src/CreatorCompetitorsPage.jsx index 5662a81..fa584d0 100644 --- a/web/src/CreatorCompetitorsPage.jsx +++ b/web/src/CreatorCompetitorsPage.jsx @@ -14,6 +14,7 @@ import { Textarea, dateTime, conflictMessage, + useUnsavedChanges, } from "./lib/ui.jsx"; const platformOptions = [ @@ -41,6 +42,9 @@ export function CreatorCompetitorsPage() { const [minLikes, setMinLikes] = useState(""); const [minComments, setMinComments] = useState(""); const [minShares, setMinShares] = useState(""); + const [publishedAtStatus, setPublishedAtStatus] = useState(""); + const [publishedAfter, setPublishedAfter] = useState(""); + const [publishedBefore, setPublishedBefore] = useState(""); const [pending, setPending] = useState(true); const [busy, setBusy] = useState(false); const [preview, setPreview] = useState(null); @@ -53,6 +57,9 @@ export function CreatorCompetitorsPage() { const [rewriteRequirement, setRewriteRequirement] = useState(""); const [rewriteTitle, setRewriteTitle] = useState(""); const [rewriteScript, setRewriteScript] = useState(""); + const [workDetail, setWorkDetail] = useState(null); + const [workMetrics, setWorkMetrics] = useState([]); + const [workDetailPending, setWorkDetailPending] = useState(false); const load = async () => { setPending(true); @@ -66,6 +73,21 @@ export function CreatorCompetitorsPage() { filters.push({ field: "min_comments", value: Number(minComments) }); if (minShares !== "") filters.push({ field: "min_shares", value: Number(minShares) }); + if (publishedAtStatus) + filters.push({ + field: "published_at_status", + value: publishedAtStatus, + }); + if (publishedAfter) + filters.push({ + field: "published_after", + value: `${publishedAfter}T00:00:00Z`, + }); + if (publishedBefore) + filters.push({ + field: "published_before", + value: `${publishedBefore}T23:59:59Z`, + }); const [competitorResult, workResult] = await Promise.all([ dataProvider.getList({ resource: "creator-competitors", @@ -88,10 +110,27 @@ export function CreatorCompetitorsPage() { }; useEffect(() => { load(); - }, [platform, minLikes, minComments, minShares, workPage]); + }, [ + platform, + minLikes, + minComments, + minShares, + publishedAtStatus, + publishedAfter, + publishedBefore, + workPage, + ]); useEffect(() => { setWorkPage(1); - }, [platform, minLikes, minComments, minShares]); + }, [ + platform, + minLikes, + minComments, + minShares, + publishedAtStatus, + publishedAfter, + publishedBefore, + ]); useEffect(() => { dataProvider .getList({ resource: "creator-accounts" }) @@ -107,7 +146,8 @@ export function CreatorCompetitorsPage() { setPreview(null); setPreviewConfirmed(false); }; - const parseHomepage = () => { + const parseHomepage = async () => { + setBusy(true); try { const parsed = new URL(form.homepage_url); const parts = parsed.pathname.split("/").filter(Boolean); @@ -115,14 +155,12 @@ export function CreatorCompetitorsPage() { const allowed = isXiaohongshu ? parsed.protocol === "https:" && parsed.hostname === "www.xiaohongshu.com" - : parsed.hostname === "www.douyin.com"; - let candidate = parts.at(-1) || ""; - if ( - isXiaohongshu && - !(parts.length === 3 && parts[0] === "user" && parts[1] === "profile") - ) { - candidate = ""; - } + : parsed.protocol === "https:" && parsed.hostname === "www.douyin.com"; + const validAccountRoute = isXiaohongshu + ? parts.length === 3 && parts[0] === "user" && parts[1] === "profile" + : (parts.length === 2 && parts[0] === "user") || + (parts.length === 4 && parts[0] === "user" && parts[1] === "profile"); + const candidate = validAccountRoute ? parts.at(-1) || "" : ""; if ( !allowed || !candidate || @@ -132,19 +170,54 @@ export function CreatorCompetitorsPage() { "链接不是受支持的平台主页格式，请人工填写平台返回的稳定标识", ); } - const next = { ...form, platform_account_key: candidate }; - setForm(next); - setPreview(next); + if (isXiaohongshu) { + const next = { ...form, platform_account_key: candidate }; + setForm(next); + setPreview(next); + setPreviewConfirmed(false); + setNotice({ + variant: "info", + text: `已解析候选标识 ${candidate}，请核对平台返回值后确认。`, + }); + return; + } + const account = accounts.find( + (value) => + value.platform === "douyin" && + value.authorization_status === "authorized" && + (value.business_status === "normal" || + value.business_status === "muted") && + value.login_status === "logged_in", + ); + if (!account) throw new Error("没有可用于校验抖音主页的已登录账号"); + const result = await dataProvider.create({ + resource: "creator-competitor-preview", + variables: { + ...form, + platform_account_key: candidate, + account_id: account.id, + }, + }); + const verified = result.data; + setForm((value) => ({ + ...value, + platform_account_key: verified.platform_account_key, + nickname: verified.nickname || value.nickname, + homepage_url: verified.homepage_url || value.homepage_url, + })); + setPreview(verified); setPreviewConfirmed(false); setNotice({ variant: "info", - text: `已解析候选标识 ${candidate}，请核对平台返回值后确认。`, + text: `已通过抖音账号核验 ${verified.nickname || verified.platform_account_key}，请确认后加入监测。`, }); } catch (parseError) { setNotice({ variant: "warning", text: parseError.message || "主页链接解析失败", }); + } finally { + setBusy(false); } }; const create = async (event) => { @@ -158,7 +231,12 @@ export function CreatorCompetitorsPage() { try { const result = await dataProvider.create({ resource: "creator-competitors", - variables: form, + variables: { + ...form, + ...(form.platform === "douyin" && preview?.account_id + ? { account_id: preview.account_id } + : {}), + }, }); setCompetitors((items) => [result.data, ...items]); setForm((value) => ({ @@ -234,6 +312,27 @@ export function CreatorCompetitorsPage() { setBusy(false); } }; + const openWorkDetail = async (workID) => { + setWorkDetailPending(true); + setNotice(null); + try { + const [detail, metrics] = await Promise.all([ + dataProvider.creatorGet(`/creator/works/${encodeURIComponent(workID)}`), + dataProvider.creatorGet( + `/creator/works/${encodeURIComponent(workID)}/metrics`, + ), + ]); + setWorkDetail(detail); + setWorkMetrics(metrics || []); + } catch (actionError) { + setNotice({ + variant: "destructive", + text: conflictMessage(actionError, "作品详情读取失败"), + }); + } finally { + setWorkDetailPending(false); + } + }; const selectMaterial = async (workID) => { const dirty = material && @@ -275,13 +374,16 @@ export function CreatorCompetitorsPage() { `/creator/works/${encodeURIComponent(material.work_id)}/material/process`, ); setMaterial(result); + const failedStep = + result.failed_step || + ["download", "audio", "transcription"].find( + (step) => result[`${step}_status`] === "failed", + ); setNotice({ - variant: - result.transcription_status === "failed" ? "warning" : "success", - text: - result.transcription_status === "failed" - ? "素材已处理，但转写供应商尚未配置。" - : "素材处理完成。", + variant: failedStep ? "warning" : "success", + text: failedStep + ? `素材处理未完成（${failedStep}）：${result.failure_reason || "请查看步骤状态后重试。"}` + : "素材处理完成。", }); } catch (actionError) { setNotice({ @@ -338,6 +440,20 @@ export function CreatorCompetitorsPage() { setMaterialPending(false); } }; + const materialDirty = + material && + (rewriteRequirement !== (material.rewrite_requirement || "") || + rewriteTitle !== (material.generated_title || "") || + rewriteScript !== (material.generated_script || "")); + useUnsavedChanges( + Boolean( + form.platform_account_key || + form.nickname || + form.homepage_url || + materialDirty, + ), + ); + const saveRewrite = async () => { if (!material) return; setMaterialPending(true); @@ -559,6 +675,36 @@ export function CreatorCompetitorsPage() { onChange={(event) => setMinShares(event.target.value)} /> + + setPublishedAfter(event.target.value)} + /> + + + setPublishedBefore(event.target.value)} + /> + {accountError ? ( @@ -590,9 +736,14 @@ export function CreatorCompetitorsPage() { {works.map((work) => ( -

+

{work.author_name || work.source_id} ·{" "} {work.published_at_status} @@ -613,12 +764,22 @@ export function CreatorCompetitorsPage() { : work.metric_stop_reason || "—"} - +

+ + +
))} @@ -649,6 +810,80 @@ export function CreatorCompetitorsPage() { + {workDetail ? ( + + +
+
+

作品详情

+

+ {workDetail.work_key || workDetail.id} +

+
+ +
+ {workDetail.cover_url ? ( + 作品封面 + ) : null} +

+ {workDetail.body || workDetail.title || "暂无正文"} +

+
+ 来源：{workDetail.source_type || "—"} + 状态：{workDetail.published_at_status || "—"} + {workDetail.original_url ? ( + + 打开原文 + + ) : null} +
+
+ + + + + + + + + + + {workMetrics.map((point) => ( + + + + + + + ))} + +
采集时间点赞评论转发
+ {dateTime(point.collected_at)} + {point.likes ?? "—"} + {point.comments_count ?? "—"} + {point.shares ?? "—"}
+
+ + + ) : null} {material ? (
diff --git a/web/src/CreatorPages.test.jsx b/web/src/CreatorPages.test.jsx index 4d7dd9f..186d680 100644 --- a/web/src/CreatorPages.test.jsx +++ b/web/src/CreatorPages.test.jsx @@ -67,6 +67,7 @@ function provider(overrides = {}) { create: vi.fn().mockResolvedValue({ data: { id: "competitor-a", + account_id: "account-a", platform: "douyin", platform_account_key: "sec-b", homepage_url: "https://www.douyin.com/user/sec-b", @@ -120,6 +121,71 @@ describe("creator pages", () => { ); }); + it("verifies account identity and toggles big-account mode", async () => { + const dataProvider = provider({ + creatorAction: vi.fn((path) => + Promise.resolve( + path.endsWith("/verify") + ? { ...profile, login_status: "logged_in" } + : { ...profile, big_account: true }, + ), + ), + }); + renderPage(, dataProvider); + expect( + await screen.findByRole("button", { name: "开启大号模式" }), + ).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "核验浏览器身份" })); + await waitFor(() => + expect(dataProvider.creatorAction).toHaveBeenCalledWith( + "/creator/accounts/account-a/verify", + ), + ); + fireEvent.click(screen.getByRole("button", { name: "开启大号模式" })); + await waitFor(() => + expect(dataProvider.creatorAction).toHaveBeenCalledWith( + "/creator/accounts/account-a/big-account", + { enabled: true }, + ), + ); + expect(await screen.findByText("已开启大号模式。")).toBeTruthy(); + }); + + it("shows retryable account and strategy loading failures", async () => { + const dataProvider = provider({ + getList: vi.fn().mockRejectedValue(new Error("accounts offline")), + creatorGet: vi.fn().mockRejectedValue(new Error("creator offline")), + }); + renderPage(, dataProvider); + expect(await screen.findByText("accounts offline")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "重试" })); + await waitFor(() => expect(dataProvider.getList).toHaveBeenCalledTimes(2)); + }); + + it("shows settings save conflicts", async () => { + const dataProvider = provider({ + creatorUpdate: vi.fn().mockRejectedValue({ + status: 409, + body: { reason: "version changed" }, + }), + }); + renderPage(, dataProvider); + expect(await screen.findByText("采集设置")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "保存设置" })); + expect(await screen.findByText("冲突（409）：设置保存失败")).toBeTruthy(); + }); + + it("saves settings successfully", async () => { + const dataProvider = provider({ + creatorUpdate: vi.fn().mockResolvedValue({}), + }); + renderPage(, dataProvider); + expect(await screen.findByText("采集设置")).toBeTruthy(); + fireEvent.click(screen.getByLabelText("已完成 AI 配置审批")); + fireEvent.click(screen.getByRole("button", { name: "保存设置" })); + expect(await screen.findByText("采集与 AI 配置已保存。")).toBeTruthy(); + }); + it("does not render stale data using another tab's shape", async () => { const dataProvider = provider({ getList: vi.fn(({ resource }) => @@ -183,21 +249,38 @@ describe("creator pages", () => { it("requires preview confirmation before importing a competitor", async () => { const dataProvider = provider(); renderPage(, dataProvider); + await waitFor(() => + expect(dataProvider.getList).toHaveBeenCalledWith( + expect.objectContaining({ resource: "creator-accounts" }), + ), + ); fireEvent.change(screen.getByLabelText("主页 URL", { exact: false }), { target: { value: "https://www.douyin.com/user/sec-b" }, }); fireEvent.click(screen.getByRole("button", { name: "解析链接预览" })); expect(await screen.findByText("导入预览")).toBeTruthy(); - expect(dataProvider.create).not.toHaveBeenCalled(); + await waitFor(() => + expect(dataProvider.create).toHaveBeenCalledWith( + expect.objectContaining({ + resource: "creator-competitor-preview", + variables: expect.objectContaining({ + account_id: "account-a", + platform: "douyin", + platform_account_key: "sec-b", + }), + }), + ), + ); fireEvent.click(screen.getByRole("button", { name: "确认预览内容" })); fireEvent.click(screen.getByRole("button", { name: "加入监测" })); - await waitFor(() => expect(dataProvider.create).toHaveBeenCalled()); - expect(dataProvider.create).toHaveBeenCalledWith( + await waitFor(() => expect(dataProvider.create).toHaveBeenCalledTimes(2)); + expect(dataProvider.create).toHaveBeenLastCalledWith( expect.objectContaining({ resource: "creator-competitors", variables: expect.objectContaining({ platform: "douyin", platform_account_key: "sec-b", + account_id: "account-a", }), }), ); diff --git a/web/src/CreatorSettingsPage.jsx b/web/src/CreatorSettingsPage.jsx index f7ccbbb..9886283 100644 --- a/web/src/CreatorSettingsPage.jsx +++ b/web/src/CreatorSettingsPage.jsx @@ -10,6 +10,7 @@ import { PageHeader, PageState, conflictMessage, + useUnsavedChanges, } from "./lib/ui.jsx"; const editableFields = [ @@ -48,6 +49,7 @@ const defaults = { export function CreatorSettingsPage() { const dataProvider = useDataProvider()("default"); const [form, setForm] = useState(defaults); + const [savedForm, setSavedForm] = useState(defaults); const [pending, setPending] = useState(true); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); @@ -59,6 +61,7 @@ export function CreatorSettingsPage() { try { const result = await dataProvider.creatorGet("/creator/settings"); setForm((value) => ({ ...value, ...editableSettings(result) })); + setSavedForm((value) => ({ ...value, ...editableSettings(result) })); } catch (loadError) { setError(loadError); } finally { @@ -96,6 +99,7 @@ export function CreatorSettingsPage() { editableSettings(form), ); setForm((value) => ({ ...value, ...editableSettings(result) })); + setSavedForm((value) => ({ ...value, ...editableSettings(result) })); setNotice({ variant: "success", text: "采集与 AI 配置已保存。" }); } catch (saveError) { setNotice({ @@ -107,6 +111,11 @@ export function CreatorSettingsPage() { } }; + useUnsavedChanges( + JSON.stringify(editableSettings(form)) !== + JSON.stringify(editableSettings(savedForm)), + ); + return ( <> item.trim()); + const values = value + .split(",") + .map((item) => item.replace(/^[ \t\r\n]+|[ \t\r\n]+$/g, "")); if (values.some((item) => !item)) throw new Error("关键词不能为空"); return values; } @@ -143,6 +146,7 @@ export function CreatorWorkbenchPage() { ai_requirement: "", }); const [editingRuleID, setEditingRuleID] = useState(""); + const [replyMode, setReplyMode] = useState("comment"); const [reply, setReply] = useState({ account_id: "", text: "", @@ -158,6 +162,11 @@ export function CreatorWorkbenchPage() { const visibleData = dataTab === tab ? data : []; const visibleConversations = conversationAccountID === dmAccountID ? conversations : []; + const replyTarget = + visibleData.find((item) => item.id === reply.target_comment_id) || + visibleData.find((item) => item.comment?.id === reply.target_comment_id) + ?.comment; + useUnsavedChanges(Boolean(reply.text.trim())); const load = async () => { const sequence = ++loadSequence.current; @@ -494,7 +503,8 @@ export function CreatorWorkbenchPage() { setBusy(false); } }; - const requestReply = (comment) => { + const requestReply = (comment, mode = "comment") => { + setReplyMode(mode); const draft = readDraft(replyDraftKey(comment.id)); setReply({ account_id: draft?.account_id || "", @@ -505,7 +515,10 @@ export function CreatorWorkbenchPage() { }); setNotice({ variant: "info", - text: "请填写发送账号和文案，再进行逐次确认。", + text: + mode === "dm" + ? "请填写发送账号和私信文案，再进行逐次确认。" + : "请填写发送账号和文案，再进行逐次确认。", }); }; const askReplyConfirm = (comment) => { @@ -539,11 +552,13 @@ export function CreatorWorkbenchPage() { const created = await dataProvider.creatorCreate("/creator/operations", { idempotency_key: reply.operation_key, source: "manual", - action: "reply_comment", + action: replyMode === "dm" ? "dm" : "reply_comment", platform: confirm.comment.platform, account_id: reply.account_id, target_uid: reply.target_uid, - target_comment_id: reply.target_comment_id, + ...(replyMode === "comment" + ? { target_comment_id: reply.target_comment_id } + : {}), text: reply.text, }); const result = await dataProvider.creatorAction( @@ -654,7 +669,7 @@ export function CreatorWorkbenchPage() { emptyText="暂无一级评论。" onRetry={load} > - {reply.target_comment_id ? ( + {reply.target_comment_id && replyTarget ? ( @@ -670,11 +685,7 @@ export function CreatorWorkbenchPage() { } options={accounts .filter( - (account) => - account.platform === - visibleData.find( - (item) => item.id === reply.target_comment_id, - )?.platform, + (account) => account.platform === replyTarget.platform, ) .map((account) => ({ value: account.id, @@ -683,7 +694,11 @@ export function CreatorWorkbenchPage() { placeholder="选择同平台账号" /> - + @@ -773,6 +782,13 @@ export function CreatorWorkbenchPage() { +
+ setReply((value) => ({ + ...value, + account_id: event.target.value, + operation_key: "", + })) + } + options={accounts + .filter( + (account) => account.platform === replyTarget.platform, + ) + .map((account) => ({ + value: account.id, + label: account.name || account.id, + }))} + placeholder="选择同平台账号" + /> + + + + setReply((value) => ({ + ...value, + text: event.target.value, + operation_key: "", + })) + } + /> + +
+ +
+

+ 目标 UID：{reply.target_uid || "不可用，无法发送"} +

+ + + ) : null}
{visibleData.map((lead) => ( @@ -825,9 +897,16 @@ export function CreatorWorkbenchPage() { +
@@ -835,6 +914,21 @@ export function CreatorWorkbenchPage() {

规则：{lead.rule_ids.join("、")}

+
+ {lead.results?.length ? ( + lead.results.map((result) => ( +
+ {result.status} + {result.reason ? `：${result.reason}` : ""} + {result.matched_keywords?.length + ? `（关键词：${result.matched_keywords.join("、")}）` + : ""} +
+ )) + ) : ( + 尚未分析 + )} +
))} diff --git a/web/src/NetworkExitsPage.jsx b/web/src/NetworkExitsPage.jsx index 729a96b..7923100 100644 --- a/web/src/NetworkExitsPage.jsx +++ b/web/src/NetworkExitsPage.jsx @@ -1,4 +1,4 @@ -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Link, useParams } from "react-router"; import { useDataProvider, useList, useOne } from "@refinedev/core"; import { @@ -59,8 +59,8 @@ function ExitHealthPill({ exit }) { ); } -function ExitActions({ exit, busy, onAction }) { - const disabled = busy || exit.health_status === "disabled"; +function ExitActions({ exit, busy, onAction, onEdit, onDelete }) { + const disabled = busy; return (
+ {exit.health_status === "disabled" ? ( + + ) : ( + + )} +
); } -function ExitCreateModal({ open, onClose, onSubmit, busy, error }) { - const [form, setForm] = useState(createInitial); +function ExitCreateModal({ + open, + onClose, + onSubmit, + busy, + error, + initialValue = createInitial, + title = "创建网络出口", + submitLabel = "创建网络出口", +}) { + const [form, setForm] = useState({ ...createInitial, ...initialValue }); + useEffect(() => { + if (open) + setForm({ + ...createInitial, + ...initialValue, + port: initialValue.port ? String(initialValue.port) : "", + }); + }, [open, initialValue]); const update = (key, value) => setForm((current) => ({ ...current, [key]: value })); const port = Number(form.port); @@ -119,7 +163,7 @@ function ExitCreateModal({ open, onClose, onSubmit, busy, error }) { - 创建网络出口 + {submitLabel} } @@ -240,7 +284,15 @@ function ExitCreateModal({ open, onClose, onSubmit, busy, error }) { ); } -function ExitTable({ exits, accountsByExit, bindingsError, busy, onAction }) { +function ExitTable({ + exits, + accountsByExit, + bindingsError, + busy, + onAction, + onEdit, + onDelete, +}) { const columns = [ { header: "出口", @@ -313,7 +365,13 @@ function ExitTable({ exits, accountsByExit, bindingsError, busy, onAction }) { width: "8%", align: "right", render: (exit) => ( - + ), }, ]; @@ -334,6 +392,8 @@ export function NetworkExitList() { const [createError, setCreateError] = useState(null); const [notice, setNotice] = useState(null); const [disabling, setDisabling] = useState(null); + const [editing, setEditing] = useState(null); + const [deleting, setDeleting] = useState(null); const { result, query } = useList({ resource: "network-exits" }); const error = query.error; const isPending = query.isPending; @@ -377,6 +437,51 @@ export function NetworkExitList() { } } + async function updateExit(exit, data) { + setBusy(`edit:${exit.id}`); + setNotice(null); + try { + await dataProvider.update({ + resource: "network-exits", + id: exit.id, + variables: data, + }); + await query.refetch(); + setEditing(null); + setNotice({ + variant: "success", + text: "网络出口已更新，需重新检测后才能使用。", + }); + return true; + } catch (reason) { + setNotice({ + variant: "destructive", + text: conflictMessage(reason, "出口当前不能修改。"), + }); + return false; + } finally { + setBusy(""); + } + } + + async function deleteExit(exit) { + setBusy(`delete:${exit.id}`); + setNotice(null); + try { + await dataProvider.delete({ resource: "network-exits", id: exit.id }); + await query.refetch(); + setDeleting(null); + setNotice({ variant: "success", text: "网络出口已删除。" }); + } catch (reason) { + setNotice({ + variant: "destructive", + text: conflictMessage(reason, "出口仍有绑定或运行实例，不能删除。"), + }); + } finally { + setBusy(""); + } + } + async function runAction(exit, action) { setBusy(exit.id); setNotice(null); @@ -388,7 +493,9 @@ export function NetworkExitList() { text: action === "check" ? "出口检测完成。" - : "出口已停用；绑定账号不会自动恢复。", + : action === "enable" + ? "出口已启用；重新检测通过后才能绑定。" + : "出口已停用；绑定账号不会自动恢复。", }); } catch (reason) { setNotice({ @@ -451,6 +558,8 @@ export function NetworkExitList() { onAction={(exit, action) => action === "disable" ? setDisabling(exit) : runAction(exit, action) } + onEdit={(exit) => setEditing(exit)} + onDelete={(exit) => setDeleting(exit)} /> + setEditing(null)} + onSubmit={(data) => updateExit(editing, data)} + busy={busy === `edit:${editing?.id}`} + error={null} + initialValue={editing || createInitial} + title="编辑网络出口" + submitLabel="保存网络出口" + /> + setDeleting(null)} + onConfirm={() => deleteExit(deleting)} + title="删除网络出口" + confirmLabel="确认删除" + busy={busy === `delete:${deleting?.id}`} + body={`删除网络出口 ${deleting?.id}？有绑定或运行实例时会拒绝。`} + /> setDisabling(null)} diff --git a/web/src/dataProvider.js b/web/src/dataProvider.js index baba8dc..3d4f5c4 100644 --- a/web/src/dataProvider.js +++ b/web/src/dataProvider.js @@ -57,6 +57,7 @@ const resourcePaths = { "network-exits": "/network-exits", "creator-accounts": "/creator/accounts", "creator-competitors": "/creator/competitors", + "creator-competitor-preview": "/creator/competitors/preview", "creator-works": "/creator/works", "creator-comments": "/creator/comments", "creator-leads": "/creator/leads", @@ -107,10 +108,7 @@ export const dataProvider = { data: data.map((record, index) => withID(record, index)), total: records.total ?? data.length, }; - if ( - !Array.isArray(records) && - Object.prototype.hasOwnProperty.call(records, "has_next") - ) { + if (!Array.isArray(records) && Object.hasOwn(records, "has_next")) { result.hasNext = Boolean(records.has_next); } return result; @@ -230,7 +228,7 @@ export const dataProvider = { ); }, async networkExitAction(id, action) { - if (action !== "check" && action !== "disable") + if (!["check", "disable", "enable"].includes(action)) throw new Error(`未知网络出口操作: ${action}`); return request(`/network-exits/${encodeURIComponent(id)}/${action}`, { method: "POST", @@ -275,17 +273,23 @@ export const dataProvider = { onStatus?.("connected"); const decoder = new TextDecoder(); let buffer = ""; + const dispatch = (frame) => { + if (frame.split(/\r\n|\n|\r/).some((line) => line.startsWith("data:"))) + onMessage(frame); + }; while (true) { const { value, done } = await reader.read(); buffer += decoder.decode(value || new Uint8Array(), { stream: !done }); - let boundary; - while ((boundary = buffer.indexOf("\\n\\n")) >= 0) { - const frame = buffer.slice(0, boundary); - buffer = buffer.slice(boundary + 2); - if (frame.split(/\\r?\\n/).some((line) => line.startsWith("data:"))) - onMessage(frame); + let match; + while ((match = buffer.match(/\r\n\r\n|\n\n|\r\r/))) { + dispatch(buffer.slice(0, match.index)); + buffer = buffer.slice(match.index + match[0].length); + } + if (done) { + if (buffer.trim()) dispatch(buffer); + onStatus?.("disconnected"); + return; } - if (done) return; } }, creatorUpdate(path, data) { diff --git a/web/src/dataProvider.test.js b/web/src/dataProvider.test.js index a898ced..4ad4296 100644 --- a/web/src/dataProvider.test.js +++ b/web/src/dataProvider.test.js @@ -1,173 +1,339 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' -import { dataProvider } from './dataProvider' +import { afterEach, describe, expect, it, vi } from "vitest"; +import { dataProvider } from "./dataProvider"; -afterEach(() => vi.unstubAllGlobals()) +afterEach(() => vi.unstubAllGlobals()); -const httpError = (message, status, body) => Object.assign(new Error(message), { status, body }) +describe("dataProvider", () => { + it("maps the existing browser list into refine records", async () => { + const fetch = vi + .fn() + .mockResolvedValue( + new Response( + JSON.stringify([{ name: "account-a", state: "running" }]), + { status: 200 }, + ), + ); + vi.stubGlobal("fetch", fetch); -describe('dataProvider', () => { - it('maps the existing browser list into refine records', async () => { - const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify([{ name: 'account-a', state: 'running' }]), { status: 200 })) - vi.stubGlobal('fetch', fetch) - - await expect(dataProvider.getList({ resource: 'browsers' })).resolves.toEqual({ - data: [{ id: 'account-a', name: 'account-a', state: 'running' }], + await expect( + dataProvider.getList({ resource: "browsers" }), + ).resolves.toEqual({ + data: [{ id: "account-a", name: "account-a", state: "running" }], total: 1, - }) - expect(fetch).toHaveBeenCalledWith('/api/browsers', { headers: {} }) - }) + }); + expect(fetch).toHaveBeenCalledWith("/api/browsers", { headers: {} }); + }); - it('attaches Basic credentials from localStorage', async () => { - localStorage.setItem('creatorhub.auth', 'user:pass') - const fetch = vi.fn().mockResolvedValue(new Response('[]', { status: 200 })) - vi.stubGlobal('fetch', fetch) + it("attaches Basic credentials from localStorage", async () => { + localStorage.setItem("creatorhub.auth", "user:pass"); + const fetch = vi + .fn() + .mockResolvedValue(new Response("[]", { status: 200 })); + vi.stubGlobal("fetch", fetch); - await dataProvider.getList({ resource: 'browsers' }) - expect(fetch).toHaveBeenCalledWith('/api/browsers', { headers: { Authorization: 'Basic dXNlcjpwYXNz' } }) - localStorage.removeItem('creatorhub.auth') - }) + await dataProvider.getList({ resource: "browsers" }); + expect(fetch).toHaveBeenCalledWith("/api/browsers", { + headers: { Authorization: "Basic dXNlcjpwYXNz" }, + }); + localStorage.removeItem("creatorhub.auth"); + }); - it('clears credentials and redirects to login on 401', async () => { - localStorage.setItem('creatorhub.auth', 'user:pass') - const fetch = vi.fn().mockResolvedValue(new Response('', { status: 401 })) - vi.stubGlobal('fetch', fetch) + it("clears credentials and redirects to login on 401", async () => { + localStorage.setItem("creatorhub.auth", "user:pass"); + const fetch = vi.fn().mockResolvedValue(new Response("", { status: 401 })); + vi.stubGlobal("fetch", fetch); - await expect(dataProvider.getList({ resource: 'browsers' })).rejects.toMatchObject({ status: 401 }) - expect(localStorage.getItem('creatorhub.auth')).toBeNull() - expect(location.hash).toBe('#/login') - }) + await expect( + dataProvider.getList({ resource: "browsers" }), + ).rejects.toMatchObject({ status: 401 }); + expect(localStorage.getItem("creatorhub.auth")).toBeNull(); + expect(location.hash).toBe("#/login"); + }); it.each([ - ['accounts', '/api/phase-a/accounts'], - ['network-exits', '/api/network-exits'], - ])('loads %s for the environment create contract', async (resource, path) => { - const fetch = vi.fn().mockResolvedValue(new Response('[{"id":"record-1"}]', { status: 200 })) - vi.stubGlobal('fetch', fetch) + ["accounts", "/api/phase-a/accounts"], + ["network-exits", "/api/network-exits"], + ])("loads %s for the environment create contract", async (resource, path) => { + const fetch = vi + .fn() + .mockResolvedValue(new Response('[{"id":"record-1"}]', { status: 200 })); + vi.stubGlobal("fetch", fetch); - await expect(dataProvider.getList({ resource })).resolves.toMatchObject({ data: [{ id: 'record-1' }] }) - expect(fetch).toHaveBeenCalledWith(path, { headers: {} }) - }) + await expect(dataProvider.getList({ resource })).resolves.toMatchObject({ + data: [{ id: "record-1" }], + }); + expect(fetch).toHaveBeenCalledWith(path, { headers: {} }); + }); it.each([ - ['accounts', 'account-a', '/api/phase-a/accounts/account-a'], - ['network-exits', 'exit/one', '/api/network-exits/exit%2Fone'], - ['browsers', 'environment-one', '/api/browsers/environment-one'], - ['drafts', 'draft/one', '/api/phase-a/drafts/draft%2Fone'], - ['confirmations', 'confirmation/one', '/api/phase-a/confirmations/confirmation%2Fone'], - ['tasks', 'task/one', '/api/phase-a/tasks/task%2Fone'], - ['attempts', 'attempt/one', '/api/phase-a/attempts/attempt%2Fone'], - ])('loads %s detail through its stable API path', async (resource, id, path) => { - const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({ id }), { status: 200 })) - vi.stubGlobal('fetch', fetch) + ["accounts", "account-a", "/api/phase-a/accounts/account-a"], + ["network-exits", "exit/one", "/api/network-exits/exit%2Fone"], + ["browsers", "environment-one", "/api/browsers/environment-one"], + ["drafts", "draft/one", "/api/phase-a/drafts/draft%2Fone"], + [ + "confirmations", + "confirmation/one", + "/api/phase-a/confirmations/confirmation%2Fone", + ], + ["tasks", "task/one", "/api/phase-a/tasks/task%2Fone"], + ["attempts", "attempt/one", "/api/phase-a/attempts/attempt%2Fone"], + ])("loads %s detail through its stable API path", async (resource, id, path) => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(JSON.stringify({ id }), { status: 200 })); + vi.stubGlobal("fetch", fetch); - await expect(dataProvider.getOne({ resource, id })).resolves.toEqual({ data: { id } }) - expect(fetch).toHaveBeenCalledWith(path, { headers: {} }) - }) + await expect(dataProvider.getOne({ resource, id })).resolves.toEqual({ + data: { id }, + }); + expect(fetch).toHaveBeenCalledWith(path, { headers: {} }); + }); - it('creates an account without a client-generated technical id', async () => { - const fetch = vi.fn().mockResolvedValue(new Response('{"id":"account-generated"}', { status: 201 })) - vi.stubGlobal('fetch', fetch) - const data = { name: '店铺一号', platform: 'douyin', platform_account_key: 'shop-a', tags: ['主账号'], cookies: 'sessionid=value' } + it("creates an account without a client-generated technical id", async () => { + const fetch = vi + .fn() + .mockResolvedValue( + new Response('{"id":"account-generated"}', { status: 201 }), + ); + vi.stubGlobal("fetch", fetch); + const data = { + name: "店铺一号", + platform: "douyin", + platform_account_key: "shop-a", + tags: ["主账号"], + cookies: "sessionid=value", + }; - await expect(dataProvider.create({ resource: 'accounts', variables: data })).resolves.toEqual({ data: { ...data, id: 'account-generated' } }) - expect(fetch).toHaveBeenCalledWith('/api/phase-a/accounts', expect.objectContaining({ method: 'POST' })) - expect(JSON.parse(fetch.mock.calls[0][1].body)).toEqual(data) - }) + await expect( + dataProvider.create({ resource: "accounts", variables: data }), + ).resolves.toEqual({ data: { ...data, id: "account-generated" } }); + expect(fetch).toHaveBeenCalledWith( + "/api/phase-a/accounts", + expect.objectContaining({ method: "POST" }), + ); + expect(JSON.parse(fetch.mock.calls[0][1].body)).toEqual(data); + }); - it('filters drafts by account through the server list contract', async () => { - const fetch = vi.fn().mockResolvedValue(new Response('[]', { status: 200 })) - vi.stubGlobal('fetch', fetch) + it("filters drafts by account through the server list contract", async () => { + const fetch = vi + .fn() + .mockResolvedValue(new Response("[]", { status: 200 })); + vi.stubGlobal("fetch", fetch); - await dataProvider.getList({ resource: 'drafts', filters: [{ field: 'account_id', value: 'account-a' }] }) - expect(fetch).toHaveBeenCalledWith('/api/phase-a/drafts?account_id=account-a', { headers: {} }) - }) + await dataProvider.getList({ + resource: "drafts", + filters: [{ field: "account_id", value: "account-a" }], + }); + expect(fetch).toHaveBeenCalledWith( + "/api/phase-a/drafts?account_id=account-a", + { headers: {} }, + ); + }); - it('maps paginated audit filters without exposing a CRUD mutation', async () => { - const fetch = vi.fn().mockResolvedValue(new Response('{"data":[{"id":7,"event_type":"task_verified"}],"total":31}', { status: 200 })) - vi.stubGlobal('fetch', fetch) + it("maps paginated audit filters without exposing a CRUD mutation", async () => { + const fetch = vi + .fn() + .mockResolvedValue( + new Response( + '{"data":[{"id":7,"event_type":"task_verified"}],"total":31}', + { status: 200 }, + ), + ); + vi.stubGlobal("fetch", fetch); - await expect(dataProvider.getList({ resource: 'audit', - pagination: { currentPage: 2, pageSize: 25 }, - filters: [ - { field: 'task_id', value: 'task-a' }, { field: 'attempt_id', value: 'attempt-a' }, - { field: 'browser_env_alias', value: 'env-a' }, { field: 'network_exit_id', value: 'exit-a' }, + await expect( + dataProvider.getList({ + resource: "audit", + pagination: { currentPage: 2, pageSize: 25 }, + filters: [ + { field: "task_id", value: "task-a" }, + { field: "attempt_id", value: "attempt-a" }, + { field: "browser_env_alias", value: "env-a" }, + { field: "network_exit_id", value: "exit-a" }, + ], + }), + ).resolves.toEqual({ + data: [{ id: 7, event_type: "task_verified" }], + total: 31, + }); + expect(fetch).toHaveBeenCalledWith( + "/api/phase-a/audit?task_id=task-a&attempt_id=attempt-a&browser_env_alias=env-a&network_exit_id=exit-a&page=2&page_size=25", + { headers: {} }, + ); + }); + + it("keeps generated draft, confirmation and enqueue identifiers out of user input", async () => { + const fetch = vi + .fn() + .mockImplementation(() => + Promise.resolve(new Response('{"id":"generated"}', { status: 201 })), + ); + vi.stubGlobal("fetch", fetch); + + await dataProvider.createDraft("account-a", "hello"); + await dataProvider.confirmDraft("draft-a", 2, 3); + await dataProvider.enqueueConfirmation("confirmation-a"); + + expect( + fetch.mock.calls.map(([path, options]) => [ + path, + JSON.parse(options.body), + ]), + ).toEqual([ + ["/api/phase-a/drafts", { account_id: "account-a", content: "hello" }], + [ + "/api/phase-a/confirmations", + { draft_id: "draft-a", account_version: 2, draft_version: 3 }, ], - })).resolves.toEqual({ - data: [{ id: 7, event_type: 'task_verified' }], total: 31, - }) - expect(fetch).toHaveBeenCalledWith('/api/phase-a/audit?task_id=task-a&attempt_id=attempt-a&browser_env_alias=env-a&network_exit_id=exit-a&page=2&page_size=25', { headers: {} }) - }) - - it('keeps generated draft, confirmation and enqueue identifiers out of user input', async () => { - const fetch = vi.fn().mockImplementation(() => Promise.resolve(new Response('{"id":"generated"}', { status: 201 }))) - vi.stubGlobal('fetch', fetch) - - await dataProvider.createDraft('account-a', 'hello') - await dataProvider.confirmDraft('draft-a', 2, 3) - await dataProvider.enqueueConfirmation('confirmation-a') - - expect(fetch.mock.calls.map(([path, options]) => [path, JSON.parse(options.body)])).toEqual([ - ['/api/phase-a/drafts', { account_id: 'account-a', content: 'hello' }], - ['/api/phase-a/confirmations', { draft_id: 'draft-a', account_version: 2, draft_version: 3 }], - ['/api/phase-a/tasks', { confirmation_id: 'confirmation-a' }], - ]) - }) + ["/api/phase-a/tasks", { confirmation_id: "confirmation-a" }], + ]); + }); it.each([ - ['start', '/api/browsers/account-a/start', 'POST'], - ['stop', '/api/browsers/account-a/stop', 'POST'], - ['recycle', '/api/browsers/account-a', 'DELETE'], - ])('keeps %s as an explicit domain action', async (action, path, method) => { - const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 204 })) - vi.stubGlobal('fetch', fetch) + ["start", "/api/browsers/account-a/start", "POST"], + ["stop", "/api/browsers/account-a/stop", "POST"], + ["recycle", "/api/browsers/account-a", "DELETE"], + ])("keeps %s as an explicit domain action", async (action, path, method) => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetch); - await dataProvider.browserAction('account-a', action) - expect(fetch).toHaveBeenCalledWith(path, expect.objectContaining({ method })) - }) + await dataProvider.browserAction("account-a", action); + expect(fetch).toHaveBeenCalledWith( + path, + expect.objectContaining({ method }), + ); + }); - it('sends upgrade with the version payload', async () => { - const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 204 })) - vi.stubGlobal('fetch', fetch) + it("sends upgrade with the version payload", async () => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetch); - await dataProvider.browserAction('account-a', 'upgrade', { version: '144.0.0.1' }) - expect(fetch).toHaveBeenCalledWith('/api/browsers/account-a/upgrade', expect.objectContaining({ method: 'POST', body: '{"version":"144.0.0.1"}' })) - }) + await dataProvider.browserAction("account-a", "upgrade", { + version: "144.0.0.1", + }); + expect(fetch).toHaveBeenCalledWith( + "/api/browsers/account-a/upgrade", + expect.objectContaining({ + method: "POST", + body: '{"version":"144.0.0.1"}', + }), + ); + }); - it('preserves API error messages and status codes', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({ error: '网关不可用' }), { status: 502 }))) + it("preserves API error messages and status codes", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ error: "网关不可用" }), { + status: 502, + }), + ), + ); - await expect(dataProvider.getList({ resource: 'browsers' })).rejects.toMatchObject({ message: '网关不可用', status: 502 }) - }) - - it.each([409, 503])('preserves structured action errors for status %s', async status => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({ error: 'blocked', reason_code: 'network_exit_unhealthy' }), { status }))) - - await expect(dataProvider.accountAction('account-a', 'resume')).rejects.toMatchObject({ - message: 'blocked', status, body: { error: 'blocked', reason_code: 'network_exit_unhealthy' }, - }) - }) + await expect( + dataProvider.getList({ resource: "browsers" }), + ).rejects.toMatchObject({ message: "网关不可用", status: 502 }); + }); it.each([ - ['pause', '/api/phase-a/accounts/account-a/pause'], - ['resume', '/api/phase-a/accounts/account-a/resume'], - ])('sends explicit account action %s', async (action, path) => { - const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 204 })) - vi.stubGlobal('fetch', fetch) + 409, 503, + ])("preserves structured action errors for status %s", async (status) => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ + error: "blocked", + reason_code: "network_exit_unhealthy", + }), + { status }, + ), + ), + ); - await dataProvider.accountAction('account-a', action) - expect(fetch).toHaveBeenCalledWith(path, { method: 'POST', headers: {} }) - }) + await expect( + dataProvider.accountAction("account-a", "resume"), + ).rejects.toMatchObject({ + message: "blocked", + status, + body: { error: "blocked", reason_code: "network_exit_unhealthy" }, + }); + }); - it('keeps verification and recovery as separate task actions', async () => { - const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 204 })) - vi.stubGlobal('fetch', fetch) + it.each([ + ["pause", "/api/phase-a/accounts/account-a/pause"], + ["resume", "/api/phase-a/accounts/account-a/resume"], + ])("sends explicit account action %s", async (action, path) => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetch); - await dataProvider.taskAction('task/a', 'verify', { result: 'not_executed' }) - await dataProvider.taskAction('task/a', 'resume') + await dataProvider.accountAction("account-a", action); + expect(fetch).toHaveBeenCalledWith(path, { method: "POST", headers: {} }); + }); + + it("keeps verification and recovery as separate task actions", async () => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetch); + + await dataProvider.taskAction("task/a", "verify", { + result: "not_executed", + }); + await dataProvider.taskAction("task/a", "resume"); expect(fetch.mock.calls).toEqual([ - ['/api/phase-a/tasks/task%2Fa/verify', expect.objectContaining({ method: 'POST', body: '{"result":"not_executed"}' })], - ['/api/phase-a/tasks/task%2Fa/resume', { method: 'POST', headers: {} }], - ]) - }) -}) + [ + "/api/phase-a/tasks/task%2Fa/verify", + expect.objectContaining({ + method: "POST", + body: '{"result":"not_executed"}', + }), + ], + ["/api/phase-a/tasks/task%2Fa/resume", { method: "POST", headers: {} }], + ]); + }); +}); + +describe("creatorSubscribe SSE framing", () => { + it("keeps split UTF-8 messages and accepts CRLF framing", async () => { + const encoder = new TextEncoder(); + const chunks = [ + 'data: {"text":"北', + '京"}\r\n\r', + '\ndata: {"text":"ok"}\n\n', + ]; + const body = new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(encoder.encode(chunk)); + controller.close(); + }, + }); + vi.stubGlobal( + "fetch", + vi.fn(() => Promise.resolve({ status: 200, ok: true, body })), + ); + const messages = []; + const statuses = []; + + await dataProvider.creatorSubscribe( + "/creator/updates", + (frame) => messages.push(frame), + new AbortController().signal, + (status) => statuses.push(status), + ); + + expect(messages).toEqual(['data: {"text":"北京"}', 'data: {"text":"ok"}']); + expect(statuses).toEqual(["connected", "disconnected"]); + }); +}); diff --git a/web/src/lib/hooks.test.jsx b/web/src/lib/hooks.test.jsx new file mode 100644 index 0000000..aae717a --- /dev/null +++ b/web/src/lib/hooks.test.jsx @@ -0,0 +1,41 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { render } from "@testing-library/react"; +import { conflictError, displayError, useTitle } from "./hooks"; + +describe("hooks helpers", () => { + afterEach(() => { + document.title = ""; + }); + + it("returns backend messages and fallback text", () => { + expect(displayError({ message: "backend failed" })).toBe("backend failed"); + expect(displayError({})).toBe("操作失败"); + }); + + it("formats conflict and unavailable errors", () => { + expect( + conflictError( + { status: 409, body: { reason: "version changed" } }, + "retry", + ), + ).toBe("冲突（409）：version changed"); + expect(conflictError({ status: 503, message: "offline" }, "retry")).toBe( + "资源未就绪（503）：offline", + ); + expect(conflictError({ status: 400, message: "bad input" }, "retry")).toBe( + "bad input", + ); + expect(conflictError({ status: 409, body: {} }, "retry")).toBe( + "冲突（409）：retry", + ); + }); + + it("updates the document title", () => { + function TitleProbe() { + useTitle("CreatorHub 测试"); + return null; + } + render(); + expect(document.title).toBe("CreatorHub 测试"); + }); +}); diff --git a/web/src/lib/ui.jsx b/web/src/lib/ui.jsx index e0c3462..c1075ed 100644 --- a/web/src/lib/ui.jsx +++ b/web/src/lib/ui.jsx @@ -20,6 +20,29 @@ export function conflictMessage(error, fallback) { export const dateTime = (value) => new Date(value).toLocaleString("zh-CN"); +export function useUnsavedChanges(dirty) { + useEffect(() => { + if (!dirty) return undefined; + const beforeUnload = (event) => { + event.preventDefault(); + event.returnValue = ""; + }; + const click = (event) => { + const anchor = event.target.closest?.("a[href]"); + if (!anchor || !anchor.getAttribute("href")?.startsWith("#/")) return; + if (window.confirm("当前内容尚未保存，确定离开吗？")) return; + event.preventDefault(); + event.stopPropagation(); + }; + window.addEventListener("beforeunload", beforeUnload); + document.addEventListener("click", click, true); + return () => { + window.removeEventListener("beforeunload", beforeUnload); + document.removeEventListener("click", click, true); + }; + }, [dirty]); +} + /** * 页面加载/错误/空态的统一外壳。 */ @@ -186,7 +209,6 @@ export function Field({ className, }) { const helpId = helper || error ? `${id}-help` : undefined; - const describedBy = error ? `${id}-error` : helpId; const control = children; return (
diff --git a/web/vite.config.js b/web/vite.config.js index ed9f4cc..f73ac52 100644 --- a/web/vite.config.js +++ b/web/vite.config.js @@ -17,7 +17,7 @@ export default defineConfig({ isolate: false, // ponytail: one worker avoids 90s startup stalls; restore isolation if tests leak state. coverage: { provider: "v8", - reporter: ["text"], + reporter: ["text", "json-summary", "json"], thresholds: { lines: 65 }, }, },