diff --git a/AGENTS.md b/AGENTS.md index e8d6c2b..b52a4dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,12 @@ - 永不提交密钥、生产凭据或个人账号数据。 - 引用上游项目时，记录其来源与许可证；除非许可证明确允许复用，否则须独立实现。 +## 当前产品方向 + +- 当前业务范围与验收以 [docs/plan01.md](docs/plan01.md) 为准：竞品分析、账号与大小号响应、环境与代理、评论线索和私信；先完成抖音完整流程，再完成小红书。 +- 与旧探索规划或现有功能冲突时，以上述需求及使用者最新确认为准。自动响应按策略和 UID 冷却执行；人工发送逐次确认，两者不得混淆。自有账号互动与私信采用事件监听，不用轮询或 Mock 冒充实际能力。 +- 上述内容是目标范围，不代表已经实现。平台能力缺口、新的业务歧义必须向使用者确认；禁止自行删减需求、隐藏失败或以未要求的通用框架扩大实现范围。技术栈保持不变。 + ## 已批准的技术栈 ### Go 后端 diff --git a/cmd/control-plane/creator.go b/cmd/control-plane/creator.go new file mode 100644 index 0000000..d38a7f0 --- /dev/null +++ b/cmd/control-plane/creator.go @@ -0,0 +1,892 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/phasea" + "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" +) + +func registerCreator(app *fiber.App, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge) { + registerCreatorWithServices(app, store, phaseAStore, hubStore, credentials, nil, nil, nil) +} + +func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, executor creator.ActionExecutor, generator creator.TextGenerator, analyzer creator.ThemeAnalyzer) { + app.Get("/api/creator/settings", func(c fiber.Ctx) error { + settings, err := store.GetSettings(c.Context()) + if err != nil { + return creatorError(c, err) + } + return c.JSON(settings) + }) + app.Put("/api/creator/settings", func(c fiber.Ctx) error { + var input creator.SettingsUpdate + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + settings, err := store.UpdateSettings(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + return c.JSON(settings) + }) + + app.Get("/api/creator/accounts", func(c fiber.Ctx) error { + profiles, err := store.ListAccountProfiles(c.Context()) + if err != nil { + return creatorError(c, err) + } + return c.JSON(profiles) + }) + app.Get("/api/creator/accounts/:id/profile", func(c fiber.Ctx) error { + profile, err := store.GetAccountProfile(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(profile) + }) + app.Put("/api/creator/accounts/:id/profile", func(c fiber.Ctx) error { + var input creator.AccountProfileUpdate + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + profile, err := store.UpdateAccountProfile(c.Context(), c.Params("id"), input) + if err != nil { + return creatorError(c, err) + } + return c.JSON(profile) + }) + app.Post("/api/creator/accounts/:id/login-result", func(c fiber.Ctx) error { + var input struct { + Status string `json:"status"` + Reason string `json:"reason"` + ActualKey string `json:"actual_platform_account_key"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + result, err := store.RecordLoginResult(c.Context(), c.Params("id"), input.Status, input.Reason, input.ActualKey) + if err != nil { + return creatorError(c, err) + } + return c.JSON(result) + }) + app.Post("/api/creator/accounts/:id/big-account", func(c fiber.Ctx) error { + var input struct { + Enabled bool `json:"enabled"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + profile, err := store.SetBigAccount(c.Context(), c.Params("id"), input.Enabled) + if err != nil { + return creatorError(c, err) + } + return c.JSON(profile) + }) + app.Get("/api/creator/relations", func(c fiber.Ctx) error { + relations, err := store.ListRelations(c.Context(), c.Query("big_account_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(relations) + }) + app.Post("/api/creator/relations", func(c fiber.Ctx) error { + var input struct { + creator.Relation + Enabled bool `json:"enabled"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + if err := store.SetRelation(c.Context(), input.BigAccountID, input.SmallAccountID, input.Enabled); err != nil { + return creatorError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) + }) + app.Get("/api/creator/accounts/:id/strategies", func(c fiber.Ctx) error { + strategies, err := store.ListStrategies(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(strategies) + }) + app.Post("/api/creator/accounts/:id/strategies", func(c fiber.Ctx) error { + var input creator.StrategyInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + strategy, err := store.CreateStrategy(c.Context(), c.Params("id"), input) + if err != nil { + return creatorError(c, err) + } + return c.Status(fiber.StatusCreated).JSON(strategy) + }) + app.Put("/api/creator/strategies/:id", func(c fiber.Ctx) error { + var input creator.StrategyInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + strategy, err := store.UpdateStrategy(c.Context(), c.Params("id"), input) + if err != nil { + return creatorError(c, err) + } + return c.JSON(strategy) + }) + app.Post("/api/creator/strategies/:id/enable", func(c fiber.Ctx) error { return setStrategyEnabled(c, store, true) }) + app.Post("/api/creator/strategies/:id/disable", func(c fiber.Ctx) error { return setStrategyEnabled(c, store, false) }) + app.Delete("/api/creator/strategies/:id", func(c fiber.Ctx) error { + if err := store.DeleteStrategy(c.Context(), c.Params("id")); err != nil { + return creatorError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) + }) + + app.Get("/api/creator/competitors", func(c fiber.Ctx) error { + items, err := store.ListCompetitors(c.Context(), c.Query("platform")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/competitors", func(c fiber.Ctx) error { + var input creator.CompetitorInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, err := store.CreateCompetitor(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + return c.Status(fiber.StatusCreated).JSON(item) + }) + app.Get("/api/creator/competitors/:id", func(c fiber.Ctx) error { + item, err := store.GetCompetitor(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/competitors/:id/pause", func(c fiber.Ctx) error { + item, err := store.SetCompetitorEnabled(c.Context(), c.Params("id"), false) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/competitors/:id/resume", func(c fiber.Ctx) error { + item, err := store.SetCompetitorEnabled(c.Context(), c.Params("id"), true) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/competitors/:id/sync", func(c fiber.Ctx) error { + var input struct { + AccountID string `json:"account_id"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + report, err := syncCreatorCompetitor(c.Context(), store, phaseAStore, hubStore, credentials, c.Params("id"), input.AccountID) + if err != nil { + return creatorError(c, err) + } + return c.Status(fiber.StatusAccepted).JSON(report) + }) + + app.Get("/api/creator/works", func(c fiber.Ctx) error { + filter, err := workFilter(c) + if err != nil { + return creatorError(c, err) + } + items, err := store.ListWorks(c.Context(), filter) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/works", func(c fiber.Ctx) error { + var input creator.WorkInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, inserted, err := store.UpsertWork(c.Context(), input, time.Now().UTC()) + if err != nil { + return creatorError(c, err) + } + status := fiber.StatusOK + if inserted { + status = fiber.StatusCreated + } + return c.Status(status).JSON(item) + }) + app.Get("/api/creator/works/:id", func(c fiber.Ctx) error { + item, err := store.GetWork(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Get("/api/creator/works/:id/metrics", func(c fiber.Ctx) error { + items, err := store.ListMetrics(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/works/:id/metrics", func(c fiber.Ctx) error { + var input struct { + CollectedAt time.Time `json:"collected_at"` + Likes *int64 `json:"likes"` + CommentsCount *int64 `json:"comments_count"` + Shares *int64 `json:"shares"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + if input.CollectedAt.IsZero() { + input.CollectedAt = time.Now().UTC() + } + settings, err := store.GetSettings(c.Context()) + if err != nil { + return creatorError(c, err) + } + point, err := store.RecordMetric(c.Context(), creator.MetricInput{WorkID: c.Params("id"), CollectedAt: input.CollectedAt, Likes: input.Likes, CommentsCount: input.CommentsCount, Shares: input.Shares}, settings, time.Now().UTC()) + if err != nil { + return creatorError(c, err) + } + return c.JSON(point) + }) + app.Get("/api/creator/works/:id/material", func(c fiber.Ctx) error { + item, err := store.GetMaterial(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/works/:id/material/select", func(c fiber.Ctx) error { + item, inserted, err := store.SelectMaterial(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + status := fiber.StatusOK + if inserted { + status = fiber.StatusCreated + } + return c.Status(status).JSON(item) + }) + app.Post("/api/creator/works/:id/material/step", func(c fiber.Ctx) error { + var input struct { + Step string `json:"step"` + Status string `json:"status"` + Reference string `json:"reference"` + Reason string `json:"reason"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, err := store.SetMaterialStep(c.Context(), c.Params("id"), input.Step, input.Status, input.Reference, input.Reason) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/works/:id/material/rewrite/confirm", func(c fiber.Ctx) error { + var input struct { + Requirement string `json:"requirement"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, err := store.ConfirmRewrite(c.Context(), c.Params("id"), input.Requirement) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Put("/api/creator/works/:id/material/rewrite", func(c fiber.Ctx) error { + var input struct { + Title string `json:"title"` + Script string `json:"script"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, err := store.SaveRewrite(c.Context(), c.Params("id"), input.Title, input.Script) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + + app.Get("/api/creator/comments", func(c fiber.Ctx) error { + items, err := store.ListComments(c.Context(), c.Query("platform"), c.Query("work_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/comments", func(c fiber.Ctx) error { + var input creator.CommentInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, inserted, err := store.SaveComment(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + status := fiber.StatusOK + if inserted { + status = fiber.StatusCreated + } + return c.Status(status).JSON(item) + }) + app.Get("/api/creator/comments/:id", func(c fiber.Ctx) error { + item, err := store.GetComment(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Get("/api/creator/rules", func(c fiber.Ctx) error { + items, err := store.ListRules(c.Context(), c.Query("enabled_only") == "true") + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/rules", func(c fiber.Ctx) error { + var input creator.LeadRuleInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, err := store.CreateRule(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + return c.Status(fiber.StatusCreated).JSON(item) + }) + app.Get("/api/creator/rules/:id", func(c fiber.Ctx) error { + item, err := store.GetRule(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Put("/api/creator/rules/:id", func(c fiber.Ctx) error { + var input creator.LeadRuleInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, err := store.UpdateRule(c.Context(), c.Params("id"), input) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/rules/:id/enable", func(c fiber.Ctx) error { + item, err := store.SetRuleEnabled(c.Context(), c.Params("id"), true) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/rules/:id/disable", func(c fiber.Ctx) error { + item, err := store.SetRuleEnabled(c.Context(), c.Params("id"), false) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Get("/api/creator/rule-results", func(c fiber.Ctx) error { + items, err := store.ListRuleResults(c.Context(), c.Query("comment_id"), c.Query("rule_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Get("/api/creator/leads", func(c fiber.Ctx) error { + items, err := store.ListLeads(c.Context(), c.Query("platform")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/comments/:id/analyze", func(c fiber.Ctx) error { + var input struct { + RuleID string `json:"rule_id"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + result, err := store.AnalyzeComment(c.Context(), c.Params("id"), input.RuleID, analyzer) + if err != nil && !errors.Is(err, creator.ErrUnavailable) { + return creatorError(c, err) + } + status := fiber.StatusOK + if errors.Is(err, creator.ErrUnavailable) { + status = fiber.StatusServiceUnavailable + } + return c.Status(status).JSON(result) + }) + + app.Get("/api/creator/events", func(c fiber.Ctx) error { + items, err := store.ListEvents(c.Context(), c.Query("account_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/events", func(c fiber.Ctx) error { + var input creator.InteractionEvent + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + result, err := store.RecordEvent(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + status := fiber.StatusOK + if !result.Duplicate { + status = fiber.StatusCreated + } + return c.Status(status).JSON(result) + }) + app.Post("/api/creator/events/process", func(c fiber.Ctx) error { + var input creator.InteractionEvent + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + result, err := store.ProcessAutomaticEvent(c.Context(), input, executor, generator) + if err != nil && !errors.Is(err, creator.ErrUnavailable) { + return creatorError(c, err) + } + status := fiber.StatusOK + if errors.Is(err, creator.ErrUnavailable) { + status = fiber.StatusServiceUnavailable + } + return c.Status(status).JSON(result) + }) + app.Post("/api/creator/events/:id/display", func(c fiber.Ctx) error { + event, err := store.SetEventDisplayed(c.Context(), c.Params("id"), time.Now().UTC()) + if err != nil { + return creatorError(c, err) + } + return c.JSON(event) + }) + app.Get("/api/creator/operations", func(c fiber.Ctx) error { + items, err := store.ListOperations(c.Context(), c.Query("account_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/operations", func(c fiber.Ctx) error { + var input creator.OperationInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, inserted, err := store.CreateOperation(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + status := fiber.StatusOK + if inserted { + status = fiber.StatusCreated + } + return c.Status(status).JSON(item) + }) + app.Get("/api/creator/operations/:id", func(c fiber.Ctx) error { + item, err := store.GetOperation(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Post("/api/creator/operations/:id/execute", func(c fiber.Ctx) error { + item, err := store.ExecuteManualOperation(c.Context(), c.Params("id"), executor) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + + app.Get("/api/creator/conversations", func(c fiber.Ctx) error { + items, err := store.ListConversations(c.Context(), c.Query("account_id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Get("/api/creator/conversations/:id/messages", func(c fiber.Ctx) error { + items, err := store.ListMessages(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.JSON(items) + }) + app.Post("/api/creator/messages", func(c fiber.Ctx) error { + var input creator.MessageInput + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + item, inserted, err := store.SaveMessage(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + status := fiber.StatusOK + if inserted { + status = fiber.StatusCreated + } + return c.Status(status).JSON(item) + }) +} + +func setStrategyEnabled(c fiber.Ctx, store *creator.Store, enabled bool) error { + item, err := store.SetStrategyEnabled(c.Context(), c.Params("id"), enabled) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) +} + +func workFilter(c fiber.Ctx) (creator.WorkFilter, error) { + filter := creator.WorkFilter{Platform: c.Query("platform"), SourceID: c.Query("source_id"), SourceType: c.Query("source_type")} + for _, field := range []struct { + name string + target **int64 + }{{"min_likes", &filter.MinLikes}, {"min_comments", &filter.MinComments}, {"min_shares", &filter.MinShares}} { + value := c.Query(field.name) + if value == "" { + continue + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return creator.WorkFilter{}, creator.ErrInvalid + } + *field.target = &parsed + } + for _, field := range []struct { + name string + target **time.Time + }{{"published_after", &filter.PublishedAfter}, {"published_before", &filter.PublishedBefore}} { + value := c.Query(field.name) + if value == "" { + continue + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return creator.WorkFilter{}, creator.ErrInvalid + } + *field.target = &parsed + } + return filter, nil +} + +func decodeCreator(c fiber.Ctx, destination any) error { + decoder := json.NewDecoder(bytes.NewReader(c.Body())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { + return creator.ErrInvalid + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return creator.ErrInvalid + } + return nil +} + +func creatorError(c fiber.Ctx, err error) error { + logrus.WithError(err).WithField("service", "control-plane").Error("creator operation failed") + status, message := fiber.StatusInternalServerError, "creator operation failed" + switch { + case errors.Is(err, creator.ErrInvalid): + status, message = fiber.StatusBadRequest, creator.ErrInvalid.Error() + case errors.Is(err, creator.ErrConflict): + status, message = fiber.StatusConflict, creator.ErrConflict.Error() + case errors.Is(err, creator.ErrNotFound): + status, message = fiber.StatusNotFound, creator.ErrNotFound.Error() + case errors.Is(err, creator.ErrUnavailable): + status, message = fiber.StatusServiceUnavailable, creator.ErrUnavailable.Error() + case errors.Is(err, creator.ErrUncertain): + status, message = fiber.StatusConflict, creator.ErrUncertain.Error() + } + return c.Status(status).JSON(map[string]string{"error": message}) +} + +type creatorGatewayBrowser struct { + gateway hub.Gateway + environment hub.EnvironmentContext +} + +func (browser creatorGatewayBrowser) SetCookies(ctx context.Context, cookies []douyin.Cookie) error { + payload := gatewayGenerationPayload(browser.environment) + payload["cookies"] = cookies + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/cookies", payload, 30*time.Second) + if err != nil { + return err + } + if status != http.StatusNoContent && status != http.StatusNotModified { + return fmt.Errorf("douyin cookie injection rejected with HTTP %d: %s", status, string(body)) + } + return nil +} + +func (browser creatorGatewayBrowser) Get(ctx context.Context, target string) (douyin.Response, error) { + payload := gatewayGenerationPayload(browser.environment) + payload["url"] = target + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/get", payload, 30*time.Second) + if err != nil { + return douyin.Response{}, err + } + if status != http.StatusOK { + return douyin.Response{}, fmt.Errorf("douyin browser request rejected with HTTP %d: %s", status, string(body)) + } + var response struct { + Status int `json:"status"` + Body string `json:"body"` + Challenge douyin.Challenge `json:"challenge"` + } + if err := json.Unmarshal(body, &response); err != nil { + return douyin.Response{}, fmt.Errorf("decode douyin browser response: %w", err) + } + return douyin.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil +} + +func syncCreatorCompetitor(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, competitorID, accountID string) (creator.CollectionReport, error) { + return syncCreatorCompetitorWithClaim(ctx, store, phaseAStore, hubStore, credentials, competitorID, accountID, true) +} + +func syncCreatorCompetitorDue(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, competitorID, accountID string) (creator.CollectionReport, error) { + return syncCreatorCompetitorWithClaim(ctx, store, phaseAStore, hubStore, credentials, competitorID, accountID, false) +} + +func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, competitorID, accountID string, force bool) (creator.CollectionReport, error) { + if store == nil || phaseAStore == nil || hubStore == nil || credentials == nil || accountID == "" { + return creator.CollectionReport{}, creator.ErrUnavailable + } + competitor, err := store.GetCompetitor(ctx, competitorID) + if err != nil { + return creator.CollectionReport{}, err + } + settings, err := store.GetSettings(ctx) + if err != nil { + return creator.CollectionReport{}, err + } + now := time.Now().UTC() + claimed, err := store.ClaimCompetitorSync(ctx, competitorID, force, now) + if err != nil { + return creator.CollectionReport{}, err + } + if !claimed { + return creator.CollectionReport{}, creator.ErrConflict + } + blocked := func(blockErr error) (creator.CollectionReport, error) { + markErr := store.MarkCompetitorSync(ctx, competitorID, "blocked", "", blockErr.Error(), nil) + return creator.CollectionReport{}, errors.Join(blockErr, markErr) + } + if competitor.Platform != creator.PlatformDouyin { + return blocked(fmt.Errorf("%w: 小红书采集器尚未完成平台能力验证", creator.ErrUnavailable)) + } + account, err := phaseAStore.GetAccount(ctx, accountID) + if err != nil { + return blocked(err) + } + if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" { + return blocked(creator.ErrConflict) + } + if _, err := store.GetAccountProfile(ctx, accountID); err != nil { + return blocked(err) + } + resolver, ok := credentials.(phasea.CredentialResolver) + if !ok { + return blocked(creator.ErrUnavailable) + } + rawCredential, err := phaseAStore.ResolveAccountCredential(ctx, accountID, resolver) + if err != nil { + return blocked(fmt.Errorf("%w: resolve account credential: %v", creator.ErrUnavailable, err)) + } + cookies, err := douyin.ParseCookieHeader(rawCredential) + if err != nil { + cookies, err = douyin.ParseCookieBundle(rawCredential) + } + if err != nil { + return blocked(fmt.Errorf("%w: invalid account cookie bundle", creator.ErrConflict)) + } + environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) + if err != nil { + return blocked(fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err)) + } + if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { + return blocked(fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable)) + } + gateway, err := hubStore.GetGateway(ctx, environment.Gateway) + if err != nil { + return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)) + } + browser := creatorGatewayBrowser{gateway: gateway, environment: environment} + if err := browser.SetCookies(ctx, cookies); err != nil { + return blocked(fmt.Errorf("%w: set account cookies: %v", creator.ErrUnavailable, err)) + } + collector := douyin.CreatorCollector{Browser: browser, AccountKey: competitor.PlatformAccountKey, SourceType: creator.SourceCompetitor, SourceID: competitor.ID} + if err := collector.VerifyIdentity(ctx, account.PlatformAccountKey); err != nil { + return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)) + } + report, collectErr := store.CollectSource(ctx, competitor.Platform, creator.SourceCompetitor, competitor.ID, collector, now) + if collectErr != nil { + status, next := "failed", now.Add(time.Duration(settings.NewWorkIntervalSeconds)*time.Second) + if errors.Is(collectErr, creator.ErrUnavailable) || errors.Is(collectErr, creator.ErrConflict) { + status, next = "blocked", time.Time{} + } + var nextAt *time.Time + if !next.IsZero() { + nextAt = &next + } + markErr := store.MarkCompetitorSync(ctx, competitorID, status, "", collectErr.Error(), nextAt) + return report, errors.Join(collectErr, markErr) + } + next := time.Now().UTC().Add(time.Duration(settings.NewWorkIntervalSeconds) * time.Second) + if err := store.MarkCompetitorSync(ctx, competitorID, "idle", "", "", &next); err != nil { + return report, err + } + return report, nil +} + +func runCreatorScheduleOnce(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge) error { + if store == nil { + return creator.ErrUnavailable + } + settings, err := store.GetSettings(ctx) + if err != nil { + return err + } + now := time.Now().UTC() + competitors, err := store.ListDueCompetitors(ctx, now) + if err != nil { + return err + } + for _, competitor := range competitors { + accountID, err := creatorCollectionAccount(ctx, store, phaseAStore, hubStore, competitor.Platform) + if err != nil { + _ = store.MarkCompetitorSync(ctx, competitor.ID, "blocked", "", err.Error(), nil) + logrus.WithError(err).WithField("competitor_id", competitor.ID).Warn("creator competitor sync blocked") + continue + } + if _, err := syncCreatorCompetitorDue(ctx, store, phaseAStore, hubStore, credentials, competitor.ID, accountID); err != nil { + logrus.WithError(err).WithField("competitor_id", competitor.ID).Warn("creator competitor scheduled sync failed") + } + } + ownedAccounts, err := store.ListDueOwnedAccounts(ctx, now, settings.NewWorkIntervalSeconds) + if err != nil { + return err + } + for _, accountID := range ownedAccounts { + if err := syncCreatorOwned(ctx, store, phaseAStore, hubStore, credentials, accountID, now); err != nil { + logrus.WithError(err).WithField("account_id", accountID).Warn("creator owned scheduled sync failed") + } + } + return nil +} + +func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, accountID string, now time.Time) error { + if store == nil || phaseAStore == nil || hubStore == nil || credentials == nil || accountID == "" { + return creator.ErrUnavailable + } + account, err := phaseAStore.GetAccount(ctx, accountID) + if err != nil { + return err + } + if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" { + return creator.ErrConflict + } + if _, err := store.GetAccountProfile(ctx, accountID); err != nil { + return err + } + resolver, ok := credentials.(phasea.CredentialResolver) + if !ok { + return creator.ErrUnavailable + } + rawCredential, err := phaseAStore.ResolveAccountCredential(ctx, accountID, resolver) + if err != nil { + return fmt.Errorf("%w: resolve account credential: %v", creator.ErrUnavailable, err) + } + cookies, err := douyin.ParseCookieHeader(rawCredential) + if err != nil { + cookies, err = douyin.ParseCookieBundle(rawCredential) + } + if err != nil { + return fmt.Errorf("%w: invalid account cookie bundle", creator.ErrConflict) + } + environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) + if err != nil { + return fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) + } + if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { + return fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable) + } + gateway, err := hubStore.GetGateway(ctx, environment.Gateway) + if err != nil { + return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) + } + browser := creatorGatewayBrowser{gateway: gateway, environment: environment} + if err := browser.SetCookies(ctx, cookies); err != nil { + return fmt.Errorf("%w: set account cookies: %v", creator.ErrUnavailable, err) + } + collector := douyin.CreatorCollector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: creator.SourceOwned, SourceID: account.ID} + if err := collector.VerifyIdentity(ctx, account.PlatformAccountKey); err != nil { + return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err) + } + _, err = store.CollectSource(ctx, account.Platform, creator.SourceOwned, account.ID, collector, now) + return err +} + +func creatorCollectionAccount(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, platform string) (string, error) { + if store == nil || phaseAStore == nil || hubStore == nil || platform == "" { + return "", creator.ErrUnavailable + } + accounts, err := phaseAStore.ListAccounts(ctx) + if err != nil { + return "", err + } + for _, account := range accounts { + if account.Platform != platform || account.AuthorizationStatus != "authorized" { + continue + } + if _, err := store.GetAccountProfile(ctx, account.ID); err != nil { + continue + } + if _, err := hubStore.GetEnvironmentContextForAccount(ctx, account.ID); err != nil { + continue + } + return account.ID, nil + } + return "", fmt.Errorf("%w: no authorized creator collection account", creator.ErrUnavailable) +} + +func runCreatorScheduler(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge) { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + if err := runCreatorScheduleOnce(ctx, store, phaseAStore, hubStore, credentials); err != nil { + logrus.WithError(err).Error("creator scheduler failed") + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} diff --git a/cmd/control-plane/credential.go b/cmd/control-plane/credential.go index bdfc707..7a965e8 100644 --- a/cmd/control-plane/credential.go +++ b/cmd/control-plane/credential.go @@ -106,6 +106,32 @@ func (bridge *persistentCredentialBridge) Delete(ctx context.Context, reference return nil } +func (bridge *persistentCredentialBridge) Resolve(ctx context.Context, reference phasea.CredentialReference, key string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if !validCredentialReference(reference.Provider, key) { + return nil, errors.New("invalid credential reference") + } + payload, err := os.ReadFile(bridge.path(reference.Provider, key)) + if err != nil { + return nil, errors.New("resolve credential") + } + aead, err := bridge.aead() + if err != nil { + return nil, err + } + if len(payload) < 1+aead.NonceSize() || payload[0] != credentialFileVersion { + return nil, errors.New("invalid credential") + } + nonceEnd := 1 + aead.NonceSize() + value, err := aead.Open(nil, payload[1:nonceEnd], payload[nonceEnd:], credentialAAD(reference.Provider, key)) + if err != nil { + return nil, errors.New("resolve credential") + } + return value, nil +} + func (bridge *persistentCredentialBridge) aead() (cipher.AEAD, error) { block, err := aes.NewCipher(bridge.key[:]) if err != nil { diff --git a/cmd/control-plane/douyin_test.go b/cmd/control-plane/douyin_test.go index 73cb85c..91aac2e 100644 --- a/cmd/control-plane/douyin_test.go +++ b/cmd/control-plane/douyin_test.go @@ -12,7 +12,7 @@ import ( "git.ipao.vip/rogee/creator-hub/internal/hub" ) -const testDouyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/" +const testDouyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp" func TestDouyinGatewayBrowserFencesAccountGeneration(t *testing.T) { requests := 0 diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index d0bb7a3..da18118 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -106,7 +106,7 @@ func gatewayCreatePayload(environment hub.EnvironmentContext, imageRef string, n fingerprint := environment.Fingerprint fingerprint.ProxyServer = "" fingerprint.DisableNonProxiedUDP = false - cmd := append(fingerprint.Args(), "about:blank") + cmd := append(fingerprint.Args(), "--remote-allow-origins=*", "about:blank") return map[string]any{ "alias": environment.Alias, "name": environment.Name, diff --git a/cmd/control-plane/hub_test.go b/cmd/control-plane/hub_test.go index 1745d46..ed918dd 100644 --- a/cmd/control-plane/hub_test.go +++ b/cmd/control-plane/hub_test.go @@ -1026,8 +1026,8 @@ func TestCreateBrowserOrchestratesGateway(t *testing.T) { t.Fatalf("platform must force the bound exit: %#v", payload) } cmd := payload["cmd"].([]any) - if len(cmd) != 4 || cmd[0] != "--fingerprint=2024" || cmd[1] != "--fingerprint-platform=windows" || - cmd[2] != "--timezone=Asia/Shanghai" || cmd[3] != "about:blank" { + if len(cmd) != 5 || cmd[0] != "--fingerprint=2024" || cmd[1] != "--fingerprint-platform=windows" || + cmd[2] != "--timezone=Asia/Shanghai" || cmd[3] != "--remote-allow-origins=*" || cmd[4] != "about:blank" { t.Fatalf("cmd must carry fingerprint args plus start url: %#v", cmd) } if stored := store.envs["account-a"].Fingerprint; stored.ProxyServer != "" || stored.DisableNonProxiedUDP { diff --git a/cmd/control-plane/main.go b/cmd/control-plane/main.go index 795ca59..135fb57 100644 --- a/cmd/control-plane/main.go +++ b/cmd/control-plane/main.go @@ -17,6 +17,7 @@ import ( "syscall" "time" + "git.ipao.vip/rogee/creator-hub/internal/creator" "git.ipao.vip/rogee/creator-hub/internal/hub" "git.ipao.vip/rogee/creator-hub/internal/phasea" "git.ipao.vip/rogee/creator-hub/internal/taskstate" @@ -74,6 +75,12 @@ func newCommand() *cobra.Command { return err } defer hubStore.Close() + creatorStore, err := creator.Open(command.Context(), cfg.databaseURL) + if err != nil { + return err + } + defer creatorStore.Close() + creatorStore.SetSecretBridge(creatorSecretBridge{bridge: credentials}) notify := newAttentionNotifier(os.Stderr) phaseAStore.SetTaskNotifier(notify) hubStore.SetTaskNotifier(notify) @@ -84,11 +91,19 @@ func newCommand() *cobra.Command { defer close(heartbeatDone) runtimeLeaseHeartbeat(heartbeatContext, hubStore) }() - listenErr := newHandlerWithCredentialBridge(cfg.webDir, cfg.username, cfg.password, phaseAStore, hubStore, credentials).Listen(cfg.listenAddr, fiber.ListenConfig{ + creatorScheduleContext, stopCreatorScheduler := context.WithCancel(command.Context()) + creatorScheduleDone := make(chan struct{}) + go func() { + defer close(creatorScheduleDone) + runCreatorScheduler(creatorScheduleContext, creatorStore, phaseAStore, hubStore, credentials) + }() + listenErr := newHandlerWithCreator(cfg.webDir, cfg.username, cfg.password, phaseAStore, hubStore, credentials, creatorStore).Listen(cfg.listenAddr, fiber.ListenConfig{ GracefulContext: command.Context(), DisableStartupMessage: true, }) + stopCreatorScheduler() stopHeartbeat() + <-creatorScheduleDone <-heartbeatDone return listenErr }, @@ -212,7 +227,29 @@ func newHandlerWithStores(webDirectory, username, password string, phaseAStore * return newHandlerWithCredentialBridge(webDirectory, username, password, phaseAStore, hubStore, nil) } +type creatorSecretBridge struct { + bridge phasea.CredentialBridge +} + +func (b creatorSecretBridge) Store(ctx context.Context, reference creator.SecretReference, key, value string) error { + if b.bridge == nil { + return errors.New("credential bridge is unavailable") + } + return b.bridge.Store(ctx, phasea.CredentialReference{ID: reference.ID, Provider: reference.Provider}, key, value) +} + +func (b creatorSecretBridge) Delete(ctx context.Context, reference creator.SecretReference, key string) error { + if b.bridge == nil { + return errors.New("credential bridge is unavailable") + } + return b.bridge.Delete(ctx, phasea.CredentialReference{ID: reference.ID, Provider: reference.Provider}, key) +} + func newHandlerWithCredentialBridge(webDirectory, username, password string, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge) *fiber.App { + return newHandlerWithCreator(webDirectory, username, password, phaseAStore, hubStore, credentials, nil) +} + +func newHandlerWithCreator(webDirectory, username, password string, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, creatorStore *creator.Store) *fiber.App { app := fiber.New(fiber.Config{ AppName: "CreatorHub control plane", BodyLimit: 1 << 20, @@ -231,6 +268,9 @@ func newHandlerWithCredentialBridge(webDirectory, username, password string, pha if phaseAStore != nil { registerPhaseA(app, phaseAStore, hubStore, credentials) } + if creatorStore != nil { + registerCreator(app, creatorStore, phaseAStore, hubStore, credentials) + } app.Get("/*", spaHandler(webDirectory)) return app } diff --git a/cmd/docker-gateway/douyin.go b/cmd/docker-gateway/douyin.go index a5b70ad..78e6a69 100644 --- a/cmd/docker-gateway/douyin.go +++ b/cmd/docker-gateway/douyin.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "net" "net/http" "net/url" "regexp" @@ -15,6 +16,7 @@ import ( "git.ipao.vip/rogee/creator-hub/internal/douyin" "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" "golang.org/x/net/websocket" ) @@ -23,6 +25,7 @@ const ( douyinOriginURL = "https://www.douyin.com/" douyinIdentityPath = "/aweme/v1/web/user/profile/self/" douyinWorksPath = "/aweme/v1/web/aweme/post/" + douyinCommentsPath = "/aweme/v1/web/comment/list/" douyinResponseLimit = 1 << 20 browserControlTimeout = 15 * time.Second ) @@ -71,7 +74,11 @@ func (api gateway) setDouyinCookies(c fiber.Ctx) error { if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { return writeError(c, statusFor(err), err) } - if api.browser == nil || api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies) != nil { + if api.browser == nil { + return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) + } + if err := api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies); err != nil { + logrus.WithError(err).WithField("browser_id", c.Params("id")).Error("restricted browser cookie operation failed") return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) } if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { @@ -99,6 +106,7 @@ func (api gateway) getDouyin(c fiber.Ctx) error { } response, err := api.browser.Get(c.Context(), c.Params("id"), input.URL) if err != nil { + logrus.WithError(err).WithField("browser_id", c.Params("id")).Error("restricted browser fetch operation failed") return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) } if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { @@ -174,10 +182,22 @@ func validDouyinURL(raw string) bool { query := parsed.Query() switch parsed.Path { case douyinIdentityPath: - return parsed.RawQuery == "" + return len(query) == 2 && len(query["aid"]) == 1 && query.Get("aid") == "6383" && + len(query["device_platform"]) == 1 && query.Get("device_platform") == "webapp" case douyinWorksPath: - return len(query) == 3 && len(query["sec_user_id"]) == 1 && douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) && - len(query["count"]) == 1 && query.Get("count") == "20" && len(query["max_cursor"]) == 1 && query.Get("max_cursor") == "0" + if len(query) != 3 || len(query["sec_user_id"]) != 1 || !douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) || + len(query["count"]) != 1 || query.Get("count") != "20" || len(query["max_cursor"]) != 1 { + return false + } + cursor, err := strconv.ParseInt(query.Get("max_cursor"), 10, 64) + return err == nil && cursor >= 0 + case douyinCommentsPath: + if len(query) != 3 || len(query["aweme_id"]) != 1 || !douyinAccountKeyPattern.MatchString(query.Get("aweme_id")) || + len(query["count"]) != 1 || query.Get("count") != "20" || len(query["cursor"]) != 1 { + return false + } + cursor, err := strconv.ParseInt(query.Get("cursor"), 10, 64) + return err == nil && cursor >= 0 default: return false } @@ -227,11 +247,15 @@ func (browser cdpBrowser) SetCookies(ctx context.Context, alias string, cookies LoaderID string `json:"loaderId"` ErrorText string `json:"errorText"` } - if err := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events); err != nil || - navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" { + navigationErr := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events) + logrus.WithFields(logrus.Fields{"frame_id": navigation.FrameID, "loader_id": navigation.LoaderID, + "error_text": navigation.ErrorText, "event_count": len(events), "command_error": navigationErr != nil}).Debug("restricted browser navigation response") + if navigationErr != nil || navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" { + logrus.WithFields(logrus.Fields{"frame_id": navigation.FrameID, "loader_id": navigation.LoaderID, + "error_text": navigation.ErrorText, "event_count": len(events)}).Error("restricted browser navigation response invalid") return errors.New("restricted browser navigation failed") } - if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, navigation.LoaderID, events); err != nil { + if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, events); err != nil { return err } return cdpCommand(connection, &commandID, "Network.setCookies", map[string]any{"cookies": cdpCookies}, nil, nil) @@ -286,6 +310,18 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket if browser.endpoint != nil { base = browser.endpoint(alias) } + baseURL, err := url.Parse(base) + if err != nil || baseURL.Scheme != "http" || baseURL.Hostname() == "" || baseURL.Port() == "" { + return nil, errors.New("restricted browser unavailable") + } + if net.ParseIP(baseURL.Hostname()) == nil { + addresses, lookupErr := net.LookupIP(baseURL.Hostname()) + if lookupErr != nil || len(addresses) == 0 { + return nil, errors.New("restricted browser unavailable") + } + baseURL.Host = net.JoinHostPort(addresses[0].String(), baseURL.Port()) + } + base = strings.TrimRight(baseURL.String(), "/") request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/json/list", nil) if err != nil { return nil, errors.New("restricted browser unavailable") @@ -300,6 +336,7 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } response, err := client.Do(request) if err != nil { + logrus.WithError(err).WithField("browser_id", alias).Error("restricted browser discovery failed") return nil, errors.New("restricted browser unavailable") } defer response.Body.Close() @@ -322,7 +359,6 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return nil, errors.New("restricted browser unavailable") } - baseURL, _ := url.Parse(base) pageTarget := "" for _, target := range targets { if target.Type != "page" { @@ -352,6 +388,7 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket } connection, err := config.DialContext(ctx) if err != nil { + logrus.WithError(err).WithField("browser_id", alias).Error("restricted browser websocket failed") return nil, errors.New("restricted browser unavailable") } deadline := time.Now().Add(browserControlTimeout) @@ -359,6 +396,7 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket deadline = contextDeadline } _ = connection.SetDeadline(deadline) + logrus.WithFields(logrus.Fields{"browser_id": alias, "cdp_endpoint": pageTarget}).Debug("restricted browser websocket connected") return connection, nil } @@ -373,11 +411,13 @@ type cdpMessage struct { func cdpCommand(connection *websocket.Conn, commandID *int, method string, parameters any, output any, events *[]cdpMessage) error { *commandID = *commandID + 1 if err := websocket.JSON.Send(connection, map[string]any{"id": *commandID, "method": method, "params": parameters}); err != nil { + logrus.WithError(err).WithField("cdp_method", method).Error("restricted browser command send failed") return errors.New("restricted browser command failed") } for range 128 { var reply cdpMessage if err := websocket.JSON.Receive(connection, &reply); err != nil { + logrus.WithError(err).WithField("cdp_method", method).Error("restricted browser command receive failed") return errors.New("restricted browser command failed") } if reply.ID != *commandID { @@ -387,6 +427,7 @@ func cdpCommand(connection *websocket.Conn, commandID *int, method string, param continue } if len(reply.Error) != 0 || len(reply.Result) == 0 || bytes.Equal(bytes.TrimSpace(reply.Result), []byte("null")) { + logrus.WithFields(logrus.Fields{"cdp_method": method, "cdp_error": string(reply.Error), "has_result": len(reply.Result) != 0}).Error("restricted browser command returned failure") return errors.New("restricted browser command failed") } if output != nil && json.Unmarshal(reply.Result, output) != nil { @@ -397,17 +438,32 @@ func cdpCommand(connection *websocket.Conn, commandID *int, method string, param return errors.New("restricted browser command failed") } -func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID, loaderID string, events []cdpMessage) error { +func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID string, events []cdpMessage) error { deadline := time.Now().Add(10 * time.Second) if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { deadline = contextDeadline } _ = connection.SetReadDeadline(deadline) - for attempts := 0; attempts < 256; attempts++ { + bufferedLifecycle := map[string]bool{} + for _, event := range events { + if event.Method != "Page.lifecycleEvent" { + continue + } + var lifecycle struct { + FrameID string `json:"frameId"` + LoaderID string `json:"loaderId"` + Name string `json:"name"` + } + if json.Unmarshal(event.Params, &lifecycle) == nil && lifecycle.FrameID == frameID && (lifecycle.Name == "DOMContentLoaded" || lifecycle.Name == "load") { + bufferedLifecycle[lifecycle.LoaderID+":"+lifecycle.Name] = true + } + } + for { var event cdpMessage if len(events) != 0 { event, events = events[0], events[1:] - } else if websocket.JSON.Receive(connection, &event) != nil { + } else if err := websocket.JSON.Receive(connection, &event); err != nil { + logrus.WithError(err).Error("restricted browser lifecycle receive failed") return errors.New("restricted browser navigation failed") } if event.Method != "Page.lifecycleEvent" { @@ -418,7 +474,11 @@ func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandI LoaderID string `json:"loaderId"` Name string `json:"name"` } - if json.Unmarshal(event.Params, &lifecycle) != nil || lifecycle.FrameID != frameID || lifecycle.LoaderID != loaderID || lifecycle.Name != "load" { + if err := json.Unmarshal(event.Params, &lifecycle); err != nil { + logrus.WithError(err).Debug("restricted browser lifecycle decode failed") + continue + } + if lifecycle.FrameID != frameID || lifecycle.Name != "DOMContentLoaded" && lifecycle.Name != "load" || bufferedLifecycle[lifecycle.LoaderID+":"+lifecycle.Name] { continue } var evaluated struct { @@ -429,11 +489,12 @@ func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandI if err := cdpCommand(connection, commandID, "Runtime.evaluate", map[string]any{ "expression": "location.origin", "returnByValue": true, }, &evaluated, nil); err != nil || evaluated.Result.Value != douyinOrigin { + logrus.WithFields(logrus.Fields{"frame_id": lifecycle.FrameID, "loader_id": lifecycle.LoaderID, + "origin": evaluated.Result.Value}).Error("restricted browser navigation origin failed") return errors.New("restricted browser navigation failed") } return nil } - return errors.New("restricted browser navigation failed") } func detectDouyinChallenge(status int, body string) douyin.Challenge { diff --git a/cmd/docker-gateway/douyin_test.go b/cmd/docker-gateway/douyin_test.go index cd9d420..f72cd6f 100644 --- a/cmd/docker-gateway/douyin_test.go +++ b/cmd/docker-gateway/douyin_test.go @@ -16,6 +16,8 @@ import ( "golang.org/x/net/websocket" ) +const douyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp" + type fakeRestrictedBrowser struct { cookies []douyin.Cookie urls []string @@ -90,7 +92,7 @@ func TestGatewayRestrictedDouyinContract(t *testing.T) { } response.Body.Close() - identityURL := "https://www.douyin.com" + douyinIdentityPath + identityURL := douyinIdentityURL getBody := `{` + generation + `,"url":"` + identityURL + `"}` response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody))) body, _ := io.ReadAll(response.Body) @@ -222,7 +224,7 @@ func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) { }}) if !onlyOldLoader { afterReply = append(afterReply, map[string]any{"method": "Page.lifecycleEvent", "params": map[string]any{ - "frameId": "frame-new", "loaderId": "loader-new", "name": "load", + "frameId": "frame-new", "loaderId": "loader-final", "name": "load", }}) } case "Runtime.evaluate": @@ -269,7 +271,7 @@ func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) { }}); err != nil { t.Fatal(err) } - result, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath) + result, err := browser.Get(context.Background(), "account-a", douyinIdentityURL) if err != nil || result.Status != 412 || result.Challenge != douyin.ChallengeCaptcha { t.Fatalf("unexpected CDP response: %#v err=%v", result, err) } @@ -297,13 +299,13 @@ func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) { } pageOrigin, fetchMode = douyinOrigin, "redirect" mu.Unlock() - if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil { + if _, err := browser.Get(context.Background(), "account-a", douyinIdentityURL); err == nil { t.Fatal("accepted a redirected fetch") } mu.Lock() fetchMode = "too_large" mu.Unlock() - if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil { + if _, err := browser.Get(context.Background(), "account-a", douyinIdentityURL); err == nil { t.Fatal("accepted a response at the 1 MiB limit") } mu.Lock() @@ -395,9 +397,9 @@ func TestCDPDiscoveryRequiresOneJSONValue(t *testing.T) { func TestDouyinURLContract(t *testing.T) { for target, want := range map[string]bool{ - "https://www.douyin.com" + douyinIdentityPath: true, + douyinIdentityURL: true, "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0": true, - "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=1": false, + "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=1": true, "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0&method=publish": false, "https://www.douyin.com/aweme/v1/web/commit/item/": false, } { diff --git a/cmd/docker-gateway/main.go b/cmd/docker-gateway/main.go index 54c692e..ebb8ca2 100644 --- a/cmd/docker-gateway/main.go +++ b/cmd/docker-gateway/main.go @@ -919,6 +919,11 @@ func (api gateway) requireGeneration(id string, input generationRequest) (string } version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) if input.RuntimeID == "" || runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkIDLabel] != input.NetworkID { + logrus.WithFields(logrus.Fields{ + "browser_id": id, "input_runtime_id": input.RuntimeID, "actual_runtime_id": runtimeID, + "input_binding_version": input.BindingVersion, "actual_binding_version": version, + "input_network_id": input.NetworkID, "actual_network_id": labels[networkIDLabel], + }).Warn("browser generation mismatch") return "", false, errGenerationConflict } return runtimeID, true, nil diff --git a/docs/architecture/container-control.md b/docs/architecture/container-control.md index f10d46c..0b87407 100644 --- a/docs/architecture/container-control.md +++ b/docs/architecture/container-control.md @@ -1,36 +1,48 @@ # 浏览器容器控制面 +> 当前实现说明，核对基线 `main@1fbf126`；不是新业务完成证明。目标范围与验收以 [plan01](../plan01.md) 为准，运行步骤见[部署说明](../deployment.md)。现状限制不自动成为新产品约束。 + ## 技术选型 -- 前端：React 19 + react-admin（ra-core）+ MUI，包含运行环境、镜像版本、网关管理三个页面。 +- 前端：React 19 + Vite 8 + Refine + shadcn/ui + Tailwind CSS，图标 RemixIcon；项目自有 Layout 与 HashRouter，当前页面见 [main.jsx](../../web/src/main.jsx)，依赖见 [package.json](../../web/package.json)。 - 后端：Go 模块化单体，Fiber v3 提供 HTTP 路由，Viper 读取并校验启动配置，Logrus 输出 JSON 结构化日志，Cobra 保持当前两个服务入口。控制面提供同源 API 和静态文件，并编排网关；受限网关单独封装 Docker Engine API，是纯执行器。 -- 数据：环境配置（别名、中文名、网关、镜像版本、指纹参数）持久化在 Postgres，运行态实时查询网关；Profile 使用命名卷持久化；阶段 A 账号、凭据引用、确认、任务、尝试和审计实体同样由控制面持久化到 Postgres。 +- 数据：环境配置（别名、中文名、网关、镜像版本、指纹参数）持久化在 Postgres，列表/详情读取持久运行记录，不触发网关实时探测；后台独立维护运行租约；Profile 使用命名卷持久化；阶段 A 账号、凭据引用、确认、任务、尝试和审计实体同样由控制面持久化到 Postgres。 - 部署：Docker Compose 启动控制面和受限网关；浏览器容器由网关按平台下发的镜像引用动态创建，缺失时自动拉取。 ## 调用链与契约 ```text -React ──> control-plane ── /api/browsers ──(Bearer token)──> docker-gateway ──> docker.sock - │ │ - │ └─> browser container - └─ /api/phase-a, /api/browser-images, /api/gateways ──> PostgreSQL +React ── /api/* (Basic Auth) ──> control-plane + ├─ /v1/browsers (Bearer token) ──> docker-gateway ──> docker.sock + │ └─> browser container + └─ 账号/环境/任务等持久记录 ──> PostgreSQL ``` 控制面是唯一事实源：网关不持有镜像清单和业务规则，镜像引用、启动命令和卷名均随请求下发。 -- `POST /api/browsers` 接受 `{alias, name, gateway, image_version, fingerprint}`（严格 JSON，未知字段拒绝），校验后先落库，再调网关创建并启动；网关失败时回滚数据库行。`name` 为中文环境名，`alias` 限 `^[a-z0-9][a-z0-9-]{0,31}$`，容器名 `creatorhub-browser-`，Profile 卷 `creatorhub-profile-`。 -- `GET /api/browsers` 合并数据库环境与网关实时状态；环境在网关无容器时显示为未部署。 -- `POST /api/browsers/{alias}/start|stop` 改变状态；`POST /api/browsers/{alias}/upgrade` 收 `{version}`，由平台编排：停止并删除旧容器（保留 Profile 卷）→ 用新镜像引用与原指纹参数重建 → 启动；失败直接重试，不做自动回滚。 -- `DELETE /api/browsers/{alias}` 回收容器并删除数据库行，保留 Profile 数据卷。 +当前生命周期契约集中如下（代码：[控制面](../../cmd/control-plane/hub.go)、[环境存储](../../internal/hub/environment.go)）： + +| 接口/动作 | 当前行为与失败语义 | +| --- | --- | +| `POST /api/browsers` / create | 严格接收 `{alias, name, gateway, image_version, fingerprint, account_id, network_exit_id}`，未知字段拒绝；`account_id` 必填，`network_exit_id` 可空表示明确选择直连。新绑定要求账号 authorized/paused、镜像启用，指定出口须健康并再次核验。先保存环境与稳定 binding，创建停止态容器；不是创建即启动。相同绑定及配置可复用，冲突拒绝；已存在且账号可运行的环境可调和/恢复运行。网关失败不删除已保存环境/binding，保留以供核验/恢复 | +| `GET /api/browsers`、`GET /api/browsers/{alias}` | 只读数据库环境/运行记录；列表的 running 表示已记录运行实例，不保证实时存活。页面可直接展示返回 status，不主动探测或轮询。后台租约调和与列表读取独立 | +| `POST /api/browsers/{alias}/start` | 要求账号可运行；有出口时重新核验，不健康则失败而非直连。当前匹配且就绪的运行容器可复用；停止/缺失容器按当前 binding 重建并激活租约；支持预先选定的直连 | +| `POST /api/browsers/{alias}/stop` | 停止容器并处理租约/清理状态；失败或结果不明可见，不据请求发出即声称已停止 | +| `POST /api/browsers/{alias}/upgrade` | 接收 `{version}`，要求启用镜像；删除旧容器并保留 Profile，更新版本后按原指纹/binding 重建，账号可运行才启动，否则为停止态。无自动回滚；失败保留结果，人工重试进入现有调和流程。当前实现无条件核验出口，空出口的直连环境不能据 create/start 成功推断 upgrade 可用 | +| `POST /api/browsers/{alias}/rebind` | 接收 `{network_exit_id}`；要求 paused、无 executing task、无活动 runtime，验证目标出口后变更。当前要求有效出口 ID，不支持以空值切回直连 | +| `DELETE /api/browsers/{alias}` / recycle | 回收容器、处理运行记录，但保留环境、稳定 binding 与命名 Profile 卷；后续 create/start 复用。不是永久删除账号/环境或素材；没有新增永久删除 API | + +`name` 是环境展示名，`alias` 限 `^[a-z0-9][a-z0-9-]{0,31}$`；容器名 `creatorhub-browser-`，Profile 卷 `creatorhub-profile-`。指纹中的代理字段拒绝，使用所选出口；已绑定代理失败不能静默直连。create/start/stop/upgrade/recycle 写同一 operation ID 的 requested/finished 审计对，网关断连且无法调和时 outcome 为 `unknown`；幂等/重试是生命周期核验，不等于 plan01 中人工/自动业务发送可重发。 + - `GET/POST /api/browser-images` 维护可用镜像版本（版本号不可改，`PUT /{version}` 仅接受 `image_ref/note/enabled`）；仅启用版本可用于创建与升级；被环境引用时拒绝删除。 - `GET/POST /api/gateways` 注册网关（`POST` 可携带令牌，否则平台生成 48 位十六进制令牌并明文存储），`DELETE /api/gateways/{name}` 删除；仍被环境引用时拒绝删除。 -- 启停接受幂等响应，不自动重试未知结果；别名唯一约束由数据库保证。 +- 别名唯一约束由数据库保证；控制面不自动重试结果不明的用户生命周期请求，后台租约恢复的现状另见下文。 ## docker.sock 安全边界 将 socket 以只读文件挂载**不会**限制 Docker API 的写操作；拥有 socket 等价于拥有宿主机 root 权限。因此： -- 只有 `docker-gateway` 挂载 socket，控制面和浏览器容器均不可见；网关加入 control 与 browser 网络，浏览器只拿到无凭据的内存转发代理地址，`/v1` 仍必须通过容器内不可见的网关令牌； +- 只有 `docker-gateway` 挂载 socket，控制面和浏览器容器均不可见；网关加入 control 并按需接入浏览器隔离网络，浏览器在代理模式只拿到无凭据的内存转发代理地址，`/v1` 仍必须通过容器内不可见的网关令牌； - 网关只暴露面向领域的路由，不提供通用 Docker 代理；`/v1` 全部接口校验 `Authorization: Bearer `（常数时间比较），令牌由部署者在网关环境变量与平台注册表中保持一致； - 网关直连的 `POST /v1/browsers/{alias}/start|stop` 仅供内部维护使用，必须提交并精确匹配容器标签中的 `{binding_version,runtime_id,network_id}`；直连 start 拒绝空 `network_id`，generation 不匹配返回 `409`，控制面生命周期编排不依赖无 fence 的直连 start； - 网关恢复内存代理时会同时 fence 隔离网络成员及网关成员 IPv4，地址变化返回 `409` 并关闭刚恢复的监听；代理移除或代际替换会立即关闭所有已 hijack 的 CONNECT 双向连接，任一端先关闭也会关闭隧道两端，不等待优雅 drain； @@ -40,24 +52,17 @@ React ──> control-plane ── /api/browsers ──(Bearer token)──> doc - 控制面发布到宿主机所有网卡；控制网络为固定名称的 Compose 网络；浏览器 bridge 按 ownership、role、driver、Internal 失败关闭校验，且拒绝复用 control 网络； - Compose 基础镜像锁定 digest；浏览器镜像推荐使用 `@sha256:` 摘要引用以获得不可变性，tag 引用由部署者自行把控。 -网关自身一旦被攻破，socket 仍允许接管宿主机；应用内校验不能消除这个平台级风险。开发阶段控制面不做认证或访问限制，安全由部署者自行把控。 +网关自身一旦被攻破，socket 仍允许接管宿主机；应用内校验不能消除这个平台级风险。开发目标不新增认证或访问限制，安全由部署者自行把控；但当前代码仍强制 HTTP Basic Auth（除 `/healthz` 外的 API 与静态页面），Compose 仍要求用户名/密码。见 [main.go](../../cmd/control-plane/main.go) 与 [compose.yaml](../../compose.yaml)；本轮未移除现有策略，也不新增 RBAC 或认证 profile。 ## 运行 -Docker socket 的 GID 因宿主机而异： +使用[部署说明](../deployment.md)中的完整变量及启动命令（含 Basic Auth、凭据主密钥与 socket GID），不维护另一套省略必填配置的命令。首次注册网关和镜像、创建账号、再创建停止态环境；恢复账号后显式启动。网关拉取镜像上限约 10 分钟；失败查看审计和保留的环境，不能按“数据库已回滚”直接假定没有资源。 -```bash -export GATEWAY_TOKEN="$(openssl rand -hex 24)" # 亦可在 .env 中设置 -DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up --build -``` +## 现有账号与阶段 A 离线闭环 -打开 ，局域网内用宿主机 IP 访问同一端口。首次使用：在「网关管理」用 `GATEWAY_TOKEN` 注册 `http://docker-gateway:8081`，在「镜像版本」添加镜像引用（缺失时网关自动拉取，拉取上限 10 分钟）。浏览器容器可访问外网。 +本节描述已运行的基础及历史 schema，不规定 plan01 的新业务范围。旧阶段 A 的 Mock 任务验证不等于 G0 平台能力或 G1 抖音完整验收；当前抖音受限读取连接器也未接成完整竞品/监听/发送流程。 -创建成功但启动失败时，网关会立即删除失败容器并保留命名 Profile 卷，控制面回滚数据库行，允许同名请求安全重试。 - -## 阶段 A 离线闭环 - -`POST /api/phase-a/accounts` 只接受 `{name, platform, platform_account_key, tags, cookies}`；`platform` 限定为 `douyin`、`xiaohongshu`、`wechat-official`、`kuaishou`，`cookies` 必须是浏览器 Cookie Header 格式。控制面通过持久 provider bridge 安全写入凭据：部署侧 Secret Manager/OS Keyring 注入 32 字节主密钥，独立凭据卷只保存 AES-GCM 密文，数据库只记录凭据引用；外部 API 不返回 Cookies、provider 或 `reference_key`。数据库明确回滚时清理凭据，提交结果未知时保留凭据并返回 `account_creation_result_unknown`，不自动破坏可能已提交的账号。内部账号 ID 由服务端生成，新账号默认 `paused`，`(platform, platform_account_key)` 全局唯一。pause/revoke 会递增账号版本并将 queued 任务置为 `policy_hold`，只有具备 binding 和 healthy 出口的未撤销账号才能 resume。账号与浏览器环境通过一对一 `environment_binding` 关联，出口可复用；运行实例保留历史，并以 binding 和外部 runtime id 的部分唯一索引限制活动实例。 +`POST /api/phase-a/accounts` 只接受 `{name, platform, platform_account_key, tags, cookies}`；`platform` 限定为 `douyin`、`xiaohongshu`、`wechat-official`、`kuaishou`，`cookies` 可空，非空时须为浏览器 Cookie Header 格式。控制面通过持久 provider bridge 安全写入凭据：部署侧 Secret Manager/OS Keyring 注入 32 字节主密钥，独立凭据卷只保存 AES-GCM 密文，数据库只记录凭据引用；外部 API 不返回 Cookies、provider 或 `reference_key`。数据库明确回滚时清理凭据，提交结果未知时保留凭据并返回 `account_creation_result_unknown`，不自动破坏可能已提交的账号。内部账号 ID 由服务端生成，新账号默认 `paused`，`(platform, platform_account_key)` 全局唯一。pause/revoke 会递增账号版本并将 queued 任务置为 `policy_hold`，resume 要求稳定 binding，若绑定出口则需 healthy，并满足无活动 runtime/待清理等条件；未绑定出口的显式直连不要求出口记录。账号与浏览器环境通过一对一 `environment_binding` 关联，出口可复用；运行实例保留历史，并以 binding 和外部 runtime id 的部分唯一索引限制活动实例。 草稿经 `POST /api/phase-a/confirmations` 显式确认后才可投递到 `/api/phase-a/tasks`。任务由幂等键去重；`POST /api/phase-a/mock/execute` 使用 `FOR UPDATE SKIP LOCKED` 领取一分钟租约，执行前统一核对账号、草稿和确认版本。缺少确认或版本不一致会进入 `needs_confirmation`，暂停账号或 Mock 策略结果会进入 `policy_hold`，不确定结果与过期租约进入 `needs_confirmation`；这些状态都不会自动重试。`GET /api/phase-a/audit` 只导出账号、确认版本、尝试和结果等非秘密证据。 @@ -65,10 +70,16 @@ DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up --build `POST /api/network-exits` 只接受协议、主机、端口、已有 `credential_reference: {id}` 和预期出口身份；新出口为 `unchecked`，由 `POST /api/network-exits/:id/check` 经实际代理链路变为 `healthy` 或 `unhealthy`，`disable` 不可被检查重新启用。credential reference 的 `reference_key` 不出现在 API、日志或审计中；OS Keyring/Secret Manager bridge 在控制面进程启动前注入 `CREATORHUB_CREDENTIAL_`（大写十六进制），值为请求期解析的 `username:password`，控制面不持久化解析值。 -`POST /api/browsers` 必须同时给出 `account_id` 和 `network_exit_id`。环境创建、启动和升级都会重新检查出口身份，只有 `healthy` 才调用网关；控制面强制下发代理和 `disable_non_proxied_udp`，fingerprint 中的代理字段会被拒绝。显式 `POST /api/browsers/:alias/rebind` 只允许 paused、无 executing task 且无活动 runtime 的账号。`DELETE /api/browsers/:alias` 回收容器但保留稳定 binding、环境和命名 Profile 卷，后续 create 复用它们。create/start/stop/upgrade/recycle 均写共享 operation ID 的 requested/finished 审计对；网关断连且无法调和时 outcome 为 `unknown`。 +环境创建/启动/回收及直连边界以上方生命周期表为唯一说明。已配置代理时由控制面下发代理信息与 `disable_non_proxied_udp`；直连并非代理失败后的替代路径。 解析后的出口凭据只存在于控制面单次请求和网关内存转发器中；Docker inspect、容器环境、标签、挂载、`Config.Cmd` 与进程参数只包含 `docker-gateway` 的无凭据本地代理地址。网关内存代理以 alias、binding version 和 exit ID 共同标识 generation；生命周期操作按 alias 串行，重启恢复或重建必须重新核对该 generation，旧出口代理不能被新容器复用。 -stopped 环境启动时先删除旧容器并确认 runtime lease 释放，再按当前 binding 重建；控制面每 20 秒及列表读取时调和网关，续租 running runtime、释放 stopped/missing runtime，过期 lease 也会在绑定事务中回收。控制面用 PostgreSQL advisory transaction lock 按 alias 协调多副本；每个 Store 最多允许 5 个锁会话占用 10 连接池的一半，为锁内数据库调用保留连接。create 同时锁定账号 ID、alias、请求出口和请求镜像；start、reconcile/rebuild、rebind 和 upgrade 锁定 alias、当前出口及当前镜像（upgrade 还锁目标镜像），拿锁后重新读取出口与镜像版本。账号 pause/resume/revoke 使用账号 ID 与当前 binding alias 加入同一协调域；镜像禁用、引用更新或账号状态变更不能穿透在途生命周期。 +控制面后台每 20 秒调和网关（[runtimeLeaseHeartbeat](../../cmd/control-plane/main.go)），续租 running runtime、释放 stopped/missing runtime；列表/详情读取不触发调和，过期 lease 也会在绑定事务中回收。控制面用 PostgreSQL advisory transaction lock 按 alias 协调多副本；每个 Store 最多允许 5 个锁会话占用 10 连接池的一半，为锁内数据库调用保留连接。create 同时锁定账号 ID、alias、请求出口和请求镜像；start、reconcile/rebuild、rebind 和 upgrade 锁定 alias、当前出口及当前镜像（upgrade 还锁目标镜像），拿锁后重新读取出口与镜像版本。账号 pause/resume/revoke 使用账号 ID 与当前 binding alias 加入同一协调域；镜像禁用、引用更新或账号状态变更不能穿透在途生命周期。 非法 upgrade/rebind 目标在进入 advisory lock key 前按公开格式校验；审计仅保留环境原有的非秘密资源关联，并以 `upgrade_input_rejected` / `rebind_input_rejected` 写同一 operation ID 的 requested/finished 对。reconcile 恢复或重建后会重新读取 context，finished 事件关联实际激活的 runtime instance、binding version 与出口；后台释放 runtime 的成功或失败也写独立的 `reconcile` 审计对。 + +## 与新业务计划的边界 + +- 当前受限抖音读取仅允许自身身份与 `count=20/max_cursor=0` 首批作品，JSON 响应限制为 1 MiB，见 [douyin.go](../../cmd/docker-gateway/douyin.go)。这些是现状，不是竞品分页或媒体下载的完成证据；后续媒体按 plan01 C4 分步保存产物引用，不复用通用 JSON 响应承载二进制。 +- 当前账号凭据入口处理 Cookie；plan01 要求的可选登录密码应允许输入现有凭据保存流程，读取不回显、不进入日志，并非新增“禁止收密码”的接口限制。 +- 新业务平台监听、前端业务推送和现有后台运行租约是三件事；前两者要求见 plan01 A6，列表不主动探测的约定不禁止业务事件推送。现有生命周期/租约可能核验出口，不应误写成已完成 plan01 的手动代理管理目标。 diff --git a/docs/deployment.md b/docs/deployment.md index 0cda0c6..09eaa71 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,6 +1,8 @@ # CreatorHub 部署 -本文档适用于当前阶段 A：在一台 Linux 主机上通过 Docker Compose 部署。控制面使用单用户 HTTP Basic Auth；当前不提供 RBAC 或多租户隔离。 +本文档按 `main@1fbf126` 当前可运行代码说明单台 Linux 主机 Docker Compose 部署及旧阶段 A Mock 检查，不证明新产品功能已实现。[plan01](plan01.md) 是业务范围与验收依据，旧阶段 A 离线结果不等于 G0/G1，也不能替代抖音/小红书最终真机验收。 + +当前控制面仍使用单用户 HTTP Basic Auth（除 `/healthz` 外，包括静态页面），不提供 RBAC 或多租户隔离。开发目标不新增认证/访问限制，但本轮未删除现有代码或配置；以下变量仍须填写，不新增认证 profile。 ## 部署内容 @@ -60,7 +62,8 @@ docker compose up --detach --build 1. 「网关管理」页注册网关：名称如 `gw-main`，Endpoint `http://docker-gateway:8081`，令牌填 `GATEWAY_TOKEN` 的值（即 `openssl rand -hex 24` 生成的值）。 2. 「镜像版本」页添加可用镜像，如版本 `148.0.7778.215`、引用 `git.ipao.vip/rogee/fingerprint-chromium:148.0.7778.215`（或 `@sha256:` 摘要引用）。 -3. 「运行环境」页创建环境：中文名 + 小写别名 + 指纹参数，容器名 `creatorhub-browser-<别名>`，Profile 卷 `creatorhub-profile-<别名>`。 +3. 「社媒账号」页先创建账号（默认暂停），再在「运行环境」选该账号、网关、镜像，填写中文名、小写别名及指纹参数。代理可选；指定代理须先手动检测为健康，留空是明确直连，不是失败回退。 +4. 创建得到停止态容器；在账号页恢复账号后回环境页显式启动。容器名 `creatorhub-browser-<别名>`，Profile 卷 `creatorhub-profile-<别名>`。回收只删除容器、保留环境/binding/Profile；完整契约见[架构说明](architecture/container-control.md)。当前直连 create/start 不代表 upgrade/rebind 已支持空出口。 ## 部署验证 @@ -84,8 +87,7 @@ docker compose ps 健康检查应成功，浏览器列表接口应返回 JSON，迁移查询当前应输出 `1`，三个 Compose 服务应为运行状态。`CREATORHUB_CREDENTIAL_MASTER_KEY` 必须由部署侧 Secret Manager/OS Keyring 持久保存并在每次启动时注入同一值；账号凭据以 AES-GCM 密文写入独立 `creatorhub_credentials` 卷，轮换主密钥前必须先迁移已有凭据。然后访问 ；修改过 `CREATORHUB_PORT` 时使用对应端口。 -若任何持久环境曾运行 PR 中间版本 `a37732c`，发布前先执行 -`SELECT to_regclass(current_schema() || '.account_credential_reconciliation');`。结果非空表示遗留了当前代码不再使用的非秘密对账表；不要手工改写 `schema_migration` 或直接删表。应在确认不再回滚到该中间版本并完成数据库备份后，以单独的前向迁移 v15 删除该表并先在备份副本演练；该清理不影响当前账号创建链路，但属于正式发布前检查项。 +旧实验版本的数据库异常应先记录版本、备份并核实，不把历史文档中的建议迁移当成本期开发要求；本期不新增兼容迁移、回填或双写。下文更新/恢复命令仅描述现有部署的数据操作，不改变 plan01 验收范围。 排障时读取结构化服务日志： @@ -95,11 +97,11 @@ docker compose logs --tail=200 creator-hub docker-gateway postgres ## 手工业务验证（阶段 A Mock） -当前阶段 A 是单用户、虚拟平台 mock 的离线闭环，不连接真实社交平台。最小业务路径是：账号 → 草稿 → 显式确认 → 任务入队 → Mock 执行 → 审计回溯。 +本节的旧阶段 A 业务执行器是 Mock，不向真实社交平台发送；容器、数据库、网关和可选代理则是实际运行资源。最小业务路径是：账号 → 草稿 → 显式确认 → 任务入队 → Mock 执行 → 审计回溯。 页面路径如下： -1. 登录后在「网关管理」注册 GATEWAY_TOKEN。 +1. 登录后在「网关管理」注册 ，令牌必须等于 GATEWAY_TOKEN。 2. 在「镜像版本」添加并启用一个可拉取的 fingerprint-chromium 镜像。 3. 可选：在「网络出口」创建出口并点击「检测」，健康状态必须为「健康」；这里只填写凭据引用 ID，不填写密码、Cookie 或 token。留空则使用网关所在机器的网络出口直连。 4. 在「社媒账号」创建平台为抖音的账号；创建后默认暂停，后续仍可使用 Mock 执行器验证离线闭环。 @@ -213,7 +215,7 @@ printf 'PASS account=%s environment=%s task=%s state=succeeded audit=task_queued 通过标准：/healthz 返回 204；受保护的 /api/browsers 返回 JSON；网关、控制面、PostgreSQL 均为运行状态；运行环境有 runtime_id 和 runtime_instance_id；任务最终为 succeeded，且执行尝试的脱敏证据为 mock_outcome=succeeded；审计至少包含 task_queued、task_claimed、task_finished。重复提交同一确认时应返回同一任务 ID，不应产生第二条任务。 -验证完成后的安全清理（不删除 PostgreSQL 或 Profile 卷）： +验证完成后的容器回收（保留账号、环境/binding、PostgreSQL 数据与 Profile 卷，不是永久删除环境）： ~~~bash api() { @@ -234,11 +236,11 @@ docker compose stop | 手工验证脚本在 `:?` 处退出 | CREATORHUB_PORT、GATEWAY_TOKEN、CONTROL_PLANE_USERNAME、CONTROL_PLANE_PASSWORD、CREATORHUB_CREDENTIAL_MASTER_KEY 是否都已 export | 在启动 Compose 的同一个 shell 中 export 完整变量集；不要只依赖 `.env` 或 Compose 默认值 | | creator-hub 未启动 | docker compose ps、docker compose logs --tail=200 postgres docker-gateway creator-hub | 先确认 PostgreSQL 与网关 health 为 healthy；网关需能访问 /var/run/docker.sock，DOCKER_GID 使用 stat -c '%g' /var/run/docker.sock 的实际值 | | API 返回 401 | curl 是否带 --user CONTROL_PLANE_USERNAME:CONTROL_PLANE_PASSWORD | /healthz 不需要认证，其余 /api/* 需要控制面 Basic Auth | -| /api/browsers 返回 503 或网关不可用 | 网关注册的 Endpoint、令牌与 Compose 的 GATEWAY_TOKEN | Endpoint 在 Compose 网络内应为 | +| 生命周期动作报网关不可用 | 网关注册的 Endpoint、令牌与 Compose 的 GATEWAY_TOKEN | Endpoint 应为 ；令牌必须完全一致。列表/详情只读持久记录，其成功不能证明网关在线 | | 出口一直是 unchecked/unhealthy | 出口协议、主机、端口；控制面容器到代理的连通性；last_check_reason | 先用无认证代理完成最小验证；有认证时只提供已配置的凭据引用，不把认证值放到请求、日志或文档 | -| 创建环境时报 image_unavailable 或拉取超时 | image_ref 格式、镜像架构、Docker daemon 的 registry 登录和网络 | 版本表中的镜像必须可被 Docker daemon 拉取；缺失镜像会由网关按引用拉取，最长约 10 分钟 | +| 创建环境时报 image_unavailable 或拉取超时 | image_ref 格式、镜像架构、Docker daemon 的 registry 登录和网络 | 版本表中的镜像必须可被 Docker daemon 拉取，最长约 10 分钟；已保存环境/binding 不会因网关失败自动删掉，核对记录后按原配置恢复 | | 恢复/入队返回 503 | readiness、GET /api/browsers/、GET /api/network-exits/ | binding_missing、network_exit_unhealthy、runtime_missing 表示固定资源未就绪；先修复出口并启动原环境，不要换出口重试 | -| Mock 执行没有领取任务 | GET /api/phase-a/tasks/ 的 state、hold_reason | POST /api/phase-a/mock/execute 只领取满足账号、确认、健康出口、活动 runtime 和租约条件的 queued 任务；policy_hold/needs_confirmation 不会自动重试 | +| Mock 执行没有领取任务 | GET /api/phase-a/tasks/ 的 state、hold_reason | POST /api/phase-a/mock/execute 只领取满足账号、确认、已绑定出口健康（显式直连无此项）、活动 runtime 和租约条件的 queued 任务；policy_hold/needs_confirmation 不会自动重试 | | 停止 Compose 后浏览器仍在运行 | docker ps --filter 'name=^creatorhub-browser-' | 动态浏览器不由 Compose 管理；先通过「运行环境」停止/回收，再 docker compose stop，不要误删 Profile 卷 | 不要执行 docker compose down --volumes 作为普通排障手段；它会删除 PostgreSQL 数据卷。不要把控制面密码、网关令牌、代理密码、Cookie 或 token 写入仓库、截图、日志或审计查询。 @@ -314,7 +316,7 @@ docker compose config --quiet docker compose up --detach --build ``` -镜像版本升级在页面「运行环境 → 升级」完成：平台会停止并删除旧容器（保留 Profile 卷），用新镜像引用与原指纹参数重建后启动；失败时直接重试即可，无需回滚。 +镜像版本升级在页面「运行环境 → 升级」完成：删除旧容器并保留 Profile，用新镜像与原指纹/binding 重建，账号可运行才启动，否则保持停止态；无自动回滚。失败先查看记录，人工重试由现有流程调和，不承诺所有失败都可直接重试消除。当前 upgrade 无条件要求有效代理出口，直连环境会失败；rebind 也不支持空出口切回直连。此为现状限制，不是新产品范围裁决。 数据库迁移只支持安全前进，不提供自动破坏性回滚。需要同时恢复旧代码和更新前数据库时，修改下面两个变量后**整块执行一次**；不要逐行或拆块执行。预检、恢复演练、动态容器停止、停服、主库恢复、提交切换和启动都位于同一个 fail-fast subshell 中。 diff --git a/docs/plan01.md b/docs/plan01.md new file mode 100644 index 0000000..08d00d4 --- /dev/null +++ b/docs/plan01.md @@ -0,0 +1,461 @@ +# CreatorHub 需求细化与验收标准 + +> 版本：v1.2 · 2026-09-12 +> 状态：业务范围已逐项确认；本文是实现与验收依据，不代表功能已经完成。 +> 现状基线：`feat/plan01-implementation` 基于 `main@1fbf126`，已同步远程；本轮完成一批本地 G1.1/G1.2 实现与离线验证，未完成真实平台验收。 +> 优先级：本需求高于现有实现及旧产品规划；遇到平台能力不足或新的业务歧义，必须向使用者确认，不得自行删减、替换或假装成功。 + +## 1. 目标、范围与完成定义 + +在现有账号、浏览器环境与代理管理基础上，形成四个可实际使用的模块： + +1. **竞品分析**：导入竞品账号，发现新作品，监测指标，筛选爆款，选择素材并生成仿写文案。 +2. **账号管理**：登记实名与登录资料，管理账号状态、大号与小号关系，监听大号收到的互动并按策略响应。 +3. **环境管理**：一账号一环境，自动配置稳定指纹，手工管理与分配代理。 +4. **工作台**：采集自有与竞品账号的一级评论，识别线索，人工回复/私信；展示并操作所选平台账号的私信会话。 + +### 1.1 交付范围 + +- 平台为**抖音、小红书**。分阶段实现：先验证抖音完整流程，再完成小红书；最终交付必须逐平台验收，不能以“能登记小红书账号”替代业务功能。 +- 本文及相关旧文档同步是需求基线；当前工作树已补充一批 CreatorHub 本地实现，但不部署、不操作真实平台账号，也不把离线结果当作平台成功证据。 +- 后续功能交付以第 8 节的可观察结果为准，既要覆盖成功，也要覆盖缺数据、重复操作、失败和恢复。 +- 平台不提供的事件、UID、指标或动作能力属于**待验证的外部依赖**，不是默认排除项。必须给出证据并由使用者决定替代或范围调整；未经确认，该项不能通过验收。 + +### 1.2 统一术语 + +| 名称 | 含义 | +| --- | --- | +| 自有账号 | 在本系统登记、由使用者登录操作的平台账号 | +| 竞品账号 | 通过链接加入监测的目标账号，不要求也不收集其登录凭据 | +| 大号 | 开启大号模式、接收互动并配置响应策略的自有账号 | +| 小号 | 被一个同平台大号选中，用于执行响应动作的自有账号 | +| UID | 平台提供的稳定用户标识；不得用昵称、头像或显示名称代替 | +| 账号状态 | 人工维护的正常、禁言、封禁、注销等业务状态，不等于登录状态或环境状态 | +| 自动响应 | 大号收到互动后，系统按启用策略选择一个小号执行一个动作 | +| 人工发送 | 使用者选择账号、确认目标和内容后发送；与自动响应分开记录 | +| 结果不明 | 已尝试执行，但未获得足以判断成功或失败的证据；不得标为成功或自动补发 | + +## 2. 当前系统与需求差距 + +以下来自代码检查，**不是本轮真实平台验证结论**。路径用于定位现状，不冻结后续实现方式。 + +| 范围 | 已有基础 | 本需求需要补齐或改变 | 代码依据 | +| --- | --- | --- | --- | +| 页面入口 | 账号、任务、审计、环境、代理、镜像、网关等页面 | 竞品池、作品、素材仿写、评论线索、响应策略、私信会话尚无完整入口 | [页面路由](../web/src/main.jsx) | +| 账号 | 平台账号 ID、标签、Cookie、暂停/恢复等 | 实名资料、登录用户名/密码、备注、人工业务状态、自动登录与大小号关系 | [账号页面](../web/src/AccountsPage.jsx)、[账号存储](../internal/phasea/store.go)、[账号接口](../cmd/control-plane/phasea.go) | +| 平台范围 | 登记项还包含公众号、快手 | 新业务仅承诺抖音、小红书；其他登记项不是业务能力验收结果，也不因此要求删除无关已有功能 | [账号页面](../web/src/AccountsPage.jsx) | +| 作品读取 | 有抖音连接器、指标字段与测试 | 连接器只核验登录者自身、读取首批 20 条，未接成运行中的竞品采集链路；需目标账号解析、分页、保存、查询和定时更新 | [抖音连接器](../internal/douyin/connector.go)、[连接器测试](../internal/douyin/connector_test.go)、[网关读取限制](../cmd/docker-gateway/douyin.go) | +| 指纹与环境 | 结构化指纹、独立 Profile、启动/停止/升级及代理接入 | 默认固定 seed 不是自动分配；需首次自动生成、地区匹配及稳定性验证 | [环境页面](../web/src/BrowsersPage.jsx)、[指纹参数](../internal/hub/fingerprint.go)、[环境存储](../internal/hub/environment.go) | +| 代理 | 添加、列表、手动检测、停用、实际转发 | 补齐编辑、删除、重新启用及引用约束；不增加自动轮换 | [代理页面](../web/src/NetworkExitsPage.jsx)、[代理转发](../cmd/docker-gateway/proxy.go) | +| 任务与发送 | 有草稿确认、任务记录、Mock 执行 | 不能作为真实回复、私信、点赞、关注、转发成功的证据；需接通实际平台执行及结果核验 | [任务页面](../web/src/TasksPage.jsx)、[任务接口](../cmd/control-plane/phasea.go) | +| 事件、线索、私信、AI | 未发现完整可运行链路 | 均需新增业务能力，不能把旧文档、类型定义或模拟返回记为已实现 | [页面路由](../web/src/main.jsx)、[抖音连接器](../internal/douyin/connector.go) | + +### 2.1 已确认的冲突裁决 + +- [旧探索方案](product/compliance-product-plan.md)排除了自动评论、自动关注、批量私信等；本文明确纳入**由互动触发、按用户冷却限制的单次自动响应**。不得沿用旧排除项阻止实现，也不得扩展为无差别群发。 +- 旧流程强调每次草稿人工确认；本文的自动响应由人工配置、启用策略后自动执行，工作台与私信页面则逐次人工确认。不能混为一个审批流程。 +- 旧读取代码仅支持登录账号自身与首批作品；本文要求竞品账号和完整范围内分页，后续须接通真实能力，不能只在界面标注支持。竞品仅采公开可访问或已授权的数据，不要求竞品登录，也不以旧“仅自有账号”限制删减竞品范围。 +- 账号实名状态、业务状态、登录状态分别记录；禁止把“授权成功”当作“已实名”，或把一次网络错误当作“封禁”。 +- 自有账号的互动和私信采用**事件监听**，不以定时轮询替代；竞品作品与评论仍按固定计划采集。现有账号/环境列表不主动检测、不轮询实时状态的约定保持不变。 + +### 2.2 文档职责与现状边界 + +- 本文规定目标业务；[旧探索方案](product/compliance-product-plan.md)仅保留历史背景，其阶段 A 离线 Mock 不等于本文 G0/G1。尚未实现是开发差距，不是计划缺陷。 +- [架构说明](architecture/container-control.md)集中记录当前生命周期 API；[部署说明](deployment.md)只验证当前可运行代码，不证明本文四模块已验收。当前控制面仍有 Basic Auth；本期开发目标不新增认证、RBAC 或认证 profile，本轮也不删除现有认证。 + +## 3. 竞品分析 + +### C1. 账号导入与监测管理 + +- 用户输入抖音或小红书账号主页链接；平台可解析的作品分享链接可先解析作者，再展示确认。无法确定作者时要求填写主页链接，不猜测账号。 +- 保存前展示平台、账号昵称、稳定标识、主页链接及可取得的头像；使用者确认后加入监测。缺少稳定标识不得创建成功。 +- 以“平台＋目标账号稳定标识”去重；短链接、长链接指向同一账号时不产生两条监测记录。 +- 支持列表查询、启用/暂停监测及手动更新。暂停只停止后续采集，已采数据保留；不默认扩展批量导入、自动发现相似账号。 +- 首次默认回溯最近 **30 天**作品，回溯天数可在统一设置中修改。采集全部符合范围且平台可提供的作品，不限制为首页或固定 20 条；可展示进度并在中断后继续。 +- 作品以“平台＋作品标识”去重，展示作者、标题/正文、发布时间、原链接、可取得的封面、点赞/评论/转发数及最近采集时间。 +- 不可获取的指标显示“不可获取”，未采集显示“待采集”，不能填 0。平台真实返回 0 则显示 0；指标以当前真实值更新，不强制只能增加。 + +### C2. 两种独立的检查计划 + +| 配置 | 已确认默认值 | 约束 | +| --- | --- | --- | +| 历史回溯天数 | 30 天 | 正整数；定义首次采集范围，不等于数据删除期限 | +| 账号新作品检查间隔 | 30 分钟 | 正数；与作品指标计划分开 | +| 指标初始间隔 | 1 小时 | 正数 | +| 指标间隔倍数 | 2 | 大于 1，作品越旧检查越少 | +| 指标最大间隔 | 24 小时 | 不小于初始间隔 | +| 指标监测年龄 | 发布满 30 天停止 | 正数；停止采集不删除历史数据 | + +- 以上为一组全局设置，不增加每账号覆盖、热度自适应或公式编辑器。 +- 以作品发布时间为起点，间隔为 `min(初始间隔 × 倍数^n, 最大间隔)`，`n` 从 0 开始；累计时刻是发布时间后 **1、3、7、15、31、55、79……小时**。 +- 计算统一使用 UTC 时刻和固定时长，1 天 = 24 小时，不按本地自然日或夏令时换日；界面可转为本地时间但注明时区。首次回溯以开始时刻 `T` 固定窗口 `[T - 回溯时长, T]`，分页期间不滑动窗口。 +- 账号启用时立即检查新作品，以该次启用时刻为固定间隔起点；一级评论补充采集同理。正常到期条件为 `当前时刻 >= 计划时刻`，同一计划点只启动一次；暂停不启动，恢复或重启最多采一次当前结果，再取严格晚于当前时刻的计划点。 +- 首次发现先采一次当前指标，再等待该作品严格晚于当前时刻的计划点；发现时恰好到期，这次当前采集即覆盖该点，不再重复。已经错过的历史时刻不逐次补跑。 +- 派发前再次检查发布年龄；`当前时刻 >= 发布时间 + 监测年龄` 时不再安排自动指标检查（与计划点重合时停止优先）。首次回溯到比监测年龄更旧的作品，仍采初始资料和当前可得指标，但不创建持续指标计划。 +- 暂停、进程重启或短暂中断后，每部作品最多补一次当前采集，不集中回放历史计划；已有作品不重复入库。登录失效、验证挑战等需人工处理的阻断不得靠重试消除。 +- 修改设置后以保存时刻重设账号/评论未来检查起点，作品指标仍以发布时间计算未来点；不重写历史指标时间，也不补造过去的数据。 +- 缺少可靠发布时间或时区、时间无效/位于未来时，资料标为“发布时间待核验”，不猜日期、不把发现时间当发布时间，不纳入已确认回溯结果或安排年龄相关计划；保留待核验项及范围不完整提示。获得可靠时间后按当前窗口及未来计划处理，不回填错过的采集。 + +### C3. 爆款筛选 + +- 支持平台、监测账号、作品发布时间，以及点赞、评论、转发最低值筛选。 +- 只对用户填写的条件进行判断，所有已填写条件**同时满足**才入选；数值阈值为非负数，等于阈值也满足。 +- 未配置某项阈值不限制该项；配置了阈值但对应指标不可获取，则不进入该筛选结果，并说明缺少指标。 +- 不增加系统综合评分、增长预测、AI 爆款推荐或排行榜算法。“爆款”在本期就是满足用户筛选条件的作品。 + +### C4. 素材与仿写：两次人工确认 + +固定流程： + +`标题/正文 → 人工选取素材 → 下载视频、提取音频、语音转写 → 人工确认是否仿写 → 系统生成标题与口播文案` + +- 未被人工选取的作品只获取作品资料，不自动下载媒体、转写或生成文案。 +- 选取后可查看各步骤进度、视频、音频、转写文本及失败原因；下载保留平台可获取文件，不包含去水印、视频画面 OCR、配音合成或剪辑。 +- “语音转写”是对音轨中说话内容的识别，不能拿作品描述冒充；“分析”用于理解现有标题、正文与转写，不额外输出未经要求的复杂分析报告。 +- 无音轨或无可识别语音时明确标注，不编造转写；其他下载/转写失败显示具体失败步骤，不能标成全部素材完成。 +- 素材准备成功后，用户可放弃或填写仿写要求并确认。下载或转写失败时，必须先解决失败步骤才允许仿写，不提供“部分素材继续”分支。已确认无音轨/无说话内容属于明确的素材结果，不等于任务失败，仍须如实展示。 +- 输出仅为**可编辑的标题与口播文案**，保留来源作品、采用的素材及用户要求；不自动发布、不生成视频、不做多代理创作流程。 +- 重复点击同一次选取/确认不得重复启动同一任务。失败允许人工重试失败步骤，已有成功素材不无故重新下载；AI 失败不得用原文或空内容冒充成功。 +- 作品/评论分页与媒体下载是不同链路；任务按下载、提音、转写、生成保存步骤状态和产物引用，页面按需预览，不把媒体二进制塞进通用 JSON 读取响应，也不靠放大当前 JSON 上限冒充媒体支持。最终须用超过一页数据及真实媒体证明可用；当前读取限制只是后续接通点。 +- 空间不足、文件过大或服务配额不足须明确失败步骤、原因和已有产物；未经用户授权不得自动删除素材腾空间，不静默丢弃成功产物。 + +## 4. 账号管理与自动响应 + +### A1. 账号资料与状态 + +| 字段 | 要求 | +| --- | --- | +| 平台 | 抖音或小红书；和平台稳定账号标识共同确定账号身份 | +| 登录账号、密码 | 可选；与平台 UID 分开，密码输入后不由读取接口回显，也不进入列表、日志或错误信息 | +| 实名状态、姓名、身份证 | 人工记录，用于区分账号；不校验平台真实实名、不代办认证、不上传身份证资料 | +| 备注 | 用户补充说明，不用标签代替 | +| 账号状态 | 人工维护正常、禁言、封禁、注销；需要其他状态时先确认，不设计可编程状态系统 | +| 大号模式 | 开关；开启后可管理本大号的小号、响应策略、冷却时间与 AI 回复要求 | +| 登录情况 | 独立展示已登录、需登录、登录失败/待人工处理等实际结果与时间，不推断处罚状态 | + +- 支持账号资料新增、查看、编辑；必填平台及能区分账号的标识，已获得平台 UID 后检查身份重复和登录一致性，不通过改昵称掩盖冲突。 +- 登录密码允许由新增/编辑请求输入并进入现有凭据保存流程，账号资料仅关联凭据；读取只显示是否已配置，编辑空白不覆盖已有密码。当前接口只接收 Cookie，不代表密码能力已实现，也不能以“API 禁止接收密码”阻断本需求。 +- 实名资料只做输入类型、长度等基本校验；不把人工记录包装成已核验身份，不要求用户为运行普通功能提交身份证证据。 +- 正常账号按实际登录及平台能力执行。**非正常账号暂停自动动作**；禁言账号禁止人工评论、私信及带文案转发；封禁、注销账号不参与采集和发送。已有数据仍可查看，人工登录/检查入口保留。 +- 账号恢复正常后需人工重新启用受影响策略，不补发暂停期间的历史互动。登录失败不擅自修改上述业务状态。 + +### A2. 登录 + +- 已存在有效登录时复用；未登录且提供用户名与密码时尝试自动登录，否则直接进入可见浏览器人工登录。 +- 自动登录失败显示真实原因：例如凭据错误、平台不支持该登录方式、验证码/扫码/设备确认、网络失败或页面变化；未知原因明确写“无法确认”，不能猜测。 +- 失败后保留同一环境交用户人工处理，不循环尝试、不自动更换账号/代理/指纹，也不绕过验证码。 +- 登录后核对实际平台账号身份；与选定账号不一致时停止该账号任务并提示纠正，不能把别的账号登录结果保存为成功。 + +### A3. 大小号关系与策略 + +- 一个小号只归属一个**同平台**大号；不能自己关联自己，大号不能再作为其他大号的小号，不建立多层关系。 +- 大号可关联多个小号，并配置有明确先后顺序的策略。每条策略包含：启停、适用事件类型、一个执行小号、**一个动作**及动作所需内容。 +- 事件是大号**收到**评论、点赞、转发、关注等互动，不是跟随大号主动操作。以上四类全部纳入，不因平台获取困难自行删除；其他事件类型须说明含义后确认。 +- 同一事件按顺序找到第一条匹配、账号可用且目标明确的策略执行。较早策略因登录/业务状态/缺少目标而不可用时可检查下一条，并记录原因；一旦选中并开始执行，不再换小号或换动作补发。 +- 大号模式关闭、策略停用、关系调整时停止受影响的未执行动作；关系调整后需人工重新配置并启用。已发送动作不能假装撤回；在途动作实际结果继续保留。 + +| 响应动作 | 目标与必要条件 | +| --- | --- | +| 给用户发私信 | 互动者的稳定 UID；需要文本 | +| 回复评论 | 事件中明确关联的评论；需要文本 | +| 点赞评论或作品 | 策略明确选择评论或作品，事件中存在对应目标标识 | +| 关注互动用户 | 互动者的稳定 UID | +| 转发作品 | 事件中明确关联的作品；平台要求文案时还需文本 | + +所有自动动作都必须另有互动者稳定 UID 供 A5 冷却使用，即使点赞/转发只需作品 ID 也不例外。缺少必要目标时不猜测、不搜索用户的其他作品代替。例如关注事件只有用户而没有评论，不能执行“回复评论”；只有聚合点赞数却没有互动者 UID 时，不能触发面向用户的响应。 + +### A4. 文本来源与 AI + +- 需要文本的动作允许预填多条非空候选文本；每次随机选择其中一条。只有一条时直接使用，不要求轮换不重复。 +- 未填写候选文本时，使用 AI 按**大号统一的回复要求**生成。启用此类策略前必须具备有效 AI 配置及该大号回复要求。 +- AI 输入限于当前事件、相关作品/评论、允许使用的业务说明和回复要求；不新增知识库、长期记忆、多轮自动聊天。 +- AI 失败、内容为空或超出平台限制时停止本次响应并显示原因，不改用隐藏的默认话术、不截断后强发、不自动重发。 +- 用户内容是待分析材料，不应改变大号预设要求、执行账号或目标；保存实际选中文本或生成文本与执行结果的对应关系。 + +### A5. 同一用户只自动响应一次 + +- 冷却范围为**平台＋大号＋互动用户 UID**，该大号下所有小号、所有事件和所有动作共同计算。 +- 大号可配置正数冷却时间，默认 **24 小时**；不能把 0 解释成禁用冷却。 +- 从选中小号、开始执行时占用响应名额并计时。一个 UID 在冷却内不论触发什么事件，都只允许一次自动动作，不执行“关注后再私信”等组合。 +- 冷却检查和占用必须不可被并发穿透；服务重启、监听重连、策略重启、修改候选文本不能清除已占用的冷却。事件身份及永久去重按 A6；冷却到期只允许新事件，不允许旧事件重放。 +- 失败、AI 生成失败、结果不明均保留已经开始执行的冷却，不自动尝试下一个小号。没有匹配策略、账号不可用等尚未开始执行的情况不占用名额。 +- 修改冷却时长只作用于新开始的响应；已占用名额保留其原到期时间，避免修改设置意外提前解锁。 +- **人工回复和人工私信不受自动响应冷却限制**，不拿自动冷却阻止用户逐次确认的发送；必须明确标识人工/自动来源，仍防止同一次提交重复发送。 + +### A6. 事件监听与接近实时 + +- 自有账号登录后由后台监听大号互动和私信新消息；关闭页面不停止后台监听。私信事件更新会话，不触发自动聊天；竞品采集不依赖竞品登录。 +- **事件身份**为“平台＋接收账号＋平台稳定事件 ID”；也可使用经真机验证能唯一定位事件、跨重连/重启仍稳定的游标。普通翻页游标不当然是事件身份；不得用昵称、时间/内容哈希猜唯一。缺少可靠身份属于 G0 阻塞，不启用该类自动响应。 +- 首次监听及每次重新启用先保存已验证的历史基线（平台事件时间边界或可验证的事件位置），基线建立期间不执行自动动作；从确定的新事件边界开始响应，旧通知只作历史展示。平台没有时间标识时必须证明游标/事件位置可区分新旧；两者都没有则标为边界不明，不猜测。 +- 持久保存已接收事件的最小身份、边界归类和处理结论（含未匹配、跳过、失败与结果不明），同一旧事件永不因策略修改、冷却到期或重启再次执行；该最小去重记录不随冷却到期删除，不要求永久保存原始消息全文。 +- 断连、登录失效、解析失败可见；恢复时核对持久基线和平台恢复位置，重复只更新原记录。**迟到/待核验事件**指恢复时取得的断连期间事件，或无法证实发生在持续启用的新事件边界内的事件，不以“晚了几秒”任意划线。默认只记录、不补发自动动作；是否能可靠恢复、缺失范围以及恢复事件是否允许触发，须在 G0 提交平台证据并取得用户批准后明确，不默认授权历史补发。停用期间及启用前事件始终不触发。 +- 进程重启按断连恢复处理，不清除基线、事件去重或冷却；无法确认恢复连续性时显示数据缺口，从重新确认的新边界开始。私信历史可按平台能力恢复显示，但不触发发送；不能以重连成功声称事件无遗漏。 +- **平台→后台的事件监听**与**后台→页面的业务更新**分层验收：前者不得用固定轮询或 Mock 替代；后者采用一种适合现有服务的推送方式即可，不要求 SSE/WS 双实现，不引入新基础设施。禁止账号/环境列表实时状态探测不等于禁止业务事件推送。 +- 监听正常时，从系统收到事件起 **5 秒内开始处理**；已打开且正常连接的相关页面 **30 秒内可见**。自动响应展示接收/处理中，不承诺 30 秒内 AI 或动作完成。页面关闭、离线时不测页面可见时限；重新打开先读取持久结果，再接后续更新，页面断连须可见，不能记成准时展示。 +- 分别记录平台事件时间（若提供）、系统接收、处理开始、已打开页面展示及动作结束；缺少平台时间则来源延迟不可测，不填 0。只有系统处理快不证明平台投递及时；每类事件、UID、恢复边界和接近实时能力须经真机证据验收，缺口交用户裁决而非自行删除。 + +## 5. 环境与代理 + +### E1. 自动配置环境指纹 + +- 一账号一环境，首次创建自动生成不同 seed 及相容参数；沿用现有指纹 Chromium 与独立 Profile，不再要求用户手填所有参数。 +- 语言、时区按所选代理的明确地区匹配；地区未知或不足以唯一判断时显示待确认并要求人工选择，不默默填入猜测地区。 +- 指纹与 Profile 在重启、普通升级后保持；必要手工调整只在环境停止后进行，并显示影响。不每次启动自动换指纹。 +- 指纹必须在真实浏览器中生效，不能只验证保存了 JSON；不同账号不串 Profile 或登录资料。 +- 不承诺“无法识别”“永不封号”，不开发额外反检测评分或新指纹内核。 + +### E2. 代理管理 + +- 支持新增、查看、编辑、删除、停用、重新启用及**手动检测**；支持 HTTP、HTTPS、SOCKS4、SOCKS5 与协议允许的认证配置。 +- 显示名称、协议、地址、端口、启停状态、可取得的地区/出口 IP、最近一次检测时间和结果。历史检测不是实时状态，不自动轮询检测。 +- 一个环境指定一个代理，多个环境可共用；绑定代理时明确使用哪个记录。环境原有的显式直连能力不作为故障替代路径。 +- 更换代理、修改正在被环境使用的连接参数前，必须停止受影响环境；再次启动使用用户明确选定的配置。 +- 被环境引用的代理不能直接删除，先解除或更换。停用后不能用于新启动；被运行环境使用时提示先停环境，避免管理状态与实际连接不一致。 +- 代理失败显示原因，不静默直连、不自动换代理，不增加批量导入、供应商采购、自动轮换或代理池调度。 + +## 6. 工作台 + +### W1. 评论来源与采集 + +- 来源可区分自有账号、已启用的竞品账号；默认覆盖其最近 30 天作品，复用 C2 的回溯天数及 30 分钟更新间隔。 +- 本期**只采集一级评论，不采集楼中楼回复**；即使自有账号事件监听能及时发现评论，也不取消这一范围内的定时补充采集与去重。 +- 分页获取范围内全部平台可提供的一级评论，支持中断后继续；不能以首屏、热门评论或前 N 条宣称完整。 +- 展示来源类型、平台、作品、评论内容、作者 UID/可得昵称、发表时间、采集时间；没有 UID 可查看文字，但不可执行面向该用户的动作。 +- 同一平台评论只保存一个逻辑对象；来源同时命中自有和竞品范围时保留来源关联，不生成重复评论或重复线索。删除、隐藏或平台限制导致不可获取时如实说明。 + +### W2. 主题、关键词及 AI 线索规则 + +每条规则只包含：名称、启停、来源范围、内容主题说明、包含关键词、排除关键词、AI 线索判定说明。 + +执行顺序固定： + +1. **作品主题**：AI 根据作品标题与正文判断是否符合该规则的主题，不先下载整个视频做主题分析。 +2. **评论关键词**：作品主题通过后，评论包含任一包含词且不含任何排除词才通过；词按文本包含匹配，不增加正则表达式编辑器。 +3. **评论 AI 判断**：仅对前两步通过的评论判断是否符合线索要求；全部通过才标记线索。 + +- 关键词采用确定的原文子串匹配：保存时仅去除每个词首尾的普通空格、制表符、CR/LF，空词拒绝；不改变评论原文。区分大小写，不折叠内部空白、不合并全半角、不做 Unicode 规范化。例如 `AI` 不匹配 `ai`，`买 车` 不匹配 `买车`，`é` 不匹配 `e` 加组合重音；包含与排除使用同一规则，不增加可配置归一化系统。 +- 必填主题、至少一个包含词和 AI 判定说明，排除词可空。规则可创建、编辑、启停，禁用规则不再参与新判断。 +- 同一评论命中多条规则时仍为一条线索，展示所有命中规则、关键词和简短判定原因，不复制成多条待联系对象。 +- 显示未分析、非线索、已识别线索、分析失败等真实结果。AI 不可用、结果无法解析或依据不足不能标成“无符合线索”。 +- 规则修改不偷偷重写历史结论；记录当时规则内容，用户可明确对已采评论重新分析，并展示最新结果，不增加复杂版本管理界面。 +- 线索可按来源、平台、账号、规则、时间筛选；本期不增加销售漏斗、CRM、自动打分和跟进任务系统。 + +### W3. 人工回复与私信 + +- 从评论或线索选择发送账号、确认目标，输入/编辑文本后显式发送评论回复或向评论作者发私信。 +- 只能选择同平台且具备相应操作条件的自有账号；执行前核对实际登录身份和目标。平台不允许跨账号回复时明确失败，不假装已发送。 +- 用户人工确认的操作**不受 A5 自动冷却限制**；线索识别本身不自动发送。 +- 成功必须有平台侧可核验证据；未返回确认则标为结果不明，提示先检查原生平台，不能自动重发。 +- 一次人工确认在提交前取得持久操作标识，固定账号、目标和实际文本；双击、网络重试、页面重开与进程重启均查询/提交同一标识，不再生成第二次发送。服务端保证同标识至多执行一次，不能只靠按钮禁用。 +- 结果不明只提供查询/人工核验并记录证据，不将“重试”变成重新发送；确认未发送也不重用原操作触发。用户确需另发时须重新核对账号、目标和文本、明确再次确认，产生新的操作标识。记录两次操作关系及原因，不能把新发送伪装成结果核验。 + +### M1. 私信会话 + +- “登录用户”指当前选择的**已登录平台账号**，不是 CreatorHub 的登录用户名。支持切换账号，展示会话列表、对方、最近消息、时间和消息详情。 +- 详情区分收发方向、文本、时间及发送状态；历史消息按平台可提供范围加载，不承诺恢复已删除、不可见或平台未返回的完整历史。 +- 新消息通过 A6 的监听更新，不需要持续手动刷新；切换账号和查看历史时可主动读取。监听断开时展示状态并保留已有消息。 +- 支持人工输入、确认并发送文本，遵循 W3 的身份、状态、重复提交与结果核验要求，不受自动冷却限制。 +- 不新增群聊管理、附件发送、已读回执、AI 自动聊天。平台收到非文本消息时至少标明消息类型/不支持展示，不当作空白文本或丢失会话。 + +## 7. 必须遵守的最小实现约束 + +### 7.1 保留必要能力，不扩展通用平台 + +- 复用当前 Go 控制面、Docker 网关、数据库、浏览器环境以及 React/Refine/shadcn/ui；先核实已安装依赖能力，再决定是否新增依赖。 +- 为实际需要的采集、监听、执行和 AI 调用分别划清职责，但**不预建插件市场、通用工作流引擎、任意条件编辑器或多代理框架**。 +- 先完成一个平台、少量真实账号的完整流程，不为了假定的海量并发引入微服务拆分、额外 MQ、Redis、分布式调度或多租户系统。确需新增基础设施时说明已有方案不足并确认。 +- 复用已有持久任务与记录能力前，必须确认它能处理真实任务；不能把 Mock 换个名字接到界面。不继续保留已被替代的旧业务代码路径或兼容层。 +- 指数监测用确定的间隔规则；冷却用明确的用户范围；策略用有序单动作列表；线索用固定三步判断。不要实现能表达任意业务的通用规则系统。 +- AI 仅服务于已确认的语音转写/素材理解、仿写、主题与线索判断、无候选文本时的单次响应；配置只满足真实调用，不增加模型市场、自动供应商切换或隐藏降级。 + +### 7.2 不允许以“简单”为由省略 + +- 不能省略真实平台接通、分页、断连可见、身份核对、重复事件去重、并发冷却、人工确认、失败与结果不明区分。 +- 关键记录能关联平台、账号、作品/评论/事件、命中策略或规则、所选小号、人工确认和最终结果；已有任务/日志页面能承载的内容不另建监控系统。 +- 记录排查所需的时间和原因，但不把密码、身份证、Cookie、代理凭据或无关完整私信写入日志。 +- 只读采集的临时中断可按下一计划或人工操作恢复；明确的登录/平台验证阻断需人工解决；已经开始的自动写动作及结果不明动作不自动重试。 +- 同一执行账号的所有写操作（人工回复/私信及自动动作）统一串行协调，不只锁小号自动任务；实际写入前重新核对停用、业务状态、当前登录身份及目标，自动动作另核对关系/策略。等待执行时条件变化即阻止未开始的操作；已开始结果如实保留，不以停止冒充撤销。并发竞争、重启恢复须有可重复测试。 +- 表格、筛选、操作按钮保持克制，沿用现有布局与 RemixIcon；不另造大屏、营销式卡片或装饰性流程编辑器。键盘操作、加载/空/错误/禁用状态必须可用。 + +### 7.3 成熟产品参考的使用边界 + +参考 Hootsuite 的[消息条件自动处理](https://help.hootsuite.com/s/article/automatic-actions)、[统一收件箱](https://help.hootsuite.com/s/article/inbox-faq)及[关键词包含匹配](https://www.hootsuite.com/whats-new/partial-keyword-matching-for-inbox-automations)：采用常规的会话列表、保存文本、有序条件判断和可解释分类，不复制其完整企业功能。 + +这些资料仅作产品行为参考，**不证明抖音/小红书提供相同能力**，也不授权复制源码或界面资产。本次未引入上游代码。 + +### 7.4 页面地图与操作闭环（推荐路由，不代表已实现） + +沿用现有 HashRouter、Layout、表格及抽屉/页签；下列路径省略 `#`。主导航为竞品分析、账号管理、环境管理、工作台；任务/审计保留共用入口，镜像/网关留在现有环境管理入口。设置只含 C2 采集计划与**批准后**的 AI/转写调用配置，不建模型市场。 + +| 页面与推荐入口 | 列表、详情及关键字段 | 编辑、按钮与跳转 | 对应验收 | +| --- | --- | --- | --- | +| 竞品 `/competitors`，详情 `/competitors/:id` | 平台、昵称/稳定标识、主页、监测启停、最近/下次采集、进度和失败；详情用“作品/采集记录”页签 | 导入链接→解析预览→确认保存/取消；启用/暂停、手动更新；账号进入作品列表 | AC-C1、AC-C2、AC-C3、AC-U1 | +| 作品（竞品详情作品页签，跨账号筛选复用同一列表） | C1 字段、C3 筛选、多页加载、已采页/条数及总量未知提示；详情展示原文、指标时间、下一指标计划/停止原因 | 筛选/清空、分页、打开原平台、选取素材；详情抽屉返回保留筛选与页码，进度不以首屏当完成 | AC-C2、AC-C4、AC-C5、AC-C6、AC-U1 | +| 素材（作品详情“素材/文稿”页签） | 选中来源、分步状态、视频/音频预览、转写、无音轨/无语音结果、产物引用与失败原因 | 第一次确认选取后才准备；只重试失败步骤；准备完成后填要求并第二次确认仿写/放弃；标题/口播编辑、保存，保存失败保留输入，无发布按钮 | AC-C7、AC-C8、AC-C9、AC-C10、AC-U2 | +| 账号 `/accounts`、`/accounts/:id` | A1 资料、业务/登录/环境状态分列；详情“资料/登录/大小号与策略/监听记录”页签 | 新增/编辑抽屉、保存/取消；密码只显示已配置；打开绑定的同一浏览器人工登录、重核身份，冲突显示期望/实际标识并停止；跳转环境与任务 | AC-A1、AC-A2、AC-A3、AC-U3 | +| 大小号与策略（账号详情页签） | 同平台小号归属、策略顺序/事件/执行号/单动作、候选文本、大号 AI 要求、冷却时长与占用记录 | 大号开关、关联/解除、策略编辑/启停、上移/下移、保存/取消；文本列表增删；缺 UID/目标/AI 条件显示禁用原因；监听记录可跳转事件任务 | AC-A4、AC-A5、AC-A6、AC-A7、AC-A8、AC-A9、AC-A10、AC-A11、AC-A14、AC-U3 | +| 环境 `/browsers`、`/browsers/:id`；代理 `/network-exits`、`/network-exits/:id` | 账号绑定、指纹、代理地区、历史检测时间和状态；代理引用列表 | 创建/启动/停止/升级/回收保持显式动作；停止后修改、待定地区人工选择；代理增改删/停启/手动检测，引用冲突跳转相关环境。回收不宣称永久删除 | AC-E1、AC-E2、AC-E3、AC-E4、AC-E5、AC-U1 | +| 评论/线索 `/workbench` 的“评论/线索/规则”页签 | W1 字段、W2 筛选、分析状态、命中原因；评论详情展示来源作品与原规则依据 | 规则新增/编辑/启停；明确选择已采评论范围后确认重新分析，展示进度/失败；回复/私信抽屉选择同平台账号、目标/文本预览并确认；跳转作品、账号、任务 | AC-W1、AC-W2、AC-W3、AC-W4、AC-W5、AC-U4 | +| 私信 `/workbench` 的“私信”页签 | 顶部账号选择器、会话列表及消息详情双栏；收发方向/时间/发送结果、历史范围、非文本类型、监听连接状态 | 账号/会话切换隔离请求与草稿；旧请求返回不能覆盖新账号，草稿不带到别的目标；切换有未保存文本时提示保留当前并取消切换或放弃再切换；发送前确认，历史加载/读取失败可重试 | AC-M1、AC-M2、AC-M3、AC-U5 | +| 任务 `/tasks`、`/tasks/:id`，关联 `/audit` | 来源人工/自动、操作/事件标识、账号与目标、策略/规则、实际文本、步骤/时间、证据引用、失败/结果不明 | 由任一业务结果跳转任务，再跳回原作品/评论/会话；只读/素材失败可按规则重试，写结果不明仅“查看平台/核验结果”，新发送需回业务页重新确认 | AC-A12、AC-A13、AC-W5、AC-M3、AC-B1、AC-B2、AC-B3 | +| 设置 `/settings` | 全局采集计划；获批 AI/转写服务、模型/版本、参数和配置记录；秘密仅显示是否配置 | 编辑/保存/取消；校验 C2 边界，保存计划提示只改未来；未批准调用配置不启用相关 AI 动作 | AC-C4、AC-C5、AC-A11、AC-U6 | + +**各页共用交互约定**：首次加载显示加载状态，空数据区别于筛选无结果及读取失败；错误保留已有数据和输入并提供适用的重试。翻页失败停在原页；提交中禁用重复按钮、说明具体禁用原因，不把失败清成空列表。只读重试不触发写入；写入结果不明遵循 W3。表单离开、取消或切换页签前提示未保存内容，可留在原处继续编辑或明确放弃；可保存的资料/文稿先保存成功才离开，发送草稿不能以“保存”代替发送确认。键盘可达、焦点回到触发入口，详情返回保留原列表筛选和分页；无权取得的字段明确不可获取，不能编造默认值。 + +## 8. 验收清单 + +下表是功能验收要求。本轮仅执行本地单元测试、构建和静态诊断，未执行真实平台功能测试。每项记录平台、输入、操作、预期、实际结果和证据；“不支持/待验证”不得填写通过。 + +### 8.1 竞品与素材 + +| 编号 | 场景 | 必须观察到的结果 | +| --- | --- | --- | +| AC-C1 | 主页链接、可解析分享链接、重复链接、无效链接 | 正确确认目标；同账号不重复；无效/身份不明有具体错误，不创建假账号 | +| AC-C2 | 在回溯范围内准备超过一页作品，包含窗口内外作品 | 页间无漏项/重复；范围边界按发布时间正确处理；首批 20 条不是完成标志 | +| AC-C3 | 采集中断、恢复，暂停后到检查时间 | 可继续采集，已完成内容不重复；暂停不再启动新采集，历史数据仍可查看 | +| AC-C4 | 使用可控时钟测试新作品与指标计划 | 新作品默认每 30 分钟检查；指标累计 1/3/7/15/31/55 小时，24 小时封顶，满 30 天停止；两种计划独立 | +| AC-C5 | 首次发现旧作品、服务中断、修改计划 | 即时采一次后按未来时刻执行；不补跑全部错过计划，不制造历史指标 | +| AC-C6 | 阈值以下/等于/以上、缺失指标、真实 0、指标下降 | 等值入选；所有已填条件同时满足；缺值不当 0；真实值正确更新 | +| AC-C7 | 浏览爆款但不选取；选取后不确认仿写 | 前者没有媒体/AI 转写任务；后者只准备素材，不生成仿写稿 | +| AC-C8 | 素材成功、无音轨、转写失败、重复点击 | 视频/音频/真实转写可核验；异常分步显示；一次确认不产生重复任务 | +| AC-C9 | 素材准备后确认或放弃仿写 | 确认后才生成可编辑标题及口播文案；来源、用户要求对应；放弃不调用生成；不发生自动发布 | +| AC-C10 | AI 失败或素材步骤失败 | 失败不冒充成功；下载/转写失败时禁止确认仿写，解决失败后才可确认；无音轨与任务失败明确区分 | + +### 8.2 账号、自动响应与监听 + +| 编号 | 场景 | 必须观察到的结果 | +| --- | --- | --- | +| AC-A1 | 新增/修改资料，区分账号 UID、用户名、实名与业务状态 | 字段分别保存与显示；不伪称平台已实名；凭据不进入日志 | +| AC-A2 | 有/无密码、有效登录、密码错误、验证码、身份不符 | 对应复用/自动尝试/人工登录；错误有原因，不循环尝试、不误认别的账号 | +| AC-A3 | 正常→禁言/封禁/注销→正常 | 执行限制符合 A1；已有数据可查；恢复后不擅自重启策略或补发历史互动 | +| AC-A4 | 自关联、跨平台、重复归属、大小号循环、调整关系 | 非法关系被拒绝；变更停用受影响策略，不产生错误账号动作 | +| AC-A5 | 四类大号收到的互动 | 评论、点赞、转发、关注均用真实事件证明；大号主动操作不误触发；无 UID/目标明确提示 | +| AC-A6 | 多条策略匹配、首条不可用、选中后失败 | 按优先顺序仅执行一个可用策略的单个动作；开始后不换小号补发 | +| AC-A7 | 私信、回复、点赞评论/作品、关注、转发 | 各动作在真实平台确认目标、账号及结果；不相容事件/动作不猜目标，不能用一次私信证明所有动作可用 | +| AC-A8 | 同 UID 多事件、多小号并发及跨动作 | 同一大号冷却内总共只开始一个自动动作；不是每个小号各执行一次 | +| AC-A9 | 不同 UID、不同大号、到期边界、重复事件、重启 | 冷却范围正确；到期可响应新事件；旧事件永不重放；重启不清空冷却 | +| AC-A10 | 失败、结果不明、AI 失败、修改冷却时长 | 已开始的名额保留到原到期；不补发；未开始的无匹配事件不占用冷却 | +| AC-A11 | 多条候选文本、单条文本、空候选、无 AI 配置 | 文本来自候选集；空候选才使用大号 AI 要求；配置不足不能启用；生成失败/空/超长停止 | +| AC-A12 | 监听建立、启用、断连、重连、登录失效 | 新互动被处理；历史通知不触发；断连/缺口可见；重连无重复动作，不用轮询冒充监听 | +| AC-A13 | 监听正常时收到互动与私信 | 收到后 5 秒内开始处理、30 秒内页面可见；平台推送与最终动作耗时单独记录，不掩盖来源延迟 | +| AC-A14 | 关闭大号模式、停用策略、账号状态改变与执行竞争 | 尚未开始的动作被阻止；已开始的实际结果可追踪，不假装撤销或再执行 | + +### 8.3 环境与代理 + +| 编号 | 场景 | 必须观察到的结果 | +| --- | --- | --- | +| AC-E1 | 至少两个账号新建环境、重启、升级 | 自动生成不同 seed；单账号指纹/Profile 保持，真实浏览器参数生效，登录不串号 | +| AC-E2 | 代理地区明确、未知、多时区 | 可确定时正确匹配；不确定时请求人工确认，不静默猜测 | +| AC-E3 | 四类代理按合法配置连接、认证失败、网络失败 | 真实浏览器走选定出口；支持的认证方式有证据；失败不静默直连或轮换 | +| AC-E4 | 编辑、停用、重新启用、删除被引用代理 | 运行引用须先停环境；引用未解除不得删除；重新启动使用明确配置 | +| AC-E5 | 打开账号/环境/代理列表，点击手动检测 | 打开列表不触发实时状态探测或轮询；只在用户明确操作时检测，显示检测时间 | + +### 8.4 评论线索与私信 + +| 编号 | 场景 | 必须观察到的结果 | +| --- | --- | --- | +| AC-W1 | 自有/竞品来源、多页评论、楼中楼、重复来源 | 范围内一级评论无页间遗漏/重复；不采楼中楼；来源正确，缺 UID 禁止定向动作 | +| AC-W2 | 主题不匹配、主题匹配但无关键词、命中排除词、AI 否定/肯定 | 严格按“作品主题→关键词→评论 AI”执行；只有全部通过才生成线索 | +| AC-W3 | AI 异常、规则禁用/修改、显式重新分析、多规则命中 | 失败不当非线索；禁用不再判断；历史依据保留；多规则只产生一个评论线索 | +| AC-W4 | 人工回复/私信，自动冷却尚未到期 | 人工确认仍可发送；线索出现不自动发送；账号、目标、文本及平台结果一致 | +| AC-W5 | 双击发送、错账号、超时、平台拒绝 | 一次操作至多发送一次；身份不符拒绝；超时不明不自动重发；平台失败如实显示 | +| AC-M1 | 切换两个已登录账号、查看列表与消息详情 | 会话按所选账号隔离；收发方向、时间、历史范围正确，不串消息 | +| AC-M2 | 收到新文本、非文本、断连后重连 | 文本接近实时更新；非文本有类型提示；断连可见、重连不重复，不自动回复 | +| AC-M3 | 会话内人工文本发送、禁言账号、结果不明 | 正常账号人工确认后发送；禁言被阻止；证据不足保持结果不明、不补发 | + +### 8.5 新增页面与边界验收（原 37 项编号及含义保留） + +| 编号 | 场景 | 必须观察到的结果 | +| --- | --- | --- | +| AC-U1 | 页面地图列表/详情跳转，加载/空/错误/禁用，分页失败及返回 | 导航可达；字段、进度、计划和原因可读；错误不当空数据；筛选/页码保留，不触发状态探测 | +| AC-U2 | 素材两次确认、分步失败/重试、无语音、文稿保存失败及离开 | 两道确认独立；成功产物复用；无音轨/无语音可继续；保存失败保留编辑，未保存离开有提示 | +| AC-U3 | 账号编辑、密码配置、人工登录、身份冲突、策略排序与监听记录 | 资料保存准确；密码可输入而不回显/泄露；同环境处理登录；策略顺序可见可保存，身份冲突阻止执行 | +| AC-U4 | 规则编辑、显式选择范围重新分析，查看线索并人工确认 | 原依据可查；新分析范围/进度/失败可见；线索不自动发送，确认中账号与目标明确 | +| AC-U5 | 两账号/会话切换，草稿未保存、旧请求迟返、非文本/断连 | 消息与草稿不串号；离开可取消/放弃；旧响应不覆盖新视图，非文本及断连可见 | +| AC-U6 | 修改设置、未批准 AI 配置、表单校验/保存失败 | 只含批准范围；未批准不调用；无效时间拒绝，失败保留输入且未来计划不误改 | +| AC-B1 | 缺事件身份、首次历史、缺时间、断连/停用/重启、冷却到期旧事件 | 按 A6 基线与永久去重；不猜唯一、不擅自补发，缺口/迟到可见；缺 UID 的所有自动动作均阻止 | +| AC-B2 | 人工/自动同账号并发、等待中停用/换身份/目标变化、发送后响应丢失 | 写操作不串扰且执行前重核；同操作跨重启不再发送；不明仅核验，新发送另行确认 | +| AC-B3 | 后台监听时关页、打开正常页面、页面独立断连再打开 | 关页不停止监听；5 秒处理与正常已开页面 30 秒可见分别有时间证据；重开恢复持久结果，来源延迟不可测不填 0 | +| AC-B4 | UTC 跨夏令时、恰好计划/回溯/30 天边界、缺发布时间 | 按 C2 固定时长及包含边界；到期一次、停止优先、不猜时间、不回填历史 | +| AC-B5 | 大小写、内部空白、全半角、组合字符、空关键词 | 严格按 W2 原文子串匹配与保存规则，可用确定样例重现，不隐含规范化 | +| AC-B6 | 多页读取、真实媒体产物预览、磁盘/配额不足 | 读取和媒体分离且有真实证据；资源不足分步失败可查，不未经授权删除素材 | + +### 8.6 AI 质量与真实平台门槛 + +- 在相关功能开发前，与使用者共同选取脱敏的主题/非主题、包含词/排除词、线索/非线索、边界评论样本，逐条标注预期。硬性三步执行、停用、失败和禁发规则必须全部通过，不能交给 AI 自行决定。 +- 转写样本包含清晰语音、背景音乐、无语音；用户核对关键内容及缺失提示。仿写样本核对标题/口播结构、用户要求、来源关联和可编辑性；自动文本样本核对大号回复要求、目标与平台长度限制。 +- 已确认采用**使用者批准的样本逐项验收**；AI 语义准确率、转写误差率和创作质量尚无量化阈值，不编造 95% 等承诺。在相关功能开发前共同标注样本与可接受误差，再确定质量门槛；未经样本确认不能将 AI 质量标为已验收。 +- 每个平台至少使用一个大号、两个小号及可控互动账号验证关系和冷却；竞品/评论样本需包含超过一页数据。只读样本与可控写入样本分别准备，不向无关真实用户发送测试消息。 +- 为每个平台填写下方能力证据表；全部未接通项均保持未通过。 +- 本期不承诺任意规模吞吐或永久不受平台限制；生产规模需要真实使用数据后再确认，不预先添加容量平台。 + +### 8.7 逐平台能力证据矩阵模板 + +状态仅为**待验证 / 支持 / 条件支持 / 不支持**；支持不等于验收已通过，仍须有对应 AC 结果。每个分项逐平台记录测试日期、客户端/平台版本、授权样本、复现操作、脱敏证据位置及限制；条件支持/不支持不得自动删项，范围改变另记用户批准。下表为初始登记，**无真机结论**；复合行执行时按行内分项各填一行，不以某一事件/动作成功代表全部。 + +| 能力（须逐分项记录） | AC 关联 | 抖音状态 | 小红书状态 | 证据 | 限制 | 用户变更批准 | +| --- | --- | --- | --- | --- | --- | --- | +| 主页/分享链接解析、稳定账号标识 | AC-C1 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 作品/一级评论全部分页、发布时间/窗口 | AC-C2、AC-W1、AC-B4 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 点赞数、评论数、转发数 | AC-C6 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 视频下载、音频提取、无语音结果 | AC-C8、AC-C10、AC-B6 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 登录复用、用户名密码、同环境人工登录/身份核对 | AC-A2、AC-U3 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 评论、点赞、转发、关注事件各自的 ID/UID/目标/时间 | AC-A5、AC-A12、AC-B1 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 基线、断连恢复/缺口、游标稳定性、迟到分类、投递延迟 | AC-A9、AC-A12、AC-A13、AC-B1、AC-B3 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 私信、回复评论、点赞评论/作品、关注、转发动作及成功核验证据 | AC-A7、AC-W4、AC-W5、AC-B2 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 私信账号隔离、历史范围、文本/非文本监听、人工文本发送 | AC-M1、AC-M2、AC-M3、AC-U5 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | +| 指纹/Profile 稳定、代理实际出口及认证方式 | AC-E1、AC-E2、AC-E3 | 待验证 | 待验证 | 待提供 | 待核实 | 无 | + +## 9. 实现顺序与检查要求 + +| 阶段 | 可交付结果 | 通过条件 | +| --- | --- | --- | +| G0：证据与输入冻结 | 抖音能力矩阵、事件身份/基线/恢复边界、AI 验收包 | 缺可靠事件身份/UID/目标/监听证据即阻塞相应自动能力；不以 Mock 通过。冻结脱敏样本及逐条预期、允许误差、服务商/模型版本/参数/提示要求等配置记录；服务商、样本、质量阈值或费用未获批准仍为前置待确认，不作已批准声明 | +| G1.1：抖音账号与环境 | 账号资料、凭据、同环境登录/身份核对、稳定指纹与代理 | AC-A1、AC-A2、AC-A3、AC-E1、AC-E2、AC-E3、AC-E4、AC-E5、AC-U3 的相关部分独立验证 | +| G1.2：抖音只读采集 | 竞品导入、作品/一级评论分页、计划与筛选 | AC-C1、AC-C2、AC-C3、AC-C4、AC-C5、AC-C6、AC-W1、AC-U1、AC-B4 | +| G1.3：抖音媒体与 AI | 两次素材确认、文稿编辑保存、主题/关键词/线索及重新分析 | AC-C7、AC-C8、AC-C9、AC-C10、AC-W2、AC-W3、AC-U2、AC-U4、AC-U6、AC-B5、AC-B6；批准样本逐项核验 | +| G1.4：抖音人工动作 | 评论/线索逐次确认、真实执行、任务证据与不明核验 | AC-W4、AC-W5、AC-B2 的人工部分；私信动作先验证发送，不以此代替完整会话 | +| G1.5：抖音监听与自动响应 | 大小号、有序五动作、四事件、AI 候选、冷却与恢复 | AC-A4、AC-A5、AC-A6、AC-A7、AC-A8、AC-A9、AC-A10、AC-A11、AC-A12、AC-A13、AC-A14、AC-B1、AC-B2、AC-B3；补齐人工/自动竞争证据 | +| G1.6：抖音私信及完整回归 | 账号隔离会话、历史/非文本、新消息、人工文本发送 | AC-M1、AC-M2、AC-M3、AC-U5；G1 全部片段及第 8 节所有抖音适用项通过后才称抖音完整完成 | +| G2：小红书完整流程 | 先独立重复 G0 证据核实，再按 G1.1 至 G1.6 同范围验证 | 每项取得小红书证据；差异须用户批准，不以抖音、账号登记或部分功能替代 | +| G3：整体回归 | 两平台共存、错误恢复、无串号及无重复发送 | 所有适用项有证据，未通过项清零或经用户明确调整需求；分段验收不替代最终完整范围 | + +后续代码变更遵循 [AGENTS.md](../AGENTS.md)： + +- 非平凡行为先写能在未实现时失败的最小回归测试，单元测试覆盖率至少 65%；尤其覆盖时间边界、并发冷却、监听重复、平台失败及人工/自动区别。 +- 后端通过 `go test ./...`、`go vet ./...`，并构建 `./cmd/control-plane`、`./cmd/docker-gateway`；并发、生命周期与共享状态变更运行 `go test -race ./...`；Compose 变更运行 `docker compose config --quiet`。 +- 前端从 lockfile 安装、通过非交互测试及 `npm --prefix web run build`；关键页面覆盖加载、空数据、错误、禁用、确认/取消和重复点击。 +- 离线测试只证明本地逻辑，真实平台必须另外验收；请求返回 200、任务入队、模拟连接器成功均不能代替平台动作成功。 + +### 9.1 本轮文档完成标准 + +- 原始四模块需求均有对应细化、边界及验收项；逐项确认的业务选择无遗漏、无相反表述。 +- 区分现状、目标、已确认规则与待真实验证的外部条件，不将建议数值或 AI 质量写成未经确认的承诺。 +- 相对链接有效、验收编号唯一、Markdown 检查通过；没有功能代码改动，原有未提交修改保留。 + +### 9.2 后续必须确认而不得猜测的事项 + +业务流程已按本轮对话确定。以下依赖真实验证或尚未提供的材料，**不阻塞本文交付，但阻塞相应功能最终验收**： + +1. 两平台每类事件是否提供可靠事件身份、互动者 UID、明确目标、历史基线和接近实时投递；断连恢复范围及迟到事件处理须按 A6/G0 验证并批准，无法提供时由使用者决定该项替代或范围调整。 +2. 平台未提供的指标、下载能力、动作限制、账号密码登录方式及历史消息范围，须逐项给证据，不能通用地写成“尽力支持”。 +3. 使用者指定/批准的 AI、转写服务及调用配置、脱敏评价样本与质量门槛；不擅自选择付费服务或上传整库资料。 +4. 遇到本文之外的事件类型、业务状态、自动聊天、批量发送、视频生成或规模要求，重新确认需求，不顺带扩展。 + +### 9.3 本轮评审处理表 + +| 问题或评审说法 | 修复位置 | 处理结论 | +| --- | --- | --- | +| 旧“仅自有/阶段 A 离线”覆盖新范围，架构栈/生命周期互相矛盾 | 2.1、2.2；旧产品首页、架构、部署 | 历史与目标分开，当前代码事实单独说明，不新增永久删除 API 或认证方案 | +| 功能尚未实现即计划缺陷 | 第 2 节、第 9 节 | 评审误判：属于实现差距，不用 Mock 冒充完成，也不删需求 | +| 无音轨/无语音必须重新决定；API 应禁止接收密码 | C4、A1、AC-U2、AC-U3 | 评审误判：已确认可继续；密码允许输入凭据流程，禁止回显/日志泄露，不阻断录入 | +| 页面不够可执行，缺跳转与异常状态 | 7.4、8.5 | 紧凑页面地图与共用交互，优先现有页签/抽屉，无新通用系统 | +| 去重/冷却、基线/断连、人工重试与并发边界不足 | A3、A5、A6、W3、7.2、AC-B1 至 AC-B3 | 可靠身份、永久最小去重、迟到核验、持久操作标识及同账号写协调 | + +### 9.4 当前本地实现状态 + +- 已有：CreatorHub 数据表与 API、账号/大小号关系约束、规则与线索判断、作品/一级评论分页采集边界、断点租约、来源关联、指数指标计划、抖音自有/竞品只读采集调度，以及工作台的错误/重试/确认状态。 +- 下一轮 TODO：真实平台事件监听与断连恢复；抖音实际点赞/评论/关注/转发/私信写入；素材真实下载/提音/转写；小红书采集与动作；生产 AI/转写供应商、模型、密钥与质量样本；以及对应的 G0/G1 真机证据。 +- 本地限制：仓库默认未配置 `CREATORHUB_POSTGRES_TEST_URL`；本轮使用运行中的 PostgreSQL 完成了 CreatorHub 数据行为、来源关联、指标、操作、消息和 checkpoint 租约集成测试，覆盖率为 65.6%。CI 或后续环境需提供专用测试库以重复该验证。 +| 时间/文本匹配不确定，媒体读取限制误当需求缺陷 | C2、W2、C4、AC-B4 至 AC-B6 | 固定 UTC 时长、原文匹配、分步产物引用；真实分页/媒体另验，不预设扩容或自动删素材 | +| AI 输入未冻结、平台能力无证据模板、G1 过大 | 8.6、8.7、第 9 节 | G0 保留外部批准前置，矩阵全为待验证；G1 分段可验但不替代最终完整流程 | diff --git a/docs/product/compliance-product-plan.md b/docs/product/compliance-product-plan.md index 211acba..3cff3b5 100644 --- a/docs/product/compliance-product-plan.md +++ b/docs/product/compliance-product-plan.md @@ -1,6 +1,9 @@ # CreatorHub 内部技术探索方案 -> 状态：已入库，阶段 A 按单用户范围验证 +> 状态：历史探索记录，不再作为当前产品范围与验收依据。 +> 当前权威入口：[需求细化与验收标准](../plan01.md)。冲突以该文档及使用者最新确认为准；以下“本轮”“当前”均指 2026-08-27 的旧探索阶段。 +> 旧阶段 A 仅离线 Mock，不等于新计划 G0/G1；旧“仅自有账号”、自动动作排除项和逐次草稿确认不限制新计划中的公开可访问/已授权竞品采集及按策略自动响应。人工发送仍逐次确认。 +> 本文中的 RBAC、Secret Manager 等历史设想不构成本期新增系统要求；实际部署与现有认证见[部署说明](../deployment.md)，不因新开发目标而宣称旧认证已移除。 > 日期：2026-08-27 > 上游参考快照：`3441293738/creatorhub@11df1389ee557169bb2381e5ea08de96f14f80c7` @@ -46,7 +49,7 @@ CreatorHub 是一个**内部、非公开**的多账号运行环境与运营任 P0 是首轮验证必需，P1 仅在 P0 通过后开展。 | 领域 | P0 | P1 | 本轮停止线 | -|---|---|---|---| +| --- | --- | --- | --- | | 运行环境 | 每账号独立 Profile；运行时版本固定；单实例占用锁；环境变更留痕；浏览器崩溃可见 | 升级灰度、备份恢复、跨 OS 兼容矩阵 | Profile 串用、版本漂移、设备校验或未知挑战页 | | 网络出口 | 账号稳定绑定；连通性与认证观测；出口失效时禁止静默直连；凭据脱敏 | 容量、延迟、可用率和成本统计 | 出口地址/地区异常变化、认证失败、网络策略不明确 | | 账号池 | 自有/授权标记；平台账号唯一键；凭据只存引用；状态和撤销 | 标签、归档、登录态健康 | 授权不明、账号映射冲突、凭据进入日志 | @@ -97,7 +100,7 @@ Secret Manager / OS Keyring 只保存凭据，数据库仅存引用。 ### 高层威胁与控制 | 威胁 | 可观测现象 | 系统控制 | -|---|---|---| +| --- | --- | --- | | Profile 或账号串用 | 当前账号标识与任务账号不一致 | 执行前双重校验、每账号锁、立即停机 | | 凭据泄露 | 日志/错误/导出中出现敏感字段 | Secret 引用、结构化脱敏、敏感字段扫描 | | 环境兼容性漂移 | 浏览器版本、启动结果、页面能力或登录态变化 | 固定版本、记录变更、按兼容矩阵回归；不猜测检测机制 | diff --git a/internal/creator/accounts.go b/internal/creator/accounts.go new file mode 100644 index 0000000..4239203 --- /dev/null +++ b/internal/creator/accounts.go @@ -0,0 +1,291 @@ +package creator + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" + "unicode/utf8" +) + +func (s *Store) EnsureAccountProfile(ctx context.Context, accountID string) error { + if strings.TrimSpace(accountID) == "" { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO creator_account_profile (account_id) VALUES ($1) + ON CONFLICT (account_id) DO NOTHING`, accountID) + return databaseError(err) +} + +func accountProfileQuery() string { + return ` + SELECT a.id, a.name, a.platform, a.platform_account_key, + a.authorization_status, a.status, + p.login_username, p.password_configured, p.real_name_status, + p.real_name, p.identity_number, p.note, p.business_status, + p.big_account, p.reply_requirements, p.login_status, p.login_reason, + p.login_checked_at, p.cooldown_seconds, p.updated_at + FROM social_account a + JOIN creator_account_profile p ON p.account_id = a.id + WHERE a.id = $1` +} + +func scanAccountProfile(scanner interface{ Scan(...any) error }) (AccountProfile, error) { + var result AccountProfile + var checkedAt sql.NullTime + if err := scanner.Scan( + &result.ID, &result.Name, &result.Platform, &result.PlatformAccountKey, + &result.AuthorizationStatus, &result.RuntimeStatus, + &result.LoginUsername, &result.PasswordConfigured, &result.RealNameStatus, + &result.RealName, &result.IdentityNumber, &result.Note, &result.BusinessStatus, + &result.BigAccount, &result.ReplyRequirements, &result.LoginStatus, &result.LoginReason, + &checkedAt, &result.CooldownSeconds, &result.UpdatedAt, + ); err != nil { + return AccountProfile{}, err + } + result.LoginCheckedAt = nullableTime(checkedAt) + return result, nil +} + +func (s *Store) GetAccountProfile(ctx context.Context, accountID string) (AccountProfile, error) { + if err := s.EnsureAccountProfile(ctx, accountID); err != nil { + return AccountProfile{}, err + } + result, err := scanAccountProfile(s.db.QueryRowContext(ctx, accountProfileQuery(), accountID)) + return result, rowError(err) +} + +func (s *Store) ListAccountProfiles(ctx context.Context) ([]AccountProfile, error) { + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO creator_account_profile (account_id) + SELECT id FROM social_account + ON CONFLICT (account_id) DO NOTHING`); err != nil { + return nil, databaseError(err) + } + rows, err := s.db.QueryContext(ctx, strings.Replace(accountProfileQuery(), "WHERE a.id = $1", "ORDER BY a.created_at DESC, a.id", 1)) + if err != nil { + return nil, fmt.Errorf("list creator account profiles: %w", err) + } + defer rows.Close() + profiles := make([]AccountProfile, 0) + for rows.Next() { + profile, err := scanAccountProfile(rows) + if err != nil { + return nil, fmt.Errorf("decode creator account profile: %w", err) + } + profiles = append(profiles, profile) + } + return profiles, rows.Err() +} + +func validateProfileUpdate(input AccountProfileUpdate) error { + if input.RealNameStatus != "unknown" && input.RealNameStatus != "not_real_name" && input.RealNameStatus != "recorded" { + return ErrInvalid + } + if input.BusinessStatus != "normal" && input.BusinessStatus != "muted" && input.BusinessStatus != "banned" && input.BusinessStatus != "deleted" { + return ErrInvalid + } + if input.CooldownSeconds <= 0 || utf8.RuneCountInString(input.LoginUsername) > 255 || + utf8.RuneCountInString(input.RealName) > 100 || utf8.RuneCountInString(input.IdentityNumber) > 64 || + utf8.RuneCountInString(input.Note) > 1000 || utf8.RuneCountInString(input.ReplyRequirements) > 4000 { + return ErrInvalid + } + return nil +} + +func (s *Store) UpdateAccountProfile(ctx context.Context, accountID string, input AccountProfileUpdate) (AccountProfile, error) { + input.LoginUsername = strings.TrimSpace(input.LoginUsername) + input.RealName = strings.TrimSpace(input.RealName) + input.IdentityNumber = strings.TrimSpace(input.IdentityNumber) + input.Note = strings.TrimSpace(input.Note) + input.ReplyRequirements = strings.TrimSpace(input.ReplyRequirements) + if err := validateProfileUpdate(input); err != nil { + return AccountProfile{}, err + } + if err := s.EnsureAccountProfile(ctx, accountID); err != nil { + return AccountProfile{}, err + } + + passwordConfigured := false + var secret SecretReference + var secretKey string + if input.Password != "" { + if s.secrets == nil { + return AccountProfile{}, ErrUnavailable + } + secret = SecretReference{ID: accountID + "-password", Provider: "os_keyring"} + secretKey = "creatorhub/" + accountID + "/password" + if err := s.secrets.Store(ctx, secret, secretKey, input.Password); err != nil { + return AccountProfile{}, fmt.Errorf("store account password: %w", err) + } + passwordConfigured = true + } + _, err := s.db.ExecContext(ctx, ` + UPDATE creator_account_profile + SET login_username = $2, password_configured = CASE WHEN $3 THEN true ELSE password_configured END, + real_name_status = $4, real_name = $5, identity_number = $6, + note = $7, business_status = $8, big_account = $9, + reply_requirements = $10, cooldown_seconds = $11, updated_at = now() + WHERE account_id = $1`, accountID, input.LoginUsername, passwordConfigured, input.RealNameStatus, + input.RealName, input.IdentityNumber, input.Note, input.BusinessStatus, input.BigAccount, + input.ReplyRequirements, input.CooldownSeconds) + if err != nil { + if passwordConfigured { + if deleteErr := s.secrets.Delete(ctx, secret, secretKey); deleteErr != nil { + return AccountProfile{}, fmt.Errorf("save account profile: %w; remove password after failure: %v", databaseError(err), deleteErr) + } + } + return AccountProfile{}, databaseError(err) + } + return s.GetAccountProfile(ctx, accountID) +} + +func (s *Store) RecordLoginResult(ctx context.Context, accountID, status, reason, actualKey string) (LoginResult, error) { + status = strings.TrimSpace(status) + reason = strings.TrimSpace(reason) + if status != "logged_in" && status != "needs_login" && status != "failed" && status != "manual_required" { + return LoginResult{}, ErrInvalid + } + if len(actualKey) > 128 || utf8.RuneCountInString(reason) > 1000 { + return LoginResult{}, ErrInvalid + } + profile, err := s.GetAccountProfile(ctx, accountID) + if err != nil { + return LoginResult{}, err + } + if status == "logged_in" && strings.TrimSpace(actualKey) != "" && actualKey != profile.PlatformAccountKey { + status = "manual_required" + reason = "登录身份与登记账号不一致" + } + now := time.Now().UTC() + _, err = s.db.ExecContext(ctx, ` + UPDATE creator_account_profile + SET login_status = $2, login_reason = $3, login_checked_at = $4, updated_at = $4 + WHERE account_id = $1`, accountID, status, reason, now) + if err != nil { + return LoginResult{}, databaseError(err) + } + return LoginResult{AccountID: accountID, Status: status, Reason: reason, ActualKey: actualKey, CheckedAt: now}, nil +} + +func (s *Store) SetBigAccount(ctx context.Context, accountID string, enabled bool) (AccountProfile, error) { + if accountID == "" { + return AccountProfile{}, ErrInvalid + } + if err := s.EnsureAccountProfile(ctx, accountID); err != nil { + return AccountProfile{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return AccountProfile{}, fmt.Errorf("begin creator big-account update: %w", err) + } + defer tx.Rollback() + if enabled { + var isSmall bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM creator_relation WHERE small_account_id=$1)`, accountID).Scan(&isSmall); err != nil { + return AccountProfile{}, databaseError(err) + } + if isSmall { + return AccountProfile{}, ErrConflict + } + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_account_profile SET big_account = $2, updated_at = now() WHERE account_id = $1`, accountID, enabled); err != nil { + return AccountProfile{}, databaseError(err) + } + if !enabled { + if _, err := tx.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1`, accountID); err != nil { + return AccountProfile{}, databaseError(err) + } + } + if err := tx.Commit(); err != nil { + return AccountProfile{}, fmt.Errorf("commit creator big-account update: %w", err) + } + return s.GetAccountProfile(ctx, accountID) +} + +func (s *Store) ListRelations(ctx context.Context, bigAccountID string) ([]Relation, error) { + query := `SELECT big_account_id, small_account_id FROM creator_relation ORDER BY created_at, small_account_id` + args := []any{} + if bigAccountID != "" { + query = `SELECT big_account_id, small_account_id FROM creator_relation WHERE big_account_id = $1 ORDER BY created_at, small_account_id` + args = append(args, bigAccountID) + } + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Relation, 0) + for rows.Next() { + var relation Relation + if err := rows.Scan(&relation.BigAccountID, &relation.SmallAccountID); err != nil { + return nil, err + } + result = append(result, relation) + } + return result, rows.Err() +} + +func (s *Store) SetRelation(ctx context.Context, bigAccountID, smallAccountID string, enabled bool) error { + if bigAccountID == "" || smallAccountID == "" || bigAccountID == smallAccountID { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin creator relation: %w", err) + } + defer tx.Rollback() + var bigPlatform, smallPlatform string + if err := tx.QueryRowContext(ctx, `SELECT platform FROM social_account WHERE id = $1`, bigAccountID).Scan(&bigPlatform); err != nil { + return rowError(err) + } + if err := tx.QueryRowContext(ctx, `SELECT platform FROM social_account WHERE id = $1`, smallAccountID).Scan(&smallPlatform); err != nil { + return rowError(err) + } + if !ValidatePlatform(bigPlatform) || bigPlatform != smallPlatform { + return ErrInvalid + } + var bigEnabled bool + if err := tx.QueryRowContext(ctx, `SELECT COALESCE((SELECT big_account FROM creator_account_profile WHERE account_id=$1), false)`, bigAccountID).Scan(&bigEnabled); err != nil { + return databaseError(err) + } + if enabled && !bigEnabled { + return ErrConflict + } + if enabled { + var smallIsBig, bigIsSmall bool + if err := tx.QueryRowContext(ctx, `SELECT COALESCE((SELECT big_account FROM creator_account_profile WHERE account_id = $1), false)`, smallAccountID).Scan(&smallIsBig); err != nil { + return rowError(err) + } + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM creator_relation WHERE small_account_id=$1)`, bigAccountID).Scan(&bigIsSmall); err != nil { + return databaseError(err) + } + if smallIsBig || bigIsSmall { + return ErrConflict + } + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_relation (big_account_id, small_account_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, bigAccountID, smallAccountID); err != nil { + return databaseError(err) + } + } else { + if _, err := tx.ExecContext(ctx, `DELETE FROM creator_relation WHERE big_account_id = $1 AND small_account_id = $2`, bigAccountID, smallAccountID); err != nil { + return databaseError(err) + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_strategy SET enabled=false, updated_at=now() WHERE big_account_id=$1 AND execution_account_id=$2`, bigAccountID, smallAccountID); err != nil { + return databaseError(err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit creator relation: %w", err) + } + return nil +} + +func (s *Store) AccountWriteCheck(ctx context.Context, accountID string, automatic bool, action string) (AccountProfile, error) { + profile, err := s.GetAccountProfile(ctx, accountID) + if err != nil { + return AccountProfile{}, err + } + return profile, CanWrite(profile, automatic, action) +} diff --git a/internal/creator/actions.go b/internal/creator/actions.go new file mode 100644 index 0000000..b06496d --- /dev/null +++ b/internal/creator/actions.go @@ -0,0 +1,854 @@ +package creator + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math/big" + "strings" + "time" +) + +func validateStrategyInput(input StrategyInput) (StrategyInput, error) { + input.Action = strings.TrimSpace(input.Action) + input.TargetType = strings.TrimSpace(input.TargetType) + if input.Position <= 0 || !ValidAction(input.Action) || len(input.EventTypes) == 0 { + return StrategyInput{}, ErrInvalid + } + for i, eventType := range input.EventTypes { + input.EventTypes[i] = strings.TrimSpace(eventType) + if !ValidEventType(input.EventTypes[i]) || input.EventTypes[i] == "dm" { + return StrategyInput{}, ErrInvalid + } + } + for i, text := range input.CandidateTexts { + input.CandidateTexts[i] = strings.TrimSpace(text) + if input.CandidateTexts[i] == "" { + return StrategyInput{}, ErrInvalid + } + } + if input.Action == ActionLikeComment && input.TargetType != "comment" { + return StrategyInput{}, ErrInvalid + } + if (input.Action == ActionLikeWork || input.Action == ActionRepost) && input.TargetType != "work" { + return StrategyInput{}, ErrInvalid + } + if input.Action != ActionLikeComment && input.Action != ActionLikeWork && input.Action != ActionRepost { + input.TargetType = "user" + } + if len(input.CandidateTexts) > 20 { + return StrategyInput{}, ErrInvalid + } + return input, nil +} + +func encodeStrategyLists(input StrategyInput) (string, string, error) { + events, err := jsonText(input.EventTypes) + if err != nil { + return "", "", err + } + texts, err := jsonText(input.CandidateTexts) + if err != nil { + return "", "", err + } + return events, texts, nil +} + +func scanStrategy(scanner interface{ Scan(...any) error }) (Strategy, error) { + var result Strategy + var events, texts []byte + if err := scanner.Scan(&result.ID, &result.BigAccountID, &result.ExecutionAccountID, &result.Position, &result.Enabled, &events, &result.Action, &result.TargetType, &texts, &result.CreatedAt, &result.UpdatedAt); err != nil { + return Strategy{}, err + } + var err error + result.EventTypes, err = decodeStringList(events) + if err != nil { + return Strategy{}, err + } + result.CandidateTexts, err = decodeStringList(texts) + if err != nil { + return Strategy{}, err + } + return result, nil +} + +const strategySelect = `SELECT id,big_account_id,execution_account_id,position,enabled,event_types,action,target_type,candidate_texts,created_at,updated_at FROM creator_strategy` + +func (s *Store) CreateStrategy(ctx context.Context, bigAccountID string, input StrategyInput) (Strategy, error) { + input, err := validateStrategyInput(input) + if err != nil { + return Strategy{}, err + } + big, err := s.GetAccountProfile(ctx, bigAccountID) + if err != nil { + return Strategy{}, err + } + if !big.BigAccount { + return Strategy{}, ErrConflict + } + small, err := s.GetAccountProfile(ctx, input.ExecutionAccountID) + if err != nil { + return Strategy{}, err + } + if small.Platform != "" { + if err := s.requireRelation(ctx, bigAccountID, input.ExecutionAccountID); err != nil { + return Strategy{}, err + } + } + events, texts, err := encodeStrategyLists(input) + if err != nil { + return Strategy{}, err + } + id := newID("strategy") + if _, err := s.db.ExecContext(ctx, `INSERT INTO creator_strategy (id,big_account_id,execution_account_id,position,enabled,event_types,action,target_type,candidate_texts) VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9::jsonb)`, id, bigAccountID, input.ExecutionAccountID, input.Position, input.Enabled, events, input.Action, input.TargetType, texts); err != nil { + return Strategy{}, databaseError(err) + } + return s.GetStrategy(ctx, id) +} + +func (s *Store) requireRelation(ctx context.Context, bigID, smallID string) error { + var exists bool + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM creator_relation WHERE big_account_id=$1 AND small_account_id=$2)`, bigID, smallID).Scan(&exists); err != nil { + return databaseError(err) + } + if !exists { + return ErrConflict + } + return nil +} + +func (s *Store) GetStrategy(ctx context.Context, id string) (Strategy, error) { + result, err := scanStrategy(s.db.QueryRowContext(ctx, strategySelect+` WHERE id=$1`, id)) + return result, rowError(err) +} +func (s *Store) ListStrategies(ctx context.Context, bigID string) ([]Strategy, error) { + query := strategySelect + args := []any{} + if bigID != "" { + query += ` WHERE big_account_id=$1` + args = append(args, bigID) + } + query += ` ORDER BY position,id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Strategy, 0) + for rows.Next() { + item, err := scanStrategy(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} +func (s *Store) UpdateStrategy(ctx context.Context, id string, input StrategyInput) (Strategy, error) { + input, err := validateStrategyInput(input) + if err != nil { + return Strategy{}, err + } + events, texts, err := encodeStrategyLists(input) + if err != nil { + return Strategy{}, err + } + strategy, err := s.GetStrategy(ctx, id) + if err != nil { + return Strategy{}, err + } + if err := s.requireRelation(ctx, strategy.BigAccountID, input.ExecutionAccountID); err != nil { + return Strategy{}, err + } + if _, err := s.db.ExecContext(ctx, `UPDATE creator_strategy SET execution_account_id=$2,position=$3,enabled=$4,event_types=$5::jsonb,action=$6,target_type=$7,candidate_texts=$8::jsonb,updated_at=now() WHERE id=$1`, id, input.ExecutionAccountID, input.Position, input.Enabled, events, input.Action, input.TargetType, texts); err != nil { + return Strategy{}, databaseError(err) + } + return s.GetStrategy(ctx, id) +} +func (s *Store) SetStrategyEnabled(ctx context.Context, id string, enabled bool) (Strategy, error) { + if _, err := s.db.ExecContext(ctx, `UPDATE creator_strategy SET enabled=$2,updated_at=now() WHERE id=$1`, id, enabled); err != nil { + return Strategy{}, databaseError(err) + } + return s.GetStrategy(ctx, id) +} +func (s *Store) DeleteStrategy(ctx context.Context, id string) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM creator_strategy WHERE id=$1`, id) + return databaseError(err) +} + +func scanEvent(scanner interface{ Scan(...any) error }) (InteractionEvent, error) { + var result InteractionEvent + var platformAt, receivedAt, startedAt, displayedAt sql.NullTime + var commentID, workID, strategyID, executionID sql.NullString + if err := scanner.Scan(&result.ID, &result.Platform, &result.ReceivingAccountID, &result.EventKey, &result.EventType, &result.InteractorUID, &commentID, &workID, &platformAt, &receivedAt, &startedAt, &displayedAt, &result.State, &result.Reason, &strategyID, &executionID); err != nil { + return InteractionEvent{}, err + } + result.CommentID, result.WorkID = commentID.String, workID.String + result.StrategyID, result.ExecutionAccountID = strategyID.String, executionID.String + result.PlatformEventAt, result.ProcessingStartedAt, result.DisplayedAt = nullableTime(platformAt), nullableTime(startedAt), nullableTime(displayedAt) + if receivedAt.Valid { + result.ReceivedAt = receivedAt.Time.UTC() + } + return result, nil +} + +const eventSelect = `SELECT id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,platform_event_at,received_at,processing_started_at,displayed_at,state,reason,strategy_id,execution_account_id FROM creator_event` + +func (s *Store) GetEvent(ctx context.Context, id string) (InteractionEvent, error) { + result, err := scanEvent(s.db.QueryRowContext(ctx, eventSelect+` WHERE id=$1`, id)) + return result, rowError(err) +} +func (s *Store) ListEvents(ctx context.Context, accountID string) ([]InteractionEvent, error) { + query := eventSelect + args := []any{} + if accountID != "" { + query += ` WHERE receiving_account_id=$1` + args = append(args, accountID) + } + query += ` ORDER BY received_at DESC,id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]InteractionEvent, 0) + for rows.Next() { + item, err := scanEvent(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func (s *Store) RecordEvent(ctx context.Context, input InteractionEvent) (AutomaticResult, error) { + input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID = strings.TrimSpace(input.Platform), strings.TrimSpace(input.ReceivingAccountID), strings.TrimSpace(input.EventKey), strings.TrimSpace(input.EventType), strings.TrimSpace(input.InteractorUID), strings.TrimSpace(input.CommentID), strings.TrimSpace(input.WorkID) + if !ValidatePlatform(input.Platform) || input.ReceivingAccountID == "" || input.EventKey == "" || !ValidEventType(input.EventType) { + return AutomaticResult{}, ErrInvalid + } + profile, err := s.GetAccountProfile(ctx, input.ReceivingAccountID) + if err != nil { + return AutomaticResult{}, err + } + if profile.Platform != input.Platform { + return AutomaticResult{}, ErrInvalid + } + id := newID("event") + state := "received" + reason := "" + if input.Baseline { + state = "baseline" + reason = "监听基线" + } + if input.EventType != "dm" && input.InteractorUID == "" { + state = "blocked" + reason = "缺少互动用户 UID" + } + var returnedID string + var inserted bool + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_event (id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,platform_event_at,state,reason) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT (platform,receiving_account_id,event_key) DO NOTHING RETURNING id,(xmax=0)`, id, input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.PlatformEventAt, state, reason).Scan(&returnedID, &inserted) + if errors.Is(err, sql.ErrNoRows) { + existingErr := s.db.QueryRowContext(ctx, `SELECT id FROM creator_event WHERE platform=$1 AND receiving_account_id=$2 AND event_key=$3`, input.Platform, input.ReceivingAccountID, input.EventKey).Scan(&returnedID) + if existingErr != nil { + return AutomaticResult{}, rowError(existingErr) + } + event, getErr := s.GetEvent(ctx, returnedID) + return AutomaticResult{Event: event, Duplicate: true}, getErr + } + if err != nil { + return AutomaticResult{}, databaseError(err) + } + event, err := s.GetEvent(ctx, returnedID) + return AutomaticResult{Event: event, Duplicate: !inserted}, err +} + +func (s *Store) markEvent(ctx context.Context, eventID, state, reason, strategyID, executionID string, started, displayed *time.Time) error { + if state != "received" && state != "baseline" && state != "ignored" && state != "unmatched" && state != "blocked" && state != "processing" && state != "succeeded" && state != "failed" && state != "uncertain" { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, `UPDATE creator_event SET state=$2,reason=$3,strategy_id=NULLIF($4,''),execution_account_id=NULLIF($5,''),processing_started_at=COALESCE($6,processing_started_at),displayed_at=COALESCE($7,displayed_at) WHERE id=$1`, eventID, state, reason, strategyID, executionID, started, displayed) + return databaseError(err) +} + +func selectCandidate(values []string) (string, error) { + if len(values) == 0 { + return "", nil + } + if len(values) == 1 { + return values[0], nil + } + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(values)))) + if err != nil { + return "", fmt.Errorf("choose automatic response candidate: %w", err) + } + return values[n.Int64()], nil +} + +func (s *Store) ProcessAutomaticEvent(ctx context.Context, input InteractionEvent, executor ActionExecutor, generator TextGenerator) (AutomaticResult, error) { + recorded, err := s.RecordEvent(ctx, input) + if err != nil { + return recorded, err + } + eventID := recorded.Event.ID + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return AutomaticResult{}, fmt.Errorf("begin automatic event: %w", err) + } + defer tx.Rollback() + var storedPlatform, storedReceivingAccountID, storedEventKey, storedEventType, storedInteractorUID, storedCommentID, storedWorkID, eventState string + if err := tx.QueryRowContext(ctx, `SELECT platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,state FROM creator_event WHERE id=$1 FOR UPDATE`, eventID).Scan(&storedPlatform, &storedReceivingAccountID, &storedEventKey, &storedEventType, &storedInteractorUID, &storedCommentID, &storedWorkID, &eventState); err != nil { + return AutomaticResult{}, rowError(err) + } + if eventState != "received" { + tx.Rollback() + event, getErr := s.GetEvent(ctx, eventID) + return AutomaticResult{Event: event, Duplicate: recorded.Duplicate}, getErr + } + input.Platform = storedPlatform + input.ReceivingAccountID = storedReceivingAccountID + input.EventKey = storedEventKey + input.EventType = storedEventType + input.InteractorUID = storedInteractorUID + input.CommentID = storedCommentID + input.WorkID = storedWorkID + var bigProfile AccountProfile + if err := s.scanAccountTx(ctx, tx, input.ReceivingAccountID, &bigProfile); err != nil { + return AutomaticResult{}, err + } + if !bigProfile.BigAccount { + if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='ignored', reason='大号模式未开启' WHERE id=$1`, eventID); err != nil { + return AutomaticResult{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return AutomaticResult{}, fmt.Errorf("commit ignored event: %w", err) + } + event, err := s.GetEvent(ctx, eventID) + return AutomaticResult{Event: event}, err + } + var chosen Strategy + var execution AccountProfile + reason := "" + strategies, err := s.listStrategiesTx(ctx, tx, input.ReceivingAccountID) + if err != nil { + return AutomaticResult{}, err + } + for _, strategy := range strategies { + if !strategy.Enabled || !contains(strategy.EventTypes, input.EventType) { + continue + } + var profile AccountProfile + if err := s.scanAccountTx(ctx, tx, strategy.ExecutionAccountID, &profile); err != nil { + reason = "执行账号不可用" + continue + } + var related bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM creator_relation WHERE big_account_id=$1 AND small_account_id=$2)`, input.ReceivingAccountID, strategy.ExecutionAccountID).Scan(&related); err != nil { + return AutomaticResult{}, databaseError(err) + } + if !related { + reason = "执行账号未绑定" + continue + } + if err := CanWrite(profile, true, strategy.Action); err != nil { + reason = "执行账号不可用" + continue + } + if !ActionTargetValid(strategy.Action, input.InteractorUID, input.CommentID, input.WorkID, strategy.TargetType) { + reason = "事件缺少动作目标" + continue + } + if ActionRequiresText(strategy.Action) && len(strategy.CandidateTexts) == 0 && strings.TrimSpace(bigProfile.ReplyRequirements) == "" { + reason = "缺少候选文本和大号 AI 回复要求" + continue + } + chosen, execution = strategy, profile + break + } + if chosen.ID == "" { + if reason == "" { + reason = "没有匹配策略" + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='unmatched',reason=$2 WHERE id=$1`, eventID, reason); err != nil { + return AutomaticResult{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return AutomaticResult{}, err + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event}, nil + } + now := time.Now().UTC() + cooldownSeconds := bigProfile.CooldownSeconds + if cooldownSeconds <= 0 { + cooldownSeconds = 86400 + } + expires := now.Add(time.Duration(cooldownSeconds) * time.Second) + var cooldownID string + if err := tx.QueryRowContext(ctx, `INSERT INTO creator_cooldown (big_account_id,interactor_uid,event_id,strategy_id,execution_account_id,started_at,expires_at) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (big_account_id,interactor_uid) DO UPDATE SET event_id=EXCLUDED.event_id,strategy_id=EXCLUDED.strategy_id,execution_account_id=EXCLUDED.execution_account_id,started_at=EXCLUDED.started_at,expires_at=EXCLUDED.expires_at WHERE creator_cooldown.expires_at <= $6 RETURNING event_id`, input.ReceivingAccountID, input.InteractorUID, eventID, chosen.ID, execution.ID, now, expires).Scan(&cooldownID); errors.Is(err, sql.ErrNoRows) { + if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='blocked',reason='自动响应冷却中' WHERE id=$1`, eventID); err != nil { + return AutomaticResult{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return AutomaticResult{}, err + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event}, nil + } + if err != nil { + return AutomaticResult{}, databaseError(err) + } + text, selectionErr := selectCandidate(chosen.CandidateTexts) + if selectionErr != nil { + if _, updateErr := tx.ExecContext(ctx, `UPDATE creator_event SET state='failed', reason=$2, strategy_id=$3, execution_account_id=$4 WHERE id=$1`, eventID, selectionErr.Error(), chosen.ID, execution.ID); updateErr != nil { + return AutomaticResult{}, databaseError(updateErr) + } + if commitErr := tx.Commit(); commitErr != nil { + return AutomaticResult{}, commitErr + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event}, selectionErr + } + if text == "" { + if generator == nil { + if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='blocked',reason='AI 生成不可用',strategy_id=$2,execution_account_id=$3 WHERE id=$1`, eventID, chosen.ID, execution.ID); err != nil { + return AutomaticResult{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return AutomaticResult{}, err + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event}, ErrUnavailable + } + text, err = generator.Generate(ctx, bigProfile.ReplyRequirements, input.EventType) + if err != nil { + if _, updateErr := tx.ExecContext(ctx, `UPDATE creator_event SET state='failed',reason=$2,strategy_id=$3,execution_account_id=$4 WHERE id=$1`, eventID, err.Error(), chosen.ID, execution.ID); updateErr != nil { + return AutomaticResult{}, databaseError(updateErr) + } + if commitErr := tx.Commit(); commitErr != nil { + return AutomaticResult{}, commitErr + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event}, err + } + text = strings.TrimSpace(text) + if text == "" { + if _, updateErr := tx.ExecContext(ctx, `UPDATE creator_event SET state='failed',reason='AI 返回空内容',strategy_id=$2,execution_account_id=$3 WHERE id=$1`, eventID, chosen.ID, execution.ID); updateErr != nil { + return AutomaticResult{}, databaseError(updateErr) + } + if commitErr := tx.Commit(); commitErr != nil { + return AutomaticResult{}, commitErr + } + event, getErr := s.GetEvent(ctx, eventID) + if getErr != nil { + return AutomaticResult{}, getErr + } + return AutomaticResult{Event: event}, ErrInvalid + } + } + opInput := OperationInput{IdempotencyKey: "event:" + input.Platform + ":" + input.ReceivingAccountID + ":" + input.EventKey, Source: "automatic", Action: chosen.Action, Platform: input.Platform, AccountID: execution.ID, TargetUID: input.InteractorUID, TargetCommentID: input.CommentID, TargetWorkID: input.WorkID, Text: text, EventID: eventID, StrategyID: chosen.ID} + opID := newID("operation") + hash := operationHash(opInput) + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_operation (id,idempotency_key,source,action,platform,account_id,target_uid,target_comment_id,target_work_id,text,event_id,strategy_id,request_hash,state) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,'processing')`, opID, opInput.IdempotencyKey, opInput.Source, opInput.Action, opInput.Platform, opInput.AccountID, opInput.TargetUID, opInput.TargetCommentID, opInput.TargetWorkID, opInput.Text, opInput.EventID, opInput.StrategyID, hash); err != nil { + return AutomaticResult{}, databaseError(err) + } + started := now + if _, err := tx.ExecContext(ctx, `UPDATE creator_event SET state='processing',reason='',strategy_id=$2,execution_account_id=$3,processing_started_at=$4 WHERE id=$1`, eventID, chosen.ID, execution.ID, started); err != nil { + return AutomaticResult{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return AutomaticResult{}, fmt.Errorf("commit automatic event: %w", err) + } + result := ActionResult{} + if executor == nil { + result = ActionResult{State: "uncertain", Reason: "平台执行器不可用"} + } else { + result, err = executor.Execute(ctx, ActionRequest{OperationID: opID, Action: chosen.Action, Platform: input.Platform, AccountID: execution.ID, TargetUID: input.InteractorUID, TargetCommentID: input.CommentID, TargetWorkID: input.WorkID, Text: text}) + if err != nil && result.State == "" { + result.State = "uncertain" + result.Reason = err.Error() + } + } + if result.State != "succeeded" && result.State != "failed" && result.State != "uncertain" { + result.State = "uncertain" + if result.Reason == "" { + result.Reason = "执行器未返回明确结果" + } + } + if err := s.UpdateOperationResult(ctx, opID, result); err != nil { + return AutomaticResult{}, fmt.Errorf("save automatic operation result: %w", err) + } + if err := s.markEvent(ctx, eventID, result.State, result.Reason, chosen.ID, execution.ID, nil, ptrTime(time.Now().UTC())); err != nil { + return AutomaticResult{}, fmt.Errorf("save automatic event result: %w", err) + } + event, err := s.GetEvent(ctx, eventID) + if err != nil { + return AutomaticResult{}, err + } + op, err := s.GetOperation(ctx, opID) + if err != nil { + return AutomaticResult{}, err + } + return AutomaticResult{Event: event, Operation: &op}, nil +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} +func (s *Store) scanAccountTx(ctx context.Context, tx *sql.Tx, id string, result *AccountProfile) error { + var checkedAt sql.NullTime + if err := tx.QueryRowContext(ctx, accountProfileQuery(), id).Scan(&result.ID, &result.Name, &result.Platform, &result.PlatformAccountKey, &result.AuthorizationStatus, &result.RuntimeStatus, &result.LoginUsername, &result.PasswordConfigured, &result.RealNameStatus, &result.RealName, &result.IdentityNumber, &result.Note, &result.BusinessStatus, &result.BigAccount, &result.ReplyRequirements, &result.LoginStatus, &result.LoginReason, &checkedAt, &result.CooldownSeconds, &result.UpdatedAt); err != nil { + return rowError(err) + } + result.LoginCheckedAt = nullableTime(checkedAt) + return nil +} +func (s *Store) listStrategiesTx(ctx context.Context, tx *sql.Tx, bigID string) ([]Strategy, error) { + rows, err := tx.QueryContext(ctx, strategySelect+` WHERE big_account_id=$1 ORDER BY position,id`, bigID) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Strategy, 0) + for rows.Next() { + item, err := scanStrategy(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func operationHash(input OperationInput) string { + encoded, _ := json.Marshal(input) + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} +func scanOperation(scanner interface{ Scan(...any) error }) (Operation, error) { + var result Operation + var evidence []byte + var eventID, strategyID sql.NullString + if err := scanner.Scan(&result.ID, &result.IdempotencyKey, &result.Source, &result.Action, &result.Platform, &result.AccountID, &result.TargetUID, &result.TargetCommentID, &result.TargetWorkID, &result.Text, &eventID, &strategyID, &result.State, &evidence, &result.Reason, &result.CreatedAt, &result.UpdatedAt); err != nil { + return Operation{}, err + } + result.EventID, result.StrategyID = eventID.String, strategyID.String + result.Evidence = map[string]string{} + if len(evidence) > 0 { + if err := json.Unmarshal(evidence, &result.Evidence); err != nil { + return Operation{}, err + } + } + return result, nil +} + +const operationSelect = `SELECT id,idempotency_key,source,action,platform,account_id,target_uid,target_comment_id,target_work_id,text,event_id,strategy_id,state,evidence,reason,created_at,updated_at FROM creator_operation` + +func (s *Store) GetOperation(ctx context.Context, id string) (Operation, error) { + result, err := scanOperation(s.db.QueryRowContext(ctx, operationSelect+` WHERE id=$1`, id)) + return result, rowError(err) +} +func (s *Store) ListOperations(ctx context.Context, accountID string) ([]Operation, error) { + query := operationSelect + args := []any{} + if accountID != "" { + query += ` WHERE account_id=$1` + args = append(args, accountID) + } + query += ` ORDER BY created_at DESC,id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Operation, 0) + for rows.Next() { + item, err := scanOperation(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} +func (s *Store) validateOperationTarget(ctx context.Context, input OperationInput) error { + if input.TargetCommentID != "" { + var platform, authorUID string + if err := s.db.QueryRowContext(ctx, `SELECT platform,author_uid FROM creator_comment WHERE id=$1`, input.TargetCommentID).Scan(&platform, &authorUID); err != nil { + return rowError(err) + } + if platform != input.Platform || authorUID != input.TargetUID { + return ErrInvalid + } + } + if input.TargetWorkID != "" { + var platform string + if err := s.db.QueryRowContext(ctx, `SELECT platform FROM creator_work WHERE id=$1`, input.TargetWorkID).Scan(&platform); err != nil { + return rowError(err) + } + if platform != input.Platform { + return ErrInvalid + } + } + return nil +} + +func (s *Store) CreateOperation(ctx context.Context, input OperationInput) (Operation, bool, error) { + input.IdempotencyKey = strings.TrimSpace(input.IdempotencyKey) + input.Source = strings.TrimSpace(input.Source) + input.Action = strings.TrimSpace(input.Action) + input.Platform = strings.TrimSpace(input.Platform) + input.AccountID = strings.TrimSpace(input.AccountID) + input.TargetUID = strings.TrimSpace(input.TargetUID) + input.TargetCommentID = strings.TrimSpace(input.TargetCommentID) + input.TargetWorkID = strings.TrimSpace(input.TargetWorkID) + input.Text = strings.TrimSpace(input.Text) + if input.IdempotencyKey == "" || input.Source != "manual" || !ValidAction(input.Action) || !ValidatePlatform(input.Platform) || input.AccountID == "" || !ActionTargetValid(input.Action, input.TargetUID, input.TargetCommentID, input.TargetWorkID, "") || ActionRequiresText(input.Action) && input.Text == "" { + return Operation{}, false, ErrInvalid + } + hash := operationHash(input) + var returnedID, existingHash string + lookupErr := s.db.QueryRowContext(ctx, `SELECT id,request_hash FROM creator_operation WHERE idempotency_key=$1`, input.IdempotencyKey).Scan(&returnedID, &existingHash) + if lookupErr == nil { + if existingHash != hash { + return Operation{}, false, ErrConflict + } + op, err := s.GetOperation(ctx, returnedID) + return op, false, err + } + if !errors.Is(lookupErr, sql.ErrNoRows) { + return Operation{}, false, databaseError(lookupErr) + } + profile, err := s.AccountWriteCheck(ctx, input.AccountID, false, input.Action) + if err != nil { + return Operation{}, false, err + } + if profile.Platform != input.Platform { + return Operation{}, false, ErrInvalid + } + if err := s.validateOperationTarget(ctx, input); err != nil { + return Operation{}, false, err + } + id := newID("operation") + var inserted bool + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_operation (id,idempotency_key,source,action,platform,account_id,target_uid,target_comment_id,target_work_id,text,event_id,strategy_id,request_hash) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) ON CONFLICT (idempotency_key) DO NOTHING RETURNING id,(xmax=0)`, id, input.IdempotencyKey, input.Source, input.Action, input.Platform, input.AccountID, input.TargetUID, input.TargetCommentID, input.TargetWorkID, input.Text, input.EventID, input.StrategyID, hash).Scan(&returnedID, &inserted) + if errors.Is(err, sql.ErrNoRows) { + if err := s.db.QueryRowContext(ctx, `SELECT id,request_hash FROM creator_operation WHERE idempotency_key=$1`, input.IdempotencyKey).Scan(&returnedID, &existingHash); err != nil { + return Operation{}, false, rowError(err) + } + if existingHash != hash { + return Operation{}, false, ErrConflict + } + op, err := s.GetOperation(ctx, returnedID) + return op, false, err + } + if err != nil { + return Operation{}, false, databaseError(err) + } + op, err := s.GetOperation(ctx, returnedID) + return op, inserted, err +} +func (s *Store) UpdateOperationResult(ctx context.Context, id string, result ActionResult) error { + if result.State != "succeeded" && result.State != "failed" && result.State != "uncertain" && result.State != "blocked" { + return ErrInvalid + } + evidence, err := jsonText(result.Evidence) + if err != nil { + return err + } + _, err = s.db.ExecContext(ctx, `UPDATE creator_operation SET state=$2,evidence=$3::jsonb,reason=$4,updated_at=now() WHERE id=$1`, id, result.State, evidence, result.Reason) + return databaseError(err) +} +func (s *Store) ExecuteManualOperation(ctx context.Context, id string, executor ActionExecutor) (Operation, error) { + op, err := s.GetOperation(ctx, id) + if err != nil { + return Operation{}, err + } + if op.Source != "manual" { + return Operation{}, ErrInvalid + } + if op.State != "created" { + return op, nil + } + if _, err := s.AccountWriteCheck(ctx, op.AccountID, false, op.Action); err != nil { + if updateErr := s.UpdateOperationResult(ctx, id, ActionResult{State: "blocked", Reason: err.Error()}); updateErr != nil { + return Operation{}, updateErr + } + return s.GetOperation(ctx, id) + } + if executor == nil { + if updateErr := s.UpdateOperationResult(ctx, id, ActionResult{State: "uncertain", Reason: "平台执行器不可用"}); updateErr != nil { + return Operation{}, updateErr + } + return s.GetOperation(ctx, id) + } + var claimedID string + if err := s.db.QueryRowContext(ctx, `UPDATE creator_operation SET state='processing',updated_at=now() WHERE id=$1 AND state='created' RETURNING id`, id).Scan(&claimedID); errors.Is(err, sql.ErrNoRows) { + return s.GetOperation(ctx, id) + } else if err != nil { + return Operation{}, databaseError(err) + } + result, execErr := executor.Execute(ctx, ActionRequest{OperationID: claimedID, Action: op.Action, Platform: op.Platform, AccountID: op.AccountID, TargetUID: op.TargetUID, TargetCommentID: op.TargetCommentID, TargetWorkID: op.TargetWorkID, Text: op.Text}) + if execErr != nil && result.State == "" { + result.State = "uncertain" + result.Reason = execErr.Error() + } + if result.State == "" { + result.State = "uncertain" + } + if err := s.UpdateOperationResult(ctx, id, result); err != nil { + return Operation{}, err + } + return s.GetOperation(ctx, id) +} + +func scanConversation(scanner interface{ Scan(...any) error }) (Conversation, error) { + var c Conversation + var last sql.NullTime + if err := scanner.Scan(&c.ID, &c.Platform, &c.AccountID, &c.PeerUID, &c.PeerName, &last); err != nil { + return Conversation{}, err + } + c.LastMessageAt = nullableTime(last) + return c, nil +} + +const conversationSelect = `SELECT id,platform,account_id,peer_uid,peer_name,last_message_at FROM creator_conversation` + +func (s *Store) UpsertConversation(ctx context.Context, input MessageInput) (Conversation, error) { + if !ValidatePlatform(input.Platform) || input.AccountID == "" || input.PeerUID == "" { + return Conversation{}, ErrInvalid + } + var accountPlatform string + if err := s.db.QueryRowContext(ctx, `SELECT platform FROM social_account WHERE id=$1`, input.AccountID).Scan(&accountPlatform); err != nil { + return Conversation{}, rowError(err) + } + if accountPlatform != input.Platform { + return Conversation{}, ErrInvalid + } + id := newID("conversation") + var returned string + if err := s.db.QueryRowContext(ctx, `INSERT INTO creator_conversation (id,platform,account_id,peer_uid,peer_name,last_message_at) VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (account_id,peer_uid) DO UPDATE SET peer_name=CASE WHEN EXCLUDED.peer_name='' THEN creator_conversation.peer_name ELSE EXCLUDED.peer_name END, last_message_at=COALESCE(EXCLUDED.last_message_at,creator_conversation.last_message_at) RETURNING id`, id, input.Platform, input.AccountID, input.PeerUID, input.PeerName, input.MessageAt).Scan(&returned); err != nil { + return Conversation{}, databaseError(err) + } + return s.GetConversation(ctx, returned) +} +func (s *Store) GetConversation(ctx context.Context, id string) (Conversation, error) { + result, err := scanConversation(s.db.QueryRowContext(ctx, conversationSelect+` WHERE id=$1`, id)) + return result, rowError(err) +} +func (s *Store) ListConversations(ctx context.Context, accountID string) ([]Conversation, error) { + query := conversationSelect + args := []any{} + if accountID != "" { + query += ` WHERE account_id=$1` + args = append(args, accountID) + } + query += ` ORDER BY last_message_at DESC NULLS LAST,id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Conversation, 0) + for rows.Next() { + item, err := scanConversation(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} +func (s *Store) SaveMessage(ctx context.Context, input MessageInput) (Message, bool, error) { + input.PlatformMessageKey = strings.TrimSpace(input.PlatformMessageKey) + input.Direction = strings.TrimSpace(input.Direction) + input.MessageType = strings.TrimSpace(input.MessageType) + if input.PlatformMessageKey == "" || input.Direction != "inbound" && input.Direction != "outbound" || input.MessageType == "" || len(input.Text) > 100000 { + return Message{}, false, ErrInvalid + } + state := input.SentState + if state == "" { + state = "received" + } + if state != "received" && state != "pending" && state != "succeeded" && state != "failed" && state != "uncertain" { + return Message{}, false, ErrInvalid + } + input.SentState = state + conversation, err := s.UpsertConversation(ctx, input) + if err != nil { + return Message{}, false, err + } + id := newID("message") + var returned string + var inserted bool + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_message (id,conversation_id,platform_message_key,direction,message_type,text,sent_state,message_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (conversation_id,platform_message_key) DO NOTHING RETURNING id,(xmax=0)`, id, conversation.ID, input.PlatformMessageKey, input.Direction, input.MessageType, input.Text, state, input.MessageAt).Scan(&returned, &inserted) + if errors.Is(err, sql.ErrNoRows) { + if err := s.db.QueryRowContext(ctx, `SELECT id FROM creator_message WHERE conversation_id=$1 AND platform_message_key=$2`, conversation.ID, input.PlatformMessageKey).Scan(&returned); err != nil { + return Message{}, false, rowError(err) + } + message, err := s.GetMessage(ctx, returned) + return message, false, err + } + if err != nil { + return Message{}, false, databaseError(err) + } + message, err := s.GetMessage(ctx, returned) + return message, inserted, err +} +func scanMessage(scanner interface{ Scan(...any) error }) (Message, error) { + var m Message + var at sql.NullTime + if err := scanner.Scan(&m.ID, &m.ConversationID, &m.PlatformMessageKey, &m.Direction, &m.MessageType, &m.Text, &m.SentState, &at, &m.CreatedAt); err != nil { + return Message{}, err + } + m.MessageAt = nullableTime(at) + return m, nil +} + +const messageSelect = `SELECT id,conversation_id,platform_message_key,direction,message_type,text,sent_state,message_at,created_at FROM creator_message` + +func (s *Store) GetMessage(ctx context.Context, id string) (Message, error) { + result, err := scanMessage(s.db.QueryRowContext(ctx, messageSelect+` WHERE id=$1`, id)) + return result, rowError(err) +} +func (s *Store) ListMessages(ctx context.Context, conversationID string) ([]Message, error) { + rows, err := s.db.QueryContext(ctx, messageSelect+` WHERE conversation_id=$1 ORDER BY message_at NULLS LAST,created_at,id`, conversationID) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Message, 0) + for rows.Next() { + item, err := scanMessage(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} diff --git a/internal/creator/bailian.go b/internal/creator/bailian.go new file mode 100644 index 0000000..bba9ed9 --- /dev/null +++ b/internal/creator/bailian.go @@ -0,0 +1,180 @@ +package creator + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const defaultBailianBaseURL = "https://dashscope.aliyuncs.com/compatible-mode/v1" + +type BailianClient struct { + BaseURL string + APIKey string + Model string + HTTPClient *http.Client +} + +func NewBailianClient(baseURL, apiKey, model string, client *http.Client) (*BailianClient, error) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" { + baseURL = defaultBailianBaseURL + } + if strings.TrimSpace(apiKey) == "" || strings.TrimSpace(model) == "" { + return nil, fmt.Errorf("%w: BAILIAN_API_KEY and AI model are required", ErrUnavailable) + } + if client == nil { + client = &http.Client{Timeout: 60 * time.Second} + } + return &BailianClient{BaseURL: baseURL, APIKey: apiKey, Model: model, HTTPClient: client}, nil +} + +type bailianChatRequest struct { + Model string `json:"model"` + Messages []bailianChatMessage `json:"messages"` +} + +type bailianChatMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type bailianChatResponse struct { + Choices []struct { + Message bailianChatMessage `json:"message"` + } `json:"choices"` +} + +func (c *BailianClient) chat(ctx context.Context, instruction, input string) (string, error) { + if c == nil || c.HTTPClient == nil || strings.TrimSpace(c.APIKey) == "" || strings.TrimSpace(c.Model) == "" { + return "", fmt.Errorf("%w: BAILIAN client is not configured", ErrUnavailable) + } + body, err := json.Marshal(bailianChatRequest{ + Model: c.Model, + Messages: []bailianChatMessage{ + {Role: "system", Content: instruction}, + {Role: "user", Content: input}, + }, + }) + if err != nil { + return "", err + } + endpoint := c.BaseURL + "/chat/completions" + if strings.HasSuffix(c.BaseURL, "/chat/completions") { + endpoint = c.BaseURL + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Bearer "+c.APIKey) + request.Header.Set("Content-Type", "application/json") + response, err := c.HTTPClient.Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + responseBody, err := io.ReadAll(io.LimitReader(response.Body, 2<<20)) + if err != nil { + return "", err + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return "", fmt.Errorf("bailian request failed with HTTP %d: %s", response.StatusCode, strings.TrimSpace(string(responseBody))) + } + var result bailianChatResponse + if err := json.Unmarshal(responseBody, &result); err != nil { + return "", fmt.Errorf("decode bailian response: %w", err) + } + if len(result.Choices) == 0 || strings.TrimSpace(result.Choices[0].Message.Content) == "" { + return "", fmt.Errorf("bailian response contained no message") + } + return strings.TrimSpace(result.Choices[0].Message.Content), nil +} + +func (c *BailianClient) Generate(ctx context.Context, instruction, input string) (string, error) { + return c.chat(ctx, instruction, input) +} + +func (c *BailianClient) MatchTheme(ctx context.Context, title, body, topic string) (bool, string, error) { + content, err := c.chat(ctx, + "判断作品是否符合给定主题。只返回 JSON，不要 Markdown 或额外文字，格式必须是 {\"match\":true或false,\"reason\":\"简短原因\"}。", + fmt.Sprintf("主题：%s\n标题：%s\n正文：%s", topic, title, body)) + if err != nil { + return false, "", err + } + var result struct { + Match bool `json:"match"` + Reason string `json:"reason"` + } + if err := json.Unmarshal([]byte(content), &result); err != nil { + return false, "", fmt.Errorf("decode bailian theme result: %w", err) + } + return result.Match, strings.TrimSpace(result.Reason), nil +} + +func (c *BailianClient) MatchLead(ctx context.Context, work, comment, requirement string) (bool, string, error) { + content, err := c.chat(ctx, + "判断评论是否是有效业务线索。只返回 JSON，不要 Markdown 或额外文字，格式必须是 {\"match\":true或false,\"reason\":\"简短原因\"}。", + fmt.Sprintf("判定要求：%s\n作品：%s\n评论：%s", requirement, work, comment)) + if err != nil { + return false, "", err + } + var result struct { + Match bool `json:"match"` + Reason string `json:"reason"` + } + if err := json.Unmarshal([]byte(content), &result); err != nil { + return false, "", fmt.Errorf("decode bailian lead result: %w", err) + } + return result.Match, strings.TrimSpace(result.Reason), nil +} + +type ConfiguredBailian struct { + Store *Store + APIKey string + BaseURL string + HTTPClient *http.Client +} + +func (b *ConfiguredBailian) client(ctx context.Context) (*BailianClient, error) { + if b == nil || b.Store == nil { + return nil, fmt.Errorf("%w: BAILIAN client is not configured", ErrUnavailable) + } + settings, err := b.Store.GetSettings(ctx) + if err != nil { + return nil, err + } + if !settings.AIConfigured || settings.AIProvider != "bailian" { + return nil, fmt.Errorf("%w: BAILIAN is not enabled in creator settings", ErrUnavailable) + } + return NewBailianClient(b.BaseURL, b.APIKey, settings.AIModel, b.HTTPClient) +} + +func (b *ConfiguredBailian) Generate(ctx context.Context, instruction, input string) (string, error) { + client, err := b.client(ctx) + if err != nil { + return "", err + } + return client.Generate(ctx, instruction, input) +} + +func (b *ConfiguredBailian) MatchTheme(ctx context.Context, title, body, topic string) (bool, string, error) { + client, err := b.client(ctx) + if err != nil { + return false, "", err + } + return client.MatchTheme(ctx, title, body, topic) +} + +func (b *ConfiguredBailian) MatchLead(ctx context.Context, work, comment, requirement string) (bool, string, error) { + client, err := b.client(ctx) + if err != nil { + return false, "", err + } + return client.MatchLead(ctx, work, comment, requirement) +} diff --git a/internal/creator/bailian_test.go b/internal/creator/bailian_test.go new file mode 100644 index 0000000..c996114 --- /dev/null +++ b/internal/creator/bailian_test.go @@ -0,0 +1,53 @@ +package creator + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestBailianClientGeneratesAndParsesDecisions(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/chat/completions" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer test-key" { + t.Fatal("missing authorization") + } + var request bailianChatRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + if request.Model != "qwen-test" || len(request.Messages) != 2 { + t.Fatalf("unexpected request body: %#v", request) + } + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{ + "message": map[string]string{"role": "assistant", "content": `{"match":true,"reason":"相关"}`}, + }}}) + })) + defer server.Close() + + client, err := NewBailianClient(server.URL+"/v1", "test-key", "qwen-test", server.Client()) + if err != nil { + t.Fatal(err) + } + text, err := client.Generate(context.Background(), "要求", "输入") + if err != nil || text == "" { + t.Fatalf("Generate() = %q, %v", text, err) + } + match, reason, err := client.MatchLead(context.Background(), "作品", "评论", "要求") + if err != nil || !match || reason != "相关" { + t.Fatalf("MatchLead() = %v, %q, %v", match, reason, err) + } +} + +func TestBailianClientRejectsIncompleteConfiguration(t *testing.T) { + if _, err := NewBailianClient("", "", "qwen-test", nil); err == nil { + t.Fatal("expected missing key to fail") + } + if _, err := NewBailianClient("", "test-key", "", nil); err == nil { + t.Fatal("expected missing model to fail") + } +} diff --git a/internal/creator/collection.go b/internal/creator/collection.go new file mode 100644 index 0000000..cac62a1 --- /dev/null +++ b/internal/creator/collection.go @@ -0,0 +1,436 @@ +package creator + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +type WorkPage struct { + Items []WorkInput + NextCursor string + HasMore bool +} + +type CommentPage struct { + Items []CommentInput + NextCursor string + HasMore bool +} + +type PlatformCollector interface { + ListWorks(context.Context, string, string) (WorkPage, error) + ListTopLevelComments(context.Context, string, string) (CommentPage, error) +} + +type CollectionReport struct { + SourceType string `json:"source_type"` + SourceID string `json:"source_id"` + WorksSeen int `json:"works_seen"` + WorksSaved int `json:"works_saved"` + CommentsSeen int `json:"comments_seen"` + CommentsSaved int `json:"comments_saved"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + PaginationComplete bool `json:"pagination_complete"` +} + +func collectPages[T any](ctx context.Context, fetch func(context.Context, string) ([]T, string, bool, error)) ([]T, error) { + items := make([]T, 0) + seenCursors := map[string]struct{}{"": {}} + cursor := "" + for page := 0; page < 100; page++ { + pageItems, nextCursor, hasMore, err := fetch(ctx, cursor) + if err != nil { + return nil, err + } + items = append(items, pageItems...) + if !hasMore { + return items, nil + } + if nextCursor == "" { + return nil, fmt.Errorf("%w: pagination cursor missing", ErrInvalid) + } + if _, exists := seenCursors[nextCursor]; exists { + return nil, fmt.Errorf("%w: pagination cursor did not advance", ErrInvalid) + } + seenCursors[nextCursor] = struct{}{} + cursor = nextCursor + } + return nil, fmt.Errorf("%w: pagination exceeded 100 pages", ErrInvalid) +} + +func CollectWorkPages(ctx context.Context, collector PlatformCollector) ([]WorkInput, error) { + if collector == nil { + return nil, ErrUnavailable + } + return collectPages(ctx, func(ctx context.Context, cursor string) ([]WorkInput, string, bool, error) { + page, err := collector.ListWorks(ctx, "", cursor) + if err != nil { + return nil, "", false, err + } + return page.Items, page.NextCursor, page.HasMore, nil + }) +} + +func CollectCommentPages(ctx context.Context, collector PlatformCollector, workKey string) ([]CommentInput, error) { + if collector == nil || strings.TrimSpace(workKey) == "" { + return nil, ErrInvalid + } + return collectPages(ctx, func(ctx context.Context, cursor string) ([]CommentInput, string, bool, error) { + page, err := collector.ListTopLevelComments(ctx, workKey, cursor) + if err != nil { + return nil, "", false, err + } + for _, item := range page.Items { + if item.CommentType == "reply" { + return nil, "", false, fmt.Errorf("%w: reply comment returned by top-level collector", ErrInvalid) + } + } + return page.Items, page.NextCursor, page.HasMore, nil + }) +} + +func NewCollectionWindow(now time.Time, lookbackDays int) (time.Time, time.Time, error) { + if now.IsZero() || lookbackDays <= 0 || int64(lookbackDays) > maxDurationSeconds/(24*60*60) { + return time.Time{}, time.Time{}, ErrInvalid + } + end := now.UTC() + return end.Add(-time.Duration(lookbackDays) * 24 * time.Hour), end, nil +} + +func checkpointID(sourceType, sourceID, kind string) string { + return sourceType + ":" + sourceID + ":" + kind +} + +type checkpointState struct { + Cursor string + LeaseToken string + WindowStart time.Time + WindowEnd time.Time + Status string +} + +func (s *Store) beginCheckpoint(ctx context.Context, sourceType, sourceID, kind string, start, end time.Time) (string, error) { + if sourceType != SourceOwned && sourceType != SourceCompetitor || sourceID == "" || kind != "works" && kind != "comments" || start.IsZero() || end.IsZero() || !start.Before(end) { + return "", ErrInvalid + } + leaseToken := newID("lease") + var claimedToken string + err := s.db.QueryRowContext(ctx, ` + INSERT INTO creator_collection_checkpoint + (id, source_type, source_id, collection_kind, window_start, window_end, status, lease_token, lease_until, last_started_at, last_error) + VALUES ($1, $2, $3, $4, $5, $6, 'running', $7, now() + interval '10 minutes', now(), '') + ON CONFLICT (source_type, source_id, collection_kind) DO UPDATE SET + window_start = CASE WHEN creator_collection_checkpoint.status IN ('failed', 'running') THEN creator_collection_checkpoint.window_start ELSE EXCLUDED.window_start END, + window_end = CASE WHEN creator_collection_checkpoint.status IN ('failed', 'running') THEN creator_collection_checkpoint.window_end ELSE EXCLUDED.window_end END, + cursor = CASE WHEN creator_collection_checkpoint.status IN ('failed', 'running') THEN creator_collection_checkpoint.cursor ELSE '' END, + status = 'running', lease_token = EXCLUDED.lease_token, lease_until = EXCLUDED.lease_until, + last_started_at = EXCLUDED.last_started_at, last_error = '' + WHERE creator_collection_checkpoint.status <> 'running' + OR creator_collection_checkpoint.lease_until IS NULL + OR creator_collection_checkpoint.lease_until <= now() + RETURNING lease_token`, checkpointID(sourceType, sourceID, kind), sourceType, sourceID, kind, start.UTC(), end.UTC(), leaseToken).Scan(&claimedToken) + if errors.Is(err, sql.ErrNoRows) { + return "", ErrConflict + } + if err != nil { + return "", databaseError(err) + } + return claimedToken, nil +} + +func (s *Store) checkpoint(ctx context.Context, sourceType, sourceID, kind string) (checkpointState, error) { + var state checkpointState + err := s.db.QueryRowContext(ctx, `SELECT cursor,lease_token,window_start,window_end,status FROM creator_collection_checkpoint WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3`, sourceType, sourceID, kind).Scan(&state.Cursor, &state.LeaseToken, &state.WindowStart, &state.WindowEnd, &state.Status) + if err != nil { + return checkpointState{}, rowError(err) + } + state.WindowStart, state.WindowEnd = state.WindowStart.UTC(), state.WindowEnd.UTC() + return state, nil +} + +func (s *Store) saveCheckpointCursor(ctx context.Context, sourceType, sourceID, kind, leaseToken, cursor string) error { + if leaseToken == "" { + return ErrInvalid + } + result, err := s.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET cursor=$5 WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3 AND lease_token=$4 AND status='running' AND lease_until > now()`, sourceType, sourceID, kind, leaseToken, cursor) + if err != nil { + return databaseError(err) + } + if affected, err := result.RowsAffected(); err != nil { + return err + } else if affected != 1 { + return ErrConflict + } + return nil +} + +func (s *Store) finishCheckpoint(ctx context.Context, sourceType, sourceID, kind, leaseToken, status, reason string) error { + if status != "succeeded" && status != "failed" && status != "blocked" || leaseToken == "" { + return ErrInvalid + } + result, err := s.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET status=$5, cursor=CASE WHEN $5='succeeded' THEN '' ELSE cursor END, lease_until=NULL, last_completed_at=CASE WHEN $5='succeeded' THEN now() ELSE last_completed_at END, last_error=$6 WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3 AND lease_token=$4 AND status='running' AND lease_until > now()`, sourceType, sourceID, kind, leaseToken, status, reason) + if err != nil { + return databaseError(err) + } + if affected, err := result.RowsAffected(); err != nil { + return err + } else if affected != 1 { + return ErrConflict + } + return nil +} + +func (s *Store) failCheckpoint(ctx context.Context, sourceType, sourceID, kind, leaseToken string, primary error) error { + if err := s.finishCheckpoint(ctx, sourceType, sourceID, kind, leaseToken, "failed", primary.Error()); err != nil { + return errors.Join(primary, err) + } + return primary +} + +func collectPagesFromCursor[T any](ctx context.Context, initialCursor string, fetch func(context.Context, string) ([]T, string, bool, error), afterPage func([]T, string, bool) error) ([]T, error) { + items := make([]T, 0) + seenCursors := map[string]struct{}{initialCursor: {}} + cursor := initialCursor + for page := 0; page < 100; page++ { + pageItems, nextCursor, hasMore, err := fetch(ctx, cursor) + if err != nil { + return nil, err + } + items = append(items, pageItems...) + if afterPage != nil { + if err := afterPage(pageItems, nextCursor, hasMore); err != nil { + return nil, err + } + } + if !hasMore { + return items, nil + } + if nextCursor == "" { + return nil, fmt.Errorf("%w: pagination cursor missing", ErrInvalid) + } + if _, exists := seenCursors[nextCursor]; exists { + return nil, fmt.Errorf("%w: pagination cursor did not advance", ErrInvalid) + } + seenCursors[nextCursor] = struct{}{} + cursor = nextCursor + } + return nil, fmt.Errorf("%w: pagination exceeded 100 pages", ErrInvalid) +} + +type commentCheckpointCursor struct { + WorkKey string `json:"work_key"` + Cursor string `json:"cursor"` +} + +func encodeCommentCheckpoint(workKey, cursor string) (string, error) { + value, err := json.Marshal(commentCheckpointCursor{WorkKey: workKey, Cursor: cursor}) + if err != nil { + return "", fmt.Errorf("encode comment checkpoint: %w", err) + } + return string(value), nil +} + +func decodeCommentCheckpoint(value string) (commentCheckpointCursor, error) { + var result commentCheckpointCursor + if value == "" { + return result, nil + } + if err := json.Unmarshal([]byte(value), &result); err != nil || result.WorkKey == "" { + return commentCheckpointCursor{}, fmt.Errorf("%w: invalid comment checkpoint", ErrInvalid) + } + return result, nil +} + +func (s *Store) ListDueOwnedAccounts(ctx context.Context, now time.Time, intervalSeconds int64) ([]string, error) { + if now.IsZero() || intervalSeconds <= 0 || intervalSeconds > maxDurationSeconds { + return nil, ErrInvalid + } + cutoff := now.UTC().Add(-time.Duration(intervalSeconds) * time.Second) + rows, err := s.db.QueryContext(ctx, ` + SELECT account.id + FROM social_account account + JOIN creator_account_profile profile ON profile.account_id=account.id AND profile.big_account=true + LEFT JOIN creator_collection_checkpoint works_checkpoint + ON works_checkpoint.source_type='owned' AND works_checkpoint.source_id=account.id AND works_checkpoint.collection_kind='works' + LEFT JOIN creator_collection_checkpoint comments_checkpoint + ON comments_checkpoint.source_type='owned' AND comments_checkpoint.source_id=account.id AND comments_checkpoint.collection_kind='comments' + WHERE account.platform='douyin' AND account.authorization_status='authorized' + AND COALESCE(works_checkpoint.status, '') <> 'blocked' + AND COALESCE(comments_checkpoint.status, '') <> 'blocked' + AND (works_checkpoint.id IS NULL OR comments_checkpoint.id IS NULL + OR works_checkpoint.status='failed' OR comments_checkpoint.status='failed' + OR works_checkpoint.last_completed_at IS NULL OR comments_checkpoint.last_completed_at IS NULL + OR works_checkpoint.last_completed_at <= $1 + OR comments_checkpoint.last_completed_at <= $1) + ORDER BY account.id`, cutoff) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + accounts := make([]string, 0) + for rows.Next() { + var accountID string + if err := rows.Scan(&accountID); err != nil { + return nil, err + } + accounts = append(accounts, accountID) + } + return accounts, rows.Err() +} + +func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceID string, collector PlatformCollector, now time.Time) (CollectionReport, error) { + if !ValidatePlatform(platform) || sourceID == "" || (sourceType != SourceOwned && sourceType != SourceCompetitor) || collector == nil { + return CollectionReport{}, ErrInvalid + } + settings, err := s.GetSettings(ctx) + if err != nil { + return CollectionReport{}, err + } + start, end, err := NewCollectionWindow(now, settings.LookbackDays) + if err != nil { + return CollectionReport{}, err + } + report := CollectionReport{SourceType: sourceType, SourceID: sourceID, WindowStart: start, WindowEnd: end} + worksLease, err := s.beginCheckpoint(ctx, sourceType, sourceID, "works", start, end) + if err != nil { + return report, err + } + worksCheckpoint, err := s.checkpoint(ctx, sourceType, sourceID, "works") + if err != nil { + return report, err + } + report.WindowStart, report.WindowEnd = worksCheckpoint.WindowStart, worksCheckpoint.WindowEnd + seenWorks := map[string]string{} + _, err = collectPagesFromCursor(ctx, worksCheckpoint.Cursor, func(ctx context.Context, cursor string) ([]WorkInput, string, bool, error) { + page, err := collector.ListWorks(ctx, sourceID, cursor) + if err != nil { + return nil, "", false, err + } + return page.Items, page.NextCursor, page.HasMore, nil + }, func(pageItems []WorkInput, nextCursor string, hasMore bool) error { + for _, work := range pageItems { + report.WorksSeen++ + if work.PublishedAt != nil && !InWindow(*work.PublishedAt, report.WindowStart, report.WindowEnd) { + continue + } + work.Platform, work.SourceType, work.SourceID = platform, sourceType, sourceID + if _, exists := seenWorks[work.WorkKey]; exists { + continue + } + savedWork, _, err := s.UpsertWork(ctx, work, report.WindowEnd) + if err != nil { + return err + } + if err := s.EnsureMetricPlan(ctx, savedWork.ID, settings); err != nil { + return err + } + if work.Likes != nil || work.CommentsCount != nil || work.Shares != nil { + if _, metricErr := s.RecordMetric(ctx, MetricInput{WorkID: savedWork.ID, CollectedAt: now.UTC(), Likes: work.Likes, CommentsCount: work.CommentsCount, Shares: work.Shares}, settings, now.UTC()); metricErr != nil && !errors.Is(metricErr, ErrConflict) { + return metricErr + } + } + seenWorks[work.WorkKey] = savedWork.ID + report.WorksSaved++ + } + if hasMore { + return s.saveCheckpointCursor(ctx, sourceType, sourceID, "works", worksLease, nextCursor) + } + return nil + }) + if err != nil { + return report, s.failCheckpoint(ctx, sourceType, sourceID, "works", worksLease, err) + } + if err := s.finishCheckpoint(ctx, sourceType, sourceID, "works", worksLease, "succeeded", ""); err != nil { + return report, err + } + commentsLease, err := s.beginCheckpoint(ctx, sourceType, sourceID, "comments", report.WindowStart, report.WindowEnd) + if err != nil { + return report, err + } + commentsCheckpoint, err := s.checkpoint(ctx, sourceType, sourceID, "comments") + if err != nil { + return report, err + } + report.WindowStart, report.WindowEnd = commentsCheckpoint.WindowStart, commentsCheckpoint.WindowEnd + works, err := s.ListWorks(ctx, WorkFilter{Platform: platform, SourceType: sourceType, SourceID: sourceID, PublishedAfter: &report.WindowStart, PublishedBefore: &report.WindowEnd}) + if err != nil { + return report, err + } + startIndex, initialCommentCursor := 0, "" + if commentsCheckpoint.Cursor != "" { + position, err := decodeCommentCheckpoint(commentsCheckpoint.Cursor) + if err != nil { + return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err) + } + initialCommentCursor = position.Cursor + for index := range works { + if works[index].WorkKey == position.WorkKey { + startIndex = index + break + } + if index == len(works)-1 { + return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, fmt.Errorf("%w: comment checkpoint work missing", ErrInvalid)) + } + } + } + for index := startIndex; index < len(works); index++ { + work := works[index] + cursor := initialCommentCursor + initialCommentCursor = "" + _, err := collectPagesFromCursor(ctx, cursor, func(ctx context.Context, cursor string) ([]CommentInput, string, bool, error) { + page, err := collector.ListTopLevelComments(ctx, work.WorkKey, cursor) + if err != nil { + return nil, "", false, err + } + for _, item := range page.Items { + if item.CommentType == "reply" { + return nil, "", false, fmt.Errorf("%w: reply comment returned by top-level collector", ErrInvalid) + } + } + return page.Items, page.NextCursor, page.HasMore, nil + }, func(pageItems []CommentInput, nextCursor string, hasMore bool) error { + for _, comment := range pageItems { + report.CommentsSeen++ + if comment.PublishedAt != nil && !InWindow(*comment.PublishedAt, report.WindowStart, report.WindowEnd) || comment.CommentType == "reply" { + continue + } + comment.Platform, comment.WorkID = platform, work.ID + if _, _, err := s.SaveComment(ctx, comment); err != nil { + return err + } + report.CommentsSaved++ + } + if hasMore { + checkpoint, err := encodeCommentCheckpoint(work.WorkKey, nextCursor) + if err != nil { + return err + } + return s.saveCheckpointCursor(ctx, sourceType, sourceID, "comments", commentsLease, checkpoint) + } + return nil + }) + if err != nil { + return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err) + } + if index+1 < len(works) { + checkpoint, err := encodeCommentCheckpoint(works[index+1].WorkKey, "") + if err != nil { + return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err) + } + if err := s.saveCheckpointCursor(ctx, sourceType, sourceID, "comments", commentsLease, checkpoint); err != nil { + return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err) + } + } + } + if err := s.finishCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, "succeeded", ""); err != nil { + return report, err + } + report.PaginationComplete = true + return report, nil +} diff --git a/internal/creator/collection_test.go b/internal/creator/collection_test.go new file mode 100644 index 0000000..2cf1242 --- /dev/null +++ b/internal/creator/collection_test.go @@ -0,0 +1,46 @@ +package creator + +import ( + "context" + "errors" + "testing" + "time" +) + +type pageCollector struct { + works map[string]WorkPage + comments map[string]CommentPage +} + +func (c pageCollector) ListWorks(_ context.Context, _ string, cursor string) (WorkPage, error) { + return c.works[cursor], nil +} +func (c pageCollector) ListTopLevelComments(_ context.Context, workKey, cursor string) (CommentPage, error) { + return c.comments[workKey+":"+cursor], nil +} + +func TestCollectPagesRequiresAdvancingCursor(t *testing.T) { + collector := pageCollector{works: map[string]WorkPage{ + "": {Items: []WorkInput{{WorkKey: "one"}}, NextCursor: "next", HasMore: true}, + "next": {Items: []WorkInput{{WorkKey: "two"}}, HasMore: false}, + }} + items, err := CollectWorkPages(context.Background(), collector) + if err != nil || len(items) != 2 { + t.Fatalf("collect pages: %#v err=%v", items, err) + } + bad := pageCollector{works: map[string]WorkPage{"": {HasMore: true}}} + if _, err := CollectWorkPages(context.Background(), bad); !errors.Is(err, ErrInvalid) { + t.Fatalf("missing cursor error = %v", err) + } +} + +func TestCollectionWindowIsFixedInUTC(t *testing.T) { + now := time.Date(2026, 3, 2, 8, 0, 0, 0, time.FixedZone("CST", 8*60*60)) + start, end, err := NewCollectionWindow(now, 30) + if err != nil || !end.Equal(now.UTC()) || !start.Equal(now.UTC().Add(-30*24*time.Hour)) { + t.Fatalf("window = %s..%s err=%v", start, end, err) + } + if InWindow(end.Add(time.Nanosecond), start, end) || !InWindow(start, start, end) { + t.Fatal("window must include both exact boundaries only") + } +} diff --git a/internal/creator/content.go b/internal/creator/content.go new file mode 100644 index 0000000..a0387c3 --- /dev/null +++ b/internal/creator/content.go @@ -0,0 +1,604 @@ +package creator + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "strings" + "time" + "unicode/utf8" +) + +func validateHomepage(value string) error { + parsed, err := url.Parse(strings.TrimSpace(value)) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil { + return ErrInvalid + } + return nil +} + +func (s *Store) CreateCompetitor(ctx context.Context, input CompetitorInput) (Competitor, error) { + input.Platform = strings.TrimSpace(input.Platform) + input.PlatformAccountKey = strings.TrimSpace(input.PlatformAccountKey) + input.Nickname = strings.TrimSpace(input.Nickname) + input.AvatarURL = strings.TrimSpace(input.AvatarURL) + input.HomepageURL = strings.TrimSpace(input.HomepageURL) + if !ValidatePlatform(input.Platform) || input.PlatformAccountKey == "" || input.HomepageURL == "" || + utf8.RuneCountInString(input.PlatformAccountKey) > 255 || utf8.RuneCountInString(input.Nickname) > 255 || + utf8.RuneCountInString(input.AvatarURL) > 1000 || validateHomepage(input.HomepageURL) != nil { + return Competitor{}, ErrInvalid + } + id := newID("competitor") + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO creator_competitor (id, platform, platform_account_key, nickname, avatar_url, homepage_url, next_sync_at) + VALUES ($1, $2, $3, $4, $5, $6, now())`, + id, input.Platform, input.PlatformAccountKey, input.Nickname, input.AvatarURL, input.HomepageURL); err != nil { + return Competitor{}, databaseError(err) + } + return s.GetCompetitor(ctx, id) +} + +func scanCompetitor(scanner interface{ Scan(...any) error }) (Competitor, error) { + var result Competitor + var leaseUntil, lastSync, nextSync sql.NullTime + if err := scanner.Scan(&result.ID, &result.Platform, &result.PlatformAccountKey, &result.Nickname, + &result.AvatarURL, &result.HomepageURL, &result.Enabled, &result.SyncStatus, &result.SyncCursor, + &result.SyncError, &leaseUntil, &lastSync, &nextSync, &result.CreatedAt, &result.UpdatedAt); err != nil { + return Competitor{}, err + } + result.SyncLeaseUntil = nullableTime(leaseUntil) + result.LastSyncAt = nullableTime(lastSync) + result.NextSyncAt = nullableTime(nextSync) + return result, nil +} + +func (s *Store) GetCompetitor(ctx context.Context, id string) (Competitor, error) { + result, err := scanCompetitor(s.db.QueryRowContext(ctx, ` + SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, + enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at + FROM creator_competitor WHERE id = $1`, id)) + return result, rowError(err) +} + +func (s *Store) ListCompetitors(ctx context.Context, platform string) ([]Competitor, error) { + query := `SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, + enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at + FROM creator_competitor` + args := []any{} + if platform != "" { + if !ValidatePlatform(platform) { + return nil, ErrInvalid + } + query += ` WHERE platform = $1` + args = append(args, platform) + } + query += ` ORDER BY created_at DESC, id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Competitor, 0) + for rows.Next() { + item, err := scanCompetitor(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func (s *Store) SetCompetitorEnabled(ctx context.Context, id string, enabled bool) (Competitor, error) { + status := "paused" + if enabled { + status = "idle" + } + _, err := s.db.ExecContext(ctx, ` + UPDATE creator_competitor SET enabled = $2, sync_status = $3, sync_lease_until = NULL, + next_sync_at = CASE WHEN $2 THEN now() ELSE NULL END, updated_at = now() + WHERE id = $1`, id, enabled, status) + if err != nil { + return Competitor{}, databaseError(err) + } + return s.GetCompetitor(ctx, id) +} + +func (s *Store) MarkCompetitorSync(ctx context.Context, id, status, cursor, syncError string, nextAt *time.Time) error { + if status != "idle" && status != "running" && status != "paused" && status != "failed" && status != "blocked" { + return ErrInvalid + } + if len(syncError) > 2000 || len(cursor) > 1000 { + return ErrInvalid + } + var next any + if nextAt != nil { + next = nextAt.UTC() + } + _, err := s.db.ExecContext(ctx, ` + UPDATE creator_competitor + SET sync_status = $2, sync_cursor = $3, sync_error = $4, + sync_lease_until = CASE WHEN $2 = 'running' THEN now() + interval '10 minutes' ELSE NULL END, + last_sync_at = CASE WHEN $2 IN ('idle', 'failed', 'blocked') THEN now() ELSE last_sync_at END, + next_sync_at = $5, updated_at = now() + WHERE id = $1`, id, status, cursor, syncError, next) + return databaseError(err) +} + +func (s *Store) ClaimCompetitorSync(ctx context.Context, id string, force bool, now time.Time) (bool, error) { + if id == "" || now.IsZero() { + return false, ErrInvalid + } + condition := `enabled AND (next_sync_at IS NULL OR next_sync_at <= $2)` + if force { + condition = `enabled` + } + var claimed string + err := s.db.QueryRowContext(ctx, `UPDATE creator_competitor SET sync_status='running', sync_lease_until=$2 + interval '10 minutes', sync_error='', updated_at=$2 WHERE id=$1 AND `+condition+` AND (sync_status <> 'running' OR sync_lease_until IS NULL OR sync_lease_until <= $2) RETURNING id`, id, now.UTC()).Scan(&claimed) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, databaseError(err) + } + return claimed != "", nil +} + +func (s *Store) ListDueCompetitors(ctx context.Context, now time.Time) ([]Competitor, error) { + if now.IsZero() { + return nil, ErrInvalid + } + rows, err := s.db.QueryContext(ctx, `SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, + enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at + FROM creator_competitor + WHERE enabled AND next_sync_at IS NOT NULL AND next_sync_at <= $1 + AND (sync_status <> 'running' OR sync_lease_until IS NULL OR sync_lease_until <= $1) + ORDER BY COALESCE(next_sync_at, created_at), id`, now.UTC()) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Competitor, 0) + for rows.Next() { + item, err := scanCompetitor(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) (Work, bool, error) { + input.Platform = strings.TrimSpace(input.Platform) + input.WorkKey = strings.TrimSpace(input.WorkKey) + input.SourceType = strings.TrimSpace(input.SourceType) + input.SourceID = strings.TrimSpace(input.SourceID) + input.AuthorName = strings.TrimSpace(input.AuthorName) + input.Title = strings.TrimSpace(input.Title) + input.OriginalURL = strings.TrimSpace(input.OriginalURL) + input.CoverURL = strings.TrimSpace(input.CoverURL) + if !ValidatePlatform(input.Platform) || input.WorkKey == "" || input.SourceID == "" || + (input.SourceType != SourceOwned && input.SourceType != SourceCompetitor) || + utf8.RuneCountInString(input.WorkKey) > 255 || utf8.RuneCountInString(input.Title) > 1000 || + utf8.RuneCountInString(input.Body) > 100000 || utf8.RuneCountInString(input.OriginalURL) > 2000 || + utf8.RuneCountInString(input.CoverURL) > 2000 { + return Work{}, false, ErrInvalid + } + if now.IsZero() { + now = time.Now().UTC() + } + status := input.PublishedAtStatus + if status == "" { + status = "verified" + if input.PublishedAt == nil { + status = "pending_verification" + } + } + if status != "verified" && status != "pending_verification" && status != "invalid" && status != "future" { + return Work{}, false, ErrInvalid + } + if input.PublishedAt != nil { + input.PublishedAt = ptrTime(input.PublishedAt.UTC()) + if input.PublishedAt.After(now.UTC()) && status == "verified" { + status = "future" + } + } + if input.Likes != nil && *input.Likes < 0 || input.CommentsCount != nil && *input.CommentsCount < 0 || input.Shares != nil && *input.Shares < 0 { + return Work{}, false, ErrInvalid + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Work{}, false, fmt.Errorf("begin creator work upsert: %w", err) + } + defer tx.Rollback() + id := newID("work") + var returnedID string + var inserted bool + err = tx.QueryRowContext(ctx, ` + INSERT INTO creator_work (id, platform, work_key, source_type, source_id, author_name, title, body, + published_at, published_at_status, original_url, cover_url, likes, comments_count, shares) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT (platform, work_key) DO UPDATE SET + author_name = CASE WHEN EXCLUDED.author_name = '' THEN creator_work.author_name ELSE EXCLUDED.author_name END, + title = CASE WHEN EXCLUDED.title = '' THEN creator_work.title ELSE EXCLUDED.title END, + body = CASE WHEN EXCLUDED.body = '' THEN creator_work.body ELSE EXCLUDED.body END, + published_at = COALESCE(EXCLUDED.published_at, creator_work.published_at), + published_at_status = CASE WHEN EXCLUDED.published_at IS NULL THEN creator_work.published_at_status ELSE EXCLUDED.published_at_status END, + original_url = CASE WHEN EXCLUDED.original_url = '' THEN creator_work.original_url ELSE EXCLUDED.original_url END, + cover_url = CASE WHEN EXCLUDED.cover_url = '' THEN creator_work.cover_url ELSE EXCLUDED.cover_url END, + likes = COALESCE(EXCLUDED.likes, creator_work.likes), + comments_count = COALESCE(EXCLUDED.comments_count, creator_work.comments_count), + shares = COALESCE(EXCLUDED.shares, creator_work.shares), updated_at = now() + RETURNING id, (xmax = 0)`, id, input.Platform, input.WorkKey, input.SourceType, input.SourceID, + input.AuthorName, input.Title, input.Body, input.PublishedAt, status, input.OriginalURL, input.CoverURL, + input.Likes, input.CommentsCount, input.Shares).Scan(&returnedID, &inserted) + if err != nil { + return Work{}, false, databaseError(err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO creator_work_source (work_id, platform, source_type, source_id) + VALUES ($1, $2, $3, $4) ON CONFLICT (work_id, source_type, source_id) DO NOTHING`, + returnedID, input.Platform, input.SourceType, input.SourceID); err != nil { + return Work{}, false, databaseError(err) + } + result, err := scanWork(tx.QueryRowContext(ctx, workSelect+` WHERE id = $1`, returnedID)) + if err != nil { + return Work{}, false, rowError(err) + } + if err := tx.Commit(); err != nil { + return Work{}, false, fmt.Errorf("commit creator work upsert: %w", err) + } + if err := s.loadWorkSources(ctx, &result); err != nil { + return Work{}, false, err + } + return result, inserted, nil +} + +func ptrTime(value time.Time) *time.Time { return &value } + +func scanWork(scanner interface{ Scan(...any) error }) (Work, error) { + var result Work + var publishedAt, latestAt, nextAt sql.NullTime + var likes, commentsCount, shares sql.NullInt64 + if err := scanner.Scan(&result.ID, &result.Platform, &result.WorkKey, &result.SourceType, &result.SourceID, + &result.AuthorName, &result.Title, &result.Body, &publishedAt, &result.PublishedAtStatus, + &result.OriginalURL, &result.CoverURL, &likes, &commentsCount, &shares, &latestAt, &nextAt, + &result.MetricStopReason, &result.CreatedAt, &result.UpdatedAt); err != nil { + return Work{}, err + } + result.PublishedAt = nullableTime(publishedAt) + result.Likes, result.CommentsCount, result.Shares = nullableInt64(likes), nullableInt64(commentsCount), nullableInt64(shares) + result.LatestMetricsAt, result.NextMetricAt = nullableTime(latestAt), nullableTime(nextAt) + return result, nil +} + +const workSelect = `SELECT id, platform, work_key, source_type, source_id, author_name, title, body, + published_at, published_at_status, original_url, cover_url, likes, comments_count, shares, + latest_metrics_at, next_metric_at, metric_stop_reason, created_at, updated_at FROM creator_work` + +func (s *Store) loadWorkSources(ctx context.Context, work *Work) error { + rows, err := s.db.QueryContext(ctx, ` + SELECT platform, source_type, source_id + FROM creator_work_source WHERE work_id = $1 + ORDER BY source_type, source_id`, work.ID) + if err != nil { + return databaseError(err) + } + defer rows.Close() + work.Sources = make([]WorkSource, 0) + for rows.Next() { + var source WorkSource + if err := rows.Scan(&source.Platform, &source.SourceType, &source.SourceID); err != nil { + return err + } + work.Sources = append(work.Sources, source) + } + return rows.Err() +} + +func (s *Store) GetWork(ctx context.Context, id string) (Work, error) { + result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE id = $1`, id)) + if err != nil { + return Work{}, rowError(err) + } + if err := s.loadWorkSources(ctx, &result); err != nil { + return Work{}, err + } + return result, nil +} + +func (s *Store) ListWorks(ctx context.Context, filter WorkFilter) ([]Work, error) { + query, args := workSelect, make([]any, 0, 8) + where := make([]string, 0, 7) + add := func(clause string, value any) { + args = append(args, value) + where = append(where, fmt.Sprintf(clause, len(args))) + } + if filter.Platform != "" { + if !ValidatePlatform(filter.Platform) { + return nil, ErrInvalid + } + add("platform = $%d", filter.Platform) + } + if filter.SourceType != "" { + if filter.SourceType != SourceOwned && filter.SourceType != SourceCompetitor { + return nil, ErrInvalid + } + args = append(args, filter.SourceType) + where = append(where, fmt.Sprintf(`EXISTS (SELECT 1 FROM creator_work_source ws WHERE ws.work_id = creator_work.id AND ws.source_type = $%d)`, len(args))) + } + if filter.SourceID != "" { + args = append(args, filter.SourceID) + where = append(where, fmt.Sprintf(`EXISTS (SELECT 1 FROM creator_work_source ws WHERE ws.work_id = creator_work.id AND ws.source_id = $%d)`, len(args))) + } + if filter.PublishedAfter != nil { + add("published_at >= $%d", filter.PublishedAfter.UTC()) + } + if filter.PublishedBefore != nil { + add("published_at <= $%d", filter.PublishedBefore.UTC()) + } + if filter.MinLikes != nil { + if *filter.MinLikes < 0 { + return nil, ErrInvalid + } + where = append(where, fmt.Sprintf("likes IS NOT NULL AND likes >= $%d", len(args)+1)) + args = append(args, *filter.MinLikes) + } + if filter.MinComments != nil { + if *filter.MinComments < 0 { + return nil, ErrInvalid + } + where = append(where, fmt.Sprintf("comments_count IS NOT NULL AND comments_count >= $%d", len(args)+1)) + args = append(args, *filter.MinComments) + } + if filter.MinShares != nil { + if *filter.MinShares < 0 { + return nil, ErrInvalid + } + where = append(where, fmt.Sprintf("shares IS NOT NULL AND shares >= $%d", len(args)+1)) + args = append(args, *filter.MinShares) + } + if len(where) > 0 { + query += ` WHERE ` + strings.Join(where, ` AND `) + } + query += ` ORDER BY published_at DESC NULLS LAST, id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Work, 0) + for rows.Next() { + item, err := scanWork(rows) + if err != nil { + return nil, err + } + if err := s.loadWorkSources(ctx, &item); err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func (s *Store) RecordMetric(ctx context.Context, input MetricInput, settings Settings, now time.Time) (MetricPoint, error) { + if input.WorkID == "" || input.CollectedAt.IsZero() || input.Likes != nil && *input.Likes < 0 || input.CommentsCount != nil && *input.CommentsCount < 0 || input.Shares != nil && *input.Shares < 0 { + return MetricPoint{}, ErrInvalid + } + if err := ValidateSettings(SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds}); err != nil { + return MetricPoint{}, err + } + if now.IsZero() { + now = time.Now().UTC() + } + return s.recordMetricWithPlan(ctx, input, settings, now) +} + +func nullableArg(value time.Time) any { + if value.IsZero() { + return nil + } + return value.UTC() +} + +func (s *Store) ListMetrics(ctx context.Context, workID string) ([]MetricPoint, error) { + rows, err := s.db.QueryContext(ctx, `SELECT collected_at, likes, comments_count, shares FROM creator_work_metric WHERE work_id = $1 ORDER BY collected_at`, workID) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]MetricPoint, 0) + for rows.Next() { + var point MetricPoint + var likes, commentsCount, shares sql.NullInt64 + if err := rows.Scan(&point.CollectedAt, &likes, &commentsCount, &shares); err != nil { + return nil, err + } + point.CollectedAt = point.CollectedAt.UTC() + point.Likes, point.CommentsCount, point.Shares = nullableInt64(likes), nullableInt64(commentsCount), nullableInt64(shares) + result = append(result, point) + } + return result, rows.Err() +} + +func (s *Store) GetMaterial(ctx context.Context, workID string) (MaterialJob, error) { + if _, err := s.db.ExecContext(ctx, `INSERT INTO creator_material_job (work_id) VALUES ($1) ON CONFLICT (work_id) DO NOTHING`, workID); err != nil { + return MaterialJob{}, databaseError(err) + } + return s.scanMaterial(s.db.QueryRowContext(ctx, `SELECT work_id, selected, select_confirmed_at, download_status, video_reference, audio_status, audio_reference, transcription_status, transcript, failed_step, failure_reason, rewrite_confirmed_at, rewrite_requirement, generated_title, generated_script, created_at, updated_at FROM creator_material_job WHERE work_id = $1`, workID)) +} + +func (s *Store) scanMaterial(scanner interface{ Scan(...any) error }) (MaterialJob, error) { + var result MaterialJob + var selectedAt, rewriteAt sql.NullTime + if err := scanner.Scan(&result.WorkID, &result.Selected, &selectedAt, &result.DownloadStatus, &result.VideoReference, &result.AudioStatus, &result.AudioReference, &result.TranscriptionStatus, &result.Transcript, &result.FailedStep, &result.FailureReason, &rewriteAt, &result.RewriteRequirement, &result.GeneratedTitle, &result.GeneratedScript, &result.CreatedAt, &result.UpdatedAt); err != nil { + return MaterialJob{}, rowError(err) + } + result.SelectConfirmedAt, result.RewriteConfirmedAt = nullableTime(selectedAt), nullableTime(rewriteAt) + return result, nil +} + +func (s *Store) SelectMaterial(ctx context.Context, workID string) (MaterialJob, bool, error) { + if _, err := s.GetWork(ctx, workID); err != nil { + return MaterialJob{}, false, err + } + var inserted bool + if err := s.db.QueryRowContext(ctx, `INSERT INTO creator_material_job (work_id, selected, select_confirmed_at) VALUES ($1, true, now()) ON CONFLICT (work_id) DO UPDATE SET selected = true, select_confirmed_at = COALESCE(creator_material_job.select_confirmed_at, now()), updated_at = now() RETURNING (xmax = 0)`, workID).Scan(&inserted); err != nil { + return MaterialJob{}, false, databaseError(err) + } + job, err := s.GetMaterial(ctx, workID) + return job, inserted, err +} + +func (s *Store) SetMaterialStep(ctx context.Context, workID, step, status, reference, reason string) (MaterialJob, error) { + if step != "download" && step != "audio" && step != "transcription" { + return MaterialJob{}, ErrInvalid + } + valid := map[string]map[string]bool{ + "download": {"not_started": true, "running": true, "succeeded": true, "failed": true}, + "audio": {"not_started": true, "running": true, "succeeded": true, "no_audio": true, "failed": true}, + "transcription": {"not_started": true, "running": true, "succeeded": true, "no_speech": true, "failed": true}, + } + if !valid[step][status] || len(reference) > 2000 || len(reason) > 2000 { + return MaterialJob{}, ErrInvalid + } + job, err := s.GetMaterial(ctx, workID) + if err != nil { + return MaterialJob{}, err + } + if !job.Selected { + return MaterialJob{}, ErrConflict + } + var query string + var args []any + switch step { + case "download": + query = `UPDATE creator_material_job SET download_status = $2, video_reference = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'download' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1` + args = []any{workID, status, reference, reason} + case "audio": + query = `UPDATE creator_material_job SET audio_status = $2, audio_reference = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'audio' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1` + args = []any{workID, status, reference, reason} + case "transcription": + query = `UPDATE creator_material_job SET transcription_status = $2, transcript = $3, failed_step = CASE WHEN $2 = 'failed' THEN 'transcription' ELSE failed_step END, failure_reason = CASE WHEN $2 = 'failed' THEN $4 ELSE failure_reason END, updated_at = now() WHERE work_id = $1` + args = []any{workID, status, reference, reason} + } + if _, err := s.db.ExecContext(ctx, query, args...); err != nil { + return MaterialJob{}, databaseError(err) + } + return s.GetMaterial(ctx, workID) +} + +func (s *Store) ConfirmRewrite(ctx context.Context, workID, requirement string) (MaterialJob, error) { + requirement = strings.TrimSpace(requirement) + if utf8.RuneCountInString(requirement) > 4000 { + return MaterialJob{}, ErrInvalid + } + job, err := s.GetMaterial(ctx, workID) + if err != nil { + return MaterialJob{}, err + } + if !job.Selected || job.DownloadStatus != "succeeded" || job.AudioStatus == "failed" || job.TranscriptionStatus == "failed" || job.AudioStatus == "running" || job.TranscriptionStatus == "running" || job.AudioStatus == "not_started" || job.TranscriptionStatus == "not_started" { + return MaterialJob{}, ErrConflict + } + if _, err := s.db.ExecContext(ctx, `UPDATE creator_material_job SET rewrite_confirmed_at = now(), rewrite_requirement = $2, updated_at = now() WHERE work_id = $1`, workID, requirement); err != nil { + return MaterialJob{}, databaseError(err) + } + return s.GetMaterial(ctx, workID) +} + +func (s *Store) SaveRewrite(ctx context.Context, workID, title, script string) (MaterialJob, error) { + title, script = strings.TrimSpace(title), strings.TrimSpace(script) + if title == "" || script == "" || utf8.RuneCountInString(title) > 1000 || utf8.RuneCountInString(script) > 100000 { + return MaterialJob{}, ErrInvalid + } + job, err := s.GetMaterial(ctx, workID) + if err != nil { + return MaterialJob{}, err + } + if job.RewriteConfirmedAt == nil { + return MaterialJob{}, ErrConflict + } + if _, err := s.db.ExecContext(ctx, `UPDATE creator_material_job SET generated_title = $2, generated_script = $3, updated_at = now() WHERE work_id = $1`, workID, title, script); err != nil { + return MaterialJob{}, databaseError(err) + } + return s.GetMaterial(ctx, workID) +} + +func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, bool, error) { + input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.CommentType = strings.TrimSpace(input.Platform), strings.TrimSpace(input.CommentKey), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.AuthorUID), strings.TrimSpace(input.AuthorName), strings.TrimSpace(input.CommentType) + if !ValidatePlatform(input.Platform) || input.CommentKey == "" || input.WorkID == "" || strings.TrimSpace(input.Content) == "" || (input.CommentType != "top_level" && input.CommentType != "unknown") || utf8.RuneCountInString(input.Content) > 10000 { + return Comment{}, false, ErrInvalid + } + work, err := s.GetWork(ctx, input.WorkID) + if err != nil { + return Comment{}, false, err + } + if work.Platform != input.Platform { + return Comment{}, false, ErrInvalid + } + id := newID("comment") + var returnedID string + var inserted bool + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_comment (id, platform, comment_key, work_id, author_uid, author_name, content, published_at, comment_type) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (platform, comment_key) DO UPDATE SET work_id = EXCLUDED.work_id, author_uid = EXCLUDED.author_uid, author_name = EXCLUDED.author_name, content = EXCLUDED.content, published_at = EXCLUDED.published_at, comment_type = EXCLUDED.comment_type RETURNING id, (xmax = 0)`, id, input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.Content, input.PublishedAt, input.CommentType).Scan(&returnedID, &inserted) + if err != nil { + return Comment{}, false, databaseError(err) + } + result, err := s.GetComment(ctx, returnedID) + return result, inserted, err +} + +func scanComment(scanner interface{ Scan(...any) error }) (Comment, error) { + var result Comment + var publishedAt sql.NullTime + if err := scanner.Scan(&result.ID, &result.Platform, &result.CommentKey, &result.WorkID, &result.AuthorUID, &result.AuthorName, &result.Content, &publishedAt, &result.CollectedAt, &result.CommentType); err != nil { + return Comment{}, err + } + result.PublishedAt = nullableTime(publishedAt) + result.CollectedAt = result.CollectedAt.UTC() + return result, nil +} + +const commentSelect = `SELECT id, platform, comment_key, work_id, author_uid, author_name, content, published_at, collected_at, comment_type FROM creator_comment` + +func (s *Store) GetComment(ctx context.Context, id string) (Comment, error) { + result, err := scanComment(s.db.QueryRowContext(ctx, commentSelect+` WHERE id = $1`, id)) + return result, rowError(err) +} + +func (s *Store) ListComments(ctx context.Context, platform, workID string) ([]Comment, error) { + query, args := commentSelect, []any{} + where := []string{} + if platform != "" { + if !ValidatePlatform(platform) { + return nil, ErrInvalid + } + args = append(args, platform) + where = append(where, fmt.Sprintf("platform = $%d", len(args))) + } + if workID != "" { + args = append(args, workID) + where = append(where, fmt.Sprintf("work_id = $%d", len(args))) + } + if len(where) > 0 { + query += ` WHERE ` + strings.Join(where, ` AND `) + } + query += ` ORDER BY published_at DESC NULLS LAST, collected_at DESC, id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Comment, 0) + for rows.Next() { + item, err := scanComment(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} diff --git a/internal/creator/integration_test.go b/internal/creator/integration_test.go new file mode 100644 index 0000000..e958153 --- /dev/null +++ b/internal/creator/integration_test.go @@ -0,0 +1,427 @@ +package creator + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "os" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/phasea" + _ "github.com/jackc/pgx/v5/stdlib" +) + +type integrationCredentialBridge struct{} + +func (integrationCredentialBridge) Store(context.Context, phasea.CredentialReference, string, string) error { + return nil +} + +func (integrationCredentialBridge) Delete(context.Context, phasea.CredentialReference, string) error { + return nil +} + +type integrationAnalyzer struct{} + +func (integrationAnalyzer) MatchTheme(context.Context, string, string, string) (bool, string, error) { + return true, "主题匹配", nil +} + +func (integrationAnalyzer) MatchLead(context.Context, string, string, string) (bool, string, error) { + return true, "具备线索意向", nil +} + +type integrationExecutor struct{} + +func (integrationExecutor) Execute(context.Context, ActionRequest) (ActionResult, error) { + return ActionResult{State: "succeeded", Evidence: map[string]string{"platform_id": "creator-it"}}, nil +} + +type integrationCollector struct { + work WorkInput + comment CommentInput +} + +func (c integrationCollector) VerifyIdentity(context.Context, string) error { return nil } + +func (c integrationCollector) ListWorks(_ context.Context, _, cursor string) (WorkPage, error) { + if cursor != "" { + return WorkPage{Items: nil, HasMore: false}, nil + } + return WorkPage{Items: []WorkInput{c.work}, HasMore: true, NextCursor: "1"}, nil +} + +func (c integrationCollector) ListTopLevelComments(_ context.Context, _, cursor string) (CommentPage, error) { + if cursor != "" { + return CommentPage{Items: nil, HasMore: false}, nil + } + return CommentPage{Items: []CommentInput{c.comment}, HasMore: false}, nil +} + +func openCreatorIntegrationStore(t *testing.T) (*Store, *phasea.Store, context.Context) { + t.Helper() + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run CreatorHub PostgreSQL integration coverage") + } + ctx := context.Background() + admin, err := sql.Open("pgx", databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = admin.Close() }) + if err := admin.PingContext(ctx); err != nil { + t.Fatal(err) + } + schema := fmt.Sprintf("creatorhub_it_%d", time.Now().UnixNano()) + if _, err := admin.ExecContext(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.ExecContext(ctx, "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Errorf("drop test schema: %v", err) + } + }) + parsed, err := url.Parse(databaseURL) + if err != nil { + t.Fatal(err) + } + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + testURL := parsed.String() + + phaseAStore, err := phasea.Open(ctx, testURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = phaseAStore.Close() }) + hubStore, err := hub.Open(ctx, testURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hubStore.Close() }) + store, err := Open(ctx, testURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + return store, phaseAStore, ctx +} + +func createIntegrationAccount(t *testing.T, ctx context.Context, phaseAStore *phasea.Store, suffix string) string { + t.Helper() + id := "cit" + suffix + account := phasea.Account{ + ID: id, + Name: "Creator integration " + suffix, + Platform: PlatformDouyin, + PlatformAccountKey: "sec_uid_" + id, + Tags: []string{"integration"}, + CredentialReference: phasea.CredentialReference{ID: "credential_" + id, Provider: "os_keyring"}, + CredentialKey: "creatorhub/" + id, + } + if err := phaseAStore.CreateAccount(ctx, account, integrationCredentialBridge{}); err != nil { + t.Fatal(err) + } + return id +} + +func TestCreatorPostgresContentAndWorkflow(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := fmt.Sprintf("%d", time.Now().UnixNano()) + bigID := createIntegrationAccount(t, ctx, phaseAStore, "big"+stamp) + smallID := createIntegrationAccount(t, ctx, phaseAStore, "small"+stamp) + if err := store.EnsureAccountProfile(ctx, bigID); err != nil { + t.Fatal(err) + } + if err := store.EnsureAccountProfile(ctx, smallID); err != nil { + t.Fatal(err) + } + profiles, err := store.ListAccountProfiles(ctx) + if err != nil || len(profiles) != 2 { + t.Fatalf("list account profiles: profiles=%+v err=%v", profiles, err) + } + if _, err := store.UpdateAccountProfile(ctx, bigID, AccountProfileUpdate{RealNameStatus: "unknown", BusinessStatus: "normal", BigAccount: true, ReplyRequirements: "保持准确", CooldownSeconds: 86400}); err != nil { + t.Fatal(err) + } + if _, err := store.UpdateAccountProfile(ctx, smallID, AccountProfileUpdate{RealNameStatus: "unknown", BusinessStatus: "normal", CooldownSeconds: 86400}); err != nil { + t.Fatal(err) + } + if _, err := store.RecordLoginResult(ctx, bigID, "logged_in", "", "sec_uid_"+bigID); err != nil { + t.Fatal(err) + } + if _, err := store.RecordLoginResult(ctx, smallID, "logged_in", "", "sec_uid_"+smallID); err != nil { + t.Fatal(err) + } + if err := store.SetRelation(ctx, bigID, smallID, true); err != nil { + t.Fatal(err) + } + relations, err := store.ListRelations(ctx, bigID) + if err != nil || len(relations) != 1 { + t.Fatalf("list relations: relations=%+v err=%v", relations, err) + } + if _, err := store.SetBigAccount(ctx, smallID, true); !errors.Is(err, ErrConflict) { + t.Fatalf("expected small-account promotion to be rejected, got %v", err) + } + + now := time.Now().UTC().Truncate(time.Microsecond) + ownedDue, err := store.ListDueOwnedAccounts(ctx, now, 1800) + if err != nil || len(ownedDue) != 1 || ownedDue[0] != bigID { + t.Fatalf("list due owned accounts: accounts=%+v err=%v", ownedDue, err) + } + published := now.Add(-2 * time.Hour) + likes, comments, shares := int64(10), int64(2), int64(1) + workKey := "creator-it-work-" + stamp + work, inserted, err := store.UpsertWork(ctx, WorkInput{Platform: PlatformDouyin, WorkKey: workKey, SourceType: SourceOwned, SourceID: bigID, Title: "Topic", Body: "Need consultation", PublishedAt: &published, PublishedAtStatus: "verified", OriginalURL: "https://www.douyin.com/video/" + workKey, Likes: &likes, CommentsCount: &comments, Shares: &shares}, now) + if err != nil || !inserted { + t.Fatalf("insert work: work=%+v inserted=%v err=%v", work, inserted, err) + } + competitor, err := store.CreateCompetitor(ctx, CompetitorInput{Platform: PlatformDouyin, PlatformAccountKey: "sec_uid_competitor_" + stamp, Nickname: "Competitor", HomepageURL: "https://www.douyin.com/user/sec_uid_competitor_" + stamp}) + if err != nil { + t.Fatal(err) + } + competitors, err := store.ListCompetitors(ctx, PlatformDouyin) + if err != nil || len(competitors) != 1 { + t.Fatalf("list competitors: competitors=%+v err=%v", competitors, err) + } + if _, err := store.SetCompetitorEnabled(ctx, competitor.ID, false); err != nil { + t.Fatal(err) + } + if _, err := store.SetCompetitorEnabled(ctx, competitor.ID, true); err != nil { + t.Fatal(err) + } + dueNow := time.Now().UTC().Add(time.Minute) + if due, err := store.ListDueCompetitors(ctx, dueNow); err != nil || len(due) != 1 { + t.Fatalf("list due competitors: due=%+v err=%v", due, err) + } + claimed, err := store.ClaimCompetitorSync(ctx, competitor.ID, false, dueNow) + if err != nil || !claimed { + t.Fatalf("claim competitor sync: claimed=%v err=%v", claimed, err) + } + if err := store.MarkCompetitorSync(ctx, competitor.ID, "idle", "", "", nil); err != nil { + t.Fatal(err) + } + work, inserted, err = store.UpsertWork(ctx, WorkInput{Platform: PlatformDouyin, WorkKey: workKey, SourceType: SourceCompetitor, SourceID: competitor.ID, Title: "", Body: "", PublishedAt: nil, Likes: nil, CommentsCount: nil, Shares: nil}, now) + if err != nil || inserted || len(work.Sources) != 2 { + t.Fatalf("work source association was not retained: work=%+v inserted=%v err=%v", work, inserted, err) + } + filtered, err := store.ListWorks(ctx, WorkFilter{Platform: PlatformDouyin, SourceType: SourceCompetitor}) + if err != nil || len(filtered) != 1 || filtered[0].ID != work.ID { + t.Fatalf("competitor source filter failed: works=%+v err=%v", filtered, err) + } + + comment, inserted, err := store.SaveComment(ctx, CommentInput{Platform: PlatformDouyin, CommentKey: "creator-it-comment-" + stamp, WorkID: work.ID, Content: "咨询价格", CommentType: "top_level"}) + if err != nil || !inserted || comment.AuthorUID != "" { + t.Fatalf("save comment with missing UID: comment=%+v inserted=%v err=%v", comment, inserted, err) + } + if _, duplicate, err := store.SaveComment(ctx, CommentInput{Platform: PlatformDouyin, CommentKey: comment.CommentKey, WorkID: work.ID, Content: "咨询价格", CommentType: "top_level"}); err != nil || duplicate { + t.Fatalf("comment deduplication failed: duplicate=%v err=%v", duplicate, err) + } + if _, err := store.RecordMetric(ctx, MetricInput{WorkID: work.ID, CollectedAt: now, Likes: &likes, CommentsCount: &comments, Shares: &shares}, Settings{LookbackDays: 30, NewWorkIntervalSeconds: 1800, MetricInitialIntervalSeconds: 3600, MetricMultiplier: 2, MetricMaxIntervalSeconds: 86400, MetricAgeSeconds: 30 * 24 * 60 * 60}, now); err != nil { + t.Fatal(err) + } + metrics, err := store.ListMetrics(ctx, work.ID) + if err != nil || len(metrics) != 1 { + t.Fatalf("initial metric snapshot failed: metrics=%+v err=%v", metrics, err) + } + if _, err := store.RecordMetric(ctx, MetricInput{WorkID: work.ID, CollectedAt: now.Add(-time.Minute), Likes: &likes, CommentsCount: &comments, Shares: &shares}, Settings{LookbackDays: 30, NewWorkIntervalSeconds: 1800, MetricInitialIntervalSeconds: 3600, MetricMultiplier: 2, MetricMaxIntervalSeconds: 86400, MetricAgeSeconds: 30 * 24 * 60 * 60}, now); !errors.Is(err, ErrConflict) { + t.Fatalf("expected early metric point to be rejected, got %v", err) + } + if _, err := store.RecordMetric(ctx, MetricInput{WorkID: work.ID, CollectedAt: now.Add(time.Minute), Likes: &likes, CommentsCount: &comments, Shares: &shares}, Settings{LookbackDays: 30, NewWorkIntervalSeconds: 1800, MetricInitialIntervalSeconds: 3600, MetricMultiplier: 2, MetricMaxIntervalSeconds: 86400, MetricAgeSeconds: 30 * 24 * 60 * 60}, now.Add(-time.Minute)); !errors.Is(err, ErrInvalid) { + t.Fatalf("expected future metric point to be rejected, got %v", err) + } + + rule, err := store.CreateRule(ctx, LeadRuleInput{Name: "咨询线索", Enabled: true, SourceType: SourceOwned, Topic: "咨询", IncludeKeywords: []string{"咨询"}, ExcludeKeywords: []string{"招聘"}, AIRequirement: "判断购买意向"}) + if err != nil { + t.Fatal(err) + } + rules, err := store.ListRules(ctx, false) + if err != nil || len(rules) != 1 { + t.Fatalf("list rules: rules=%+v err=%v", rules, err) + } + updatedRule, err := store.UpdateRule(ctx, rule.ID, LeadRuleInput{Name: rule.Name, Enabled: true, SourceType: SourceOwned, Topic: rule.Topic, IncludeKeywords: []string{"咨询"}, ExcludeKeywords: []string{"招聘"}, AIRequirement: rule.AIRequirement}) + if err != nil || updatedRule.ID != rule.ID { + t.Fatalf("update rule: rule=%+v err=%v", updatedRule, err) + } + result, err := store.AnalyzeComment(ctx, comment.ID, rule.ID, integrationAnalyzer{}) + if err != nil || result.Status != "lead" { + t.Fatalf("analyze comment: result=%+v err=%v", result, err) + } + ruleResults, err := store.ListRuleResults(ctx, comment.ID, rule.ID) + if err != nil || len(ruleResults) != 1 { + t.Fatalf("list rule results: results=%+v err=%v", ruleResults, err) + } + leads, err := store.ListLeads(ctx, PlatformDouyin) + if err != nil || len(leads) != 1 { + t.Fatalf("list leads: leads=%+v err=%v", leads, err) + } + if _, err := store.SetRuleEnabled(ctx, rule.ID, false); err != nil { + t.Fatal(err) + } + + material, selected, err := store.SelectMaterial(ctx, work.ID) + if err != nil || !selected || !material.Selected { + t.Fatalf("select material: material=%+v selected=%v err=%v", material, selected, err) + } + for step, status := range map[string]string{"download": "succeeded", "audio": "no_audio", "transcription": "no_speech"} { + if _, err := store.SetMaterialStep(ctx, work.ID, step, status, "ref-"+step, ""); err != nil { + t.Fatal(err) + } + } + if _, err := store.ConfirmRewrite(ctx, work.ID, "仿写同主题但不复制原文"); err != nil { + t.Fatal(err) + } + if _, err := store.SaveRewrite(ctx, work.ID, "新标题", "新口播"); err != nil { + t.Fatal(err) + } + + collector := integrationCollector{work: WorkInput{Platform: PlatformDouyin, WorkKey: "creator-it-collected-" + stamp, SourceType: SourceOwned, SourceID: bigID, Title: "Collected", Body: "body", PublishedAt: &published, PublishedAtStatus: "verified", Likes: &likes}, comment: CommentInput{Platform: PlatformDouyin, CommentKey: "creator-it-collected-comment-" + stamp, WorkID: "", Content: "hello", CommentType: "top_level"}} + collector.comment.WorkID = "" + report, err := store.CollectSource(ctx, PlatformDouyin, SourceOwned, bigID, collector, now) + if err != nil || !report.PaginationComplete || report.WorksSeen != 1 || report.CommentsSaved != 2 { + t.Fatalf("collect source workflow failed: report=%+v err=%v", report, err) + } +} + +func prepareIntegrationActionFixture(t *testing.T, store *Store, phaseAStore *phasea.Store, ctx context.Context, stamp string) (string, string, Work, Comment, Strategy) { + t.Helper() + bigID := createIntegrationAccount(t, ctx, phaseAStore, "big"+stamp) + smallID := createIntegrationAccount(t, ctx, phaseAStore, "small"+stamp) + for _, accountID := range []string{bigID, smallID} { + if err := store.EnsureAccountProfile(ctx, accountID); err != nil { + t.Fatal(err) + } + if _, err := store.UpdateAccountProfile(ctx, accountID, AccountProfileUpdate{RealNameStatus: "unknown", BusinessStatus: "normal", CooldownSeconds: 86400}); err != nil { + t.Fatal(err) + } + if _, err := store.RecordLoginResult(ctx, accountID, "logged_in", "", "sec_uid_"+accountID); err != nil { + t.Fatal(err) + } + } + if _, err := store.SetBigAccount(ctx, bigID, true); err != nil { + t.Fatal(err) + } + if err := store.SetRelation(ctx, bigID, smallID, true); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Microsecond) + published := now.Add(-time.Hour) + likes, comments, shares := int64(1), int64(1), int64(1) + work, _, err := store.UpsertWork(ctx, WorkInput{Platform: PlatformDouyin, WorkKey: "creator-it-action-work-" + stamp, SourceType: SourceOwned, SourceID: bigID, Title: "Action", Body: "body", PublishedAt: &published, PublishedAtStatus: "verified", Likes: &likes, CommentsCount: &comments, Shares: &shares}, now) + if err != nil { + t.Fatal(err) + } + comment, _, err := store.SaveComment(ctx, CommentInput{Platform: PlatformDouyin, CommentKey: "creator-it-action-comment-" + stamp, WorkID: work.ID, AuthorUID: "interactor-" + stamp, Content: "hello", CommentType: "top_level"}) + if err != nil { + t.Fatal(err) + } + strategy, err := store.CreateStrategy(ctx, bigID, StrategyInput{ExecutionAccountID: smallID, Position: 1, Enabled: true, EventTypes: []string{"comment"}, Action: ActionReplyComment, TargetType: "comment", CandidateTexts: []string{"已收到"}}) + if err != nil { + t.Fatal(err) + } + strategies, err := store.ListStrategies(ctx, bigID) + if err != nil || len(strategies) != 1 { + t.Fatalf("list strategies: strategies=%+v err=%v", strategies, err) + } + return bigID, smallID, work, comment, strategy +} + +func TestCreatorPostgresActionsAndMessaging(t *testing.T) { + store, phaseAStore, ctx := openCreatorIntegrationStore(t) + stamp := fmt.Sprintf("%d", time.Now().UnixNano()) + bigID, smallID, work, comment, strategy := prepareIntegrationActionFixture(t, store, phaseAStore, ctx, stamp) + event := InteractionEvent{Platform: PlatformDouyin, ReceivingAccountID: bigID, EventKey: "creator-it-event-" + stamp, EventType: "comment", InteractorUID: comment.AuthorUID, CommentID: comment.ID, WorkID: work.ID} + automatic, err := store.ProcessAutomaticEvent(ctx, event, integrationExecutor{}, nil) + if err != nil || automatic.Operation == nil || automatic.Operation.State != "succeeded" || automatic.Event.State != "succeeded" { + t.Fatalf("automatic action failed: result=%+v err=%v", automatic, err) + } + events, err := store.ListEvents(ctx, bigID) + if err != nil || len(events) != 1 { + t.Fatalf("list events: events=%+v err=%v", events, err) + } + duplicate, err := store.ProcessAutomaticEvent(ctx, event, integrationExecutor{}, nil) + if err != nil || !duplicate.Duplicate || duplicate.Event.ID != automatic.Event.ID { + t.Fatalf("automatic event deduplication failed: result=%+v err=%v", duplicate, err) + } + cooldownEvent := event + cooldownEvent.EventKey += "-cooldown" + blocked, err := store.ProcessAutomaticEvent(ctx, cooldownEvent, integrationExecutor{}, nil) + if err != nil || blocked.Event.State != "blocked" { + t.Fatalf("cooldown did not block second event: result=%+v err=%v", blocked, err) + } + manualInput := OperationInput{IdempotencyKey: "creator-it-manual-" + stamp, Source: "manual", Action: ActionReplyComment, Platform: PlatformDouyin, AccountID: smallID, TargetUID: comment.AuthorUID, TargetCommentID: comment.ID, Text: "人工回复"} + manual, inserted, err := store.CreateOperation(ctx, manualInput) + if err != nil || !inserted { + t.Fatalf("create manual operation: operation=%+v inserted=%v err=%v", manual, inserted, err) + } + manualDuplicate, inserted, err := store.CreateOperation(ctx, manualInput) + if err != nil || inserted || manualDuplicate.ID != manual.ID { + t.Fatalf("manual operation idempotency failed: operation=%+v inserted=%v err=%v", manualDuplicate, inserted, err) + } + operations, err := store.ListOperations(ctx, smallID) + if err != nil || len(operations) != 2 { + t.Fatalf("list operations: operations=%+v err=%v", operations, err) + } + manual, err = store.ExecuteManualOperation(ctx, manual.ID, integrationExecutor{}) + if err != nil || manual.State != "succeeded" { + t.Fatalf("execute manual operation: operation=%+v err=%v", manual, err) + } + messageAt := time.Now().UTC().Truncate(time.Microsecond) + message, inserted, err := store.SaveMessage(ctx, MessageInput{Platform: PlatformDouyin, AccountID: smallID, PeerUID: "peer-" + stamp, PeerName: "Peer", PlatformMessageKey: "creator-it-message-" + stamp, Direction: "inbound", MessageType: "text", Text: "hello", MessageAt: &messageAt}) + if err != nil || !inserted { + t.Fatalf("save message: message=%+v inserted=%v err=%v", message, inserted, err) + } + if _, inserted, err := store.SaveMessage(ctx, MessageInput{Platform: PlatformDouyin, AccountID: smallID, PeerUID: "peer-" + stamp, PlatformMessageKey: message.PlatformMessageKey, Direction: "inbound", MessageType: "text", Text: "hello", MessageAt: &messageAt}); err != nil || inserted { + t.Fatalf("message deduplication failed: inserted=%v err=%v", inserted, err) + } + conversations, err := store.ListConversations(ctx, smallID) + if err != nil || len(conversations) != 1 { + t.Fatalf("list conversations: conversations=%+v err=%v", conversations, err) + } + messages, err := store.ListMessages(ctx, conversations[0].ID) + if err != nil || len(messages) != 1 { + t.Fatalf("list messages: messages=%+v err=%v", messages, err) + } + if _, err := store.SetEventDisplayed(ctx, automatic.Event.ID, time.Now().UTC()); err != nil { + t.Fatal(err) + } + updated, err := store.UpdateStrategy(ctx, strategy.ID, StrategyInput{ExecutionAccountID: smallID, Position: 2, Enabled: true, EventTypes: []string{"comment", "like"}, Action: ActionReplyComment, TargetType: "comment", CandidateTexts: []string{"已更新"}}) + if err != nil || updated.Position != 2 || len(updated.EventTypes) != 2 { + t.Fatalf("update strategy: strategy=%+v err=%v", updated, err) + } + if _, err := store.SetStrategyEnabled(ctx, strategy.ID, false); err != nil { + t.Fatal(err) + } + if err := store.DeleteStrategy(ctx, strategy.ID); err != nil { + t.Fatal(err) + } + if err := store.SetRelation(ctx, bigID, smallID, false); err != nil { + t.Fatal(err) + } + if _, err := store.SetBigAccount(ctx, bigID, false); err != nil { + t.Fatal(err) + } + + start, end := time.Now().UTC().Add(-time.Hour), time.Now().UTC() + lease, err := store.beginCheckpoint(ctx, SourceOwned, bigID, "works", start, end) + if err != nil { + t.Fatal(err) + } + if _, err := store.beginCheckpoint(ctx, SourceOwned, bigID, "works", start, end); !errors.Is(err, ErrConflict) { + t.Fatalf("expected concurrent checkpoint claim conflict, got %v", err) + } + if err := store.finishCheckpoint(ctx, SourceOwned, bigID, "works", "stale-lease", "succeeded", ""); !errors.Is(err, ErrConflict) { + t.Fatalf("expected stale checkpoint completion conflict, got %v", err) + } + if err := store.saveCheckpointCursor(ctx, SourceOwned, bigID, "works", lease, "1"); err != nil { + t.Fatal(err) + } + if err := store.finishCheckpoint(ctx, SourceOwned, bigID, "works", lease, "succeeded", ""); err != nil { + t.Fatal(err) + } +} diff --git a/internal/creator/logic.go b/internal/creator/logic.go new file mode 100644 index 0000000..2b02101 --- /dev/null +++ b/internal/creator/logic.go @@ -0,0 +1,218 @@ +package creator + +import ( + "errors" + "math" + "strings" + "time" +) + +var ( + ErrInvalid = errors.New("invalid creator input") + ErrConflict = errors.New("creator resource conflicts with existing state") + ErrNotFound = errors.New("creator resource not found") + ErrUnavailable = errors.New("creator capability is unavailable") + ErrUncertain = errors.New("creator operation result is uncertain") +) + +func ValidatePlatform(platform string) bool { + return platform == PlatformDouyin || platform == PlatformXiaohongshu +} + +const maxDurationSeconds = int64(math.MaxInt64 / int64(time.Second)) + +func ValidateSettings(input SettingsUpdate) error { + if input.LookbackDays <= 0 || int64(input.LookbackDays) > maxDurationSeconds/(24*60*60) || + input.NewWorkIntervalSeconds <= 0 || input.NewWorkIntervalSeconds > maxDurationSeconds || + input.MetricInitialIntervalSeconds <= 0 || input.MetricInitialIntervalSeconds > maxDurationSeconds || + math.IsNaN(input.MetricMultiplier) || math.IsInf(input.MetricMultiplier, 0) || input.MetricMultiplier <= 1 || + input.MetricMaxIntervalSeconds < input.MetricInitialIntervalSeconds || input.MetricMaxIntervalSeconds > maxDurationSeconds || + input.MetricAgeSeconds <= 0 || input.MetricAgeSeconds > maxDurationSeconds { + return ErrInvalid + } + return nil +} + +// NextMetricAt returns the next scheduled point strictly after now. A point +// exactly at now is considered covered by the current collection and is not +// returned again. +func NextMetricAt(publishedAt, now time.Time, initial, max time.Duration, multiplier float64, monitoringAge time.Duration) (time.Time, string) { + publishedAt, now = publishedAt.UTC(), now.UTC() + if publishedAt.IsZero() || !publishedAt.Before(now) || initial <= 0 || max < initial || math.IsNaN(multiplier) || math.IsInf(multiplier, 0) || multiplier <= 1 || monitoringAge <= 0 { + return time.Time{}, "published_at_pending_verification" + } + if !now.Before(publishedAt.Add(monitoringAge)) { + return time.Time{}, "monitoring_age_reached" + } + + elapsed := now.Sub(publishedAt) + interval := initial + total := initial + for total <= elapsed { + candidate := float64(interval) * multiplier + var next time.Duration + if math.IsNaN(candidate) || math.IsInf(candidate, 0) || candidate < float64(interval) || candidate > float64(max) { + next = max + } else { + next = time.Duration(candidate) + if next < interval { + next = max + } + } + interval = next + if total > time.Duration(math.MaxInt64)-interval { + return time.Time{}, "monitoring_age_reached" + } + total += interval + } + candidate := publishedAt.Add(total) + if !candidate.Before(publishedAt.Add(monitoringAge)) { + return time.Time{}, "monitoring_age_reached" + } + return candidate, "" +} + +func MetricSchedule(publishedAt time.Time, initial, max time.Duration, multiplier float64, count int) []time.Time { + if count <= 0 || publishedAt.IsZero() || initial <= 0 || max < initial || math.IsNaN(multiplier) || math.IsInf(multiplier, 0) || multiplier <= 1 { + return []time.Time{} + } + points := make([]time.Time, 0, count) + interval, total := initial, initial + for len(points) < count { + points = append(points, publishedAt.UTC().Add(total)) + candidate := float64(interval) * multiplier + var next time.Duration + if math.IsNaN(candidate) || math.IsInf(candidate, 0) || candidate < float64(interval) || candidate > float64(max) { + next = max + } else { + next = time.Duration(candidate) + if next < interval { + next = max + } + } + interval = next + if total > time.Duration(math.MaxInt64)-interval { + break + } + total += interval + } + return points +} + +// TrimKeyword removes only the edge whitespace allowed by the product rule. +// It intentionally does not normalize case, Unicode, full-width characters, +// or whitespace inside a keyword. +func TrimKeyword(value string) string { return strings.Trim(value, " \t\r\n") } + +func NormalizeKeywords(values []string) ([]string, error) { + result := make([]string, len(values)) + for index, value := range values { + result[index] = TrimKeyword(value) + if result[index] == "" { + return nil, ErrInvalid + } + } + return result, nil +} + +func ValidateRuleInput(input LeadRuleInput) (LeadRuleInput, error) { + input.Name = strings.TrimSpace(input.Name) + input.Topic = strings.TrimSpace(input.Topic) + input.AIRequirement = strings.TrimSpace(input.AIRequirement) + input.SourceType = strings.TrimSpace(input.SourceType) + if input.SourceType == "" { + input.SourceType = "all" + } + if input.Name == "" || input.Topic == "" || input.AIRequirement == "" || + (input.SourceType != "all" && input.SourceType != SourceOwned && input.SourceType != SourceCompetitor) { + return LeadRuleInput{}, ErrInvalid + } + var err error + input.IncludeKeywords, err = NormalizeKeywords(input.IncludeKeywords) + if err != nil || len(input.IncludeKeywords) == 0 { + return LeadRuleInput{}, ErrInvalid + } + input.ExcludeKeywords, err = NormalizeKeywords(input.ExcludeKeywords) + if err != nil { + return LeadRuleInput{}, ErrInvalid + } + return input, nil +} + +func MatchKeywords(content string, include, exclude []string) (bool, []string) { + matched := make([]string, 0, len(include)) + for _, keyword := range include { + if strings.Contains(content, keyword) { + matched = append(matched, keyword) + } + } + if len(matched) == 0 { + return false, matched + } + for _, keyword := range exclude { + if strings.Contains(content, keyword) { + return false, matched + } + } + return true, matched +} + +func ValidEventType(eventType string) bool { + switch eventType { + case "comment", "like", "repost", "follow", "dm": + return true + default: + return false + } +} + +func ValidAction(action string) bool { + switch action { + case ActionDM, ActionReplyComment, ActionLikeComment, ActionLikeWork, ActionFollow, ActionRepost: + return true + default: + return false + } +} + +func ActionRequiresText(action string) bool { + return action == ActionDM || action == ActionReplyComment || action == ActionRepost +} + +func ActionTargetValid(action string, interactorUID, commentID, workID, targetType string) bool { + if interactorUID == "" { + return false + } + switch action { + case ActionDM, ActionFollow: + return true + case ActionReplyComment, ActionLikeComment: + return commentID != "" + case ActionLikeWork: + return workID != "" + case ActionRepost: + return workID != "" + default: + return false + } +} + +func CanWrite(profile AccountProfile, automatic bool, action string) error { + if !ValidAction(action) { + return ErrInvalid + } + if profile.BusinessStatus != "normal" { + if automatic || profile.BusinessStatus == "banned" || profile.BusinessStatus == "deleted" || + (action == ActionReplyComment || action == ActionDM || action == ActionRepost) { + return ErrConflict + } + } + if profile.LoginStatus != "logged_in" || profile.AuthorizationStatus != "authorized" { + return ErrConflict + } + return nil +} + +func InWindow(publishedAt, start, end time.Time) bool { + return !publishedAt.IsZero() && !publishedAt.Before(start.UTC()) && !publishedAt.After(end.UTC()) +} diff --git a/internal/creator/logic_test.go b/internal/creator/logic_test.go new file mode 100644 index 0000000..fcd0892 --- /dev/null +++ b/internal/creator/logic_test.go @@ -0,0 +1,80 @@ +package creator + +import ( + "math" + "testing" + "time" +) + +func TestMetricScheduleUsesUTCAndCumulativeIntervals(t *testing.T) { + published := time.Date(2026, 1, 1, 0, 0, 0, 0, time.FixedZone("CST", 8*60*60)) + points := MetricSchedule(published, time.Hour, 24*time.Hour, 2, 6) + want := []time.Duration{time.Hour, 3 * time.Hour, 7 * time.Hour, 15 * time.Hour, 31 * time.Hour, 55 * time.Hour} + for index, duration := range want { + if got := points[index].Sub(published.UTC()); got != duration { + t.Fatalf("point %d: got %s, want %s", index, got, duration) + } + } + if next, reason := NextMetricAt(published, published.Add(3*time.Hour), time.Hour, 24*time.Hour, 2, 30*24*time.Hour); next != published.UTC().Add(7*time.Hour) || reason != "" { + t.Fatalf("strict next point: got %s/%q", next, reason) + } + if next, reason := NextMetricAt(published, published.Add(30*24*time.Hour), time.Hour, 24*time.Hour, 2, 30*24*time.Hour); !next.IsZero() || reason != "monitoring_age_reached" { + t.Fatalf("age boundary: got %s/%q", next, reason) + } +} + +func TestKeywordMatchingDoesNotNormalizeText(t *testing.T) { + if got := TrimKeyword(" \tAI \r\n"); got != "AI" { + t.Fatalf("trimmed keyword = %q", got) + } + if ok, _ := MatchKeywords("AI 买 车", []string{"AI"}, nil); !ok { + t.Fatal("expected exact substring match") + } + if ok, _ := MatchKeywords("ai 买车", []string{"AI"}, nil); ok { + t.Fatal("case should remain significant") + } + if ok, _ := MatchKeywords("AI 买车", []string{"买 车"}, nil); ok { + t.Fatal("internal whitespace should remain significant") + } + if ok, _ := MatchKeywords("AI 买 车", []string{"AI"}, []string{"买 车"}); ok { + t.Fatal("exclude keyword should veto the match") + } +} + +func TestValidateSettingsRejectsUnsafeNumbers(t *testing.T) { + base := SettingsUpdate{LookbackDays: 30, NewWorkIntervalSeconds: 1800, MetricInitialIntervalSeconds: 3600, MetricMultiplier: 2, MetricMaxIntervalSeconds: 86400, MetricAgeSeconds: 30 * 24 * 60 * 60} + for _, mutate := range []func(*SettingsUpdate){ + func(value *SettingsUpdate) { value.MetricMultiplier = math.NaN() }, + func(value *SettingsUpdate) { value.MetricAgeSeconds = maxDurationSeconds + 1 }, + func(value *SettingsUpdate) { value.LookbackDays = int(maxDurationSeconds/(24*60*60) + 1) }, + } { + value := base + mutate(&value) + if err := ValidateSettings(value); err != ErrInvalid { + t.Fatalf("expected invalid settings, got %v", err) + } + } +} + +func TestNextMetricIntervalRejectsNonFiniteMultiplier(t *testing.T) { + if _, err := nextMetricInterval(3600, math.NaN(), 86400); err != ErrInvalid { + t.Fatalf("expected NaN multiplier to be invalid, got %v", err) + } + if next, err := nextMetricInterval(3600, math.Inf(1), 86400); err != nil || next != 86400 { + t.Fatalf("expected infinite growth to clamp at maximum, got %d/%v", next, err) + } +} + +func TestActionTargetAndBusinessStatusGates(t *testing.T) { + if ActionTargetValid(ActionReplyComment, "uid-1", "", "", "comment") { + t.Fatal("reply without a comment target must be blocked") + } + profile := AccountProfile{AuthorizationStatus: "authorized", LoginStatus: "logged_in", BusinessStatus: "muted"} + if err := CanWrite(profile, false, ActionDM); err == nil { + t.Fatal("muted accounts cannot manually send DMs") + } + profile.BusinessStatus = "normal" + if err := CanWrite(profile, false, ActionDM); err != nil { + t.Fatalf("normal logged-in account should be writable: %v", err) + } +} diff --git a/internal/creator/metrics.go b/internal/creator/metrics.go new file mode 100644 index 0000000..1712329 --- /dev/null +++ b/internal/creator/metrics.go @@ -0,0 +1,212 @@ +package creator + +import ( + "context" + "database/sql" + "fmt" + "math" + "time" +) + +func (s *Store) EnsureMetricPlan(ctx context.Context, workID string, settings Settings) error { + if err := ValidateSettings(SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds}); err != nil { + return err + } + work, err := s.GetWork(ctx, workID) + if err != nil { + return err + } + if work.PublishedAt == nil || work.PublishedAtStatus != "verified" { + return nil + } + now := time.Now().UTC() + publishedAt := work.PublishedAt.UTC() + monitoringEnd := publishedAt.Add(time.Duration(settings.MetricAgeSeconds) * time.Second) + if !publishedAt.Before(now) { + _, err := s.db.ExecContext(ctx, `UPDATE creator_work SET next_metric_at=NULL, metric_stop_reason='published_at_pending_verification', updated_at=now() WHERE id=$1`, workID) + return databaseError(err) + } + if !now.Before(monitoringEnd) { + if _, err := s.db.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=NULL, stopped=true, stop_reason='monitoring_age_reached', updated_at=now() WHERE work_id=$1 AND NOT stopped`, workID); err != nil { + return databaseError(err) + } + _, err := s.db.ExecContext(ctx, `UPDATE creator_work SET next_metric_at=NULL, metric_stop_reason='monitoring_age_reached', updated_at=now() WHERE id=$1`, workID) + return databaseError(err) + } + nextAt := publishedAt.Add(time.Duration(settings.MetricInitialIntervalSeconds) * time.Second) + if !nextAt.Before(monitoringEnd) { + _, err := s.db.ExecContext(ctx, `UPDATE creator_work SET next_metric_at=NULL, metric_stop_reason='monitoring_age_reached', updated_at=now() WHERE id=$1`, workID) + return databaseError(err) + } + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO creator_metric_plan (work_id, published_at, monitoring_end, next_plan_at, + interval_seconds, multiplier, max_interval_seconds) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (work_id) DO NOTHING`, workID, publishedAt, monitoringEnd, nextAt, + settings.MetricInitialIntervalSeconds, settings.MetricMultiplier, settings.MetricMaxIntervalSeconds); err != nil { + return databaseError(err) + } + _, err = s.db.ExecContext(ctx, ` + UPDATE creator_work SET + next_metric_at=COALESCE((SELECT next_plan_at FROM creator_metric_plan WHERE work_id=$1 AND NOT stopped), next_metric_at), + metric_stop_reason=CASE WHEN EXISTS (SELECT 1 FROM creator_metric_plan WHERE work_id=$1 AND NOT stopped) THEN '' ELSE metric_stop_reason END, + updated_at=now() WHERE id=$1`, workID) + return databaseError(err) +} + +func nextMetricInterval(interval int64, multiplier float64, maximum int64) (int64, error) { + if interval <= 0 || math.IsNaN(multiplier) || multiplier <= 1 || maximum < interval { + return 0, ErrInvalid + } + if interval == maximum || math.IsInf(multiplier, 0) { + return maximum, nil + } + candidate := float64(interval) * multiplier + if math.IsNaN(candidate) { + return 0, ErrInvalid + } + if math.IsInf(candidate, 0) || candidate >= float64(maximum) || candidate < float64(interval) { + return maximum, nil + } + result := int64(math.Round(candidate)) + if result <= interval { + result = interval + 1 + } + if result > maximum { + result = maximum + } + return result, nil +} + +func (s *Store) recordMetricWithPlan(ctx context.Context, input MetricInput, settings Settings, now time.Time) (MetricPoint, error) { + collectedAt := input.CollectedAt.UTC() + now = now.UTC() + if collectedAt.After(now) { + return MetricPoint{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return MetricPoint{}, fmt.Errorf("begin creator metric: %w", err) + } + defer tx.Rollback() + var publishedAt sql.NullTime + var publishedStatus string + var latestAt sql.NullTime + if err := tx.QueryRowContext(ctx, `SELECT published_at, published_at_status, latest_metrics_at FROM creator_work WHERE id=$1 FOR UPDATE`, input.WorkID).Scan(&publishedAt, &publishedStatus, &latestAt); err != nil { + return MetricPoint{}, rowError(err) + } + point := MetricPoint{CollectedAt: collectedAt, Likes: input.Likes, CommentsCount: input.CommentsCount, Shares: input.Shares} + if !publishedAt.Valid || publishedStatus != "verified" { + if _, err := tx.ExecContext(ctx, `UPDATE creator_work SET likes=$2, comments_count=$3, shares=$4, latest_metrics_at=$5, next_metric_at=NULL, metric_stop_reason='published_at_pending_verification', updated_at=now() WHERE id=$1`, input.WorkID, input.Likes, input.CommentsCount, input.Shares, collectedAt); err != nil { + return MetricPoint{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return MetricPoint{}, fmt.Errorf("commit creator metric: %w", err) + } + return point, nil + } + published := publishedAt.Time.UTC() + monitoringEnd := published.Add(time.Duration(settings.MetricAgeSeconds) * time.Second) + if !published.Before(collectedAt) { + return MetricPoint{}, ErrConflict + } + if latestAt.Valid && latestAt.Time.UTC().Equal(collectedAt) { + return point, nil + } + + var nextPlan sql.NullTime + var interval int64 + var multiplier float64 + var maximum int64 + var pointIndex int + var stopped bool + var stopReason string + if latestAt.Valid { + if !now.Before(monitoringEnd) || !collectedAt.Before(monitoringEnd) { + if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=NULL, stopped=true, stop_reason='monitoring_age_reached', updated_at=now() WHERE work_id=$1 AND NOT stopped`, input.WorkID); err != nil { + return MetricPoint{}, databaseError(err) + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_work SET next_metric_at=NULL, metric_stop_reason='monitoring_age_reached', updated_at=now() WHERE id=$1`, input.WorkID); err != nil { + return MetricPoint{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return MetricPoint{}, fmt.Errorf("commit stopped creator metric plan: %w", err) + } + return MetricPoint{}, ErrConflict + } + } + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_metric_plan (work_id,published_at,monitoring_end,next_plan_at,interval_seconds,multiplier,max_interval_seconds) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (work_id) DO NOTHING`, input.WorkID, published, monitoringEnd, published.Add(time.Duration(settings.MetricInitialIntervalSeconds)*time.Second), settings.MetricInitialIntervalSeconds, settings.MetricMultiplier, settings.MetricMaxIntervalSeconds); err != nil { + return MetricPoint{}, databaseError(err) + } + if err := tx.QueryRowContext(ctx, `SELECT next_plan_at, interval_seconds, multiplier, max_interval_seconds, point_index, stopped, stop_reason FROM creator_metric_plan WHERE work_id=$1 FOR UPDATE`, input.WorkID).Scan(&nextPlan, &interval, &multiplier, &maximum, &pointIndex, &stopped, &stopReason); err != nil { + return MetricPoint{}, rowError(err) + } + if !latestAt.Valid { + nextAt, nextReason := NextMetricAt(published, now, time.Duration(settings.MetricInitialIntervalSeconds)*time.Second, time.Duration(settings.MetricMaxIntervalSeconds)*time.Second, settings.MetricMultiplier, time.Duration(settings.MetricAgeSeconds)*time.Second) + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_work_metric (work_id,collected_at,likes,comments_count,shares) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (work_id,collected_at) DO UPDATE SET likes=EXCLUDED.likes,comments_count=EXCLUDED.comments_count,shares=EXCLUDED.shares`, input.WorkID, collectedAt, input.Likes, input.CommentsCount, input.Shares); err != nil { + return MetricPoint{}, databaseError(err) + } + if nextAt.IsZero() || stopped || !nextPlan.Valid { + if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=NULL, stopped=true, stop_reason=$2, updated_at=now() WHERE work_id=$1`, input.WorkID, coalesceReason(nextReason, "monitoring_age_reached")); err != nil { + return MetricPoint{}, databaseError(err) + } + nextAt = time.Time{} + nextReason = coalesceReason(nextReason, "monitoring_age_reached") + } else if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=$2, stopped=false, stop_reason='', updated_at=now() WHERE work_id=$1`, input.WorkID, nextAt); err != nil { + return MetricPoint{}, databaseError(err) + } else { + nextReason = "" + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_work SET likes=$2,comments_count=$3,shares=$4,latest_metrics_at=$5,next_metric_at=$6,metric_stop_reason=$7,updated_at=now() WHERE id=$1`, input.WorkID, input.Likes, input.CommentsCount, input.Shares, collectedAt, nullableArg(nextAt), nextReason); err != nil { + return MetricPoint{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return MetricPoint{}, fmt.Errorf("commit initial creator metric: %w", err) + } + return point, nil + } + if stopped || !nextPlan.Valid || collectedAt.Before(nextPlan.Time.UTC()) { + return MetricPoint{}, ErrConflict + } + newInterval, err := nextMetricInterval(interval, multiplier, maximum) + if err != nil { + return MetricPoint{}, err + } + nextAt := nextPlan.Time.UTC().Add(time.Duration(newInterval) * time.Second) + steps := 1 + for !nextAt.After(now) && nextAt.Before(monitoringEnd) { + newInterval, err = nextMetricInterval(newInterval, multiplier, maximum) + if err != nil { + return MetricPoint{}, err + } + nextAt = nextAt.Add(time.Duration(newInterval) * time.Second) + steps++ + } + stopped = !nextAt.Before(monitoringEnd) + if stopped { + nextAt = time.Time{} + stopReason = "monitoring_age_reached" + } else { + stopReason = "" + } + if _, err := tx.ExecContext(ctx, `INSERT INTO creator_work_metric (work_id,collected_at,likes,comments_count,shares) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (work_id,collected_at) DO UPDATE SET likes=EXCLUDED.likes,comments_count=EXCLUDED.comments_count,shares=EXCLUDED.shares`, input.WorkID, collectedAt, input.Likes, input.CommentsCount, input.Shares); err != nil { + return MetricPoint{}, databaseError(err) + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_metric_plan SET next_plan_at=$2, interval_seconds=$3, point_index=point_index+$4, stopped=$5, stop_reason=$6, updated_at=now() WHERE work_id=$1`, input.WorkID, nullableArg(nextAt), newInterval, steps, stopped, stopReason); err != nil { + return MetricPoint{}, databaseError(err) + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_work SET likes=$2,comments_count=$3,shares=$4,latest_metrics_at=$5,next_metric_at=$6,metric_stop_reason=$7,updated_at=now() WHERE id=$1`, input.WorkID, input.Likes, input.CommentsCount, input.Shares, collectedAt, nullableArg(nextAt), stopReason); err != nil { + return MetricPoint{}, databaseError(err) + } + if err := tx.Commit(); err != nil { + return MetricPoint{}, fmt.Errorf("commit creator metric plan: %w", err) + } + return point, nil +} + +func coalesceReason(value, fallback string) string { + if value != "" { + return value + } + return fallback +} diff --git a/internal/creator/migrations/017_creator.sql b/internal/creator/migrations/017_creator.sql new file mode 100644 index 0000000..ddaba71 --- /dev/null +++ b/internal/creator/migrations/017_creator.sql @@ -0,0 +1,244 @@ +CREATE TABLE IF NOT EXISTS creator_settings ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + lookback_days integer NOT NULL DEFAULT 30 CHECK (lookback_days > 0), + new_work_interval_seconds bigint NOT NULL DEFAULT 1800 CHECK (new_work_interval_seconds > 0), + metric_initial_interval_seconds bigint NOT NULL DEFAULT 3600 CHECK (metric_initial_interval_seconds > 0), + metric_multiplier double precision NOT NULL DEFAULT 2 CHECK (metric_multiplier > 1), + metric_max_interval_seconds bigint NOT NULL DEFAULT 86400 CHECK (metric_max_interval_seconds >= metric_initial_interval_seconds), + metric_age_seconds bigint NOT NULL DEFAULT 2592000 CHECK (metric_age_seconds > 0), + ai_provider text NOT NULL DEFAULT '', + ai_model text NOT NULL DEFAULT '', + ai_configured boolean NOT NULL DEFAULT false, + transcription_provider text NOT NULL DEFAULT '', + transcription_model text NOT NULL DEFAULT '', + transcription_configured boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now() +); +INSERT INTO creator_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING; + +CREATE TABLE IF NOT EXISTS creator_account_profile ( + account_id text PRIMARY KEY REFERENCES social_account(id) ON DELETE CASCADE, + login_username text NOT NULL DEFAULT '', + password_configured boolean NOT NULL DEFAULT false, + real_name_status text NOT NULL DEFAULT 'unknown' CHECK (real_name_status IN ('unknown', 'not_real_name', 'recorded')), + real_name text NOT NULL DEFAULT '', + identity_number text NOT NULL DEFAULT '', + note text NOT NULL DEFAULT '', + business_status text NOT NULL DEFAULT 'normal' CHECK (business_status IN ('normal', 'muted', 'banned', 'deleted')), + big_account boolean NOT NULL DEFAULT false, + reply_requirements text NOT NULL DEFAULT '', + login_status text NOT NULL DEFAULT 'unknown' CHECK (login_status IN ('unknown', 'logged_in', 'needs_login', 'failed', 'manual_required')), + login_reason text NOT NULL DEFAULT '', + login_checked_at timestamptz, + cooldown_seconds bigint NOT NULL DEFAULT 86400 CHECK (cooldown_seconds > 0), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS creator_competitor ( + id text PRIMARY KEY, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + platform_account_key text NOT NULL, + nickname text NOT NULL DEFAULT '', + avatar_url text NOT NULL DEFAULT '', + homepage_url text NOT NULL, + enabled boolean NOT NULL DEFAULT true, + sync_status text NOT NULL DEFAULT 'idle' CHECK (sync_status IN ('idle', 'running', 'paused', 'failed', 'blocked')), + sync_cursor text NOT NULL DEFAULT '', + sync_error text NOT NULL DEFAULT '', + last_sync_at timestamptz, + next_sync_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (platform, platform_account_key) +); + +CREATE TABLE IF NOT EXISTS creator_work ( + id text PRIMARY KEY, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + work_key text NOT NULL, + source_type text NOT NULL CHECK (source_type IN ('owned', 'competitor')), + source_id text NOT NULL, + author_name text NOT NULL DEFAULT '', + title text NOT NULL DEFAULT '', + body text NOT NULL DEFAULT '', + published_at timestamptz, + published_at_status text NOT NULL DEFAULT 'verified' CHECK (published_at_status IN ('verified', 'pending_verification', 'invalid', 'future')), + original_url text NOT NULL DEFAULT '', + cover_url text NOT NULL DEFAULT '', + likes bigint, + comments_count bigint, + shares bigint, + latest_metrics_at timestamptz, + next_metric_at timestamptz, + metric_stop_reason text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (platform, work_key) +); +CREATE INDEX IF NOT EXISTS creator_work_source_idx ON creator_work (source_type, source_id, published_at DESC); + +CREATE TABLE IF NOT EXISTS creator_work_metric ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + work_id text NOT NULL REFERENCES creator_work(id) ON DELETE CASCADE, + collected_at timestamptz NOT NULL DEFAULT now(), + likes bigint, + comments_count bigint, + shares bigint, + UNIQUE (work_id, collected_at) +); + +CREATE TABLE IF NOT EXISTS creator_material_job ( + work_id text PRIMARY KEY REFERENCES creator_work(id) ON DELETE CASCADE, + selected boolean NOT NULL DEFAULT false, + select_confirmed_at timestamptz, + download_status text NOT NULL DEFAULT 'not_started' CHECK (download_status IN ('not_started', 'running', 'succeeded', 'failed')), + video_reference text NOT NULL DEFAULT '', + audio_status text NOT NULL DEFAULT 'not_started' CHECK (audio_status IN ('not_started', 'running', 'succeeded', 'no_audio', 'failed')), + audio_reference text NOT NULL DEFAULT '', + transcription_status text NOT NULL DEFAULT 'not_started' CHECK (transcription_status IN ('not_started', 'running', 'succeeded', 'no_speech', 'failed')), + transcript text NOT NULL DEFAULT '', + failed_step text NOT NULL DEFAULT '', + failure_reason text NOT NULL DEFAULT '', + rewrite_confirmed_at timestamptz, + rewrite_requirement text NOT NULL DEFAULT '', + generated_title text NOT NULL DEFAULT '', + generated_script text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS creator_comment ( + id text PRIMARY KEY, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + comment_key text NOT NULL, + work_id text NOT NULL REFERENCES creator_work(id) ON DELETE CASCADE, + author_uid text NOT NULL DEFAULT '', + author_name text NOT NULL DEFAULT '', + content text NOT NULL, + published_at timestamptz, + collected_at timestamptz NOT NULL DEFAULT now(), + comment_type text NOT NULL DEFAULT 'top_level' CHECK (comment_type IN ('top_level', 'reply', 'unknown')), + UNIQUE (platform, comment_key) +); +CREATE INDEX IF NOT EXISTS creator_comment_work_idx ON creator_comment (work_id, published_at DESC); + +CREATE TABLE IF NOT EXISTS creator_lead_rule ( + id text PRIMARY KEY, + name text NOT NULL, + enabled boolean NOT NULL DEFAULT true, + source_type text NOT NULL DEFAULT 'all' CHECK (source_type IN ('all', 'owned', 'competitor')), + topic text NOT NULL, + include_keywords jsonb NOT NULL, + exclude_keywords jsonb NOT NULL DEFAULT '[]'::jsonb, + ai_requirement text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS creator_comment_rule_result ( + comment_id text NOT NULL REFERENCES creator_comment(id) ON DELETE CASCADE, + rule_id text NOT NULL REFERENCES creator_lead_rule(id) ON DELETE CASCADE, + status text NOT NULL CHECK (status IN ('not_analysed', 'topic_no_match', 'keyword_no_match', 'not_lead', 'lead', 'failed')), + reason text NOT NULL DEFAULT '', + matched_keywords jsonb NOT NULL DEFAULT '[]'::jsonb, + rule_snapshot jsonb NOT NULL, + analysed_at timestamptz, + PRIMARY KEY (comment_id, rule_id) +); + +CREATE TABLE IF NOT EXISTS creator_relation ( + big_account_id text NOT NULL REFERENCES social_account(id) ON DELETE CASCADE, + small_account_id text PRIMARY KEY REFERENCES social_account(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT now(), + CHECK (big_account_id <> small_account_id) +); + +CREATE TABLE IF NOT EXISTS creator_strategy ( + id text PRIMARY KEY, + big_account_id text NOT NULL REFERENCES social_account(id) ON DELETE CASCADE, + execution_account_id text NOT NULL REFERENCES social_account(id) ON DELETE RESTRICT, + position integer NOT NULL CHECK (position > 0), + enabled boolean NOT NULL DEFAULT true, + event_types jsonb NOT NULL, + action text NOT NULL CHECK (action IN ('dm', 'reply_comment', 'like_comment', 'like_work', 'follow', 'repost')), + target_type text NOT NULL DEFAULT '', + candidate_texts jsonb NOT NULL DEFAULT '[]'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (big_account_id, position) +); + +CREATE TABLE IF NOT EXISTS creator_cooldown ( + big_account_id text NOT NULL REFERENCES social_account(id) ON DELETE CASCADE, + interactor_uid text NOT NULL, + event_id text NOT NULL, + strategy_id text NOT NULL, + execution_account_id text NOT NULL, + started_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (big_account_id, interactor_uid) +); + +CREATE TABLE IF NOT EXISTS creator_event ( + id text PRIMARY KEY, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + receiving_account_id text NOT NULL REFERENCES social_account(id) ON DELETE CASCADE, + event_key text NOT NULL, + event_type text NOT NULL, + interactor_uid text NOT NULL DEFAULT '', + comment_id text, + work_id text, + platform_event_at timestamptz, + received_at timestamptz NOT NULL DEFAULT now(), + processing_started_at timestamptz, + displayed_at timestamptz, + state text NOT NULL DEFAULT 'received' CHECK (state IN ('received', 'baseline', 'ignored', 'unmatched', 'blocked', 'processing', 'succeeded', 'failed', 'uncertain')), + reason text NOT NULL DEFAULT '', + strategy_id text, + execution_account_id text, + UNIQUE (platform, receiving_account_id, event_key) +); + +CREATE TABLE IF NOT EXISTS creator_operation ( + id text PRIMARY KEY, + idempotency_key text NOT NULL UNIQUE, + source text NOT NULL CHECK (source IN ('manual', 'automatic')), + action text NOT NULL, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + account_id text NOT NULL REFERENCES social_account(id) ON DELETE RESTRICT, + target_uid text NOT NULL DEFAULT '', + target_comment_id text NOT NULL DEFAULT '', + target_work_id text NOT NULL DEFAULT '', + text text NOT NULL DEFAULT '', + event_id text, + strategy_id text, + request_hash text NOT NULL, + state text NOT NULL DEFAULT 'created' CHECK (state IN ('created', 'processing', 'succeeded', 'failed', 'uncertain', 'blocked')), + evidence jsonb NOT NULL DEFAULT '{}'::jsonb, + reason text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS creator_conversation ( + id text PRIMARY KEY, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + account_id text NOT NULL REFERENCES social_account(id) ON DELETE CASCADE, + peer_uid text NOT NULL, + peer_name text NOT NULL DEFAULT '', + last_message_at timestamptz, + UNIQUE (account_id, peer_uid) +); + +CREATE TABLE IF NOT EXISTS creator_message ( + id text PRIMARY KEY, + conversation_id text NOT NULL REFERENCES creator_conversation(id) ON DELETE CASCADE, + platform_message_key text NOT NULL DEFAULT '', + direction text NOT NULL CHECK (direction IN ('inbound', 'outbound')), + message_type text NOT NULL DEFAULT 'text', + text text NOT NULL DEFAULT '', + sent_state text NOT NULL DEFAULT 'received' CHECK (sent_state IN ('received', 'pending', 'succeeded', 'failed', 'uncertain')), + message_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (conversation_id, platform_message_key) +); diff --git a/internal/creator/migrations/018_collection_checkpoints.sql b/internal/creator/migrations/018_collection_checkpoints.sql new file mode 100644 index 0000000..b09fef2 --- /dev/null +++ b/internal/creator/migrations/018_collection_checkpoints.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS creator_collection_checkpoint ( + id text PRIMARY KEY, + source_type text NOT NULL CHECK (source_type IN ('owned', 'competitor')), + source_id text NOT NULL, + collection_kind text NOT NULL CHECK ( + collection_kind IN ('works', 'comments') + ), + cursor text NOT NULL DEFAULT '', + window_start timestamptz NOT NULL, + window_end timestamptz NOT NULL, + status text NOT NULL DEFAULT 'idle' CHECK ( + status IN ('idle', 'running', 'succeeded', 'failed', 'blocked') + ), + lease_until timestamptz, + last_started_at timestamptz, + last_completed_at timestamptz, + last_error text NOT NULL DEFAULT '', + UNIQUE (source_type, source_id, collection_kind) +); +CREATE INDEX IF NOT EXISTS creator_collection_due_idx ON creator_collection_checkpoint ( + status, lease_until, last_completed_at +); diff --git a/internal/creator/migrations/019_metric_plans.sql b/internal/creator/migrations/019_metric_plans.sql new file mode 100644 index 0000000..f40aa75 --- /dev/null +++ b/internal/creator/migrations/019_metric_plans.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS creator_metric_plan ( + work_id text PRIMARY KEY REFERENCES creator_work (id) ON DELETE CASCADE, + published_at timestamptz NOT NULL, + monitoring_end timestamptz NOT NULL, + next_plan_at timestamptz, + interval_seconds bigint NOT NULL, + multiplier double precision NOT NULL, + max_interval_seconds bigint NOT NULL, + point_index integer NOT NULL DEFAULT 0 CHECK (point_index >= 0), + stopped boolean NOT NULL DEFAULT false, + stop_reason text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); diff --git a/internal/creator/migrations/020_competitor_sync_leases.sql b/internal/creator/migrations/020_competitor_sync_leases.sql new file mode 100644 index 0000000..fc4995e --- /dev/null +++ b/internal/creator/migrations/020_competitor_sync_leases.sql @@ -0,0 +1,2 @@ +ALTER TABLE creator_competitor + ADD COLUMN IF NOT EXISTS sync_lease_until timestamptz; diff --git a/internal/creator/migrations/021_work_sources.sql b/internal/creator/migrations/021_work_sources.sql new file mode 100644 index 0000000..a0362bf --- /dev/null +++ b/internal/creator/migrations/021_work_sources.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS creator_work_source ( + work_id text NOT NULL REFERENCES creator_work(id) ON DELETE CASCADE, + platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')), + source_type text NOT NULL CHECK (source_type IN ('owned', 'competitor')), + source_id text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (work_id, source_type, source_id) +); +CREATE INDEX IF NOT EXISTS creator_work_source_lookup_idx + ON creator_work_source (platform, source_type, source_id, work_id); + +INSERT INTO creator_work_source (work_id, platform, source_type, source_id) +SELECT id, platform, source_type, source_id +FROM creator_work +ON CONFLICT (work_id, source_type, source_id) DO NOTHING; diff --git a/internal/creator/migrations/022_collection_lease_tokens.sql b/internal/creator/migrations/022_collection_lease_tokens.sql new file mode 100644 index 0000000..f5d0da1 --- /dev/null +++ b/internal/creator/migrations/022_collection_lease_tokens.sql @@ -0,0 +1,2 @@ +ALTER TABLE creator_collection_checkpoint + ADD COLUMN IF NOT EXISTS lease_token text NOT NULL DEFAULT ''; diff --git a/internal/creator/models.go b/internal/creator/models.go new file mode 100644 index 0000000..96b9a3c --- /dev/null +++ b/internal/creator/models.go @@ -0,0 +1,431 @@ +package creator + +import ( + "context" + "time" +) + +const ( + PlatformDouyin = "douyin" + PlatformXiaohongshu = "xiaohongshu" + + SourceOwned = "owned" + SourceCompetitor = "competitor" + + ActionDM = "dm" + ActionReplyComment = "reply_comment" + ActionLikeComment = "like_comment" + ActionLikeWork = "like_work" + ActionFollow = "follow" + ActionRepost = "repost" +) + +type Settings struct { + LookbackDays int `json:"lookback_days"` + NewWorkIntervalSeconds int64 `json:"new_work_interval_seconds"` + MetricInitialIntervalSeconds int64 `json:"metric_initial_interval_seconds"` + MetricMultiplier float64 `json:"metric_multiplier"` + MetricMaxIntervalSeconds int64 `json:"metric_max_interval_seconds"` + MetricAgeSeconds int64 `json:"metric_age_seconds"` + AIProvider string `json:"ai_provider"` + AIModel string `json:"ai_model"` + AIConfigured bool `json:"ai_configured"` + TranscriptionProvider string `json:"transcription_provider"` + TranscriptionModel string `json:"transcription_model"` + TranscriptionConfigured bool `json:"transcription_configured"` + UpdatedAt time.Time `json:"updated_at"` +} + +type SettingsUpdate struct { + LookbackDays int `json:"lookback_days"` + NewWorkIntervalSeconds int64 `json:"new_work_interval_seconds"` + MetricInitialIntervalSeconds int64 `json:"metric_initial_interval_seconds"` + MetricMultiplier float64 `json:"metric_multiplier"` + MetricMaxIntervalSeconds int64 `json:"metric_max_interval_seconds"` + MetricAgeSeconds int64 `json:"metric_age_seconds"` + AIProvider string `json:"ai_provider"` + AIModel string `json:"ai_model"` + AIConfigured bool `json:"ai_configured"` + TranscriptionProvider string `json:"transcription_provider"` + TranscriptionModel string `json:"transcription_model"` + TranscriptionConfigured bool `json:"transcription_configured"` +} + +type AccountProfile struct { + ID string `json:"id"` + Name string `json:"name"` + Platform string `json:"platform"` + PlatformAccountKey string `json:"platform_account_key"` + AuthorizationStatus string `json:"authorization_status"` + RuntimeStatus string `json:"runtime_status"` + LoginUsername string `json:"login_username"` + PasswordConfigured bool `json:"password_configured"` + RealNameStatus string `json:"real_name_status"` + RealName string `json:"real_name"` + IdentityNumber string `json:"identity_number"` + Note string `json:"note"` + BusinessStatus string `json:"business_status"` + BigAccount bool `json:"big_account"` + ReplyRequirements string `json:"reply_requirements"` + LoginStatus string `json:"login_status"` + LoginReason string `json:"login_reason"` + LoginCheckedAt *time.Time `json:"login_checked_at,omitempty"` + CooldownSeconds int64 `json:"cooldown_seconds"` + UpdatedAt time.Time `json:"updated_at"` +} + +type AccountProfileUpdate struct { + LoginUsername string `json:"login_username"` + Password string `json:"password"` + RealNameStatus string `json:"real_name_status"` + RealName string `json:"real_name"` + IdentityNumber string `json:"identity_number"` + Note string `json:"note"` + BusinessStatus string `json:"business_status"` + BigAccount bool `json:"big_account"` + ReplyRequirements string `json:"reply_requirements"` + CooldownSeconds int64 `json:"cooldown_seconds"` +} + +type LoginResult struct { + AccountID string `json:"account_id"` + Status string `json:"status"` + Reason string `json:"reason"` + ActualKey string `json:"actual_platform_account_key,omitempty"` + CheckedAt time.Time `json:"checked_at"` +} + +type Competitor struct { + ID string `json:"id"` + Platform string `json:"platform"` + PlatformAccountKey string `json:"platform_account_key"` + Nickname string `json:"nickname"` + AvatarURL string `json:"avatar_url"` + HomepageURL string `json:"homepage_url"` + Enabled bool `json:"enabled"` + SyncStatus string `json:"sync_status"` + SyncCursor string `json:"sync_cursor,omitempty"` + SyncError string `json:"sync_error,omitempty"` + SyncLeaseUntil *time.Time `json:"sync_lease_until,omitempty"` + LastSyncAt *time.Time `json:"last_sync_at,omitempty"` + NextSyncAt *time.Time `json:"next_sync_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type CompetitorInput struct { + Platform string `json:"platform"` + PlatformAccountKey string `json:"platform_account_key"` + Nickname string `json:"nickname"` + AvatarURL string `json:"avatar_url"` + HomepageURL string `json:"homepage_url"` +} + +type WorkSource struct { + Platform string `json:"platform"` + SourceType string `json:"source_type"` + SourceID string `json:"source_id"` +} + +type Work struct { + ID string `json:"id"` + Platform string `json:"platform"` + WorkKey string `json:"work_key"` + SourceType string `json:"source_type"` + SourceID string `json:"source_id"` + Sources []WorkSource `json:"sources,omitempty"` + AuthorName string `json:"author_name"` + Title string `json:"title"` + Body string `json:"body"` + PublishedAt *time.Time `json:"published_at,omitempty"` + PublishedAtStatus string `json:"published_at_status"` + OriginalURL string `json:"original_url"` + CoverURL string `json:"cover_url"` + Likes *int64 `json:"likes"` + CommentsCount *int64 `json:"comments_count"` + Shares *int64 `json:"shares"` + LatestMetricsAt *time.Time `json:"latest_metrics_at,omitempty"` + NextMetricAt *time.Time `json:"next_metric_at,omitempty"` + MetricStopReason string `json:"metric_stop_reason,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type WorkInput struct { + Platform string `json:"platform"` + WorkKey string `json:"work_key"` + SourceType string `json:"source_type"` + SourceID string `json:"source_id"` + AuthorName string `json:"author_name"` + Title string `json:"title"` + Body string `json:"body"` + PublishedAt *time.Time `json:"published_at"` + PublishedAtStatus string `json:"published_at_status"` + OriginalURL string `json:"original_url"` + CoverURL string `json:"cover_url"` + Likes *int64 `json:"likes"` + CommentsCount *int64 `json:"comments_count"` + Shares *int64 `json:"shares"` +} + +type WorkFilter struct { + Platform string + SourceID string + SourceType string + PublishedAfter *time.Time + PublishedBefore *time.Time + MinLikes *int64 + MinComments *int64 + MinShares *int64 +} + +type MetricInput struct { + WorkID string + CollectedAt time.Time + Likes *int64 + CommentsCount *int64 + Shares *int64 +} + +type MetricPoint struct { + CollectedAt time.Time `json:"collected_at"` + Likes *int64 `json:"likes"` + CommentsCount *int64 `json:"comments_count"` + Shares *int64 `json:"shares"` +} + +type MaterialJob struct { + WorkID string `json:"work_id"` + Selected bool `json:"selected"` + SelectConfirmedAt *time.Time `json:"select_confirmed_at,omitempty"` + DownloadStatus string `json:"download_status"` + VideoReference string `json:"video_reference,omitempty"` + AudioStatus string `json:"audio_status"` + AudioReference string `json:"audio_reference,omitempty"` + TranscriptionStatus string `json:"transcription_status"` + Transcript string `json:"transcript,omitempty"` + FailedStep string `json:"failed_step,omitempty"` + FailureReason string `json:"failure_reason,omitempty"` + RewriteConfirmedAt *time.Time `json:"rewrite_confirmed_at,omitempty"` + RewriteRequirement string `json:"rewrite_requirement,omitempty"` + GeneratedTitle string `json:"generated_title,omitempty"` + GeneratedScript string `json:"generated_script,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type Comment struct { + ID string `json:"id"` + Platform string `json:"platform"` + CommentKey string `json:"comment_key"` + WorkID string `json:"work_id"` + AuthorUID string `json:"author_uid,omitempty"` + AuthorName string `json:"author_name,omitempty"` + Content string `json:"content"` + PublishedAt *time.Time `json:"published_at,omitempty"` + CollectedAt time.Time `json:"collected_at"` + CommentType string `json:"comment_type"` +} + +type CommentInput struct { + Platform string `json:"platform"` + CommentKey string `json:"comment_key"` + WorkID string `json:"work_id"` + AuthorUID string `json:"author_uid"` + AuthorName string `json:"author_name"` + Content string `json:"content"` + PublishedAt *time.Time `json:"published_at"` + CommentType string `json:"comment_type"` +} + +type LeadRule struct { + ID string `json:"id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + SourceType string `json:"source_type"` + Topic string `json:"topic"` + IncludeKeywords []string `json:"include_keywords"` + ExcludeKeywords []string `json:"exclude_keywords"` + AIRequirement string `json:"ai_requirement"` + UpdatedAt time.Time `json:"updated_at"` + CreatedAt time.Time `json:"created_at"` +} + +type LeadRuleInput struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + SourceType string `json:"source_type"` + Topic string `json:"topic"` + IncludeKeywords []string `json:"include_keywords"` + ExcludeKeywords []string `json:"exclude_keywords"` + AIRequirement string `json:"ai_requirement"` +} + +type RuleResult struct { + CommentID string `json:"comment_id"` + RuleID string `json:"rule_id"` + Status string `json:"status"` + Reason string `json:"reason"` + MatchedKeywords []string `json:"matched_keywords"` + RuleSnapshot LeadRule `json:"rule_snapshot"` + AnalysedAt *time.Time `json:"analysed_at,omitempty"` +} + +type Lead struct { + Comment Comment `json:"comment"` + RuleIDs []string `json:"rule_ids"` + Results []RuleResult `json:"results"` +} + +type Strategy struct { + ID string `json:"id"` + BigAccountID string `json:"big_account_id"` + ExecutionAccountID string `json:"execution_account_id"` + Position int `json:"position"` + Enabled bool `json:"enabled"` + EventTypes []string `json:"event_types"` + Action string `json:"action"` + TargetType string `json:"target_type"` + CandidateTexts []string `json:"candidate_texts"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type StrategyInput struct { + ExecutionAccountID string `json:"execution_account_id"` + Position int `json:"position"` + Enabled bool `json:"enabled"` + EventTypes []string `json:"event_types"` + Action string `json:"action"` + TargetType string `json:"target_type"` + CandidateTexts []string `json:"candidate_texts"` +} + +type Relation struct { + BigAccountID string `json:"big_account_id"` + SmallAccountID string `json:"small_account_id"` +} + +type InteractionEvent struct { + ID string `json:"id"` + Platform string `json:"platform"` + ReceivingAccountID string `json:"receiving_account_id"` + EventKey string `json:"event_key"` + EventType string `json:"event_type"` + Baseline bool `json:"baseline,omitempty"` + InteractorUID string `json:"interactor_uid"` + CommentID string `json:"comment_id"` + WorkID string `json:"work_id"` + PlatformEventAt *time.Time `json:"platform_event_at,omitempty"` + ReceivedAt time.Time `json:"received_at"` + ProcessingStartedAt *time.Time `json:"processing_started_at,omitempty"` + DisplayedAt *time.Time `json:"displayed_at,omitempty"` + State string `json:"state"` + Reason string `json:"reason,omitempty"` + StrategyID string `json:"strategy_id,omitempty"` + ExecutionAccountID string `json:"execution_account_id,omitempty"` +} + +type Operation struct { + ID string `json:"id"` + IdempotencyKey string `json:"idempotency_key"` + Source string `json:"source"` + Action string `json:"action"` + Platform string `json:"platform"` + AccountID string `json:"account_id"` + TargetUID string `json:"target_uid,omitempty"` + TargetCommentID string `json:"target_comment_id,omitempty"` + TargetWorkID string `json:"target_work_id,omitempty"` + Text string `json:"text,omitempty"` + EventID string `json:"event_id,omitempty"` + StrategyID string `json:"strategy_id,omitempty"` + State string `json:"state"` + Evidence map[string]string `json:"evidence"` + Reason string `json:"reason,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type OperationInput struct { + IdempotencyKey string `json:"idempotency_key"` + Source string `json:"source"` + Action string `json:"action"` + Platform string `json:"platform"` + AccountID string `json:"account_id"` + TargetUID string `json:"target_uid"` + TargetCommentID string `json:"target_comment_id"` + TargetWorkID string `json:"target_work_id"` + Text string `json:"text"` + EventID string `json:"event_id"` + StrategyID string `json:"strategy_id"` +} + +type Conversation struct { + ID string `json:"id"` + Platform string `json:"platform"` + AccountID string `json:"account_id"` + PeerUID string `json:"peer_uid"` + PeerName string `json:"peer_name"` + LastMessageAt *time.Time `json:"last_message_at,omitempty"` +} + +type Message struct { + ID string `json:"id"` + ConversationID string `json:"conversation_id"` + PlatformMessageKey string `json:"platform_message_key"` + Direction string `json:"direction"` + MessageType string `json:"message_type"` + Text string `json:"text,omitempty"` + SentState string `json:"sent_state"` + MessageAt *time.Time `json:"message_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +type MessageInput struct { + Platform string `json:"platform"` + AccountID string `json:"account_id"` + PeerUID string `json:"peer_uid"` + PeerName string `json:"peer_name"` + PlatformMessageKey string `json:"platform_message_key"` + Direction string `json:"direction"` + MessageType string `json:"message_type"` + Text string `json:"text"` + SentState string `json:"sent_state"` + MessageAt *time.Time `json:"message_at"` +} + +type ActionRequest struct { + OperationID string + Action string + Platform string + AccountID string + TargetUID string + TargetCommentID string + TargetWorkID string + Text string +} + +type ActionResult struct { + State string + Evidence map[string]string + Reason string +} + +type AutomaticResult struct { + Event InteractionEvent `json:"event"` + Operation *Operation `json:"operation,omitempty"` + Duplicate bool `json:"duplicate"` +} + +type ActionExecutor interface { + Execute(context.Context, ActionRequest) (ActionResult, error) +} + +type TextGenerator interface { + Generate(context.Context, string, string) (string, error) +} + +type ThemeAnalyzer interface { + MatchTheme(context.Context, string, string, string) (bool, string, error) + MatchLead(context.Context, string, string, string) (bool, string, error) +} diff --git a/internal/creator/rules.go b/internal/creator/rules.go new file mode 100644 index 0000000..55addc1 --- /dev/null +++ b/internal/creator/rules.go @@ -0,0 +1,267 @@ +package creator + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +func encodeRuleLists(input LeadRuleInput) (string, string, error) { + include, err := jsonText(input.IncludeKeywords) + if err != nil { + return "", "", err + } + exclude, err := jsonText(input.ExcludeKeywords) + if err != nil { + return "", "", err + } + return include, exclude, nil +} + +func scanRule(scanner interface{ Scan(...any) error }) (LeadRule, error) { + var result LeadRule + var include, exclude []byte + if err := scanner.Scan(&result.ID, &result.Name, &result.Enabled, &result.SourceType, &result.Topic, &include, &exclude, &result.AIRequirement, &result.UpdatedAt, &result.CreatedAt); err != nil { + return LeadRule{}, err + } + var err error + result.IncludeKeywords, err = decodeStringList(include) + if err != nil { + return LeadRule{}, err + } + result.ExcludeKeywords, err = decodeStringList(exclude) + if err != nil { + return LeadRule{}, err + } + return result, nil +} + +const ruleSelect = `SELECT id, name, enabled, source_type, topic, include_keywords, exclude_keywords, ai_requirement, updated_at, created_at FROM creator_lead_rule` + +func (s *Store) CreateRule(ctx context.Context, input LeadRuleInput) (LeadRule, error) { + input, err := ValidateRuleInput(input) + if err != nil { + return LeadRule{}, err + } + include, exclude, err := encodeRuleLists(input) + if err != nil { + return LeadRule{}, err + } + id := newID("rule") + if _, err := s.db.ExecContext(ctx, `INSERT INTO creator_lead_rule (id,name,enabled,source_type,topic,include_keywords,exclude_keywords,ai_requirement) VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7::jsonb,$8)`, id, input.Name, input.Enabled, input.SourceType, input.Topic, include, exclude, input.AIRequirement); err != nil { + return LeadRule{}, databaseError(err) + } + return s.GetRule(ctx, id) +} + +func (s *Store) GetRule(ctx context.Context, id string) (LeadRule, error) { + result, err := scanRule(s.db.QueryRowContext(ctx, ruleSelect+` WHERE id = $1`, id)) + return result, rowError(err) +} + +func (s *Store) ListRules(ctx context.Context, enabledOnly bool) ([]LeadRule, error) { + query := ruleSelect + if enabledOnly { + query += ` WHERE enabled` + } + query += ` ORDER BY created_at DESC,id` + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]LeadRule, 0) + for rows.Next() { + item, err := scanRule(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func (s *Store) UpdateRule(ctx context.Context, id string, input LeadRuleInput) (LeadRule, error) { + input, err := ValidateRuleInput(input) + if err != nil { + return LeadRule{}, err + } + include, exclude, err := encodeRuleLists(input) + if err != nil { + return LeadRule{}, err + } + _, err = s.db.ExecContext(ctx, `UPDATE creator_lead_rule SET name=$2,enabled=$3,source_type=$4,topic=$5,include_keywords=$6::jsonb,exclude_keywords=$7::jsonb,ai_requirement=$8,updated_at=now() WHERE id=$1`, id, input.Name, input.Enabled, input.SourceType, input.Topic, include, exclude, input.AIRequirement) + if err != nil { + return LeadRule{}, databaseError(err) + } + return s.GetRule(ctx, id) +} + +func (s *Store) SetRuleEnabled(ctx context.Context, id string, enabled bool) (LeadRule, error) { + if _, err := s.db.ExecContext(ctx, `UPDATE creator_lead_rule SET enabled=$2,updated_at=now() WHERE id=$1`, id, enabled); err != nil { + return LeadRule{}, databaseError(err) + } + return s.GetRule(ctx, id) +} + +func (s *Store) upsertRuleResult(ctx context.Context, commentID string, rule LeadRule, status, reason string, matched []string, analysedAt *time.Time) (RuleResult, error) { + snapshot, err := jsonText(rule) + if err != nil { + return RuleResult{}, err + } + matchedJSON, err := jsonText(matched) + if err != nil { + return RuleResult{}, err + } + _, err = s.db.ExecContext(ctx, `INSERT INTO creator_comment_rule_result (comment_id,rule_id,status,reason,matched_keywords,rule_snapshot,analysed_at) VALUES ($1,$2,$3,$4,$5::jsonb,$6::jsonb,$7) ON CONFLICT (comment_id,rule_id) DO UPDATE SET status=EXCLUDED.status,reason=EXCLUDED.reason,matched_keywords=EXCLUDED.matched_keywords,rule_snapshot=EXCLUDED.rule_snapshot,analysed_at=EXCLUDED.analysed_at`, commentID, rule.ID, status, reason, matchedJSON, snapshot, analysedAt) + if err != nil { + return RuleResult{}, databaseError(err) + } + return RuleResult{CommentID: commentID, RuleID: rule.ID, Status: status, Reason: reason, MatchedKeywords: matched, RuleSnapshot: rule, AnalysedAt: analysedAt}, nil +} + +func (s *Store) AnalyzeComment(ctx context.Context, commentID, ruleID string, analyzer ThemeAnalyzer) (RuleResult, error) { + comment, err := s.GetComment(ctx, commentID) + if err != nil { + return RuleResult{}, err + } + rule, err := s.GetRule(ctx, ruleID) + if err != nil { + return RuleResult{}, err + } + if !rule.Enabled { + return RuleResult{}, ErrConflict + } + work, err := s.GetWork(ctx, comment.WorkID) + if err != nil { + return RuleResult{}, err + } + if rule.SourceType != "all" && rule.SourceType != work.SourceType { + return RuleResult{}, ErrConflict + } + if analyzer == nil { + result, saveErr := s.upsertRuleResult(ctx, commentID, rule, "failed", "AI 分析不可用", nil, ptrTime(time.Now().UTC())) + if saveErr != nil { + return RuleResult{}, saveErr + } + return result, ErrUnavailable + } + theme, reason, err := analyzer.MatchTheme(ctx, work.Title, work.Body, rule.Topic) + if err != nil { + result, saveErr := s.upsertRuleResult(ctx, commentID, rule, "failed", err.Error(), nil, ptrTime(time.Now().UTC())) + if saveErr != nil { + return RuleResult{}, saveErr + } + return result, err + } + if !theme { + return s.upsertRuleResult(ctx, commentID, rule, "topic_no_match", strings.TrimSpace(reason), nil, ptrTime(time.Now().UTC())) + } + keywordsOK, matched := MatchKeywords(comment.Content, rule.IncludeKeywords, rule.ExcludeKeywords) + if !keywordsOK { + return s.upsertRuleResult(ctx, commentID, rule, "keyword_no_match", "关键词不满足", matched, ptrTime(time.Now().UTC())) + } + lead, reason, err := analyzer.MatchLead(ctx, work.Title+"\n"+work.Body, comment.Content, rule.AIRequirement) + if err != nil { + result, saveErr := s.upsertRuleResult(ctx, commentID, rule, "failed", err.Error(), matched, ptrTime(time.Now().UTC())) + if saveErr != nil { + return RuleResult{}, saveErr + } + return result, err + } + if !lead { + return s.upsertRuleResult(ctx, commentID, rule, "not_lead", strings.TrimSpace(reason), matched, ptrTime(time.Now().UTC())) + } + return s.upsertRuleResult(ctx, commentID, rule, "lead", strings.TrimSpace(reason), matched, ptrTime(time.Now().UTC())) +} + +func scanRuleResult(scanner interface{ Scan(...any) error }) (RuleResult, error) { + var r RuleResult + var matched, snapshot []byte + var analysed sql.NullTime + if err := scanner.Scan(&r.CommentID, &r.RuleID, &r.Status, &r.Reason, &matched, &snapshot, &analysed); err != nil { + return RuleResult{}, err + } + var err error + r.MatchedKeywords, err = decodeStringList(matched) + if err != nil { + return RuleResult{}, err + } + if err := jsonUnmarshal(snapshot, &r.RuleSnapshot); err != nil { + return RuleResult{}, err + } + r.AnalysedAt = nullableTime(analysed) + return r, nil +} + +func jsonUnmarshal(data []byte, value any) error { + if len(data) == 0 { + return nil + } + return json.Unmarshal(data, value) +} + +func (s *Store) ListRuleResults(ctx context.Context, commentID, ruleID string) ([]RuleResult, error) { + query := `SELECT comment_id,rule_id,status,reason,matched_keywords,rule_snapshot,analysed_at FROM creator_comment_rule_result` + where := []string{} + args := []any{} + if commentID != "" { + args = append(args, commentID) + where = append(where, fmt.Sprintf("comment_id = $%d", len(args))) + } + if ruleID != "" { + args = append(args, ruleID) + where = append(where, fmt.Sprintf("rule_id = $%d", len(args))) + } + if len(where) > 0 { + query += ` WHERE ` + strings.Join(where, ` AND `) + } + query += ` ORDER BY analysed_at DESC NULLS LAST,comment_id,rule_id` + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]RuleResult, 0) + for rows.Next() { + item, err := scanRuleResult(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +func (s *Store) ListLeads(ctx context.Context, platform string) ([]Lead, error) { + comments, err := s.ListComments(ctx, platform, "") + if err != nil { + return nil, err + } + results, err := s.ListRuleResults(ctx, "", "") + if err != nil { + return nil, err + } + byComment := map[string][]RuleResult{} + for _, result := range results { + if result.Status == "lead" { + byComment[result.CommentID] = append(byComment[result.CommentID], result) + } + } + leads := make([]Lead, 0) + for _, comment := range comments { + matched := byComment[comment.ID] + if len(matched) == 0 { + continue + } + ids := make([]string, 0, len(matched)) + for _, result := range matched { + ids = append(ids, result.RuleID) + } + leads = append(leads, Lead{Comment: comment, RuleIDs: ids, Results: matched}) + } + return leads, nil +} diff --git a/internal/creator/scheduler.go b/internal/creator/scheduler.go new file mode 100644 index 0000000..5112a5f --- /dev/null +++ b/internal/creator/scheduler.go @@ -0,0 +1,39 @@ +package creator + +import "time" + +type ScheduleKind string + +const ( + ScheduleNewWorks ScheduleKind = "new_works" + ScheduleComments ScheduleKind = "comments" + ScheduleMetrics ScheduleKind = "metrics" +) + +func IsDue(lastCompleted, now time.Time, interval time.Duration) bool { + if now.IsZero() || interval <= 0 { + return false + } + if lastCompleted.IsZero() { + return true + } + return !now.UTC().Before(lastCompleted.UTC().Add(interval)) +} + +func NextFixedRun(lastCompleted, now time.Time, interval time.Duration) time.Time { + if now.IsZero() || interval <= 0 { + return time.Time{} + } + if lastCompleted.IsZero() { + return now.UTC() + } + next := lastCompleted.UTC().Add(interval) + if next.Before(now.UTC()) { + return now.UTC() + } + return next +} + +func MetricDue(work Work, now time.Time) bool { + return work.NextMetricAt != nil && !now.UTC().Before(work.NextMetricAt.UTC()) +} diff --git a/internal/creator/scheduler_test.go b/internal/creator/scheduler_test.go new file mode 100644 index 0000000..4bd02aa --- /dev/null +++ b/internal/creator/scheduler_test.go @@ -0,0 +1,31 @@ +package creator + +import ( + "testing" + "time" +) + +func TestFixedScheduleUsesDueBoundary(t *testing.T) { + now := time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC) + last := now.Add(-time.Hour) + if !IsDue(last, now, time.Hour) || IsDue(last.Add(time.Minute), now, time.Hour) { + t.Fatal("schedule due check must include the exact boundary") + } + if got := NextFixedRun(last, now, time.Hour); !got.Equal(now) { + t.Fatalf("next fixed run = %s", got) + } + if got := NextFixedRun(time.Time{}, now, time.Hour); !got.Equal(now) { + t.Fatalf("first fixed run = %s", got) + } +} + +func TestMetricDuePreservesUnavailableState(t *testing.T) { + now := time.Now().UTC() + if MetricDue(Work{}, now) { + t.Fatal("missing metric schedule is not due") + } + past := now.Add(-time.Minute) + if !MetricDue(Work{NextMetricAt: &past}, now) { + t.Fatal("past metric schedule is due") + } +} diff --git a/internal/creator/settings.go b/internal/creator/settings.go new file mode 100644 index 0000000..dcd0462 --- /dev/null +++ b/internal/creator/settings.go @@ -0,0 +1,76 @@ +package creator + +import ( + "context" + "fmt" + "strings" + "time" +) + +func scanSettings(scanner interface{ Scan(...any) error }) (Settings, error) { + var result Settings + if err := scanner.Scan(&result.LookbackDays, &result.NewWorkIntervalSeconds, &result.MetricInitialIntervalSeconds, + &result.MetricMultiplier, &result.MetricMaxIntervalSeconds, &result.MetricAgeSeconds, + &result.AIProvider, &result.AIModel, &result.AIConfigured, &result.TranscriptionProvider, + &result.TranscriptionModel, &result.TranscriptionConfigured, &result.UpdatedAt); err != nil { + return Settings{}, err + } + result.UpdatedAt = result.UpdatedAt.UTC() + return result, nil +} + +const settingsSelect = `SELECT lookback_days, new_work_interval_seconds, metric_initial_interval_seconds, + metric_multiplier, metric_max_interval_seconds, metric_age_seconds, ai_provider, ai_model, + ai_configured, transcription_provider, transcription_model, transcription_configured, updated_at + FROM creator_settings WHERE id = true` + +func (s *Store) GetSettings(ctx context.Context) (Settings, error) { + result, err := scanSettings(s.db.QueryRowContext(ctx, settingsSelect)) + return result, rowError(err) +} + +func (s *Store) UpdateSettings(ctx context.Context, input SettingsUpdate) (Settings, error) { + input.AIProvider = strings.TrimSpace(input.AIProvider) + input.AIModel = strings.TrimSpace(input.AIModel) + input.TranscriptionProvider = strings.TrimSpace(input.TranscriptionProvider) + input.TranscriptionModel = strings.TrimSpace(input.TranscriptionModel) + if err := ValidateSettings(input); err != nil { + return Settings{}, err + } + if input.AIConfigured && (input.AIProvider == "" || input.AIModel == "") { + return Settings{}, ErrInvalid + } + if input.TranscriptionConfigured && (input.TranscriptionProvider == "" || input.TranscriptionModel == "") { + return Settings{}, ErrInvalid + } + _, err := s.db.ExecContext(ctx, ` + UPDATE creator_settings SET lookback_days=$1, new_work_interval_seconds=$2, + metric_initial_interval_seconds=$3, metric_multiplier=$4, metric_max_interval_seconds=$5, + metric_age_seconds=$6, ai_provider=$7, ai_model=$8, ai_configured=$9, + transcription_provider=$10, transcription_model=$11, transcription_configured=$12, + updated_at=now() WHERE id=true`, input.LookbackDays, input.NewWorkIntervalSeconds, + input.MetricInitialIntervalSeconds, input.MetricMultiplier, input.MetricMaxIntervalSeconds, + input.MetricAgeSeconds, input.AIProvider, input.AIModel, input.AIConfigured, + input.TranscriptionProvider, input.TranscriptionModel, input.TranscriptionConfigured) + if err != nil { + return Settings{}, databaseError(err) + } + return s.GetSettings(ctx) +} + +func (s *Store) SetEventDisplayed(ctx context.Context, eventID string, displayedAt time.Time) (InteractionEvent, error) { + if displayedAt.IsZero() { + displayedAt = time.Now().UTC() + } + if _, err := s.db.ExecContext(ctx, `UPDATE creator_event SET displayed_at=$2 WHERE id=$1`, eventID, displayedAt.UTC()); err != nil { + return InteractionEvent{}, databaseError(err) + } + return s.GetEvent(ctx, eventID) +} + +func (s *Store) EnsureSchema(ctx context.Context) error { + if _, err := s.db.ExecContext(ctx, `SELECT 1 FROM creator_settings WHERE id=true`); err != nil { + return fmt.Errorf("check creator schema: %w", err) + } + return nil +} diff --git a/internal/creator/store.go b/internal/creator/store.go new file mode 100644 index 0000000..a10e4d3 --- /dev/null +++ b/internal/creator/store.go @@ -0,0 +1,186 @@ +package creator + +import ( + "context" + "crypto/rand" + "database/sql" + _ "embed" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgconn" + _ "github.com/jackc/pgx/v5/stdlib" +) + +//go:embed migrations/017_creator.sql +var migration017 string + +//go:embed migrations/018_collection_checkpoints.sql +var migration018 string + +//go:embed migrations/019_metric_plans.sql +var migration019 string + +//go:embed migrations/020_competitor_sync_leases.sql +var migration020 string + +//go:embed migrations/021_work_sources.sql +var migration021 string + +//go:embed migrations/022_collection_lease_tokens.sql +var migration022 string + +type SecretReference struct { + ID string + Provider string +} + +type SecretBridge interface { + Store(context.Context, SecretReference, string, string) error + Delete(context.Context, SecretReference, string) error +} + +type Store struct { + db *sql.DB + secrets SecretBridge +} + +func Open(ctx context.Context, databaseURL string) (*Store, error) { + db, err := sql.Open("pgx", databaseURL) + if err != nil { + return nil, errors.New("open creator database") + } + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(2) + db.SetConnMaxIdleTime(5 * time.Minute) + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, errors.New("connect to creator database") + } + store := &Store{db: db} + if err := store.migrate(ctx); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (s *Store) Close() error { return s.db.Close() } + +func (s *Store) SetSecretBridge(bridge SecretBridge) { s.secrets = bridge } + +func (s *Store) migrate(ctx context.Context) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin creator schema migration") + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1542738017)`); err != nil { + return errors.New("lock creator schema migration") + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { + return errors.New("create creator schema migration table") + } + migrations := []struct { + version int + sql string + }{ + {version: 17, sql: migration017}, + {version: 18, sql: migration018}, + {version: 19, sql: migration019}, + {version: 20, sql: migration020}, + {version: 21, sql: migration021}, + {version: 22, sql: migration022}, + } + for _, migration := range migrations { + var applied bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { + return errors.New("read creator schema migration state") + } + if applied { + continue + } + if _, err := tx.ExecContext(ctx, migration.sql); err != nil { + return fmt.Errorf("apply creator schema migration %d: %w", migration.version, err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migration (version) VALUES ($1)`, migration.version); err != nil { + return fmt.Errorf("record creator schema migration %d: %w", migration.version, err) + } + } + if err := tx.Commit(); err != nil { + return errors.New("commit creator schema migration") + } + return nil +} + +func newID(prefix string) string { + var bytes [12]byte + if _, err := rand.Read(bytes[:]); err != nil { + panic(fmt.Sprintf("generate creator id: %v", err)) + } + return prefix + "-" + hex.EncodeToString(bytes[:]) +} + +func databaseError(err error) error { + if err == nil { + return nil + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505": + return ErrConflict + case "23503", "23514", "22P02": + return ErrInvalid + } + } + return err +} + +func rowError(err error) error { + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + return databaseError(err) +} + +func jsonText(value any) (string, error) { + encoded, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode creator json: %w", err) + } + return string(encoded), nil +} + +func decodeStringList(encoded []byte) ([]string, error) { + if len(encoded) == 0 { + return []string{}, nil + } + var result []string + if err := json.Unmarshal(encoded, &result); err != nil { + return nil, fmt.Errorf("decode creator string list: %w", err) + } + if result == nil { + result = []string{} + } + return result, nil +} + +func nullableTime(value sql.NullTime) *time.Time { + if !value.Valid { + return nil + } + result := value.Time.UTC() + return &result +} + +func nullableInt64(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + result := value.Int64 + return &result +} diff --git a/internal/douyin/connector.go b/internal/douyin/connector.go index 98eb2ac..69676f6 100644 --- a/internal/douyin/connector.go +++ b/internal/douyin/connector.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "regexp" + "strconv" "strings" "time" "unicode/utf8" @@ -27,7 +28,7 @@ const ( ReasonIdentityMatch = "douyin_identity_mismatch" ReasonSucceeded = "douyin_sync_succeeded" - identityEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/self/" + identityEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp" worksEndpoint = "https://www.douyin.com/aweme/v1/web/aweme/post/" ) @@ -90,11 +91,13 @@ type Work struct { } type Evidence struct { - Phase string `json:"phase"` - HTTPStatus int `json:"http_status,omitempty"` - IdentityVerified bool `json:"identity_verified"` - WorksSeen int `json:"works_seen,omitempty"` - HasMore bool `json:"has_more,omitempty"` + Phase string `json:"phase"` + HTTPStatus int `json:"http_status,omitempty"` + IdentityVerified bool `json:"identity_verified"` + WorksSeen int `json:"works_seen,omitempty"` + HasMore bool `json:"has_more,omitempty"` + Pages int `json:"pages,omitempty"` + PaginationComplete bool `json:"pagination_complete"` } type Result struct { @@ -159,21 +162,50 @@ func (connector Connector) Sync(ctx context.Context, request Request) (Result, e Evidence{Phase: "identity", HTTPStatus: identityResponse.Status}) } - query := url.Values{"sec_user_id": {identity.User.SecUID}, "count": {"20"}, "max_cursor": {"0"}} - worksResponse, err := connector.Browser.Get(ctx, worksEndpoint+"?"+query.Encode()) - if err != nil { - return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, - Evidence{Phase: "works", IdentityVerified: true}) - } - if state, reason := classify(worksResponse); state != "" { - return connector.stop(ctx, request.AccountID, state, reason, - Evidence{Phase: "works", HTTPStatus: worksResponse.Status, IdentityVerified: true}) - } - works, hasMore, ok := parseWorks(worksResponse.Body) - evidence := Evidence{Phase: "works", HTTPStatus: worksResponse.Status, IdentityVerified: true, WorksSeen: len(works), HasMore: hasMore} - if !ok { - return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + var works []Work + seen := make(map[string]struct{}) + evidence := Evidence{Phase: "works", IdentityVerified: true} + cursor := int64(0) + for page := 0; page < 100; page++ { + query := url.Values{"sec_user_id": {identity.User.SecUID}, "count": {"20"}, "max_cursor": {strconv.FormatInt(cursor, 10)}} + worksResponse, err := connector.Browser.Get(ctx, worksEndpoint+"?"+query.Encode()) + if err != nil { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + } + if state, reason := classify(worksResponse); state != "" { + evidence.HTTPStatus = worksResponse.Status + return connector.stop(ctx, request.AccountID, state, reason, evidence) + } + pageWorks, hasMore, nextCursor, ok := parseWorksPage(worksResponse.Body) + evidence.HTTPStatus, evidence.Pages, evidence.HasMore = worksResponse.Status, page+1, hasMore + if !ok { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + } + for _, work := range pageWorks { + if _, exists := seen[work.ID]; exists { + continue + } + seen[work.ID] = struct{}{} + works = append(works, work) + } + if !hasMore { + evidence.PaginationComplete = true + break + } + if nextCursor == nil { + // Older gateway responses omitted the cursor. Keep the existing + // read-only behavior, but expose that pagination was incomplete. + break + } + if *nextCursor <= cursor { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + } + cursor = *nextCursor + if page == 99 { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + } } + evidence.WorksSeen = len(works) if err := connector.Store.Complete(ctx, request.AccountID, works, evidence); err != nil { result, holdErr := connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) return result, errors.Join(err, holdErr) @@ -210,6 +242,28 @@ func classify(response Response) (string, string) { } } +func ParseCookieBundle(raw []byte) ([]Cookie, error) { + return parseCredential(raw) +} + +func ParseCookieHeader(raw []byte) ([]Cookie, error) { + if len(raw) == 0 || len(raw) > 64<<10 { + return nil, ErrInvalid + } + parsed, err := http.ParseCookie(string(raw)) + if err != nil || len(parsed) == 0 || len(parsed) > 64 { + return nil, ErrInvalid + } + cookies := make([]Cookie, 0, len(parsed)) + for _, cookie := range parsed { + if cookie == nil || cookie.Name == "" || cookie.Value == "" { + return nil, ErrInvalid + } + cookies = append(cookies, Cookie{Name: cookie.Name, Value: cookie.Value, Domain: ".douyin.com", Path: "/", Secure: true, SameSite: "Lax"}) + } + return cookies, nil +} + func parseCredential(raw []byte) ([]Cookie, error) { if len(raw) == 0 || len(raw) > 64<<10 { return nil, ErrInvalid @@ -264,8 +318,9 @@ func parseIdentity(body []byte) (identityEnvelope, bool) { } type worksEnvelope struct { - StatusCode *int `json:"status_code"` - HasMore *bool `json:"has_more"` + StatusCode *int `json:"status_code"` + HasMore *bool `json:"has_more"` + MaxCursor *int64 `json:"max_cursor"` Works []struct { ID string `json:"aweme_id"` Description string `json:"desc"` @@ -279,11 +334,11 @@ type worksEnvelope struct { } `json:"aweme_list"` } -func parseWorks(body []byte) ([]Work, bool, bool) { +func parseWorksPage(body []byte) ([]Work, bool, *int64, bool) { var envelope worksEnvelope if len(body) > 4<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || - envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 { - return nil, false, false + envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 { + return nil, false, nil, false } works := make([]Work, 0, len(envelope.Works)) seen := make(map[string]bool, len(envelope.Works)) @@ -293,12 +348,17 @@ func parseWorks(body []byte) ([]Work, bool, bool) { candidate.Statistics.ShareCount == nil || candidate.Statistics.PlayCount == nil || utf8.RuneCountInString(candidate.Description) > 4096 || *candidate.Statistics.DiggCount < 0 || *candidate.Statistics.CommentCount < 0 || *candidate.Statistics.ShareCount < 0 || *candidate.Statistics.PlayCount < 0 { - return nil, false, false + return nil, false, nil, false } seen[candidate.ID] = true works = append(works, Work{ID: candidate.ID, Description: candidate.Description, CreatedAt: *candidate.CreatedAt, DiggCount: *candidate.Statistics.DiggCount, CommentCount: *candidate.Statistics.CommentCount, ShareCount: *candidate.Statistics.ShareCount, PlayCount: *candidate.Statistics.PlayCount}) } - return works, *envelope.HasMore, true + return works, *envelope.HasMore, envelope.MaxCursor, true +} + +func parseWorks(body []byte) ([]Work, bool, bool) { + works, hasMore, _, ok := parseWorksPage(body) + return works, hasMore, ok } diff --git a/internal/douyin/connector_test.go b/internal/douyin/connector_test.go index 76f48e1..1048bc4 100644 --- a/internal/douyin/connector_test.go +++ b/internal/douyin/connector_test.go @@ -96,6 +96,22 @@ func TestSyncLogsInVerifiesIdentityAndReadsOwnWorks(t *testing.T) { } } +func TestSyncPaginatesAndDeduplicatesAcrossPages(t *testing.T) { + browser := &fakeBrowser{responses: []Response{ + {Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, + {Status: 200, Body: []byte(`{"status_code":0,"has_more":true,"max_cursor":20,"aweme_list":[{"aweme_id":"work-1","desc":"one","create_time":123,"statistics":{"digg_count":1,"comment_count":1,"share_count":1,"play_count":1}},{"aweme_id":"work-2","desc":"two","create_time":124,"statistics":{"digg_count":2,"comment_count":2,"share_count":2,"play_count":2}}]}`)}, + {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"max_cursor":40,"aweme_list":[{"aweme_id":"work-2","desc":"two-new","create_time":124,"statistics":{"digg_count":20,"comment_count":20,"share_count":20,"play_count":20}},{"aweme_id":"work-3","desc":"three","create_time":125,"statistics":{"digg_count":3,"comment_count":3,"share_count":3,"play_count":3}}]}`)}, + }} + store := &fakeStore{} + result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference}) + if err != nil || result.State != StateSucceeded || !result.Evidence.PaginationComplete || result.Evidence.Pages != 2 || result.Evidence.WorksSeen != 3 || result.Evidence.HasMore || len(store.works) != 3 { + t.Fatalf("unexpected paginated result: %#v works=%#v urls=%#v remaining=%d err=%v", result, store.works, browser.urls, len(browser.responses), err) + } + if len(browser.urls) != 3 || !strings.Contains(browser.urls[2], "max_cursor=20") { + t.Fatalf("connector did not follow the cursor: %#v", browser.urls) + } +} + func TestSyncMapsRiskSignalsAndNeverRetries(t *testing.T) { tests := []struct { name string diff --git a/internal/douyin/creator_collector.go b/internal/douyin/creator_collector.go new file mode 100644 index 0000000..36aca84 --- /dev/null +++ b/internal/douyin/creator_collector.go @@ -0,0 +1,215 @@ +package douyin + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +const commentsEndpoint = "https://www.douyin.com/aweme/v1/web/comment/list/" + +type CreatorCollector struct { + Browser Browser + AccountKey string + SourceType string + SourceID string +} + +func (c CreatorCollector) VerifyIdentity(ctx context.Context, expectedKey string) error { + if c.Browser == nil || !keyPattern.MatchString(expectedKey) { + return fmt.Errorf("%w: invalid identity verification request", ErrInvalid) + } + response, err := c.Browser.Get(ctx, identityEndpoint) + if err != nil { + return err + } + if err := creatorResponseError(response, "identity"); err != nil { + return err + } + identity, ok := parseIdentity(response.Body) + if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID { + return fmt.Errorf("%w: douyin identity mismatch", ErrInvalid) + } + return nil +} + +func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) { + if c.AccountKey != "" { + accountKey = c.AccountKey + } + if c.Browser == nil || !keyPattern.MatchString(accountKey) || cursor != "" && !cursorPattern.MatchString(cursor) { + return creator.WorkPage{}, fmt.Errorf("%w: invalid creator work collection request", ErrInvalid) + } + maxCursor := "0" + if cursor != "" { + maxCursor = cursor + } + query := url.Values{"sec_user_id": {accountKey}, "count": {"20"}, "max_cursor": {maxCursor}} + response, err := c.Browser.Get(ctx, worksEndpoint+"?"+query.Encode()) + if err != nil { + return creator.WorkPage{}, err + } + if err := creatorResponseError(response, "works"); err != nil { + return creator.WorkPage{}, err + } + works, hasMore, nextCursor, ok := parseCreatorWorksPage(response.Body) + if !ok { + return creator.WorkPage{}, fmt.Errorf("%w: invalid douyin works response", ErrInvalid) + } + items := make([]creator.WorkInput, 0, len(works)) + for _, work := range works { + var published *time.Time + if work.CreatedAt != nil && *work.CreatedAt > 0 { + value := time.Unix(*work.CreatedAt, 0).UTC() + published = &value + } + likes, comments, shares := work.DiggCount, work.CommentCount, work.ShareCount + sourceType, sourceID := c.SourceType, c.SourceID + if sourceType == "" { + sourceType = creator.SourceCompetitor + } + if sourceID == "" { + sourceID = accountKey + } + status := "pending_verification" + if published != nil { + status = "verified" + } + items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, Likes: likes, CommentsCount: comments, Shares: shares}) + } + page := creator.WorkPage{Items: items, HasMore: hasMore} + if nextCursor != nil { + page.NextCursor = strconv.FormatInt(*nextCursor, 10) + } + if page.HasMore && page.NextCursor == "" { + return creator.WorkPage{}, fmt.Errorf("%w: douyin works cursor missing", ErrInvalid) + } + return page, nil +} + +func (c CreatorCollector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) { + if c.Browser == nil || !keyPattern.MatchString(workKey) || cursor != "" && !cursorPattern.MatchString(cursor) { + return creator.CommentPage{}, fmt.Errorf("%w: invalid creator comment collection request", ErrInvalid) + } + cursorValue := "0" + if cursor != "" { + cursorValue = cursor + } + query := url.Values{"aweme_id": {workKey}, "count": {"20"}, "cursor": {cursorValue}} + response, err := c.Browser.Get(ctx, commentsEndpoint+"?"+query.Encode()) + if err != nil { + return creator.CommentPage{}, err + } + if err := creatorResponseError(response, "comments"); err != nil { + return creator.CommentPage{}, err + } + var envelope commentEnvelope + if len(response.Body) > 4<<20 || json.Unmarshal(response.Body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Comments == nil { + return creator.CommentPage{}, fmt.Errorf("%w: invalid douyin comments response", ErrInvalid) + } + items := make([]creator.CommentInput, 0, len(envelope.Comments)) + for _, item := range envelope.Comments { + if item.ID == "" || strings.TrimSpace(item.Text) == "" || item.ReplyID != "" { + return creator.CommentPage{}, fmt.Errorf("%w: non-top-level or incomplete douyin comment", ErrInvalid) + } + var published *time.Time + if item.CreateTime > 0 { + value := time.Unix(item.CreateTime, 0).UTC() + published = &value + } + items = append(items, creator.CommentInput{Platform: creator.PlatformDouyin, CommentKey: item.ID, AuthorUID: item.UserUID, AuthorName: item.UserName, Content: item.Text, PublishedAt: published, CommentType: "top_level"}) + } + page := creator.CommentPage{Items: items, HasMore: *envelope.HasMore} + if envelope.Cursor != nil { + page.NextCursor = strconv.FormatInt(*envelope.Cursor, 10) + } + if page.HasMore && page.NextCursor == "" { + return creator.CommentPage{}, fmt.Errorf("%w: douyin comments cursor missing", ErrInvalid) + } + return page, nil +} + +type creatorWorkPageItem struct { + ID string + Description string + CreatedAt *int64 + DiggCount *int64 + CommentCount *int64 + ShareCount *int64 +} + +func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) { + if len(body) > 4<<20 { + return nil, false, nil, false + } + var envelope worksEnvelope + if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 || *envelope.HasMore && envelope.MaxCursor == nil { + return nil, false, nil, false + } + items := make([]creatorWorkPageItem, 0, len(envelope.Works)) + seen := make(map[string]struct{}, len(envelope.Works)) + for _, item := range envelope.Works { + if !keyPattern.MatchString(item.ID) { + return nil, false, nil, false + } + if _, exists := seen[item.ID]; exists { + return nil, false, nil, false + } + seen[item.ID] = struct{}{} + var likes, comments, shares *int64 + if item.Statistics != nil { + likes, comments, shares = item.Statistics.DiggCount, item.Statistics.CommentCount, item.Statistics.ShareCount + } + for _, value := range []*int64{likes, comments, shares} { + if value != nil && *value < 0 { + return nil, false, nil, false + } + } + createdAt := item.CreatedAt + if createdAt != nil && *createdAt <= 0 { + createdAt = nil + } + items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, DiggCount: likes, CommentCount: comments, ShareCount: shares}) + } + return items, *envelope.HasMore, envelope.MaxCursor, true +} + +type commentEnvelope struct { + StatusCode *int `json:"status_code"` + HasMore *bool `json:"has_more"` + Cursor *int64 `json:"cursor"` + Comments []struct { + ID string `json:"cid"` + Text string `json:"text"` + CreateTime int64 `json:"create_time"` + ReplyID string `json:"reply_id"` + UserUID string `json:"user_uid"` + UserName string `json:"user_name"` + } `json:"comments"` +} + +var cursorPattern = regexp.MustCompile(`^[0-9]{1,20}$`) + +func creatorResponseError(response Response, resource string) error { + if response.Challenge != ChallengeNone { + return fmt.Errorf("%w: douyin %s challenge %s", creator.ErrUnavailable, resource, response.Challenge) + } + if response.Status >= 200 && response.Status < 300 { + return nil + } + switch response.Status { + case 401, 403: + return fmt.Errorf("%w: douyin %s authentication rejected with HTTP %d", creator.ErrConflict, resource, response.Status) + case 429: + return fmt.Errorf("%w: douyin %s rate limited", creator.ErrUnavailable, resource) + default: + return fmt.Errorf("douyin %s returned HTTP %d", resource, response.Status) + } +} diff --git a/internal/douyin/creator_collector_test.go b/internal/douyin/creator_collector_test.go new file mode 100644 index 0000000..bb37fa9 --- /dev/null +++ b/internal/douyin/creator_collector_test.go @@ -0,0 +1,21 @@ +package douyin + +import "testing" + +func TestParseCreatorWorksPageKeepsPartialMetadata(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"partial","statistics":{"digg_count":7}}]}`) + works, hasMore, cursor, ok := parseCreatorWorksPage(body) + if !ok || hasMore || cursor != nil || len(works) != 1 { + t.Fatalf("unexpected parsed page: ok=%v hasMore=%v cursor=%v works=%d", ok, hasMore, cursor, len(works)) + } + if works[0].CreatedAt != nil || works[0].DiggCount == nil || *works[0].DiggCount != 7 || works[0].CommentCount != nil || works[0].ShareCount != nil { + t.Fatalf("partial metadata was not preserved: %+v", works[0]) + } +} + +func TestParseCreatorWorksPageRejectsMissingWorkIdentity(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"desc":"no id"}]}`) + if _, _, _, ok := parseCreatorWorksPage(body); ok { + t.Fatal("expected missing work identity to be rejected") + } +} diff --git a/internal/phasea/store.go b/internal/phasea/store.go index 9ddeeb5..77db8c6 100644 --- a/internal/phasea/store.go +++ b/internal/phasea/store.go @@ -69,6 +69,10 @@ type CredentialBridge interface { Delete(context.Context, CredentialReference, string) error } +type CredentialResolver interface { + Resolve(context.Context, CredentialReference, string) ([]byte, error) +} + type Draft struct { ID string `json:"id"` AccountID string `json:"account_id"` @@ -366,6 +370,29 @@ func (s *Store) GetAccount(ctx context.Context, id string) (Account, error) { WHERE account.id = $1`, id)) } +func (s *Store) ResolveAccountCredential(ctx context.Context, id string, resolver CredentialResolver) ([]byte, error) { + if !idPattern.MatchString(id) || resolver == nil { + return nil, ErrInvalid + } + var reference CredentialReference + var key string + if err := s.db.QueryRowContext(ctx, ` + SELECT credential.id, credential.provider, credential.reference_key + FROM social_account account + JOIN credential_reference credential ON credential.id = account.credential_reference_id + WHERE account.id = $1`, id).Scan(&reference.ID, &reference.Provider, &key); err != nil { + return nil, rowError(err) + } + value, err := resolver.Resolve(ctx, reference, key) + if err != nil { + return nil, err + } + if len(value) == 0 { + return nil, ErrInvalid + } + return value, nil +} + type accountScanner interface{ Scan(...any) error } func scanAccount(row accountScanner) (Account, error) { diff --git a/web/src/BrowsersPage.test.jsx b/web/src/BrowsersPage.test.jsx index 3d2f504..5aa67da 100644 --- a/web/src/BrowsersPage.test.jsx +++ b/web/src/BrowsersPage.test.jsx @@ -1,223 +1,479 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' -import { cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/react' -import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { Refine } from '@refinedev/core' -import { MemoryRouter } from 'react-router' -import { BrowserCreatePage, BrowserList } from './BrowsersPage' +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + cleanup, + fireEvent, + render, + screen, + waitFor, + within, +} from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { Refine } from "@refinedev/core"; +import { MemoryRouter } from "react-router"; +import { BrowserCreatePage, BrowserList } from "./BrowsersPage"; -afterEach(() => { cleanup(); vi.restoreAllMocks() }) +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); -const httpError = (message, status, body) => Object.assign(new Error(message), { status, body }) +const httpError = (message, status, body) => + Object.assign(new Error(message), { status, body }); const runtimes = [ - { id: 'account-a', alias: 'account-a', name: '店铺一号', state: 'running', status: 'Up', image_version: '148.0.0.1', fingerprint: { seed: 1000 }, endpoint: 'http://account-a:9222', account_id: 'social-a', network_exit_id: 'exit-1', network_exit_health: 'healthy', schedule_status: 'ready' }, - { id: 'account-b', alias: 'account-b', name: '店铺二号', state: 'exited', status: 'Exited', image_version: '148.0.0.1', fingerprint: { seed: 2000 }, endpoint: 'http://account-b:9222' }, -] -const gateways = [{ id: 'gw-1', name: 'gw-1', endpoint: 'http://gw:8081', token: 't' }] + { + id: "account-a", + alias: "account-a", + name: "店铺一号", + state: "running", + status: "Up", + image_version: "148.0.0.1", + fingerprint: { seed: 1000 }, + endpoint: "http://account-a:9222", + account_id: "social-a", + network_exit_id: "exit-1", + network_exit_health: "healthy", + schedule_status: "ready", + }, + { + id: "account-b", + alias: "account-b", + name: "店铺二号", + state: "exited", + status: "Exited", + image_version: "148.0.0.1", + fingerprint: { seed: 2000 }, + endpoint: "http://account-b:9222", + }, +]; +const gateways = [ + { id: "gw-1", name: "gw-1", endpoint: "http://gw:8081", token: "t" }, +]; const images = [ - { id: '148.0.0.1', version: '148.0.0.1', image_ref: 'reg/img:148', enabled: true }, - { id: '144.0.0.1', version: '144.0.0.1', image_ref: 'reg/img:144', enabled: true }, - { id: '139.0.0.1', version: '139.0.0.1', image_ref: 'reg/img:139', enabled: false }, -] -const accounts = [{ id: 'social-a', platform: 'douyin', authorization_status: 'authorized', runtime_status: 'paused' }] -const networkExits = [{ id: 'exit-1', protocol: 'socks5', host: 'proxy.example', port: 1080, health_status: 'healthy' }] + { + id: "148.0.0.1", + version: "148.0.0.1", + image_ref: "reg/img:148", + enabled: true, + }, + { + id: "144.0.0.1", + version: "144.0.0.1", + image_ref: "reg/img:144", + enabled: true, + }, + { + id: "139.0.0.1", + version: "139.0.0.1", + image_ref: "reg/img:139", + enabled: false, + }, +]; +const accounts = [ + { + id: "social-a", + platform: "douyin", + authorization_status: "authorized", + runtime_status: "paused", + }, +]; +const networkExits = [ + { + id: "exit-1", + protocol: "socks5", + host: "proxy.example", + port: 1080, + health_status: "healthy", + }, +]; function provider(overrides = {}) { return { getList: vi.fn(({ resource }) => { - if (resource === 'browsers') return Promise.resolve({ data: runtimes, total: runtimes.length }) - if (resource === 'gateways') return Promise.resolve({ data: gateways, total: gateways.length }) - if (resource === 'browser-images') return Promise.resolve({ data: images, total: images.length }) - if (resource === 'accounts') return Promise.resolve({ data: accounts, total: accounts.length }) - if (resource === 'network-exits') return Promise.resolve({ data: networkExits, total: networkExits.length }) - return Promise.reject(new Error(`unsupported ${resource}`)) + if (resource === "browsers") + return Promise.resolve({ data: runtimes, total: runtimes.length }); + if (resource === "gateways") + return Promise.resolve({ data: gateways, total: gateways.length }); + if (resource === "browser-images") + return Promise.resolve({ data: images, total: images.length }); + if (resource === "accounts") + return Promise.resolve({ data: accounts, total: accounts.length }); + if (resource === "network-exits") + return Promise.resolve({ + data: networkExits, + total: networkExits.length, + }); + return Promise.reject(new Error(`unsupported ${resource}`)); }), - create: vi.fn().mockResolvedValue({ data: { id: 'account-a', alias: 'account-a' } }), + create: vi + .fn() + .mockResolvedValue({ data: { id: "account-a", alias: "account-a" } }), browserAction: vi.fn().mockResolvedValue(undefined), - getOne: vi.fn(), getMany: vi.fn(), getManyReference: vi.fn(), update: vi.fn(), updateMany: vi.fn(), delete: vi.fn(), deleteMany: vi.fn(), + getOne: vi.fn(), + getMany: vi.fn(), + getManyReference: vi.fn(), + update: vi.fn(), + updateMany: vi.fn(), + delete: vi.fn(), + deleteMany: vi.fn(), ...overrides, - } + }; } -const queryClient = () => new QueryClient({ defaultOptions: { queries: { retry: false } } }) +const queryClient = () => + new QueryClient({ defaultOptions: { queries: { retry: false } } }); -const renderList = dataProvider => render( - - - - - , -) +const renderList = (dataProvider) => + render( + + + + + + + , + ); -const renderCreate = dataProvider => render( - - - - - , -) +const renderCreate = (dataProvider) => + render( + + + + + + + , + ); -describe('BrowserList', () => { - it('keeps CDP data in desktop and mobile views', async () => { - renderList(provider()) +describe("BrowserList", () => { + it("keeps CDP data in desktop and mobile views", async () => { + renderList(provider()); - expect(await screen.findAllByText('http://account-a:9222')).toHaveLength(2) - expect(screen.getAllByText('CDP')).toHaveLength(1) - }) + expect(await screen.findAllByText("http://account-a:9222")).toHaveLength(2); + expect(screen.getAllByText("CDP")).toHaveLength(1); + }); - it('does not poll the browser list', async () => { - const setInterval = vi.spyOn(globalThis, 'setInterval') - renderList(provider()) + it("does not poll the browser list", async () => { + const setInterval = vi.spyOn(globalThis, "setInterval"); + renderList(provider()); - await screen.findAllByText('店铺一号') - expect(setInterval.mock.calls.some(([, delay]) => delay === 3000)).toBe(false) - }) + await screen.findAllByText("店铺一号"); + expect(setInterval.mock.calls.some(([, delay]) => delay === 3000)).toBe( + false, + ); + }); - it('shows unknown states with their raw value as secondary evidence', async () => { - const unknownRuntime = { ...runtimes[0], state: 'dead' } - renderList(provider({ getList: vi.fn().mockResolvedValue({ data: [unknownRuntime], total: 1 }) })) + it("shows unknown states with their raw value as secondary evidence", async () => { + const unknownRuntime = { ...runtimes[0], state: "dead" }; + renderList( + provider({ + getList: vi + .fn() + .mockResolvedValue({ data: [unknownRuntime], total: 1 }), + }), + ); - expect(await screen.findAllByText('未知状态')).toHaveLength(2) - const rawStates = screen.getAllByText('dead') - expect(rawStates).toHaveLength(2) - }) + expect(await screen.findAllByText("未知状态")).toHaveLength(2); + const rawStates = screen.getAllByText("dead"); + expect(rawStates).toHaveLength(2); + }); - it.each(['constructor', 'toString', '__proto__'])('treats prototype key %s as an unknown state', async state => { - const protoRuntime = { ...runtimes[0], state } - renderList(provider({ getList: vi.fn().mockResolvedValue({ data: [protoRuntime], total: 1 }) })) + it.each([ + "constructor", + "toString", + "__proto__", + ])("treats prototype key %s as an unknown state", async (state) => { + const protoRuntime = { ...runtimes[0], state }; + renderList( + provider({ + getList: vi.fn().mockResolvedValue({ data: [protoRuntime], total: 1 }), + }), + ); - expect(await screen.findAllByText('未知状态')).toHaveLength(2) - expect(screen.getAllByText(state)).toHaveLength(2) - }) + expect(await screen.findAllByText("未知状态")).toHaveLength(2); + expect(screen.getAllByText(state)).toHaveLength(2); + }); - it('disables invalid lifecycle actions and sends explicit domain actions', async () => { - const dataProvider = provider() - renderList(dataProvider) + it("disables invalid lifecycle actions and sends explicit domain actions", async () => { + const dataProvider = provider(); + renderList(dataProvider); - await screen.findAllByText('店铺一号') - expect(screen.getAllByLabelText('启动 店铺一号')[0].disabled).toBe(true) - expect(screen.getAllByLabelText('停止 店铺二号')[0].disabled).toBe(true) - fireEvent.click(screen.getAllByLabelText('停止 店铺一号')[0]) + await screen.findAllByText("店铺一号"); + expect(screen.getAllByLabelText("启动 店铺一号")[0].disabled).toBe(true); + expect(screen.getAllByLabelText("停止 店铺二号")[0].disabled).toBe(true); + fireEvent.click(screen.getAllByLabelText("停止 店铺一号")[0]); - await waitFor(() => expect(dataProvider.browserAction).toHaveBeenCalledWith('account-a', 'stop', undefined)) - }) + await waitFor(() => + expect(dataProvider.browserAction).toHaveBeenCalledWith( + "account-a", + "stop", + undefined, + ), + ); + }); - it('starts ready or runtime-missing environments and preserves resource gates', async () => { + it("starts ready or runtime-missing environments and preserves resource gates", async () => { const environments = [ - { ...runtimes[1], id: 'ready', alias: 'ready', name: '就绪环境', account_id: 'social-a', network_exit_id: 'exit-1', network_exit_health: 'healthy', schedule_status: 'ready' }, - { ...runtimes[1], id: 'stopped', alias: 'stopped', name: '已停止环境', account_id: 'social-a', network_exit_id: 'exit-1', network_exit_health: 'healthy', schedule_status: 'blocked', schedule_block_reason: 'runtime_missing' }, - { ...runtimes[1], id: 'revoked', alias: 'revoked', name: '已撤权环境', schedule_status: 'blocked', schedule_block_reason: 'account_revoked' }, - { ...runtimes[1], id: 'paused', alias: 'paused', name: '账号暂停环境', schedule_status: 'blocked', schedule_block_reason: 'account_paused' }, - { ...runtimes[1], id: 'unbound', alias: 'unbound', name: '未绑定环境', schedule_status: 'blocked', schedule_block_reason: 'binding_missing' }, - { ...runtimes[1], id: 'no-exit', alias: 'no-exit', name: '无出口环境', schedule_status: 'blocked', schedule_block_reason: 'network_exit_missing' }, - { ...runtimes[1], id: 'unhealthy', alias: 'unhealthy', name: '出口异常环境', account_id: 'social-a', network_exit_id: 'exit-1', network_exit_health: 'unhealthy', schedule_status: 'blocked', schedule_block_reason: 'network_exit_unhealthy' }, - { ...runtimes[1], id: 'cleanup', alias: 'cleanup', name: '清理中环境', schedule_status: 'blocked', schedule_block_reason: 'runtime_stop_pending' }, - ] - const dataProvider = provider({ getList: vi.fn(({ resource }) => Promise.resolve({ data: resource === 'browsers' ? environments : [], total: resource === 'browsers' ? environments.length : 0 })) }) - renderList(dataProvider) - - fireEvent.click((await screen.findAllByLabelText('启动 就绪环境'))[0]) - fireEvent.click(screen.getAllByLabelText('启动 已停止环境')[0]) - for (const [name, reason] of [['已撤权环境', '授权已撤销'], ['账号暂停环境', '账号已暂停'], ['未绑定环境', '绑定缺失'], ['无出口环境', '固定出口缺失'], ['出口异常环境', '固定出口不健康'], ['清理中环境', '停止结果待确认']]) { - expect(screen.getAllByLabelText(`启动 ${name}（${reason}）`).every(button => button.disabled)).toBe(true) - } - await waitFor(() => expect(dataProvider.browserAction.mock.calls).toEqual(expect.arrayContaining([ - ['ready', 'start', undefined], - ['stopped', 'start', undefined], - ]))) - }) - - it('keeps the backend 409 fallback for stale readiness', async () => { - const stale = { ...runtimes[1], account_id: 'social-a', network_exit_id: 'exit-1', network_exit_health: 'healthy', schedule_status: 'ready' } + { + ...runtimes[1], + id: "ready", + alias: "ready", + name: "就绪环境", + account_id: "social-a", + network_exit_id: "exit-1", + network_exit_health: "healthy", + schedule_status: "ready", + }, + { + ...runtimes[1], + id: "stopped", + alias: "stopped", + name: "已停止环境", + account_id: "social-a", + network_exit_id: "exit-1", + network_exit_health: "healthy", + schedule_status: "blocked", + schedule_block_reason: "runtime_missing", + }, + { + ...runtimes[1], + id: "revoked", + alias: "revoked", + name: "已撤权环境", + schedule_status: "blocked", + schedule_block_reason: "account_revoked", + }, + { + ...runtimes[1], + id: "paused", + alias: "paused", + name: "账号暂停环境", + schedule_status: "blocked", + schedule_block_reason: "account_paused", + }, + { + ...runtimes[1], + id: "unbound", + alias: "unbound", + name: "未绑定环境", + schedule_status: "blocked", + schedule_block_reason: "binding_missing", + }, + { + ...runtimes[1], + id: "no-exit", + alias: "no-exit", + name: "无出口环境", + schedule_status: "blocked", + schedule_block_reason: "network_exit_missing", + }, + { + ...runtimes[1], + id: "unhealthy", + alias: "unhealthy", + name: "出口异常环境", + account_id: "social-a", + network_exit_id: "exit-1", + network_exit_health: "unhealthy", + schedule_status: "blocked", + schedule_block_reason: "network_exit_unhealthy", + }, + { + ...runtimes[1], + id: "cleanup", + alias: "cleanup", + name: "清理中环境", + schedule_status: "blocked", + schedule_block_reason: "runtime_stop_pending", + }, + ]; const dataProvider = provider({ - getList: vi.fn(({ resource }) => Promise.resolve({ data: resource === 'browsers' ? [stale] : [], total: resource === 'browsers' ? 1 : 0 })), - browserAction: vi.fn().mockRejectedValue(httpError('readiness changed', 409)), - }) - renderList(dataProvider) + getList: vi.fn(({ resource }) => + Promise.resolve({ + data: resource === "browsers" ? environments : [], + total: resource === "browsers" ? environments.length : 0, + }), + ), + }); + renderList(dataProvider); - fireEvent.click((await screen.findAllByLabelText('启动 店铺二号'))[0]) + fireEvent.click((await screen.findAllByLabelText("启动 就绪环境"))[0]); + fireEvent.click(screen.getAllByLabelText("启动 已停止环境")[0]); + for (const [name, reason] of [ + ["已撤权环境", "授权已撤销"], + ["账号暂停环境", "账号已暂停"], + ["未绑定环境", "绑定缺失"], + ["无出口环境", "固定出口缺失"], + ["出口异常环境", "固定出口不健康"], + ["清理中环境", "停止结果待确认"], + ]) { + expect( + screen + .getAllByLabelText(`启动 ${name}（${reason}）`) + .every((button) => button.disabled), + ).toBe(true); + } + await waitFor(() => + expect(dataProvider.browserAction.mock.calls).toEqual( + expect.arrayContaining([ + ["ready", "start", undefined], + ["stopped", "start", undefined], + ]), + ), + ); + }); - expect((await screen.findByRole('alert')).textContent).toContain('readiness changed') - }) + it("keeps the backend 409 fallback for stale readiness", async () => { + const stale = { + ...runtimes[1], + account_id: "social-a", + network_exit_id: "exit-1", + network_exit_health: "healthy", + schedule_status: "ready", + }; + const dataProvider = provider({ + getList: vi.fn(({ resource }) => + Promise.resolve({ + data: resource === "browsers" ? [stale] : [], + total: resource === "browsers" ? 1 : 0, + }), + ), + browserAction: vi + .fn() + .mockRejectedValue(httpError("readiness changed", 409)), + }); + renderList(dataProvider); - it('creates an env with an account and healthy network exit without legacy proxy fields', async () => { - const dataProvider = provider() - renderCreate(dataProvider) - await screen.findByRole('button', { name: '创建环境' }) + fireEvent.click((await screen.findAllByLabelText("启动 店铺二号"))[0]); - fireEvent.change(screen.getByRole('textbox', { name: /环境名称/ }), { target: { value: '店铺三号' } }) - fireEvent.change(screen.getByRole('textbox', { name: /别名/ }), { target: { value: 'shop-03' } }) - fireEvent.click(screen.getByRole('combobox', { name: '网络出口' })) - fireEvent.click(await screen.findByRole('option', { name: /exit-1/ })) - fireEvent.click(screen.getByRole('button', { name: '创建环境' })) + expect((await screen.findByRole("alert")).textContent).toContain( + "readiness changed", + ); + }); - await waitFor(() => expect(dataProvider.create).toHaveBeenCalledWith({ resource: 'browsers', variables: { - alias: 'shop-03', - name: '店铺三号', - gateway: 'gw-1', - image_version: '148.0.0.1', - account_id: 'social-a', - network_exit_id: 'exit-1', - fingerprint: { seed: 1000 }, - } })) - expect(screen.queryByLabelText('代理 proxy-server')).toBeNull() - expect(screen.queryByLabelText('禁用非代理 UDP')).toBeNull() - }) + it("creates an env with an account and healthy network exit without legacy proxy fields", async () => { + const dataProvider = provider(); + renderCreate(dataProvider); + await screen.findByRole("button", { name: "创建环境" }); - it('creates an env with the current machine exit when network exit is empty', async () => { - const dataProvider = provider({ getList: vi.fn(({ resource }) => { - if (resource === 'network-exits') return Promise.resolve({ data: [], total: 0 }) - return provider().getList({ resource }) - }) }) - renderCreate(dataProvider) - await screen.findByRole('button', { name: '创建环境' }) + fireEvent.change(screen.getByRole("textbox", { name: /环境名称/ }), { + target: { value: "店铺三号" }, + }); + fireEvent.change(screen.getByRole("textbox", { name: /别名/ }), { + target: { value: "shop-03" }, + }); + fireEvent.click(screen.getByRole("combobox", { name: "网络出口" })); + fireEvent.click(await screen.findByRole("option", { name: /exit-1/ })); + fireEvent.click(screen.getByRole("button", { name: "创建环境" })); - fireEvent.change(screen.getByRole('textbox', { name: /环境名称/ }), { target: { value: '直连环境' } }) - fireEvent.change(screen.getByRole('textbox', { name: /别名/ }), { target: { value: 'direct-env' } }) - fireEvent.click(screen.getByRole('button', { name: '创建环境' })) + await waitFor(() => + expect(dataProvider.create).toHaveBeenCalledWith({ + resource: "browsers", + variables: { + alias: "shop-03", + name: "店铺三号", + gateway: "gw-1", + image_version: "148.0.0.1", + account_id: "social-a", + network_exit_id: "exit-1", + fingerprint: { seed: 1000 }, + }, + }), + ); + expect(screen.queryByLabelText("代理 proxy-server")).toBeNull(); + expect(screen.queryByLabelText("禁用非代理 UDP")).toBeNull(); + }); - await waitFor(() => expect(dataProvider.create).toHaveBeenCalledWith({ resource: 'browsers', variables: expect.objectContaining({ - alias: 'direct-env', - network_exit_id: '', - }) })) - expect(screen.getByText('留空时使用当前机器网络出口直连')).toBeTruthy() - }) + it("creates an env with the current machine exit when network exit is empty", async () => { + const dataProvider = provider({ + getList: vi.fn(({ resource }) => { + if (resource === "network-exits") + return Promise.resolve({ data: [], total: 0 }); + return provider().getList({ resource }); + }), + }); + renderCreate(dataProvider); + await screen.findByRole("button", { name: "创建环境" }); - it('keeps binding input after a 409 conflict', async () => { - const dataProvider = provider({ create: vi.fn().mockRejectedValue(httpError('conflict', 409)) }) - renderCreate(dataProvider) - await screen.findByRole('button', { name: '创建环境' }) + fireEvent.change(screen.getByRole("textbox", { name: /环境名称/ }), { + target: { value: "直连环境" }, + }); + fireEvent.change(screen.getByRole("textbox", { name: /别名/ }), { + target: { value: "direct-env" }, + }); + fireEvent.click(screen.getByRole("button", { name: "创建环境" })); - fireEvent.change(screen.getByRole('textbox', { name: /环境名称/ }), { target: { value: '冲突环境' } }) - fireEvent.change(screen.getByRole('textbox', { name: /别名/ }), { target: { value: 'conflict-env' } }) - fireEvent.click(screen.getByRole('button', { name: '创建环境' })) + await waitFor(() => + expect(dataProvider.create).toHaveBeenCalledWith({ + resource: "browsers", + variables: expect.objectContaining({ + alias: "direct-env", + network_exit_id: "", + }), + }), + ); + expect(screen.getByText("留空时使用当前机器网络出口直连")).toBeTruthy(); + }); - expect((await screen.findByRole('alert')).textContent).toContain('冲突（409）') - expect(screen.getByRole('textbox', { name: /环境名称/ }).value).toBe('冲突环境') - }) + it("keeps binding input after a 409 conflict", async () => { + const dataProvider = provider({ + create: vi.fn().mockRejectedValue(httpError("conflict", 409)), + }); + renderCreate(dataProvider); + await screen.findByRole("button", { name: "创建环境" }); - it('upgrades an env through the version dialog', async () => { - const dataProvider = provider() - renderList(dataProvider) - await screen.findAllByText('店铺一号') + fireEvent.change(screen.getByRole("textbox", { name: /环境名称/ }), { + target: { value: "冲突环境" }, + }); + fireEvent.change(screen.getByRole("textbox", { name: /别名/ }), { + target: { value: "conflict-env" }, + }); + fireEvent.click(screen.getByRole("button", { name: "创建环境" })); - fireEvent.click(screen.getAllByLabelText('升级 店铺一号')[0]) - const dialog = await screen.findByRole('dialog') - fireEvent.click(within(dialog).getByRole('combobox', { name: /目标镜像版本/ })) - fireEvent.click(await within(document.body).findByRole('option', { name: '144.0.0.1' })) - fireEvent.click(within(dialog).getByRole('button', { name: '确认升级' })) + expect((await screen.findByRole("alert")).textContent).toContain( + "冲突（409）", + ); + expect(screen.getByRole("textbox", { name: /环境名称/ }).value).toBe( + "冲突环境", + ); + }); - await waitFor(() => expect(dataProvider.browserAction).toHaveBeenCalledWith('account-a', 'upgrade', { version: '144.0.0.1' })) - }) + it("upgrades an env through the version dialog", async () => { + const dataProvider = provider(); + renderList(dataProvider); + await screen.findAllByText("店铺一号"); - it('shows action errors and keeps a retry control', async () => { - const dataProvider = provider({ browserAction: vi.fn().mockRejectedValue(new Error('停止失败')) }) - renderList(dataProvider) - await screen.findAllByText('店铺一号') + fireEvent.click(screen.getAllByLabelText("升级 店铺一号")[0]); + const dialog = await screen.findByRole("dialog"); + fireEvent.click( + within(dialog).getByRole("combobox", { name: /目标镜像版本/ }), + ); + fireEvent.click( + await within(document.body).findByRole("option", { name: "144.0.0.1" }), + ); + fireEvent.click(within(dialog).getByRole("button", { name: "确认升级" })); - fireEvent.click(screen.getAllByLabelText('停止 店铺一号')[0]) + await waitFor(() => + expect(dataProvider.browserAction).toHaveBeenCalledWith( + "account-a", + "upgrade", + { version: "144.0.0.1" }, + ), + ); + }); - expect((await screen.findByRole('alert')).textContent).toContain('停止失败') - expect(screen.getByRole('button', { name: '重试' })).toBeTruthy() - }) -}) + it("shows action errors and keeps a retry control", async () => { + const dataProvider = provider({ + browserAction: vi.fn().mockRejectedValue(new Error("停止失败")), + }); + renderList(dataProvider); + await screen.findAllByText("店铺一号"); + + fireEvent.click(screen.getAllByLabelText("停止 店铺一号")[0]); + + expect((await screen.findByRole("alert")).textContent).toContain( + "停止失败", + ); + expect(screen.getByRole("button", { name: "重试" })).toBeTruthy(); + }); +}); diff --git a/web/src/CreatorAccountsPage.jsx b/web/src/CreatorAccountsPage.jsx new file mode 100644 index 0000000..5e73555 --- /dev/null +++ b/web/src/CreatorAccountsPage.jsx @@ -0,0 +1,647 @@ +import { useEffect, useMemo, useState } from "react"; +import { useDataProvider } from "@refinedev/core"; +import { + Alert, + Button, + Card, + CardContent, + Checkbox, + Field, + Input, + PageHeader, + PageState, + Select, + StatusPill, + Textarea, + dateTime, + conflictMessage, +} from "./lib/ui.jsx"; + +const emptyProfile = { + login_username: "", + password: "", + real_name_status: "unknown", + real_name: "", + identity_number: "", + note: "", + business_status: "normal", + big_account: false, + reply_requirements: "", + cooldown_seconds: 86400, +}; + +const platformLabel = { douyin: "抖音", xiaohongshu: "小红书" }; +const statusLabel = { + normal: "正常", + muted: "禁言", + banned: "封禁", + deleted: "注销", +}; +const eventTypeOptions = [ + ["comment", "评论"], + ["like", "点赞"], + ["repost", "转发"], + ["follow", "关注"], +]; + +function profileForm(profile) { + return { + login_username: profile.login_username || "", + password: "", + real_name_status: profile.real_name_status || "unknown", + real_name: profile.real_name || "", + identity_number: profile.identity_number || "", + note: profile.note || "", + business_status: profile.business_status || "normal", + big_account: Boolean(profile.big_account), + reply_requirements: profile.reply_requirements || "", + cooldown_seconds: profile.cooldown_seconds || 86400, + }; +} + +export function CreatorAccountsPage() { + const dataProvider = useDataProvider()("default"); + const [profiles, setProfiles] = useState([]); + const [selectedID, setSelectedID] = useState(""); + const [form, setForm] = useState(emptyProfile); + const [pending, setPending] = useState(true); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + const [busy, setBusy] = useState(false); + const [strategies, setStrategies] = useState([]); + const [strategyForm, setStrategyForm] = useState({ + execution_account_id: "", + position: 1, + enabled: true, + event_types: ["comment"], + action: "reply_comment", + target_type: "comment", + candidate_texts: "", + }); + + const load = async () => { + setPending(true); + setError(null); + try { + const result = await dataProvider.getList({ + resource: "creator-accounts", + }); + setProfiles(result.data); + if (selectedID) { + const selected = result.data.find((item) => item.id === selectedID); + if (selected) setForm(profileForm(selected)); + } else if (result.data[0]) { + setSelectedID(result.data[0].id); + setForm(profileForm(result.data[0])); + } + } catch (loadError) { + setError(loadError); + } finally { + setPending(false); + } + }; + + useEffect(() => { + load(); + }, []); + useEffect(() => { + if (!selectedID) return; + dataProvider + .creatorGet( + `/creator/accounts/${encodeURIComponent(selectedID)}/strategies`, + ) + .then(setStrategies) + .catch(() => setStrategies([])); + }, [selectedID]); + + const selected = useMemo( + () => profiles.find((item) => item.id === selectedID), + [profiles, selectedID], + ); + const choose = (profile) => { + setSelectedID(profile.id); + setForm(profileForm(profile)); + setNotice(null); + }; + const change = (field) => (event) => + setForm((value) => ({ ...value, [field]: event.target.value })); + const save = async (event) => { + event.preventDefault(); + if (!selected) return; + setBusy(true); + setNotice(null); + try { + const result = await dataProvider.creatorUpdate( + `/creator/accounts/${encodeURIComponent(selected.id)}/profile`, + { + ...form, + cooldown_seconds: Number(form.cooldown_seconds), + }, + ); + setProfiles((items) => + items.map((item) => (item.id === selected.id ? result : item)), + ); + setForm(profileForm(result)); + setNotice({ variant: "success", text: "账号资料已保存。" }); + } catch (saveError) { + setNotice({ + variant: "destructive", + text: conflictMessage(saveError, "账号资料保存失败"), + }); + } finally { + setBusy(false); + } + }; + const toggleBig = async () => { + if (!selected) return; + setBusy(true); + setNotice(null); + try { + const result = await dataProvider.creatorAction( + `/creator/accounts/${encodeURIComponent(selected.id)}/big-account`, + { enabled: !selected.big_account }, + ); + setProfiles((items) => + items.map((item) => (item.id === selected.id ? result : item)), + ); + setForm(profileForm(result)); + setNotice({ + variant: "success", + text: result.big_account ? "已开启大号模式。" : "已关闭大号模式。", + }); + } catch (actionError) { + setNotice({ + variant: "destructive", + text: conflictMessage(actionError, "大号模式更新失败"), + }); + } finally { + setBusy(false); + } + }; + const createStrategy = async (event) => { + event.preventDefault(); + if (!selected || !strategyForm.execution_account_id) { + setNotice({ variant: "warning", text: "请先选择同平台小号。" }); + return; + } + if (!strategyForm.event_types.length) { + setNotice({ variant: "warning", text: "至少选择一种触发事件。" }); + return; + } + setBusy(true); + setNotice(null); + try { + await dataProvider.creatorCreate("/creator/relations", { + big_account_id: selected.id, + small_account_id: strategyForm.execution_account_id, + enabled: true, + }); + const result = await dataProvider.creatorCreate( + `/creator/accounts/${encodeURIComponent(selected.id)}/strategies`, + { + ...strategyForm, + position: Number(strategyForm.position), + candidate_texts: strategyForm.candidate_texts + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + }, + ); + setStrategies((items) => + [...items, result].sort( + (left, right) => left.position - right.position, + ), + ); + setNotice({ variant: "success", text: "自动响应策略已保存。" }); + } catch (createError) { + setNotice({ + variant: "destructive", + text: conflictMessage( + createError, + "策略保存失败；请确认执行账号已绑定且平台一致", + ), + }); + } finally { + setBusy(false); + } + }; + const setStrategyEnabled = async (strategy, enabled) => { + setBusy(true); + setNotice(null); + try { + const result = await dataProvider.creatorAction( + `/creator/strategies/${encodeURIComponent(strategy.id)}/${enabled ? "enable" : "disable"}`, + ); + setStrategies((items) => + items.map((item) => (item.id === strategy.id ? result : item)), + ); + } catch (actionError) { + setNotice({ + variant: "destructive", + text: conflictMessage(actionError, "策略状态更新失败"), + }); + } finally { + setBusy(false); + } + }; + const deleteStrategy = async (strategy) => { + setBusy(true); + setNotice(null); + try { + await dataProvider.creatorRequest( + `/creator/strategies/${encodeURIComponent(strategy.id)}`, + { method: "DELETE" }, + ); + setStrategies((items) => items.filter((item) => item.id !== strategy.id)); + setNotice({ variant: "success", text: "策略已删除。" }); + } catch (actionError) { + setNotice({ + variant: "destructive", + text: conflictMessage(actionError, "策略删除失败"), + }); + } finally { + setBusy(false); + } + }; + + return ( + <> + + +
+ + +
    + {profiles.map((profile) => ( +
  • + +
    • + ))} +
+ + + {selected ? ( + + +
+
+

+ {selected.name || selected.platform_account_key} +

+

+ {platformLabel[selected.platform] || selected.platform} ·{" "} + {selected.platform_account_key} +

+
+ +
+ {notice ? ( + + {notice.text} + + ) : null} +
+ + + + + + + + + + + + + + + + + + + +