feat: integrate creator hub douyin workflows

This commit is contained in:
2026-09-14 19:22:07 +08:00
parent 025fe62c37
commit 44a28954cf
42 changed files with 2850 additions and 880 deletions
+75 -26
View File
@@ -125,10 +125,11 @@ func (s *Store) CreateStrategy(ctx context.Context, bigAccountID string, input S
if err != nil {
return Strategy{}, err
}
if small.Platform != "" {
if err := s.requireRelation(ctx, bigAccountID, input.ExecutionAccountID); err != nil {
return Strategy{}, err
}
if big.Platform != small.Platform {
return Strategy{}, ErrConflict
}
if err := s.requireRelation(ctx, bigAccountID, input.ExecutionAccountID); err != nil {
return Strategy{}, err
}
events, texts, err := encodeStrategyLists(input)
if err != nil {
@@ -191,6 +192,17 @@ func (s *Store) UpdateStrategy(ctx context.Context, id string, input StrategyInp
if err := s.validateEnabledStrategy(ctx, strategy.BigAccountID, input); err != nil {
return Strategy{}, err
}
big, err := s.GetAccountProfile(ctx, strategy.BigAccountID)
if err != nil {
return Strategy{}, err
}
small, err := s.GetAccountProfile(ctx, input.ExecutionAccountID)
if err != nil {
return Strategy{}, err
}
if big.Platform != small.Platform {
return Strategy{}, ErrConflict
}
events, texts, err := encodeStrategyLists(input)
if err != nil {
return Strategy{}, err
@@ -225,21 +237,21 @@ func (s *Store) DeleteStrategy(ctx context.Context, id string) error {
func scanEvent(scanner interface{ Scan(...any) error }) (InteractionEvent, error) {
var result InteractionEvent
var platformAt, receivedAt, startedAt, finishedAt, displayedAt sql.NullTime
var commentID, workID, messageText, strategyID, executionID sql.NullString
if err := scanner.Scan(&result.ID, &result.Platform, &result.ReceivingAccountID, &result.EventKey, &result.EventType, &result.InteractorUID, &commentID, &workID, &messageText, &platformAt, &receivedAt, &startedAt, &finishedAt, &displayedAt, &result.State, &result.Reason, &strategyID, &executionID); err != nil {
var platformAt, gatewayReceivedAt, receivedAt, startedAt, finishedAt, displayedAt sql.NullTime
var commentID, workID, messageType, messageText, strategyID, executionID sql.NullString
if err := scanner.Scan(&result.ID, &result.Platform, &result.ReceivingAccountID, &result.EventKey, &result.EventType, &result.InteractorUID, &commentID, &workID, &messageType, &messageText, &platformAt, &gatewayReceivedAt, &receivedAt, &startedAt, &finishedAt, &displayedAt, &result.State, &result.Reason, &strategyID, &executionID); err != nil {
return InteractionEvent{}, err
}
result.CommentID, result.WorkID, result.MessageText = commentID.String, workID.String, messageText.String
result.CommentID, result.WorkID, result.MessageType, result.MessageText = commentID.String, workID.String, messageType.String, messageText.String
result.StrategyID, result.ExecutionAccountID = strategyID.String, executionID.String
result.PlatformEventAt, result.ProcessingStartedAt, result.ProcessingFinishedAt, result.DisplayedAt = nullableTime(platformAt), nullableTime(startedAt), nullableTime(finishedAt), nullableTime(displayedAt)
result.PlatformEventAt, result.GatewayReceivedAt, result.ProcessingStartedAt, result.ProcessingFinishedAt, result.DisplayedAt = nullableTime(platformAt), nullableTime(gatewayReceivedAt), nullableTime(startedAt), nullableTime(finishedAt), nullableTime(displayedAt)
if receivedAt.Valid {
result.ReceivedAt = receivedAt.Time.UTC()
}
return result, nil
}
const eventSelect = `SELECT id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_text,platform_event_at,received_at,processing_started_at,processing_finished_at,displayed_at,state,reason,strategy_id,execution_account_id FROM creator_event`
const eventSelect = `SELECT id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_type,message_text,platform_event_at,gateway_received_at,received_at,processing_started_at,processing_finished_at,displayed_at,state,reason,strategy_id,execution_account_id FROM creator_event`
func (s *Store) GetEvent(ctx context.Context, id string) (InteractionEvent, error) {
result, err := scanEvent(s.db.QueryRowContext(ctx, eventSelect+` WHERE id=$1`, id))
@@ -270,8 +282,11 @@ func (s *Store) ListEvents(ctx context.Context, accountID string) ([]Interaction
}
func (s *Store) RecordEvent(ctx context.Context, input InteractionEvent) (AutomaticResult, error) {
input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageText = strings.TrimSpace(input.Platform), strings.TrimSpace(input.ReceivingAccountID), strings.TrimSpace(input.EventKey), strings.TrimSpace(input.EventType), strings.TrimSpace(input.InteractorUID), strings.TrimSpace(input.CommentID), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.MessageText)
if !ValidatePlatform(input.Platform) || input.ReceivingAccountID == "" || input.EventKey == "" || !ValidEventType(input.EventType) || len(input.MessageText) > 100000 {
input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageType, input.MessageText = strings.TrimSpace(input.Platform), strings.TrimSpace(input.ReceivingAccountID), strings.TrimSpace(input.EventKey), strings.TrimSpace(input.EventType), strings.TrimSpace(input.InteractorUID), strings.TrimSpace(input.CommentID), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.MessageType), strings.TrimSpace(input.MessageText)
if input.MessageType == "" {
input.MessageType = MessageTypeText
}
if !ValidatePlatform(input.Platform) || input.ReceivingAccountID == "" || input.EventKey == "" || !ValidEventType(input.EventType) || !ValidMessageType(input.MessageType) || len(input.MessageText) > 100000 {
return AutomaticResult{}, ErrInvalid
}
profile, err := s.GetAccountProfile(ctx, input.ReceivingAccountID)
@@ -301,7 +316,7 @@ func (s *Store) RecordEvent(ctx context.Context, input InteractionEvent) (Automa
}
var returnedID string
var inserted bool
err = s.db.QueryRowContext(ctx, `INSERT INTO creator_event (id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_text,platform_event_at,received_at,state,reason) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) ON CONFLICT (platform,receiving_account_id,event_key) DO NOTHING RETURNING id,(xmax=0)`, id, input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageText, input.PlatformEventAt, receivedAt, state, reason).Scan(&returnedID, &inserted)
err = s.db.QueryRowContext(ctx, `INSERT INTO creator_event (id,platform,receiving_account_id,event_key,event_type,interactor_uid,comment_id,work_id,message_type,message_text,platform_event_at,gateway_received_at,received_at,state,reason) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (platform,receiving_account_id,event_key) DO NOTHING RETURNING id,(xmax=0)`, id, input.Platform, input.ReceivingAccountID, input.EventKey, input.EventType, input.InteractorUID, input.CommentID, input.WorkID, input.MessageType, input.MessageText, input.PlatformEventAt, input.GatewayReceivedAt, receivedAt, state, reason).Scan(&returnedID, &inserted)
if errors.Is(err, sql.ErrNoRows) {
existingErr := s.db.QueryRowContext(ctx, `SELECT id FROM creator_event WHERE platform=$1 AND receiving_account_id=$2 AND event_key=$3`, input.Platform, input.ReceivingAccountID, input.EventKey).Scan(&returnedID)
if existingErr != nil {
@@ -851,6 +866,19 @@ func (s *Store) UpdateOperationResult(ctx context.Context, id string, result Act
_, err = s.db.ExecContext(ctx, `UPDATE creator_operation SET state=$2,evidence=$3::jsonb,reason=$4,updated_at=now() WHERE id=$1`, id, result.State, evidence, result.Reason)
return databaseError(err)
}
func (s *Store) persistDirectMessageResult(ctx context.Context, op Operation, result ActionResult) error {
if op.Action != ActionDM {
return nil
}
messageState := result.State
if messageState == "blocked" {
messageState = "failed"
}
messageAt := time.Now().UTC()
_, _, err := s.SaveMessage(ctx, MessageInput{Platform: op.Platform, AccountID: op.AccountID, PeerUID: op.TargetUID, PlatformMessageKey: "operation:" + op.ID, Direction: "outbound", MessageType: "text", Text: op.Text, SentState: messageState, MessageAt: &messageAt})
return err
}
func (s *Store) ExecuteManualOperation(ctx context.Context, id string, executor ActionExecutor) (Operation, error) {
op, err := s.GetOperation(ctx, id)
if err != nil {
@@ -872,21 +900,33 @@ func (s *Store) ExecuteManualOperation(ctx context.Context, id string, executor
executionLock.Lock()
defer executionLock.Unlock()
if _, checkErr := s.AccountWriteCheck(ctx, op.AccountID, false, op.Action); checkErr != nil {
if updateErr := s.UpdateOperationResult(ctx, id, actionPreconditionResult(checkErr, "写入前条件已变化")); updateErr != nil {
result := actionPreconditionResult(checkErr, "写入前条件已变化")
if updateErr := s.UpdateOperationResult(ctx, id, result); updateErr != nil {
return Operation{}, updateErr
}
if messageErr := s.persistDirectMessageResult(ctx, op, result); messageErr != nil {
return Operation{}, fmt.Errorf("persist direct message result: %w", messageErr)
}
return s.GetOperation(ctx, id)
}
if checkErr := s.validateOperationTarget(ctx, &OperationInput{Platform: op.Platform, AccountID: op.AccountID, Action: op.Action, TargetUID: op.TargetUID, TargetCommentID: op.TargetCommentID, TargetWorkID: op.TargetWorkID}); checkErr != nil {
if updateErr := s.UpdateOperationResult(ctx, id, actionPreconditionResult(checkErr, "写入目标已变化")); updateErr != nil {
result := actionPreconditionResult(checkErr, "写入目标已变化")
if updateErr := s.UpdateOperationResult(ctx, id, result); updateErr != nil {
return Operation{}, updateErr
}
if messageErr := s.persistDirectMessageResult(ctx, op, result); messageErr != nil {
return Operation{}, fmt.Errorf("persist direct message result: %w", messageErr)
}
return s.GetOperation(ctx, id)
}
if executor == nil {
if updateErr := s.UpdateOperationResult(ctx, id, ActionResult{State: "uncertain", Reason: "平台执行器不可用"}); updateErr != nil {
result := ActionResult{State: "uncertain", Reason: "平台执行器不可用"}
if updateErr := s.UpdateOperationResult(ctx, id, result); updateErr != nil {
return Operation{}, updateErr
}
if messageErr := s.persistDirectMessageResult(ctx, op, result); messageErr != nil {
return Operation{}, fmt.Errorf("persist direct message result: %w", messageErr)
}
return s.GetOperation(ctx, id)
}
result, execErr := executor.Execute(ctx, ActionRequest{OperationID: claimedID, Action: op.Action, Platform: op.Platform, AccountID: op.AccountID, TargetUID: op.TargetUID, TargetCommentID: op.TargetCommentID, TargetWorkID: op.TargetWorkID, Text: op.Text})
@@ -894,15 +934,8 @@ func (s *Store) ExecuteManualOperation(ctx context.Context, id string, executor
if err := s.UpdateOperationResult(ctx, id, result); err != nil {
return Operation{}, err
}
if op.Action == "dm" {
messageState := result.State
if messageState == "blocked" {
messageState = "failed"
}
messageAt := time.Now().UTC()
if _, _, messageErr := s.SaveMessage(ctx, MessageInput{Platform: op.Platform, AccountID: op.AccountID, PeerUID: op.TargetUID, PlatformMessageKey: "operation:" + op.ID, Direction: "outbound", MessageType: "text", Text: op.Text, SentState: messageState, MessageAt: &messageAt}); messageErr != nil {
return Operation{}, fmt.Errorf("persist direct message result: %w", messageErr)
}
if err := s.persistDirectMessageResult(ctx, op, result); err != nil {
return Operation{}, fmt.Errorf("persist direct message result: %w", err)
}
return s.GetOperation(ctx, id)
}
@@ -968,7 +1001,7 @@ func (s *Store) SaveMessage(ctx context.Context, input MessageInput) (Message, b
input.PlatformMessageKey = strings.TrimSpace(input.PlatformMessageKey)
input.Direction = strings.TrimSpace(input.Direction)
input.MessageType = strings.TrimSpace(input.MessageType)
if input.PlatformMessageKey == "" || input.Direction != "inbound" && input.Direction != "outbound" || input.MessageType == "" || len(input.Text) > 100000 {
if input.PlatformMessageKey == "" || len(input.PlatformMessageKey) > 500 || input.Direction != "inbound" && input.Direction != "outbound" || input.MessageType == "" || len(input.MessageType) > 100 || len(input.Text) > 100000 {
return Message{}, false, ErrInvalid
}
state := input.SentState
@@ -1016,6 +1049,22 @@ func (s *Store) GetMessage(ctx context.Context, id string) (Message, error) {
result, err := scanMessage(s.db.QueryRowContext(ctx, messageSelect+` WHERE id=$1`, id))
return result, rowError(err)
}
func (s *Store) ListEventsPage(ctx context.Context, accountID string, page, pageSize int) (Page[InteractionEvent], error) {
items, err := s.ListEvents(ctx, accountID)
if err != nil {
return Page[InteractionEvent]{}, err
}
return slicePage(items, page, pageSize)
}
func (s *Store) ListMessagesPage(ctx context.Context, conversationID string, page, pageSize int) (Page[Message], error) {
items, err := s.ListMessages(ctx, conversationID)
if err != nil {
return Page[Message]{}, err
}
return slicePage(items, page, pageSize)
}
func (s *Store) ListMessages(ctx context.Context, conversationID string) ([]Message, error) {
rows, err := s.db.QueryContext(ctx, messageSelect+` WHERE conversation_id=$1 ORDER BY message_at NULLS LAST,created_at,id`, conversationID)
if err != nil {
+22 -3
View File
@@ -27,6 +27,12 @@ type PlatformCollector interface {
ListTopLevelComments(context.Context, string, string) (CommentPage, error)
}
// WorkContextCollector lets a platform retain per-work request context across
// the separate works and comments phases, including a resumed collection.
type WorkContextCollector interface {
SetWorkContext(string, string) error
}
type CollectionReport struct {
SourceType string `json:"source_type"`
SourceID string `json:"source_id"`
@@ -103,6 +109,14 @@ func NewCollectionWindow(now time.Time, lookbackDays int) (time.Time, time.Time,
return end.Add(-time.Duration(lookbackDays) * 24 * time.Hour), end, nil
}
func publishedAtInCollectionWindow(publishedAt *time.Time, start, end time.Time) bool {
if publishedAt == nil {
return true
}
published := publishedAt.UTC()
return !published.Before(start.UTC()) && !published.After(end.UTC())
}
func checkpointID(sourceType, sourceID, kind string) string {
return sourceType + ":" + sourceID + ":" + kind
}
@@ -330,7 +344,7 @@ func (s *Store) ListDueOwnedAccounts(ctx context.Context, now time.Time, interva
ON works_checkpoint.source_type='owned' AND works_checkpoint.source_id=account.id AND works_checkpoint.collection_kind='works'
LEFT JOIN creator_collection_checkpoint comments_checkpoint
ON comments_checkpoint.source_type='owned' AND comments_checkpoint.source_id=account.id AND comments_checkpoint.collection_kind='comments'
WHERE account.platform='douyin' AND account.authorization_status='authorized'
WHERE account.platform IN ('douyin', 'xiaohongshu') AND account.authorization_status='authorized'
AND profile.login_status='logged_in' AND profile.big_account=true
AND COALESCE(works_checkpoint.status, '') <> 'blocked'
AND COALESCE(comments_checkpoint.status, '') <> 'blocked'
@@ -393,7 +407,7 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI
return err
}
report.WorksSeen++
if work.PublishedAt != nil && work.PublishedAt.Before(report.WindowStart) {
if !publishedAtInCollectionWindow(work.PublishedAt, report.WindowStart, report.WindowEnd) {
continue
}
work.Platform, work.SourceType, work.SourceID = platform, sourceType, sourceID
@@ -460,6 +474,11 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI
}
for index := startIndex; index < len(works); index++ {
work := works[index]
if contextual, ok := collector.(WorkContextCollector); ok {
if err := contextual.SetWorkContext(work.WorkKey, work.OriginalURL); err != nil {
return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err)
}
}
cursor := initialCommentCursor
initialCommentCursor = ""
_, err := collectPagesFromCursor(ctx, cursor, func(ctx context.Context, cursor string) ([]CommentInput, string, bool, error) {
@@ -482,7 +501,7 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI
return err
}
report.CommentsSeen++
if comment.PublishedAt != nil && !InWindow(*comment.PublishedAt, report.WindowStart, report.WindowEnd) || comment.CommentType == "reply" {
if !publishedAtInCollectionWindow(comment.PublishedAt, report.WindowStart, report.WindowEnd) || comment.CommentType == "reply" {
continue
}
comment.Platform, comment.WorkID = platform, work.ID
+23
View File
@@ -44,3 +44,26 @@ func TestCollectionWindowIsFixedInUTC(t *testing.T) {
t.Fatal("window must include both exact boundaries only")
}
}
func TestPublishedAtCollectionWindowRejectsFutureItems(t *testing.T) {
start := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
end := start.Add(24 * time.Hour)
before, atStart, atEnd, after := start.Add(-time.Nanosecond), start, end, end.Add(time.Nanosecond)
for _, test := range []struct {
name string
at *time.Time
want bool
}{
{name: "unknown", at: nil, want: true},
{name: "before", at: &before, want: false},
{name: "start", at: &atStart, want: true},
{name: "end", at: &atEnd, want: true},
{name: "future", at: &after, want: false},
} {
t.Run(test.name, func(t *testing.T) {
if got := publishedAtInCollectionWindow(test.at, start, end); got != test.want {
t.Fatalf("publishedAtInCollectionWindow(%v) = %v, want %v", test.at, got, test.want)
}
})
}
}
+119
View File
@@ -3,6 +3,7 @@ package creator
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/url"
@@ -184,6 +185,26 @@ func (s *Store) ListDueCompetitors(ctx context.Context, now time.Time) ([]Compet
return result, rows.Err()
}
func (s *Store) validateWorkSource(ctx context.Context, platform, sourceType, sourceID string) error {
var query string
switch sourceType {
case SourceOwned:
query = `SELECT 1 FROM social_account WHERE id=$1 AND platform=$2`
case SourceCompetitor:
query = `SELECT 1 FROM creator_competitor WHERE id=$1 AND platform=$2`
default:
return ErrInvalid
}
var exists int
if err := s.db.QueryRowContext(ctx, query, sourceID, platform).Scan(&exists); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return ErrInvalid
}
return databaseError(err)
}
return nil
}
func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) (Work, bool, error) {
input.Platform = strings.TrimSpace(input.Platform)
input.WorkKey = strings.TrimSpace(input.WorkKey)
@@ -200,6 +221,9 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time)
utf8.RuneCountInString(input.CoverURL) > 2000 {
return Work{}, false, ErrInvalid
}
if err := s.validateWorkSource(ctx, input.Platform, input.SourceType, input.SourceID); err != nil {
return Work{}, false, err
}
if now.IsZero() {
now = time.Now().UTC()
}
@@ -532,6 +556,59 @@ func (s *Store) ConfirmRewrite(ctx context.Context, workID, requirement string)
return s.GetMaterial(ctx, workID)
}
func parseGeneratedRewrite(content string) (string, string, error) {
var result struct {
Title string `json:"title"`
Script string `json:"script"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(content)), &result); err != nil {
return "", "", fmt.Errorf("decode generated rewrite: %w", err)
}
result.Title, result.Script = strings.TrimSpace(result.Title), strings.TrimSpace(result.Script)
if result.Title == "" || result.Script == "" || utf8.RuneCountInString(result.Title) > 1000 || utf8.RuneCountInString(result.Script) > 100000 {
return "", "", ErrInvalid
}
return result.Title, result.Script, nil
}
func (s *Store) GenerateRewrite(ctx context.Context, workID string, generator TextGenerator) (MaterialJob, error) {
if generator == nil {
return MaterialJob{}, fmt.Errorf("%w: text generator is not configured", ErrUnavailable)
}
settings, err := s.GetSettings(ctx)
if err != nil {
return MaterialJob{}, err
}
if !settings.AIConfigured || strings.TrimSpace(settings.AIProvider) == "" || strings.TrimSpace(settings.AIModel) == "" {
return MaterialJob{}, fmt.Errorf("%w: AI settings are not approved", ErrUnavailable)
}
job, err := s.GetMaterial(ctx, workID)
if err != nil {
return MaterialJob{}, err
}
if job.RewriteConfirmedAt == nil {
return MaterialJob{}, ErrConflict
}
work, err := s.GetWork(ctx, workID)
if err != nil {
return MaterialJob{}, err
}
content, err := generator.Generate(ctx,
"根据原作品和转写内容生成仿写草稿。只返回 JSON,不要 Markdown 或额外文字,格式必须是 {\"title\":\"标题\",\"script\":\"脚本\"}。不得编造原文没有的事实。",
fmt.Sprintf("改写要求:%s\n原标题:%s\n原文案:%s\n转写:%s", job.RewriteRequirement, work.Title, work.Body, job.Transcript))
if err != nil {
return MaterialJob{}, err
}
title, script, err := parseGeneratedRewrite(content)
if err != nil {
return MaterialJob{}, err
}
if _, err := s.db.ExecContext(ctx, `UPDATE creator_material_job SET generated_title = $2, generated_script = $3, updated_at = now() WHERE work_id = $1`, workID, title, script); err != nil {
return MaterialJob{}, databaseError(err)
}
return s.GetMaterial(ctx, workID)
}
func (s *Store) SaveRewrite(ctx context.Context, workID, title, script string) (MaterialJob, error) {
title, script = strings.TrimSpace(title), strings.TrimSpace(script)
if title == "" || script == "" || utf8.RuneCountInString(title) > 1000 || utf8.RuneCountInString(script) > 100000 {
@@ -591,6 +668,48 @@ func (s *Store) GetComment(ctx context.Context, id string) (Comment, error) {
return result, rowError(err)
}
func pageBounds(page, pageSize int) (int, int, error) {
if page < 1 || pageSize < 1 || pageSize > 100 {
return 0, 0, ErrInvalid
}
if page > int(maxDurationSeconds) || pageSize > int(maxDurationSeconds)/page {
return 0, 0, ErrInvalid
}
return (page - 1) * pageSize, pageSize, nil
}
func slicePage[T any](items []T, page, pageSize int) (Page[T], error) {
offset, limit, err := pageBounds(page, pageSize)
if err != nil {
return Page[T]{}, err
}
if offset > len(items) {
offset = len(items)
}
end := offset + limit
if end > len(items) {
end = len(items)
}
data := append([]T(nil), items[offset:end]...)
return Page[T]{Data: data, Page: page, PageSize: pageSize, Total: len(items), HasNext: end < len(items)}, nil
}
func (s *Store) ListWorksPage(ctx context.Context, filter WorkFilter, page, pageSize int) (Page[Work], error) {
items, err := s.ListWorks(ctx, filter)
if err != nil {
return Page[Work]{}, err
}
return slicePage(items, page, pageSize)
}
func (s *Store) ListCommentsPage(ctx context.Context, platform, workID string, page, pageSize int) (Page[Comment], error) {
items, err := s.ListComments(ctx, platform, workID)
if err != nil {
return Page[Comment]{}, err
}
return slicePage(items, page, pageSize)
}
func (s *Store) ListComments(ctx context.Context, platform, workID string) ([]Comment, error) {
query, args := commentSelect, []any{}
where := []string{}
+20
View File
@@ -0,0 +1,20 @@
package creator
import "testing"
func TestParseGeneratedRewriteRequiresBoundedJSON(t *testing.T) {
title, script, err := parseGeneratedRewrite(`{"title":"新标题","script":"新脚本"}`)
if err != nil || title != "新标题" || script != "新脚本" {
t.Fatalf("parseGeneratedRewrite() = %q, %q, %v", title, script, err)
}
for _, content := range []string{
"",
"不是 JSON",
`{"title":"有标题"}`,
`{"title":"标题","script":""}`,
} {
if _, _, err := parseGeneratedRewrite(content); err == nil {
t.Fatalf("parseGeneratedRewrite(%q) accepted invalid output", content)
}
}
}
+85
View File
@@ -0,0 +1,85 @@
package creator
import (
"context"
"database/sql"
"strings"
)
const listenerStateSelect = `SELECT account_id,platform,generation,status,boundary_at,last_delivery_id,reason,updated_at FROM creator_listener_state`
func validListenerState(input ListenerState) bool {
if input.AccountID == "" || !ValidatePlatform(input.Platform) || strings.TrimSpace(input.Generation) == "" {
return false
}
switch input.Status {
case "starting", "gap", "stopped", "error":
return len(input.LastDeliveryID) <= 500 && len(input.Reason) <= 2000
case "ready":
return input.BoundaryAt != nil && !input.BoundaryAt.IsZero() && len(input.LastDeliveryID) <= 500 && len(input.Reason) <= 2000
default:
return false
}
}
func scanListenerState(scanner interface{ Scan(...any) error }) (ListenerState, error) {
var result ListenerState
var boundaryAt sql.NullTime
if err := scanner.Scan(&result.AccountID, &result.Platform, &result.Generation, &result.Status, &boundaryAt, &result.LastDeliveryID, &result.Reason, &result.UpdatedAt); err != nil {
return ListenerState{}, err
}
result.BoundaryAt = nullableTime(boundaryAt)
result.UpdatedAt = result.UpdatedAt.UTC()
return result, nil
}
func (s *Store) UpsertListenerState(ctx context.Context, input ListenerState) (ListenerState, error) {
input.Generation = strings.TrimSpace(input.Generation)
input.LastDeliveryID = strings.TrimSpace(input.LastDeliveryID)
input.Reason = strings.TrimSpace(input.Reason)
if !validListenerState(input) {
return ListenerState{}, ErrInvalid
}
if _, err := s.db.ExecContext(ctx, `
INSERT INTO creator_listener_state (account_id,platform,generation,status,boundary_at,last_delivery_id,reason,updated_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,now())
ON CONFLICT (account_id) DO UPDATE SET platform=EXCLUDED.platform,generation=EXCLUDED.generation,status=EXCLUDED.status,
boundary_at=COALESCE(EXCLUDED.boundary_at, creator_listener_state.boundary_at),
last_delivery_id=CASE WHEN EXCLUDED.last_delivery_id = '' THEN creator_listener_state.last_delivery_id ELSE EXCLUDED.last_delivery_id END,
reason=EXCLUDED.reason,updated_at=now()`,
input.AccountID, input.Platform, input.Generation, input.Status, input.BoundaryAt, input.LastDeliveryID, input.Reason); err != nil {
return ListenerState{}, databaseError(err)
}
return s.GetListenerState(ctx, input.AccountID)
}
func (s *Store) GetListenerState(ctx context.Context, accountID string) (ListenerState, error) {
if strings.TrimSpace(accountID) == "" {
return ListenerState{}, ErrInvalid
}
result, err := scanListenerState(s.db.QueryRowContext(ctx, listenerStateSelect+` WHERE account_id=$1`, accountID))
return result, rowError(err)
}
func (s *Store) ListListenerStates(ctx context.Context, accountID string) ([]ListenerState, error) {
query, args := listenerStateSelect, []any{}
if strings.TrimSpace(accountID) != "" {
query += ` WHERE account_id=$1`
args = append(args, accountID)
}
query += ` ORDER BY account_id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]ListenerState, 0)
for rows.Next() {
item, err := scanListenerState(rows)
if err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}
+29
View File
@@ -0,0 +1,29 @@
package creator
import "testing"
func TestValidListenerStateRequiresVerifiedGeneration(t *testing.T) {
base := ListenerState{AccountID: "account", Platform: PlatformDouyin, Generation: "runtime:network:1", Status: "starting"}
if !validListenerState(base) {
t.Fatal("expected a complete listener state to be valid")
}
for _, mutate := range []func(*ListenerState){
func(state *ListenerState) { state.Generation = "" },
func(state *ListenerState) { state.Platform = "unknown" },
func(state *ListenerState) { state.Status = "ready"; state.BoundaryAt = nil },
func(state *ListenerState) { state.Status = "lost" },
} {
state := base
mutate(&state)
if state.Status == "ready" && state.BoundaryAt == nil {
// A ready state without a boundary is never a valid continuity claim.
if validListenerState(state) {
t.Fatalf("accepted ready state without boundary: %+v", state)
}
continue
}
if validListenerState(state) {
t.Fatalf("accepted invalid listener state: %+v", state)
}
}
}
+9
View File
@@ -166,6 +166,15 @@ func ValidEventType(eventType string) bool {
}
}
func ValidMessageType(messageType string) bool {
switch messageType {
case MessageTypeText, MessageTypeImage, MessageTypeVoice, MessageTypeVideo, MessageTypeSticker, MessageTypeUnknown:
return true
default:
return false
}
}
func ValidAction(action string) bool {
switch action {
case ActionDM, ActionReplyComment, ActionLikeComment, ActionLikeWork, ActionFollow, ActionRepost:
+11
View File
@@ -78,3 +78,14 @@ func TestActionTargetAndBusinessStatusGates(t *testing.T) {
t.Fatalf("normal logged-in account should be writable: %v", err)
}
}
func TestValidMessageTypeKeepsNonTextMessagesExplicit(t *testing.T) {
for _, messageType := range []string{MessageTypeText, MessageTypeImage, MessageTypeVoice, MessageTypeVideo, MessageTypeSticker, MessageTypeUnknown} {
if !ValidMessageType(messageType) {
t.Fatalf("message type %q should be accepted", messageType)
}
}
if ValidMessageType("text-like") {
t.Fatal("unknown message type must not be accepted")
}
}
@@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS creator_listener_state (
account_id text PRIMARY KEY REFERENCES social_account (
id
) ON DELETE CASCADE,
platform text NOT NULL CHECK (platform IN ('douyin', 'xiaohongshu')),
generation text NOT NULL DEFAULT '',
status text NOT NULL CHECK (
status IN ('starting', 'ready', 'gap', 'stopped', 'error')
),
boundary_at timestamptz,
last_delivery_id text NOT NULL DEFAULT '',
reason text NOT NULL DEFAULT '',
updated_at timestamptz NOT NULL DEFAULT now()
);
@@ -0,0 +1,9 @@
ALTER TABLE creator_event
ADD COLUMN IF NOT EXISTS message_type text NOT NULL DEFAULT 'text';
ALTER TABLE creator_event
DROP CONSTRAINT IF EXISTS creator_event_message_type_check;
ALTER TABLE creator_event
ADD CONSTRAINT creator_event_message_type_check
CHECK (message_type IN ('text', 'image', 'voice', 'video', 'sticker', 'non_text'));
@@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS creator_source_sync_lease (
source_type text NOT NULL CHECK (source_type IN ('owned', 'competitor')),
source_id text NOT NULL,
lease_token text NOT NULL,
lease_until timestamptz NOT NULL,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (source_type, source_id)
);
@@ -0,0 +1,2 @@
ALTER TABLE creator_event
ADD COLUMN IF NOT EXISTS gateway_received_at timestamptz;
+34
View File
@@ -18,6 +18,13 @@ const (
ActionLikeWork = "like_work"
ActionFollow = "follow"
ActionRepost = "repost"
MessageTypeText = "text"
MessageTypeImage = "image"
MessageTypeVoice = "voice"
MessageTypeVideo = "video"
MessageTypeSticker = "sticker"
MessageTypeUnknown = "non_text"
)
type Settings struct {
@@ -271,6 +278,12 @@ type RuleResult struct {
AnalysedAt *time.Time `json:"analysed_at,omitempty"`
}
type RuleAnalysisItem struct {
CommentID string `json:"comment_id"`
Result *RuleResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
type Lead struct {
Comment Comment `json:"comment"`
RuleIDs []string `json:"rule_ids"`
@@ -306,6 +319,25 @@ type Relation struct {
SmallAccountID string `json:"small_account_id"`
}
type ListenerState struct {
AccountID string `json:"account_id"`
Platform string `json:"platform"`
Generation string `json:"generation"`
Status string `json:"status"`
BoundaryAt *time.Time `json:"boundary_at,omitempty"`
LastDeliveryID string `json:"last_delivery_id,omitempty"`
Reason string `json:"reason,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
type Page[T any] struct {
Data []T `json:"data"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int `json:"total"`
HasNext bool `json:"has_next"`
}
type InteractionEvent struct {
ID string `json:"id"`
Platform string `json:"platform"`
@@ -317,8 +349,10 @@ type InteractionEvent struct {
InteractorUID string `json:"interactor_uid"`
CommentID string `json:"comment_id"`
WorkID string `json:"work_id"`
MessageType string `json:"message_type"`
MessageText string `json:"message_text,omitempty"`
PlatformEventAt *time.Time `json:"platform_event_at,omitempty"`
GatewayReceivedAt *time.Time `json:"gateway_received_at,omitempty"`
ReceivedAt time.Time `json:"received_at"`
ProcessingStartedAt *time.Time `json:"processing_started_at,omitempty"`
ProcessingFinishedAt *time.Time `json:"processing_finished_at,omitempty"`
+29
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
@@ -123,6 +124,34 @@ func (s *Store) upsertRuleResult(ctx context.Context, commentID string, rule Lea
return RuleResult{CommentID: commentID, RuleID: rule.ID, Status: status, Reason: reason, MatchedKeywords: matched, RuleSnapshot: rule, AnalysedAt: analysedAt}, nil
}
func (s *Store) AnalyzeComments(ctx context.Context, commentIDs []string, ruleID string, analyzer ThemeAnalyzer) ([]RuleAnalysisItem, error) {
if len(commentIDs) == 0 || len(commentIDs) > 100 || strings.TrimSpace(ruleID) == "" {
return nil, ErrInvalid
}
seen := make(map[string]struct{}, len(commentIDs))
items := make([]RuleAnalysisItem, 0, len(commentIDs))
var batchErr error
for _, commentID := range commentIDs {
commentID = strings.TrimSpace(commentID)
if commentID == "" {
return nil, ErrInvalid
}
if _, ok := seen[commentID]; ok {
continue
}
seen[commentID] = struct{}{}
result, err := s.AnalyzeComment(ctx, commentID, ruleID, analyzer)
item := RuleAnalysisItem{CommentID: commentID, Result: &result}
if err != nil {
item.Result = nil
item.Error = err.Error()
batchErr = errors.Join(batchErr, err)
}
items = append(items, item)
}
return items, batchErr
}
func (s *Store) AnalyzeComment(ctx context.Context, commentID, ruleID string, analyzer ThemeAnalyzer) (RuleResult, error) {
comment, err := s.GetComment(ctx, commentID)
if err != nil {
+44
View File
@@ -0,0 +1,44 @@
package creator
import (
"context"
"database/sql"
)
func (s *Store) ClaimSourceSync(ctx context.Context, sourceType, sourceID string) (string, error) {
if sourceType != SourceOwned && sourceType != SourceCompetitor || sourceID == "" {
return "", ErrInvalid
}
token := newID("source-lease")
var claimed string
err := s.db.QueryRowContext(ctx, `
INSERT INTO creator_source_sync_lease (source_type,source_id,lease_token,lease_until,updated_at)
VALUES ($1,$2,$3,now() + interval '10 minutes',now())
ON CONFLICT (source_type,source_id) DO UPDATE SET lease_token=EXCLUDED.lease_token,
lease_until=EXCLUDED.lease_until,updated_at=now()
WHERE creator_source_sync_lease.lease_until <= now()
RETURNING lease_token`, sourceType, sourceID, token).Scan(&claimed)
if err == sql.ErrNoRows {
return "", ErrConflict
}
if err != nil {
return "", databaseError(err)
}
return claimed, nil
}
func (s *Store) ReleaseSourceSync(ctx context.Context, sourceType, sourceID, token string) error {
if sourceType != SourceOwned && sourceType != SourceCompetitor || sourceID == "" || token == "" {
return ErrInvalid
}
result, err := s.db.ExecContext(ctx, `DELETE FROM creator_source_sync_lease WHERE source_type=$1 AND source_id=$2 AND lease_token=$3`, sourceType, sourceID, token)
if err != nil {
return databaseError(err)
}
if rows, err := result.RowsAffected(); err != nil {
return err
} else if rows != 1 {
return ErrConflict
}
return nil
}
+16
View File
@@ -46,6 +46,18 @@ var migration025 string
//go:embed migrations/026_event_message_text.sql
var migration026 string
//go:embed migrations/027_creator_listener_state.sql
var migration027 string
//go:embed migrations/028_creator_event_message_type.sql
var migration028 string
//go:embed migrations/029_creator_source_sync_lease.sql
var migration029 string
//go:embed migrations/030_creator_event_gateway_time.sql
var migration030 string
type SecretReference struct {
ID string
Provider string
@@ -119,6 +131,10 @@ func (s *Store) migrate(ctx context.Context) error {
{version: 24, sql: migration024},
{version: 25, sql: migration025},
{version: 26, sql: migration026},
{version: 27, sql: migration027},
{version: 28, sql: migration028},
{version: 29, sql: migration029},
{version: 30, sql: migration030},
}
for _, migration := range migrations {
var applied bool
+5 -27
View File
@@ -32,10 +32,7 @@ const (
worksEndpoint = "https://www.douyin.com/aweme/v1/web/aweme/post/"
)
var (
keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`)
credentialKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._/-]{0,126}$`)
)
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`)
var ErrInvalid = errors.New("invalid douyin connector input")
@@ -65,9 +62,9 @@ type Response struct {
}
// Browser is the deliberately narrow contract the restricted browser control
// plane must implement. It does not permit arbitrary CDP commands.
// plane must implement. Login happens in the managed browser session; the
// connector never injects stored credentials or cookies.
type Browser interface {
SetCookies(context.Context, []Cookie) error
Get(context.Context, string) (Response, error)
}
@@ -127,9 +124,7 @@ type Request struct {
func (connector Connector) Sync(ctx context.Context, request Request) (Result, error) {
if connector.Browser == nil || connector.Store == nil || !keyPattern.MatchString(request.AccountID) ||
!keyPattern.MatchString(request.PlatformAccountKey) ||
(request.Credential.Provider != "os_keyring" && request.Credential.Provider != "secret_manager") ||
!credentialKeyPattern.MatchString(request.Credential.Key) {
!keyPattern.MatchString(request.PlatformAccountKey) {
return Result{}, ErrInvalid
}
if err := ctx.Err(); err != nil {
@@ -140,24 +135,7 @@ func (connector Connector) Sync(ctx context.Context, request Request) (Result, e
if ctx.Err() != nil {
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
}
if connector.Secrets == nil {
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
}
credential, credentialErr := connector.Secrets.Resolve(ctx, request.Credential)
if credentialErr != nil {
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
}
cookies, credentialErr := ParseCredential(credential)
if credentialErr != nil {
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
}
if credentialErr = connector.Browser.SetCookies(ctx, cookies); credentialErr != nil {
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "login"})
}
identityResponse, err = connector.Browser.Get(ctx, identityEndpoint)
if err != nil {
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "identity"})
}
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "login"})
}
if state, reason := classify(identityResponse); state != "" {
return connector.stop(ctx, request.AccountID, state, reason, Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
+12 -8
View File
@@ -83,7 +83,7 @@ func TestSyncLogsInVerifiesIdentityAndReadsOwnWorks(t *testing.T) {
store := &fakeStore{}
resolveCalls := 0
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential), resolveCalls: &resolveCalls}, Store: store}).Sync(context.Background(), Request{
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
AccountID: "account-a", PlatformAccountKey: "sec-a",
})
if err != nil || result.State != StateSucceeded || !result.Evidence.IdentityVerified || result.Evidence.WorksSeen != 1 || !result.Evidence.HasMore {
t.Fatalf("unexpected result: %#v err=%v", result, err)
@@ -193,7 +193,7 @@ func TestSyncRejectsInvalidCredentialWithoutLeakingIt(t *testing.T) {
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(`{"cookies":[{"name":"sessionid","value":"secret","domain":"evil.example"}]}`)}, Store: store}).Sync(context.Background(), Request{
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
})
if err != nil || result.State != StatePolicyHold || result.ReasonCode != ReasonAuthInvalid || len(browser.urls) != 1 || len(browser.cookies) != 0 || len(store.holds) != 1 {
if err != nil || result.State != StateNeedsConfirmation || result.ReasonCode != ReasonUnknown || len(browser.urls) != 1 || len(browser.cookies) != 0 || len(store.holds) != 1 {
t.Fatalf("unexpected invalid credential result: %#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err)
}
encoded, _ := json.Marshal(result)
@@ -208,8 +208,8 @@ func TestSyncStopsWhenSecretReferenceCannotResolve(t *testing.T) {
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{err: errors.New("secret unavailable")}, Store: store}).Sync(context.Background(), Request{
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
})
if err != nil || result.State != StatePolicyHold || result.ReasonCode != ReasonAuthInvalid || len(browser.urls) != 1 || len(store.holds) != 1 {
t.Fatalf("unavailable secret did not fail closed: result=%#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err)
if err != nil || result.State != StateNeedsConfirmation || result.ReasonCode != ReasonUnknown || len(browser.urls) != 1 || len(store.holds) != 1 {
t.Fatalf("browser session did not fail closed without credential injection: result=%#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err)
}
}
@@ -269,11 +269,15 @@ func TestSyncHoldsWithCancelledRequestContext(t *testing.T) {
}
}
func TestSyncRejectsNonSecretCredentialReference(t *testing.T) {
_, err := (Connector{Browser: &fakeBrowser{}, Secrets: fakeSecrets{}, Store: &fakeStore{}}).Sync(context.Background(), Request{
func TestSyncIgnoresLegacyCredentialReference(t *testing.T) {
browser := &fakeBrowser{responses: []Response{
{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")},
{Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[]}`)},
}}
result, err := (Connector{Browser: browser, Store: &fakeStore{}}).Sync(context.Background(), Request{
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: SecretReference{Provider: "plain_text", Key: "raw-secret"},
})
if !errors.Is(err, ErrInvalid) {
t.Fatalf("accepted non-secret credential reference: %v", err)
if errors.Is(err, ErrInvalid) || result.State != StateSucceeded {
t.Fatalf("legacy credential reference affected browser-session sync: result=%#v err=%v", result, err)
}
}