rogee/creator-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate creator hub douyin workflows

Browse Source
This commit is contained in:
rogee
2026-09-14 19:22:07 +08:00
parent 025fe62c37
commit 44a28954cf
42 changed files with 2850 additions and 880 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/creator/collection.go
+22 -3
View File
@@ -27,6 +27,12 @@ type PlatformCollector interface {
ListTopLevelComments(context.Context, string, string) (CommentPage, error)
}
// WorkContextCollector lets a platform retain per-work request context across
// the separate works and comments phases, including a resumed collection.
type WorkContextCollector interface {
SetWorkContext(string, string) error
}
type CollectionReport struct {
SourceType string `json:"source_type"`
SourceID string `json:"source_id"`
@@ -103,6 +109,14 @@ func NewCollectionWindow(now time.Time, lookbackDays int) (time.Time, time.Time,
return end.Add(-time.Duration(lookbackDays) * 24 * time.Hour), end, nil
}
func publishedAtInCollectionWindow(publishedAt *time.Time, start, end time.Time) bool {
if publishedAt == nil {
return true
}
published := publishedAt.UTC()
return !published.Before(start.UTC()) && !published.After(end.UTC())
}
func checkpointID(sourceType, sourceID, kind string) string {
return sourceType + ":" + sourceID + ":" + kind
}
@@ -330,7 +344,7 @@ func (s *Store) ListDueOwnedAccounts(ctx context.Context, now time.Time, interva
ON works_checkpoint.source_type='owned' AND works_checkpoint.source_id=account.id AND works_checkpoint.collection_kind='works'
LEFT JOIN creator_collection_checkpoint comments_checkpoint
ON comments_checkpoint.source_type='owned' AND comments_checkpoint.source_id=account.id AND comments_checkpoint.collection_kind='comments'
WHERE account.platform='douyin' AND account.authorization_status='authorized'
WHERE account.platform IN ('douyin', 'xiaohongshu') AND account.authorization_status='authorized'
AND profile.login_status='logged_in' AND profile.big_account=true
AND COALESCE(works_checkpoint.status, '') <> 'blocked'
AND COALESCE(comments_checkpoint.status, '') <> 'blocked'
@@ -393,7 +407,7 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI
return err
}
report.WorksSeen++
if work.PublishedAt != nil && work.PublishedAt.Before(report.WindowStart) {
if !publishedAtInCollectionWindow(work.PublishedAt, report.WindowStart, report.WindowEnd) {
continue
}
work.Platform, work.SourceType, work.SourceID = platform, sourceType, sourceID
@@ -460,6 +474,11 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI
}
for index := startIndex; index < len(works); index++ {
work := works[index]
if contextual, ok := collector.(WorkContextCollector); ok {
if err := contextual.SetWorkContext(work.WorkKey, work.OriginalURL); err != nil {
return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err)
}
}
cursor := initialCommentCursor
initialCommentCursor = ""
_, err := collectPagesFromCursor(ctx, cursor, func(ctx context.Context, cursor string) ([]CommentInput, string, bool, error) {
@@ -482,7 +501,7 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI
return err
}
report.CommentsSeen++
if comment.PublishedAt != nil && !InWindow(*comment.PublishedAt, report.WindowStart, report.WindowEnd) || comment.CommentType == "reply" {
if !publishedAtInCollectionWindow(comment.PublishedAt, report.WindowStart, report.WindowEnd) || comment.CommentType == "reply" {
continue
}
comment.Platform, comment.WorkID = platform, work.ID