Files
creator-hub/internal/creator/content.go
T
rogee 0879d9fd25 refactor(schema): 迁移 043 schema 收敛——42→19 张表,统一迁移 runner
- 迁移 043 一次到位:runtime_instance/environment_binding 并入 browser_env(账号即环境)、
  credential_reference 并入 social_account、work_source/metric_plan 并入 creator_work、
  sync 租约并入 creator_collection_checkpoint(kind='sync')、audit_event 瘦列、
  creator_settings 去转写列、TRUNCATE 账号域(竞品采集数据无损保留)
- 统一迁移 runner:environment.Open 串起 001 账号基座 + hub 链 + 038 creator 版本回填
  (v+1000) + creator 1017-1042 + 043;account/creator Open 不再迁移
- 三域 store SQL 全量适配:CreateBoundEnv 就绪门禁、SetRuntimeCleanupPending fence
  语义(未知代 MissingRuntimeID 允许登记并释放活跃实例)、metric_plan 列并入 work、
  checkpoint sync 租约、deletion 链路重写
- api 适配:envView 去 runtime_instance_id、audit filter 去任务列、browser_env 单表查询
- 测试:迁移 043 形态探针、迁移不可变映射新路径、legacy 迁移重放子测试重写、
  fence/出口生命周期/单来源 upsert 语义覆盖;CREATORHUB_POSTGRES_TEST_URL 全绿,
  environment 70%/creator 67.5% 覆盖率,tsc 通过
2026-09-28 19:48:55 +08:00

852 lines
33 KiB
Go

package creator
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
"time"
"unicode/utf8"
"github.com/jackc/pgx/v5/pgtype"
)
const maxRawPayloadBytes = 4 << 20
func validateHomepage(value string) error {
parsed, err := url.Parse(strings.TrimSpace(value))
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil {
return ErrInvalid
}
return nil
}
func validateCreatorTags(tags []string) error {
if len(tags) > 20 {
return ErrInvalid
}
for _, tag := range tags {
if tag == "" || strings.TrimSpace(tag) != tag || !utf8.ValidString(tag) || utf8.RuneCountInString(tag) > 32 {
return ErrInvalid
}
}
return nil
}
func normalizeCompetitorInput(input CompetitorInput) (CompetitorInput, error) {
input.Platform = strings.TrimSpace(input.Platform)
input.PlatformAccountKey = strings.TrimSpace(input.PlatformAccountKey)
input.UniqueID = strings.TrimSpace(input.UniqueID)
input.Nickname = strings.TrimSpace(input.Nickname)
input.AvatarURL = strings.TrimSpace(input.AvatarURL)
input.HomepageURL = strings.TrimSpace(input.HomepageURL)
if input.Tags == nil {
input.Tags = []string{}
}
if !ValidatePlatform(input.Platform) || input.PlatformAccountKey == "" || input.HomepageURL == "" ||
utf8.RuneCountInString(input.PlatformAccountKey) > 255 || utf8.RuneCountInString(input.UniqueID) > 255 || utf8.RuneCountInString(input.Nickname) > 255 ||
utf8.RuneCountInString(input.AvatarURL) > 1000 || validateHomepage(input.HomepageURL) != nil ||
validateCreatorTags(input.Tags) != nil {
return CompetitorInput{}, ErrInvalid
}
return input, nil
}
func (s *Store) CreateCompetitor(ctx context.Context, input CompetitorInput) (Competitor, error) {
input, err := normalizeCompetitorInput(input)
if err != nil {
return Competitor{}, err
}
id := newID("competitor")
if _, err := s.db.ExecContext(ctx, `
INSERT INTO creator_competitor (id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags, next_sync_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())`,
id, input.Platform, input.PlatformAccountKey, input.UniqueID, input.Nickname, input.AvatarURL, input.HomepageURL, input.Tags); err != nil {
return Competitor{}, databaseError(err)
}
return s.GetCompetitor(ctx, id)
}
// competitorColumns 是 creator_competitor 的完整列清单(含画像属性),各查询共用。
const competitorColumns = `id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags,
enabled, follower_count, following_count, aweme_count, sync_status, sync_cursor, sync_error, sync_lease_until,
last_sync_at, next_sync_at, created_at, updated_at`
type competitorScan struct {
competitor Competitor
tags pgtype.FlatArray[string]
leaseUntil, lastSync, nextSync sql.NullTime
follower, following, aweme sql.NullInt64
}
func competitorScanDestinations(scan *competitorScan) []any {
return []any{&scan.competitor.ID, &scan.competitor.Platform, &scan.competitor.PlatformAccountKey, &scan.competitor.UniqueID, &scan.competitor.Nickname,
&scan.competitor.AvatarURL, &scan.competitor.HomepageURL, pgtype.NewMap().SQLScanner(&scan.tags), &scan.competitor.Enabled,
&scan.follower, &scan.following, &scan.aweme, &scan.competitor.SyncStatus, &scan.competitor.SyncCursor,
&scan.competitor.SyncError, &scan.leaseUntil, &scan.lastSync, &scan.nextSync, &scan.competitor.CreatedAt, &scan.competitor.UpdatedAt}
}
func (scan *competitorScan) materialize() Competitor {
scan.competitor.Tags = []string(scan.tags)
scan.competitor.FollowerCount = nullableInt64(scan.follower)
scan.competitor.FollowingCount = nullableInt64(scan.following)
scan.competitor.AwemeCount = nullableInt64(scan.aweme)
scan.competitor.SyncLeaseUntil = nullableTime(scan.leaseUntil)
scan.competitor.LastSyncAt = nullableTime(scan.lastSync)
scan.competitor.NextSyncAt = nullableTime(scan.nextSync)
return scan.competitor
}
func scanCompetitor(scanner interface{ Scan(...any) error }) (Competitor, error) {
var scan competitorScan
if err := scanner.Scan(competitorScanDestinations(&scan)...); err != nil {
return Competitor{}, err
}
return scan.materialize(), nil
}
func (s *Store) GetCompetitor(ctx context.Context, id string) (Competitor, error) {
result, err := scanCompetitor(s.db.QueryRowContext(ctx, `SELECT `+competitorColumns+` FROM creator_competitor WHERE id = $1`, id))
return result, rowError(err)
}
// ListCompetitorsWithProfile 监控账号列表视图:账号基础信息 + 作品数/最近发布聚合统计。
func (s *Store) ListCompetitorsWithProfile(ctx context.Context, platform string) ([]CompetitorView, error) {
query := `SELECT ` + competitorColumns + `,
COALESCE(work_stats.work_count, 0),
work_stats.latest_published_at
FROM creator_competitor
LEFT JOIN (
SELECT w.source_id, COUNT(*) AS work_count, MAX(w.published_at) AS latest_published_at
FROM creator_work w
WHERE w.source_type = 'competitor'
GROUP BY w.source_id
) work_stats ON work_stats.source_id = creator_competitor.id`
args := []any{}
if platform != "" {
if !ValidatePlatform(platform) {
return nil, ErrInvalid
}
query += ` WHERE creator_competitor.platform = $1`
args = append(args, platform)
}
query += ` ORDER BY creator_competitor.created_at DESC, creator_competitor.id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]CompetitorView, 0)
for rows.Next() {
item, err := scanCompetitorView(rows)
if err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}
func scanCompetitorView(scanner interface{ Scan(...any) error }) (CompetitorView, error) {
var scan competitorScan
var latestPublished sql.NullTime
var workCount sql.NullInt64
destinations := append(competitorScanDestinations(&scan), &workCount, &latestPublished)
if err := scanner.Scan(destinations...); err != nil {
return CompetitorView{}, err
}
result := CompetitorView{
Competitor: scan.materialize(),
WorkCount: workCount.Int64,
LatestPublishedAt: nullableTime(latestPublished),
}
return result, nil
}
// UpdateCompetitorProfile 更新监控账号画像属性(头像与粉丝/关注/作品计数)。
// 当前仅提供数据通道(字段定义与接口返回),采集侧回填由后续接入。
func (s *Store) UpdateCompetitorProfile(ctx context.Context, id, avatarURL string, followerCount, followingCount, awemeCount int64) (Competitor, error) {
id = strings.TrimSpace(id)
avatarURL = strings.TrimSpace(avatarURL)
if id == "" || utf8.RuneCountInString(avatarURL) > 2000 || followerCount < 0 || followingCount < 0 || awemeCount < 0 {
return Competitor{}, ErrInvalid
}
result, err := s.db.ExecContext(ctx, `
UPDATE creator_competitor
SET avatar_url = $2, follower_count = $3, following_count = $4, aweme_count = $5, updated_at = now()
WHERE id = $1`, id, avatarURL, followerCount, followingCount, awemeCount)
if err != nil {
return Competitor{}, databaseError(err)
}
affected, err := result.RowsAffected()
if err != nil {
return Competitor{}, databaseError(err)
}
if affected != 1 {
return Competitor{}, ErrNotFound
}
return s.GetCompetitor(ctx, id)
}
func (s *Store) UpdateCompetitorTags(ctx context.Context, id string, tags []string) (Competitor, error) {
id = strings.TrimSpace(id)
if id == "" || validateCreatorTags(tags) != nil {
return Competitor{}, ErrInvalid
}
if tags == nil {
tags = []string{}
}
result, err := s.db.ExecContext(ctx, `
UPDATE creator_competitor SET tags = $2, updated_at = now() WHERE id = $1`, id, tags)
if err != nil {
return Competitor{}, databaseError(err)
}
affected, err := result.RowsAffected()
if err != nil {
return Competitor{}, databaseError(err)
}
if affected != 1 {
return Competitor{}, ErrNotFound
}
return s.GetCompetitor(ctx, id)
}
func (s *Store) SetCompetitorEnabled(ctx context.Context, id string, enabled bool) (Competitor, error) {
status := "paused"
if enabled {
status = "idle"
}
_, err := s.db.ExecContext(ctx, `
UPDATE creator_competitor SET enabled = $2, sync_status = $3, sync_lease_until = NULL,
next_sync_at = CASE WHEN $2 THEN now() ELSE NULL END, updated_at = now()
WHERE id = $1`, id, enabled, status)
if err != nil {
return Competitor{}, databaseError(err)
}
return s.GetCompetitor(ctx, id)
}
func (s *Store) MarkCompetitorSync(ctx context.Context, id, leaseToken, status, cursor, syncError string, nextAt *time.Time) error {
if id == "" || leaseToken == "" {
return ErrInvalid
}
if status != "idle" && status != "running" && status != "paused" && status != "failed" && status != "blocked" {
return ErrInvalid
}
if len(syncError) > 2000 || len(cursor) > 1000 {
return ErrInvalid
}
var next any
if nextAt != nil {
next = nextAt.UTC()
}
result, err := s.db.ExecContext(ctx, `
UPDATE creator_competitor
SET sync_status = $3, sync_cursor = $4, sync_error = $5,
sync_lease_token = CASE WHEN $3 = 'running' THEN $2 ELSE NULL END,
sync_lease_until = CASE WHEN $3 = 'running' THEN now() + interval '10 minutes' ELSE NULL END,
last_sync_at = CASE WHEN $3 IN ('idle', 'failed', 'blocked') THEN now() ELSE last_sync_at END,
next_sync_at = $6, updated_at = now()
WHERE id = $1 AND sync_lease_token = $2`, id, leaseToken, status, cursor, syncError, next)
if err != nil {
return databaseError(err)
}
if affected, err := result.RowsAffected(); err != nil {
return databaseError(err)
} else if affected != 1 {
return ErrConflict
}
return nil
}
func (s *Store) ClaimCompetitorSync(ctx context.Context, id string, force bool, now time.Time) (string, bool, error) {
if id == "" || now.IsZero() {
return "", false, ErrInvalid
}
condition := `enabled AND (next_sync_at IS NULL OR next_sync_at <= $2)`
if force {
condition = `enabled`
}
token := newID("competitor-lease")
var claimed string
err := s.db.QueryRowContext(ctx, `UPDATE creator_competitor SET sync_status='running', sync_lease_token=$2, sync_lease_until=$3 + interval '10 minutes', sync_error='', updated_at=$3 WHERE id=$1 AND `+strings.ReplaceAll(condition, "$2", "$3")+` AND (sync_status <> 'running' OR sync_lease_until IS NULL OR sync_lease_until <= $3) RETURNING id`, id, token, now.UTC()).Scan(&claimed)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, databaseError(err)
}
return token, claimed != "", nil
}
func (s *Store) ListDueCompetitors(ctx context.Context, now time.Time) ([]Competitor, error) {
if now.IsZero() {
return nil, ErrInvalid
}
rows, err := s.db.QueryContext(ctx, `SELECT `+competitorColumns+`
FROM creator_competitor
WHERE enabled AND next_sync_at IS NOT NULL AND next_sync_at <= $1
AND (sync_status <> 'running' OR sync_lease_until IS NULL OR sync_lease_until <= $1)
ORDER BY COALESCE(next_sync_at, created_at), id`, now.UTC())
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]Competitor, 0)
for rows.Next() {
item, err := scanCompetitor(rows)
if err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}
func (s *Store) validateWorkSource(ctx context.Context, platform, sourceType, sourceID string) error {
var query string
switch sourceType {
case SourceOwned:
query = `SELECT 1 FROM social_account WHERE id=$1 AND platform=$2`
case SourceCompetitor:
query = `SELECT 1 FROM creator_competitor WHERE id=$1 AND platform=$2`
default:
return ErrInvalid
}
var exists int
if err := s.db.QueryRowContext(ctx, query, sourceID, platform).Scan(&exists); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return ErrInvalid
}
return databaseError(err)
}
return nil
}
func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) (Work, bool, error) {
input.Platform = strings.TrimSpace(input.Platform)
input.WorkKey = strings.TrimSpace(input.WorkKey)
input.SourceType = strings.TrimSpace(input.SourceType)
input.SourceID = strings.TrimSpace(input.SourceID)
input.AuthorName = strings.TrimSpace(input.AuthorName)
input.Title = strings.TrimSpace(input.Title)
input.OriginalURL = strings.TrimSpace(input.OriginalURL)
input.CoverURL = strings.TrimSpace(input.CoverURL)
if !ValidatePlatform(input.Platform) || input.WorkKey == "" || input.SourceID == "" ||
(input.SourceType != SourceOwned && input.SourceType != SourceCompetitor) ||
utf8.RuneCountInString(input.WorkKey) > 255 || utf8.RuneCountInString(input.Title) > 1000 ||
utf8.RuneCountInString(input.Body) > 100000 || utf8.RuneCountInString(input.OriginalURL) > 2000 ||
utf8.RuneCountInString(input.CoverURL) > 2000 || len(input.RawPayload) > maxRawPayloadBytes ||
input.RawPayload != "" && !json.Valid([]byte(input.RawPayload)) {
return Work{}, false, ErrInvalid
}
if err := s.validateWorkSource(ctx, input.Platform, input.SourceType, input.SourceID); err != nil {
return Work{}, false, err
}
if now.IsZero() {
now = time.Now().UTC()
}
status := input.PublishedAtStatus
if status == "" {
status = "verified"
if input.PublishedAt == nil {
status = "pending_verification"
}
}
if status != "verified" && status != "pending_verification" && status != "invalid" && status != "future" {
return Work{}, false, ErrInvalid
}
if input.PublishedAt != nil {
input.PublishedAt = ptrTime(input.PublishedAt.UTC())
if input.PublishedAt.After(now.UTC()) && status == "verified" {
status = "future"
}
}
if input.Likes != nil && *input.Likes < 0 || input.CommentsCount != nil && *input.CommentsCount < 0 || input.Shares != nil && *input.Shares < 0 {
return Work{}, false, ErrInvalid
}
if input.CollectCount != nil && *input.CollectCount < 0 || input.PlayCount != nil && *input.PlayCount < 0 {
return Work{}, false, ErrInvalid
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return Work{}, false, fmt.Errorf("begin creator work upsert: %w", err)
}
defer tx.Rollback()
id := newID("work")
var returnedID string
var inserted bool
err = tx.QueryRowContext(ctx, `
INSERT INTO creator_work (id, platform, work_key, source_type, source_id, author_name, title, body,
published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares, collect_count, play_count)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
ON CONFLICT (platform, work_key) DO UPDATE SET
author_name = CASE WHEN EXCLUDED.author_name = '' THEN creator_work.author_name ELSE EXCLUDED.author_name END,
title = CASE WHEN EXCLUDED.title = '' THEN creator_work.title ELSE EXCLUDED.title END,
body = CASE WHEN EXCLUDED.body = '' THEN creator_work.body ELSE EXCLUDED.body END,
published_at = COALESCE(EXCLUDED.published_at, creator_work.published_at),
published_at_status = CASE WHEN EXCLUDED.published_at IS NULL THEN creator_work.published_at_status ELSE EXCLUDED.published_at_status END,
original_url = CASE WHEN EXCLUDED.original_url = '' THEN creator_work.original_url ELSE EXCLUDED.original_url END,
cover_url = CASE WHEN EXCLUDED.cover_url = '' THEN creator_work.cover_url ELSE EXCLUDED.cover_url END,
raw_payload = COALESCE(EXCLUDED.raw_payload, creator_work.raw_payload),
likes = COALESCE(EXCLUDED.likes, creator_work.likes),
comments_count = COALESCE(EXCLUDED.comments_count, creator_work.comments_count),
shares = COALESCE(EXCLUDED.shares, creator_work.shares),
collect_count = COALESCE(EXCLUDED.collect_count, creator_work.collect_count),
play_count = COALESCE(EXCLUDED.play_count, creator_work.play_count),
updated_at = now()
RETURNING id, (xmax = 0)`, id, input.Platform, input.WorkKey, input.SourceType, input.SourceID,
input.AuthorName, input.Title, input.Body, input.PublishedAt, status, input.OriginalURL, input.CoverURL,
nullableRawPayload(input.RawPayload), input.Likes, input.CommentsCount, input.Shares, input.CollectCount, input.PlayCount).Scan(&returnedID, &inserted)
if err != nil {
return Work{}, false, databaseError(err)
}
result, err := scanWork(tx.QueryRowContext(ctx, workSelect+` WHERE id = $1`, returnedID))
if err != nil {
return Work{}, false, rowError(err)
}
if err := tx.Commit(); err != nil {
return Work{}, false, fmt.Errorf("commit creator work upsert: %w", err)
}
loadWorkSources(&result)
return result, inserted, nil
}
func ptrTime(value time.Time) *time.Time { return &value }
func nullableRawPayload(value string) any {
if value == "" {
return nil
}
return value
}
func scanWork(scanner interface{ Scan(...any) error }) (Work, error) {
var result Work
var publishedAt, latestAt, nextAt sql.NullTime
var likes, commentsCount, shares, collectCount, playCount sql.NullInt64
var rawPayload sql.NullString
if err := scanner.Scan(&result.ID, &result.Platform, &result.WorkKey, &result.SourceType, &result.SourceID,
&result.AuthorName, &result.Title, &result.Body, &publishedAt, &result.PublishedAtStatus,
&result.OriginalURL, &result.CoverURL, &rawPayload, &likes, &commentsCount, &shares, &collectCount, &playCount, &latestAt, &nextAt,
&result.MetricStopReason, &result.CreatedAt, &result.UpdatedAt); err != nil {
return Work{}, err
}
result.PublishedAt = nullableTime(publishedAt)
if rawPayload.Valid {
result.RawPayload = rawPayload.String
}
result.Likes, result.CommentsCount, result.Shares = nullableInt64(likes), nullableInt64(commentsCount), nullableInt64(shares)
result.CollectCount, result.PlayCount = nullableInt64(collectCount), nullableInt64(playCount)
result.LatestMetricsAt, result.NextMetricAt = nullableTime(latestAt), nullableTime(nextAt)
return result, nil
}
const workSelect = `SELECT id, platform, work_key, source_type, source_id, author_name, title, body,
published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares, collect_count, play_count,
latest_metrics_at, next_metric_at, metric_stop_reason, created_at, updated_at FROM creator_work`
func loadWorkSources(work *Work) {
work.Sources = []WorkSource{{Platform: work.Platform, SourceType: work.SourceType, SourceID: work.SourceID}}
}
func (s *Store) GetWork(ctx context.Context, id string) (Work, error) {
result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE id = $1`, id))
if err != nil {
return Work{}, rowError(err)
}
loadWorkSources(&result)
return result, nil
}
func (s *Store) GetWorkByKey(ctx context.Context, platform, workKey string) (Work, error) {
platform, workKey = strings.TrimSpace(platform), strings.TrimSpace(workKey)
if !ValidatePlatform(platform) || workKey == "" {
return Work{}, ErrInvalid
}
result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE platform = $1 AND work_key = $2`, platform, workKey))
if err != nil {
return Work{}, rowError(err)
}
loadWorkSources(&result)
return result, nil
}
func (s *Store) ListWorks(ctx context.Context, filter WorkFilter) ([]Work, error) {
query, args := workSelect, make([]any, 0, 8)
where := make([]string, 0, 7)
add := func(clause string, value any) {
args = append(args, value)
where = append(where, fmt.Sprintf(clause, len(args)))
}
if filter.Platform != "" {
if !ValidatePlatform(filter.Platform) {
return nil, ErrInvalid
}
add("platform = $%d", filter.Platform)
}
if filter.PublishedAtStatus != "" {
if filter.PublishedAtStatus != "verified" && filter.PublishedAtStatus != "future" && filter.PublishedAtStatus != "pending_verification" && filter.PublishedAtStatus != "invalid" {
return nil, ErrInvalid
}
add("published_at_status = $%d", filter.PublishedAtStatus)
}
if filter.SourceType != "" {
if filter.SourceType != SourceOwned && filter.SourceType != SourceCompetitor {
return nil, ErrInvalid
}
args = append(args, filter.SourceType)
where = append(where, fmt.Sprintf(`creator_work.source_type = $%d`, len(args)))
}
if filter.SourceID != "" {
args = append(args, filter.SourceID)
where = append(where, fmt.Sprintf(`creator_work.source_id = $%d`, len(args)))
}
if filter.PublishedAfter != nil {
add("published_at >= $%d", filter.PublishedAfter.UTC())
}
if filter.PublishedBefore != nil {
add("published_at <= $%d", filter.PublishedBefore.UTC())
}
if filter.MinLikes != nil {
if *filter.MinLikes < 0 {
return nil, ErrInvalid
}
where = append(where, fmt.Sprintf("likes IS NOT NULL AND likes >= $%d", len(args)+1))
args = append(args, *filter.MinLikes)
}
if filter.MinComments != nil {
if *filter.MinComments < 0 {
return nil, ErrInvalid
}
where = append(where, fmt.Sprintf("comments_count IS NOT NULL AND comments_count >= $%d", len(args)+1))
args = append(args, *filter.MinComments)
}
if filter.MinShares != nil {
if *filter.MinShares < 0 {
return nil, ErrInvalid
}
where = append(where, fmt.Sprintf("shares IS NOT NULL AND shares >= $%d", len(args)+1))
args = append(args, *filter.MinShares)
}
if len(where) > 0 {
query += ` WHERE ` + strings.Join(where, ` AND `)
}
query += ` ORDER BY published_at DESC NULLS LAST, id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, databaseError(err)
}
result := make([]Work, 0)
for rows.Next() {
item, err := scanWork(rows)
if err != nil {
_ = rows.Close()
return nil, err
}
result = append(result, item)
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return nil, err
}
if err := rows.Close(); err != nil {
return nil, err
}
for index := range result {
loadWorkSources(&result[index])
}
return result, nil
}
// ListWorksMissingCover 返回已采集到远程封面但本地尚无缓存的作品(用于封面回填)。
// 单次返回上限 40 条,超出部分由后续同步继续回填。
func (s *Store) ListWorksMissingCover(ctx context.Context, sourceID string) ([]Work, error) {
if sourceID == "" {
return nil, ErrInvalid
}
rows, err := s.db.QueryContext(ctx, workSelect+` WHERE source_type = $1 AND source_id = $2 AND cover_url <> ''
AND NOT EXISTS (SELECT 1 FROM creator_work_cover c WHERE c.work_id = creator_work.id AND c.variant = 'cover')
ORDER BY created_at DESC LIMIT 40`, SourceCompetitor, sourceID)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]Work, 0)
for rows.Next() {
item, err := scanWork(rows)
if err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}
func (s *Store) RecordMetric(ctx context.Context, input MetricInput, settings Settings, now time.Time) (MetricPoint, error) {
if input.WorkID == "" || input.CollectedAt.IsZero() || input.Likes != nil && *input.Likes < 0 || input.CommentsCount != nil && *input.CommentsCount < 0 || input.Shares != nil && *input.Shares < 0 || input.CollectCount != nil && *input.CollectCount < 0 || input.PlayCount != nil && *input.PlayCount < 0 {
return MetricPoint{}, ErrInvalid
}
if err := ValidateSettings(SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds}); err != nil {
return MetricPoint{}, err
}
if now.IsZero() {
now = time.Now().UTC()
}
return s.recordMetricWithPlan(ctx, input, settings, now)
}
func nullableArg(value time.Time) any {
if value.IsZero() {
return nil
}
return value.UTC()
}
// RecordAccountMetric 落库账号画像快照。同一 (account_id, collected_at) 重复采集覆盖更新。
func (s *Store) RecordAccountMetric(ctx context.Context, input AccountMetricInput) error {
if input.AccountID == "" || input.CollectedAt.IsZero() || input.FollowerCount != nil && *input.FollowerCount < 0 ||
input.FollowingCount != nil && *input.FollowingCount < 0 || input.TotalFavorited != nil && *input.TotalFavorited < 0 || input.AwemeCount != nil && *input.AwemeCount < 0 {
return ErrInvalid
}
if _, err := s.db.ExecContext(ctx, `
INSERT INTO creator_account_metric (account_id, collected_at, follower_count, following_count, total_favorited, aweme_count)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (account_id, collected_at) DO UPDATE SET
follower_count = EXCLUDED.follower_count, following_count = EXCLUDED.following_count,
total_favorited = EXCLUDED.total_favorited, aweme_count = EXCLUDED.aweme_count`,
input.AccountID, input.CollectedAt.UTC(), input.FollowerCount, input.FollowingCount, input.TotalFavorited, input.AwemeCount); err != nil {
return databaseError(err)
}
return nil
}
// ListAccountMetrics 返回账号画像时序(时间升序),用于粉丝/获赞趋势曲线。
func (s *Store) ListAccountMetrics(ctx context.Context, accountID string) ([]AccountMetricPoint, error) {
accountID = strings.TrimSpace(accountID)
if accountID == "" {
return nil, ErrInvalid
}
rows, err := s.db.QueryContext(ctx, `
SELECT collected_at, follower_count, following_count, total_favorited, aweme_count
FROM creator_account_metric WHERE account_id = $1 ORDER BY collected_at`, accountID)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]AccountMetricPoint, 0)
for rows.Next() {
var point AccountMetricPoint
var follower, following, favorited, aweme sql.NullInt64
if err := rows.Scan(&point.CollectedAt, &follower, &following, &favorited, &aweme); err != nil {
return nil, err
}
point.CollectedAt = point.CollectedAt.UTC()
point.FollowerCount, point.FollowingCount = nullableInt64(follower), nullableInt64(following)
point.TotalFavorited, point.AwemeCount = nullableInt64(favorited), nullableInt64(aweme)
result = append(result, point)
}
return result, rows.Err()
}
func (s *Store) ListMetrics(ctx context.Context, workID string) ([]MetricPoint, error) {
rows, err := s.db.QueryContext(ctx, `SELECT collected_at, likes, comments_count, shares, collect_count, play_count FROM creator_work_metric WHERE work_id = $1 ORDER BY collected_at`, workID)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]MetricPoint, 0)
for rows.Next() {
var point MetricPoint
var likes, commentsCount, shares, collectCount, playCount sql.NullInt64
if err := rows.Scan(&point.CollectedAt, &likes, &commentsCount, &shares, &collectCount, &playCount); err != nil {
return nil, err
}
point.CollectedAt = point.CollectedAt.UTC()
point.Likes, point.CommentsCount, point.Shares = nullableInt64(likes), nullableInt64(commentsCount), nullableInt64(shares)
point.CollectCount, point.PlayCount = nullableInt64(collectCount), nullableInt64(playCount)
result = append(result, point)
}
return result, rows.Err()
}
func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, bool, error) {
input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.CommentType = strings.TrimSpace(input.Platform), strings.TrimSpace(input.CommentKey), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.AuthorUID), strings.TrimSpace(input.AuthorName), strings.TrimSpace(input.CommentType)
if !ValidatePlatform(input.Platform) || input.CommentKey == "" || input.WorkID == "" || strings.TrimSpace(input.Content) == "" || (input.CommentType != "top_level" && input.CommentType != "unknown") || utf8.RuneCountInString(input.Content) > 10000 || len(input.RawPayload) > maxRawPayloadBytes || input.RawPayload != "" && !json.Valid([]byte(input.RawPayload)) {
return Comment{}, false, ErrInvalid
}
work, err := s.GetWork(ctx, input.WorkID)
if err != nil {
return Comment{}, false, err
}
if work.Platform != input.Platform {
return Comment{}, false, ErrInvalid
}
id := newID("comment")
var returnedID string
var inserted bool
err = s.db.QueryRowContext(ctx, `INSERT INTO creator_comment (id, platform, comment_key, work_id, author_uid, author_name, content, raw_payload, published_at, comment_type) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (platform, comment_key) DO UPDATE SET work_id = EXCLUDED.work_id, author_uid = EXCLUDED.author_uid, author_name = EXCLUDED.author_name, content = EXCLUDED.content, raw_payload = COALESCE(EXCLUDED.raw_payload, creator_comment.raw_payload), published_at = EXCLUDED.published_at, comment_type = EXCLUDED.comment_type RETURNING id, (xmax = 0)`, id, input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.Content, nullableRawPayload(input.RawPayload), input.PublishedAt, input.CommentType).Scan(&returnedID, &inserted)
if err != nil {
return Comment{}, false, databaseError(err)
}
result, err := s.GetComment(ctx, returnedID)
return result, inserted, err
}
func scanComment(scanner interface{ Scan(...any) error }) (Comment, error) {
var result Comment
var publishedAt sql.NullTime
var rawPayload sql.NullString
if err := scanner.Scan(&result.ID, &result.Platform, &result.CommentKey, &result.WorkID, &result.AuthorUID, &result.AuthorName, &result.Content, &rawPayload, &publishedAt, &result.CollectedAt, &result.CommentType); err != nil {
return Comment{}, err
}
result.PublishedAt = nullableTime(publishedAt)
if rawPayload.Valid {
result.RawPayload = rawPayload.String
}
result.CollectedAt = result.CollectedAt.UTC()
return result, nil
}
const commentSelect = `SELECT id, platform, comment_key, work_id, author_uid, author_name, content, raw_payload, published_at, collected_at, comment_type FROM creator_comment`
func (s *Store) GetComment(ctx context.Context, id string) (Comment, error) {
result, err := scanComment(s.db.QueryRowContext(ctx, commentSelect+` WHERE id = $1`, id))
return result, rowError(err)
}
func (s *Store) GetCommentByKey(ctx context.Context, platform, commentKey string) (Comment, error) {
platform, commentKey = strings.TrimSpace(platform), strings.TrimSpace(commentKey)
if !ValidatePlatform(platform) || commentKey == "" {
return Comment{}, ErrInvalid
}
result, err := scanComment(s.db.QueryRowContext(ctx, commentSelect+` WHERE platform = $1 AND comment_key = $2`, platform, commentKey))
return result, rowError(err)
}
// SaveWorkCover 缓存作品封面图(带签名的 douyinpic URL 会过期,存字节后经本地接口展示)。
func (s *Store) SaveWorkCover(ctx context.Context, workID, variant, contentType string, data []byte) error {
if workID == "" || variant == "" || contentType == "" || len(data) == 0 || len(data) > maxWorkCoverBytes {
return ErrInvalid
}
_, err := s.db.ExecContext(ctx, `
INSERT INTO creator_work_cover (work_id, variant, content_type, data, fetched_at)
VALUES ($1, $2, $3, $4, now())
ON CONFLICT (work_id, variant) DO UPDATE SET content_type = EXCLUDED.content_type, data = EXCLUDED.data, fetched_at = now()`,
workID, variant, contentType, data)
if err != nil {
return databaseError(err)
}
return nil
}
func (s *Store) GetWorkCover(ctx context.Context, workID, variant string) (contentType string, data []byte, err error) {
if workID == "" || variant == "" {
return "", nil, ErrInvalid
}
err = s.db.QueryRowContext(ctx, `SELECT content_type, data FROM creator_work_cover WHERE work_id = $1 AND variant = $2`, workID, variant).Scan(&contentType, &data)
if errors.Is(err, sql.ErrNoRows) {
return "", nil, ErrNotFound
}
if err != nil {
return "", nil, databaseError(err)
}
return contentType, data, nil
}
const maxWorkCoverBytes = 4 << 20
func pageBounds(page, pageSize int) (int, int, error) {
if page < 1 || pageSize < 1 || pageSize > 100 {
return 0, 0, ErrInvalid
}
if page > int(maxDurationSeconds) || pageSize > int(maxDurationSeconds)/page {
return 0, 0, ErrInvalid
}
return (page - 1) * pageSize, pageSize, nil
}
func slicePage[T any](items []T, page, pageSize int) (Page[T], error) {
offset, limit, err := pageBounds(page, pageSize)
if err != nil {
return Page[T]{}, err
}
if offset > len(items) {
offset = len(items)
}
end := offset + limit
if end > len(items) {
end = len(items)
}
data := append([]T{}, items[offset:end]...)
return Page[T]{Data: data, Page: page, PageSize: pageSize, Total: len(items), HasNext: end < len(items)}, nil
}
func (s *Store) ListWorksPage(ctx context.Context, filter WorkFilter, page, pageSize int) (Page[Work], error) {
items, err := s.ListWorks(ctx, filter)
if err != nil {
return Page[Work]{}, err
}
if filter.Growth.Hours != 0 || filter.Growth.MinLikes != nil {
ids := make([]string, 0, len(items))
for _, item := range items {
ids = append(ids, item.ID)
}
growth, growthErr := s.LoadWorksGrowth(ctx, ids, filter.Growth)
if growthErr != nil {
return Page[Work]{}, growthErr
}
items, err = ApplyWorkGrowthFilter(items, growth, filter.Growth)
if err != nil {
return Page[Work]{}, err
}
}
return slicePage(items, page, pageSize)
}
func (s *Store) ListCommentsPage(ctx context.Context, platform, workID string, page, pageSize int) (Page[Comment], error) {
items, err := s.ListComments(ctx, platform, workID)
if err != nil {
return Page[Comment]{}, err
}
return slicePage(items, page, pageSize)
}
func (s *Store) ListComments(ctx context.Context, platform, workID string) ([]Comment, error) {
query, args := commentSelect, []any{}
where := []string{}
if platform != "" {
if !ValidatePlatform(platform) {
return nil, ErrInvalid
}
args = append(args, platform)
where = append(where, fmt.Sprintf("platform = $%d", len(args)))
}
if workID != "" {
args = append(args, workID)
where = append(where, fmt.Sprintf("work_id = $%d", len(args)))
}
if len(where) > 0 {
query += ` WHERE ` + strings.Join(where, ` AND `)
}
query += ` ORDER BY published_at DESC NULLS LAST, collected_at DESC, id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]Comment, 0)
for rows.Next() {
item, err := scanComment(rows)
if err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}