From 143f3b263ebd505445d2ecad9b6e1c088777ec5c Mon Sep 17 00:00:00 2001 From: Rogee Date: Mon, 28 Sep 2026 10:39:39 +0800 Subject: [PATCH] =?UTF-8?q?fix(creator):=20=E7=AB=9E=E5=93=81=E5=88=86?= =?UTF-8?q?=E6=9E=90=E9=A1=B5=E8=B5=B0=E6=9F=A5=E4=BF=AE=E5=A4=8D=E2=80=94?= =?UTF-8?q?=E2=80=94=E5=B0=81=E9=9D=A2=E6=9C=AC=E5=9C=B0=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E3=80=81=E4=BD=9C=E8=80=85=E5=90=8D=E3=80=81=E6=8C=87=E6=A0=87?= =?UTF-8?q?=E5=85=A8=E5=AE=BD=E8=A1=8C=E3=80=81=E6=95=B0=E9=87=8F=E5=85=A5?= =?UTF-8?q?=E5=8D=A1=E7=89=87=E6=A0=87=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 后端：works 采集解析 author.nickname 与 video.cover.url_list（migration 042 新增 creator_work_cover 本地缓存表）；竞品同步成功后回填缺失封面（单次上限 40，余量自愈） - 网关：新增 /douyin/image 浏览器内 fetch 图片端点（douyinpic CDN 白名单、4MB 限额） - API：GET /api/creator/works/:id/cover 输出本地缓存封面（private 缓存头） - 前端：封面改读本地缓存（认证 fetch + blob）；点赞/评论/分享全宽行贴封面下方； 作者名独立展示；'下次采集'移除；作品总数移入卡片标题右侧 extra - '已核验' 状态标签补充 Tooltip 释义（发布时间已从平台数据成功解析核验） 测试：douyin 解析 3 例、cover 存储 PG 集成 1 例、网关 image 3 例全绿 --- browser_gateway/platform/douyin.py | 93 +++++++++++++++ browser_gateway/server/http.py | 26 ++++- browser_gateway/test_gateway.py | 47 ++++++++ internal/controlplane/api/creator.go | 72 ++++++++++++ .../controlplane/api/creator_collector.go | 1 + internal/creator/content.go | 56 +++++++++ internal/creator/coverage_integration_test.go | 54 +++++++++ .../migrations/042_work_cover_cache.sql | 10 ++ internal/creator/store.go | 4 + internal/platform/douyin/connector.go | 62 +++++++++- internal/platform/douyin/creator_collector.go | 49 +------- .../platform/douyin/creator_collector_test.go | 23 ++++ .../pages/accounts/monitoring/$id/index.tsx | 61 ++++++---- web/src/pages/creator/competitors/index.tsx | 108 ++++++++++-------- web/src/services/api.ts | 14 +++ 15 files changed, 564 insertions(+), 116 deletions(-) create mode 100644 internal/creator/migrations/042_work_cover_cache.sql diff --git a/browser_gateway/platform/douyin.py b/browser_gateway/platform/douyin.py index ecbd1a6..64b2a05 100644 --- a/browser_gateway/platform/douyin.py +++ b/browser_gateway/platform/douyin.py @@ -54,6 +54,25 @@ COMMENTS_PATH = "/aweme/v1/web/comment/list/" RESPONSE_LIMIT = 8 << 20 MEDIA_RESPONSE_LIMIT = 32 << 20 MEDIA_SOURCE_WAIT_MS = 15000 +# 封面/头像图片体积小，限额远低于视频。 +IMAGE_RESPONSE_LIMIT = 4 << 20 +# 抖音图片 CDN 主机（封面、头像等静态资源）。 +IMAGE_HOSTS = frozenset( + { + "p3-pc-sign.douyinpic.com", + "p9-pc-sign.douyinpic.com", + "p26-pc-sign.douyinpic.com", + "p3-pc.douyinpic.com", + "p9-pc.douyinpic.com", + "p26-pc.douyinpic.com", + "p3.douyinpic.com", + "p9.douyinpic.com", + "p26.douyinpic.com", + "p3-sign.douyinpic.com", + "p9-sign.douyinpic.com", + "p26-sign.douyinpic.com", + } +) UID_RE = re.compile(r"^[1-9][0-9]{0,19}$") ID_RE = re.compile(r"^[1-9][0-9]{0,63}$") ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$") @@ -83,6 +102,7 @@ class DouyinBrowser: url_validator: Callable[[object], bool] | None = None, media_validator: Callable[[object], bool] | None = None, media_selector: str = "video", + image_validator: Callable[[object], bool] | None = None, target_id: str = "", ) -> None: self.endpoint = endpoint @@ -90,6 +110,7 @@ class DouyinBrowser: self.url_validator = url_validator or is_douyin_url self.media_validator = media_validator or is_douyin_media_url self.media_selector = media_selector + self.image_validator = image_validator or is_douyin_image_url self.target_id = target_id.strip() @contextmanager @@ -508,6 +529,10 @@ class DouyinBrowser: raise DouyinError("Douyin media response is too large") return BrowserMediaResponse(status, content_type, body) + def get_image(self, alias: str, target: str) -> BrowserMediaResponse: + """浏览器内 fetch 图片资源；与 get_media 分离，避免视频校验语义混用。""" + return fetch_douyin_image(self, alias, target) + def identity(self, alias: str, expected_uid: str | None = None) -> dict: response = self.get(alias, IDENTITY_URL) try: @@ -1497,6 +1522,74 @@ def is_douyin_media_url(value: object) -> bool: return False +def is_douyin_image_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + return ( + parsed.scheme == "https" + and parsed.hostname in IMAGE_HOSTS + and parsed.port is None + and parsed.username is None + and parsed.password is None + and len(value) <= 2048 + ) + except (TypeError, ValueError): + return False + + +def fetch_douyin_image(self: object, alias: str, target: str) -> BrowserMediaResponse: + """在浏览器内 fetch 图片（封面/头像），返回 base64 字节；复用 media 的响应结构。""" + if not self.image_validator(target): # type: ignore[attr-defined] + raise DouyinError("restricted browser image target is invalid") + with self.connection(alias) as cdp: # type: ignore[attr-defined] + result = cdp.evaluate( + f"""(async()=>{{ + try {{ + const r=await fetch({json.dumps(target)},{{credentials:'include',redirect:'error'}}); + if(!r.body)return {{status:r.status,content_type:r.headers.get('content-type')||'',body:'',too_large:false}}; + const reader=r.body.getReader(), chunks=[]; let size=0; + for(;;){{const item=await reader.read();if(item.done)break; + if(size+item.value.byteLength>{IMAGE_RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}} + size+=item.value.byteLength;chunks.push(item.value); + }} + const bytes=new Uint8Array(size); let offset=0; + for(const chunk of chunks){{bytes.set(chunk,offset);offset+=chunk.length;}} + let binary=''; + for(let offset=0;offset= 300: + raise DouyinError("Douyin image response was not successful") + try: + decoded_size = len(base64.b64decode(body, validate=True)) + except (ValueError, binascii.Error) as exc: + raise DouyinError("Douyin image response is invalid") from exc + if decoded_size > IMAGE_RESPONSE_LIMIT: + raise DouyinError("Douyin image response is too large") + if not content_type.startswith("image/"): + raise DouyinError("Douyin image response is not an image") + return BrowserMediaResponse(status, content_type, body) + + def notice_ids(event: dict) -> list[str]: try: payload = json.loads(event["payload"]) diff --git a/browser_gateway/server/http.py b/browser_gateway/server/http.py index ba6e575..91a160f 100644 --- a/browser_gateway/server/http.py +++ b/browser_gateway/server/http.py @@ -224,6 +224,28 @@ class Gateway: "body_base64": response.body_base64, } + def get_douyin_image(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_douyin_generation(input) or not isinstance(target, str): + raise RequestError("invalid restricted browser request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.browser.get_image(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Douyin image download failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError("restricted browser image download failed") from exc + return { + "status": response.status, + "content_type": response.content_type, + "body_base64": response.body_base64, + } + def douyin_identity(self, alias: str, input: dict) -> dict: expected_account_key = input.get("expected_account_key", "") if ( @@ -617,7 +639,7 @@ class GatewayHandler(BaseHTTPRequestHandler): gateway.restore_proxy(alias, body) return None match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|resolve|login-qr|action|messages|events)", + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|image|identity|resolve|login-qr|action|messages|events)", path, ) if match: @@ -628,6 +650,8 @@ class GatewayHandler(BaseHTTPRequestHandler): return gateway.resolve_douyin(alias, body) if action == "media" and method == "POST": return gateway.get_douyin_media(alias, body) + if action == "image" and method == "POST": + return gateway.get_douyin_image(alias, body) if action == "identity" and method == "POST": return gateway.douyin_identity(alias, body) if action == "login-qr" and method == "POST": diff --git a/browser_gateway/test_gateway.py b/browser_gateway/test_gateway.py index 75dc8e2..ff58587 100644 --- a/browser_gateway/test_gateway.py +++ b/browser_gateway/test_gateway.py @@ -904,6 +904,53 @@ class BrowserTests(unittest.TestCase): self.assertIn("querySelectorAll", cdp.expressions[-1]) self.assertIn("uuu_265.mp4", cdp.expressions[-1]) + def test_image_download_accepts_douyinpic_and_rejects_other(self) -> None: + # 封面/头像走 douyinpic CDN：浏览器内 fetch + base64，无导航无签名。 + payload = base64.b64encode(b"jpeg-bytes").decode("ascii") + cdp = BrowserCDP( + [ + {"status": 200, "content_type": "image/jpeg", "body": payload}, + ] + ) + browser = DouyinBrowser() + self._with_connection(browser, cdp) + response = browser.get_image( + "safe", + "https://p3-pc-sign.douyinpic.com/tos-cn-i-0813/cover.jpeg?x-expires=2105848800&x-signature=abc", + ) + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "image/jpeg") + self.assertEqual(base64.b64decode(response.body_base64), b"jpeg-bytes") + self.assertIn("fetch", cdp.expressions[-1]) + + cdp = BrowserCDP([]) + self._with_connection(browser, cdp) + with self.assertRaisesRegex(DouyinError, "image target is invalid"): + browser.get_image("safe", "https://evil.example.com/cover.jpeg") + + def test_image_download_rejects_non_image_content(self) -> None: + payload = base64.b64encode(b"").decode("ascii") + cdp = BrowserCDP( + [ + {"status": 200, "content_type": "text/html", "body": payload}, + ] + ) + browser = DouyinBrowser() + self._with_connection(browser, cdp) + with self.assertRaisesRegex(DouyinError, "not an image"): + browser.get_image("safe", "https://p3-pc-sign.douyinpic.com/cover.jpeg") + + def test_image_download_rejects_too_large(self) -> None: + cdp = BrowserCDP( + [ + {"too_large": True}, + ] + ) + browser = DouyinBrowser() + self._with_connection(browser, cdp) + with self.assertRaisesRegex(DouyinError, "image download failed"): + browser.get_image("safe", "https://p9-pc-sign.douyinpic.com/big.jpeg") + def test_connect_identity_and_actions(self) -> None: target = [ { diff --git a/internal/controlplane/api/creator.go b/internal/controlplane/api/creator.go index 9a8ba8f..4c07ea1 100644 --- a/internal/controlplane/api/creator.go +++ b/internal/controlplane/api/creator.go @@ -461,6 +461,15 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto } return c.JSON(point) }) + app.Get("/api/creator/works/:id/cover", func(c fiber.Ctx) error { + contentType, data, err := store.GetWorkCover(c.Context(), c.Params("id"), "cover") + if err != nil { + return creatorError(c, err) + } + c.Set(fiber.HeaderContentType, contentType) + c.Set(fiber.HeaderCacheControl, "private, max-age=86400") + return c.Send(data) + }) app.Get("/api/creator/works/:id/material", func(c fiber.Ctx) error { item, err := store.GetMaterial(c.Context(), c.Params("id")) if err != nil { @@ -1435,6 +1444,63 @@ func (browser creatorGatewayBrowser) Get(ctx context.Context, target string) (do return douyin.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil } +// GetImage 经网关浏览器内 fetch 拉取图片（封面等静态资源），返回 content_type 与字节。 +func (browser creatorGatewayBrowser) GetImage(ctx context.Context, target string) (string, []byte, error) { + payload := gatewayGenerationPayload(browser.environment) + payload["url"] = target + status, body, err := gatewayCallWithLimit(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/image", payload, 60*time.Second, largeGatewayResponseLimit) + if err != nil { + return "", nil, err + } + if status != http.StatusOK { + return "", nil, fmt.Errorf("douyin image request rejected with HTTP %d: %s", status, string(body)) + } + var response struct { + Status int `json:"status"` + ContentType string `json:"content_type"` + BodyBase64 string `json:"body_base64"` + } + if err := json.Unmarshal(body, &response); err != nil { + return "", nil, fmt.Errorf("decode douyin image response: %w", err) + } + contentType := strings.ToLower(strings.TrimSpace(response.ContentType)) + if response.Status < 200 || response.Status >= 300 || !strings.HasPrefix(contentType, "image/") { + return "", nil, fmt.Errorf("douyin image response is not an image (HTTP %d, content_type %q)", response.Status, response.ContentType) + } + data, err := base64.StdEncoding.DecodeString(response.BodyBase64) + if err != nil { + return "", nil, fmt.Errorf("decode douyin image bytes: %w", err) + } + if len(data) > maxCreatorCoverBytes { + return "", nil, fmt.Errorf("douyin image response is too large") + } + return contentType, data, nil +} + +const maxCreatorCoverBytes = 4 << 20 + +// cacheCompetitorWorkCovers 把竞品作品封面拉取到本地缓存（带签名的 douyinpic URL 会过期，前端读本地缓存）。 +// 单次同步最多回填 40 张，超出部分留待后续同步自愈，避免首次采集时长时间占用匿名浏览器。 +func cacheCompetitorWorkCovers(ctx context.Context, store *creator.Store, browser creatorGatewayBrowser, competitorID string) error { + works, err := store.ListWorksMissingCover(ctx, competitorID) + if err != nil { + return err + } + for _, work := range works { + contentType, data, fetchErr := browser.GetImage(ctx, work.CoverURL) + if fetchErr != nil { + return fmt.Errorf("fetch cover for work %s: %w", work.ID, fetchErr) + } + if saveErr := store.SaveWorkCover(ctx, work.ID, "cover", contentType, data); saveErr != nil { + return fmt.Errorf("save cover for work %s: %w", work.ID, saveErr) + } + } + if len(works) > 0 { + logrus.WithFields(logrus.Fields{"competitor_id": competitorID, "covers_cached": len(works)}).Info("creator competitor covers cached") + } + return nil +} + func (browser creatorGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) { payload := gatewayGenerationPayload(browser.environment) payload["url"] = target @@ -1878,6 +1944,12 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, h collectionNow = competitor.NextSyncAt.UTC() } report, collectErr := store.CollectSource(ctx, competitor.Platform, creator.SourceCompetitor, competitor.ID, collector, collectionNow) + if collectErr == nil { + // 采集成功后把缺封面的作品封面缓存到本地（失败只记日志，不影响本次同步结果）。 + if coverErr := cacheCompetitorWorkCovers(ctx, store, creatorGatewayBrowser{gateway: lease.gateway, environment: lease.environment}, competitor.ID); coverErr != nil { + logrus.WithError(coverErr).WithField("competitor_id", competitorID).Warn("creator competitor cover cache failed") + } + } if closeErr := lease.close(); closeErr != nil { collectErr = errors.Join(collectErr, closeErr) } diff --git a/internal/controlplane/api/creator_collector.go b/internal/controlplane/api/creator_collector.go index b7bfa00..22673b0 100644 --- a/internal/controlplane/api/creator_collector.go +++ b/internal/controlplane/api/creator_collector.go @@ -37,6 +37,7 @@ func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gatew func douyinCollector(browser creatorGatewayBrowser, accountKey, sourceType, sourceID string) douyin.CreatorCollector { return douyin.CreatorCollector{Browser: browser, AccountKey: accountKey, SourceType: sourceType, SourceID: sourceID} } + // validateDouyinCompetitor 监控账号入队前的平台一致性校验（平台收敛后仅抖音）。 func validateDouyinCompetitor(input creator.CompetitorInput) error { if input.Platform != creator.PlatformDouyin { diff --git a/internal/creator/content.go b/internal/creator/content.go index 325b02c..3dce4a7 100644 --- a/internal/creator/content.go +++ b/internal/creator/content.go @@ -593,6 +593,30 @@ func (s *Store) ListWorks(ctx context.Context, filter WorkFilter) ([]Work, error return result, nil } +// ListWorksMissingCover 返回已采集到远程封面但本地尚无缓存的作品（用于封面回填）。 +// 单次返回上限 40 条，超出部分由后续同步继续回填。 +func (s *Store) ListWorksMissingCover(ctx context.Context, sourceID string) ([]Work, error) { + if sourceID == "" { + return nil, ErrInvalid + } + rows, err := s.db.QueryContext(ctx, workSelect+` WHERE source_type = $1 AND source_id = $2 AND cover_url <> '' + AND NOT EXISTS (SELECT 1 FROM creator_work_cover c WHERE c.work_id = creator_work.id AND c.variant = 'cover') + ORDER BY created_at DESC LIMIT 40`, SourceCompetitor, sourceID) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + result := make([]Work, 0) + for rows.Next() { + item, err := scanWork(rows) + if err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + func (s *Store) RecordMetric(ctx context.Context, input MetricInput, settings Settings, now time.Time) (MetricPoint, error) { if input.WorkID == "" || input.CollectedAt.IsZero() || input.Likes != nil && *input.Likes < 0 || input.CommentsCount != nil && *input.CommentsCount < 0 || input.Shares != nil && *input.Shares < 0 || input.CollectCount != nil && *input.CollectCount < 0 || input.PlayCount != nil && *input.PlayCount < 0 { return MetricPoint{}, ErrInvalid @@ -1002,6 +1026,38 @@ func (s *Store) GetCommentByKey(ctx context.Context, platform, commentKey string return result, rowError(err) } +// SaveWorkCover 缓存作品封面图（带签名的 douyinpic URL 会过期，存字节后经本地接口展示）。 +func (s *Store) SaveWorkCover(ctx context.Context, workID, variant, contentType string, data []byte) error { + if workID == "" || variant == "" || contentType == "" || len(data) == 0 || len(data) > maxWorkCoverBytes { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO creator_work_cover (work_id, variant, content_type, data, fetched_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (work_id, variant) DO UPDATE SET content_type = EXCLUDED.content_type, data = EXCLUDED.data, fetched_at = now()`, + workID, variant, contentType, data) + if err != nil { + return databaseError(err) + } + return nil +} + +func (s *Store) GetWorkCover(ctx context.Context, workID, variant string) (contentType string, data []byte, err error) { + if workID == "" || variant == "" { + return "", nil, ErrInvalid + } + err = s.db.QueryRowContext(ctx, `SELECT content_type, data FROM creator_work_cover WHERE work_id = $1 AND variant = $2`, workID, variant).Scan(&contentType, &data) + if errors.Is(err, sql.ErrNoRows) { + return "", nil, ErrNotFound + } + if err != nil { + return "", nil, databaseError(err) + } + return contentType, data, nil +} + +const maxWorkCoverBytes = 4 << 20 + func pageBounds(page, pageSize int) (int, int, error) { if page < 1 || pageSize < 1 || pageSize > 100 { return 0, 0, ErrInvalid diff --git a/internal/creator/coverage_integration_test.go b/internal/creator/coverage_integration_test.go index 5673d92..98f4031 100644 --- a/internal/creator/coverage_integration_test.go +++ b/internal/creator/coverage_integration_test.go @@ -2,10 +2,64 @@ package creator import ( "errors" + "fmt" "testing" "time" ) +func TestCreatorPostgresWorkCoverCache(t *testing.T) { + store, _, ctx := openCreatorIntegrationStore(t) + stamp := fmt.Sprintf("%d", time.Now().UnixNano()) + competitor, err := store.UpsertCompetitor(ctx, CompetitorInput{ + Platform: PlatformDouyin, PlatformAccountKey: "MS4wLjABAAAAcover" + stamp, Nickname: "温江菜花小院", + HomepageURL: "https://www.douyin.com/user/cover" + stamp, + }) + if err != nil { + t.Fatalf("upsert competitor: %v", err) + } + now := time.Now().UTC().Add(-time.Hour) + work, inserted, err := store.UpsertWork(ctx, WorkInput{ + Platform: PlatformDouyin, WorkKey: "cover-work-" + stamp, SourceType: SourceCompetitor, SourceID: competitor.ID, + AuthorName: "木子不写代码", Title: "标题", PublishedAt: &now, PublishedAtStatus: "verified", + CoverURL: "https://p3-pc-sign.douyinpic.com/cover.jpeg", + }, now) + if err != nil || !inserted { + t.Fatalf("upsert work: inserted=%v err=%v", inserted, err) + } + // 采集到封面 URL 但尚未缓存 → 出现在回填列表。 + missing, err := store.ListWorksMissingCover(ctx, competitor.ID) + if err != nil || len(missing) != 1 || missing[0].ID != work.ID { + t.Fatalf("list missing covers: works=%+v err=%v", missing, err) + } + if err := store.SaveWorkCover(ctx, work.ID, "cover", "image/jpeg", []byte("jpeg-bytes")); err != nil { + t.Fatalf("save cover: %v", err) + } + contentType, data, err := store.GetWorkCover(ctx, work.ID, "cover") + if err != nil || contentType != "image/jpeg" || string(data) != "jpeg-bytes" { + t.Fatalf("get cover: contentType=%q data=%q err=%v", contentType, data, err) + } + // 缓存后不再出现在回填列表。 + missing, err = store.ListWorksMissingCover(ctx, competitor.ID) + if err != nil || len(missing) != 0 { + t.Fatalf("missing covers after cache: works=%+v err=%v", missing, err) + } + // 重复写入覆盖旧内容（新采集周期封面可能更新）。 + if err := store.SaveWorkCover(ctx, work.ID, "cover", "image/webp", []byte("webp-bytes")); err != nil { + t.Fatalf("overwrite cover: %v", err) + } + contentType, data, err = store.GetWorkCover(ctx, work.ID, "cover") + if err != nil || contentType != "image/webp" || string(data) != "webp-bytes" { + t.Fatalf("overwritten cover: contentType=%q data=%q err=%v", contentType, data, err) + } + if _, _, err := store.GetWorkCover(ctx, "missing", "cover"); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing cover must be ErrNotFound: err=%v", err) + } + // 非法输入拒绝。 + if err := store.SaveWorkCover(ctx, work.ID, "cover", "image/jpeg", nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("empty cover data must be rejected: err=%v", err) + } +} + func TestCreatorPostgresPageAndConversationState(t *testing.T) { store, phaseAStore, ctx := openCreatorIntegrationStore(t) if err := store.Ping(ctx); err != nil { diff --git a/internal/creator/migrations/042_work_cover_cache.sql b/internal/creator/migrations/042_work_cover_cache.sql new file mode 100644 index 0000000..13905e7 --- /dev/null +++ b/internal/creator/migrations/042_work_cover_cache.sql @@ -0,0 +1,10 @@ +-- 作品封面本地缓存（2026-09-28 竞品页走查）：封面图存本地 cover 表，按 (work_id, variant) 唯一。 +-- 原始 douyinpic URL 带签名会过期，前端改为经 /api/creator/works/:id/cover 读取本地缓存。 +CREATE TABLE IF NOT EXISTS creator_work_cover ( + work_id text NOT NULL REFERENCES creator_work(id) ON DELETE CASCADE, + variant text NOT NULL, + content_type text NOT NULL, + data bytea NOT NULL, + fetched_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (work_id, variant) +); diff --git a/internal/creator/store.go b/internal/creator/store.go index 09db35a..05bc9d3 100644 --- a/internal/creator/store.go +++ b/internal/creator/store.go @@ -95,6 +95,9 @@ var migration040 string //go:embed migrations/041_creator_account_metric.sql var migration041 string +//go:embed migrations/042_work_cover_cache.sql +var migration042 string + type SecretReference struct { ID string Provider string @@ -197,6 +200,7 @@ func (s *Store) migrate(ctx context.Context) error { {version: 39, sql: migration039}, {version: 40, sql: migration040}, {version: 41, sql: migration041}, + {version: 42, sql: migration042}, } for _, migration := range migrations { var applied bool diff --git a/internal/platform/douyin/connector.go b/internal/platform/douyin/connector.go index 2a472c2..cbb7198 100644 --- a/internal/platform/douyin/connector.go +++ b/internal/platform/douyin/connector.go @@ -440,7 +440,15 @@ type worksEnvelope struct { ID string `json:"aweme_id"` Description string `json:"desc"` CreatedAt *int64 `json:"create_time"` - Statistics *struct { + Author *struct { + Nickname string `json:"nickname"` + } `json:"author"` + Video *struct { + Cover *struct { + URLList []string `json:"url_list"` + } `json:"cover"` + } `json:"video"` + Statistics *struct { DiggCount *int64 `json:"digg_count"` CommentCount *int64 `json:"comment_count"` ShareCount *int64 `json:"share_count"` @@ -474,6 +482,58 @@ func parseWorksPage(body []byte) ([]Work, bool, *int64, bool) { return works, bool(*envelope.HasMore), envelope.MaxCursor, true } +func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) { + // 作品项携带播放信息等大字段，单页真实响应可达 2MB+（见网关 RESPONSE_LIMIT 同步放宽）。 + if len(body) > 8<<20 { + return nil, false, nil, false + } + var envelope worksEnvelope + // 平台风控语义（2026 年实测，与 TikTokDownloader #706 等公开项目一致）： + // - 匿名会话第一页正常（16-20 条，has_more=1 且带 max_cursor）； + // - 第二页起一律返回空 envelope {"status_code":0}（无 aweme_list/has_more/max_cursor）， + // 登录 Cookie 也无法翻页。无 aweme_list 的空 envelope 视为正常空页（分页终止）， + // 而非无效响应；只有响应同时带 max_cursor 时才继续翻页。 + if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore != nil && envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 { + return nil, false, nil, false + } + items := make([]creatorWorkPageItem, 0, len(envelope.Works)) + seen := make(map[string]struct{}, len(envelope.Works)) + for _, candidate := range envelope.Works { + if !keyPattern.MatchString(candidate.ID) { + return nil, false, nil, false + } + if _, exists := seen[candidate.ID]; exists { + return nil, false, nil, false + } + seen[candidate.ID] = struct{}{} + var likes, comments, shares, collects, plays *int64 + if candidate.Statistics != nil { + likes, comments, shares, collects, plays = candidate.Statistics.DiggCount, candidate.Statistics.CommentCount, candidate.Statistics.ShareCount, candidate.Statistics.CollectCount, candidate.Statistics.PlayCount + } + for _, value := range []*int64{likes, comments, shares, collects, plays} { + if value != nil && *value < 0 { + return nil, false, nil, false + } + } + createdAt := candidate.CreatedAt + createdAtInvalid := createdAt != nil && *createdAt <= 0 + if createdAtInvalid { + createdAt = nil + } + item := creatorWorkPageItem{ID: candidate.ID, Description: candidate.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares, CollectCount: collects, PlayCount: plays} + if candidate.Author != nil { + item.AuthorName = candidate.Author.Nickname + } + if candidate.Video != nil && candidate.Video.Cover != nil && len(candidate.Video.Cover.URLList) > 0 { + item.CoverURL = candidate.Video.Cover.URLList[0] + } + items = append(items, item) + } + // 仅当响应带有效 max_cursor 时才声明翻页；无 cursor 的 has_more 不产生翻页游标。 + hasMore := envelope.HasMore != nil && bool(*envelope.HasMore) && envelope.MaxCursor != nil + return items, hasMore, envelope.MaxCursor, true +} + func parseWorks(body []byte) ([]Work, bool, bool) { works, hasMore, _, ok := parseWorksPage(body) return works, hasMore, ok diff --git a/internal/platform/douyin/creator_collector.go b/internal/platform/douyin/creator_collector.go index 6db3c21..f263db2 100644 --- a/internal/platform/douyin/creator_collector.go +++ b/internal/platform/douyin/creator_collector.go @@ -263,7 +263,7 @@ func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor stri } else if published != nil { status = "verified" } - items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, Likes: likes, CommentsCount: comments, Shares: shares, CollectCount: collects, PlayCount: plays}) + items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, AuthorName: work.AuthorName, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, CoverURL: work.CoverURL, Likes: likes, CommentsCount: comments, Shares: shares, CollectCount: collects, PlayCount: plays}) } page := creator.WorkPage{Items: items, HasMore: hasMore} if nextCursor != nil { @@ -339,8 +339,10 @@ func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) { type creatorWorkPageItem struct { ID string Description string + AuthorName string CreatedAt *int64 CreatedAtInvalid bool + CoverURL string DiggCount *int64 CommentCount *int64 ShareCount *int64 @@ -348,51 +350,6 @@ type creatorWorkPageItem struct { PlayCount *int64 } -func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) { - // 作品项携带播放信息等大字段，单页真实响应可达 2MB+（见网关 RESPONSE_LIMIT 同步放宽）。 - if len(body) > 8<<20 { - return nil, false, nil, false - } - var envelope worksEnvelope - // 平台风控语义（2026 年实测，与 TikTokDownloader #706 等公开项目一致）： - // - 匿名会话第一页正常（16-20 条，has_more=1 且带 max_cursor）； - // - 第二页起一律返回空 envelope {"status_code":0}（无 aweme_list/has_more/max_cursor）， - // 登录 Cookie 也无法翻页。无 aweme_list 的空 envelope 视为正常空页（分页终止）， - // 而非无效响应；只有响应同时带 max_cursor 时才继续翻页。 - if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore != nil && envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 { - return nil, false, nil, false - } - items := make([]creatorWorkPageItem, 0, len(envelope.Works)) - seen := make(map[string]struct{}, len(envelope.Works)) - for _, item := range envelope.Works { - if !keyPattern.MatchString(item.ID) { - return nil, false, nil, false - } - if _, exists := seen[item.ID]; exists { - return nil, false, nil, false - } - seen[item.ID] = struct{}{} - var likes, comments, shares, collects, plays *int64 - if item.Statistics != nil { - likes, comments, shares, collects, plays = item.Statistics.DiggCount, item.Statistics.CommentCount, item.Statistics.ShareCount, item.Statistics.CollectCount, item.Statistics.PlayCount - } - for _, value := range []*int64{likes, comments, shares, collects, plays} { - if value != nil && *value < 0 { - return nil, false, nil, false - } - } - createdAt := item.CreatedAt - createdAtInvalid := createdAt != nil && *createdAt <= 0 - if createdAtInvalid { - createdAt = nil - } - items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares, CollectCount: collects, PlayCount: plays}) - } - // 仅当响应带有效 max_cursor 时才声明翻页；无 cursor 的 has_more 不产生翻页游标。 - hasMore := envelope.HasMore != nil && bool(*envelope.HasMore) && envelope.MaxCursor != nil - return items, hasMore, envelope.MaxCursor, true -} - type commentEnvelope struct { StatusCode *int `json:"status_code"` HasMore *douyinBool `json:"has_more"` diff --git a/internal/platform/douyin/creator_collector_test.go b/internal/platform/douyin/creator_collector_test.go index 72b4bba..bd3a627 100644 --- a/internal/platform/douyin/creator_collector_test.go +++ b/internal/platform/douyin/creator_collector_test.go @@ -127,6 +127,29 @@ func TestParseCreatorWorksPageMarksInvalidTimestamp(t *testing.T) { } } +func TestParseCreatorWorksPageExtractsAuthorAndCover(t *testing.T) { + // 匿名 works 响应实测：author.nickname 与 video.cover.url_list[0] 都在作品项上。 + body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"first page","create_time":1700000000,"author":{"nickname":"温江菜花小院"},"video":{"cover":{"url_list":["https://p3-pc-sign.douyinpic.com/cover.jpeg","https://p9-pc-sign.douyinpic.com/backup.jpeg"]}},"statistics":{"digg_count":1,"comment_count":2,"share_count":3,"collect_count":4,"play_count":55}}]}`) + works, _, _, ok := parseCreatorWorksPage(body) + if !ok || len(works) != 1 { + t.Fatalf("page not parsed: ok=%v works=%+v", ok, works) + } + if works[0].AuthorName != "温江菜花小院" { + t.Fatalf("author name missing: %+v", works[0]) + } + if works[0].CoverURL != "https://p3-pc-sign.douyinpic.com/cover.jpeg" { + t.Fatalf("cover url missing: %+v", works[0]) + } +} + +func TestParseCreatorWorksPageToleratesMissingAuthorAndCover(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"no media","create_time":1700000000,"statistics":{"digg_count":1}}]}`) + works, _, _, ok := parseCreatorWorksPage(body) + if !ok || len(works) != 1 || works[0].AuthorName != "" || works[0].CoverURL != "" { + t.Fatalf("missing author/cover must be tolerated: ok=%v works=%+v", ok, works) + } +} + func TestParseCreatorWorksPageAcceptsNumericHasMore(t *testing.T) { body := []byte(`{"status_code":0,"has_more":0,"aweme_list":[]}`) works, hasMore, cursor, ok := parseCreatorWorksPage(body) diff --git a/web/src/pages/accounts/monitoring/$id/index.tsx b/web/src/pages/accounts/monitoring/$id/index.tsx index 527dc70..625a6da 100644 --- a/web/src/pages/accounts/monitoring/$id/index.tsx +++ b/web/src/pages/accounts/monitoring/$id/index.tsx @@ -139,28 +139,45 @@ export default function Page() { return ( - {competitor.sync_error} - ) : ( - '—' - ) }, - ]} - /> + + + {(competitor.nickname || competitor.platform_account_key).slice(0, 1).toUpperCase()} + + + + + {competitor.nickname || competitor.platform_account_key} + + {syncMeta.label} + + + {competitor.unique_id || competitor.platform_account_key} · {platformLabel(competitor.platform)} + + {competitor.sync_error} + ) : ( + '—' + ) }, + ]} + /> + + diff --git a/web/src/pages/creator/competitors/index.tsx b/web/src/pages/creator/competitors/index.tsx index 380b025..cc8ff11 100644 --- a/web/src/pages/creator/competitors/index.tsx +++ b/web/src/pages/creator/competitors/index.tsx @@ -1,11 +1,11 @@ // 竞品分析：作品列表对齐 ant-design-pro card-list——antd Row/Col 响应式栅格 + Card 原样组合； // 作品详情与素材处理均走 Drawer；日期筛选用 antd DatePicker；筛选自动刷新、手动翻页不变。 -// 平台筛选已随 migration 038（platform 收敛 douyin）移除。 +// 封面读本地缓存（/works/:id/cover，认证 fetch + blob）；发布时间状态详情见 publishedAtStatusMeta 注释。 import { useEffect, useState } from 'react'; import { CommentOutlined, LikeOutlined, ShareAltOutlined } from '@ant-design/icons'; -import { Alert, App, Button, Card, Col, DatePicker, Drawer, Empty, Flex, Input, InputNumber, Modal, Row, Select, Space, Spin, Table, Tag, Typography } from 'antd'; +import { Alert, App, Button, Card, Col, DatePicker, Drawer, Empty, Flex, Input, InputNumber, Modal, Row, Select, Space, Spin, Table, Tag, Tooltip, Typography } from 'antd'; import type { Dayjs } from 'dayjs'; -import { creatorAction, creatorGet, creatorUpdate, getList } from '@/services/api'; +import { creatorAction, creatorGet, creatorUpdate, fetchCreatorBinary, getList } from '@/services/api'; import { conflictMessage, dateTime } from '@/utils/helpers'; const publishedAtStatusOptions = [ @@ -15,11 +15,12 @@ const publishedAtStatusOptions = [ { value: 'invalid', label: '无效时间' }, ]; -const publishedAtStatusMeta: Record = { - verified: { color: 'green', label: '已核验' }, - future: { color: 'orange', label: '未来时间' }, - pending_verification: { label: '待核验' }, - invalid: { color: 'red', label: '无效时间' } }; +const publishedAtStatusMeta: Record = { + // verified：发布时间从平台原始数据中成功解析且已核验（非未来时间、非无效值）。 + verified: { color: 'green', label: '已核验', hint: '发布时间已从平台数据成功解析核验' }, + future: { color: 'orange', label: '未来时间', hint: '发布时间晚于当前时间，可能是定时发布' }, + pending_verification: { label: '待核验', hint: '平台未返回发布时间，待后续采集补齐' }, + invalid: { color: 'red', label: '无效时间', hint: '平台返回的发布时间无效（如 0 或负值）' } }; interface Filters { minLikes?: number; @@ -32,7 +33,36 @@ interface Filters { function statusTag(status?: string) { const meta = publishedAtStatusMeta[status || ''] || { label: status || '—' }; - return {meta.label}; + const tag = {meta.label}; + return meta.hint ? {tag} : tag; +} + +// 作品封面：从本地缓存接口读取（认证 fetch + blob URL）；缓存缺失时展示占位。 +function WorkCover({ workID, title }: { workID: string; title?: string }) { + const [src, setSrc] = useState(null); + const [missing, setMissing] = useState(false); + useEffect(() => { + let objectURL: string | null = null; + setSrc(null); + setMissing(false); + fetchCreatorBinary(`/creator/works/${encodeURIComponent(workID)}/cover`) + .then((blob) => { + objectURL = URL.createObjectURL(blob); + setSrc(objectURL); + }) + .catch(() => setMissing(true)); + return () => { + if (objectURL) URL.revokeObjectURL(objectURL); + }; + }, [workID]); + if (src) { + return {title; + } + return ( + + {missing ? '封面未缓存' : '封面加载中'} + + ); } function stepTag(status: string, label: string) { @@ -241,13 +271,10 @@ export default function Page() { /> - + 共 {workPageInfo.total} 个作品}> {error ? ( 重试} /> ) : null} - - 共 {workPageInfo.total} 个作品 - {works.length ? ( @@ -255,15 +282,8 @@ export default function Page() { - ) : ( - - 无封面 - - ) - } + styles={{ body: { padding: '0 0 12px' } }} + cover={} actions={[