fix(creator): 竞品分析页走查修复——封面本地缓存、作者名、指标全宽行、数量入卡片标题

- 后端:works 采集解析 author.nickname 与 video.cover.url_list(migration 042 新增
  creator_work_cover 本地缓存表);竞品同步成功后回填缺失封面(单次上限 40,余量自愈)
- 网关:新增 /douyin/image 浏览器内 fetch 图片端点(douyinpic CDN 白名单、4MB 限额)
- API:GET /api/creator/works/:id/cover 输出本地缓存封面(private 缓存头)
- 前端:封面改读本地缓存(认证 fetch + blob);点赞/评论/分享全宽行贴封面下方;
  作者名独立展示;'下次采集'移除;作品总数移入卡片标题右侧 extra
- '已核验' 状态标签补充 Tooltip 释义(发布时间已从平台数据成功解析核验)

测试:douyin 解析 3 例、cover 存储 PG 集成 1 例、网关 image 3 例全绿
This commit is contained in:
2026-09-28 10:39:56 +08:00
parent 5e391b70e9
commit 143f3b263e
15 changed files with 564 additions and 116 deletions
+93
View File
@@ -54,6 +54,25 @@ COMMENTS_PATH = "/aweme/v1/web/comment/list/"
RESPONSE_LIMIT = 8 << 20
MEDIA_RESPONSE_LIMIT = 32 << 20
MEDIA_SOURCE_WAIT_MS = 15000
# 封面/头像图片体积小,限额远低于视频。
IMAGE_RESPONSE_LIMIT = 4 << 20
# 抖音图片 CDN 主机(封面、头像等静态资源)。
IMAGE_HOSTS = frozenset(
{
"p3-pc-sign.douyinpic.com",
"p9-pc-sign.douyinpic.com",
"p26-pc-sign.douyinpic.com",
"p3-pc.douyinpic.com",
"p9-pc.douyinpic.com",
"p26-pc.douyinpic.com",
"p3.douyinpic.com",
"p9.douyinpic.com",
"p26.douyinpic.com",
"p3-sign.douyinpic.com",
"p9-sign.douyinpic.com",
"p26-sign.douyinpic.com",
}
)
UID_RE = re.compile(r"^[1-9][0-9]{0,19}$")
ID_RE = re.compile(r"^[1-9][0-9]{0,63}$")
ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$")
@@ -83,6 +102,7 @@ class DouyinBrowser:
url_validator: Callable[[object], bool] | None = None,
media_validator: Callable[[object], bool] | None = None,
media_selector: str = "video",
image_validator: Callable[[object], bool] | None = None,
target_id: str = "",
) -> None:
self.endpoint = endpoint
@@ -90,6 +110,7 @@ class DouyinBrowser:
self.url_validator = url_validator or is_douyin_url
self.media_validator = media_validator or is_douyin_media_url
self.media_selector = media_selector
self.image_validator = image_validator or is_douyin_image_url
self.target_id = target_id.strip()
@contextmanager
@@ -508,6 +529,10 @@ class DouyinBrowser:
raise DouyinError("Douyin media response is too large")
return BrowserMediaResponse(status, content_type, body)
def get_image(self, alias: str, target: str) -> BrowserMediaResponse:
"""浏览器内 fetch 图片资源;与 get_media 分离,避免视频校验语义混用。"""
return fetch_douyin_image(self, alias, target)
def identity(self, alias: str, expected_uid: str | None = None) -> dict:
response = self.get(alias, IDENTITY_URL)
try:
@@ -1497,6 +1522,74 @@ def is_douyin_media_url(value: object) -> bool:
return False
def is_douyin_image_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
return (
parsed.scheme == "https"
and parsed.hostname in IMAGE_HOSTS
and parsed.port is None
and parsed.username is None
and parsed.password is None
and len(value) <= 2048
)
except (TypeError, ValueError):
return False
def fetch_douyin_image(self: object, alias: str, target: str) -> BrowserMediaResponse:
"""在浏览器内 fetch 图片(封面/头像),返回 base64 字节;复用 media 的响应结构。"""
if not self.image_validator(target): # type: ignore[attr-defined]
raise DouyinError("restricted browser image target is invalid")
with self.connection(alias) as cdp: # type: ignore[attr-defined]
result = cdp.evaluate(
f"""(async()=>{{
try {{
const r=await fetch({json.dumps(target)},{{credentials:'include',redirect:'error'}});
if(!r.body)return {{status:r.status,content_type:r.headers.get('content-type')||'',body:'',too_large:false}};
const reader=r.body.getReader(), chunks=[]; let size=0;
for(;;){{const item=await reader.read();if(item.done)break;
if(size+item.value.byteLength>{IMAGE_RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}}
size+=item.value.byteLength;chunks.push(item.value);
}}
const bytes=new Uint8Array(size); let offset=0;
for(const chunk of chunks){{bytes.set(chunk,offset);offset+=chunk.length;}}
let binary='';
for(let offset=0;offset<bytes.length;offset+=0x8000)binary+=String.fromCharCode(...bytes.subarray(offset,Math.min(offset+0x8000,bytes.length)));
return {{status:r.status,content_type:r.headers.get('content-type')||'',body:btoa(binary),too_large:false}};
}} catch(e) {{
return {{status:0,content_type:'',body:'',error:String((e&&e.message)||e)}};
}}
}})()"""
)
if not isinstance(result, dict) or result.get("too_large"):
raise DouyinError("Douyin image download failed")
if result.get("error"):
raise DouyinError(f"Douyin image download failed: {result['error']}")
status = result.get("status")
content_type = result.get("content_type")
body = result.get("body")
if (
not isinstance(status, int)
or not isinstance(content_type, str)
or not isinstance(body, str)
):
raise DouyinError("Douyin image response is invalid")
if status < 200 or status >= 300:
raise DouyinError("Douyin image response was not successful")
try:
decoded_size = len(base64.b64decode(body, validate=True))
except (ValueError, binascii.Error) as exc:
raise DouyinError("Douyin image response is invalid") from exc
if decoded_size > IMAGE_RESPONSE_LIMIT:
raise DouyinError("Douyin image response is too large")
if not content_type.startswith("image/"):
raise DouyinError("Douyin image response is not an image")
return BrowserMediaResponse(status, content_type, body)
def notice_ids(event: dict) -> list[str]:
try:
payload = json.loads(event["payload"])
+25 -1
View File
@@ -224,6 +224,28 @@ class Gateway:
"body_base64": response.body_base64,
}
def get_douyin_image(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_douyin_generation(input) or not isinstance(target, str):
raise RequestError("invalid restricted browser request", 400)
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
try:
response = self.browser.get_image(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning(
"Douyin image download failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError("restricted browser image download failed") from exc
return {
"status": response.status,
"content_type": response.content_type,
"body_base64": response.body_base64,
}
def douyin_identity(self, alias: str, input: dict) -> dict:
expected_account_key = input.get("expected_account_key", "")
if (
@@ -617,7 +639,7 @@ class GatewayHandler(BaseHTTPRequestHandler):
gateway.restore_proxy(alias, body)
return None
match = re.fullmatch(
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|resolve|login-qr|action|messages|events)",
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|image|identity|resolve|login-qr|action|messages|events)",
path,
)
if match:
@@ -628,6 +650,8 @@ class GatewayHandler(BaseHTTPRequestHandler):
return gateway.resolve_douyin(alias, body)
if action == "media" and method == "POST":
return gateway.get_douyin_media(alias, body)
if action == "image" and method == "POST":
return gateway.get_douyin_image(alias, body)
if action == "identity" and method == "POST":
return gateway.douyin_identity(alias, body)
if action == "login-qr" and method == "POST":
+47
View File
@@ -904,6 +904,53 @@ class BrowserTests(unittest.TestCase):
self.assertIn("querySelectorAll", cdp.expressions[-1])
self.assertIn("uuu_265.mp4", cdp.expressions[-1])
def test_image_download_accepts_douyinpic_and_rejects_other(self) -> None:
# 封面/头像走 douyinpic CDN:浏览器内 fetch + base64,无导航无签名。
payload = base64.b64encode(b"jpeg-bytes").decode("ascii")
cdp = BrowserCDP(
[
{"status": 200, "content_type": "image/jpeg", "body": payload},
]
)
browser = DouyinBrowser()
self._with_connection(browser, cdp)
response = browser.get_image(
"safe",
"https://p3-pc-sign.douyinpic.com/tos-cn-i-0813/cover.jpeg?x-expires=2105848800&x-signature=abc",
)
self.assertEqual(response.status, 200)
self.assertEqual(response.content_type, "image/jpeg")
self.assertEqual(base64.b64decode(response.body_base64), b"jpeg-bytes")
self.assertIn("fetch", cdp.expressions[-1])
cdp = BrowserCDP([])
self._with_connection(browser, cdp)
with self.assertRaisesRegex(DouyinError, "image target is invalid"):
browser.get_image("safe", "https://evil.example.com/cover.jpeg")
def test_image_download_rejects_non_image_content(self) -> None:
payload = base64.b64encode(b"<html>").decode("ascii")
cdp = BrowserCDP(
[
{"status": 200, "content_type": "text/html", "body": payload},
]
)
browser = DouyinBrowser()
self._with_connection(browser, cdp)
with self.assertRaisesRegex(DouyinError, "not an image"):
browser.get_image("safe", "https://p3-pc-sign.douyinpic.com/cover.jpeg")
def test_image_download_rejects_too_large(self) -> None:
cdp = BrowserCDP(
[
{"too_large": True},
]
)
browser = DouyinBrowser()
self._with_connection(browser, cdp)
with self.assertRaisesRegex(DouyinError, "image download failed"):
browser.get_image("safe", "https://p9-pc-sign.douyinpic.com/big.jpeg")
def test_connect_identity_and_actions(self) -> None:
target = [
{
+72
View File
@@ -461,6 +461,15 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto
}
return c.JSON(point)
})
app.Get("/api/creator/works/:id/cover", func(c fiber.Ctx) error {
contentType, data, err := store.GetWorkCover(c.Context(), c.Params("id"), "cover")
if err != nil {
return creatorError(c, err)
}
c.Set(fiber.HeaderContentType, contentType)
c.Set(fiber.HeaderCacheControl, "private, max-age=86400")
return c.Send(data)
})
app.Get("/api/creator/works/:id/material", func(c fiber.Ctx) error {
item, err := store.GetMaterial(c.Context(), c.Params("id"))
if err != nil {
@@ -1435,6 +1444,63 @@ func (browser creatorGatewayBrowser) Get(ctx context.Context, target string) (do
return douyin.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil
}
// GetImage 经网关浏览器内 fetch 拉取图片(封面等静态资源),返回 content_type 与字节。
func (browser creatorGatewayBrowser) GetImage(ctx context.Context, target string) (string, []byte, error) {
payload := gatewayGenerationPayload(browser.environment)
payload["url"] = target
status, body, err := gatewayCallWithLimit(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/image", payload, 60*time.Second, largeGatewayResponseLimit)
if err != nil {
return "", nil, err
}
if status != http.StatusOK {
return "", nil, fmt.Errorf("douyin image request rejected with HTTP %d: %s", status, string(body))
}
var response struct {
Status int `json:"status"`
ContentType string `json:"content_type"`
BodyBase64 string `json:"body_base64"`
}
if err := json.Unmarshal(body, &response); err != nil {
return "", nil, fmt.Errorf("decode douyin image response: %w", err)
}
contentType := strings.ToLower(strings.TrimSpace(response.ContentType))
if response.Status < 200 || response.Status >= 300 || !strings.HasPrefix(contentType, "image/") {
return "", nil, fmt.Errorf("douyin image response is not an image (HTTP %d, content_type %q)", response.Status, response.ContentType)
}
data, err := base64.StdEncoding.DecodeString(response.BodyBase64)
if err != nil {
return "", nil, fmt.Errorf("decode douyin image bytes: %w", err)
}
if len(data) > maxCreatorCoverBytes {
return "", nil, fmt.Errorf("douyin image response is too large")
}
return contentType, data, nil
}
const maxCreatorCoverBytes = 4 << 20
// cacheCompetitorWorkCovers 把竞品作品封面拉取到本地缓存(带签名的 douyinpic URL 会过期,前端读本地缓存)。
// 单次同步最多回填 40 张,超出部分留待后续同步自愈,避免首次采集时长时间占用匿名浏览器。
func cacheCompetitorWorkCovers(ctx context.Context, store *creator.Store, browser creatorGatewayBrowser, competitorID string) error {
works, err := store.ListWorksMissingCover(ctx, competitorID)
if err != nil {
return err
}
for _, work := range works {
contentType, data, fetchErr := browser.GetImage(ctx, work.CoverURL)
if fetchErr != nil {
return fmt.Errorf("fetch cover for work %s: %w", work.ID, fetchErr)
}
if saveErr := store.SaveWorkCover(ctx, work.ID, "cover", contentType, data); saveErr != nil {
return fmt.Errorf("save cover for work %s: %w", work.ID, saveErr)
}
}
if len(works) > 0 {
logrus.WithFields(logrus.Fields{"competitor_id": competitorID, "covers_cached": len(works)}).Info("creator competitor covers cached")
}
return nil
}
func (browser creatorGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) {
payload := gatewayGenerationPayload(browser.environment)
payload["url"] = target
@@ -1878,6 +1944,12 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, h
collectionNow = competitor.NextSyncAt.UTC()
}
report, collectErr := store.CollectSource(ctx, competitor.Platform, creator.SourceCompetitor, competitor.ID, collector, collectionNow)
if collectErr == nil {
// 采集成功后把缺封面的作品封面缓存到本地(失败只记日志,不影响本次同步结果)。
if coverErr := cacheCompetitorWorkCovers(ctx, store, creatorGatewayBrowser{gateway: lease.gateway, environment: lease.environment}, competitor.ID); coverErr != nil {
logrus.WithError(coverErr).WithField("competitor_id", competitorID).Warn("creator competitor cover cache failed")
}
}
if closeErr := lease.close(); closeErr != nil {
collectErr = errors.Join(collectErr, closeErr)
}
@@ -37,6 +37,7 @@ func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gatew
func douyinCollector(browser creatorGatewayBrowser, accountKey, sourceType, sourceID string) douyin.CreatorCollector {
return douyin.CreatorCollector{Browser: browser, AccountKey: accountKey, SourceType: sourceType, SourceID: sourceID}
}
// validateDouyinCompetitor 监控账号入队前的平台一致性校验(平台收敛后仅抖音)。
func validateDouyinCompetitor(input creator.CompetitorInput) error {
if input.Platform != creator.PlatformDouyin {
+56
View File
@@ -593,6 +593,30 @@ func (s *Store) ListWorks(ctx context.Context, filter WorkFilter) ([]Work, error
return result, nil
}
// ListWorksMissingCover 返回已采集到远程封面但本地尚无缓存的作品(用于封面回填)。
// 单次返回上限 40 条,超出部分由后续同步继续回填。
func (s *Store) ListWorksMissingCover(ctx context.Context, sourceID string) ([]Work, error) {
if sourceID == "" {
return nil, ErrInvalid
}
rows, err := s.db.QueryContext(ctx, workSelect+` WHERE source_type = $1 AND source_id = $2 AND cover_url <> ''
AND NOT EXISTS (SELECT 1 FROM creator_work_cover c WHERE c.work_id = creator_work.id AND c.variant = 'cover')
ORDER BY created_at DESC LIMIT 40`, SourceCompetitor, sourceID)
if err != nil {
return nil, databaseError(err)
}
defer rows.Close()
result := make([]Work, 0)
for rows.Next() {
item, err := scanWork(rows)
if err != nil {
return nil, err
}
result = append(result, item)
}
return result, rows.Err()
}
func (s *Store) RecordMetric(ctx context.Context, input MetricInput, settings Settings, now time.Time) (MetricPoint, error) {
if input.WorkID == "" || input.CollectedAt.IsZero() || input.Likes != nil && *input.Likes < 0 || input.CommentsCount != nil && *input.CommentsCount < 0 || input.Shares != nil && *input.Shares < 0 || input.CollectCount != nil && *input.CollectCount < 0 || input.PlayCount != nil && *input.PlayCount < 0 {
return MetricPoint{}, ErrInvalid
@@ -1002,6 +1026,38 @@ func (s *Store) GetCommentByKey(ctx context.Context, platform, commentKey string
return result, rowError(err)
}
// SaveWorkCover 缓存作品封面图(带签名的 douyinpic URL 会过期,存字节后经本地接口展示)。
func (s *Store) SaveWorkCover(ctx context.Context, workID, variant, contentType string, data []byte) error {
if workID == "" || variant == "" || contentType == "" || len(data) == 0 || len(data) > maxWorkCoverBytes {
return ErrInvalid
}
_, err := s.db.ExecContext(ctx, `
INSERT INTO creator_work_cover (work_id, variant, content_type, data, fetched_at)
VALUES ($1, $2, $3, $4, now())
ON CONFLICT (work_id, variant) DO UPDATE SET content_type = EXCLUDED.content_type, data = EXCLUDED.data, fetched_at = now()`,
workID, variant, contentType, data)
if err != nil {
return databaseError(err)
}
return nil
}
func (s *Store) GetWorkCover(ctx context.Context, workID, variant string) (contentType string, data []byte, err error) {
if workID == "" || variant == "" {
return "", nil, ErrInvalid
}
err = s.db.QueryRowContext(ctx, `SELECT content_type, data FROM creator_work_cover WHERE work_id = $1 AND variant = $2`, workID, variant).Scan(&contentType, &data)
if errors.Is(err, sql.ErrNoRows) {
return "", nil, ErrNotFound
}
if err != nil {
return "", nil, databaseError(err)
}
return contentType, data, nil
}
const maxWorkCoverBytes = 4 << 20
func pageBounds(page, pageSize int) (int, int, error) {
if page < 1 || pageSize < 1 || pageSize > 100 {
return 0, 0, ErrInvalid
@@ -2,10 +2,64 @@ package creator
import (
"errors"
"fmt"
"testing"
"time"
)
func TestCreatorPostgresWorkCoverCache(t *testing.T) {
store, _, ctx := openCreatorIntegrationStore(t)
stamp := fmt.Sprintf("%d", time.Now().UnixNano())
competitor, err := store.UpsertCompetitor(ctx, CompetitorInput{
Platform: PlatformDouyin, PlatformAccountKey: "MS4wLjABAAAAcover" + stamp, Nickname: "温江菜花小院",
HomepageURL: "https://www.douyin.com/user/cover" + stamp,
})
if err != nil {
t.Fatalf("upsert competitor: %v", err)
}
now := time.Now().UTC().Add(-time.Hour)
work, inserted, err := store.UpsertWork(ctx, WorkInput{
Platform: PlatformDouyin, WorkKey: "cover-work-" + stamp, SourceType: SourceCompetitor, SourceID: competitor.ID,
AuthorName: "木子不写代码", Title: "标题", PublishedAt: &now, PublishedAtStatus: "verified",
CoverURL: "https://p3-pc-sign.douyinpic.com/cover.jpeg",
}, now)
if err != nil || !inserted {
t.Fatalf("upsert work: inserted=%v err=%v", inserted, err)
}
// 采集到封面 URL 但尚未缓存 → 出现在回填列表。
missing, err := store.ListWorksMissingCover(ctx, competitor.ID)
if err != nil || len(missing) != 1 || missing[0].ID != work.ID {
t.Fatalf("list missing covers: works=%+v err=%v", missing, err)
}
if err := store.SaveWorkCover(ctx, work.ID, "cover", "image/jpeg", []byte("jpeg-bytes")); err != nil {
t.Fatalf("save cover: %v", err)
}
contentType, data, err := store.GetWorkCover(ctx, work.ID, "cover")
if err != nil || contentType != "image/jpeg" || string(data) != "jpeg-bytes" {
t.Fatalf("get cover: contentType=%q data=%q err=%v", contentType, data, err)
}
// 缓存后不再出现在回填列表。
missing, err = store.ListWorksMissingCover(ctx, competitor.ID)
if err != nil || len(missing) != 0 {
t.Fatalf("missing covers after cache: works=%+v err=%v", missing, err)
}
// 重复写入覆盖旧内容(新采集周期封面可能更新)。
if err := store.SaveWorkCover(ctx, work.ID, "cover", "image/webp", []byte("webp-bytes")); err != nil {
t.Fatalf("overwrite cover: %v", err)
}
contentType, data, err = store.GetWorkCover(ctx, work.ID, "cover")
if err != nil || contentType != "image/webp" || string(data) != "webp-bytes" {
t.Fatalf("overwritten cover: contentType=%q data=%q err=%v", contentType, data, err)
}
if _, _, err := store.GetWorkCover(ctx, "missing", "cover"); !errors.Is(err, ErrNotFound) {
t.Fatalf("missing cover must be ErrNotFound: err=%v", err)
}
// 非法输入拒绝。
if err := store.SaveWorkCover(ctx, work.ID, "cover", "image/jpeg", nil); !errors.Is(err, ErrInvalid) {
t.Fatalf("empty cover data must be rejected: err=%v", err)
}
}
func TestCreatorPostgresPageAndConversationState(t *testing.T) {
store, phaseAStore, ctx := openCreatorIntegrationStore(t)
if err := store.Ping(ctx); err != nil {
@@ -0,0 +1,10 @@
-- 作品封面本地缓存(2026-09-28 竞品页走查):封面图存本地 cover 表,按 (work_id, variant) 唯一。
-- 原始 douyinpic URL 带签名会过期,前端改为经 /api/creator/works/:id/cover 读取本地缓存。
CREATE TABLE IF NOT EXISTS creator_work_cover (
work_id text NOT NULL REFERENCES creator_work(id) ON DELETE CASCADE,
variant text NOT NULL,
content_type text NOT NULL,
data bytea NOT NULL,
fetched_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (work_id, variant)
);
+4
View File
@@ -95,6 +95,9 @@ var migration040 string
//go:embed migrations/041_creator_account_metric.sql
var migration041 string
//go:embed migrations/042_work_cover_cache.sql
var migration042 string
type SecretReference struct {
ID string
Provider string
@@ -197,6 +200,7 @@ func (s *Store) migrate(ctx context.Context) error {
{version: 39, sql: migration039},
{version: 40, sql: migration040},
{version: 41, sql: migration041},
{version: 42, sql: migration042},
}
for _, migration := range migrations {
var applied bool
+61 -1
View File
@@ -440,7 +440,15 @@ type worksEnvelope struct {
ID string `json:"aweme_id"`
Description string `json:"desc"`
CreatedAt *int64 `json:"create_time"`
Statistics *struct {
Author *struct {
Nickname string `json:"nickname"`
} `json:"author"`
Video *struct {
Cover *struct {
URLList []string `json:"url_list"`
} `json:"cover"`
} `json:"video"`
Statistics *struct {
DiggCount *int64 `json:"digg_count"`
CommentCount *int64 `json:"comment_count"`
ShareCount *int64 `json:"share_count"`
@@ -474,6 +482,58 @@ func parseWorksPage(body []byte) ([]Work, bool, *int64, bool) {
return works, bool(*envelope.HasMore), envelope.MaxCursor, true
}
func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) {
// 作品项携带播放信息等大字段,单页真实响应可达 2MB+(见网关 RESPONSE_LIMIT 同步放宽)。
if len(body) > 8<<20 {
return nil, false, nil, false
}
var envelope worksEnvelope
// 平台风控语义(2026 年实测,与 TikTokDownloader #706 等公开项目一致):
// - 匿名会话第一页正常(16-20 条,has_more=1 且带 max_cursor);
// - 第二页起一律返回空 envelope {"status_code":0}(无 aweme_list/has_more/max_cursor),
// 登录 Cookie 也无法翻页。无 aweme_list 的空 envelope 视为正常空页(分页终止),
// 而非无效响应;只有响应同时带 max_cursor 时才继续翻页。
if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore != nil && envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 {
return nil, false, nil, false
}
items := make([]creatorWorkPageItem, 0, len(envelope.Works))
seen := make(map[string]struct{}, len(envelope.Works))
for _, candidate := range envelope.Works {
if !keyPattern.MatchString(candidate.ID) {
return nil, false, nil, false
}
if _, exists := seen[candidate.ID]; exists {
return nil, false, nil, false
}
seen[candidate.ID] = struct{}{}
var likes, comments, shares, collects, plays *int64
if candidate.Statistics != nil {
likes, comments, shares, collects, plays = candidate.Statistics.DiggCount, candidate.Statistics.CommentCount, candidate.Statistics.ShareCount, candidate.Statistics.CollectCount, candidate.Statistics.PlayCount
}
for _, value := range []*int64{likes, comments, shares, collects, plays} {
if value != nil && *value < 0 {
return nil, false, nil, false
}
}
createdAt := candidate.CreatedAt
createdAtInvalid := createdAt != nil && *createdAt <= 0
if createdAtInvalid {
createdAt = nil
}
item := creatorWorkPageItem{ID: candidate.ID, Description: candidate.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares, CollectCount: collects, PlayCount: plays}
if candidate.Author != nil {
item.AuthorName = candidate.Author.Nickname
}
if candidate.Video != nil && candidate.Video.Cover != nil && len(candidate.Video.Cover.URLList) > 0 {
item.CoverURL = candidate.Video.Cover.URLList[0]
}
items = append(items, item)
}
// 仅当响应带有效 max_cursor 时才声明翻页;无 cursor 的 has_more 不产生翻页游标。
hasMore := envelope.HasMore != nil && bool(*envelope.HasMore) && envelope.MaxCursor != nil
return items, hasMore, envelope.MaxCursor, true
}
func parseWorks(body []byte) ([]Work, bool, bool) {
works, hasMore, _, ok := parseWorksPage(body)
return works, hasMore, ok
+3 -46
View File
@@ -263,7 +263,7 @@ func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor stri
} else if published != nil {
status = "verified"
}
items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, Likes: likes, CommentsCount: comments, Shares: shares, CollectCount: collects, PlayCount: plays})
items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, AuthorName: work.AuthorName, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, CoverURL: work.CoverURL, Likes: likes, CommentsCount: comments, Shares: shares, CollectCount: collects, PlayCount: plays})
}
page := creator.WorkPage{Items: items, HasMore: hasMore}
if nextCursor != nil {
@@ -339,8 +339,10 @@ func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) {
type creatorWorkPageItem struct {
ID string
Description string
AuthorName string
CreatedAt *int64
CreatedAtInvalid bool
CoverURL string
DiggCount *int64
CommentCount *int64
ShareCount *int64
@@ -348,51 +350,6 @@ type creatorWorkPageItem struct {
PlayCount *int64
}
func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) {
// 作品项携带播放信息等大字段,单页真实响应可达 2MB+(见网关 RESPONSE_LIMIT 同步放宽)。
if len(body) > 8<<20 {
return nil, false, nil, false
}
var envelope worksEnvelope
// 平台风控语义(2026 年实测,与 TikTokDownloader #706 等公开项目一致):
// - 匿名会话第一页正常(16-20 条,has_more=1 且带 max_cursor);
// - 第二页起一律返回空 envelope {"status_code":0}(无 aweme_list/has_more/max_cursor),
// 登录 Cookie 也无法翻页。无 aweme_list 的空 envelope 视为正常空页(分页终止),
// 而非无效响应;只有响应同时带 max_cursor 时才继续翻页。
if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore != nil && envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 {
return nil, false, nil, false
}
items := make([]creatorWorkPageItem, 0, len(envelope.Works))
seen := make(map[string]struct{}, len(envelope.Works))
for _, item := range envelope.Works {
if !keyPattern.MatchString(item.ID) {
return nil, false, nil, false
}
if _, exists := seen[item.ID]; exists {
return nil, false, nil, false
}
seen[item.ID] = struct{}{}
var likes, comments, shares, collects, plays *int64
if item.Statistics != nil {
likes, comments, shares, collects, plays = item.Statistics.DiggCount, item.Statistics.CommentCount, item.Statistics.ShareCount, item.Statistics.CollectCount, item.Statistics.PlayCount
}
for _, value := range []*int64{likes, comments, shares, collects, plays} {
if value != nil && *value < 0 {
return nil, false, nil, false
}
}
createdAt := item.CreatedAt
createdAtInvalid := createdAt != nil && *createdAt <= 0
if createdAtInvalid {
createdAt = nil
}
items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares, CollectCount: collects, PlayCount: plays})
}
// 仅当响应带有效 max_cursor 时才声明翻页;无 cursor 的 has_more 不产生翻页游标。
hasMore := envelope.HasMore != nil && bool(*envelope.HasMore) && envelope.MaxCursor != nil
return items, hasMore, envelope.MaxCursor, true
}
type commentEnvelope struct {
StatusCode *int `json:"status_code"`
HasMore *douyinBool `json:"has_more"`
@@ -127,6 +127,29 @@ func TestParseCreatorWorksPageMarksInvalidTimestamp(t *testing.T) {
}
}
func TestParseCreatorWorksPageExtractsAuthorAndCover(t *testing.T) {
// 匿名 works 响应实测:author.nickname 与 video.cover.url_list[0] 都在作品项上。
body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"first page","create_time":1700000000,"author":{"nickname":"温江菜花小院"},"video":{"cover":{"url_list":["https://p3-pc-sign.douyinpic.com/cover.jpeg","https://p9-pc-sign.douyinpic.com/backup.jpeg"]}},"statistics":{"digg_count":1,"comment_count":2,"share_count":3,"collect_count":4,"play_count":55}}]}`)
works, _, _, ok := parseCreatorWorksPage(body)
if !ok || len(works) != 1 {
t.Fatalf("page not parsed: ok=%v works=%+v", ok, works)
}
if works[0].AuthorName != "温江菜花小院" {
t.Fatalf("author name missing: %+v", works[0])
}
if works[0].CoverURL != "https://p3-pc-sign.douyinpic.com/cover.jpeg" {
t.Fatalf("cover url missing: %+v", works[0])
}
}
func TestParseCreatorWorksPageToleratesMissingAuthorAndCover(t *testing.T) {
body := []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"123","desc":"no media","create_time":1700000000,"statistics":{"digg_count":1}}]}`)
works, _, _, ok := parseCreatorWorksPage(body)
if !ok || len(works) != 1 || works[0].AuthorName != "" || works[0].CoverURL != "" {
t.Fatalf("missing author/cover must be tolerated: ok=%v works=%+v", ok, works)
}
}
func TestParseCreatorWorksPageAcceptsNumericHasMore(t *testing.T) {
body := []byte(`{"status_code":0,"has_more":0,"aweme_list":[]}`)
works, hasMore, cursor, ok := parseCreatorWorksPage(body)
+39 -22
View File
@@ -139,28 +139,45 @@ export default function Page() {
return (
<Flex vertical gap={16}>
<Card title="账号画像">
<Descriptions
column={{ xs: 1, sm: 2, md: 3 }}
size="small"
items={[
{ key: 'key', label: '平台账号', children: competitor.platform_account_key },
{ key: 'uid', label: '抖音号', children: competitor.unique_id || '—' },
{ key: 'follower', label: '粉丝', children: formatCount(competitor.follower_count) },
{ key: 'following', label: '关注', children: formatCount(competitor.following_count) },
{ key: 'aweme', label: '作品总数', children: formatCount(competitor.aweme_count) },
{ key: 'tags', label: '标签', children: competitor.tags?.length ? competitor.tags.join('、') : '无' },
{ key: 'lastSync', label: '上次同步', children: competitor.last_sync_at ? dateTime(competitor.last_sync_at) : '—' },
{ key: 'nextSync', label: '下次自动同步', children: competitor.next_sync_at ? dateTime(competitor.next_sync_at) : '—' },
{
key: 'syncError',
label: '同步异常',
children: competitor.sync_error ? (
<Typography.Text type="danger" style={{ fontSize: 12 }}>{competitor.sync_error}</Typography.Text>
) : (
'—'
) },
]}
/>
<Flex gap={24} align="flex-start" wrap="wrap">
<Avatar size={100} shape="square" src={competitor.avatar_url || undefined} style={{ flexShrink: 0 }}>
{(competitor.nickname || competitor.platform_account_key).slice(0, 1).toUpperCase()}
</Avatar>
<Flex vertical gap={4} style={{ flex: 1, minWidth: 260 }}>
<Flex align="center" gap={8} wrap="wrap">
<Typography.Title level={4} style={{ marginBottom: 0 }}>
{competitor.nickname || competitor.platform_account_key}
</Typography.Title>
<Tag color={syncMeta.color}>{syncMeta.label}</Tag>
</Flex>
<Typography.Text type="secondary">
{competitor.unique_id || competitor.platform_account_key} · {platformLabel(competitor.platform)}
</Typography.Text>
<Descriptions
column={{ xs: 1, sm: 2 }}
size="small"
style={{ marginTop: 8 }}
items={[
{ key: 'key', label: '平台账号', children: competitor.platform_account_key },
{ key: 'uid', label: '抖音号', children: competitor.unique_id || '—' },
{ key: 'follower', label: '粉丝', children: formatCount(competitor.follower_count) },
{ key: 'following', label: '关注', children: formatCount(competitor.following_count) },
{ key: 'aweme', label: '作品总数', children: formatCount(competitor.aweme_count) },
{ key: 'tags', label: '标签', children: competitor.tags?.length ? competitor.tags.join('、') : '无' },
{ key: 'lastSync', label: '上次同步', children: competitor.last_sync_at ? dateTime(competitor.last_sync_at) : '—' },
{ key: 'nextSync', label: '下次自动同步', children: competitor.next_sync_at ? dateTime(competitor.next_sync_at) : '—' },
{
key: 'syncError',
label: '同步异常',
children: competitor.sync_error ? (
<Typography.Text type="danger" style={{ fontSize: 12 }}>{competitor.sync_error}</Typography.Text>
) : (
'—'
) },
]}
/>
</Flex>
</Flex>
</Card>
<WorksGallery sourceId={competitor.id} sourceType="competitor" />
</Flex>
+62 -46
View File
@@ -1,11 +1,11 @@
// 竞品分析:作品列表对齐 ant-design-pro card-list——antd Row/Col 响应式栅格 + Card 原样组合;
// 作品详情与素材处理均走 Drawer;日期筛选用 antd DatePicker;筛选自动刷新、手动翻页不变。
// 平台筛选已随 migration 038(platform 收敛 douyin)移除。
// 封面读本地缓存(/works/:id/cover,认证 fetch + blob);发布时间状态详情见 publishedAtStatusMeta 注释。
import { useEffect, useState } from 'react';
import { CommentOutlined, LikeOutlined, ShareAltOutlined } from '@ant-design/icons';
import { Alert, App, Button, Card, Col, DatePicker, Drawer, Empty, Flex, Input, InputNumber, Modal, Row, Select, Space, Spin, Table, Tag, Typography } from 'antd';
import { Alert, App, Button, Card, Col, DatePicker, Drawer, Empty, Flex, Input, InputNumber, Modal, Row, Select, Space, Spin, Table, Tag, Tooltip, Typography } from 'antd';
import type { Dayjs } from 'dayjs';
import { creatorAction, creatorGet, creatorUpdate, getList } from '@/services/api';
import { creatorAction, creatorGet, creatorUpdate, fetchCreatorBinary, getList } from '@/services/api';
import { conflictMessage, dateTime } from '@/utils/helpers';
const publishedAtStatusOptions = [
@@ -15,11 +15,12 @@ const publishedAtStatusOptions = [
{ value: 'invalid', label: '无效时间' },
];
const publishedAtStatusMeta: Record<string, { color?: string; label: string }> = {
verified: { color: 'green', label: '已核验' },
future: { color: 'orange', label: '未来时间' },
pending_verification: { label: '待核验' },
invalid: { color: 'red', label: '无效时间' } };
const publishedAtStatusMeta: Record<string, { color?: string; label: string; hint?: string }> = {
// verified:发布时间从平台原始数据中成功解析且已核验(非未来时间、非无效值)。
verified: { color: 'green', label: '已核验', hint: '发布时间已从平台数据成功解析核验' },
future: { color: 'orange', label: '未来时间', hint: '发布时间晚于当前时间,可能是定时发布' },
pending_verification: { label: '待核验', hint: '平台未返回发布时间,待后续采集补齐' },
invalid: { color: 'red', label: '无效时间', hint: '平台返回的发布时间无效(如 0 或负值)' } };
interface Filters {
minLikes?: number;
@@ -32,7 +33,36 @@ interface Filters {
function statusTag(status?: string) {
const meta = publishedAtStatusMeta[status || ''] || { label: status || '—' };
return <Tag color={meta.color}>{meta.label}</Tag>;
const tag = <Tag color={meta.color}>{meta.label}</Tag>;
return meta.hint ? <Tooltip title={meta.hint}>{tag}</Tooltip> : tag;
}
// 作品封面:从本地缓存接口读取(认证 fetch + blob URL);缓存缺失时展示占位。
function WorkCover({ workID, title }: { workID: string; title?: string }) {
const [src, setSrc] = useState<string | null>(null);
const [missing, setMissing] = useState(false);
useEffect(() => {
let objectURL: string | null = null;
setSrc(null);
setMissing(false);
fetchCreatorBinary(`/creator/works/${encodeURIComponent(workID)}/cover`)
.then((blob) => {
objectURL = URL.createObjectURL(blob);
setSrc(objectURL);
})
.catch(() => setMissing(true));
return () => {
if (objectURL) URL.revokeObjectURL(objectURL);
};
}, [workID]);
if (src) {
return <img src={src} alt={title || '作品封面'} style={{ width: '100%', aspectRatio: '16 / 9', objectFit: 'cover' }} />;
}
return (
<Flex align="center" justify="center" style={{ aspectRatio: '16 / 9', background: '#f5f5f5' }}>
<Typography.Text type="secondary" style={{ fontSize: 12 }}>{missing ? '封面未缓存' : '封面加载中'}</Typography.Text>
</Flex>
);
}
function stepTag(status: string, label: string) {
@@ -241,13 +271,10 @@ export default function Page() {
/>
</Space>
</Card>
<Card title="作品与指标">
<Card title="作品与指标" extra={<Typography.Text type="secondary">共 {workPageInfo.total} 个作品</Typography.Text>}>
{error ? (
<Alert type="error" showIcon style={{ marginBottom: 16 }} title={error.message} action={<Button size="small" onClick={load}>重试</Button>} />
) : null}
<Flex justify="space-between" align="center" style={{ marginBottom: 16 }}>
<Typography.Text type="secondary">共 {workPageInfo.total} 个作品</Typography.Text>
</Flex>
<Spin spinning={pending}>
{works.length ? (
<Row gutter={[16, 16]}>
@@ -255,15 +282,8 @@ export default function Page() {
<Col key={work.id} xs={24} sm={12} lg={8} xl={6}>
<Card
style={{ height: '100%' }}
cover={
work.cover_url ? (
<img src={work.cover_url} alt={work.title || '作品封面'} style={{ width: '100%', aspectRatio: '16 / 9', objectFit: 'cover' }} />
) : (
<Flex align="center" justify="center" style={{ aspectRatio: '16 / 9', background: '#f5f5f5' }}>
<Typography.Text type="secondary" style={{ fontSize: 12 }}>无封面</Typography.Text>
</Flex>
)
}
styles={{ body: { padding: '0 0 12px' } }}
cover={<WorkCover workID={work.id} title={work.title} />}
actions={[
<Button key="detail" type="link" size="small" onClick={() => openWorkDetail(work.id)} disabled={workDetailPending}>
详情
@@ -279,29 +299,27 @@ export default function Page() {
...(work.original_url ? [<Typography.Link key="origin" href={work.original_url} target="_blank">原文</Typography.Link>] : []),
]}
>
<Flex vertical gap={8} style={{ minWidth: 0 }}>
<Typography.Paragraph strong ellipsis={{ rows: 2 }} style={{ marginBottom: 0 }}>
{work.title || '无标题'}
</Typography.Paragraph>
<Typography.Text type="secondary" style={{ fontSize: 12 }} ellipsis>
{work.author_name || work.source_id}
</Typography.Text>
<Space size={8} wrap>
{statusTag(work.published_at_status)}
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
{work.published_at ? dateTime(work.published_at) : '—'}
</Typography.Text>
</Space>
<Flex gap={16}>
<Typography.Text style={{ fontSize: 12 }}><LikeOutlined /> {work.likes ?? '—'}</Typography.Text>
<Typography.Text style={{ fontSize: 12 }}><CommentOutlined /> {work.comments_count ?? '—'}</Typography.Text>
<Typography.Text style={{ fontSize: 12 }}><ShareAltOutlined /> {work.shares ?? '—'}</Typography.Text>
</Flex>
{work.next_metric_at || work.metric_stop_reason ? (
<div style={{ padding: '12px 16px 0' }}>
<Flex vertical gap={8} style={{ minWidth: 0 }}>
<Typography.Paragraph strong ellipsis={{ rows: 2 }} style={{ marginBottom: 0 }}>
{work.title || '无标题'}
</Typography.Paragraph>
<Typography.Text type="secondary" style={{ fontSize: 12 }} ellipsis>
{work.next_metric_at ? `下次采集 ${dateTime(work.next_metric_at)}` : work.metric_stop_reason}
作者:{work.author_name || '—'}
</Typography.Text>
) : null}
<Space size={8} wrap>
{statusTag(work.published_at_status)}
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
{work.published_at ? dateTime(work.published_at) : '—'}
</Typography.Text>
</Space>
</Flex>
</div>
{/* 点赞/评论/分享全宽贴在封面下方(body padding 置 0,本行自带上下边框) */}
<Flex gap={16} justify="space-between" style={{ margin: '10px 0 0', padding: '8px 16px', borderTop: '1px solid #f0f0f0', borderBottom: '1px solid #f0f0f0' }}>
<Typography.Text style={{ fontSize: 12 }}><LikeOutlined /> {work.likes ?? '—'}</Typography.Text>
<Typography.Text style={{ fontSize: 12 }}><CommentOutlined /> {work.comments_count ?? '—'}</Typography.Text>
<Typography.Text style={{ fontSize: 12 }}><ShareAltOutlined /> {work.shares ?? '—'}</Typography.Text>
</Flex>
</Card>
</Col>
@@ -395,9 +413,7 @@ export default function Page() {
<Drawer open={Boolean(workDetail)} onClose={() => setWorkDetail(null)} size="large" title="作品详情">
{workDetail ? (
<Flex vertical gap={12}>
{workDetail.cover_url ? (
<img src={workDetail.cover_url} alt="作品封面" style={{ maxHeight: 256, borderRadius: 4, border: '1px solid #f0f0f0' }} />
) : null}
<WorkCover workID={workDetail.id} title={workDetail.title || '作品封面'} />
<Typography.Text type="secondary" style={{ fontSize: 12 }}>ID:{workDetail.work_key || workDetail.id}</Typography.Text>
<Typography.Paragraph style={{ whiteSpace: 'pre-wrap', marginBottom: 0 }}>{workDetail.body || workDetail.title || '暂无正文'}</Typography.Paragraph>
<Space wrap>
+14
View File
@@ -137,6 +137,20 @@ export function creatorRequest(path: string, options?: RequestInit) {
return request(path, options);
}
// 二进制资源(如作品封面缓存):与 creatorSubscribe 同样的认证 fetch,返回 Blob 供 objectURL 展示。
export async function fetchCreatorBinary(path: string): Promise<Blob> {
const auth = localStorage.getItem(AUTH_STORAGE_KEY);
const headers: Record<string, string> = {};
if (auth) headers.Authorization = `Basic ${btoa(auth)}`;
const response = await fetch(`/api${path}`, { headers });
if (response.status === 401) unauthorized();
if (!response.ok) {
const body = await response.json().catch(() => ({}));
throw new RequestError(body.error || `请求失败 (${response.status})`, response.status, body);
}
return response.blob();
}
export function creatorAction(path: string, data?: unknown) {
return request(path, data === undefined ? { method: 'POST' } : jsonOptions('POST', data));
}