410 lines
15 KiB
Go
410 lines
15 KiB
Go
package douyin
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
|
)
|
|
|
|
const (
|
|
commentsEndpoint = "https://www.douyin.com/aweme/v1/web/comment/list/"
|
|
profileOtherEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/other/"
|
|
)
|
|
|
|
var workKeyPattern = regexp.MustCompile(`^[1-9][0-9]{0,63}$`)
|
|
|
|
type CreatorCollector struct {
|
|
Browser Browser
|
|
AccountKey string
|
|
SourceType string
|
|
SourceID string
|
|
}
|
|
|
|
type TargetProfile struct {
|
|
UID string
|
|
SecUID string
|
|
UniqueID string
|
|
Nickname string
|
|
AvatarURL string
|
|
Signature string
|
|
FollowerCount *int64
|
|
FollowingCount *int64
|
|
TotalFavorited *int64
|
|
AwemeCount *int64
|
|
}
|
|
|
|
// WorkStatistics 是作品详情接口返回的指标快照，用于增长曲线时序采集。
|
|
type WorkStatistics struct {
|
|
Likes *int64
|
|
CommentsCount *int64
|
|
Shares *int64
|
|
CollectCount *int64
|
|
PlayCount *int64
|
|
}
|
|
|
|
// FetchWorkStatistics 经网关受限 fetch 拉取作品详情并解析指标快照。
|
|
// 作品指标是公开数据，匿名浏览器即可访问（实测 detail 返回完整 statistics）。
|
|
func FetchWorkStatistics(ctx context.Context, browser Browser, workKey string) (WorkStatistics, error) {
|
|
if browser == nil || !workKeyPattern.MatchString(workKey) {
|
|
return WorkStatistics{}, fmt.Errorf("%w: invalid work statistics request", ErrInvalid)
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("aweme_id", workKey)
|
|
response, err := browser.Get(ctx, workDetailEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return WorkStatistics{}, err
|
|
}
|
|
if err := creatorResponseError(response, "work detail"); err != nil {
|
|
return WorkStatistics{}, err
|
|
}
|
|
stats, ok := parseWorkStatistics(response.Body, workKey)
|
|
if !ok {
|
|
return WorkStatistics{}, fmt.Errorf("%w: invalid douyin work detail response (status=%d body_len=%d head=%.160s)", ErrInvalid, response.Status, len(response.Body), response.Body)
|
|
}
|
|
return stats, nil
|
|
}
|
|
|
|
func parseWorkStatistics(body []byte, expectedWorkKey string) (WorkStatistics, bool) {
|
|
var envelope workDetailEnvelope
|
|
if len(body) > 2<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 ||
|
|
envelope.AwemeDetail == nil || envelope.AwemeDetail.AwemeID != expectedWorkKey || envelope.AwemeDetail.Statistics == nil {
|
|
return WorkStatistics{}, false
|
|
}
|
|
stats := envelope.AwemeDetail.Statistics
|
|
for _, value := range []*int64{stats.DiggCount, stats.CommentCount, stats.ShareCount, stats.CollectCount, stats.PlayCount} {
|
|
if value != nil && *value < 0 {
|
|
return WorkStatistics{}, false
|
|
}
|
|
}
|
|
if stats.DiggCount == nil && stats.CommentCount == nil && stats.ShareCount == nil && stats.CollectCount == nil && stats.PlayCount == nil {
|
|
return WorkStatistics{}, false
|
|
}
|
|
return WorkStatistics{Likes: stats.DiggCount, CommentsCount: stats.CommentCount, Shares: stats.ShareCount, CollectCount: stats.CollectCount, PlayCount: stats.PlayCount}, true
|
|
}
|
|
|
|
type workDetailEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
AwemeDetail *struct {
|
|
AwemeID string `json:"aweme_id"`
|
|
Author *struct {
|
|
UID string `json:"uid"`
|
|
SecUID string `json:"sec_uid"`
|
|
UniqueID string `json:"unique_id"`
|
|
Nickname string `json:"nickname"`
|
|
AvatarThumb *struct {
|
|
URLList []string `json:"url_list"`
|
|
} `json:"avatar_thumb"`
|
|
} `json:"author"`
|
|
Statistics *struct {
|
|
DiggCount *int64 `json:"digg_count"`
|
|
CommentCount *int64 `json:"comment_count"`
|
|
ShareCount *int64 `json:"share_count"`
|
|
CollectCount *int64 `json:"collect_count"`
|
|
PlayCount *int64 `json:"play_count"`
|
|
} `json:"statistics"`
|
|
} `json:"aweme_detail"`
|
|
}
|
|
|
|
func parseWorkTarget(body []byte, expectedWorkKey string) (TargetProfile, bool) {
|
|
var envelope workDetailEnvelope
|
|
if len(body) > 2<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 ||
|
|
envelope.AwemeDetail == nil || envelope.AwemeDetail.AwemeID != expectedWorkKey || envelope.AwemeDetail.Author == nil ||
|
|
!keyPattern.MatchString(envelope.AwemeDetail.Author.UID) || !keyPattern.MatchString(envelope.AwemeDetail.Author.SecUID) ||
|
|
(envelope.AwemeDetail.Author.UniqueID != "" && !keyPattern.MatchString(envelope.AwemeDetail.Author.UniqueID)) {
|
|
return TargetProfile{}, false
|
|
}
|
|
avatarURL := ""
|
|
if envelope.AwemeDetail.Author.AvatarThumb != nil && len(envelope.AwemeDetail.Author.AvatarThumb.URLList) > 0 {
|
|
avatarURL = envelope.AwemeDetail.Author.AvatarThumb.URLList[0]
|
|
}
|
|
return TargetProfile{
|
|
UID: envelope.AwemeDetail.Author.UID,
|
|
SecUID: envelope.AwemeDetail.Author.SecUID,
|
|
UniqueID: envelope.AwemeDetail.Author.UniqueID,
|
|
Nickname: envelope.AwemeDetail.Author.Nickname,
|
|
AvatarURL: avatarURL,
|
|
}, true
|
|
}
|
|
|
|
func (c CreatorCollector) CanonicalSecUID(ctx context.Context, expectedKey string) (string, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return "", fmt.Errorf("%w: invalid identity verification request", ErrInvalid)
|
|
}
|
|
response, err := c.Browser.Get(ctx, identityEndpoint)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return canonicalSecUID(response, expectedKey)
|
|
}
|
|
|
|
func (c CreatorCollector) CanonicalTargetSecUID(ctx context.Context, expectedKey string) (string, error) {
|
|
profile, err := c.ResolveTarget(ctx, expectedKey)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return profile.SecUID, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ResolveTarget(ctx context.Context, expectedKey string) (TargetProfile, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return TargetProfile{}, fmt.Errorf("%w: invalid target identity request", ErrInvalid)
|
|
}
|
|
field := "user_id"
|
|
if _, err := strconv.ParseUint(expectedKey, 10, 64); err != nil {
|
|
if !strings.HasPrefix(expectedKey, "MS4") {
|
|
return TargetProfile{}, fmt.Errorf("%w: target requires a Douyin UID or sec UID", ErrInvalid)
|
|
}
|
|
field = "sec_user_id"
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set(field, expectedKey)
|
|
response, err := c.Browser.Get(ctx, profileOtherEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
if err := creatorResponseError(response, "target profile"); err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
identity, ok := parseIdentity(response.Body)
|
|
if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID {
|
|
return TargetProfile{}, fmt.Errorf("%w: douyin target identity mismatch", ErrInvalid)
|
|
}
|
|
avatarURL := ""
|
|
if identity.User.AvatarThumb != nil && len(identity.User.AvatarThumb.URLList) > 0 {
|
|
avatarURL = identity.User.AvatarThumb.URLList[0]
|
|
}
|
|
return TargetProfile{
|
|
UID: identity.User.UID, SecUID: identity.User.SecUID, UniqueID: identity.User.UniqueID, Nickname: identity.User.Nickname, AvatarURL: avatarURL,
|
|
Signature: identity.User.Signature, FollowerCount: identity.User.FollowerCount, FollowingCount: identity.User.FollowingCount,
|
|
TotalFavorited: flexibleIntPtr(identity.User.TotalFavorited), AwemeCount: identity.User.AwemeCount,
|
|
}, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ResolveWork(ctx context.Context, workKey string) (TargetProfile, error) {
|
|
if c.Browser == nil || !workKeyPattern.MatchString(workKey) {
|
|
return TargetProfile{}, fmt.Errorf("%w: invalid work request", ErrInvalid)
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("aweme_id", workKey)
|
|
response, err := c.Browser.Get(ctx, workDetailEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
if err := creatorResponseError(response, "work detail"); err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
profile, ok := parseWorkTarget(response.Body, workKey)
|
|
if !ok {
|
|
return TargetProfile{}, fmt.Errorf("%w: invalid douyin work detail response", ErrInvalid)
|
|
}
|
|
return profile, nil
|
|
}
|
|
|
|
func (c CreatorCollector) VerifyIdentity(ctx context.Context, expectedKey string) error {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return fmt.Errorf("%w: invalid identity verification request", ErrInvalid)
|
|
}
|
|
_, err := c.CanonicalSecUID(ctx, expectedKey)
|
|
return err
|
|
}
|
|
|
|
func canonicalSecUID(response Response, expectedKey string) (string, error) {
|
|
if err := creatorResponseError(response, "identity"); err != nil {
|
|
return "", err
|
|
}
|
|
identity, ok := parseIdentity(response.Body)
|
|
if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID {
|
|
return "", fmt.Errorf("%w: douyin identity mismatch", ErrInvalid)
|
|
}
|
|
return identity.User.SecUID, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) {
|
|
if c.AccountKey != "" {
|
|
accountKey = c.AccountKey
|
|
}
|
|
if c.Browser == nil || !keyPattern.MatchString(accountKey) || cursor != "" && !cursorPattern.MatchString(cursor) {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid creator work collection request", ErrInvalid)
|
|
}
|
|
maxCursor := "0"
|
|
if cursor != "" {
|
|
maxCursor = cursor
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("channel", "channel_pc_web")
|
|
query.Set("sec_user_id", accountKey)
|
|
query.Set("count", "18")
|
|
query.Set("max_cursor", maxCursor)
|
|
query.Set("locate_query", "false")
|
|
query.Set("show_live_replay_strategy", "1")
|
|
query.Set("need_time_list", "1")
|
|
query.Set("time_list_query", "0")
|
|
query.Set("cut_version", "1")
|
|
query.Set("publish_video_strategy_type", "2")
|
|
query.Set("from_user_page", "0")
|
|
response, err := c.Browser.Get(ctx, worksEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
if err := creatorResponseError(response, "works"); err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
works, hasMore, nextCursor, ok := parseCreatorWorksPage(response.Body)
|
|
if !ok {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid douyin works response (status=%d body_len=%d head=%.160s)", ErrInvalid, response.Status, len(response.Body), response.Body)
|
|
}
|
|
items := make([]creator.WorkInput, 0, len(works))
|
|
for _, work := range works {
|
|
var published *time.Time
|
|
if work.CreatedAt != nil && *work.CreatedAt > 0 {
|
|
value := time.Unix(*work.CreatedAt, 0).UTC()
|
|
published = &value
|
|
}
|
|
likes, comments, shares, collects, plays := work.DiggCount, work.CommentCount, work.ShareCount, work.CollectCount, work.PlayCount
|
|
sourceType, sourceID := c.SourceType, c.SourceID
|
|
if sourceType == "" {
|
|
sourceType = creator.SourceCompetitor
|
|
}
|
|
if sourceID == "" {
|
|
sourceID = accountKey
|
|
}
|
|
status := "pending_verification"
|
|
if work.CreatedAtInvalid {
|
|
status = "invalid"
|
|
} else if published != nil {
|
|
status = "verified"
|
|
}
|
|
items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, AuthorName: work.AuthorName, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, CoverURL: work.CoverURL, Likes: likes, CommentsCount: comments, Shares: shares, CollectCount: collects, PlayCount: plays})
|
|
}
|
|
page := creator.WorkPage{Items: items, HasMore: hasMore}
|
|
if nextCursor != nil {
|
|
page.NextCursor = strconv.FormatInt(*nextCursor, 10)
|
|
}
|
|
if page.HasMore && page.NextCursor == "" {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: douyin works cursor missing", ErrInvalid)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(workKey) || cursor != "" && !cursorPattern.MatchString(cursor) {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid creator comment collection request", ErrInvalid)
|
|
}
|
|
cursorValue := "0"
|
|
if cursor != "" {
|
|
cursorValue = cursor
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("aweme_id", workKey)
|
|
query.Set("count", "20")
|
|
query.Set("cursor", cursorValue)
|
|
response, err := c.Browser.Get(ctx, commentsEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
if err := creatorResponseError(response, "comments"); err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
return parseCreatorCommentsPage(response.Body)
|
|
}
|
|
|
|
func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) {
|
|
var envelope commentEnvelope
|
|
if len(body) > 4<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid douyin comments response", ErrInvalid)
|
|
}
|
|
items := make([]creator.CommentInput, 0, len(envelope.Comments))
|
|
for _, item := range envelope.Comments {
|
|
hasImage := len(item.Images) > 0
|
|
if item.ID == "" || !hasImage && strings.TrimSpace(item.Text) == "" || item.ReplyID != "" && item.ReplyID != "0" {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: non-top-level or incomplete douyin comment", ErrInvalid)
|
|
}
|
|
text := item.Text
|
|
if strings.TrimSpace(text) == "" && hasImage {
|
|
text = "[图片]"
|
|
}
|
|
authorUID, authorName := item.UserUID, item.UserName
|
|
if authorUID == "" {
|
|
authorUID = item.User.UID
|
|
}
|
|
if authorName == "" {
|
|
authorName = item.User.Nickname
|
|
}
|
|
var published *time.Time
|
|
if item.CreateTime > 0 {
|
|
value := time.Unix(item.CreateTime, 0).UTC()
|
|
published = &value
|
|
}
|
|
items = append(items, creator.CommentInput{Platform: creator.PlatformDouyin, CommentKey: item.ID, AuthorUID: authorUID, AuthorName: authorName, Content: text, PublishedAt: published, CommentType: "top_level"})
|
|
}
|
|
page := creator.CommentPage{Items: items, HasMore: bool(*envelope.HasMore)}
|
|
if envelope.Cursor != nil {
|
|
page.NextCursor = strconv.FormatInt(*envelope.Cursor, 10)
|
|
}
|
|
if page.HasMore && page.NextCursor == "" {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: douyin comments cursor missing", ErrInvalid)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
type creatorWorkPageItem struct {
|
|
ID string
|
|
Description string
|
|
AuthorName string
|
|
CreatedAt *int64
|
|
CreatedAtInvalid bool
|
|
CoverURL string
|
|
DiggCount *int64
|
|
CommentCount *int64
|
|
ShareCount *int64
|
|
CollectCount *int64
|
|
PlayCount *int64
|
|
}
|
|
|
|
type commentEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
HasMore *douyinBool `json:"has_more"`
|
|
Cursor *int64 `json:"cursor"`
|
|
Comments []struct {
|
|
ID string `json:"cid"`
|
|
Text string `json:"text"`
|
|
CreateTime int64 `json:"create_time"`
|
|
// 抖音用 reply_id="0" 表示顶层评论；只有非零 reply_id 才是回复评论。
|
|
ReplyID string `json:"reply_id"`
|
|
// 图片评论 text 为空，仅以 image_list 承载内容。
|
|
Images []json.RawMessage `json:"image_list"`
|
|
// 登录态响应曾提供顶层 user_uid/user_name；匿名响应作者信息在 user 对象里。
|
|
UserUID string `json:"user_uid"`
|
|
UserName string `json:"user_name"`
|
|
User struct {
|
|
UID string `json:"uid"`
|
|
Nickname string `json:"nickname"`
|
|
} `json:"user"`
|
|
} `json:"comments"`
|
|
}
|
|
|
|
var cursorPattern = regexp.MustCompile(`^[0-9]{1,20}$`)
|
|
|
|
func creatorResponseError(response Response, resource string) error {
|
|
if response.Challenge != ChallengeNone {
|
|
return fmt.Errorf("%w: douyin %s challenge %s", creator.ErrUnavailable, resource, response.Challenge)
|
|
}
|
|
if response.Status >= 200 && response.Status < 300 {
|
|
return nil
|
|
}
|
|
switch response.Status {
|
|
case 401, 403:
|
|
return fmt.Errorf("%w: douyin %s authentication rejected with HTTP %d", creator.ErrConflict, resource, response.Status)
|
|
case 429:
|
|
return fmt.Errorf("%w: douyin %s rate limited", creator.ErrUnavailable, resource)
|
|
default:
|
|
return fmt.Errorf("douyin %s returned HTTP %d", resource, response.Status)
|
|
}
|
|
}
|