350 lines
13 KiB
Go
350 lines
13 KiB
Go
package douyin
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
|
)
|
|
|
|
const (
|
|
commentsEndpoint = "https://www.douyin.com/aweme/v1/web/comment/list/"
|
|
profileOtherEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/other/"
|
|
)
|
|
|
|
var workKeyPattern = regexp.MustCompile(`^[1-9][0-9]{0,63}$`)
|
|
|
|
type CreatorCollector struct {
|
|
Browser Browser
|
|
AccountKey string
|
|
SourceType string
|
|
SourceID string
|
|
}
|
|
|
|
type TargetProfile struct {
|
|
UID string
|
|
SecUID string
|
|
UniqueID string
|
|
Nickname string
|
|
AvatarURL string
|
|
}
|
|
|
|
type workDetailEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
AwemeDetail *struct {
|
|
AwemeID string `json:"aweme_id"`
|
|
Author *struct {
|
|
UID string `json:"uid"`
|
|
SecUID string `json:"sec_uid"`
|
|
UniqueID string `json:"unique_id"`
|
|
Nickname string `json:"nickname"`
|
|
AvatarThumb *struct {
|
|
URLList []string `json:"url_list"`
|
|
} `json:"avatar_thumb"`
|
|
} `json:"author"`
|
|
} `json:"aweme_detail"`
|
|
}
|
|
|
|
func parseWorkTarget(body []byte, expectedWorkKey string) (TargetProfile, bool) {
|
|
var envelope workDetailEnvelope
|
|
if len(body) > 2<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 ||
|
|
envelope.AwemeDetail == nil || envelope.AwemeDetail.AwemeID != expectedWorkKey || envelope.AwemeDetail.Author == nil ||
|
|
!keyPattern.MatchString(envelope.AwemeDetail.Author.UID) || !keyPattern.MatchString(envelope.AwemeDetail.Author.SecUID) ||
|
|
(envelope.AwemeDetail.Author.UniqueID != "" && !keyPattern.MatchString(envelope.AwemeDetail.Author.UniqueID)) {
|
|
return TargetProfile{}, false
|
|
}
|
|
avatarURL := ""
|
|
if envelope.AwemeDetail.Author.AvatarThumb != nil && len(envelope.AwemeDetail.Author.AvatarThumb.URLList) > 0 {
|
|
avatarURL = envelope.AwemeDetail.Author.AvatarThumb.URLList[0]
|
|
}
|
|
return TargetProfile{
|
|
UID: envelope.AwemeDetail.Author.UID,
|
|
SecUID: envelope.AwemeDetail.Author.SecUID,
|
|
UniqueID: envelope.AwemeDetail.Author.UniqueID,
|
|
Nickname: envelope.AwemeDetail.Author.Nickname,
|
|
AvatarURL: avatarURL,
|
|
}, true
|
|
}
|
|
|
|
func (c CreatorCollector) CanonicalSecUID(ctx context.Context, expectedKey string) (string, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return "", fmt.Errorf("%w: invalid identity verification request", ErrInvalid)
|
|
}
|
|
response, err := c.Browser.Get(ctx, identityEndpoint)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return canonicalSecUID(response, expectedKey)
|
|
}
|
|
|
|
func (c CreatorCollector) CanonicalTargetSecUID(ctx context.Context, expectedKey string) (string, error) {
|
|
profile, err := c.ResolveTarget(ctx, expectedKey)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return profile.SecUID, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ResolveTarget(ctx context.Context, expectedKey string) (TargetProfile, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return TargetProfile{}, fmt.Errorf("%w: invalid target identity request", ErrInvalid)
|
|
}
|
|
field := "user_id"
|
|
if _, err := strconv.ParseUint(expectedKey, 10, 64); err != nil {
|
|
if !strings.HasPrefix(expectedKey, "MS4") {
|
|
return TargetProfile{}, fmt.Errorf("%w: target requires a Douyin UID or sec UID", ErrInvalid)
|
|
}
|
|
field = "sec_user_id"
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set(field, expectedKey)
|
|
response, err := c.Browser.Get(ctx, profileOtherEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
if err := creatorResponseError(response, "target profile"); err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
identity, ok := parseIdentity(response.Body)
|
|
if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID {
|
|
return TargetProfile{}, fmt.Errorf("%w: douyin target identity mismatch", ErrInvalid)
|
|
}
|
|
avatarURL := ""
|
|
if identity.User.AvatarThumb != nil && len(identity.User.AvatarThumb.URLList) > 0 {
|
|
avatarURL = identity.User.AvatarThumb.URLList[0]
|
|
}
|
|
return TargetProfile{UID: identity.User.UID, SecUID: identity.User.SecUID, UniqueID: identity.User.UniqueID, Nickname: identity.User.Nickname, AvatarURL: avatarURL}, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ResolveWork(ctx context.Context, workKey string) (TargetProfile, error) {
|
|
if c.Browser == nil || !workKeyPattern.MatchString(workKey) {
|
|
return TargetProfile{}, fmt.Errorf("%w: invalid work request", ErrInvalid)
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("aweme_id", workKey)
|
|
response, err := c.Browser.Get(ctx, workDetailEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
if err := creatorResponseError(response, "work detail"); err != nil {
|
|
return TargetProfile{}, err
|
|
}
|
|
profile, ok := parseWorkTarget(response.Body, workKey)
|
|
if !ok {
|
|
return TargetProfile{}, fmt.Errorf("%w: invalid douyin work detail response", ErrInvalid)
|
|
}
|
|
return profile, nil
|
|
}
|
|
|
|
func (c CreatorCollector) VerifyIdentity(ctx context.Context, expectedKey string) error {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return fmt.Errorf("%w: invalid identity verification request", ErrInvalid)
|
|
}
|
|
_, err := c.CanonicalSecUID(ctx, expectedKey)
|
|
return err
|
|
}
|
|
|
|
func canonicalSecUID(response Response, expectedKey string) (string, error) {
|
|
if err := creatorResponseError(response, "identity"); err != nil {
|
|
return "", err
|
|
}
|
|
identity, ok := parseIdentity(response.Body)
|
|
if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID {
|
|
return "", fmt.Errorf("%w: douyin identity mismatch", ErrInvalid)
|
|
}
|
|
return identity.User.SecUID, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) {
|
|
if c.AccountKey != "" {
|
|
accountKey = c.AccountKey
|
|
}
|
|
if c.Browser == nil || !keyPattern.MatchString(accountKey) || cursor != "" && !cursorPattern.MatchString(cursor) {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid creator work collection request", ErrInvalid)
|
|
}
|
|
maxCursor := "0"
|
|
if cursor != "" {
|
|
maxCursor = cursor
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("sec_user_id", accountKey)
|
|
query.Set("count", "20")
|
|
query.Set("max_cursor", maxCursor)
|
|
response, err := c.Browser.Get(ctx, worksEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
if err := creatorResponseError(response, "works"); err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
works, hasMore, nextCursor, ok := parseCreatorWorksPage(response.Body)
|
|
if !ok {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid douyin works response", ErrInvalid)
|
|
}
|
|
items := make([]creator.WorkInput, 0, len(works))
|
|
for _, work := range works {
|
|
var published *time.Time
|
|
if work.CreatedAt != nil && *work.CreatedAt > 0 {
|
|
value := time.Unix(*work.CreatedAt, 0).UTC()
|
|
published = &value
|
|
}
|
|
likes, comments, shares := work.DiggCount, work.CommentCount, work.ShareCount
|
|
sourceType, sourceID := c.SourceType, c.SourceID
|
|
if sourceType == "" {
|
|
sourceType = creator.SourceCompetitor
|
|
}
|
|
if sourceID == "" {
|
|
sourceID = accountKey
|
|
}
|
|
status := "pending_verification"
|
|
if work.CreatedAtInvalid {
|
|
status = "invalid"
|
|
} else if published != nil {
|
|
status = "verified"
|
|
}
|
|
items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, Likes: likes, CommentsCount: comments, Shares: shares})
|
|
}
|
|
page := creator.WorkPage{Items: items, HasMore: hasMore}
|
|
if nextCursor != nil {
|
|
page.NextCursor = strconv.FormatInt(*nextCursor, 10)
|
|
}
|
|
if page.HasMore && page.NextCursor == "" {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: douyin works cursor missing", ErrInvalid)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(workKey) || cursor != "" && !cursorPattern.MatchString(cursor) {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid creator comment collection request", ErrInvalid)
|
|
}
|
|
cursorValue := "0"
|
|
if cursor != "" {
|
|
cursorValue = cursor
|
|
}
|
|
query := douyinAPIQuery()
|
|
query.Set("aweme_id", workKey)
|
|
query.Set("count", "20")
|
|
query.Set("cursor", cursorValue)
|
|
response, err := c.Browser.Get(ctx, commentsEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
if err := creatorResponseError(response, "comments"); err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
return parseCreatorCommentsPage(response.Body)
|
|
}
|
|
|
|
func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) {
|
|
var envelope commentEnvelope
|
|
if len(body) > 4<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid douyin comments response", ErrInvalid)
|
|
}
|
|
items := make([]creator.CommentInput, 0, len(envelope.Comments))
|
|
for _, item := range envelope.Comments {
|
|
if item.ID == "" || strings.TrimSpace(item.Text) == "" || item.ReplyID != "" {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: non-top-level or incomplete douyin comment", ErrInvalid)
|
|
}
|
|
var published *time.Time
|
|
if item.CreateTime > 0 {
|
|
value := time.Unix(item.CreateTime, 0).UTC()
|
|
published = &value
|
|
}
|
|
items = append(items, creator.CommentInput{Platform: creator.PlatformDouyin, CommentKey: item.ID, AuthorUID: item.UserUID, AuthorName: item.UserName, Content: item.Text, PublishedAt: published, CommentType: "top_level"})
|
|
}
|
|
page := creator.CommentPage{Items: items, HasMore: bool(*envelope.HasMore)}
|
|
if envelope.Cursor != nil {
|
|
page.NextCursor = strconv.FormatInt(*envelope.Cursor, 10)
|
|
}
|
|
if page.HasMore && page.NextCursor == "" {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: douyin comments cursor missing", ErrInvalid)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
type creatorWorkPageItem struct {
|
|
ID string
|
|
Description string
|
|
CreatedAt *int64
|
|
CreatedAtInvalid bool
|
|
DiggCount *int64
|
|
CommentCount *int64
|
|
ShareCount *int64
|
|
}
|
|
|
|
func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) {
|
|
if len(body) > 4<<20 {
|
|
return nil, false, nil, false
|
|
}
|
|
var envelope worksEnvelope
|
|
if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 || bool(*envelope.HasMore) && envelope.MaxCursor == nil {
|
|
return nil, false, nil, false
|
|
}
|
|
items := make([]creatorWorkPageItem, 0, len(envelope.Works))
|
|
seen := make(map[string]struct{}, len(envelope.Works))
|
|
for _, item := range envelope.Works {
|
|
if !keyPattern.MatchString(item.ID) {
|
|
return nil, false, nil, false
|
|
}
|
|
if _, exists := seen[item.ID]; exists {
|
|
return nil, false, nil, false
|
|
}
|
|
seen[item.ID] = struct{}{}
|
|
var likes, comments, shares *int64
|
|
if item.Statistics != nil {
|
|
likes, comments, shares = item.Statistics.DiggCount, item.Statistics.CommentCount, item.Statistics.ShareCount
|
|
}
|
|
for _, value := range []*int64{likes, comments, shares} {
|
|
if value != nil && *value < 0 {
|
|
return nil, false, nil, false
|
|
}
|
|
}
|
|
createdAt := item.CreatedAt
|
|
createdAtInvalid := createdAt != nil && *createdAt <= 0
|
|
if createdAtInvalid {
|
|
createdAt = nil
|
|
}
|
|
items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares})
|
|
}
|
|
return items, bool(*envelope.HasMore), envelope.MaxCursor, true
|
|
}
|
|
|
|
type commentEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
HasMore *douyinBool `json:"has_more"`
|
|
Cursor *int64 `json:"cursor"`
|
|
Comments []struct {
|
|
ID string `json:"cid"`
|
|
Text string `json:"text"`
|
|
CreateTime int64 `json:"create_time"`
|
|
ReplyID string `json:"reply_id"`
|
|
UserUID string `json:"user_uid"`
|
|
UserName string `json:"user_name"`
|
|
} `json:"comments"`
|
|
}
|
|
|
|
var cursorPattern = regexp.MustCompile(`^[0-9]{1,20}$`)
|
|
|
|
func creatorResponseError(response Response, resource string) error {
|
|
if response.Challenge != ChallengeNone {
|
|
return fmt.Errorf("%w: douyin %s challenge %s", creator.ErrUnavailable, resource, response.Challenge)
|
|
}
|
|
if response.Status >= 200 && response.Status < 300 {
|
|
return nil
|
|
}
|
|
switch response.Status {
|
|
case 401, 403:
|
|
return fmt.Errorf("%w: douyin %s authentication rejected with HTTP %d", creator.ErrConflict, resource, response.Status)
|
|
case 429:
|
|
return fmt.Errorf("%w: douyin %s rate limited", creator.ErrUnavailable, resource)
|
|
default:
|
|
return fmt.Errorf("douyin %s returned HTTP %d", resource, response.Status)
|
|
}
|
|
}
|