224 lines
8.1 KiB
Go
224 lines
8.1 KiB
Go
package douyin
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
|
)
|
|
|
|
const commentsEndpoint = "https://www.douyin.com/aweme/v1/web/comment/list/"
|
|
|
|
type CreatorCollector struct {
|
|
Browser Browser
|
|
AccountKey string
|
|
SourceType string
|
|
SourceID string
|
|
}
|
|
|
|
func (c CreatorCollector) CanonicalSecUID(ctx context.Context, expectedKey string) (string, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return "", fmt.Errorf("%w: invalid identity verification request", ErrInvalid)
|
|
}
|
|
response, err := c.Browser.Get(ctx, identityEndpoint)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := creatorResponseError(response, "identity"); err != nil {
|
|
return "", err
|
|
}
|
|
identity, ok := parseIdentity(response.Body)
|
|
if !ok || expectedKey != identity.User.UID && expectedKey != identity.User.SecUID && expectedKey != identity.User.UniqueID {
|
|
return "", fmt.Errorf("%w: douyin identity mismatch", ErrInvalid)
|
|
}
|
|
return identity.User.SecUID, nil
|
|
}
|
|
|
|
func (c CreatorCollector) VerifyIdentity(ctx context.Context, expectedKey string) error {
|
|
if c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return fmt.Errorf("%w: invalid identity verification request", ErrInvalid)
|
|
}
|
|
_, err := c.CanonicalSecUID(ctx, expectedKey)
|
|
return err
|
|
}
|
|
|
|
func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) {
|
|
if c.AccountKey != "" {
|
|
accountKey = c.AccountKey
|
|
}
|
|
if c.Browser == nil || !keyPattern.MatchString(accountKey) || cursor != "" && !cursorPattern.MatchString(cursor) {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid creator work collection request", ErrInvalid)
|
|
}
|
|
maxCursor := "0"
|
|
if cursor != "" {
|
|
maxCursor = cursor
|
|
}
|
|
query := url.Values{"sec_user_id": {accountKey}, "count": {"20"}, "max_cursor": {maxCursor}}
|
|
response, err := c.Browser.Get(ctx, worksEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
if err := creatorResponseError(response, "works"); err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
works, hasMore, nextCursor, ok := parseCreatorWorksPage(response.Body)
|
|
if !ok {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid douyin works response", ErrInvalid)
|
|
}
|
|
items := make([]creator.WorkInput, 0, len(works))
|
|
for _, work := range works {
|
|
var published *time.Time
|
|
if work.CreatedAt != nil && *work.CreatedAt > 0 {
|
|
value := time.Unix(*work.CreatedAt, 0).UTC()
|
|
published = &value
|
|
}
|
|
likes, comments, shares := work.DiggCount, work.CommentCount, work.ShareCount
|
|
sourceType, sourceID := c.SourceType, c.SourceID
|
|
if sourceType == "" {
|
|
sourceType = creator.SourceCompetitor
|
|
}
|
|
if sourceID == "" {
|
|
sourceID = accountKey
|
|
}
|
|
status := "pending_verification"
|
|
if published != nil {
|
|
status = "verified"
|
|
}
|
|
items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, Likes: likes, CommentsCount: comments, Shares: shares})
|
|
}
|
|
page := creator.WorkPage{Items: items, HasMore: hasMore}
|
|
if nextCursor != nil {
|
|
page.NextCursor = strconv.FormatInt(*nextCursor, 10)
|
|
}
|
|
if page.HasMore && page.NextCursor == "" {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: douyin works cursor missing", ErrInvalid)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
func (c CreatorCollector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) {
|
|
if c.Browser == nil || !keyPattern.MatchString(workKey) || cursor != "" && !cursorPattern.MatchString(cursor) {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid creator comment collection request", ErrInvalid)
|
|
}
|
|
cursorValue := "0"
|
|
if cursor != "" {
|
|
cursorValue = cursor
|
|
}
|
|
query := url.Values{"aweme_id": {workKey}, "count": {"20"}, "cursor": {cursorValue}}
|
|
response, err := c.Browser.Get(ctx, commentsEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
if err := creatorResponseError(response, "comments"); err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
var envelope commentEnvelope
|
|
if len(response.Body) > 4<<20 || json.Unmarshal(response.Body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Comments == nil {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid douyin comments response", ErrInvalid)
|
|
}
|
|
items := make([]creator.CommentInput, 0, len(envelope.Comments))
|
|
for _, item := range envelope.Comments {
|
|
if item.ID == "" || strings.TrimSpace(item.Text) == "" || item.ReplyID != "" {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: non-top-level or incomplete douyin comment", ErrInvalid)
|
|
}
|
|
var published *time.Time
|
|
if item.CreateTime > 0 {
|
|
value := time.Unix(item.CreateTime, 0).UTC()
|
|
published = &value
|
|
}
|
|
items = append(items, creator.CommentInput{Platform: creator.PlatformDouyin, CommentKey: item.ID, AuthorUID: item.UserUID, AuthorName: item.UserName, Content: item.Text, PublishedAt: published, CommentType: "top_level"})
|
|
}
|
|
page := creator.CommentPage{Items: items, HasMore: *envelope.HasMore}
|
|
if envelope.Cursor != nil {
|
|
page.NextCursor = strconv.FormatInt(*envelope.Cursor, 10)
|
|
}
|
|
if page.HasMore && page.NextCursor == "" {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: douyin comments cursor missing", ErrInvalid)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
type creatorWorkPageItem struct {
|
|
ID string
|
|
Description string
|
|
CreatedAt *int64
|
|
DiggCount *int64
|
|
CommentCount *int64
|
|
ShareCount *int64
|
|
}
|
|
|
|
func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bool) {
|
|
if len(body) > 4<<20 {
|
|
return nil, false, nil, false
|
|
}
|
|
var envelope worksEnvelope
|
|
if json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 || *envelope.HasMore && envelope.MaxCursor == nil {
|
|
return nil, false, nil, false
|
|
}
|
|
items := make([]creatorWorkPageItem, 0, len(envelope.Works))
|
|
seen := make(map[string]struct{}, len(envelope.Works))
|
|
for _, item := range envelope.Works {
|
|
if !keyPattern.MatchString(item.ID) {
|
|
return nil, false, nil, false
|
|
}
|
|
if _, exists := seen[item.ID]; exists {
|
|
return nil, false, nil, false
|
|
}
|
|
seen[item.ID] = struct{}{}
|
|
var likes, comments, shares *int64
|
|
if item.Statistics != nil {
|
|
likes, comments, shares = item.Statistics.DiggCount, item.Statistics.CommentCount, item.Statistics.ShareCount
|
|
}
|
|
for _, value := range []*int64{likes, comments, shares} {
|
|
if value != nil && *value < 0 {
|
|
return nil, false, nil, false
|
|
}
|
|
}
|
|
createdAt := item.CreatedAt
|
|
if createdAt != nil && *createdAt <= 0 {
|
|
createdAt = nil
|
|
}
|
|
items = append(items, creatorWorkPageItem{ID: item.ID, Description: item.Description, CreatedAt: createdAt, DiggCount: likes, CommentCount: comments, ShareCount: shares})
|
|
}
|
|
return items, *envelope.HasMore, envelope.MaxCursor, true
|
|
}
|
|
|
|
type commentEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
HasMore *bool `json:"has_more"`
|
|
Cursor *int64 `json:"cursor"`
|
|
Comments []struct {
|
|
ID string `json:"cid"`
|
|
Text string `json:"text"`
|
|
CreateTime int64 `json:"create_time"`
|
|
ReplyID string `json:"reply_id"`
|
|
UserUID string `json:"user_uid"`
|
|
UserName string `json:"user_name"`
|
|
} `json:"comments"`
|
|
}
|
|
|
|
var cursorPattern = regexp.MustCompile(`^[0-9]{1,20}$`)
|
|
|
|
func creatorResponseError(response Response, resource string) error {
|
|
if response.Challenge != ChallengeNone {
|
|
return fmt.Errorf("%w: douyin %s challenge %s", creator.ErrUnavailable, resource, response.Challenge)
|
|
}
|
|
if response.Status >= 200 && response.Status < 300 {
|
|
return nil
|
|
}
|
|
switch response.Status {
|
|
case 401, 403:
|
|
return fmt.Errorf("%w: douyin %s authentication rejected with HTTP %d", creator.ErrConflict, resource, response.Status)
|
|
case 429:
|
|
return fmt.Errorf("%w: douyin %s rate limited", creator.ErrUnavailable, resource)
|
|
default:
|
|
return fmt.Errorf("douyin %s returned HTTP %d", resource, response.Status)
|
|
}
|
|
}
|