391 lines
14 KiB
Go
391 lines
14 KiB
Go
package douyin
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
const (
|
|
StateSucceeded = "succeeded"
|
|
StatePolicyHold = "policy_hold"
|
|
StateNeedsConfirmation = "needs_confirmation"
|
|
|
|
ReasonAuthInvalid = "douyin_auth_invalid"
|
|
ReasonForbidden = "douyin_forbidden"
|
|
ReasonRateLimited = "douyin_rate_limited"
|
|
ReasonChallenge = "douyin_challenge"
|
|
ReasonUnknown = "douyin_result_unknown"
|
|
ReasonIdentityMatch = "douyin_identity_mismatch"
|
|
ReasonSucceeded = "douyin_sync_succeeded"
|
|
|
|
identityEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp"
|
|
worksEndpoint = "https://www.douyin.com/aweme/v1/web/aweme/post/"
|
|
workDetailEndpoint = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
|
|
)
|
|
|
|
func douyinAPIQuery() url.Values {
|
|
return url.Values{"aid": {"6383"}, "device_platform": {"webapp"}}
|
|
}
|
|
|
|
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`)
|
|
|
|
var ErrInvalid = errors.New("invalid douyin connector input")
|
|
|
|
type Challenge string
|
|
|
|
const (
|
|
ChallengeNone Challenge = ""
|
|
ChallengeCaptcha Challenge = "captcha"
|
|
ChallengeDevice Challenge = "device"
|
|
)
|
|
|
|
type Cookie struct {
|
|
Name string `json:"name"`
|
|
Value string `json:"value"`
|
|
Domain string `json:"domain"`
|
|
Path string `json:"path"`
|
|
Secure bool `json:"secure,omitempty"`
|
|
HTTPOnly bool `json:"http_only,omitempty"`
|
|
SameSite string `json:"same_site,omitempty"`
|
|
Expires float64 `json:"expires,omitempty"`
|
|
}
|
|
|
|
type Response struct {
|
|
Status int
|
|
Body []byte
|
|
Challenge Challenge
|
|
}
|
|
|
|
// Browser is the deliberately narrow contract the restricted browser control
|
|
// plane must implement. Login happens in the managed browser session; the
|
|
// connector never injects stored credentials or cookies.
|
|
type Browser interface {
|
|
Get(context.Context, string) (Response, error)
|
|
}
|
|
|
|
type SecretReference struct {
|
|
Provider string
|
|
Key string
|
|
}
|
|
|
|
type SecretResolver interface {
|
|
Resolve(context.Context, SecretReference) ([]byte, error)
|
|
}
|
|
|
|
type Work struct {
|
|
ID string `json:"id"`
|
|
Description string `json:"description"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
DiggCount int64 `json:"digg_count"`
|
|
CommentCount int64 `json:"comment_count"`
|
|
ShareCount int64 `json:"share_count"`
|
|
PlayCount int64 `json:"play_count"`
|
|
}
|
|
|
|
type Evidence struct {
|
|
Phase string `json:"phase"`
|
|
HTTPStatus int `json:"http_status,omitempty"`
|
|
IdentityVerified bool `json:"identity_verified"`
|
|
WorksSeen int `json:"works_seen,omitempty"`
|
|
HasMore bool `json:"has_more,omitempty"`
|
|
Pages int `json:"pages,omitempty"`
|
|
PaginationComplete bool `json:"pagination_complete"`
|
|
}
|
|
|
|
type Result struct {
|
|
State string `json:"state"`
|
|
ReasonCode string `json:"reason_code"`
|
|
Evidence Evidence `json:"evidence"`
|
|
}
|
|
|
|
// Store owns both persistence/audit and fail-closed account/runtime handling.
|
|
// Hold must pause the account and stop its existing bound runtime without retry.
|
|
type Store interface {
|
|
Complete(context.Context, string, []Work, Evidence) error
|
|
Hold(context.Context, string, string, string, Evidence) error
|
|
}
|
|
|
|
type Connector struct {
|
|
Browser Browser
|
|
Secrets SecretResolver
|
|
Store Store
|
|
}
|
|
|
|
type Request struct {
|
|
AccountID string
|
|
PlatformAccountKey string
|
|
Credential SecretReference
|
|
}
|
|
|
|
func (connector Connector) Sync(ctx context.Context, request Request) (Result, error) {
|
|
if connector.Browser == nil || connector.Store == nil || !keyPattern.MatchString(request.AccountID) ||
|
|
!keyPattern.MatchString(request.PlatformAccountKey) {
|
|
return Result{}, ErrInvalid
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
|
|
}
|
|
identityResponse, err := connector.Browser.Get(ctx, identityEndpoint)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
|
|
}
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "login"})
|
|
}
|
|
if state, reason := classify(identityResponse); state != "" {
|
|
return connector.stop(ctx, request.AccountID, state, reason, Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
|
|
}
|
|
identity, ok := parseIdentity(identityResponse.Body)
|
|
if !ok {
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown,
|
|
Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
|
|
}
|
|
if request.PlatformAccountKey != identity.User.UID && request.PlatformAccountKey != identity.User.SecUID &&
|
|
request.PlatformAccountKey != identity.User.UniqueID {
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonIdentityMatch,
|
|
Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
|
|
}
|
|
|
|
var works []Work
|
|
seen := make(map[string]struct{})
|
|
evidence := Evidence{Phase: "works", IdentityVerified: true}
|
|
cursor := int64(0)
|
|
for page := 0; page < 100; page++ {
|
|
query := douyinAPIQuery()
|
|
query.Set("sec_user_id", identity.User.SecUID)
|
|
query.Set("count", "20")
|
|
query.Set("max_cursor", strconv.FormatInt(cursor, 10))
|
|
worksResponse, err := connector.Browser.Get(ctx, worksEndpoint+"?"+query.Encode())
|
|
if err != nil {
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
|
}
|
|
if state, reason := classify(worksResponse); state != "" {
|
|
evidence.HTTPStatus = worksResponse.Status
|
|
return connector.stop(ctx, request.AccountID, state, reason, evidence)
|
|
}
|
|
pageWorks, hasMore, nextCursor, ok := parseWorksPage(worksResponse.Body)
|
|
evidence.HTTPStatus, evidence.Pages, evidence.HasMore = worksResponse.Status, page+1, hasMore
|
|
if !ok {
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
|
}
|
|
for _, work := range pageWorks {
|
|
if _, exists := seen[work.ID]; exists {
|
|
continue
|
|
}
|
|
seen[work.ID] = struct{}{}
|
|
works = append(works, work)
|
|
}
|
|
if !hasMore {
|
|
evidence.PaginationComplete = true
|
|
break
|
|
}
|
|
if nextCursor == nil {
|
|
// Older gateway responses omitted the cursor. Keep the existing
|
|
// read-only behavior, but expose that pagination was incomplete.
|
|
break
|
|
}
|
|
if *nextCursor <= cursor {
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
|
}
|
|
cursor = *nextCursor
|
|
if page == 99 {
|
|
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
|
}
|
|
}
|
|
evidence.WorksSeen = len(works)
|
|
if err := connector.Store.Complete(ctx, request.AccountID, works, evidence); err != nil {
|
|
result, holdErr := connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
|
return result, errors.Join(err, holdErr)
|
|
}
|
|
return Result{State: StateSucceeded, ReasonCode: ReasonSucceeded, Evidence: evidence}, nil
|
|
}
|
|
|
|
func (connector Connector) stop(ctx context.Context, accountID, state, reason string, evidence Evidence) (Result, error) {
|
|
result := Result{State: state, ReasonCode: reason, Evidence: evidence}
|
|
holdContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
return result, connector.Store.Hold(holdContext, accountID, state, reason, evidence)
|
|
}
|
|
|
|
func classify(response Response) (string, string) {
|
|
switch response.Challenge {
|
|
case ChallengeCaptcha, ChallengeDevice:
|
|
return StateNeedsConfirmation, ReasonChallenge
|
|
case ChallengeNone:
|
|
default:
|
|
return StateNeedsConfirmation, ReasonUnknown
|
|
}
|
|
switch response.Status {
|
|
case http.StatusUnauthorized:
|
|
return StatePolicyHold, ReasonAuthInvalid
|
|
case http.StatusForbidden:
|
|
return StatePolicyHold, ReasonForbidden
|
|
case http.StatusTooManyRequests:
|
|
return StatePolicyHold, ReasonRateLimited
|
|
case http.StatusOK:
|
|
return "", ""
|
|
default:
|
|
return StateNeedsConfirmation, ReasonUnknown
|
|
}
|
|
}
|
|
|
|
// ParseCredential accepts either the persisted JSON cookie bundle or the
|
|
// manually captured Cookie header used by an already logged-in browser.
|
|
func ParseCredential(raw []byte) ([]Cookie, error) {
|
|
if cookies, err := ParseCookieHeader(raw); err == nil {
|
|
return cookies, nil
|
|
}
|
|
return ParseCookieBundle(raw)
|
|
}
|
|
|
|
func ParseCookieBundle(raw []byte) ([]Cookie, error) {
|
|
return parseCredential(raw)
|
|
}
|
|
|
|
func ParseCookieHeader(raw []byte) ([]Cookie, error) {
|
|
if len(raw) == 0 || len(raw) > 64<<10 {
|
|
return nil, ErrInvalid
|
|
}
|
|
parsed, err := http.ParseCookie(string(raw))
|
|
if err != nil || len(parsed) == 0 || len(parsed) > 64 {
|
|
return nil, ErrInvalid
|
|
}
|
|
cookies := make([]Cookie, 0, len(parsed))
|
|
for _, cookie := range parsed {
|
|
if cookie == nil || cookie.Name == "" || cookie.Value == "" {
|
|
return nil, ErrInvalid
|
|
}
|
|
cookies = append(cookies, Cookie{Name: cookie.Name, Value: cookie.Value, Domain: ".douyin.com", Path: "/", Secure: true, SameSite: "Lax"})
|
|
}
|
|
return cookies, nil
|
|
}
|
|
|
|
func parseCredential(raw []byte) ([]Cookie, error) {
|
|
if len(raw) == 0 || len(raw) > 64<<10 {
|
|
return nil, ErrInvalid
|
|
}
|
|
var bundle struct {
|
|
Cookies []Cookie `json:"cookies"`
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&bundle); err != nil || len(bundle.Cookies) == 0 || len(bundle.Cookies) > 64 {
|
|
return nil, ErrInvalid
|
|
}
|
|
var trailing json.RawMessage
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
return nil, ErrInvalid
|
|
}
|
|
for index := range bundle.Cookies {
|
|
cookie := &bundle.Cookies[index]
|
|
cookie.Domain = strings.ToLower(strings.TrimSpace(cookie.Domain))
|
|
if cookie.Path == "" {
|
|
cookie.Path = "/"
|
|
}
|
|
if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 ||
|
|
cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") ||
|
|
(cookie.Domain != "douyin.com" && !strings.HasSuffix(cookie.Domain, ".douyin.com")) ||
|
|
!strings.HasPrefix(cookie.Path, "/") || strings.ContainsAny(cookie.Path, ";\r\n\x00") ||
|
|
(cookie.SameSite != "" && cookie.SameSite != "Lax" &&
|
|
cookie.SameSite != "Strict" && cookie.SameSite != "None") {
|
|
return nil, ErrInvalid
|
|
}
|
|
}
|
|
return bundle.Cookies, nil
|
|
}
|
|
|
|
type identityEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
User *struct {
|
|
UID string `json:"uid"`
|
|
SecUID string `json:"sec_uid"`
|
|
UniqueID string `json:"unique_id"`
|
|
Nickname string `json:"nickname"`
|
|
AvatarThumb *struct {
|
|
URLList []string `json:"url_list"`
|
|
} `json:"avatar_thumb"`
|
|
} `json:"user"`
|
|
}
|
|
|
|
func parseIdentity(body []byte) (identityEnvelope, bool) {
|
|
var identity identityEnvelope
|
|
if len(body) > 1<<20 || json.Unmarshal(body, &identity) != nil || identity.StatusCode == nil || *identity.StatusCode != 0 || identity.User == nil ||
|
|
!keyPattern.MatchString(identity.User.UID) || !keyPattern.MatchString(identity.User.SecUID) ||
|
|
(identity.User.UniqueID != "" && !keyPattern.MatchString(identity.User.UniqueID)) {
|
|
return identityEnvelope{}, false
|
|
}
|
|
return identity, true
|
|
}
|
|
|
|
type douyinBool bool
|
|
|
|
func (value *douyinBool) UnmarshalJSON(data []byte) error {
|
|
var boolean bool
|
|
if err := json.Unmarshal(data, &boolean); err == nil {
|
|
*value = douyinBool(boolean)
|
|
return nil
|
|
}
|
|
var numeric int
|
|
if err := json.Unmarshal(data, &numeric); err == nil && (numeric == 0 || numeric == 1) {
|
|
*value = douyinBool(numeric == 1)
|
|
return nil
|
|
}
|
|
return errors.New("douyin boolean must be true, false, 0, or 1")
|
|
}
|
|
|
|
type worksEnvelope struct {
|
|
StatusCode *int `json:"status_code"`
|
|
HasMore *douyinBool `json:"has_more"`
|
|
MaxCursor *int64 `json:"max_cursor"`
|
|
Works []struct {
|
|
ID string `json:"aweme_id"`
|
|
Description string `json:"desc"`
|
|
CreatedAt *int64 `json:"create_time"`
|
|
Statistics *struct {
|
|
DiggCount *int64 `json:"digg_count"`
|
|
CommentCount *int64 `json:"comment_count"`
|
|
ShareCount *int64 `json:"share_count"`
|
|
PlayCount *int64 `json:"play_count"`
|
|
} `json:"statistics"`
|
|
} `json:"aweme_list"`
|
|
}
|
|
|
|
func parseWorksPage(body []byte) ([]Work, bool, *int64, bool) {
|
|
var envelope worksEnvelope
|
|
if len(body) > 4<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 ||
|
|
envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 || envelope.MaxCursor != nil && *envelope.MaxCursor < 0 {
|
|
return nil, false, nil, false
|
|
}
|
|
works := make([]Work, 0, len(envelope.Works))
|
|
seen := make(map[string]bool, len(envelope.Works))
|
|
for _, candidate := range envelope.Works {
|
|
if !keyPattern.MatchString(candidate.ID) || seen[candidate.ID] || candidate.CreatedAt == nil || *candidate.CreatedAt <= 0 ||
|
|
candidate.Statistics == nil || candidate.Statistics.DiggCount == nil || candidate.Statistics.CommentCount == nil ||
|
|
candidate.Statistics.ShareCount == nil || candidate.Statistics.PlayCount == nil ||
|
|
utf8.RuneCountInString(candidate.Description) > 4096 || *candidate.Statistics.DiggCount < 0 ||
|
|
*candidate.Statistics.CommentCount < 0 || *candidate.Statistics.ShareCount < 0 || *candidate.Statistics.PlayCount < 0 {
|
|
return nil, false, nil, false
|
|
}
|
|
seen[candidate.ID] = true
|
|
works = append(works, Work{ID: candidate.ID, Description: candidate.Description, CreatedAt: *candidate.CreatedAt,
|
|
DiggCount: *candidate.Statistics.DiggCount, CommentCount: *candidate.Statistics.CommentCount,
|
|
ShareCount: *candidate.Statistics.ShareCount, PlayCount: *candidate.Statistics.PlayCount})
|
|
}
|
|
return works, bool(*envelope.HasMore), envelope.MaxCursor, true
|
|
}
|
|
|
|
func parseWorks(body []byte) ([]Work, bool, bool) {
|
|
works, hasMore, _, ok := parseWorksPage(body)
|
|
return works, hasMore, ok
|
|
}
|