823 lines
25 KiB
Go
823 lines
25 KiB
Go
package xiaohongshu
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
|
)
|
|
|
|
const (
|
|
APIOrigin = "https://edith.xiaohongshu.com"
|
|
SearchOrigin = "https://so.xiaohongshu.com"
|
|
IdentityURL = APIOrigin + "/api/sns/web/v2/user/me"
|
|
UserPostedPath = "/api/sns/web/v1/user_posted"
|
|
CommentsPath = "/api/sns/web/v2/comment/page"
|
|
SearchNotesPath = "/api/sns/web/v2/search/notes"
|
|
FeedPath = "/api/sns/web/v1/feed"
|
|
)
|
|
|
|
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`)
|
|
|
|
type Response struct {
|
|
Status int
|
|
Body []byte
|
|
Challenge string
|
|
}
|
|
|
|
type Browser interface {
|
|
Get(context.Context, string) (Response, error)
|
|
Post(context.Context, string, []byte) (Response, error)
|
|
}
|
|
|
|
type Identity struct {
|
|
UserID string
|
|
Nickname string
|
|
}
|
|
|
|
type LinkKind string
|
|
|
|
const (
|
|
LinkHomepage LinkKind = "homepage"
|
|
LinkNote LinkKind = "note"
|
|
LinkShare LinkKind = "share"
|
|
)
|
|
|
|
type LinkContext struct {
|
|
Kind LinkKind
|
|
AccountKey string
|
|
WorkKey string
|
|
Token string
|
|
Source string
|
|
CanonicalURL string
|
|
}
|
|
|
|
type ShareResolver interface {
|
|
Resolve(context.Context, string) (string, error)
|
|
}
|
|
|
|
type Collector struct {
|
|
Browser Browser
|
|
AccountKey string
|
|
SourceType string
|
|
SourceID string
|
|
HomepageURL string
|
|
|
|
contexts sync.Map
|
|
}
|
|
|
|
type accessContext struct {
|
|
Token string
|
|
Source string
|
|
}
|
|
|
|
func (c *Collector) VerifyIdentity(ctx context.Context, expectedKey string) error {
|
|
_, err := c.Identity(ctx, expectedKey)
|
|
return err
|
|
}
|
|
|
|
func (c *Collector) Identity(ctx context.Context, expectedKey string) (Identity, error) {
|
|
if c == nil || c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
|
return Identity{}, fmt.Errorf("%w: invalid xiaohongshu identity request", creator.ErrInvalid)
|
|
}
|
|
response, err := c.Browser.Get(ctx, IdentityURL)
|
|
if err != nil {
|
|
return Identity{}, err
|
|
}
|
|
if err := responseError(response, "identity"); err != nil {
|
|
return Identity{}, err
|
|
}
|
|
identity, ok := parseIdentity(response.Body)
|
|
if !ok || identity.UserID != expectedKey {
|
|
return Identity{}, fmt.Errorf("%w: xiaohongshu identity mismatch", creator.ErrConflict)
|
|
}
|
|
return identity, nil
|
|
}
|
|
|
|
func ParseSourceURL(raw string) (LinkContext, error) {
|
|
raw = strings.TrimSpace(raw)
|
|
parsed, err := url.Parse(raw)
|
|
if err != nil || parsed.Scheme != "https" || parsed.User != nil || parsed.Fragment != "" {
|
|
return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu source URL", creator.ErrInvalid)
|
|
}
|
|
if parsed.Port() != "" {
|
|
return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu source URL", creator.ErrInvalid)
|
|
}
|
|
query, err := parseAccessQuery(parsed)
|
|
if err != nil {
|
|
return LinkContext{}, err
|
|
}
|
|
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
|
|
context := LinkContext{Token: query.Token, Source: query.Source, CanonicalURL: parsed.String()}
|
|
switch parsed.Hostname() {
|
|
case "www.xiaohongshu.com":
|
|
switch {
|
|
case len(parts) == 2 && parts[0] == "explore" && keyPattern.MatchString(parts[1]):
|
|
context.Kind, context.WorkKey = LinkNote, parts[1]
|
|
case len(parts) == 3 && parts[0] == "user" && parts[1] == "profile" && keyPattern.MatchString(parts[2]):
|
|
context.Kind, context.AccountKey = LinkHomepage, parts[2]
|
|
default:
|
|
return LinkContext{}, fmt.Errorf("%w: unsupported xiaohongshu source path", creator.ErrInvalid)
|
|
}
|
|
case "xhslink.com", "www.xhslink.com":
|
|
if len(parts) < 1 || len(parts) > 4 || parsed.RawQuery != "" {
|
|
return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu share URL", creator.ErrInvalid)
|
|
}
|
|
context.Kind = LinkShare
|
|
default:
|
|
return LinkContext{}, fmt.Errorf("%w: unsupported xiaohongshu source host", creator.ErrInvalid)
|
|
}
|
|
return context, nil
|
|
}
|
|
|
|
type accessQuery struct {
|
|
Token string
|
|
Source string
|
|
}
|
|
|
|
func parseAccessQuery(parsed *url.URL) (accessQuery, error) {
|
|
values, err := url.ParseQuery(parsed.RawQuery)
|
|
if err != nil {
|
|
return accessQuery{}, fmt.Errorf("%w: invalid xiaohongshu access query", creator.ErrInvalid)
|
|
}
|
|
for key, items := range values {
|
|
if key != "xsec_token" && key != "xsec_source" || len(items) != 1 || strings.ContainsAny(items[0], "\r\n") || len(items[0]) > 2048 {
|
|
return accessQuery{}, fmt.Errorf("%w: unsupported xiaohongshu access query", creator.ErrInvalid)
|
|
}
|
|
}
|
|
result := accessQuery{Token: values.Get("xsec_token"), Source: values.Get("xsec_source")}
|
|
if result.Source != "" && !keyPattern.MatchString(result.Source) {
|
|
return accessQuery{}, fmt.Errorf("%w: invalid xiaohongshu access source", creator.ErrInvalid)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func ValidateSourceURL(raw, expectedAccountKey string) error {
|
|
link, err := ParseSourceURL(raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if link.Kind != LinkHomepage || link.AccountKey != expectedAccountKey {
|
|
return fmt.Errorf("%w: xiaohongshu homepage does not match the account", creator.ErrConflict)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) resolveSourceURL(ctx context.Context, raw string) (LinkContext, error) {
|
|
link, err := ParseSourceURL(raw)
|
|
if err != nil {
|
|
return LinkContext{}, err
|
|
}
|
|
if link.Kind != LinkShare {
|
|
return link, nil
|
|
}
|
|
resolver, ok := c.Browser.(ShareResolver)
|
|
if !ok {
|
|
return LinkContext{}, fmt.Errorf("%w: xiaohongshu share URL resolver is unavailable", creator.ErrUnavailable)
|
|
}
|
|
canonical, err := resolver.Resolve(ctx, link.CanonicalURL)
|
|
if err != nil {
|
|
return LinkContext{}, err
|
|
}
|
|
resolved, err := ParseSourceURL(canonical)
|
|
if err != nil || resolved.Kind != LinkNote {
|
|
return LinkContext{}, fmt.Errorf("%w: xiaohongshu share URL did not resolve to a note", creator.ErrInvalid)
|
|
}
|
|
return resolved, nil
|
|
}
|
|
|
|
func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) {
|
|
if c == nil {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: xiaohongshu collector is nil", creator.ErrUnavailable)
|
|
}
|
|
if c.AccountKey != "" {
|
|
accountKey = c.AccountKey
|
|
}
|
|
if c.Browser == nil || !keyPattern.MatchString(accountKey) || !validCursor(cursor) {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu work collection request", creator.ErrInvalid)
|
|
}
|
|
access, err := c.defaultContext()
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
query := url.Values{
|
|
"user_id": {accountKey},
|
|
"cursor": {cursor},
|
|
"num": {"30"},
|
|
"image_formats": {"jpg,webp,avif"},
|
|
}
|
|
if access.Source != "" {
|
|
query.Set("xsec_source", access.Source)
|
|
}
|
|
if access.Token != "" {
|
|
query.Set("xsec_token", access.Token)
|
|
}
|
|
response, err := c.Browser.Get(ctx, APIOrigin+UserPostedPath+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
if err := responseError(response, "works"); err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, access)
|
|
if !ok {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu works response", creator.ErrInvalid)
|
|
}
|
|
return creator.WorkPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil
|
|
}
|
|
|
|
func (c *Collector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) {
|
|
if c == nil || c.Browser == nil || !keyPattern.MatchString(workKey) || !validCursor(cursor) {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comment collection request", creator.ErrInvalid)
|
|
}
|
|
access, err := c.contextFor(workKey)
|
|
if err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
query := url.Values{
|
|
"note_id": {workKey},
|
|
"cursor": {cursor},
|
|
"top_comment_id": {""},
|
|
"image_formats": {"jpg,webp,avif"},
|
|
}
|
|
if access.Source != "" {
|
|
query.Set("xsec_source", access.Source)
|
|
}
|
|
if access.Token != "" {
|
|
query.Set("xsec_token", access.Token)
|
|
}
|
|
response, err := c.Browser.Get(ctx, APIOrigin+CommentsPath+"?"+query.Encode())
|
|
if err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
if err := responseError(response, "comments"); err != nil {
|
|
return creator.CommentPage{}, err
|
|
}
|
|
items, nextCursor, hasMore, ok := parseCommentsPage(response.Body, workKey)
|
|
if !ok {
|
|
return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comments response", creator.ErrInvalid)
|
|
}
|
|
return creator.CommentPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil
|
|
}
|
|
|
|
func (c *Collector) SearchNotes(ctx context.Context, queryText string, page int) (creator.WorkPage, error) {
|
|
if c == nil || c.Browser == nil || strings.TrimSpace(queryText) == "" || page < 1 || page > 10000 || utf8.RuneCountInString(queryText) > 200 {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search request", creator.ErrInvalid)
|
|
}
|
|
searchID, err := randomID()
|
|
if err != nil {
|
|
return creator.WorkPage{}, fmt.Errorf("create xiaohongshu search id: %w", err)
|
|
}
|
|
sessionID, err := randomID()
|
|
if err != nil {
|
|
return creator.WorkPage{}, fmt.Errorf("create xiaohongshu search session id: %w", err)
|
|
}
|
|
body, err := json.Marshal(map[string]any{
|
|
"keyword": queryText,
|
|
"page": page,
|
|
"page_size": 20,
|
|
"search_id": searchID,
|
|
"sort": "general",
|
|
"note_type": 0,
|
|
"ext_flags": []any{},
|
|
"geo": "",
|
|
"image_formats": []string{"jpg", "webp", "avif"},
|
|
"session_id": sessionID,
|
|
})
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
response, err := c.Browser.Post(ctx, SearchOrigin+SearchNotesPath, body)
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
if err := responseError(response, "search"); err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
fallback, err := c.defaultContext()
|
|
if err != nil {
|
|
return creator.WorkPage{}, err
|
|
}
|
|
items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, fallback)
|
|
if !ok {
|
|
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search response", creator.ErrInvalid)
|
|
}
|
|
return creator.WorkPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil
|
|
}
|
|
|
|
func randomID() (string, error) {
|
|
value := make([]byte, 16)
|
|
if _, err := rand.Read(value); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(value), nil
|
|
}
|
|
|
|
func (c *Collector) defaultContext() (accessContext, error) {
|
|
if c == nil || strings.TrimSpace(c.HomepageURL) == "" {
|
|
return accessContext{}, nil
|
|
}
|
|
link, err := ParseSourceURL(c.HomepageURL)
|
|
if err != nil {
|
|
return accessContext{}, err
|
|
}
|
|
if link.Kind != LinkHomepage {
|
|
return accessContext{}, fmt.Errorf("%w: xiaohongshu homepage URL is required", creator.ErrInvalid)
|
|
}
|
|
return accessContext{Token: link.Token, Source: link.Source}, nil
|
|
}
|
|
|
|
func (c *Collector) SetWorkContext(workKey, originalURL string) error {
|
|
if c == nil || !keyPattern.MatchString(workKey) {
|
|
return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid)
|
|
}
|
|
access, err := c.defaultContext()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(originalURL) != "" {
|
|
link, parseErr := ParseSourceURL(originalURL)
|
|
if parseErr != nil || link.Kind != LinkNote || link.WorkKey != workKey {
|
|
return fmt.Errorf("%w: invalid xiaohongshu work URL", creator.ErrInvalid)
|
|
}
|
|
access.Token, access.Source = link.Token, link.Source
|
|
}
|
|
if len(access.Token) > 2048 || access.Source != "" && !keyPattern.MatchString(access.Source) {
|
|
return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid)
|
|
}
|
|
c.contexts.Store(workKey, access)
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) contextFor(workKey string) (accessContext, error) {
|
|
if c != nil {
|
|
if value, ok := c.contexts.Load(workKey); ok {
|
|
if access, ok := value.(accessContext); ok {
|
|
return access, nil
|
|
}
|
|
}
|
|
}
|
|
return c.defaultContext()
|
|
}
|
|
|
|
func parseIdentity(body []byte) (Identity, bool) {
|
|
if len(body) == 0 || len(body) > 1<<20 {
|
|
return Identity{}, false
|
|
}
|
|
var envelope struct {
|
|
Success *bool `json:"success"`
|
|
Data *struct {
|
|
UserID string `json:"user_id"`
|
|
Nickname string `json:"nickname"`
|
|
UserInfo *struct {
|
|
UserID string `json:"user_id"`
|
|
Nickname string `json:"nickname"`
|
|
} `json:"user_info"`
|
|
} `json:"data"`
|
|
}
|
|
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil {
|
|
return Identity{}, false
|
|
}
|
|
userID, nickname := envelope.Data.UserID, envelope.Data.Nickname
|
|
if envelope.Data.UserInfo != nil {
|
|
if userID == "" {
|
|
userID = envelope.Data.UserInfo.UserID
|
|
}
|
|
if nickname == "" {
|
|
nickname = envelope.Data.UserInfo.Nickname
|
|
}
|
|
}
|
|
if !keyPattern.MatchString(userID) || utf8.RuneCountInString(nickname) > 256 {
|
|
return Identity{}, false
|
|
}
|
|
return Identity{UserID: userID, Nickname: nickname}, true
|
|
}
|
|
|
|
func parseWorksPage(body []byte, collector *Collector, fallback accessContext) ([]creator.WorkInput, string, bool, bool) {
|
|
if len(body) == 0 || len(body) > 4<<20 {
|
|
return nil, "", false, false
|
|
}
|
|
var envelope struct {
|
|
Success *bool `json:"success"`
|
|
Data *struct {
|
|
Cursor json.RawMessage `json:"cursor"`
|
|
HasMore *bool `json:"has_more"`
|
|
Notes []json.RawMessage `json:"notes"`
|
|
Items []json.RawMessage `json:"items"`
|
|
} `json:"data"`
|
|
}
|
|
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil || envelope.Data.HasMore == nil {
|
|
return nil, "", false, false
|
|
}
|
|
rawNotes := envelope.Data.Notes
|
|
if rawNotes == nil {
|
|
rawNotes = envelope.Data.Items
|
|
}
|
|
if rawNotes == nil || len(rawNotes) > 30 {
|
|
return nil, "", false, false
|
|
}
|
|
items := make([]creator.WorkInput, 0, len(rawNotes))
|
|
seen := make(map[string]struct{}, len(rawNotes))
|
|
for _, raw := range rawNotes {
|
|
item, access, ok := parseWork(raw, fallback)
|
|
if !ok {
|
|
return nil, "", false, false
|
|
}
|
|
if _, exists := seen[item.WorkKey]; exists {
|
|
return nil, "", false, false
|
|
}
|
|
seen[item.WorkKey] = struct{}{}
|
|
if collector != nil {
|
|
item.SourceType = collector.SourceType
|
|
if item.SourceType == "" {
|
|
item.SourceType = creator.SourceCompetitor
|
|
}
|
|
item.SourceID = collector.SourceID
|
|
if item.SourceID == "" {
|
|
item.SourceID = collector.AccountKey
|
|
}
|
|
collector.contexts.Store(item.WorkKey, access)
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
nextCursor, ok := cursorValue(envelope.Data.Cursor)
|
|
if !ok {
|
|
return nil, "", false, false
|
|
}
|
|
if *envelope.Data.HasMore && nextCursor == "" {
|
|
return nil, "", false, false
|
|
}
|
|
return items, nextCursor, *envelope.Data.HasMore, true
|
|
}
|
|
|
|
func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput, accessContext, bool) {
|
|
object, ok := objectValue(raw)
|
|
if !ok {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
outer := object
|
|
if nested := firstObject(object, "note_card", "noteCard"); nested != nil {
|
|
object = nested
|
|
}
|
|
id := firstString(object, "note_id", "id")
|
|
if !keyPattern.MatchString(id) {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
title := firstString(object, "display_title", "title")
|
|
body := firstString(object, "desc", "description", "content")
|
|
if utf8.RuneCountInString(title) > 4096 || utf8.RuneCountInString(body) > 100000 {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
user := firstObject(object, "user", "user_info", "author")
|
|
authorKey := firstString(user, "user_id", "userid", "uid", "id")
|
|
authorName := firstString(user, "nickname", "name")
|
|
if authorKey != "" && !keyPattern.MatchString(authorKey) {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
published, publishedOK := optionalTimestamp(object, "time", "create_time", "last_update_time")
|
|
if !publishedOK {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
likes, likesOK := optionalInt(object, "likes", "liked_count")
|
|
comments, commentsOK := optionalInt(object, "comments_count", "comment_count")
|
|
shares, sharesOK := optionalInt(object, "shares", "shared_count")
|
|
interact := firstObject(object, "interact_info", "interactInfo", "statistics")
|
|
if interact != nil {
|
|
if likes == nil {
|
|
likes, likesOK = optionalInt(interact, "liked_count", "likes")
|
|
}
|
|
if comments == nil {
|
|
comments, commentsOK = optionalInt(interact, "comment_count", "comments_count")
|
|
}
|
|
if shares == nil {
|
|
shares, sharesOK = optionalInt(interact, "shared_count", "shares")
|
|
}
|
|
}
|
|
if !likesOK || !commentsOK || !sharesOK {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
access := fallback
|
|
if token := firstString(outer, "xsec_token"); token != "" {
|
|
access.Token = token
|
|
}
|
|
if source := firstString(outer, "xsec_source"); source != "" {
|
|
access.Source = source
|
|
}
|
|
if token := firstString(object, "xsec_token"); token != "" {
|
|
access.Token = token
|
|
}
|
|
if source := firstString(object, "xsec_source"); source != "" {
|
|
access.Source = source
|
|
}
|
|
if utf8.RuneCountInString(access.Token) > 2048 || access.Source != "" && !keyPattern.MatchString(access.Source) {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
originalURL := firstString(object, "original_url", "note_url", "url")
|
|
if originalURL == "" {
|
|
originalURL = firstString(outer, "original_url", "note_url", "url")
|
|
}
|
|
if originalURL != "" {
|
|
link, err := ParseSourceURL(originalURL)
|
|
if err != nil || link.Kind != LinkNote || link.WorkKey != id {
|
|
return creator.WorkInput{}, accessContext{}, false
|
|
}
|
|
originalURL = link.CanonicalURL
|
|
} else {
|
|
originalURL = noteURL(id, access)
|
|
}
|
|
cover := coverURL(object)
|
|
status := "pending_verification"
|
|
if published != nil {
|
|
status = "verified"
|
|
}
|
|
return creator.WorkInput{
|
|
Platform: creator.PlatformXiaohongshu,
|
|
RawPayload: string(raw),
|
|
WorkKey: id,
|
|
AuthorKey: authorKey,
|
|
AuthorName: authorName,
|
|
Title: title,
|
|
Body: body,
|
|
PublishedAt: published,
|
|
PublishedAtStatus: status,
|
|
OriginalURL: originalURL,
|
|
CoverURL: cover,
|
|
Likes: likes,
|
|
CommentsCount: comments,
|
|
Shares: shares,
|
|
}, access, true
|
|
}
|
|
|
|
func parseCommentsPage(body []byte, workKey string) ([]creator.CommentInput, string, bool, bool) {
|
|
if len(body) == 0 || len(body) > 4<<20 || !keyPattern.MatchString(workKey) {
|
|
return nil, "", false, false
|
|
}
|
|
var envelope struct {
|
|
Success *bool `json:"success"`
|
|
Data *struct {
|
|
Cursor json.RawMessage `json:"cursor"`
|
|
HasMore *bool `json:"has_more"`
|
|
Comments []json.RawMessage `json:"comments"`
|
|
} `json:"data"`
|
|
}
|
|
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil || envelope.Data.HasMore == nil || envelope.Data.Comments == nil || len(envelope.Data.Comments) > 100 {
|
|
return nil, "", false, false
|
|
}
|
|
items := make([]creator.CommentInput, 0, len(envelope.Data.Comments))
|
|
seen := make(map[string]struct{}, len(envelope.Data.Comments))
|
|
for _, raw := range envelope.Data.Comments {
|
|
object, ok := objectValue(raw)
|
|
if !ok {
|
|
return nil, "", false, false
|
|
}
|
|
id := firstString(object, "id", "comment_id", "commentId")
|
|
if !keyPattern.MatchString(id) {
|
|
return nil, "", false, false
|
|
}
|
|
if parent := firstString(object, "parent_comment_id", "parent_id", "reply_id"); parent != "" {
|
|
return nil, "", false, false
|
|
}
|
|
if typ := firstString(object, "comment_type", "type"); typ == "reply" || typ == "sub" {
|
|
return nil, "", false, false
|
|
}
|
|
if _, exists := seen[id]; exists {
|
|
return nil, "", false, false
|
|
}
|
|
seen[id] = struct{}{}
|
|
user := firstObject(object, "user_info", "user", "author")
|
|
published, publishedOK := optionalTimestamp(object, "create_time", "time", "created_at")
|
|
if !publishedOK {
|
|
return nil, "", false, false
|
|
}
|
|
content := firstString(object, "content", "text", "comment")
|
|
if content == "" || utf8.RuneCountInString(content) > 20000 {
|
|
return nil, "", false, false
|
|
}
|
|
items = append(items, creator.CommentInput{
|
|
Platform: creator.PlatformXiaohongshu,
|
|
RawPayload: string(raw),
|
|
CommentKey: id,
|
|
WorkID: workKey,
|
|
AuthorUID: firstString(user, "user_id", "uid", "id"),
|
|
AuthorName: firstString(user, "nickname", "name"),
|
|
Content: content,
|
|
PublishedAt: published,
|
|
CommentType: "top_level",
|
|
})
|
|
}
|
|
nextCursor, ok := cursorValue(envelope.Data.Cursor)
|
|
if !ok || *envelope.Data.HasMore && nextCursor == "" {
|
|
return nil, "", false, false
|
|
}
|
|
return items, nextCursor, *envelope.Data.HasMore, true
|
|
}
|
|
|
|
func responseError(response Response, resource string) error {
|
|
if strings.TrimSpace(response.Challenge) != "" {
|
|
return fmt.Errorf("%w: xiaohongshu %s challenge %s", creator.ErrUnavailable, resource, response.Challenge)
|
|
}
|
|
if response.Status >= 200 && response.Status < 300 {
|
|
return platformFailure(response.Body, resource)
|
|
}
|
|
switch response.Status {
|
|
case 401, 403, 406, 461:
|
|
return fmt.Errorf("%w: xiaohongshu %s authentication/session rejected with HTTP %d", creator.ErrConflict, resource, response.Status)
|
|
case 429:
|
|
return fmt.Errorf("%w: xiaohongshu %s rate limited", creator.ErrUnavailable, resource)
|
|
default:
|
|
return fmt.Errorf("xiaohongshu %s returned HTTP %d", resource, response.Status)
|
|
}
|
|
}
|
|
|
|
func platformFailure(body []byte, resource string) error {
|
|
var envelope struct {
|
|
Success *bool `json:"success"`
|
|
Code string `json:"code"`
|
|
Msg string `json:"msg"`
|
|
Message string `json:"message"`
|
|
}
|
|
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || *envelope.Success {
|
|
return nil
|
|
}
|
|
reason := strings.TrimSpace(envelope.Msg)
|
|
if reason == "" {
|
|
reason = strings.TrimSpace(envelope.Message)
|
|
}
|
|
if len(reason) > 512 {
|
|
reason = reason[:512]
|
|
}
|
|
if strings.Contains(strings.ToLower(reason), "rate") || strings.Contains(reason, "频繁") || strings.Contains(reason, "验证") {
|
|
return fmt.Errorf("%w: xiaohongshu %s rejected request code=%s message=%s", creator.ErrUnavailable, resource, envelope.Code, reason)
|
|
}
|
|
return fmt.Errorf("%w: xiaohongshu %s rejected request code=%s message=%s", creator.ErrConflict, resource, envelope.Code, reason)
|
|
}
|
|
|
|
func validCursor(cursor string) bool {
|
|
return cursor == "" || len(cursor) <= 512 && !strings.ContainsAny(cursor, "\r\n")
|
|
}
|
|
|
|
func objectValue(raw json.RawMessage) (map[string]json.RawMessage, bool) {
|
|
var object map[string]json.RawMessage
|
|
if len(raw) == 0 || json.Unmarshal(raw, &object) != nil || object == nil {
|
|
return nil, false
|
|
}
|
|
return object, true
|
|
}
|
|
|
|
func firstObject(object map[string]json.RawMessage, names ...string) map[string]json.RawMessage {
|
|
for _, name := range names {
|
|
if value, ok := object[name]; ok {
|
|
if nested, ok := objectValue(value); ok {
|
|
return nested
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func firstString(object map[string]json.RawMessage, names ...string) string {
|
|
for _, name := range names {
|
|
value, ok := object[name]
|
|
if !ok || string(value) == "null" {
|
|
continue
|
|
}
|
|
var text string
|
|
if json.Unmarshal(value, &text) == nil {
|
|
return strings.TrimSpace(text)
|
|
}
|
|
var number json.Number
|
|
decoder := json.NewDecoder(bytes.NewReader(value))
|
|
decoder.UseNumber()
|
|
if decoder.Decode(&number) == nil {
|
|
return number.String()
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func optionalInt(object map[string]json.RawMessage, names ...string) (*int64, bool) {
|
|
for _, name := range names {
|
|
value, ok := object[name]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if string(value) == "null" {
|
|
return nil, true
|
|
}
|
|
var number json.Number
|
|
decoder := json.NewDecoder(bytes.NewReader(value))
|
|
decoder.UseNumber()
|
|
if decoder.Decode(&number) != nil {
|
|
var text string
|
|
if json.Unmarshal(value, &text) != nil {
|
|
return nil, false
|
|
}
|
|
number = json.Number(text)
|
|
}
|
|
parsed, err := strconv.ParseInt(number.String(), 10, 64)
|
|
if err != nil || parsed < 0 {
|
|
return nil, false
|
|
}
|
|
return &parsed, true
|
|
}
|
|
return nil, true
|
|
}
|
|
|
|
func optionalTimestamp(object map[string]json.RawMessage, names ...string) (*time.Time, bool) {
|
|
for _, name := range names {
|
|
value, ok := object[name]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if string(value) == "null" {
|
|
return nil, true
|
|
}
|
|
var number json.Number
|
|
decoder := json.NewDecoder(bytes.NewReader(value))
|
|
decoder.UseNumber()
|
|
if decoder.Decode(&number) != nil {
|
|
var text string
|
|
if json.Unmarshal(value, &text) != nil {
|
|
return nil, false
|
|
}
|
|
number = json.Number(text)
|
|
}
|
|
parsed, err := strconv.ParseInt(number.String(), 10, 64)
|
|
if err != nil || parsed <= 0 {
|
|
return nil, false
|
|
}
|
|
if parsed > 1_000_000_000_000 {
|
|
parsed /= 1000
|
|
}
|
|
if parsed <= 0 || parsed > 4_102_444_800 {
|
|
return nil, false
|
|
}
|
|
when := time.Unix(parsed, 0).UTC()
|
|
return &when, true
|
|
}
|
|
return nil, true
|
|
}
|
|
|
|
func cursorValue(raw json.RawMessage) (string, bool) {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return "", true
|
|
}
|
|
var text string
|
|
if json.Unmarshal(raw, &text) == nil {
|
|
return strings.TrimSpace(text), validCursor(strings.TrimSpace(text))
|
|
}
|
|
var number json.Number
|
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
|
decoder.UseNumber()
|
|
if decoder.Decode(&number) != nil {
|
|
return "", false
|
|
}
|
|
return number.String(), validCursor(number.String())
|
|
}
|
|
|
|
func coverURL(object map[string]json.RawMessage) string {
|
|
images, ok := object["image_list"]
|
|
if ok {
|
|
var values []json.RawMessage
|
|
if json.Unmarshal(images, &values) == nil {
|
|
for _, value := range values {
|
|
if image, ok := objectValue(value); ok {
|
|
if cover := firstString(image, "url_default", "url_pre", "url_original", "url"); cover != "" {
|
|
return cover
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if cover := firstObject(object, "cover"); cover != nil {
|
|
return firstString(cover, "url_default", "url_pre", "url_original", "url")
|
|
}
|
|
return firstString(object, "cover_url", "cover")
|
|
}
|
|
|
|
func noteURL(id string, access accessContext) string {
|
|
query := url.Values{}
|
|
if access.Token != "" {
|
|
query.Set("xsec_token", access.Token)
|
|
}
|
|
if access.Source != "" {
|
|
query.Set("xsec_source", access.Source)
|
|
}
|
|
result := "https://www.xiaohongshu.com/explore/" + url.PathEscape(id)
|
|
if encoded := query.Encode(); encoded != "" {
|
|
result += "?" + encoded
|
|
}
|
|
return result
|
|
}
|