feat: restore Xiaohongshu read-only collection
This commit is contained in:
@@ -980,7 +980,7 @@ func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore
|
||||
if err != nil {
|
||||
return creator.LoginResult{}, err
|
||||
}
|
||||
if account.Platform != creator.PlatformDouyin || profile.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || profile.PlatformAccountKey == "" || account.PlatformAccountKey != profile.PlatformAccountKey {
|
||||
if account.Platform != profile.Platform || (account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu) || account.AuthorizationStatus != "authorized" || profile.PlatformAccountKey == "" || account.PlatformAccountKey != profile.PlatformAccountKey {
|
||||
return creator.LoginResult{}, creator.ErrConflict
|
||||
}
|
||||
environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID)
|
||||
@@ -994,8 +994,7 @@ func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore
|
||||
if err != nil {
|
||||
return creator.LoginResult{}, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
|
||||
}
|
||||
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
|
||||
uid, err := browser.Identity(ctx, profile.PlatformAccountKey)
|
||||
uid, err := verifyCreatorPlatformIdentity(ctx, account.Platform, gateway, environment, profile.PlatformAccountKey)
|
||||
if err != nil {
|
||||
return creator.LoginResult{}, fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, err)
|
||||
}
|
||||
@@ -1126,8 +1125,8 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p
|
||||
markErr := store.MarkCompetitorSync(ctx, competitorID, leaseToken, "blocked", "", blockErr.Error(), nil)
|
||||
return creator.CollectionReport{}, errors.Join(blockErr, markErr)
|
||||
}
|
||||
if competitor.Platform != creator.PlatformDouyin {
|
||||
return blocked(fmt.Errorf("%w: 小红书采集器尚未完成平台能力验证", creator.ErrUnavailable))
|
||||
if competitor.Platform != creator.PlatformDouyin && competitor.Platform != creator.PlatformXiaohongshu {
|
||||
return blocked(fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, competitor.Platform))
|
||||
}
|
||||
account, err := phaseAStore.GetAccount(ctx, accountID)
|
||||
if err != nil {
|
||||
@@ -1154,16 +1153,10 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p
|
||||
if err != nil {
|
||||
return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err))
|
||||
}
|
||||
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
|
||||
if _, identityErr := browser.Identity(ctx, account.PlatformAccountKey); identityErr != nil {
|
||||
return blocked(fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr))
|
||||
}
|
||||
collector := douyin.CreatorCollector{Browser: browser, AccountKey: competitor.PlatformAccountKey, SourceType: creator.SourceCompetitor, SourceID: competitor.ID}
|
||||
canonicalSecUID, err := collector.CanonicalSecUID(ctx, account.PlatformAccountKey)
|
||||
collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceCompetitor, competitor.ID)
|
||||
if err != nil {
|
||||
return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err))
|
||||
}
|
||||
collector.AccountKey = canonicalSecUID
|
||||
collectionNow := now
|
||||
if competitor.NextSyncAt != nil && !competitor.NextSyncAt.After(now) {
|
||||
collectionNow = competitor.NextSyncAt.UTC()
|
||||
@@ -1279,7 +1272,7 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" {
|
||||
if (account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu) || account.AuthorizationStatus != "authorized" || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" {
|
||||
return creator.ErrConflict
|
||||
}
|
||||
environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID)
|
||||
@@ -1290,19 +1283,13 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
|
||||
}
|
||||
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
|
||||
if _, identityErr := browser.Identity(ctx, account.PlatformAccountKey); identityErr != nil {
|
||||
return fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr)
|
||||
}
|
||||
collector := douyin.CreatorCollector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: work.SourceType, SourceID: work.SourceID}
|
||||
canonical, err := collector.CanonicalSecUID(ctx, account.PlatformAccountKey)
|
||||
collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, work.SourceType, work.SourceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)
|
||||
}
|
||||
collector.AccountKey = canonical
|
||||
cursor := ""
|
||||
for page := 0; page < 100; page++ {
|
||||
result, pageErr := collector.ListWorks(ctx, canonical, cursor)
|
||||
result, pageErr := collector.ListWorks(ctx, collectionKey, cursor)
|
||||
if pageErr != nil {
|
||||
return pageErr
|
||||
}
|
||||
@@ -1335,7 +1322,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" {
|
||||
if account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu || account.AuthorizationStatus != "authorized" {
|
||||
return creator.ErrConflict
|
||||
}
|
||||
profile, err := store.GetAccountProfile(ctx, accountID)
|
||||
@@ -1349,9 +1336,6 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blockOwned := func(blockErr error) error {
|
||||
return errors.Join(blockErr, store.MarkCollectionBlocked(ctx, creator.SourceOwned, accountID, blockErr.Error(), now, settings.LookbackDays))
|
||||
}
|
||||
environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err)
|
||||
@@ -1363,10 +1347,6 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
|
||||
}
|
||||
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
|
||||
if _, identityErr := browser.Identity(ctx, account.PlatformAccountKey); identityErr != nil {
|
||||
return blockOwned(fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr))
|
||||
}
|
||||
syncLease, err := store.ClaimSourceSync(ctx, creator.SourceOwned, account.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1376,13 +1356,11 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph
|
||||
logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed")
|
||||
}
|
||||
}()
|
||||
collector := douyin.CreatorCollector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: creator.SourceOwned, SourceID: account.ID}
|
||||
canonicalSecUID, err := collector.CanonicalSecUID(ctx, account.PlatformAccountKey)
|
||||
collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceOwned, account.ID)
|
||||
if err != nil {
|
||||
blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays)
|
||||
return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr)
|
||||
}
|
||||
collector.AccountKey = canonicalSecUID
|
||||
_, collectionNow, windowErr := store.NextCollectionWindow(ctx, creator.SourceOwned, account.ID, now, time.Duration(settings.NewWorkIntervalSeconds)*time.Second, settings.LookbackDays)
|
||||
if windowErr != nil {
|
||||
return windowErr
|
||||
|
||||
@@ -20,7 +20,7 @@ type creatorMaterialDownloader struct {
|
||||
}
|
||||
|
||||
func (downloader creatorMaterialDownloader) Download(ctx context.Context, work creator.Work, destination string) error {
|
||||
if work.Platform != creator.PlatformDouyin || downloader.store == nil || downloader.phaseAStore == nil || downloader.hubStore == nil {
|
||||
if (work.Platform != creator.PlatformDouyin && work.Platform != creator.PlatformXiaohongshu) || downloader.store == nil || downloader.phaseAStore == nil || downloader.hubStore == nil {
|
||||
return fmt.Errorf("%w: creator media gateway is unavailable", creator.ErrUnavailable)
|
||||
}
|
||||
accountID := work.SourceID
|
||||
@@ -39,7 +39,7 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if account.Platform != creator.PlatformDouyin || profile.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || profile.LoginStatus != "logged_in" || account.PlatformAccountKey != profile.PlatformAccountKey {
|
||||
if account.Platform != work.Platform || profile.Platform != work.Platform || account.AuthorizationStatus != "authorized" || profile.LoginStatus != "logged_in" || account.PlatformAccountKey != profile.PlatformAccountKey {
|
||||
return fmt.Errorf("%w: media account identity is not verified", creator.ErrConflict)
|
||||
}
|
||||
environment, err := downloader.hubStore.GetEnvironmentContextForAccount(ctx, accountID)
|
||||
@@ -50,11 +50,21 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: media gateway unavailable: %v", creator.ErrUnavailable, err)
|
||||
}
|
||||
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
|
||||
if _, err := browser.Identity(ctx, profile.PlatformAccountKey); err != nil {
|
||||
if _, err := verifyCreatorPlatformIdentity(ctx, work.Platform, gateway, environment, profile.PlatformAccountKey); err != nil {
|
||||
return fmt.Errorf("%w: media browser identity verification failed: %v", creator.ErrConflict, err)
|
||||
}
|
||||
return browser.Media(ctx, work.OriginalURL, destination)
|
||||
if work.Platform == creator.PlatformDouyin {
|
||||
return (creatorGatewayBrowser{gateway: gateway, environment: environment}).Media(ctx, work.OriginalURL, destination)
|
||||
}
|
||||
data, contentType, err := (xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}).Media(ctx, work.OriginalURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contentType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0]))
|
||||
if !strings.HasPrefix(contentType, "video/") && contentType != "application/octet-stream" {
|
||||
return fmt.Errorf("xiaohongshu media response is not a video")
|
||||
}
|
||||
return writeCreatorMedia(destination, data)
|
||||
}
|
||||
|
||||
func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, workID string) (creator.MaterialJob, error) {
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
||||
"git.ipao.vip/rogee/creator-hub/internal/douyin"
|
||||
"git.ipao.vip/rogee/creator-hub/internal/hub"
|
||||
"git.ipao.vip/rogee/creator-hub/internal/xiaohongshu"
|
||||
)
|
||||
|
||||
type xiaohongshuGatewayBrowser struct {
|
||||
gateway hub.Gateway
|
||||
environment hub.EnvironmentContext
|
||||
}
|
||||
|
||||
func (browser xiaohongshuGatewayBrowser) generation() (map[string]any, error) {
|
||||
request, err := (douyinGatewayBrowser{gateway: browser.gateway, environment: browser.environment}).request()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"binding_version": request.BindingVersion,
|
||||
"runtime_id": request.RuntimeID,
|
||||
"network_id": request.NetworkID,
|
||||
"network_exit_id": request.NetworkExitID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser xiaohongshuGatewayBrowser) Get(ctx context.Context, target string) (xiaohongshu.Response, error) {
|
||||
request, err := browser.generation()
|
||||
if err != nil {
|
||||
return xiaohongshu.Response{}, err
|
||||
}
|
||||
request["url"] = target
|
||||
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
|
||||
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/get", request, 30*time.Second)
|
||||
if err != nil || status != http.StatusOK {
|
||||
return xiaohongshu.Response{}, errors.New("restricted Xiaohongshu browser operation failed")
|
||||
}
|
||||
return decodeXiaohongshuResponse(body)
|
||||
}
|
||||
|
||||
func (browser xiaohongshuGatewayBrowser) Post(ctx context.Context, target string, payload []byte) (xiaohongshu.Response, error) {
|
||||
if len(payload) == 0 || len(payload) > 4<<20 {
|
||||
return xiaohongshu.Response{}, errors.New("invalid Xiaohongshu browser body")
|
||||
}
|
||||
var bodyValue any
|
||||
if err := json.Unmarshal(payload, &bodyValue); err != nil {
|
||||
return xiaohongshu.Response{}, fmt.Errorf("invalid Xiaohongshu browser body: %w", err)
|
||||
}
|
||||
if _, ok := bodyValue.(map[string]any); !ok {
|
||||
return xiaohongshu.Response{}, errors.New("Xiaohongshu browser body must be an object")
|
||||
}
|
||||
request, err := browser.generation()
|
||||
if err != nil {
|
||||
return xiaohongshu.Response{}, err
|
||||
}
|
||||
request["url"] = target
|
||||
request["body"] = bodyValue
|
||||
status, responseBody, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
|
||||
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/post", request, 30*time.Second)
|
||||
if err != nil || status != http.StatusOK {
|
||||
return xiaohongshu.Response{}, errors.New("restricted Xiaohongshu browser POST failed")
|
||||
}
|
||||
return decodeXiaohongshuResponse(responseBody)
|
||||
}
|
||||
|
||||
func (browser xiaohongshuGatewayBrowser) Identity(ctx context.Context, expectedKey string) (string, error) {
|
||||
request, err := browser.generation()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request["expected_account_key"] = expectedKey
|
||||
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
|
||||
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/identity", request, 30*time.Second)
|
||||
if err != nil || status != http.StatusOK {
|
||||
return "", errors.New("Xiaohongshu identity verification failed")
|
||||
}
|
||||
var identity struct {
|
||||
UID string `json:"uid"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &identity); err != nil || strings.TrimSpace(identity.UID) == "" {
|
||||
return "", errors.New("Xiaohongshu identity response omitted uid")
|
||||
}
|
||||
return identity.UID, nil
|
||||
}
|
||||
|
||||
func (browser xiaohongshuGatewayBrowser) Media(ctx context.Context, target string) ([]byte, string, error) {
|
||||
request, err := browser.generation()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
request["url"] = target
|
||||
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
|
||||
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/media", request, 90*time.Second)
|
||||
if err != nil || status != http.StatusOK {
|
||||
return nil, "", errors.New("restricted Xiaohongshu media request failed")
|
||||
}
|
||||
var response struct {
|
||||
Status int `json:"status"`
|
||||
ContentType string `json:"content_type"`
|
||||
BodyBase64 string `json:"body_base64"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &response); err != nil || response.Status < 200 || response.Status >= 300 || response.BodyBase64 == "" {
|
||||
return nil, "", errors.New("Xiaohongshu media response is invalid")
|
||||
}
|
||||
data, err := decodeBase64(response.BodyBase64)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return data, response.ContentType, nil
|
||||
}
|
||||
|
||||
func decodeXiaohongshuResponse(body []byte) (xiaohongshu.Response, error) {
|
||||
var response struct {
|
||||
Status int `json:"status"`
|
||||
Body string `json:"body"`
|
||||
Challenge string `json:"challenge"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &response); err != nil || response.Status < 100 || response.Status > 599 {
|
||||
return xiaohongshu.Response{}, errors.New("restricted Xiaohongshu browser returned an invalid response")
|
||||
}
|
||||
return xiaohongshu.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil
|
||||
}
|
||||
|
||||
func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, accountKey, sourceType, sourceID string) (creator.PlatformCollector, string, error) {
|
||||
switch platform {
|
||||
case creator.PlatformDouyin:
|
||||
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
|
||||
uid, err := browser.Identity(ctx, accountKey)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
collector := douyinCollector(browser, accountKey, sourceType, sourceID)
|
||||
canonical, err := collector.CanonicalSecUID(ctx, uid)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
collector.AccountKey = canonical
|
||||
return &collector, canonical, nil
|
||||
case creator.PlatformXiaohongshu:
|
||||
browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}
|
||||
uid, err := browser.Identity(ctx, accountKey)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return &xiaohongshu.Collector{Browser: browser, AccountKey: uid, SourceType: sourceType, SourceID: sourceID}, uid, nil
|
||||
default:
|
||||
return nil, "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform)
|
||||
}
|
||||
}
|
||||
|
||||
func douyinCollector(browser creatorGatewayBrowser, accountKey, sourceType, sourceID string) douyin.CreatorCollector {
|
||||
return douyin.CreatorCollector{Browser: browser, AccountKey: accountKey, SourceType: sourceType, SourceID: sourceID}
|
||||
}
|
||||
|
||||
func verifyCreatorPlatformIdentity(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, expectedKey string) (string, error) {
|
||||
switch platform {
|
||||
case creator.PlatformDouyin:
|
||||
return (creatorGatewayBrowser{gateway: gateway, environment: environment}).Identity(ctx, expectedKey)
|
||||
case creator.PlatformXiaohongshu:
|
||||
return (xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}).Identity(ctx, expectedKey)
|
||||
default:
|
||||
return "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeBase64(value string) ([]byte, error) {
|
||||
const maxEncoded = 96 << 20
|
||||
if len(value) > maxEncoded {
|
||||
return nil, errors.New("media response is too large")
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode media response: %w", err)
|
||||
}
|
||||
if len(data) > maxCreatorMediaBytes {
|
||||
return nil, errors.New("media response is too large")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
var _ xiaohongshu.Browser = xiaohongshuGatewayBrowser{}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/hub"
|
||||
)
|
||||
|
||||
const testXiaohongshuIdentityURL = "https://edith.xiaohongshu.com/api/sns/web/v2/user/me"
|
||||
|
||||
func TestXiaohongshuGatewayBrowserFencesAccountGeneration(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Header.Get("Authorization") != "Bearer gateway-token-1" {
|
||||
t.Fatal("missing gateway authorization")
|
||||
}
|
||||
var body map[string]any
|
||||
if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) || body["runtime_id"] != "runtime-a" || body["network_id"] != "network-a" || body["network_exit_id"] != "exit-a" {
|
||||
t.Fatalf("generation fence missing: %#v", body)
|
||||
}
|
||||
if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/get" || body["url"] != testXiaohongshuIdentityURL {
|
||||
t.Fatalf("unexpected request: path=%s body=%#v", request.URL.Path, body)
|
||||
}
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{"status": 200, "body": `{"success":true}`, "challenge": ""})
|
||||
}))
|
||||
defer server.Close()
|
||||
browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()}
|
||||
result, err := browser.Get(context.Background(), testXiaohongshuIdentityURL)
|
||||
if err != nil || result.Status != 200 || string(result.Body) != `{"success":true}` {
|
||||
t.Fatalf("unexpected result: %#v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXiaohongshuGatewayBrowserPostCarriesJSONBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
var body map[string]any
|
||||
if json.NewDecoder(request.Body).Decode(&body) != nil || body["url"] != "https://so.xiaohongshu.com/api/sns/web/v2/search/notes" {
|
||||
t.Fatalf("unexpected request body: %#v", body)
|
||||
}
|
||||
if _, ok := body["body"].(map[string]any); !ok {
|
||||
t.Fatalf("missing nested request body: %#v", body)
|
||||
}
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{"status": 200, "body": `{"success":true}`, "challenge": ""})
|
||||
}))
|
||||
defer server.Close()
|
||||
browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()}
|
||||
if _, err := browser.Post(context.Background(), "https://so.xiaohongshu.com/api/sns/web/v2/search/notes", []byte(`{"keyword":"x"}`)); err != nil {
|
||||
t.Fatalf("post failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1285,6 +1285,124 @@ def detect_challenge(status: int, body: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
XHS_ORIGIN = "https://www.xiaohongshu.com"
|
||||
XHS_API_ORIGIN = "https://edith.xiaohongshu.com"
|
||||
XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com"
|
||||
XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me"
|
||||
XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"})
|
||||
|
||||
|
||||
class XiaohongshuBrowser(DouyinBrowser):
|
||||
def __init__(self, endpoint=None) -> None:
|
||||
super().__init__(
|
||||
endpoint,
|
||||
origin=XHS_ORIGIN,
|
||||
url_validator=is_xiaohongshu_url,
|
||||
media_validator=is_xiaohongshu_media_url,
|
||||
)
|
||||
|
||||
def post(self, alias: str, target: str, body: bytes) -> BrowserResponse:
|
||||
if not is_xiaohongshu_url(target) or len(body) > RESPONSE_LIMIT:
|
||||
raise DouyinError("restricted Xiaohongshu POST request is invalid")
|
||||
try:
|
||||
body_text = body.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise DouyinError("restricted Xiaohongshu POST body is not UTF-8") from exc
|
||||
with self.connection(alias) as cdp:
|
||||
if cdp.evaluate("location.origin") != self.origin:
|
||||
raise DouyinError("restricted browser origin changed")
|
||||
expression = f"""(async()=>{{
|
||||
const r=await fetch({json.dumps(target)},{{method:'POST',headers:{{'content-type':'application/json'}},body:{json.dumps(body_text)},credentials:'include',redirect:'error'}});
|
||||
if(!r.body)return {{status:r.status,body:'',too_large:false}};
|
||||
const reader=r.body.getReader(), decoder=new TextDecoder(); let size=0, responseBody='';
|
||||
for(;;){{const item=await reader.read();if(item.done)break;
|
||||
if(size+item.value.byteLength>={RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}}
|
||||
size+=item.value.byteLength;responseBody+=decoder.decode(item.value,{{stream:true}});
|
||||
}}
|
||||
responseBody+=decoder.decode();return {{status:r.status,body:responseBody,too_large:false}};
|
||||
}})()"""
|
||||
result = cdp.evaluate(expression)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or result.get("too_large")
|
||||
or not isinstance(result.get("status"), int)
|
||||
):
|
||||
raise DouyinError("restricted Xiaohongshu POST failed")
|
||||
status = result["status"]
|
||||
if 300 <= status < 400:
|
||||
raise DouyinError("restricted Xiaohongshu POST redirected")
|
||||
response_body = result.get("body")
|
||||
if not isinstance(response_body, str):
|
||||
raise DouyinError("restricted Xiaohongshu POST returned invalid body")
|
||||
return BrowserResponse(status, response_body, detect_challenge(status, response_body))
|
||||
|
||||
def identity(self, alias: str, expected_uid: str | None = None) -> dict:
|
||||
response = self.get(alias, XHS_IDENTITY_URL)
|
||||
try:
|
||||
payload = json.loads(response.body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DouyinError("Xiaohongshu identity response is invalid") from exc
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
user_info = data.get("user_info") if isinstance(data, dict) else None
|
||||
user_id = data.get("user_id", "") if isinstance(data, dict) else ""
|
||||
nickname = data.get("nickname", "") if isinstance(data, dict) else ""
|
||||
if isinstance(user_info, dict):
|
||||
user_id = user_id or user_info.get("user_id", "")
|
||||
nickname = nickname or user_info.get("nickname", "")
|
||||
success = payload.get("success") if isinstance(payload, dict) else None
|
||||
if (
|
||||
response.status != 200
|
||||
or not isinstance(payload, dict)
|
||||
or not isinstance(success, bool)
|
||||
or not success
|
||||
or not isinstance(user_id, str)
|
||||
or not ACCOUNT_KEY_RE.fullmatch(user_id)
|
||||
or nickname is not None
|
||||
and not isinstance(nickname, str)
|
||||
):
|
||||
raise DouyinError("Xiaohongshu login is not valid")
|
||||
if expected_uid and user_id != expected_uid:
|
||||
raise DouyinError("Xiaohongshu identity does not match the expected account")
|
||||
return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""}
|
||||
|
||||
|
||||
def is_xiaohongshu_media_url(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
port = parsed.port
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
parsed.scheme == "https"
|
||||
and parsed.hostname == "www.xiaohongshu.com"
|
||||
and port is None
|
||||
and parsed.username is None
|
||||
and parsed.password is None
|
||||
and parsed.fragment == ""
|
||||
and parsed.path.startswith("/explore/")
|
||||
)
|
||||
|
||||
|
||||
def is_xiaohongshu_url(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
port = parsed.port
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
parsed.scheme == "https"
|
||||
and parsed.hostname in XHS_ALLOWED_HOSTS
|
||||
and port is None
|
||||
and parsed.username is None
|
||||
and parsed.password is None
|
||||
and parsed.fragment == ""
|
||||
)
|
||||
|
||||
|
||||
def notice_ids(event: dict) -> list[str]:
|
||||
try:
|
||||
payload = json.loads(event["payload"])
|
||||
|
||||
@@ -46,6 +46,7 @@ from .douyin import (
|
||||
DouyinBrowser,
|
||||
DouyinError,
|
||||
SubscriptionManager,
|
||||
XiaohongshuBrowser,
|
||||
)
|
||||
from .proxy import ProxyExit, ProxyRegistry
|
||||
|
||||
@@ -64,6 +65,12 @@ DOUYIN_IDENTITY_PATH = "/aweme/v1/web/user/profile/self/"
|
||||
DOUYIN_IDENTITY_URL = IDENTITY_URL
|
||||
DOUYIN_WORKS_PATH = WORKS_PATH
|
||||
DOUYIN_COMMENTS_PATH = COMMENTS_PATH
|
||||
XHS_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$")
|
||||
XHS_IDENTITY_PATH = "/api/sns/web/v2/user/me"
|
||||
XHS_USER_POSTED_PATH = "/api/sns/web/v1/user_posted"
|
||||
XHS_COMMENTS_PATH = "/api/sns/web/v2/comment/page"
|
||||
XHS_SEARCH_PATH = "/api/sns/web/v2/search/notes"
|
||||
XHS_FEED_PATH = "/api/sns/web/v1/feed"
|
||||
|
||||
|
||||
def _noop() -> None:
|
||||
@@ -85,12 +92,14 @@ class Gateway:
|
||||
token: str,
|
||||
self_name: str,
|
||||
browser: DouyinBrowser | None = None,
|
||||
xiaohongshu_browser: XiaohongshuBrowser | None = None,
|
||||
) -> None:
|
||||
self.docker = docker
|
||||
self.network = network
|
||||
self.token = token
|
||||
self.self_name = self_name
|
||||
self.browser = browser or DouyinBrowser(self._browser_endpoint)
|
||||
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint)
|
||||
self.proxies = ProxyRegistry()
|
||||
self.reservations = AliasReservationManager(docker, self_name)
|
||||
self.subscriptions = SubscriptionManager(self.browser)
|
||||
@@ -666,6 +675,76 @@ class Gateway:
|
||||
)
|
||||
return identity
|
||||
|
||||
def get_xiaohongshu(self, alias: str, input: dict) -> dict:
|
||||
target = input.get("url", "")
|
||||
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_url(target):
|
||||
raise RequestError("invalid restricted Xiaohongshu request", 400)
|
||||
with self._alias_lock(alias):
|
||||
self._require_douyin_generation(alias, input)
|
||||
try:
|
||||
response = self.xiaohongshu_browser.get(alias, target)
|
||||
self._require_douyin_generation(alias, input)
|
||||
except DouyinError as exc:
|
||||
LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc))
|
||||
raise RequestError("restricted Xiaohongshu operation failed") from exc
|
||||
return {"status": response.status, "body": response.body, "challenge": response.challenge}
|
||||
|
||||
def post_xiaohongshu(self, alias: str, input: dict) -> dict:
|
||||
target = input.get("url", "")
|
||||
body = input.get("body")
|
||||
if (
|
||||
not valid_xiaohongshu_generation(input)
|
||||
or not valid_xhs_post_url(target)
|
||||
or not isinstance(body, dict)
|
||||
):
|
||||
raise RequestError("invalid restricted Xiaohongshu POST request", 400)
|
||||
try:
|
||||
encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc
|
||||
with self._alias_lock(alias):
|
||||
self._require_douyin_generation(alias, input)
|
||||
try:
|
||||
response = self.xiaohongshu_browser.post(alias, target, encoded)
|
||||
self._require_douyin_generation(alias, input)
|
||||
except DouyinError as exc:
|
||||
LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc))
|
||||
raise RequestError("restricted Xiaohongshu operation failed") from exc
|
||||
return {"status": response.status, "body": response.body, "challenge": response.challenge}
|
||||
|
||||
def get_xiaohongshu_media(self, alias: str, input: dict) -> dict:
|
||||
target = input.get("url", "")
|
||||
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target):
|
||||
raise RequestError("invalid restricted Xiaohongshu media request", 400)
|
||||
with self._alias_lock(alias):
|
||||
self._require_douyin_generation(alias, input)
|
||||
try:
|
||||
response = self.xiaohongshu_browser.get_media(alias, target)
|
||||
self._require_douyin_generation(alias, input)
|
||||
except DouyinError as exc:
|
||||
LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc))
|
||||
raise RequestError("restricted Xiaohongshu media download failed") from exc
|
||||
return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64}
|
||||
|
||||
def xiaohongshu_identity(self, alias: str, input: dict) -> dict:
|
||||
expected_account_key = input.get("expected_account_key", "")
|
||||
if (
|
||||
not valid_xiaohongshu_generation(input)
|
||||
or not isinstance(expected_account_key, str)
|
||||
or not XHS_ACCOUNT_KEY_RE.fullmatch(expected_account_key)
|
||||
):
|
||||
raise RequestError("invalid Xiaohongshu identity request", 400)
|
||||
with self._alias_lock(alias):
|
||||
self._require_douyin_generation(alias, input)
|
||||
try:
|
||||
identity = self.xiaohongshu_browser.identity(alias)
|
||||
except DouyinError as exc:
|
||||
LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc))
|
||||
raise RequestError("Xiaohongshu login identity could not be verified") from exc
|
||||
if identity.get("uid") != expected_account_key:
|
||||
raise RequestError("Xiaohongshu identity does not match the expected account", 409)
|
||||
return identity
|
||||
|
||||
def douyin_action(self, alias: str, input: dict) -> dict:
|
||||
expected_uid = input.get("expected_uid", "")
|
||||
action = input.get("action", "")
|
||||
@@ -1130,6 +1209,20 @@ class GatewayHandler(BaseHTTPRequestHandler):
|
||||
if method == "POST" and action == "proxy":
|
||||
gateway.restore_proxy(alias, body)
|
||||
return None
|
||||
match = re.fullmatch(
|
||||
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)",
|
||||
path,
|
||||
)
|
||||
if match:
|
||||
alias, action = match.groups()
|
||||
if action == "get" and method == "POST":
|
||||
return gateway.get_xiaohongshu(alias, body)
|
||||
if action == "post" and method == "POST":
|
||||
return gateway.post_xiaohongshu(alias, body)
|
||||
if action == "media" and method == "POST":
|
||||
return gateway.get_xiaohongshu_media(alias, body)
|
||||
if action == "identity" and method == "POST":
|
||||
return gateway.xiaohongshu_identity(alias, body)
|
||||
match = re.fullmatch(
|
||||
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|action|events)",
|
||||
path,
|
||||
@@ -1419,6 +1512,97 @@ def valid_douyin_generation(value: dict) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def valid_xiaohongshu_generation(value: dict) -> bool:
|
||||
return valid_douyin_generation(value)
|
||||
|
||||
|
||||
def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool:
|
||||
if not isinstance(query, dict) or not isinstance(allowed, set):
|
||||
return False
|
||||
required = required or set()
|
||||
if not required.issubset(query) or not set(query).issubset(allowed):
|
||||
return False
|
||||
for key, values in query.items():
|
||||
if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1:
|
||||
return False
|
||||
if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _valid_xhs_host(parsed: object, host: str) -> bool:
|
||||
return (
|
||||
getattr(parsed, "scheme", "") == "https"
|
||||
and getattr(parsed, "hostname", None) == host
|
||||
and getattr(parsed, "port", None) is None
|
||||
and getattr(parsed, "username", None) is None
|
||||
and getattr(parsed, "password", None) is None
|
||||
and getattr(parsed, "fragment", "") == ""
|
||||
)
|
||||
|
||||
|
||||
def valid_xhs_url(raw: object) -> bool:
|
||||
if not isinstance(raw, str):
|
||||
return False
|
||||
try:
|
||||
parsed = urlsplit(raw)
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
except ValueError:
|
||||
return False
|
||||
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH:
|
||||
return not query
|
||||
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH:
|
||||
return valid_xhs_query(
|
||||
query,
|
||||
{"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"},
|
||||
{"user_id", "num"},
|
||||
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"]
|
||||
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH:
|
||||
return valid_xhs_query(
|
||||
query,
|
||||
{"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"},
|
||||
{"note_id", "cursor", "top_comment_id"},
|
||||
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0]))
|
||||
return False
|
||||
|
||||
|
||||
def valid_xiaohongshu_url(raw: object) -> bool:
|
||||
return valid_xhs_url(raw)
|
||||
|
||||
|
||||
def valid_xhs_post_url(raw: object) -> bool:
|
||||
if not isinstance(raw, str):
|
||||
return False
|
||||
try:
|
||||
parsed = urlsplit(raw)
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
except ValueError:
|
||||
return False
|
||||
return (
|
||||
_valid_xhs_host(parsed, "so.xiaohongshu.com")
|
||||
and parsed.path == XHS_SEARCH_PATH
|
||||
and not query
|
||||
) or (
|
||||
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
|
||||
and parsed.path == XHS_FEED_PATH
|
||||
and not query
|
||||
)
|
||||
|
||||
|
||||
def valid_xiaohongshu_media_url(raw: object) -> bool:
|
||||
if not isinstance(raw, str):
|
||||
return False
|
||||
try:
|
||||
parsed = urlsplit(raw)
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
except ValueError:
|
||||
return False
|
||||
if not _valid_xhs_host(parsed, "www.xiaohongshu.com"):
|
||||
return False
|
||||
parts = parsed.path.strip("/").split("/")
|
||||
return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"})
|
||||
|
||||
|
||||
def valid_douyin_url(raw: object) -> bool:
|
||||
if not isinstance(raw, str):
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from typing import Any, cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
from . import gateway as gateway_module
|
||||
|
||||
Gateway = gateway_module.Gateway
|
||||
RequestError = gateway_module.RequestError
|
||||
valid_xhs_post_url = gateway_module.valid_xhs_post_url
|
||||
valid_xhs_url = gateway_module.valid_xhs_url
|
||||
valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url
|
||||
valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url
|
||||
valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation
|
||||
|
||||
|
||||
class XiaohongshuValidationTests(unittest.TestCase):
|
||||
def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None:
|
||||
self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
|
||||
self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
|
||||
self.assertTrue(
|
||||
valid_xhs_url(
|
||||
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user"
|
||||
)
|
||||
)
|
||||
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"))
|
||||
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me"))
|
||||
self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes"))
|
||||
self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed"))
|
||||
self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"))
|
||||
self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment"))
|
||||
|
||||
def test_generation_shape_matches_existing_browser_fence(self) -> None:
|
||||
self.assertTrue(
|
||||
valid_xiaohongshu_generation(
|
||||
{"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""}
|
||||
)
|
||||
)
|
||||
self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}))
|
||||
|
||||
|
||||
class XiaohongshuRouteTests(unittest.TestCase):
|
||||
def test_read_only_routes_dispatch_without_action_or_event_routes(self) -> None:
|
||||
handler = gateway_module.GatewayHandler.__new__(gateway_module.GatewayHandler)
|
||||
gateway = Mock()
|
||||
gateway.get_xiaohongshu.return_value = {"status": 200}
|
||||
gateway.post_xiaohongshu.return_value = {"status": 200}
|
||||
gateway.get_xiaohongshu_media.return_value = {"status": 200}
|
||||
gateway.xiaohongshu_identity.return_value = {"uid": "u-1"}
|
||||
server = Mock()
|
||||
server.gateway = gateway
|
||||
cast(Any, handler).server = server
|
||||
cast(Any, handler).server_as_gateway = lambda: server
|
||||
|
||||
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200})
|
||||
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200})
|
||||
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200})
|
||||
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"})
|
||||
with self.assertRaises(RequestError):
|
||||
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {})
|
||||
gateway.get_xiaohongshu.assert_called_once_with("account-a", {})
|
||||
gateway.post_xiaohongshu.assert_called_once_with("account-a", {})
|
||||
gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {})
|
||||
gateway.xiaohongshu_identity.assert_called_once_with("account-a", {})
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Xiaohongshu gateway facade.
|
||||
|
||||
The browser implementation lives next to the existing Douyin browser so both
|
||||
platforms share the CDP transport and response limits without duplicating it.
|
||||
"""
|
||||
|
||||
from .douyin import (
|
||||
XHS_ALLOWED_HOSTS,
|
||||
XHS_API_ORIGIN,
|
||||
XHS_IDENTITY_URL,
|
||||
XHS_ORIGIN,
|
||||
XHS_SEARCH_ORIGIN,
|
||||
XiaohongshuBrowser,
|
||||
is_xiaohongshu_media_url,
|
||||
is_xiaohongshu_url,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"XHS_ALLOWED_HOSTS",
|
||||
"XHS_API_ORIGIN",
|
||||
"XHS_IDENTITY_URL",
|
||||
"XHS_ORIGIN",
|
||||
"XHS_SEARCH_ORIGIN",
|
||||
"XiaohongshuBrowser",
|
||||
"is_xiaohongshu_media_url",
|
||||
"is_xiaohongshu_url",
|
||||
]
|
||||
@@ -4,7 +4,7 @@
|
||||
> 上游仓库:<https://github.com/cv-cat/XHS_ALL_IN_ONE>
|
||||
> 研究快照:`e86f82b`(`master`)
|
||||
> 研究方式:只读检查源码、提交历史与公开 Issues;未向小红书真实账号发起请求,以下不等同于平台能力验收。
|
||||
> 当前结论:本文仅形成小红书后续实现的研究约束,尚未接入运行代码;未复制上游代码、签名脚本、Cookie 或指纹数据。
|
||||
> 当前结论:本文形成小红书实现约束;main 已接入受限只读适配器,但未复制上游代码、签名脚本、Cookie 或指纹数据。
|
||||
|
||||
## 1. 结论先行
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
## 1.1 当前实现边界
|
||||
|
||||
- CreatorHub 当前只保留平台枚举和通用 Creator 数据模型;小红书专用 collector、gateway 路由和动作适配器未接入。
|
||||
- CreatorHub 当前已接入小红书只读 collector、详情/搜索/作品/一级评论读取、受限 gateway 路由和控制面平台分派;写操作与事件监听仍未接入。
|
||||
- `internal/creator/collection.go` 的分页、窗口、checkpoint 和 lease 模型可作为后续适配的复用边界,但不能证明小红书平台能力。
|
||||
|
||||
**未完成的真实能力验收:**私有接口签名是否能由浏览器当前会话完成、真实 UID/作品/评论分页、媒体下载、写操作和事件监听仍需真实小红书环境分别验证。HTTP 200、离线 fixture 和本地单元测试不能替代这些证据。
|
||||
|
||||
@@ -0,0 +1,657 @@
|
||||
package xiaohongshu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
||||
)
|
||||
|
||||
const (
|
||||
APIOrigin = "https://edith.xiaohongshu.com"
|
||||
SearchOrigin = "https://so.xiaohongshu.com"
|
||||
IdentityURL = APIOrigin + "/api/sns/web/v2/user/me"
|
||||
UserPostedPath = "/api/sns/web/v1/user_posted"
|
||||
CommentsPath = "/api/sns/web/v2/comment/page"
|
||||
SearchNotesPath = "/api/sns/web/v2/search/notes"
|
||||
FeedPath = "/api/sns/web/v1/feed"
|
||||
)
|
||||
|
||||
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`)
|
||||
|
||||
type Response struct {
|
||||
Status int
|
||||
Body []byte
|
||||
Challenge string
|
||||
}
|
||||
|
||||
type Browser interface {
|
||||
Get(context.Context, string) (Response, error)
|
||||
Post(context.Context, string, []byte) (Response, error)
|
||||
}
|
||||
|
||||
type Identity struct {
|
||||
UserID string
|
||||
Nickname string
|
||||
}
|
||||
|
||||
type Collector struct {
|
||||
Browser Browser
|
||||
AccountKey string
|
||||
SourceType string
|
||||
SourceID string
|
||||
HomepageURL string
|
||||
|
||||
contexts sync.Map
|
||||
}
|
||||
|
||||
type accessContext struct {
|
||||
Token string
|
||||
Source string
|
||||
}
|
||||
|
||||
func (c *Collector) VerifyIdentity(ctx context.Context, expectedKey string) error {
|
||||
_, err := c.Identity(ctx, expectedKey)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Collector) Identity(ctx context.Context, expectedKey string) (Identity, error) {
|
||||
if c == nil || c.Browser == nil || !keyPattern.MatchString(expectedKey) {
|
||||
return Identity{}, fmt.Errorf("%w: invalid xiaohongshu identity request", creator.ErrInvalid)
|
||||
}
|
||||
response, err := c.Browser.Get(ctx, IdentityURL)
|
||||
if err != nil {
|
||||
return Identity{}, err
|
||||
}
|
||||
if err := responseError(response, "identity"); err != nil {
|
||||
return Identity{}, err
|
||||
}
|
||||
identity, ok := parseIdentity(response.Body)
|
||||
if !ok || identity.UserID != expectedKey {
|
||||
return Identity{}, fmt.Errorf("%w: xiaohongshu identity mismatch", creator.ErrConflict)
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) {
|
||||
if c == nil {
|
||||
return creator.WorkPage{}, fmt.Errorf("%w: xiaohongshu collector is nil", creator.ErrUnavailable)
|
||||
}
|
||||
if c.AccountKey != "" {
|
||||
accountKey = c.AccountKey
|
||||
}
|
||||
if c.Browser == nil || !keyPattern.MatchString(accountKey) || !validCursor(cursor) {
|
||||
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu work collection request", creator.ErrInvalid)
|
||||
}
|
||||
access := c.defaultContext()
|
||||
query := url.Values{
|
||||
"user_id": {accountKey},
|
||||
"cursor": {cursor},
|
||||
"num": {"30"},
|
||||
"image_formats": {"jpg,webp,avif"},
|
||||
"xsec_source": {access.Source},
|
||||
"xsec_token": {access.Token},
|
||||
}
|
||||
response, err := c.Browser.Get(ctx, APIOrigin+UserPostedPath+"?"+query.Encode())
|
||||
if err != nil {
|
||||
return creator.WorkPage{}, err
|
||||
}
|
||||
if err := responseError(response, "works"); err != nil {
|
||||
return creator.WorkPage{}, err
|
||||
}
|
||||
items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, access)
|
||||
if !ok {
|
||||
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu works response", creator.ErrInvalid)
|
||||
}
|
||||
return creator.WorkPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil
|
||||
}
|
||||
|
||||
func (c *Collector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) {
|
||||
if c == nil || c.Browser == nil || !keyPattern.MatchString(workKey) || !validCursor(cursor) {
|
||||
return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comment collection request", creator.ErrInvalid)
|
||||
}
|
||||
access := c.contextFor(workKey)
|
||||
query := url.Values{
|
||||
"note_id": {workKey},
|
||||
"cursor": {cursor},
|
||||
"top_comment_id": {""},
|
||||
"image_formats": {"jpg,webp,avif"},
|
||||
"xsec_source": {access.Source},
|
||||
"xsec_token": {access.Token},
|
||||
}
|
||||
response, err := c.Browser.Get(ctx, APIOrigin+CommentsPath+"?"+query.Encode())
|
||||
if err != nil {
|
||||
return creator.CommentPage{}, err
|
||||
}
|
||||
if err := responseError(response, "comments"); err != nil {
|
||||
return creator.CommentPage{}, err
|
||||
}
|
||||
items, nextCursor, hasMore, ok := parseCommentsPage(response.Body, workKey)
|
||||
if !ok {
|
||||
return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comments response", creator.ErrInvalid)
|
||||
}
|
||||
return creator.CommentPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil
|
||||
}
|
||||
|
||||
func (c *Collector) SearchNotes(ctx context.Context, queryText string, page int) (creator.WorkPage, error) {
|
||||
if c == nil || c.Browser == nil || strings.TrimSpace(queryText) == "" || page < 1 || page > 10000 || utf8.RuneCountInString(queryText) > 200 {
|
||||
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search request", creator.ErrInvalid)
|
||||
}
|
||||
searchID, err := randomID()
|
||||
if err != nil {
|
||||
return creator.WorkPage{}, fmt.Errorf("create xiaohongshu search id: %w", err)
|
||||
}
|
||||
sessionID, err := randomID()
|
||||
if err != nil {
|
||||
return creator.WorkPage{}, fmt.Errorf("create xiaohongshu search session id: %w", err)
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"keyword": queryText,
|
||||
"page": page,
|
||||
"page_size": 20,
|
||||
"search_id": searchID,
|
||||
"sort": "general",
|
||||
"note_type": 0,
|
||||
"ext_flags": []any{},
|
||||
"geo": "",
|
||||
"image_formats": []string{"jpg", "webp", "avif"},
|
||||
"session_id": sessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return creator.WorkPage{}, err
|
||||
}
|
||||
response, err := c.Browser.Post(ctx, SearchOrigin+SearchNotesPath, body)
|
||||
if err != nil {
|
||||
return creator.WorkPage{}, err
|
||||
}
|
||||
if err := responseError(response, "search"); err != nil {
|
||||
return creator.WorkPage{}, err
|
||||
}
|
||||
items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, c.defaultContext())
|
||||
if !ok {
|
||||
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search response", creator.ErrInvalid)
|
||||
}
|
||||
return creator.WorkPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil
|
||||
}
|
||||
|
||||
func randomID() (string, error) {
|
||||
value := make([]byte, 16)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
func (c *Collector) defaultContext() accessContext {
|
||||
access := accessContext{Source: "pc_user"}
|
||||
if c != nil && strings.TrimSpace(c.HomepageURL) != "" {
|
||||
parsed, err := url.Parse(c.HomepageURL)
|
||||
if err == nil {
|
||||
if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" {
|
||||
access.Token = token
|
||||
}
|
||||
if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" {
|
||||
access.Source = source
|
||||
}
|
||||
}
|
||||
}
|
||||
return access
|
||||
}
|
||||
|
||||
func (c *Collector) SetWorkContext(workKey, originalURL string) error {
|
||||
if c == nil || !keyPattern.MatchString(workKey) {
|
||||
return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid)
|
||||
}
|
||||
access := c.defaultContext()
|
||||
if strings.TrimSpace(originalURL) != "" {
|
||||
parsed, err := url.Parse(originalURL)
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" || !strings.Contains(parsed.Path, workKey) {
|
||||
return fmt.Errorf("%w: invalid xiaohongshu work URL", creator.ErrInvalid)
|
||||
}
|
||||
if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" {
|
||||
access.Token = token
|
||||
}
|
||||
if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" {
|
||||
access.Source = source
|
||||
}
|
||||
}
|
||||
if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) {
|
||||
return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid)
|
||||
}
|
||||
c.contexts.Store(workKey, access)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Collector) contextFor(workKey string) accessContext {
|
||||
if c != nil {
|
||||
if value, ok := c.contexts.Load(workKey); ok {
|
||||
if access, ok := value.(accessContext); ok {
|
||||
return access
|
||||
}
|
||||
}
|
||||
}
|
||||
return c.defaultContext()
|
||||
}
|
||||
|
||||
func parseIdentity(body []byte) (Identity, bool) {
|
||||
if len(body) == 0 || len(body) > 1<<20 {
|
||||
return Identity{}, false
|
||||
}
|
||||
var envelope struct {
|
||||
Success *bool `json:"success"`
|
||||
Data *struct {
|
||||
UserID string `json:"user_id"`
|
||||
Nickname string `json:"nickname"`
|
||||
UserInfo *struct {
|
||||
UserID string `json:"user_id"`
|
||||
Nickname string `json:"nickname"`
|
||||
} `json:"user_info"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil {
|
||||
return Identity{}, false
|
||||
}
|
||||
userID, nickname := envelope.Data.UserID, envelope.Data.Nickname
|
||||
if envelope.Data.UserInfo != nil {
|
||||
if userID == "" {
|
||||
userID = envelope.Data.UserInfo.UserID
|
||||
}
|
||||
if nickname == "" {
|
||||
nickname = envelope.Data.UserInfo.Nickname
|
||||
}
|
||||
}
|
||||
if !keyPattern.MatchString(userID) || utf8.RuneCountInString(nickname) > 256 {
|
||||
return Identity{}, false
|
||||
}
|
||||
return Identity{UserID: userID, Nickname: nickname}, true
|
||||
}
|
||||
|
||||
func parseWorksPage(body []byte, collector *Collector, fallback accessContext) ([]creator.WorkInput, string, bool, bool) {
|
||||
if len(body) == 0 || len(body) > 4<<20 {
|
||||
return nil, "", false, false
|
||||
}
|
||||
var envelope struct {
|
||||
Success *bool `json:"success"`
|
||||
Data *struct {
|
||||
Cursor json.RawMessage `json:"cursor"`
|
||||
HasMore *bool `json:"has_more"`
|
||||
Notes []json.RawMessage `json:"notes"`
|
||||
Items []json.RawMessage `json:"items"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil || envelope.Data.HasMore == nil {
|
||||
return nil, "", false, false
|
||||
}
|
||||
rawNotes := envelope.Data.Notes
|
||||
if rawNotes == nil {
|
||||
rawNotes = envelope.Data.Items
|
||||
}
|
||||
if rawNotes == nil || len(rawNotes) > 30 {
|
||||
return nil, "", false, false
|
||||
}
|
||||
items := make([]creator.WorkInput, 0, len(rawNotes))
|
||||
seen := make(map[string]struct{}, len(rawNotes))
|
||||
for _, raw := range rawNotes {
|
||||
item, access, ok := parseWork(raw, fallback)
|
||||
if !ok {
|
||||
return nil, "", false, false
|
||||
}
|
||||
if _, exists := seen[item.WorkKey]; exists {
|
||||
return nil, "", false, false
|
||||
}
|
||||
seen[item.WorkKey] = struct{}{}
|
||||
if collector != nil {
|
||||
item.SourceType = collector.SourceType
|
||||
if item.SourceType == "" {
|
||||
item.SourceType = creator.SourceCompetitor
|
||||
}
|
||||
item.SourceID = collector.SourceID
|
||||
if item.SourceID == "" {
|
||||
item.SourceID = collector.AccountKey
|
||||
}
|
||||
collector.contexts.Store(item.WorkKey, access)
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
nextCursor, ok := cursorValue(envelope.Data.Cursor)
|
||||
if !ok {
|
||||
return nil, "", false, false
|
||||
}
|
||||
if *envelope.Data.HasMore && nextCursor == "" {
|
||||
return nil, "", false, false
|
||||
}
|
||||
return items, nextCursor, *envelope.Data.HasMore, true
|
||||
}
|
||||
|
||||
func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput, accessContext, bool) {
|
||||
object, ok := objectValue(raw)
|
||||
if !ok {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
if nested := firstObject(object, "note_card", "noteCard"); nested != nil {
|
||||
object = nested
|
||||
}
|
||||
id := firstString(object, "note_id", "id")
|
||||
if !keyPattern.MatchString(id) {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
title := firstString(object, "display_title", "title")
|
||||
body := firstString(object, "desc", "description", "content")
|
||||
if utf8.RuneCountInString(title) > 4096 || utf8.RuneCountInString(body) > 100000 {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
user := firstObject(object, "user", "user_info", "author")
|
||||
authorName := firstString(user, "nickname", "name")
|
||||
published, publishedOK := optionalTimestamp(object, "time", "create_time", "last_update_time")
|
||||
if !publishedOK {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
likes, likesOK := optionalInt(object, "likes", "liked_count")
|
||||
comments, commentsOK := optionalInt(object, "comments_count", "comment_count")
|
||||
shares, sharesOK := optionalInt(object, "shares", "shared_count")
|
||||
interact := firstObject(object, "interact_info", "interactInfo", "statistics")
|
||||
if interact != nil {
|
||||
if likes == nil {
|
||||
likes, likesOK = optionalInt(interact, "liked_count", "likes")
|
||||
}
|
||||
if comments == nil {
|
||||
comments, commentsOK = optionalInt(interact, "comment_count", "comments_count")
|
||||
}
|
||||
if shares == nil {
|
||||
shares, sharesOK = optionalInt(interact, "shared_count", "shares")
|
||||
}
|
||||
}
|
||||
if !likesOK || !commentsOK || !sharesOK {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
access := fallback
|
||||
if token := firstString(object, "xsec_token"); token != "" {
|
||||
access.Token = token
|
||||
}
|
||||
if source := firstString(object, "xsec_source"); source != "" {
|
||||
access.Source = source
|
||||
}
|
||||
if utf8.RuneCountInString(access.Token) > 2048 || !keyPattern.MatchString(access.Source) {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
originalURL := firstString(object, "original_url", "note_url", "url")
|
||||
if !validOriginalURL(originalURL, id) {
|
||||
originalURL = noteURL(id, access)
|
||||
}
|
||||
cover := coverURL(object)
|
||||
status := "pending_verification"
|
||||
if published != nil {
|
||||
status = "verified"
|
||||
}
|
||||
return creator.WorkInput{
|
||||
Platform: creator.PlatformXiaohongshu,
|
||||
WorkKey: id,
|
||||
AuthorName: authorName,
|
||||
Title: title,
|
||||
Body: body,
|
||||
PublishedAt: published,
|
||||
PublishedAtStatus: status,
|
||||
OriginalURL: originalURL,
|
||||
CoverURL: cover,
|
||||
Likes: likes,
|
||||
CommentsCount: comments,
|
||||
Shares: shares,
|
||||
}, access, true
|
||||
}
|
||||
|
||||
func parseCommentsPage(body []byte, workKey string) ([]creator.CommentInput, string, bool, bool) {
|
||||
if len(body) == 0 || len(body) > 4<<20 || !keyPattern.MatchString(workKey) {
|
||||
return nil, "", false, false
|
||||
}
|
||||
var envelope struct {
|
||||
Success *bool `json:"success"`
|
||||
Data *struct {
|
||||
Cursor json.RawMessage `json:"cursor"`
|
||||
HasMore *bool `json:"has_more"`
|
||||
Comments []json.RawMessage `json:"comments"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil || envelope.Data.HasMore == nil || envelope.Data.Comments == nil || len(envelope.Data.Comments) > 100 {
|
||||
return nil, "", false, false
|
||||
}
|
||||
items := make([]creator.CommentInput, 0, len(envelope.Data.Comments))
|
||||
seen := make(map[string]struct{}, len(envelope.Data.Comments))
|
||||
for _, raw := range envelope.Data.Comments {
|
||||
object, ok := objectValue(raw)
|
||||
if !ok {
|
||||
return nil, "", false, false
|
||||
}
|
||||
id := firstString(object, "id", "comment_id", "commentId")
|
||||
if !keyPattern.MatchString(id) {
|
||||
return nil, "", false, false
|
||||
}
|
||||
if parent := firstString(object, "parent_comment_id", "parent_id", "reply_id"); parent != "" {
|
||||
return nil, "", false, false
|
||||
}
|
||||
if typ := firstString(object, "comment_type", "type"); typ == "reply" || typ == "sub" {
|
||||
return nil, "", false, false
|
||||
}
|
||||
if _, exists := seen[id]; exists {
|
||||
return nil, "", false, false
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
user := firstObject(object, "user_info", "user", "author")
|
||||
published, publishedOK := optionalTimestamp(object, "create_time", "time", "created_at")
|
||||
if !publishedOK {
|
||||
return nil, "", false, false
|
||||
}
|
||||
content := firstString(object, "content", "text", "comment")
|
||||
if content == "" || utf8.RuneCountInString(content) > 20000 {
|
||||
return nil, "", false, false
|
||||
}
|
||||
items = append(items, creator.CommentInput{
|
||||
Platform: creator.PlatformXiaohongshu,
|
||||
CommentKey: id,
|
||||
WorkID: workKey,
|
||||
AuthorUID: firstString(user, "user_id", "uid", "id"),
|
||||
AuthorName: firstString(user, "nickname", "name"),
|
||||
Content: content,
|
||||
PublishedAt: published,
|
||||
CommentType: "top_level",
|
||||
})
|
||||
}
|
||||
nextCursor, ok := cursorValue(envelope.Data.Cursor)
|
||||
if !ok || *envelope.Data.HasMore && nextCursor == "" {
|
||||
return nil, "", false, false
|
||||
}
|
||||
return items, nextCursor, *envelope.Data.HasMore, true
|
||||
}
|
||||
|
||||
func responseError(response Response, resource string) error {
|
||||
if strings.TrimSpace(response.Challenge) != "" {
|
||||
return fmt.Errorf("%w: xiaohongshu %s challenge %s", creator.ErrUnavailable, resource, response.Challenge)
|
||||
}
|
||||
if response.Status >= 200 && response.Status < 300 {
|
||||
return nil
|
||||
}
|
||||
switch response.Status {
|
||||
case 401, 403, 406, 461:
|
||||
return fmt.Errorf("%w: xiaohongshu %s authentication/session rejected with HTTP %d", creator.ErrConflict, resource, response.Status)
|
||||
case 429:
|
||||
return fmt.Errorf("%w: xiaohongshu %s rate limited", creator.ErrUnavailable, resource)
|
||||
default:
|
||||
return fmt.Errorf("xiaohongshu %s returned HTTP %d", resource, response.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func validCursor(cursor string) bool {
|
||||
return cursor == "" || len(cursor) <= 512 && !strings.ContainsAny(cursor, "\r\n")
|
||||
}
|
||||
|
||||
func objectValue(raw json.RawMessage) (map[string]json.RawMessage, bool) {
|
||||
var object map[string]json.RawMessage
|
||||
if len(raw) == 0 || json.Unmarshal(raw, &object) != nil || object == nil {
|
||||
return nil, false
|
||||
}
|
||||
return object, true
|
||||
}
|
||||
|
||||
func firstObject(object map[string]json.RawMessage, names ...string) map[string]json.RawMessage {
|
||||
for _, name := range names {
|
||||
if value, ok := object[name]; ok {
|
||||
if nested, ok := objectValue(value); ok {
|
||||
return nested
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstString(object map[string]json.RawMessage, names ...string) string {
|
||||
for _, name := range names {
|
||||
value, ok := object[name]
|
||||
if !ok || string(value) == "null" {
|
||||
continue
|
||||
}
|
||||
var text string
|
||||
if json.Unmarshal(value, &text) == nil {
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
var number json.Number
|
||||
decoder := json.NewDecoder(bytes.NewReader(value))
|
||||
decoder.UseNumber()
|
||||
if decoder.Decode(&number) == nil {
|
||||
return number.String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func optionalInt(object map[string]json.RawMessage, names ...string) (*int64, bool) {
|
||||
for _, name := range names {
|
||||
value, ok := object[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if string(value) == "null" {
|
||||
return nil, true
|
||||
}
|
||||
var number json.Number
|
||||
decoder := json.NewDecoder(bytes.NewReader(value))
|
||||
decoder.UseNumber()
|
||||
if decoder.Decode(&number) != nil {
|
||||
var text string
|
||||
if json.Unmarshal(value, &text) != nil {
|
||||
return nil, false
|
||||
}
|
||||
number = json.Number(text)
|
||||
}
|
||||
parsed, err := strconv.ParseInt(number.String(), 10, 64)
|
||||
if err != nil || parsed < 0 {
|
||||
return nil, false
|
||||
}
|
||||
return &parsed, true
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
func optionalTimestamp(object map[string]json.RawMessage, names ...string) (*time.Time, bool) {
|
||||
for _, name := range names {
|
||||
value, ok := object[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if string(value) == "null" {
|
||||
return nil, true
|
||||
}
|
||||
var number json.Number
|
||||
decoder := json.NewDecoder(bytes.NewReader(value))
|
||||
decoder.UseNumber()
|
||||
if decoder.Decode(&number) != nil {
|
||||
var text string
|
||||
if json.Unmarshal(value, &text) != nil {
|
||||
return nil, false
|
||||
}
|
||||
number = json.Number(text)
|
||||
}
|
||||
parsed, err := strconv.ParseInt(number.String(), 10, 64)
|
||||
if err != nil || parsed <= 0 {
|
||||
return nil, true
|
||||
}
|
||||
if parsed > 1_000_000_000_000 {
|
||||
parsed /= 1000
|
||||
}
|
||||
if parsed <= 0 || parsed > 4_102_444_800 {
|
||||
return nil, false
|
||||
}
|
||||
when := time.Unix(parsed, 0).UTC()
|
||||
return &when, true
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
func cursorValue(raw json.RawMessage) (string, bool) {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return "", true
|
||||
}
|
||||
var text string
|
||||
if json.Unmarshal(raw, &text) == nil {
|
||||
return strings.TrimSpace(text), validCursor(strings.TrimSpace(text))
|
||||
}
|
||||
var number json.Number
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber()
|
||||
if decoder.Decode(&number) != nil {
|
||||
return "", false
|
||||
}
|
||||
return number.String(), validCursor(number.String())
|
||||
}
|
||||
|
||||
func coverURL(object map[string]json.RawMessage) string {
|
||||
images, ok := object["image_list"]
|
||||
if ok {
|
||||
var values []json.RawMessage
|
||||
if json.Unmarshal(images, &values) == nil {
|
||||
for _, value := range values {
|
||||
if image, ok := objectValue(value); ok {
|
||||
if cover := firstString(image, "url_default", "url_pre", "url_original", "url"); cover != "" {
|
||||
return cover
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if cover := firstObject(object, "cover"); cover != nil {
|
||||
return firstString(cover, "url_default", "url_pre", "url_original", "url")
|
||||
}
|
||||
return firstString(object, "cover_url", "cover")
|
||||
}
|
||||
|
||||
func validOriginalURL(raw, id string) bool {
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
parsed, err := url.Parse(raw)
|
||||
return err == nil && parsed.Scheme == "https" && (parsed.Host == "www.xiaohongshu.com" || parsed.Host == "xhslink.com") && strings.Contains(parsed.Path, id)
|
||||
}
|
||||
|
||||
func noteURL(id string, access accessContext) string {
|
||||
query := url.Values{}
|
||||
if access.Token != "" {
|
||||
query.Set("xsec_token", access.Token)
|
||||
}
|
||||
if access.Source != "" {
|
||||
query.Set("xsec_source", access.Source)
|
||||
}
|
||||
result := "https://www.xiaohongshu.com/explore/" + url.PathEscape(id)
|
||||
if encoded := query.Encode(); encoded != "" {
|
||||
result += "?" + encoded
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package xiaohongshu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
||||
)
|
||||
|
||||
type fakeBrowser struct {
|
||||
getURL string
|
||||
postURL string
|
||||
getBody []byte
|
||||
postBody []byte
|
||||
getResp Response
|
||||
postResp Response
|
||||
}
|
||||
|
||||
func (f *fakeBrowser) Get(_ context.Context, target string) (Response, error) {
|
||||
f.getURL = target
|
||||
return f.getResp, nil
|
||||
}
|
||||
func (f *fakeBrowser) Post(_ context.Context, target string, body []byte) (Response, error) {
|
||||
f.postURL, f.postBody = target, body
|
||||
return f.postResp, nil
|
||||
}
|
||||
|
||||
func TestIdentityRequiresMatchingUser(t *testing.T) {
|
||||
browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"user_id":"u-1","nickname":"作者"}}`)}}
|
||||
collector := &Collector{Browser: browser}
|
||||
identity, err := collector.Identity(context.Background(), "u-1")
|
||||
if err != nil || identity.UserID != "u-1" {
|
||||
t.Fatalf("identity = %#v, err = %v", identity, err)
|
||||
}
|
||||
if _, err := collector.Identity(context.Background(), "u-2"); !errors.Is(err, creator.ErrConflict) {
|
||||
t.Fatalf("expected identity conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListWorksCapturesContextForComments(t *testing.T) {
|
||||
browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"next","has_more":true,"notes":[{"note_id":"n-1","title":"标题","desc":"正文","time":1710000000,"user":{"user_id":"u-1","nickname":"作者"},"interact_info":{"liked_count":"2","comment_count":3,"shared_count":4},"xsec_token":"token"}]}}`)}}
|
||||
collector := &Collector{Browser: browser, AccountKey: "u-1", SourceType: creator.SourceOwned, SourceID: "source-1"}
|
||||
page, err := collector.ListWorks(context.Background(), "ignored", "")
|
||||
if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || !page.HasMore {
|
||||
t.Fatalf("page = %#v, err = %v", page, err)
|
||||
}
|
||||
parsed, err := url.Parse(browser.getURL)
|
||||
if err != nil || parsed.Query().Get("user_id") != "u-1" {
|
||||
t.Fatalf("request URL = %s", browser.getURL)
|
||||
}
|
||||
browser.getResp = Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"comments":[{"id":"c-1","content":"评论","create_time":1710000000,"user_info":{"user_id":"u-2","nickname":"读者"}}]}}`)}
|
||||
comments, err := collector.ListTopLevelComments(context.Background(), "n-1", "")
|
||||
if err != nil || len(comments.Items) != 1 || comments.Items[0].CommentType != "top_level" {
|
||||
t.Fatalf("comments = %#v, err = %v", comments, err)
|
||||
}
|
||||
commentURL, err := url.Parse(browser.getURL)
|
||||
if err != nil || commentURL.Query().Get("xsec_token") != "token" {
|
||||
t.Fatalf("comment request URL = %s", browser.getURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchUsesBoundedPostAndRejectsMalformedPage(t *testing.T) {
|
||||
browser := &fakeBrowser{postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"notes":[]}}`)}}
|
||||
collector := &Collector{Browser: browser}
|
||||
page, err := collector.SearchNotes(context.Background(), "关键词", 1)
|
||||
if err != nil || page.HasMore || len(page.Items) != 0 || browser.postURL != SearchOrigin+SearchNotesPath {
|
||||
t.Fatalf("page = %#v, URL = %s, err = %v", page, browser.postURL, err)
|
||||
}
|
||||
browser.postResp = Response{Status: 200, Body: []byte(`{"success":true,"data":{"has_more":true,"notes":[]}}`)}
|
||||
if _, err := collector.SearchNotes(context.Background(), "关键词", 1); !errors.Is(err, creator.ErrInvalid) {
|
||||
t.Fatalf("expected malformed response error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetWorkContextRejectsWrongHost(t *testing.T) {
|
||||
collector := &Collector{}
|
||||
if err := collector.SetWorkContext("n-1", "https://evil.example/explore/n-1"); !errors.Is(err, creator.ErrInvalid) {
|
||||
t.Fatalf("expected invalid URL, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package xiaohongshu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
||||
)
|
||||
|
||||
// GetNoteDetail reads one note through the platform feed endpoint and keeps
|
||||
// the original link so xsec context survives a later comments resume.
|
||||
func (c *Collector) GetNoteDetail(ctx context.Context, originalURL string) (creator.WorkInput, error) {
|
||||
if c == nil || c.Browser == nil {
|
||||
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail request", creator.ErrInvalid)
|
||||
}
|
||||
parsed, err := url.Parse(originalURL)
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" {
|
||||
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid)
|
||||
}
|
||||
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
|
||||
if len(parts) != 2 || parts[0] != "explore" || !keyPattern.MatchString(parts[1]) {
|
||||
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid)
|
||||
}
|
||||
access := accessContext{Token: parsed.Query().Get("xsec_token"), Source: parsed.Query().Get("xsec_source")}
|
||||
if access.Source == "" {
|
||||
access.Source = "pc_search"
|
||||
}
|
||||
if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) {
|
||||
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL context", creator.ErrInvalid)
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"source_note_id": parts[1],
|
||||
"image_formats": []string{"jpg", "webp", "avif"},
|
||||
"extra": map[string]string{"need_body_topic": "1"},
|
||||
"xsec_source": access.Source,
|
||||
"xsec_token": access.Token,
|
||||
})
|
||||
if err != nil {
|
||||
return creator.WorkInput{}, fmt.Errorf("encode xiaohongshu detail request: %w", err)
|
||||
}
|
||||
response, err := c.Browser.Post(ctx, APIOrigin+FeedPath, body)
|
||||
if err != nil {
|
||||
return creator.WorkInput{}, err
|
||||
}
|
||||
if err := responseError(response, "detail"); err != nil {
|
||||
return creator.WorkInput{}, err
|
||||
}
|
||||
item, parsedAccess, ok := parseDetail(response.Body, access)
|
||||
if !ok {
|
||||
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail response", creator.ErrInvalid)
|
||||
}
|
||||
if item.WorkKey != parts[1] {
|
||||
return creator.WorkInput{}, fmt.Errorf("%w: xiaohongshu detail returned another note", creator.ErrConflict)
|
||||
}
|
||||
item.OriginalURL = originalURL
|
||||
item.SourceType = c.SourceType
|
||||
item.SourceID = c.SourceID
|
||||
if item.SourceType == "" {
|
||||
item.SourceType = creator.SourceCompetitor
|
||||
}
|
||||
if item.SourceID == "" {
|
||||
item.SourceID = c.AccountKey
|
||||
}
|
||||
c.contexts.Store(item.WorkKey, parsedAccess)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func parseDetail(body []byte, fallback accessContext) (creator.WorkInput, accessContext, bool) {
|
||||
if len(body) == 0 || len(body) > 4<<20 {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
var envelope struct {
|
||||
Success *bool `json:"success"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || len(envelope.Data) == 0 {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
data, ok := objectValue(envelope.Data)
|
||||
if !ok {
|
||||
return creator.WorkInput{}, accessContext{}, false
|
||||
}
|
||||
if items, ok := arrayValue(data["items"]); ok && len(items) > 0 {
|
||||
return parseWork(items[0], fallback)
|
||||
}
|
||||
if note := data["note"]; len(note) > 0 {
|
||||
return parseWork(note, fallback)
|
||||
}
|
||||
return parseWork(envelope.Data, fallback)
|
||||
}
|
||||
|
||||
func arrayValue(raw json.RawMessage) ([]json.RawMessage, bool) {
|
||||
var values []json.RawMessage
|
||||
if len(raw) == 0 || json.Unmarshal(raw, &values) != nil || values == nil {
|
||||
return nil, false
|
||||
}
|
||||
return values, true
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package xiaohongshu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/creator"
|
||||
)
|
||||
|
||||
func TestGetNoteDetailPreservesShareContext(t *testing.T) {
|
||||
browser := &fakeBrowser{postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"items":[{"note_id":"n-1","title":"详情","desc":"正文","time":1710000000,"user":{"user_id":"u-1","nickname":"作者"},"interact_info":{"liked_count":1,"comment_count":2,"shared_count":3}}]}}`)}}
|
||||
collector := &Collector{Browser: browser, SourceType: creator.SourceCompetitor, SourceID: "source-1"}
|
||||
item, err := collector.GetNoteDetail(context.Background(), "https://www.xiaohongshu.com/explore/n-1?xsec_token=tok-1&xsec_source=pc_search")
|
||||
if err != nil || item.WorkKey != "n-1" || item.OriginalURL == "" {
|
||||
t.Fatalf("item = %#v, err = %v", item, err)
|
||||
}
|
||||
if err := collector.SetWorkContext("n-1", item.OriginalURL); err != nil {
|
||||
t.Fatalf("set work context: %v", err)
|
||||
}
|
||||
if browser.postURL != APIOrigin+FeedPath {
|
||||
t.Fatalf("post URL = %s", browser.postURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNoteDetailRejectsWrongResponseID(t *testing.T) {
|
||||
browser := &fakeBrowser{postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"items":[{"note_id":"n-2","title":"详情","time":1710000000,"interact_info":{}}]}}`)}}
|
||||
collector := &Collector{Browser: browser}
|
||||
_, err := collector.GetNoteDetail(context.Background(), "https://www.xiaohongshu.com/explore/n-1")
|
||||
if !errors.Is(err, creator.ErrConflict) {
|
||||
t.Fatalf("expected conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user