diff --git a/cmd/control-plane/creator.go b/cmd/control-plane/creator.go index 39ffa49..060028c 100644 --- a/cmd/control-plane/creator.go +++ b/cmd/control-plane/creator.go @@ -980,7 +980,7 @@ func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore if err != nil { return creator.LoginResult{}, err } - if account.Platform != creator.PlatformDouyin || profile.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || profile.PlatformAccountKey == "" || account.PlatformAccountKey != profile.PlatformAccountKey { + if account.Platform != profile.Platform || (account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu) || account.AuthorizationStatus != "authorized" || profile.PlatformAccountKey == "" || account.PlatformAccountKey != profile.PlatformAccountKey { return creator.LoginResult{}, creator.ErrConflict } environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) @@ -994,8 +994,7 @@ func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore if err != nil { return creator.LoginResult{}, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - uid, err := browser.Identity(ctx, profile.PlatformAccountKey) + uid, err := verifyCreatorPlatformIdentity(ctx, account.Platform, gateway, environment, profile.PlatformAccountKey) if err != nil { return creator.LoginResult{}, fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, err) } @@ -1126,8 +1125,8 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p markErr := store.MarkCompetitorSync(ctx, competitorID, leaseToken, "blocked", "", blockErr.Error(), nil) return creator.CollectionReport{}, errors.Join(blockErr, markErr) } - if competitor.Platform != creator.PlatformDouyin { - return blocked(fmt.Errorf("%w: 小红书采集器尚未完成平台能力验证", creator.ErrUnavailable)) + if competitor.Platform != creator.PlatformDouyin && competitor.Platform != creator.PlatformXiaohongshu { + return blocked(fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, competitor.Platform)) } account, err := phaseAStore.GetAccount(ctx, accountID) if err != nil { @@ -1154,16 +1153,10 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p if err != nil { return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)) } - browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - if _, identityErr := browser.Identity(ctx, account.PlatformAccountKey); identityErr != nil { - return blocked(fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr)) - } - collector := douyin.CreatorCollector{Browser: browser, AccountKey: competitor.PlatformAccountKey, SourceType: creator.SourceCompetitor, SourceID: competitor.ID} - canonicalSecUID, err := collector.CanonicalSecUID(ctx, account.PlatformAccountKey) + collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceCompetitor, competitor.ID) if err != nil { return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)) } - collector.AccountKey = canonicalSecUID collectionNow := now if competitor.NextSyncAt != nil && !competitor.NextSyncAt.After(now) { collectionNow = competitor.NextSyncAt.UTC() @@ -1279,7 +1272,7 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if err != nil { return err } - if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { + if (account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu) || account.AuthorizationStatus != "authorized" || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { return creator.ErrConflict } environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) @@ -1290,19 +1283,13 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if err != nil { return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - if _, identityErr := browser.Identity(ctx, account.PlatformAccountKey); identityErr != nil { - return fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr) - } - collector := douyin.CreatorCollector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: work.SourceType, SourceID: work.SourceID} - canonical, err := collector.CanonicalSecUID(ctx, account.PlatformAccountKey) + collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, work.SourceType, work.SourceID) if err != nil { return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err) } - collector.AccountKey = canonical cursor := "" for page := 0; page < 100; page++ { - result, pageErr := collector.ListWorks(ctx, canonical, cursor) + result, pageErr := collector.ListWorks(ctx, collectionKey, cursor) if pageErr != nil { return pageErr } @@ -1335,7 +1322,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph if err != nil { return err } - if account.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" { + if account.Platform != creator.PlatformDouyin && account.Platform != creator.PlatformXiaohongshu || account.AuthorizationStatus != "authorized" { return creator.ErrConflict } profile, err := store.GetAccountProfile(ctx, accountID) @@ -1349,9 +1336,6 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph if err != nil { return err } - blockOwned := func(blockErr error) error { - return errors.Join(blockErr, store.MarkCollectionBlocked(ctx, creator.SourceOwned, accountID, blockErr.Error(), now, settings.LookbackDays)) - } environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) if err != nil { return fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) @@ -1363,10 +1347,6 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph if err != nil { return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - if _, identityErr := browser.Identity(ctx, account.PlatformAccountKey); identityErr != nil { - return blockOwned(fmt.Errorf("%w: verify the manually logged-in browser identity: %v", creator.ErrConflict, identityErr)) - } syncLease, err := store.ClaimSourceSync(ctx, creator.SourceOwned, account.ID) if err != nil { return err @@ -1376,13 +1356,11 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed") } }() - collector := douyin.CreatorCollector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: creator.SourceOwned, SourceID: account.ID} - canonicalSecUID, err := collector.CanonicalSecUID(ctx, account.PlatformAccountKey) + collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceOwned, account.ID) if err != nil { blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays) return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr) } - collector.AccountKey = canonicalSecUID _, collectionNow, windowErr := store.NextCollectionWindow(ctx, creator.SourceOwned, account.ID, now, time.Duration(settings.NewWorkIntervalSeconds)*time.Second, settings.LookbackDays) if windowErr != nil { return windowErr diff --git a/cmd/control-plane/creator_material.go b/cmd/control-plane/creator_material.go index e410ea4..058f754 100644 --- a/cmd/control-plane/creator_material.go +++ b/cmd/control-plane/creator_material.go @@ -20,7 +20,7 @@ type creatorMaterialDownloader struct { } func (downloader creatorMaterialDownloader) Download(ctx context.Context, work creator.Work, destination string) error { - if work.Platform != creator.PlatformDouyin || downloader.store == nil || downloader.phaseAStore == nil || downloader.hubStore == nil { + if (work.Platform != creator.PlatformDouyin && work.Platform != creator.PlatformXiaohongshu) || downloader.store == nil || downloader.phaseAStore == nil || downloader.hubStore == nil { return fmt.Errorf("%w: creator media gateway is unavailable", creator.ErrUnavailable) } accountID := work.SourceID @@ -39,7 +39,7 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c if err != nil { return err } - if account.Platform != creator.PlatformDouyin || profile.Platform != creator.PlatformDouyin || account.AuthorizationStatus != "authorized" || profile.LoginStatus != "logged_in" || account.PlatformAccountKey != profile.PlatformAccountKey { + if account.Platform != work.Platform || profile.Platform != work.Platform || account.AuthorizationStatus != "authorized" || profile.LoginStatus != "logged_in" || account.PlatformAccountKey != profile.PlatformAccountKey { return fmt.Errorf("%w: media account identity is not verified", creator.ErrConflict) } environment, err := downloader.hubStore.GetEnvironmentContextForAccount(ctx, accountID) @@ -50,11 +50,21 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c if err != nil { return fmt.Errorf("%w: media gateway unavailable: %v", creator.ErrUnavailable, err) } - browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - if _, err := browser.Identity(ctx, profile.PlatformAccountKey); err != nil { + if _, err := verifyCreatorPlatformIdentity(ctx, work.Platform, gateway, environment, profile.PlatformAccountKey); err != nil { return fmt.Errorf("%w: media browser identity verification failed: %v", creator.ErrConflict, err) } - return browser.Media(ctx, work.OriginalURL, destination) + if work.Platform == creator.PlatformDouyin { + return (creatorGatewayBrowser{gateway: gateway, environment: environment}).Media(ctx, work.OriginalURL, destination) + } + data, contentType, err := (xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}).Media(ctx, work.OriginalURL) + if err != nil { + return err + } + contentType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) + if !strings.HasPrefix(contentType, "video/") && contentType != "application/octet-stream" { + return fmt.Errorf("xiaohongshu media response is not a video") + } + return writeCreatorMedia(destination, data) } func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, workID string) (creator.MaterialJob, error) { diff --git a/cmd/control-plane/xiaohongshu.go b/cmd/control-plane/xiaohongshu.go new file mode 100644 index 0000000..442cdad --- /dev/null +++ b/cmd/control-plane/xiaohongshu.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/xiaohongshu" +) + +type xiaohongshuGatewayBrowser struct { + gateway hub.Gateway + environment hub.EnvironmentContext +} + +func (browser xiaohongshuGatewayBrowser) generation() (map[string]any, error) { + request, err := (douyinGatewayBrowser{gateway: browser.gateway, environment: browser.environment}).request() + if err != nil { + return nil, err + } + return map[string]any{ + "binding_version": request.BindingVersion, + "runtime_id": request.RuntimeID, + "network_id": request.NetworkID, + "network_exit_id": request.NetworkExitID, + }, nil +} + +func (browser xiaohongshuGatewayBrowser) Get(ctx context.Context, target string) (xiaohongshu.Response, error) { + request, err := browser.generation() + if err != nil { + return xiaohongshu.Response{}, err + } + request["url"] = target + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/get", request, 30*time.Second) + if err != nil || status != http.StatusOK { + return xiaohongshu.Response{}, errors.New("restricted Xiaohongshu browser operation failed") + } + return decodeXiaohongshuResponse(body) +} + +func (browser xiaohongshuGatewayBrowser) Post(ctx context.Context, target string, payload []byte) (xiaohongshu.Response, error) { + if len(payload) == 0 || len(payload) > 4<<20 { + return xiaohongshu.Response{}, errors.New("invalid Xiaohongshu browser body") + } + var bodyValue any + if err := json.Unmarshal(payload, &bodyValue); err != nil { + return xiaohongshu.Response{}, fmt.Errorf("invalid Xiaohongshu browser body: %w", err) + } + if _, ok := bodyValue.(map[string]any); !ok { + return xiaohongshu.Response{}, errors.New("Xiaohongshu browser body must be an object") + } + request, err := browser.generation() + if err != nil { + return xiaohongshu.Response{}, err + } + request["url"] = target + request["body"] = bodyValue + status, responseBody, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/post", request, 30*time.Second) + if err != nil || status != http.StatusOK { + return xiaohongshu.Response{}, errors.New("restricted Xiaohongshu browser POST failed") + } + return decodeXiaohongshuResponse(responseBody) +} + +func (browser xiaohongshuGatewayBrowser) Identity(ctx context.Context, expectedKey string) (string, error) { + request, err := browser.generation() + if err != nil { + return "", err + } + request["expected_account_key"] = expectedKey + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/identity", request, 30*time.Second) + if err != nil || status != http.StatusOK { + return "", errors.New("Xiaohongshu identity verification failed") + } + var identity struct { + UID string `json:"uid"` + } + if err := json.Unmarshal(body, &identity); err != nil || strings.TrimSpace(identity.UID) == "" { + return "", errors.New("Xiaohongshu identity response omitted uid") + } + return identity.UID, nil +} + +func (browser xiaohongshuGatewayBrowser) Media(ctx context.Context, target string) ([]byte, string, error) { + request, err := browser.generation() + if err != nil { + return nil, "", err + } + request["url"] = target + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/media", request, 90*time.Second) + if err != nil || status != http.StatusOK { + return nil, "", errors.New("restricted Xiaohongshu media request failed") + } + var response struct { + Status int `json:"status"` + ContentType string `json:"content_type"` + BodyBase64 string `json:"body_base64"` + } + if err := json.Unmarshal(body, &response); err != nil || response.Status < 200 || response.Status >= 300 || response.BodyBase64 == "" { + return nil, "", errors.New("Xiaohongshu media response is invalid") + } + data, err := decodeBase64(response.BodyBase64) + if err != nil { + return nil, "", err + } + return data, response.ContentType, nil +} + +func decodeXiaohongshuResponse(body []byte) (xiaohongshu.Response, error) { + var response struct { + Status int `json:"status"` + Body string `json:"body"` + Challenge string `json:"challenge"` + } + if err := json.Unmarshal(body, &response); err != nil || response.Status < 100 || response.Status > 599 { + return xiaohongshu.Response{}, errors.New("restricted Xiaohongshu browser returned an invalid response") + } + return xiaohongshu.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil +} + +func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, accountKey, sourceType, sourceID string) (creator.PlatformCollector, string, error) { + switch platform { + case creator.PlatformDouyin: + browser := creatorGatewayBrowser{gateway: gateway, environment: environment} + uid, err := browser.Identity(ctx, accountKey) + if err != nil { + return nil, "", err + } + collector := douyinCollector(browser, accountKey, sourceType, sourceID) + canonical, err := collector.CanonicalSecUID(ctx, uid) + if err != nil { + return nil, "", err + } + collector.AccountKey = canonical + return &collector, canonical, nil + case creator.PlatformXiaohongshu: + browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment} + uid, err := browser.Identity(ctx, accountKey) + if err != nil { + return nil, "", err + } + return &xiaohongshu.Collector{Browser: browser, AccountKey: uid, SourceType: sourceType, SourceID: sourceID}, uid, nil + default: + return nil, "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform) + } +} + +func douyinCollector(browser creatorGatewayBrowser, accountKey, sourceType, sourceID string) douyin.CreatorCollector { + return douyin.CreatorCollector{Browser: browser, AccountKey: accountKey, SourceType: sourceType, SourceID: sourceID} +} + +func verifyCreatorPlatformIdentity(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, expectedKey string) (string, error) { + switch platform { + case creator.PlatformDouyin: + return (creatorGatewayBrowser{gateway: gateway, environment: environment}).Identity(ctx, expectedKey) + case creator.PlatformXiaohongshu: + return (xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}).Identity(ctx, expectedKey) + default: + return "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform) + } +} + +func decodeBase64(value string) ([]byte, error) { + const maxEncoded = 96 << 20 + if len(value) > maxEncoded { + return nil, errors.New("media response is too large") + } + data, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, fmt.Errorf("decode media response: %w", err) + } + if len(data) > maxCreatorMediaBytes { + return nil, errors.New("media response is too large") + } + return data, nil +} + +var _ xiaohongshu.Browser = xiaohongshuGatewayBrowser{} diff --git a/cmd/control-plane/xiaohongshu_test.go b/cmd/control-plane/xiaohongshu_test.go new file mode 100644 index 0000000..b9723ff --- /dev/null +++ b/cmd/control-plane/xiaohongshu_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +const testXiaohongshuIdentityURL = "https://edith.xiaohongshu.com/api/sns/web/v2/user/me" + +func TestXiaohongshuGatewayBrowserFencesAccountGeneration(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "Bearer gateway-token-1" { + t.Fatal("missing gateway authorization") + } + var body map[string]any + if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) || body["runtime_id"] != "runtime-a" || body["network_id"] != "network-a" || body["network_exit_id"] != "exit-a" { + t.Fatalf("generation fence missing: %#v", body) + } + if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/get" || body["url"] != testXiaohongshuIdentityURL { + t.Fatalf("unexpected request: path=%s body=%#v", request.URL.Path, body) + } + _ = json.NewEncoder(response).Encode(map[string]any{"status": 200, "body": `{"success":true}`, "challenge": ""}) + })) + defer server.Close() + browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()} + result, err := browser.Get(context.Background(), testXiaohongshuIdentityURL) + if err != nil || result.Status != 200 || string(result.Body) != `{"success":true}` { + t.Fatalf("unexpected result: %#v err=%v", result, err) + } +} + +func TestXiaohongshuGatewayBrowserPostCarriesJSONBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + var body map[string]any + if json.NewDecoder(request.Body).Decode(&body) != nil || body["url"] != "https://so.xiaohongshu.com/api/sns/web/v2/search/notes" { + t.Fatalf("unexpected request body: %#v", body) + } + if _, ok := body["body"].(map[string]any); !ok { + t.Fatalf("missing nested request body: %#v", body) + } + _ = json.NewEncoder(response).Encode(map[string]any{"status": 200, "body": `{"success":true}`, "challenge": ""}) + })) + defer server.Close() + browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()} + if _, err := browser.Post(context.Background(), "https://so.xiaohongshu.com/api/sns/web/v2/search/notes", []byte(`{"keyword":"x"}`)); err != nil { + t.Fatalf("post failed: %v", err) + } +} diff --git a/cmd/docker_gateway/douyin.py b/cmd/docker_gateway/douyin.py index 2beaef7..81faa98 100644 --- a/cmd/docker_gateway/douyin.py +++ b/cmd/docker_gateway/douyin.py @@ -1285,6 +1285,124 @@ def detect_challenge(status: int, body: str) -> str: return "" +XHS_ORIGIN = "https://www.xiaohongshu.com" +XHS_API_ORIGIN = "https://edith.xiaohongshu.com" +XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com" +XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me" +XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"}) + + +class XiaohongshuBrowser(DouyinBrowser): + def __init__(self, endpoint=None) -> None: + super().__init__( + endpoint, + origin=XHS_ORIGIN, + url_validator=is_xiaohongshu_url, + media_validator=is_xiaohongshu_media_url, + ) + + def post(self, alias: str, target: str, body: bytes) -> BrowserResponse: + if not is_xiaohongshu_url(target) or len(body) > RESPONSE_LIMIT: + raise DouyinError("restricted Xiaohongshu POST request is invalid") + try: + body_text = body.decode("utf-8") + except UnicodeDecodeError as exc: + raise DouyinError("restricted Xiaohongshu POST body is not UTF-8") from exc + with self.connection(alias) as cdp: + if cdp.evaluate("location.origin") != self.origin: + raise DouyinError("restricted browser origin changed") + expression = f"""(async()=>{{ + const r=await fetch({json.dumps(target)},{{method:'POST',headers:{{'content-type':'application/json'}},body:{json.dumps(body_text)},credentials:'include',redirect:'error'}}); + if(!r.body)return {{status:r.status,body:'',too_large:false}}; + const reader=r.body.getReader(), decoder=new TextDecoder(); let size=0, responseBody=''; + for(;;){{const item=await reader.read();if(item.done)break; + if(size+item.value.byteLength>={RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}} + size+=item.value.byteLength;responseBody+=decoder.decode(item.value,{{stream:true}}); + }} + responseBody+=decoder.decode();return {{status:r.status,body:responseBody,too_large:false}}; + }})()""" + result = cdp.evaluate(expression) + if ( + not isinstance(result, dict) + or result.get("too_large") + or not isinstance(result.get("status"), int) + ): + raise DouyinError("restricted Xiaohongshu POST failed") + status = result["status"] + if 300 <= status < 400: + raise DouyinError("restricted Xiaohongshu POST redirected") + response_body = result.get("body") + if not isinstance(response_body, str): + raise DouyinError("restricted Xiaohongshu POST returned invalid body") + return BrowserResponse(status, response_body, detect_challenge(status, response_body)) + + def identity(self, alias: str, expected_uid: str | None = None) -> dict: + response = self.get(alias, XHS_IDENTITY_URL) + try: + payload = json.loads(response.body) + except json.JSONDecodeError as exc: + raise DouyinError("Xiaohongshu identity response is invalid") from exc + data = payload.get("data") if isinstance(payload, dict) else None + user_info = data.get("user_info") if isinstance(data, dict) else None + user_id = data.get("user_id", "") if isinstance(data, dict) else "" + nickname = data.get("nickname", "") if isinstance(data, dict) else "" + if isinstance(user_info, dict): + user_id = user_id or user_info.get("user_id", "") + nickname = nickname or user_info.get("nickname", "") + success = payload.get("success") if isinstance(payload, dict) else None + if ( + response.status != 200 + or not isinstance(payload, dict) + or not isinstance(success, bool) + or not success + or not isinstance(user_id, str) + or not ACCOUNT_KEY_RE.fullmatch(user_id) + or nickname is not None + and not isinstance(nickname, str) + ): + raise DouyinError("Xiaohongshu login is not valid") + if expected_uid and user_id != expected_uid: + raise DouyinError("Xiaohongshu identity does not match the expected account") + return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""} + + +def is_xiaohongshu_media_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + except (TypeError, ValueError): + return False + return ( + parsed.scheme == "https" + and parsed.hostname == "www.xiaohongshu.com" + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + and parsed.path.startswith("/explore/") + ) + + +def is_xiaohongshu_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + except (TypeError, ValueError): + return False + return ( + parsed.scheme == "https" + and parsed.hostname in XHS_ALLOWED_HOSTS + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + ) + + def notice_ids(event: dict) -> list[str]: try: payload = json.loads(event["payload"]) diff --git a/cmd/docker_gateway/gateway.py b/cmd/docker_gateway/gateway.py index 9a8a128..005fd25 100644 --- a/cmd/docker_gateway/gateway.py +++ b/cmd/docker_gateway/gateway.py @@ -46,6 +46,7 @@ from .douyin import ( DouyinBrowser, DouyinError, SubscriptionManager, + XiaohongshuBrowser, ) from .proxy import ProxyExit, ProxyRegistry @@ -64,6 +65,12 @@ DOUYIN_IDENTITY_PATH = "/aweme/v1/web/user/profile/self/" DOUYIN_IDENTITY_URL = IDENTITY_URL DOUYIN_WORKS_PATH = WORKS_PATH DOUYIN_COMMENTS_PATH = COMMENTS_PATH +XHS_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$") +XHS_IDENTITY_PATH = "/api/sns/web/v2/user/me" +XHS_USER_POSTED_PATH = "/api/sns/web/v1/user_posted" +XHS_COMMENTS_PATH = "/api/sns/web/v2/comment/page" +XHS_SEARCH_PATH = "/api/sns/web/v2/search/notes" +XHS_FEED_PATH = "/api/sns/web/v1/feed" def _noop() -> None: @@ -85,12 +92,14 @@ class Gateway: token: str, self_name: str, browser: DouyinBrowser | None = None, + xiaohongshu_browser: XiaohongshuBrowser | None = None, ) -> None: self.docker = docker self.network = network self.token = token self.self_name = self_name self.browser = browser or DouyinBrowser(self._browser_endpoint) + self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint) self.proxies = ProxyRegistry() self.reservations = AliasReservationManager(docker, self_name) self.subscriptions = SubscriptionManager(self.browser) @@ -666,6 +675,76 @@ class Gateway: ) return identity + def get_xiaohongshu(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_url(target): + raise RequestError("invalid restricted Xiaohongshu request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.xiaohongshu_browser.get(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc)) + raise RequestError("restricted Xiaohongshu operation failed") from exc + return {"status": response.status, "body": response.body, "challenge": response.challenge} + + def post_xiaohongshu(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + body = input.get("body") + if ( + not valid_xiaohongshu_generation(input) + or not valid_xhs_post_url(target) + or not isinstance(body, dict) + ): + raise RequestError("invalid restricted Xiaohongshu POST request", 400) + try: + encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode() + except (TypeError, ValueError) as exc: + raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.xiaohongshu_browser.post(alias, target, encoded) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc)) + raise RequestError("restricted Xiaohongshu operation failed") from exc + return {"status": response.status, "body": response.body, "challenge": response.challenge} + + def get_xiaohongshu_media(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target): + raise RequestError("invalid restricted Xiaohongshu media request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.xiaohongshu_browser.get_media(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc)) + raise RequestError("restricted Xiaohongshu media download failed") from exc + return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64} + + def xiaohongshu_identity(self, alias: str, input: dict) -> dict: + expected_account_key = input.get("expected_account_key", "") + if ( + not valid_xiaohongshu_generation(input) + or not isinstance(expected_account_key, str) + or not XHS_ACCOUNT_KEY_RE.fullmatch(expected_account_key) + ): + raise RequestError("invalid Xiaohongshu identity request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + identity = self.xiaohongshu_browser.identity(alias) + except DouyinError as exc: + LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc)) + raise RequestError("Xiaohongshu login identity could not be verified") from exc + if identity.get("uid") != expected_account_key: + raise RequestError("Xiaohongshu identity does not match the expected account", 409) + return identity + def douyin_action(self, alias: str, input: dict) -> dict: expected_uid = input.get("expected_uid", "") action = input.get("action", "") @@ -1130,6 +1209,20 @@ class GatewayHandler(BaseHTTPRequestHandler): if method == "POST" and action == "proxy": gateway.restore_proxy(alias, body) return None + match = re.fullmatch( + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)", + path, + ) + if match: + alias, action = match.groups() + if action == "get" and method == "POST": + return gateway.get_xiaohongshu(alias, body) + if action == "post" and method == "POST": + return gateway.post_xiaohongshu(alias, body) + if action == "media" and method == "POST": + return gateway.get_xiaohongshu_media(alias, body) + if action == "identity" and method == "POST": + return gateway.xiaohongshu_identity(alias, body) match = re.fullmatch( r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|action|events)", path, @@ -1419,6 +1512,97 @@ def valid_douyin_generation(value: dict) -> bool: ) +def valid_xiaohongshu_generation(value: dict) -> bool: + return valid_douyin_generation(value) + + +def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool: + if not isinstance(query, dict) or not isinstance(allowed, set): + return False + required = required or set() + if not required.issubset(query) or not set(query).issubset(allowed): + return False + for key, values in query.items(): + if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1: + return False + if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]: + return False + return True + + +def _valid_xhs_host(parsed: object, host: str) -> bool: + return ( + getattr(parsed, "scheme", "") == "https" + and getattr(parsed, "hostname", None) == host + and getattr(parsed, "port", None) is None + and getattr(parsed, "username", None) is None + and getattr(parsed, "password", None) is None + and getattr(parsed, "fragment", "") == "" + ) + + +def valid_xhs_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH: + return not query + if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH: + return valid_xhs_query( + query, + {"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"}, + {"user_id", "num"}, + ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"] + if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH: + return valid_xhs_query( + query, + {"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"}, + {"note_id", "cursor", "top_comment_id"}, + ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0])) + return False + + +def valid_xiaohongshu_url(raw: object) -> bool: + return valid_xhs_url(raw) + + +def valid_xhs_post_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + return ( + _valid_xhs_host(parsed, "so.xiaohongshu.com") + and parsed.path == XHS_SEARCH_PATH + and not query + ) or ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_FEED_PATH + and not query + ) + + +def valid_xiaohongshu_media_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + if not _valid_xhs_host(parsed, "www.xiaohongshu.com"): + return False + parts = parsed.path.strip("/").split("/") + return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"}) + + def valid_douyin_url(raw: object) -> bool: if not isinstance(raw, str): return False diff --git a/cmd/docker_gateway/test_xiaohongshu.py b/cmd/docker_gateway/test_xiaohongshu.py new file mode 100644 index 0000000..274bd16 --- /dev/null +++ b/cmd/docker_gateway/test_xiaohongshu.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import unittest +from typing import Any, cast +from unittest.mock import Mock + +from . import gateway as gateway_module + +Gateway = gateway_module.Gateway +RequestError = gateway_module.RequestError +valid_xhs_post_url = gateway_module.valid_xhs_post_url +valid_xhs_url = gateway_module.valid_xhs_url +valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url +valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url +valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation + + +class XiaohongshuValidationTests(unittest.TestCase): + def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None: + self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")) + self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")) + self.assertTrue( + valid_xhs_url( + "https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user" + ) + ) + self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10")) + self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me")) + self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes")) + self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed")) + self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search")) + self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment")) + + def test_generation_shape_matches_existing_browser_fence(self) -> None: + self.assertTrue( + valid_xiaohongshu_generation( + {"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""} + ) + ) + self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"})) + + +class XiaohongshuRouteTests(unittest.TestCase): + def test_read_only_routes_dispatch_without_action_or_event_routes(self) -> None: + handler = gateway_module.GatewayHandler.__new__(gateway_module.GatewayHandler) + gateway = Mock() + gateway.get_xiaohongshu.return_value = {"status": 200} + gateway.post_xiaohongshu.return_value = {"status": 200} + gateway.get_xiaohongshu_media.return_value = {"status": 200} + gateway.xiaohongshu_identity.return_value = {"uid": "u-1"} + server = Mock() + server.gateway = gateway + cast(Any, handler).server = server + cast(Any, handler).server_as_gateway = lambda: server + + self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200}) + self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200}) + self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200}) + self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"}) + with self.assertRaises(RequestError): + handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {}) + gateway.get_xiaohongshu.assert_called_once_with("account-a", {}) + gateway.post_xiaohongshu.assert_called_once_with("account-a", {}) + gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {}) + gateway.xiaohongshu_identity.assert_called_once_with("account-a", {}) diff --git a/cmd/docker_gateway/xiaohongshu.py b/cmd/docker_gateway/xiaohongshu.py new file mode 100644 index 0000000..4423abf --- /dev/null +++ b/cmd/docker_gateway/xiaohongshu.py @@ -0,0 +1,27 @@ +"""Xiaohongshu gateway facade. + +The browser implementation lives next to the existing Douyin browser so both +platforms share the CDP transport and response limits without duplicating it. +""" + +from .douyin import ( + XHS_ALLOWED_HOSTS, + XHS_API_ORIGIN, + XHS_IDENTITY_URL, + XHS_ORIGIN, + XHS_SEARCH_ORIGIN, + XiaohongshuBrowser, + is_xiaohongshu_media_url, + is_xiaohongshu_url, +) + +__all__ = [ + "XHS_ALLOWED_HOSTS", + "XHS_API_ORIGIN", + "XHS_IDENTITY_URL", + "XHS_ORIGIN", + "XHS_SEARCH_ORIGIN", + "XiaohongshuBrowser", + "is_xiaohongshu_media_url", + "is_xiaohongshu_url", +] diff --git a/docs/research/xhs-all-in-one.md b/docs/research/xhs-all-in-one.md index b263770..279f1a0 100644 --- a/docs/research/xhs-all-in-one.md +++ b/docs/research/xhs-all-in-one.md @@ -4,7 +4,7 @@ > 上游仓库： > 研究快照：`e86f82b`（`master`） > 研究方式：只读检查源码、提交历史与公开 Issues；未向小红书真实账号发起请求，以下不等同于平台能力验收。 -> 当前结论：本文仅形成小红书后续实现的研究约束，尚未接入运行代码；未复制上游代码、签名脚本、Cookie 或指纹数据。 +> 当前结论：本文形成小红书实现约束；main 已接入受限只读适配器，但未复制上游代码、签名脚本、Cookie 或指纹数据。 ## 1. 结论先行 @@ -22,7 +22,7 @@ ## 1.1 当前实现边界 -- CreatorHub 当前只保留平台枚举和通用 Creator 数据模型；小红书专用 collector、gateway 路由和动作适配器未接入。 +- CreatorHub 当前已接入小红书只读 collector、详情/搜索/作品/一级评论读取、受限 gateway 路由和控制面平台分派；写操作与事件监听仍未接入。 - `internal/creator/collection.go` 的分页、窗口、checkpoint 和 lease 模型可作为后续适配的复用边界，但不能证明小红书平台能力。 **未完成的真实能力验收：**私有接口签名是否能由浏览器当前会话完成、真实 UID/作品/评论分页、媒体下载、写操作和事件监听仍需真实小红书环境分别验证。HTTP 200、离线 fixture 和本地单元测试不能替代这些证据。 diff --git a/internal/xiaohongshu/collector.go b/internal/xiaohongshu/collector.go new file mode 100644 index 0000000..a2f47df --- /dev/null +++ b/internal/xiaohongshu/collector.go @@ -0,0 +1,657 @@ +package xiaohongshu + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" + + "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +const ( + APIOrigin = "https://edith.xiaohongshu.com" + SearchOrigin = "https://so.xiaohongshu.com" + IdentityURL = APIOrigin + "/api/sns/web/v2/user/me" + UserPostedPath = "/api/sns/web/v1/user_posted" + CommentsPath = "/api/sns/web/v2/comment/page" + SearchNotesPath = "/api/sns/web/v2/search/notes" + FeedPath = "/api/sns/web/v1/feed" +) + +var keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`) + +type Response struct { + Status int + Body []byte + Challenge string +} + +type Browser interface { + Get(context.Context, string) (Response, error) + Post(context.Context, string, []byte) (Response, error) +} + +type Identity struct { + UserID string + Nickname string +} + +type Collector struct { + Browser Browser + AccountKey string + SourceType string + SourceID string + HomepageURL string + + contexts sync.Map +} + +type accessContext struct { + Token string + Source string +} + +func (c *Collector) VerifyIdentity(ctx context.Context, expectedKey string) error { + _, err := c.Identity(ctx, expectedKey) + return err +} + +func (c *Collector) Identity(ctx context.Context, expectedKey string) (Identity, error) { + if c == nil || c.Browser == nil || !keyPattern.MatchString(expectedKey) { + return Identity{}, fmt.Errorf("%w: invalid xiaohongshu identity request", creator.ErrInvalid) + } + response, err := c.Browser.Get(ctx, IdentityURL) + if err != nil { + return Identity{}, err + } + if err := responseError(response, "identity"); err != nil { + return Identity{}, err + } + identity, ok := parseIdentity(response.Body) + if !ok || identity.UserID != expectedKey { + return Identity{}, fmt.Errorf("%w: xiaohongshu identity mismatch", creator.ErrConflict) + } + return identity, nil +} + +func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) { + if c == nil { + return creator.WorkPage{}, fmt.Errorf("%w: xiaohongshu collector is nil", creator.ErrUnavailable) + } + if c.AccountKey != "" { + accountKey = c.AccountKey + } + if c.Browser == nil || !keyPattern.MatchString(accountKey) || !validCursor(cursor) { + return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu work collection request", creator.ErrInvalid) + } + access := c.defaultContext() + query := url.Values{ + "user_id": {accountKey}, + "cursor": {cursor}, + "num": {"30"}, + "image_formats": {"jpg,webp,avif"}, + "xsec_source": {access.Source}, + "xsec_token": {access.Token}, + } + response, err := c.Browser.Get(ctx, APIOrigin+UserPostedPath+"?"+query.Encode()) + if err != nil { + return creator.WorkPage{}, err + } + if err := responseError(response, "works"); err != nil { + return creator.WorkPage{}, err + } + items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, access) + if !ok { + return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu works response", creator.ErrInvalid) + } + return creator.WorkPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil +} + +func (c *Collector) ListTopLevelComments(ctx context.Context, workKey, cursor string) (creator.CommentPage, error) { + if c == nil || c.Browser == nil || !keyPattern.MatchString(workKey) || !validCursor(cursor) { + return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comment collection request", creator.ErrInvalid) + } + access := c.contextFor(workKey) + query := url.Values{ + "note_id": {workKey}, + "cursor": {cursor}, + "top_comment_id": {""}, + "image_formats": {"jpg,webp,avif"}, + "xsec_source": {access.Source}, + "xsec_token": {access.Token}, + } + response, err := c.Browser.Get(ctx, APIOrigin+CommentsPath+"?"+query.Encode()) + if err != nil { + return creator.CommentPage{}, err + } + if err := responseError(response, "comments"); err != nil { + return creator.CommentPage{}, err + } + items, nextCursor, hasMore, ok := parseCommentsPage(response.Body, workKey) + if !ok { + return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comments response", creator.ErrInvalid) + } + return creator.CommentPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil +} + +func (c *Collector) SearchNotes(ctx context.Context, queryText string, page int) (creator.WorkPage, error) { + if c == nil || c.Browser == nil || strings.TrimSpace(queryText) == "" || page < 1 || page > 10000 || utf8.RuneCountInString(queryText) > 200 { + return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search request", creator.ErrInvalid) + } + searchID, err := randomID() + if err != nil { + return creator.WorkPage{}, fmt.Errorf("create xiaohongshu search id: %w", err) + } + sessionID, err := randomID() + if err != nil { + return creator.WorkPage{}, fmt.Errorf("create xiaohongshu search session id: %w", err) + } + body, err := json.Marshal(map[string]any{ + "keyword": queryText, + "page": page, + "page_size": 20, + "search_id": searchID, + "sort": "general", + "note_type": 0, + "ext_flags": []any{}, + "geo": "", + "image_formats": []string{"jpg", "webp", "avif"}, + "session_id": sessionID, + }) + if err != nil { + return creator.WorkPage{}, err + } + response, err := c.Browser.Post(ctx, SearchOrigin+SearchNotesPath, body) + if err != nil { + return creator.WorkPage{}, err + } + if err := responseError(response, "search"); err != nil { + return creator.WorkPage{}, err + } + items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, c.defaultContext()) + if !ok { + return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search response", creator.ErrInvalid) + } + return creator.WorkPage{Items: items, NextCursor: nextCursor, HasMore: hasMore}, nil +} + +func randomID() (string, error) { + value := make([]byte, 16) + if _, err := rand.Read(value); err != nil { + return "", err + } + return hex.EncodeToString(value), nil +} + +func (c *Collector) defaultContext() accessContext { + access := accessContext{Source: "pc_user"} + if c != nil && strings.TrimSpace(c.HomepageURL) != "" { + parsed, err := url.Parse(c.HomepageURL) + if err == nil { + if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" { + access.Token = token + } + if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" { + access.Source = source + } + } + } + return access +} + +func (c *Collector) SetWorkContext(workKey, originalURL string) error { + if c == nil || !keyPattern.MatchString(workKey) { + return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid) + } + access := c.defaultContext() + if strings.TrimSpace(originalURL) != "" { + parsed, err := url.Parse(originalURL) + if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" || !strings.Contains(parsed.Path, workKey) { + return fmt.Errorf("%w: invalid xiaohongshu work URL", creator.ErrInvalid) + } + if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" { + access.Token = token + } + if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" { + access.Source = source + } + } + if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) { + return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid) + } + c.contexts.Store(workKey, access) + return nil +} + +func (c *Collector) contextFor(workKey string) accessContext { + if c != nil { + if value, ok := c.contexts.Load(workKey); ok { + if access, ok := value.(accessContext); ok { + return access + } + } + } + return c.defaultContext() +} + +func parseIdentity(body []byte) (Identity, bool) { + if len(body) == 0 || len(body) > 1<<20 { + return Identity{}, false + } + var envelope struct { + Success *bool `json:"success"` + Data *struct { + UserID string `json:"user_id"` + Nickname string `json:"nickname"` + UserInfo *struct { + UserID string `json:"user_id"` + Nickname string `json:"nickname"` + } `json:"user_info"` + } `json:"data"` + } + if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil { + return Identity{}, false + } + userID, nickname := envelope.Data.UserID, envelope.Data.Nickname + if envelope.Data.UserInfo != nil { + if userID == "" { + userID = envelope.Data.UserInfo.UserID + } + if nickname == "" { + nickname = envelope.Data.UserInfo.Nickname + } + } + if !keyPattern.MatchString(userID) || utf8.RuneCountInString(nickname) > 256 { + return Identity{}, false + } + return Identity{UserID: userID, Nickname: nickname}, true +} + +func parseWorksPage(body []byte, collector *Collector, fallback accessContext) ([]creator.WorkInput, string, bool, bool) { + if len(body) == 0 || len(body) > 4<<20 { + return nil, "", false, false + } + var envelope struct { + Success *bool `json:"success"` + Data *struct { + Cursor json.RawMessage `json:"cursor"` + HasMore *bool `json:"has_more"` + Notes []json.RawMessage `json:"notes"` + Items []json.RawMessage `json:"items"` + } `json:"data"` + } + if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil || envelope.Data.HasMore == nil { + return nil, "", false, false + } + rawNotes := envelope.Data.Notes + if rawNotes == nil { + rawNotes = envelope.Data.Items + } + if rawNotes == nil || len(rawNotes) > 30 { + return nil, "", false, false + } + items := make([]creator.WorkInput, 0, len(rawNotes)) + seen := make(map[string]struct{}, len(rawNotes)) + for _, raw := range rawNotes { + item, access, ok := parseWork(raw, fallback) + if !ok { + return nil, "", false, false + } + if _, exists := seen[item.WorkKey]; exists { + return nil, "", false, false + } + seen[item.WorkKey] = struct{}{} + if collector != nil { + item.SourceType = collector.SourceType + if item.SourceType == "" { + item.SourceType = creator.SourceCompetitor + } + item.SourceID = collector.SourceID + if item.SourceID == "" { + item.SourceID = collector.AccountKey + } + collector.contexts.Store(item.WorkKey, access) + } + items = append(items, item) + } + nextCursor, ok := cursorValue(envelope.Data.Cursor) + if !ok { + return nil, "", false, false + } + if *envelope.Data.HasMore && nextCursor == "" { + return nil, "", false, false + } + return items, nextCursor, *envelope.Data.HasMore, true +} + +func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput, accessContext, bool) { + object, ok := objectValue(raw) + if !ok { + return creator.WorkInput{}, accessContext{}, false + } + if nested := firstObject(object, "note_card", "noteCard"); nested != nil { + object = nested + } + id := firstString(object, "note_id", "id") + if !keyPattern.MatchString(id) { + return creator.WorkInput{}, accessContext{}, false + } + title := firstString(object, "display_title", "title") + body := firstString(object, "desc", "description", "content") + if utf8.RuneCountInString(title) > 4096 || utf8.RuneCountInString(body) > 100000 { + return creator.WorkInput{}, accessContext{}, false + } + user := firstObject(object, "user", "user_info", "author") + authorName := firstString(user, "nickname", "name") + published, publishedOK := optionalTimestamp(object, "time", "create_time", "last_update_time") + if !publishedOK { + return creator.WorkInput{}, accessContext{}, false + } + likes, likesOK := optionalInt(object, "likes", "liked_count") + comments, commentsOK := optionalInt(object, "comments_count", "comment_count") + shares, sharesOK := optionalInt(object, "shares", "shared_count") + interact := firstObject(object, "interact_info", "interactInfo", "statistics") + if interact != nil { + if likes == nil { + likes, likesOK = optionalInt(interact, "liked_count", "likes") + } + if comments == nil { + comments, commentsOK = optionalInt(interact, "comment_count", "comments_count") + } + if shares == nil { + shares, sharesOK = optionalInt(interact, "shared_count", "shares") + } + } + if !likesOK || !commentsOK || !sharesOK { + return creator.WorkInput{}, accessContext{}, false + } + access := fallback + if token := firstString(object, "xsec_token"); token != "" { + access.Token = token + } + if source := firstString(object, "xsec_source"); source != "" { + access.Source = source + } + if utf8.RuneCountInString(access.Token) > 2048 || !keyPattern.MatchString(access.Source) { + return creator.WorkInput{}, accessContext{}, false + } + originalURL := firstString(object, "original_url", "note_url", "url") + if !validOriginalURL(originalURL, id) { + originalURL = noteURL(id, access) + } + cover := coverURL(object) + status := "pending_verification" + if published != nil { + status = "verified" + } + return creator.WorkInput{ + Platform: creator.PlatformXiaohongshu, + WorkKey: id, + AuthorName: authorName, + Title: title, + Body: body, + PublishedAt: published, + PublishedAtStatus: status, + OriginalURL: originalURL, + CoverURL: cover, + Likes: likes, + CommentsCount: comments, + Shares: shares, + }, access, true +} + +func parseCommentsPage(body []byte, workKey string) ([]creator.CommentInput, string, bool, bool) { + if len(body) == 0 || len(body) > 4<<20 || !keyPattern.MatchString(workKey) { + return nil, "", false, false + } + var envelope struct { + Success *bool `json:"success"` + Data *struct { + Cursor json.RawMessage `json:"cursor"` + HasMore *bool `json:"has_more"` + Comments []json.RawMessage `json:"comments"` + } `json:"data"` + } + if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || envelope.Data == nil || envelope.Data.HasMore == nil || envelope.Data.Comments == nil || len(envelope.Data.Comments) > 100 { + return nil, "", false, false + } + items := make([]creator.CommentInput, 0, len(envelope.Data.Comments)) + seen := make(map[string]struct{}, len(envelope.Data.Comments)) + for _, raw := range envelope.Data.Comments { + object, ok := objectValue(raw) + if !ok { + return nil, "", false, false + } + id := firstString(object, "id", "comment_id", "commentId") + if !keyPattern.MatchString(id) { + return nil, "", false, false + } + if parent := firstString(object, "parent_comment_id", "parent_id", "reply_id"); parent != "" { + return nil, "", false, false + } + if typ := firstString(object, "comment_type", "type"); typ == "reply" || typ == "sub" { + return nil, "", false, false + } + if _, exists := seen[id]; exists { + return nil, "", false, false + } + seen[id] = struct{}{} + user := firstObject(object, "user_info", "user", "author") + published, publishedOK := optionalTimestamp(object, "create_time", "time", "created_at") + if !publishedOK { + return nil, "", false, false + } + content := firstString(object, "content", "text", "comment") + if content == "" || utf8.RuneCountInString(content) > 20000 { + return nil, "", false, false + } + items = append(items, creator.CommentInput{ + Platform: creator.PlatformXiaohongshu, + CommentKey: id, + WorkID: workKey, + AuthorUID: firstString(user, "user_id", "uid", "id"), + AuthorName: firstString(user, "nickname", "name"), + Content: content, + PublishedAt: published, + CommentType: "top_level", + }) + } + nextCursor, ok := cursorValue(envelope.Data.Cursor) + if !ok || *envelope.Data.HasMore && nextCursor == "" { + return nil, "", false, false + } + return items, nextCursor, *envelope.Data.HasMore, true +} + +func responseError(response Response, resource string) error { + if strings.TrimSpace(response.Challenge) != "" { + return fmt.Errorf("%w: xiaohongshu %s challenge %s", creator.ErrUnavailable, resource, response.Challenge) + } + if response.Status >= 200 && response.Status < 300 { + return nil + } + switch response.Status { + case 401, 403, 406, 461: + return fmt.Errorf("%w: xiaohongshu %s authentication/session rejected with HTTP %d", creator.ErrConflict, resource, response.Status) + case 429: + return fmt.Errorf("%w: xiaohongshu %s rate limited", creator.ErrUnavailable, resource) + default: + return fmt.Errorf("xiaohongshu %s returned HTTP %d", resource, response.Status) + } +} + +func validCursor(cursor string) bool { + return cursor == "" || len(cursor) <= 512 && !strings.ContainsAny(cursor, "\r\n") +} + +func objectValue(raw json.RawMessage) (map[string]json.RawMessage, bool) { + var object map[string]json.RawMessage + if len(raw) == 0 || json.Unmarshal(raw, &object) != nil || object == nil { + return nil, false + } + return object, true +} + +func firstObject(object map[string]json.RawMessage, names ...string) map[string]json.RawMessage { + for _, name := range names { + if value, ok := object[name]; ok { + if nested, ok := objectValue(value); ok { + return nested + } + } + } + return nil +} + +func firstString(object map[string]json.RawMessage, names ...string) string { + for _, name := range names { + value, ok := object[name] + if !ok || string(value) == "null" { + continue + } + var text string + if json.Unmarshal(value, &text) == nil { + return strings.TrimSpace(text) + } + var number json.Number + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.UseNumber() + if decoder.Decode(&number) == nil { + return number.String() + } + } + return "" +} + +func optionalInt(object map[string]json.RawMessage, names ...string) (*int64, bool) { + for _, name := range names { + value, ok := object[name] + if !ok { + continue + } + if string(value) == "null" { + return nil, true + } + var number json.Number + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.UseNumber() + if decoder.Decode(&number) != nil { + var text string + if json.Unmarshal(value, &text) != nil { + return nil, false + } + number = json.Number(text) + } + parsed, err := strconv.ParseInt(number.String(), 10, 64) + if err != nil || parsed < 0 { + return nil, false + } + return &parsed, true + } + return nil, true +} + +func optionalTimestamp(object map[string]json.RawMessage, names ...string) (*time.Time, bool) { + for _, name := range names { + value, ok := object[name] + if !ok { + continue + } + if string(value) == "null" { + return nil, true + } + var number json.Number + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.UseNumber() + if decoder.Decode(&number) != nil { + var text string + if json.Unmarshal(value, &text) != nil { + return nil, false + } + number = json.Number(text) + } + parsed, err := strconv.ParseInt(number.String(), 10, 64) + if err != nil || parsed <= 0 { + return nil, true + } + if parsed > 1_000_000_000_000 { + parsed /= 1000 + } + if parsed <= 0 || parsed > 4_102_444_800 { + return nil, false + } + when := time.Unix(parsed, 0).UTC() + return &when, true + } + return nil, true +} + +func cursorValue(raw json.RawMessage) (string, bool) { + if len(raw) == 0 || string(raw) == "null" { + return "", true + } + var text string + if json.Unmarshal(raw, &text) == nil { + return strings.TrimSpace(text), validCursor(strings.TrimSpace(text)) + } + var number json.Number + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if decoder.Decode(&number) != nil { + return "", false + } + return number.String(), validCursor(number.String()) +} + +func coverURL(object map[string]json.RawMessage) string { + images, ok := object["image_list"] + if ok { + var values []json.RawMessage + if json.Unmarshal(images, &values) == nil { + for _, value := range values { + if image, ok := objectValue(value); ok { + if cover := firstString(image, "url_default", "url_pre", "url_original", "url"); cover != "" { + return cover + } + } + } + } + } + if cover := firstObject(object, "cover"); cover != nil { + return firstString(cover, "url_default", "url_pre", "url_original", "url") + } + return firstString(object, "cover_url", "cover") +} + +func validOriginalURL(raw, id string) bool { + if raw == "" { + return false + } + parsed, err := url.Parse(raw) + return err == nil && parsed.Scheme == "https" && (parsed.Host == "www.xiaohongshu.com" || parsed.Host == "xhslink.com") && strings.Contains(parsed.Path, id) +} + +func noteURL(id string, access accessContext) string { + query := url.Values{} + if access.Token != "" { + query.Set("xsec_token", access.Token) + } + if access.Source != "" { + query.Set("xsec_source", access.Source) + } + result := "https://www.xiaohongshu.com/explore/" + url.PathEscape(id) + if encoded := query.Encode(); encoded != "" { + result += "?" + encoded + } + return result +} diff --git a/internal/xiaohongshu/collector_test.go b/internal/xiaohongshu/collector_test.go new file mode 100644 index 0000000..9d02f8e --- /dev/null +++ b/internal/xiaohongshu/collector_test.go @@ -0,0 +1,82 @@ +package xiaohongshu + +import ( + "context" + "errors" + "net/url" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +type fakeBrowser struct { + getURL string + postURL string + getBody []byte + postBody []byte + getResp Response + postResp Response +} + +func (f *fakeBrowser) Get(_ context.Context, target string) (Response, error) { + f.getURL = target + return f.getResp, nil +} +func (f *fakeBrowser) Post(_ context.Context, target string, body []byte) (Response, error) { + f.postURL, f.postBody = target, body + return f.postResp, nil +} + +func TestIdentityRequiresMatchingUser(t *testing.T) { + browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"user_id":"u-1","nickname":"作者"}}`)}} + collector := &Collector{Browser: browser} + identity, err := collector.Identity(context.Background(), "u-1") + if err != nil || identity.UserID != "u-1" { + t.Fatalf("identity = %#v, err = %v", identity, err) + } + if _, err := collector.Identity(context.Background(), "u-2"); !errors.Is(err, creator.ErrConflict) { + t.Fatalf("expected identity conflict, got %v", err) + } +} + +func TestListWorksCapturesContextForComments(t *testing.T) { + browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"next","has_more":true,"notes":[{"note_id":"n-1","title":"标题","desc":"正文","time":1710000000,"user":{"user_id":"u-1","nickname":"作者"},"interact_info":{"liked_count":"2","comment_count":3,"shared_count":4},"xsec_token":"token"}]}}`)}} + collector := &Collector{Browser: browser, AccountKey: "u-1", SourceType: creator.SourceOwned, SourceID: "source-1"} + page, err := collector.ListWorks(context.Background(), "ignored", "") + if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || !page.HasMore { + t.Fatalf("page = %#v, err = %v", page, err) + } + parsed, err := url.Parse(browser.getURL) + if err != nil || parsed.Query().Get("user_id") != "u-1" { + t.Fatalf("request URL = %s", browser.getURL) + } + browser.getResp = Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"comments":[{"id":"c-1","content":"评论","create_time":1710000000,"user_info":{"user_id":"u-2","nickname":"读者"}}]}}`)} + comments, err := collector.ListTopLevelComments(context.Background(), "n-1", "") + if err != nil || len(comments.Items) != 1 || comments.Items[0].CommentType != "top_level" { + t.Fatalf("comments = %#v, err = %v", comments, err) + } + commentURL, err := url.Parse(browser.getURL) + if err != nil || commentURL.Query().Get("xsec_token") != "token" { + t.Fatalf("comment request URL = %s", browser.getURL) + } +} + +func TestSearchUsesBoundedPostAndRejectsMalformedPage(t *testing.T) { + browser := &fakeBrowser{postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"notes":[]}}`)}} + collector := &Collector{Browser: browser} + page, err := collector.SearchNotes(context.Background(), "关键词", 1) + if err != nil || page.HasMore || len(page.Items) != 0 || browser.postURL != SearchOrigin+SearchNotesPath { + t.Fatalf("page = %#v, URL = %s, err = %v", page, browser.postURL, err) + } + browser.postResp = Response{Status: 200, Body: []byte(`{"success":true,"data":{"has_more":true,"notes":[]}}`)} + if _, err := collector.SearchNotes(context.Background(), "关键词", 1); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("expected malformed response error, got %v", err) + } +} + +func TestSetWorkContextRejectsWrongHost(t *testing.T) { + collector := &Collector{} + if err := collector.SetWorkContext("n-1", "https://evil.example/explore/n-1"); !errors.Is(err, creator.ErrInvalid) { + t.Fatalf("expected invalid URL, got %v", err) + } +} diff --git a/internal/xiaohongshu/detail.go b/internal/xiaohongshu/detail.go new file mode 100644 index 0000000..b037922 --- /dev/null +++ b/internal/xiaohongshu/detail.go @@ -0,0 +1,101 @@ +package xiaohongshu + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + + "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +// GetNoteDetail reads one note through the platform feed endpoint and keeps +// the original link so xsec context survives a later comments resume. +func (c *Collector) GetNoteDetail(ctx context.Context, originalURL string) (creator.WorkInput, error) { + if c == nil || c.Browser == nil { + return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail request", creator.ErrInvalid) + } + parsed, err := url.Parse(originalURL) + if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" { + return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid) + } + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) != 2 || parts[0] != "explore" || !keyPattern.MatchString(parts[1]) { + return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid) + } + access := accessContext{Token: parsed.Query().Get("xsec_token"), Source: parsed.Query().Get("xsec_source")} + if access.Source == "" { + access.Source = "pc_search" + } + if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) { + return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL context", creator.ErrInvalid) + } + body, err := json.Marshal(map[string]any{ + "source_note_id": parts[1], + "image_formats": []string{"jpg", "webp", "avif"}, + "extra": map[string]string{"need_body_topic": "1"}, + "xsec_source": access.Source, + "xsec_token": access.Token, + }) + if err != nil { + return creator.WorkInput{}, fmt.Errorf("encode xiaohongshu detail request: %w", err) + } + response, err := c.Browser.Post(ctx, APIOrigin+FeedPath, body) + if err != nil { + return creator.WorkInput{}, err + } + if err := responseError(response, "detail"); err != nil { + return creator.WorkInput{}, err + } + item, parsedAccess, ok := parseDetail(response.Body, access) + if !ok { + return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail response", creator.ErrInvalid) + } + if item.WorkKey != parts[1] { + return creator.WorkInput{}, fmt.Errorf("%w: xiaohongshu detail returned another note", creator.ErrConflict) + } + item.OriginalURL = originalURL + item.SourceType = c.SourceType + item.SourceID = c.SourceID + if item.SourceType == "" { + item.SourceType = creator.SourceCompetitor + } + if item.SourceID == "" { + item.SourceID = c.AccountKey + } + c.contexts.Store(item.WorkKey, parsedAccess) + return item, nil +} + +func parseDetail(body []byte, fallback accessContext) (creator.WorkInput, accessContext, bool) { + if len(body) == 0 || len(body) > 4<<20 { + return creator.WorkInput{}, accessContext{}, false + } + var envelope struct { + Success *bool `json:"success"` + Data json.RawMessage `json:"data"` + } + if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || !*envelope.Success || len(envelope.Data) == 0 { + return creator.WorkInput{}, accessContext{}, false + } + data, ok := objectValue(envelope.Data) + if !ok { + return creator.WorkInput{}, accessContext{}, false + } + if items, ok := arrayValue(data["items"]); ok && len(items) > 0 { + return parseWork(items[0], fallback) + } + if note := data["note"]; len(note) > 0 { + return parseWork(note, fallback) + } + return parseWork(envelope.Data, fallback) +} + +func arrayValue(raw json.RawMessage) ([]json.RawMessage, bool) { + var values []json.RawMessage + if len(raw) == 0 || json.Unmarshal(raw, &values) != nil || values == nil { + return nil, false + } + return values, true +} diff --git a/internal/xiaohongshu/detail_test.go b/internal/xiaohongshu/detail_test.go new file mode 100644 index 0000000..b040d16 --- /dev/null +++ b/internal/xiaohongshu/detail_test.go @@ -0,0 +1,33 @@ +package xiaohongshu + +import ( + "context" + "errors" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +func TestGetNoteDetailPreservesShareContext(t *testing.T) { + browser := &fakeBrowser{postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"items":[{"note_id":"n-1","title":"详情","desc":"正文","time":1710000000,"user":{"user_id":"u-1","nickname":"作者"},"interact_info":{"liked_count":1,"comment_count":2,"shared_count":3}}]}}`)}} + collector := &Collector{Browser: browser, SourceType: creator.SourceCompetitor, SourceID: "source-1"} + item, err := collector.GetNoteDetail(context.Background(), "https://www.xiaohongshu.com/explore/n-1?xsec_token=tok-1&xsec_source=pc_search") + if err != nil || item.WorkKey != "n-1" || item.OriginalURL == "" { + t.Fatalf("item = %#v, err = %v", item, err) + } + if err := collector.SetWorkContext("n-1", item.OriginalURL); err != nil { + t.Fatalf("set work context: %v", err) + } + if browser.postURL != APIOrigin+FeedPath { + t.Fatalf("post URL = %s", browser.postURL) + } +} + +func TestGetNoteDetailRejectsWrongResponseID(t *testing.T) { + browser := &fakeBrowser{postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"items":[{"note_id":"n-2","title":"详情","time":1710000000,"interact_info":{}}]}}`)}} + collector := &Collector{Browser: browser} + _, err := collector.GetNoteDetail(context.Background(), "https://www.xiaohongshu.com/explore/n-1") + if !errors.Is(err, creator.ErrConflict) { + t.Fatalf("expected conflict, got %v", err) + } +}