diff --git a/cmd/control-plane/creator.go b/cmd/control-plane/creator.go index 060028c..a013ce9 100644 --- a/cmd/control-plane/creator.go +++ b/cmd/control-plane/creator.go @@ -255,6 +255,9 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto if err := decodeCreator(c, &input); err != nil { return creatorError(c, err) } + if err := validateXiaohongshuCompetitor(input); err != nil { + return creatorError(c, err) + } item, err := store.CreateCompetitor(c.Context(), input) if err != nil { return creatorError(c, err) @@ -296,6 +299,47 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto return c.Status(fiber.StatusAccepted).JSON(report) }) + app.Post("/api/creator/xiaohongshu/search", func(c fiber.Ctx) error { + var input struct { + AccountID string `json:"account_id"` + Query string `json:"query"` + Page int `json:"page"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + if input.Page == 0 { + input.Page = 1 + } + collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID) + if err != nil { + return creatorError(c, err) + } + page, err := collector.SearchNotes(c.Context(), input.Query, input.Page) + if err != nil { + return creatorError(c, err) + } + return c.JSON(page) + }) + app.Post("/api/creator/xiaohongshu/detail", func(c fiber.Ctx) error { + var input struct { + AccountID string `json:"account_id"` + URL string `json:"url"` + } + if err := decodeCreator(c, &input); err != nil { + return creatorError(c, err) + } + collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID) + if err != nil { + return creatorError(c, err) + } + item, err := collector.GetNoteDetail(c.Context(), input.URL) + if err != nil { + return creatorError(c, err) + } + return c.JSON(item) + }) + app.Get("/api/creator/works", func(c fiber.Ctx) error { filter, err := workFilter(c) if err != nil { @@ -1128,6 +1172,11 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p if competitor.Platform != creator.PlatformDouyin && competitor.Platform != creator.PlatformXiaohongshu { return blocked(fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, competitor.Platform)) } + if competitor.Platform == creator.PlatformXiaohongshu { + if err := validateXiaohongshuSource(competitor.HomepageURL, competitor.PlatformAccountKey); err != nil { + return blocked(err) + } + } account, err := phaseAStore.GetAccount(ctx, accountID) if err != nil { return blocked(err) @@ -1153,7 +1202,7 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p if err != nil { return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)) } - collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceCompetitor, competitor.ID) + collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, competitor.PlatformAccountKey, competitor.HomepageURL, creator.SourceCompetitor, competitor.ID) if err != nil { return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)) } @@ -1283,7 +1332,18 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if err != nil { return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, work.SourceType, work.SourceID) + targetAccountKey, homepageURL := account.PlatformAccountKey, "" + if work.SourceType == creator.SourceCompetitor { + competitor, competitorErr := store.GetCompetitor(ctx, work.SourceID) + if competitorErr != nil { + return competitorErr + } + if competitor.Platform != work.Platform { + return creator.ErrConflict + } + targetAccountKey, homepageURL = competitor.PlatformAccountKey, competitor.HomepageURL + } + collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, targetAccountKey, homepageURL, work.SourceType, work.SourceID) if err != nil { return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err) } @@ -1356,7 +1416,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed") } }() - collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceOwned, account.ID) + collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, account.PlatformAccountKey, "", creator.SourceOwned, account.ID) if err != nil { blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays) return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr) diff --git a/cmd/control-plane/xiaohongshu.go b/cmd/control-plane/xiaohongshu.go index 442cdad..c2c4875 100644 --- a/cmd/control-plane/xiaohongshu.go +++ b/cmd/control-plane/xiaohongshu.go @@ -14,6 +14,7 @@ import ( "git.ipao.vip/rogee/creator-hub/internal/creator" "git.ipao.vip/rogee/creator-hub/internal/douyin" "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/phasea" "git.ipao.vip/rogee/creator-hub/internal/xiaohongshu" ) @@ -94,6 +95,26 @@ func (browser xiaohongshuGatewayBrowser) Identity(ctx context.Context, expectedK return identity.UID, nil } +func (browser xiaohongshuGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) { + request, err := browser.generation() + if err != nil { + return "", err + } + request["url"] = target + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/resolve", request, 30*time.Second) + if err != nil || status != http.StatusOK { + return "", errors.New("restricted Xiaohongshu share resolution failed") + } + var response struct { + URL string `json:"url"` + } + if err := json.Unmarshal(body, &response); err != nil || strings.TrimSpace(response.URL) == "" { + return "", errors.New("Xiaohongshu share resolution response omitted url") + } + return response.URL, nil +} + func (browser xiaohongshuGatewayBrowser) Media(ctx context.Context, target string) ([]byte, string, error) { request, err := browser.generation() if err != nil { @@ -132,28 +153,79 @@ func decodeXiaohongshuResponse(body []byte) (xiaohongshu.Response, error) { return xiaohongshu.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil } -func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, accountKey, sourceType, sourceID string) (creator.PlatformCollector, string, error) { +func newXiaohongshuReadCollector(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, sourceType, sourceID string) (*xiaohongshu.Collector, error) { + if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" { + return nil, creator.ErrUnavailable + } + account, err := phaseAStore.GetAccount(ctx, accountID) + if err != nil { + return nil, err + } + if account.Platform != creator.PlatformXiaohongshu || account.AuthorizationStatus != "authorized" { + return nil, creator.ErrConflict + } + profile, err := store.GetAccountProfile(ctx, accountID) + if err != nil { + return nil, err + } + if profile.Platform != creator.PlatformXiaohongshu || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" { + return nil, creator.ErrConflict + } + environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) + if err != nil { + return nil, fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) + } + if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 { + return nil, fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable) + } + gateway, err := hubStore.GetGateway(ctx, environment.Gateway) + if err != nil { + return nil, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) + } + browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment} + if _, err := browser.Identity(ctx, account.PlatformAccountKey); err != nil { + return nil, fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err) + } + return &xiaohongshu.Collector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: sourceType, SourceID: sourceID}, nil +} + +func validateXiaohongshuSource(homepageURL, accountKey string) error { + return xiaohongshu.ValidateSourceURL(homepageURL, accountKey) +} + +func validateXiaohongshuCompetitor(input creator.CompetitorInput) error { + if input.Platform != creator.PlatformXiaohongshu { + return nil + } + return validateXiaohongshuSource(input.HomepageURL, input.PlatformAccountKey) +} + +func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, viewerAccountKey, targetAccountKey, homepageURL, sourceType, sourceID string) (creator.PlatformCollector, string, error) { + if strings.TrimSpace(viewerAccountKey) == "" || strings.TrimSpace(targetAccountKey) == "" { + return nil, "", fmt.Errorf("%w: creator collector account key is missing", creator.ErrInvalid) + } switch platform { case creator.PlatformDouyin: browser := creatorGatewayBrowser{gateway: gateway, environment: environment} - uid, err := browser.Identity(ctx, accountKey) - if err != nil { + if _, err := browser.Identity(ctx, viewerAccountKey); err != nil { return nil, "", err } - collector := douyinCollector(browser, accountKey, sourceType, sourceID) - canonical, err := collector.CanonicalSecUID(ctx, uid) - if err != nil { + collector := douyinCollector(browser, targetAccountKey, sourceType, sourceID) + if _, err := collector.CanonicalSecUID(ctx, viewerAccountKey); err != nil { return nil, "", err } - collector.AccountKey = canonical - return &collector, canonical, nil + return &collector, targetAccountKey, nil case creator.PlatformXiaohongshu: + if homepageURL != "" { + if err := xiaohongshu.ValidateSourceURL(homepageURL, targetAccountKey); err != nil { + return nil, "", err + } + } browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment} - uid, err := browser.Identity(ctx, accountKey) - if err != nil { + if _, err := browser.Identity(ctx, viewerAccountKey); err != nil { return nil, "", err } - return &xiaohongshu.Collector{Browser: browser, AccountKey: uid, SourceType: sourceType, SourceID: sourceID}, uid, nil + return &xiaohongshu.Collector{Browser: browser, AccountKey: targetAccountKey, HomepageURL: homepageURL, SourceType: sourceType, SourceID: sourceID}, targetAccountKey, nil default: return nil, "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform) } diff --git a/cmd/control-plane/xiaohongshu_test.go b/cmd/control-plane/xiaohongshu_test.go index b9723ff..89592a8 100644 --- a/cmd/control-plane/xiaohongshu_test.go +++ b/cmd/control-plane/xiaohongshu_test.go @@ -7,7 +7,9 @@ import ( "net/http/httptest" "testing" + "git.ipao.vip/rogee/creator-hub/internal/creator" "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/xiaohongshu" ) const testXiaohongshuIdentityURL = "https://edith.xiaohongshu.com/api/sns/web/v2/user/me" @@ -34,6 +36,39 @@ func TestXiaohongshuGatewayBrowserFencesAccountGeneration(t *testing.T) { } } +func TestXiaohongshuGatewayBrowserResolvesShareLinks(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/resolve" { + t.Fatalf("unexpected path: %s", request.URL.Path) + } + _ = json.NewEncoder(response).Encode(map[string]any{"url": "https://www.xiaohongshu.com/explore/n-1"}) + })) + defer server.Close() + browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()} + resolved, err := browser.Resolve(context.Background(), "https://xhslink.com/a/abc") + if err != nil || resolved != "https://www.xiaohongshu.com/explore/n-1" { + t.Fatalf("resolved URL=%q err=%v", resolved, err) + } +} + +func TestNewXiaohongshuCollectorKeepsViewerAndTargetSeparate(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/identity" { + t.Fatalf("unexpected path: %s", request.URL.Path) + } + _ = json.NewEncoder(response).Encode(map[string]any{"uid": "viewer-1"}) + })) + defer server.Close() + collector, target, err := newCreatorCollector(context.Background(), creator.PlatformXiaohongshu, hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, readyDouyinEnvironment(), "viewer-1", "target-1", "https://www.xiaohongshu.com/user/profile/target-1?xsec_source=pc_search", creator.SourceCompetitor, "source-1") + if err != nil { + t.Fatalf("new collector: %v", err) + } + xhsCollector, ok := collector.(*xiaohongshu.Collector) + if !ok || xhsCollector.AccountKey != "target-1" || target != "target-1" || xhsCollector.HomepageURL == "" { + t.Fatalf("collector=%#v target=%q", collector, target) + } +} + func TestXiaohongshuGatewayBrowserPostCarriesJSONBody(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { var body map[string]any diff --git a/cmd/docker_gateway/douyin.py b/cmd/docker_gateway/douyin.py index 81faa98..b6a6da8 100644 --- a/cmd/docker_gateway/douyin.py +++ b/cmd/docker_gateway/douyin.py @@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, suppress from dataclasses import dataclass from datetime import datetime, timezone -from urllib.parse import urlsplit +from urllib.parse import parse_qs, urlsplit import websocket @@ -229,6 +229,7 @@ class DouyinBrowser: origin: str = ORIGIN, url_validator: Callable[[object], bool] | None = None, media_validator: Callable[[object], bool] | None = None, + media_selector: str = "video", ) -> None: self.endpoint = endpoint or ( lambda alias: f"http://creatorhub-browser-{alias}:9222" @@ -236,6 +237,7 @@ class DouyinBrowser: self.origin = origin self.url_validator = url_validator or is_douyin_url self.media_validator = media_validator or is_douyin_media_url + self.media_selector = media_selector @contextmanager def connection(self, alias: str): @@ -396,8 +398,8 @@ class DouyinBrowser: raise DouyinError("Douyin media page did not load") result = cdp.evaluate( f"""(async()=>{{ - const video=document.querySelector('video'); - const source=video?.currentSrc||video?.src||''; + const media=document.querySelector({json.dumps(self.media_selector)}); + const source=media?.currentSrc||media?.src||''; if(!source)return {{error:'media_source_unavailable'}}; const r=await fetch(source,{{credentials:'include',redirect:'error'}}); if(!r.body)return {{status:r.status,content_type:r.headers.get('content-type')||'',body:''}}; @@ -1289,7 +1291,9 @@ XHS_ORIGIN = "https://www.xiaohongshu.com" XHS_API_ORIGIN = "https://edith.xiaohongshu.com" XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com" XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me" -XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"}) +XHS_ALLOWED_HOSTS = frozenset( + {"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"} +) class XiaohongshuBrowser(DouyinBrowser): @@ -1299,6 +1303,7 @@ class XiaohongshuBrowser(DouyinBrowser): origin=XHS_ORIGIN, url_validator=is_xiaohongshu_url, media_validator=is_xiaohongshu_media_url, + media_selector="video, img.note-slider-img", ) def post(self, alias: str, target: str, body: bytes) -> BrowserResponse: @@ -1334,7 +1339,29 @@ class XiaohongshuBrowser(DouyinBrowser): response_body = result.get("body") if not isinstance(response_body, str): raise DouyinError("restricted Xiaohongshu POST returned invalid body") - return BrowserResponse(status, response_body, detect_challenge(status, response_body)) + return BrowserResponse( + status, response_body, detect_challenge(status, response_body) + ) + + def resolve(self, alias: str, target: str) -> str: + if not is_xiaohongshu_share_url(target): + raise DouyinError("restricted Xiaohongshu share URL is invalid") + with self.connection(alias) as cdp: + if cdp.evaluate("location.origin") != self.origin: + raise DouyinError("restricted browser origin changed") + cdp.command("Page.navigate", {"url": target}) + event = cdp.wait_event( + "Page.frameNavigated", + lambda params: _is_xiaohongshu_page_url( + params.get("frame", {}).get("url", "") + ), + ) + final_url = event.get("frame", {}).get("url") + if not isinstance(final_url, str) or not _is_xiaohongshu_page_url(final_url): + raise DouyinError( + "Xiaohongshu share URL did not resolve to a supported page" + ) + return final_url def identity(self, alias: str, expected_uid: str | None = None) -> dict: response = self.get(alias, XHS_IDENTITY_URL) @@ -1362,7 +1389,9 @@ class XiaohongshuBrowser(DouyinBrowser): ): raise DouyinError("Xiaohongshu login is not valid") if expected_uid and user_id != expected_uid: - raise DouyinError("Xiaohongshu identity does not match the expected account") + raise DouyinError( + "Xiaohongshu identity does not match the expected account" + ) return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""} @@ -1403,6 +1432,62 @@ def is_xiaohongshu_url(value: object) -> bool: ) +def is_xiaohongshu_share_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + except (TypeError, ValueError): + return False + path = parsed.path.strip("/") + return ( + parsed.scheme == "https" + and parsed.hostname in {"xhslink.com", "www.xhslink.com"} + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + and bool(path) + and len(path) <= 256 + and not parsed.query + ) + + +def _is_xiaohongshu_page_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + query = parse_qs(parsed.query, keep_blank_values=True) + except (TypeError, ValueError): + return False + parts = parsed.path.strip("/").split("/") + if not ( + parsed.scheme == "https" + and parsed.hostname == "www.xiaohongshu.com" + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + and ( + len(parts) == 2 + and parts[0] == "explore" + or len(parts) == 3 + and parts[:2] == ["user", "profile"] + ) + ): + return False + return all( + key in {"xsec_token", "xsec_source"} + and len(values) == 1 + and len(values[0]) <= 2048 + and not any(char in values[0] for char in "\r\n") + for key, values in query.items() + ) + + def notice_ids(event: dict) -> list[str]: try: payload = json.loads(event["payload"]) diff --git a/cmd/docker_gateway/gateway.py b/cmd/docker_gateway/gateway.py index 005fd25..a58f288 100644 --- a/cmd/docker_gateway/gateway.py +++ b/cmd/docker_gateway/gateway.py @@ -47,6 +47,7 @@ from .douyin import ( DouyinError, SubscriptionManager, XiaohongshuBrowser, + is_xiaohongshu_share_url, ) from .proxy import ProxyExit, ProxyRegistry @@ -99,7 +100,9 @@ class Gateway: self.token = token self.self_name = self_name self.browser = browser or DouyinBrowser(self._browser_endpoint) - self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint) + self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser( + self._browser_endpoint + ) self.proxies = ProxyRegistry() self.reservations = AliasReservationManager(docker, self_name) self.subscriptions = SubscriptionManager(self.browser) @@ -685,9 +688,15 @@ class Gateway: response = self.xiaohongshu_browser.get(alias, target) self._require_douyin_generation(alias, input) except DouyinError as exc: - LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc)) + LOG.warning( + "Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc) + ) raise RequestError("restricted Xiaohongshu operation failed") from exc - return {"status": response.status, "body": response.body, "challenge": response.challenge} + return { + "status": response.status, + "body": response.body, + "challenge": response.challenge, + } def post_xiaohongshu(self, alias: str, input: dict) -> dict: target = input.get("url", "") @@ -699,7 +708,9 @@ class Gateway: ): raise RequestError("invalid restricted Xiaohongshu POST request", 400) try: - encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode() + encoded = json.dumps( + body, ensure_ascii=False, separators=(",", ":") + ).encode() except (TypeError, ValueError) as exc: raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc with self._alias_lock(alias): @@ -708,13 +719,47 @@ class Gateway: response = self.xiaohongshu_browser.post(alias, target, encoded) self._require_douyin_generation(alias, input) except DouyinError as exc: - LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc)) + LOG.warning( + "Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc) + ) raise RequestError("restricted Xiaohongshu operation failed") from exc - return {"status": response.status, "body": response.body, "challenge": response.challenge} + return { + "status": response.status, + "body": response.body, + "challenge": response.challenge, + } + + def resolve_xiaohongshu(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_source_url( + target + ): + raise RequestError("invalid restricted Xiaohongshu source URL", 400) + if not is_xiaohongshu_share_url(target): + return {"url": target} + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + resolved = self.xiaohongshu_browser.resolve(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Xiaohongshu share resolution failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "restricted Xiaohongshu share resolution failed" + ) from exc + if not valid_xiaohongshu_page_url(resolved): + raise RequestError("Xiaohongshu share resolved to an unsupported URL", 502) + return {"url": resolved} def get_xiaohongshu_media(self, alias: str, input: dict) -> dict: target = input.get("url", "") - if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target): + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url( + target + ): raise RequestError("invalid restricted Xiaohongshu media request", 400) with self._alias_lock(alias): self._require_douyin_generation(alias, input) @@ -722,9 +767,19 @@ class Gateway: response = self.xiaohongshu_browser.get_media(alias, target) self._require_douyin_generation(alias, input) except DouyinError as exc: - LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc)) - raise RequestError("restricted Xiaohongshu media download failed") from exc - return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64} + LOG.warning( + "Xiaohongshu media download failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "restricted Xiaohongshu media download failed" + ) from exc + return { + "status": response.status, + "content_type": response.content_type, + "body_base64": response.body_base64, + } def xiaohongshu_identity(self, alias: str, input: dict) -> dict: expected_account_key = input.get("expected_account_key", "") @@ -739,10 +794,18 @@ class Gateway: try: identity = self.xiaohongshu_browser.identity(alias) except DouyinError as exc: - LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc)) - raise RequestError("Xiaohongshu login identity could not be verified") from exc + LOG.warning( + "Xiaohongshu identity verification failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "Xiaohongshu login identity could not be verified" + ) from exc if identity.get("uid") != expected_account_key: - raise RequestError("Xiaohongshu identity does not match the expected account", 409) + raise RequestError( + "Xiaohongshu identity does not match the expected account", 409 + ) return identity def douyin_action(self, alias: str, input: dict) -> dict: @@ -1210,7 +1273,7 @@ class GatewayHandler(BaseHTTPRequestHandler): gateway.restore_proxy(alias, body) return None match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)", + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity|resolve)", path, ) if match: @@ -1221,6 +1284,8 @@ class GatewayHandler(BaseHTTPRequestHandler): return gateway.post_xiaohongshu(alias, body) if action == "media" and method == "POST": return gateway.get_xiaohongshu_media(alias, body) + if action == "resolve" and method == "POST": + return gateway.resolve_xiaohongshu(alias, body) if action == "identity" and method == "POST": return gateway.xiaohongshu_identity(alias, body) match = re.fullmatch( @@ -1516,7 +1581,9 @@ def valid_xiaohongshu_generation(value: dict) -> bool: return valid_douyin_generation(value) -def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool: +def valid_xhs_query( + query: object, allowed: set[str], required: set[str] | None = None +) -> bool: if not isinstance(query, dict) or not isinstance(allowed, set): return False required = required or set() @@ -1525,7 +1592,12 @@ def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None for key, values in query.items(): if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1: return False - if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]: + if ( + not isinstance(values[0], str) + or len(values[0]) > 2048 + or "\r" in values[0] + or "\n" in values[0] + ): return False return True @@ -1549,18 +1621,45 @@ def valid_xhs_url(raw: object) -> bool: query = parse_qs(parsed.query, keep_blank_values=True) except ValueError: return False - if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH: + if ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_IDENTITY_PATH + ): return not query - if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH: + if ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_USER_POSTED_PATH + ): + return ( + valid_xhs_query( + query, + { + "user_id", + "cursor", + "num", + "image_formats", + "xsec_source", + "xsec_token", + }, + {"user_id", "num"}, + ) + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) + and query["num"] == ["30"] + ) + if ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_COMMENTS_PATH + ): return valid_xhs_query( query, - {"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"}, - {"user_id", "num"}, - ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"] - if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH: - return valid_xhs_query( - query, - {"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"}, + { + "note_id", + "cursor", + "top_comment_id", + "image_formats", + "xsec_source", + "xsec_token", + }, {"note_id", "cursor", "top_comment_id"}, ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0])) return False @@ -1570,6 +1669,34 @@ def valid_xiaohongshu_url(raw: object) -> bool: return valid_xhs_url(raw) +def valid_xiaohongshu_page_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + if not _valid_xhs_host(parsed, "www.xiaohongshu.com") or not valid_xhs_query( + query, {"xsec_source", "xsec_token"} + ): + return False + parts = parsed.path.strip("/").split("/") + return ( + len(parts) == 2 + and parts[0] == "explore" + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) + ) or ( + len(parts) == 3 + and parts[:2] == ["user", "profile"] + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[2])) + ) + + +def valid_xiaohongshu_source_url(raw: object) -> bool: + return valid_xiaohongshu_page_url(raw) or is_xiaohongshu_share_url(raw) + + def valid_xhs_post_url(raw: object) -> bool: if not isinstance(raw, str): return False @@ -1600,7 +1727,12 @@ def valid_xiaohongshu_media_url(raw: object) -> bool: if not _valid_xhs_host(parsed, "www.xiaohongshu.com"): return False parts = parsed.path.strip("/").split("/") - return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"}) + return ( + len(parts) == 2 + and parts[0] == "explore" + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) + and valid_xhs_query(query, {"xsec_source", "xsec_token"}) + ) def valid_douyin_url(raw: object) -> bool: diff --git a/cmd/docker_gateway/test_xiaohongshu.py b/cmd/docker_gateway/test_xiaohongshu.py index 274bd16..e01a20b 100644 --- a/cmd/docker_gateway/test_xiaohongshu.py +++ b/cmd/docker_gateway/test_xiaohongshu.py @@ -13,31 +13,70 @@ valid_xhs_url = gateway_module.valid_xhs_url valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation +valid_xiaohongshu_page_url = gateway_module.valid_xiaohongshu_page_url +valid_xiaohongshu_source_url = gateway_module.valid_xiaohongshu_source_url class XiaohongshuValidationTests(unittest.TestCase): def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None: - self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")) - self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")) + self.assertTrue( + valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me") + ) + self.assertTrue( + valid_xiaohongshu_url( + "https://edith.xiaohongshu.com/api/sns/web/v2/user/me" + ) + ) + self.assertTrue( + valid_xiaohongshu_page_url("https://www.xiaohongshu.com/user/profile/u-1") + ) + self.assertTrue(valid_xiaohongshu_source_url("https://xhslink.com/a/abc")) self.assertTrue( valid_xhs_url( "https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user" ) ) - self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10")) - self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me")) - self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes")) - self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed")) - self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search")) - self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment")) + self.assertFalse( + valid_xhs_url( + "https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10" + ) + ) + self.assertFalse( + valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me") + ) + self.assertTrue( + valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes") + ) + self.assertTrue( + valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed") + ) + self.assertTrue( + valid_xiaohongshu_media_url( + "https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search" + ) + ) + self.assertFalse( + valid_xiaohongshu_media_url( + "https://www.xiaohongshu.com/explore/n-1#fragment" + ) + ) def test_generation_shape_matches_existing_browser_fence(self) -> None: self.assertTrue( valid_xiaohongshu_generation( - {"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""} + { + "binding_version": 1, + "runtime_id": "a" * 64, + "network_id": "network", + "network_exit_id": "", + } + ) + ) + self.assertFalse( + valid_xiaohongshu_generation( + {"binding_version": 1, "runtime_id": "runtime", "network_id": "network"} ) ) - self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"})) class XiaohongshuRouteTests(unittest.TestCase): @@ -48,18 +87,42 @@ class XiaohongshuRouteTests(unittest.TestCase): gateway.post_xiaohongshu.return_value = {"status": 200} gateway.get_xiaohongshu_media.return_value = {"status": 200} gateway.xiaohongshu_identity.return_value = {"uid": "u-1"} + gateway.resolve_xiaohongshu.return_value = { + "url": "https://www.xiaohongshu.com/explore/n-1" + } server = Mock() server.gateway = gateway cast(Any, handler).server = server cast(Any, handler).server_as_gateway = lambda: server - self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200}) - self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200}) - self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200}) - self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"}) + self.assertEqual( + handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), + {"status": 200}, + ) + self.assertEqual( + handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), + {"status": 200}, + ) + self.assertEqual( + handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), + {"status": 200}, + ) + self.assertEqual( + handler._route( + "POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {} + ), + {"uid": "u-1"}, + ) + self.assertEqual( + handler._route( + "POST", "/v1/browsers/account-a/xiaohongshu/resolve", {}, {} + ), + {"url": "https://www.xiaohongshu.com/explore/n-1"}, + ) with self.assertRaises(RequestError): handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {}) gateway.get_xiaohongshu.assert_called_once_with("account-a", {}) gateway.post_xiaohongshu.assert_called_once_with("account-a", {}) gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {}) gateway.xiaohongshu_identity.assert_called_once_with("account-a", {}) + gateway.resolve_xiaohongshu.assert_called_once_with("account-a", {}) diff --git a/cmd/docker_gateway/xiaohongshu.py b/cmd/docker_gateway/xiaohongshu.py index 4423abf..3268120 100644 --- a/cmd/docker_gateway/xiaohongshu.py +++ b/cmd/docker_gateway/xiaohongshu.py @@ -12,6 +12,7 @@ from .douyin import ( XHS_SEARCH_ORIGIN, XiaohongshuBrowser, is_xiaohongshu_media_url, + is_xiaohongshu_share_url, is_xiaohongshu_url, ) @@ -23,5 +24,6 @@ __all__ = [ "XHS_SEARCH_ORIGIN", "XiaohongshuBrowser", "is_xiaohongshu_media_url", + "is_xiaohongshu_share_url", "is_xiaohongshu_url", ] diff --git a/docs/research/xhs-all-in-one.md b/docs/research/xhs-all-in-one.md index 279f1a0..855c76d 100644 --- a/docs/research/xhs-all-in-one.md +++ b/docs/research/xhs-all-in-one.md @@ -22,7 +22,8 @@ ## 1.1 当前实现边界 -- CreatorHub 当前已接入小红书只读 collector、详情/搜索/作品/一级评论读取、受限 gateway 路由和控制面平台分派；写操作与事件监听仍未接入。 +- CreatorHub 当前已接入小红书只读 collector、详情/搜索/作品/一级评论读取、分享链接解析、原始 payload 保存、受限 gateway 路由和控制面平台分派；写操作与事件监听仍未接入。 +- 小红书竞品主页输入会校验稳定账号标识并保留主页中的 `xsec_token/xsec_source`；作品详情可通过受限浏览器解析官方分享短链。 - `internal/creator/collection.go` 的分页、窗口、checkpoint 和 lease 模型可作为后续适配的复用边界，但不能证明小红书平台能力。 **未完成的真实能力验收：**私有接口签名是否能由浏览器当前会话完成、真实 UID/作品/评论分页、媒体下载、写操作和事件监听仍需真实小红书环境分别验证。HTTP 200、离线 fixture 和本地单元测试不能替代这些证据。 diff --git a/internal/creator/content.go b/internal/creator/content.go index 430c4b1..8c573c2 100644 --- a/internal/creator/content.go +++ b/internal/creator/content.go @@ -12,6 +12,8 @@ import ( "unicode/utf8" ) +const maxRawPayloadBytes = 4 << 20 + func validateHomepage(value string) error { parsed, err := url.Parse(strings.TrimSpace(value)) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil { @@ -218,7 +220,8 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) (input.SourceType != SourceOwned && input.SourceType != SourceCompetitor) || utf8.RuneCountInString(input.WorkKey) > 255 || utf8.RuneCountInString(input.Title) > 1000 || utf8.RuneCountInString(input.Body) > 100000 || utf8.RuneCountInString(input.OriginalURL) > 2000 || - utf8.RuneCountInString(input.CoverURL) > 2000 { + utf8.RuneCountInString(input.CoverURL) > 2000 || len(input.RawPayload) > maxRawPayloadBytes || + input.RawPayload != "" && !json.Valid([]byte(input.RawPayload)) { return Work{}, false, ErrInvalid } if err := s.validateWorkSource(ctx, input.Platform, input.SourceType, input.SourceID); err != nil { @@ -257,8 +260,8 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) var inserted bool err = tx.QueryRowContext(ctx, ` INSERT INTO creator_work (id, platform, work_key, source_type, source_id, author_name, title, body, - published_at, published_at_status, original_url, cover_url, likes, comments_count, shares) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (platform, work_key) DO UPDATE SET author_name = CASE WHEN EXCLUDED.author_name = '' THEN creator_work.author_name ELSE EXCLUDED.author_name END, title = CASE WHEN EXCLUDED.title = '' THEN creator_work.title ELSE EXCLUDED.title END, @@ -267,12 +270,13 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) published_at_status = CASE WHEN EXCLUDED.published_at IS NULL THEN creator_work.published_at_status ELSE EXCLUDED.published_at_status END, original_url = CASE WHEN EXCLUDED.original_url = '' THEN creator_work.original_url ELSE EXCLUDED.original_url END, cover_url = CASE WHEN EXCLUDED.cover_url = '' THEN creator_work.cover_url ELSE EXCLUDED.cover_url END, + raw_payload = COALESCE(EXCLUDED.raw_payload, creator_work.raw_payload), likes = COALESCE(EXCLUDED.likes, creator_work.likes), comments_count = COALESCE(EXCLUDED.comments_count, creator_work.comments_count), shares = COALESCE(EXCLUDED.shares, creator_work.shares), updated_at = now() RETURNING id, (xmax = 0)`, id, input.Platform, input.WorkKey, input.SourceType, input.SourceID, input.AuthorName, input.Title, input.Body, input.PublishedAt, status, input.OriginalURL, input.CoverURL, - input.Likes, input.CommentsCount, input.Shares).Scan(&returnedID, &inserted) + nullableRawPayload(input.RawPayload), input.Likes, input.CommentsCount, input.Shares).Scan(&returnedID, &inserted) if err != nil { return Work{}, false, databaseError(err) } @@ -297,24 +301,35 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) func ptrTime(value time.Time) *time.Time { return &value } +func nullableRawPayload(value string) any { + if value == "" { + return nil + } + return value +} + func scanWork(scanner interface{ Scan(...any) error }) (Work, error) { var result Work var publishedAt, latestAt, nextAt sql.NullTime var likes, commentsCount, shares sql.NullInt64 + var rawPayload sql.NullString if err := scanner.Scan(&result.ID, &result.Platform, &result.WorkKey, &result.SourceType, &result.SourceID, &result.AuthorName, &result.Title, &result.Body, &publishedAt, &result.PublishedAtStatus, - &result.OriginalURL, &result.CoverURL, &likes, &commentsCount, &shares, &latestAt, &nextAt, + &result.OriginalURL, &result.CoverURL, &rawPayload, &likes, &commentsCount, &shares, &latestAt, &nextAt, &result.MetricStopReason, &result.CreatedAt, &result.UpdatedAt); err != nil { return Work{}, err } result.PublishedAt = nullableTime(publishedAt) + if rawPayload.Valid { + result.RawPayload = rawPayload.String + } result.Likes, result.CommentsCount, result.Shares = nullableInt64(likes), nullableInt64(commentsCount), nullableInt64(shares) result.LatestMetricsAt, result.NextMetricAt = nullableTime(latestAt), nullableTime(nextAt) return result, nil } const workSelect = `SELECT id, platform, work_key, source_type, source_id, author_name, title, body, - published_at, published_at_status, original_url, cover_url, likes, comments_count, shares, + published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares, latest_metrics_at, next_metric_at, metric_stop_reason, created_at, updated_at FROM creator_work` func (s *Store) loadWorkSources(ctx context.Context, work *Work) error { @@ -629,7 +644,7 @@ func (s *Store) SaveRewrite(ctx context.Context, workID, title, script string) ( func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, bool, error) { input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.CommentType = strings.TrimSpace(input.Platform), strings.TrimSpace(input.CommentKey), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.AuthorUID), strings.TrimSpace(input.AuthorName), strings.TrimSpace(input.CommentType) - if !ValidatePlatform(input.Platform) || input.CommentKey == "" || input.WorkID == "" || strings.TrimSpace(input.Content) == "" || (input.CommentType != "top_level" && input.CommentType != "unknown") || utf8.RuneCountInString(input.Content) > 10000 { + if !ValidatePlatform(input.Platform) || input.CommentKey == "" || input.WorkID == "" || strings.TrimSpace(input.Content) == "" || (input.CommentType != "top_level" && input.CommentType != "unknown") || utf8.RuneCountInString(input.Content) > 10000 || len(input.RawPayload) > maxRawPayloadBytes || input.RawPayload != "" && !json.Valid([]byte(input.RawPayload)) { return Comment{}, false, ErrInvalid } work, err := s.GetWork(ctx, input.WorkID) @@ -642,7 +657,7 @@ func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, b id := newID("comment") var returnedID string var inserted bool - err = s.db.QueryRowContext(ctx, `INSERT INTO creator_comment (id, platform, comment_key, work_id, author_uid, author_name, content, published_at, comment_type) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (platform, comment_key) DO UPDATE SET work_id = EXCLUDED.work_id, author_uid = EXCLUDED.author_uid, author_name = EXCLUDED.author_name, content = EXCLUDED.content, published_at = EXCLUDED.published_at, comment_type = EXCLUDED.comment_type RETURNING id, (xmax = 0)`, id, input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.Content, input.PublishedAt, input.CommentType).Scan(&returnedID, &inserted) + err = s.db.QueryRowContext(ctx, `INSERT INTO creator_comment (id, platform, comment_key, work_id, author_uid, author_name, content, raw_payload, published_at, comment_type) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (platform, comment_key) DO UPDATE SET work_id = EXCLUDED.work_id, author_uid = EXCLUDED.author_uid, author_name = EXCLUDED.author_name, content = EXCLUDED.content, raw_payload = COALESCE(EXCLUDED.raw_payload, creator_comment.raw_payload), published_at = EXCLUDED.published_at, comment_type = EXCLUDED.comment_type RETURNING id, (xmax = 0)`, id, input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.Content, nullableRawPayload(input.RawPayload), input.PublishedAt, input.CommentType).Scan(&returnedID, &inserted) if err != nil { return Comment{}, false, databaseError(err) } @@ -653,15 +668,19 @@ func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, b func scanComment(scanner interface{ Scan(...any) error }) (Comment, error) { var result Comment var publishedAt sql.NullTime - if err := scanner.Scan(&result.ID, &result.Platform, &result.CommentKey, &result.WorkID, &result.AuthorUID, &result.AuthorName, &result.Content, &publishedAt, &result.CollectedAt, &result.CommentType); err != nil { + var rawPayload sql.NullString + if err := scanner.Scan(&result.ID, &result.Platform, &result.CommentKey, &result.WorkID, &result.AuthorUID, &result.AuthorName, &result.Content, &rawPayload, &publishedAt, &result.CollectedAt, &result.CommentType); err != nil { return Comment{}, err } result.PublishedAt = nullableTime(publishedAt) + if rawPayload.Valid { + result.RawPayload = rawPayload.String + } result.CollectedAt = result.CollectedAt.UTC() return result, nil } -const commentSelect = `SELECT id, platform, comment_key, work_id, author_uid, author_name, content, published_at, collected_at, comment_type FROM creator_comment` +const commentSelect = `SELECT id, platform, comment_key, work_id, author_uid, author_name, content, raw_payload, published_at, collected_at, comment_type FROM creator_comment` func (s *Store) GetComment(ctx context.Context, id string) (Comment, error) { result, err := scanComment(s.db.QueryRowContext(ctx, commentSelect+` WHERE id = $1`, id)) diff --git a/internal/creator/migrations/031_xhs_raw_payloads.sql b/internal/creator/migrations/031_xhs_raw_payloads.sql new file mode 100644 index 0000000..40ead21 --- /dev/null +++ b/internal/creator/migrations/031_xhs_raw_payloads.sql @@ -0,0 +1,2 @@ +ALTER TABLE creator_work ADD COLUMN IF NOT EXISTS raw_payload text; +ALTER TABLE creator_comment ADD COLUMN IF NOT EXISTS raw_payload text; diff --git a/internal/creator/models.go b/internal/creator/models.go index 7421f5b..da232c0 100644 --- a/internal/creator/models.go +++ b/internal/creator/models.go @@ -156,6 +156,7 @@ type Work struct { MetricStopReason string `json:"metric_stop_reason,omitempty"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` + RawPayload string `json:"-"` } type WorkInput struct { @@ -173,6 +174,7 @@ type WorkInput struct { Likes *int64 `json:"likes"` CommentsCount *int64 `json:"comments_count"` Shares *int64 `json:"shares"` + RawPayload string `json:"-"` } type WorkFilter struct { @@ -232,6 +234,7 @@ type Comment struct { PublishedAt *time.Time `json:"published_at,omitempty"` CollectedAt time.Time `json:"collected_at"` CommentType string `json:"comment_type"` + RawPayload string `json:"-"` } type CommentInput struct { @@ -243,6 +246,7 @@ type CommentInput struct { Content string `json:"content"` PublishedAt *time.Time `json:"published_at"` CommentType string `json:"comment_type"` + RawPayload string `json:"-"` } type LeadRule struct { diff --git a/internal/creator/store.go b/internal/creator/store.go index 947a639..5938db5 100644 --- a/internal/creator/store.go +++ b/internal/creator/store.go @@ -58,6 +58,9 @@ var migration029 string //go:embed migrations/030_creator_event_gateway_time.sql var migration030 string +//go:embed migrations/031_xhs_raw_payloads.sql +var migration031 string + type SecretReference struct { ID string Provider string @@ -135,6 +138,7 @@ func (s *Store) migrate(ctx context.Context) error { {version: 28, sql: migration028}, {version: 29, sql: migration029}, {version: 30, sql: migration030}, + {version: 31, sql: migration031}, } for _, migration := range migrations { var applied bool diff --git a/internal/xiaohongshu/collector.go b/internal/xiaohongshu/collector.go index a2f47df..0dffa42 100644 --- a/internal/xiaohongshu/collector.go +++ b/internal/xiaohongshu/collector.go @@ -46,6 +46,27 @@ type Identity struct { Nickname string } +type LinkKind string + +const ( + LinkHomepage LinkKind = "homepage" + LinkNote LinkKind = "note" + LinkShare LinkKind = "share" +) + +type LinkContext struct { + Kind LinkKind + AccountKey string + WorkKey string + Token string + Source string + CanonicalURL string +} + +type ShareResolver interface { + Resolve(context.Context, string) (string, error) +} + type Collector struct { Browser Browser AccountKey string @@ -84,6 +105,98 @@ func (c *Collector) Identity(ctx context.Context, expectedKey string) (Identity, return identity, nil } +func ParseSourceURL(raw string) (LinkContext, error) { + raw = strings.TrimSpace(raw) + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "https" || parsed.User != nil || parsed.Fragment != "" { + return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu source URL", creator.ErrInvalid) + } + if parsed.Port() != "" { + return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu source URL", creator.ErrInvalid) + } + query, err := parseAccessQuery(parsed) + if err != nil { + return LinkContext{}, err + } + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + context := LinkContext{Token: query.Token, Source: query.Source, CanonicalURL: parsed.String()} + switch parsed.Hostname() { + case "www.xiaohongshu.com": + switch { + case len(parts) == 2 && parts[0] == "explore" && keyPattern.MatchString(parts[1]): + context.Kind, context.WorkKey = LinkNote, parts[1] + case len(parts) == 3 && parts[0] == "user" && parts[1] == "profile" && keyPattern.MatchString(parts[2]): + context.Kind, context.AccountKey = LinkHomepage, parts[2] + default: + return LinkContext{}, fmt.Errorf("%w: unsupported xiaohongshu source path", creator.ErrInvalid) + } + case "xhslink.com", "www.xhslink.com": + if len(parts) < 1 || len(parts) > 4 || parsed.RawQuery != "" { + return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu share URL", creator.ErrInvalid) + } + context.Kind = LinkShare + default: + return LinkContext{}, fmt.Errorf("%w: unsupported xiaohongshu source host", creator.ErrInvalid) + } + return context, nil +} + +type accessQuery struct { + Token string + Source string +} + +func parseAccessQuery(parsed *url.URL) (accessQuery, error) { + values, err := url.ParseQuery(parsed.RawQuery) + if err != nil { + return accessQuery{}, fmt.Errorf("%w: invalid xiaohongshu access query", creator.ErrInvalid) + } + for key, items := range values { + if key != "xsec_token" && key != "xsec_source" || len(items) != 1 || strings.ContainsAny(items[0], "\r\n") || len(items[0]) > 2048 { + return accessQuery{}, fmt.Errorf("%w: unsupported xiaohongshu access query", creator.ErrInvalid) + } + } + result := accessQuery{Token: values.Get("xsec_token"), Source: values.Get("xsec_source")} + if result.Source != "" && !keyPattern.MatchString(result.Source) { + return accessQuery{}, fmt.Errorf("%w: invalid xiaohongshu access source", creator.ErrInvalid) + } + return result, nil +} + +func ValidateSourceURL(raw, expectedAccountKey string) error { + link, err := ParseSourceURL(raw) + if err != nil { + return err + } + if link.Kind != LinkHomepage || link.AccountKey != expectedAccountKey { + return fmt.Errorf("%w: xiaohongshu homepage does not match the account", creator.ErrConflict) + } + return nil +} + +func (c *Collector) resolveSourceURL(ctx context.Context, raw string) (LinkContext, error) { + link, err := ParseSourceURL(raw) + if err != nil { + return LinkContext{}, err + } + if link.Kind != LinkShare { + return link, nil + } + resolver, ok := c.Browser.(ShareResolver) + if !ok { + return LinkContext{}, fmt.Errorf("%w: xiaohongshu share URL resolver is unavailable", creator.ErrUnavailable) + } + canonical, err := resolver.Resolve(ctx, link.CanonicalURL) + if err != nil { + return LinkContext{}, err + } + resolved, err := ParseSourceURL(canonical) + if err != nil || resolved.Kind != LinkNote { + return LinkContext{}, fmt.Errorf("%w: xiaohongshu share URL did not resolve to a note", creator.ErrInvalid) + } + return resolved, nil +} + func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) { if c == nil { return creator.WorkPage{}, fmt.Errorf("%w: xiaohongshu collector is nil", creator.ErrUnavailable) @@ -94,14 +207,21 @@ func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (c if c.Browser == nil || !keyPattern.MatchString(accountKey) || !validCursor(cursor) { return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu work collection request", creator.ErrInvalid) } - access := c.defaultContext() + access, err := c.defaultContext() + if err != nil { + return creator.WorkPage{}, err + } query := url.Values{ "user_id": {accountKey}, "cursor": {cursor}, "num": {"30"}, "image_formats": {"jpg,webp,avif"}, - "xsec_source": {access.Source}, - "xsec_token": {access.Token}, + } + if access.Source != "" { + query.Set("xsec_source", access.Source) + } + if access.Token != "" { + query.Set("xsec_token", access.Token) } response, err := c.Browser.Get(ctx, APIOrigin+UserPostedPath+"?"+query.Encode()) if err != nil { @@ -121,14 +241,21 @@ func (c *Collector) ListTopLevelComments(ctx context.Context, workKey, cursor st if c == nil || c.Browser == nil || !keyPattern.MatchString(workKey) || !validCursor(cursor) { return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comment collection request", creator.ErrInvalid) } - access := c.contextFor(workKey) + access, err := c.contextFor(workKey) + if err != nil { + return creator.CommentPage{}, err + } query := url.Values{ "note_id": {workKey}, "cursor": {cursor}, "top_comment_id": {""}, "image_formats": {"jpg,webp,avif"}, - "xsec_source": {access.Source}, - "xsec_token": {access.Token}, + } + if access.Source != "" { + query.Set("xsec_source", access.Source) + } + if access.Token != "" { + query.Set("xsec_token", access.Token) } response, err := c.Browser.Get(ctx, APIOrigin+CommentsPath+"?"+query.Encode()) if err != nil { @@ -178,7 +305,11 @@ func (c *Collector) SearchNotes(ctx context.Context, queryText string, page int) if err := responseError(response, "search"); err != nil { return creator.WorkPage{}, err } - items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, c.defaultContext()) + fallback, err := c.defaultContext() + if err != nil { + return creator.WorkPage{}, err + } + items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, fallback) if !ok { return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search response", creator.ErrInvalid) } @@ -193,51 +324,47 @@ func randomID() (string, error) { return hex.EncodeToString(value), nil } -func (c *Collector) defaultContext() accessContext { - access := accessContext{Source: "pc_user"} - if c != nil && strings.TrimSpace(c.HomepageURL) != "" { - parsed, err := url.Parse(c.HomepageURL) - if err == nil { - if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" { - access.Token = token - } - if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" { - access.Source = source - } - } +func (c *Collector) defaultContext() (accessContext, error) { + if c == nil || strings.TrimSpace(c.HomepageURL) == "" { + return accessContext{}, nil } - return access + link, err := ParseSourceURL(c.HomepageURL) + if err != nil { + return accessContext{}, err + } + if link.Kind != LinkHomepage { + return accessContext{}, fmt.Errorf("%w: xiaohongshu homepage URL is required", creator.ErrInvalid) + } + return accessContext{Token: link.Token, Source: link.Source}, nil } func (c *Collector) SetWorkContext(workKey, originalURL string) error { if c == nil || !keyPattern.MatchString(workKey) { return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid) } - access := c.defaultContext() + access, err := c.defaultContext() + if err != nil { + return err + } if strings.TrimSpace(originalURL) != "" { - parsed, err := url.Parse(originalURL) - if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" || !strings.Contains(parsed.Path, workKey) { + link, parseErr := ParseSourceURL(originalURL) + if parseErr != nil || link.Kind != LinkNote || link.WorkKey != workKey { return fmt.Errorf("%w: invalid xiaohongshu work URL", creator.ErrInvalid) } - if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" { - access.Token = token - } - if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" { - access.Source = source - } + access.Token, access.Source = link.Token, link.Source } - if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) { + if len(access.Token) > 2048 || access.Source != "" && !keyPattern.MatchString(access.Source) { return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid) } c.contexts.Store(workKey, access) return nil } -func (c *Collector) contextFor(workKey string) accessContext { +func (c *Collector) contextFor(workKey string) (accessContext, error) { if c != nil { if value, ok := c.contexts.Load(workKey); ok { if access, ok := value.(accessContext); ok { - return access + return access, nil } } } @@ -339,6 +466,7 @@ func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput, if !ok { return creator.WorkInput{}, accessContext{}, false } + outer := object if nested := firstObject(object, "note_card", "noteCard"); nested != nil { object = nested } @@ -376,17 +504,32 @@ func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput, return creator.WorkInput{}, accessContext{}, false } access := fallback + if token := firstString(outer, "xsec_token"); token != "" { + access.Token = token + } + if source := firstString(outer, "xsec_source"); source != "" { + access.Source = source + } if token := firstString(object, "xsec_token"); token != "" { access.Token = token } if source := firstString(object, "xsec_source"); source != "" { access.Source = source } - if utf8.RuneCountInString(access.Token) > 2048 || !keyPattern.MatchString(access.Source) { + if utf8.RuneCountInString(access.Token) > 2048 || access.Source != "" && !keyPattern.MatchString(access.Source) { return creator.WorkInput{}, accessContext{}, false } originalURL := firstString(object, "original_url", "note_url", "url") - if !validOriginalURL(originalURL, id) { + if originalURL == "" { + originalURL = firstString(outer, "original_url", "note_url", "url") + } + if originalURL != "" { + link, err := ParseSourceURL(originalURL) + if err != nil || link.Kind != LinkNote || link.WorkKey != id { + return creator.WorkInput{}, accessContext{}, false + } + originalURL = link.CanonicalURL + } else { originalURL = noteURL(id, access) } cover := coverURL(object) @@ -396,6 +539,7 @@ func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput, } return creator.WorkInput{ Platform: creator.PlatformXiaohongshu, + RawPayload: string(raw), WorkKey: id, AuthorName: authorName, Title: title, @@ -457,6 +601,7 @@ func parseCommentsPage(body []byte, workKey string) ([]creator.CommentInput, str } items = append(items, creator.CommentInput{ Platform: creator.PlatformXiaohongshu, + RawPayload: string(raw), CommentKey: id, WorkID: workKey, AuthorUID: firstString(user, "user_id", "uid", "id"), @@ -478,7 +623,7 @@ func responseError(response Response, resource string) error { return fmt.Errorf("%w: xiaohongshu %s challenge %s", creator.ErrUnavailable, resource, response.Challenge) } if response.Status >= 200 && response.Status < 300 { - return nil + return platformFailure(response.Body, resource) } switch response.Status { case 401, 403, 406, 461: @@ -490,6 +635,29 @@ func responseError(response Response, resource string) error { } } +func platformFailure(body []byte, resource string) error { + var envelope struct { + Success *bool `json:"success"` + Code string `json:"code"` + Msg string `json:"msg"` + Message string `json:"message"` + } + if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || *envelope.Success { + return nil + } + reason := strings.TrimSpace(envelope.Msg) + if reason == "" { + reason = strings.TrimSpace(envelope.Message) + } + if len(reason) > 512 { + reason = reason[:512] + } + if strings.Contains(strings.ToLower(reason), "rate") || strings.Contains(reason, "频繁") || strings.Contains(reason, "验证") { + return fmt.Errorf("%w: xiaohongshu %s rejected request code=%s message=%s", creator.ErrUnavailable, resource, envelope.Code, reason) + } + return fmt.Errorf("%w: xiaohongshu %s rejected request code=%s message=%s", creator.ErrConflict, resource, envelope.Code, reason) +} + func validCursor(cursor string) bool { return cursor == "" || len(cursor) <= 512 && !strings.ContainsAny(cursor, "\r\n") } @@ -582,7 +750,7 @@ func optionalTimestamp(object map[string]json.RawMessage, names ...string) (*tim } parsed, err := strconv.ParseInt(number.String(), 10, 64) if err != nil || parsed <= 0 { - return nil, true + return nil, false } if parsed > 1_000_000_000_000 { parsed /= 1000 @@ -633,14 +801,6 @@ func coverURL(object map[string]json.RawMessage) string { return firstString(object, "cover_url", "cover") } -func validOriginalURL(raw, id string) bool { - if raw == "" { - return false - } - parsed, err := url.Parse(raw) - return err == nil && parsed.Scheme == "https" && (parsed.Host == "www.xiaohongshu.com" || parsed.Host == "xhslink.com") && strings.Contains(parsed.Path, id) -} - func noteURL(id string, access accessContext) string { query := url.Values{} if access.Token != "" { diff --git a/internal/xiaohongshu/collector_test.go b/internal/xiaohongshu/collector_test.go index 9d02f8e..d2b42b4 100644 --- a/internal/xiaohongshu/collector_test.go +++ b/internal/xiaohongshu/collector_test.go @@ -10,12 +10,14 @@ import ( ) type fakeBrowser struct { - getURL string - postURL string - getBody []byte - postBody []byte - getResp Response - postResp Response + getURL string + postURL string + getBody []byte + postBody []byte + getResp Response + postResp Response + resolvedURL string + resolveInput string } func (f *fakeBrowser) Get(_ context.Context, target string) (Response, error) { @@ -26,6 +28,10 @@ func (f *fakeBrowser) Post(_ context.Context, target string, body []byte) (Respo f.postURL, f.postBody = target, body return f.postResp, nil } +func (f *fakeBrowser) Resolve(_ context.Context, target string) (string, error) { + f.resolveInput = target + return f.resolvedURL, nil +} func TestIdentityRequiresMatchingUser(t *testing.T) { browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"user_id":"u-1","nickname":"作者"}}`)}} @@ -43,7 +49,7 @@ func TestListWorksCapturesContextForComments(t *testing.T) { browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"next","has_more":true,"notes":[{"note_id":"n-1","title":"标题","desc":"正文","time":1710000000,"user":{"user_id":"u-1","nickname":"作者"},"interact_info":{"liked_count":"2","comment_count":3,"shared_count":4},"xsec_token":"token"}]}}`)}} collector := &Collector{Browser: browser, AccountKey: "u-1", SourceType: creator.SourceOwned, SourceID: "source-1"} page, err := collector.ListWorks(context.Background(), "ignored", "") - if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || !page.HasMore { + if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || !page.HasMore || page.Items[0].RawPayload == "" { t.Fatalf("page = %#v, err = %v", page, err) } parsed, err := url.Parse(browser.getURL) @@ -52,7 +58,7 @@ func TestListWorksCapturesContextForComments(t *testing.T) { } browser.getResp = Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"comments":[{"id":"c-1","content":"评论","create_time":1710000000,"user_info":{"user_id":"u-2","nickname":"读者"}}]}}`)} comments, err := collector.ListTopLevelComments(context.Background(), "n-1", "") - if err != nil || len(comments.Items) != 1 || comments.Items[0].CommentType != "top_level" { + if err != nil || len(comments.Items) != 1 || comments.Items[0].CommentType != "top_level" || comments.Items[0].RawPayload == "" { t.Fatalf("comments = %#v, err = %v", comments, err) } commentURL, err := url.Parse(browser.getURL) @@ -74,6 +80,44 @@ func TestSearchUsesBoundedPostAndRejectsMalformedPage(t *testing.T) { } } +func TestHomepageContextIsUsedWithoutInventingDefaults(t *testing.T) { + browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"notes":[]}}`)}} + collector := &Collector{Browser: browser, AccountKey: "u-1", HomepageURL: "https://www.xiaohongshu.com/user/profile/u-1?xsec_token=home-token&xsec_source=pc_search"} + if _, err := collector.ListWorks(context.Background(), "ignored", ""); err != nil { + t.Fatalf("list works: %v", err) + } + requestURL, err := url.Parse(browser.getURL) + if err != nil || requestURL.Query().Get("xsec_token") != "home-token" || requestURL.Query().Get("xsec_source") != "pc_search" { + t.Fatalf("homepage context was not forwarded: %s", browser.getURL) + } +} + +func TestSourceURLParsingAndShareResolution(t *testing.T) { + homepage, err := ParseSourceURL("https://www.xiaohongshu.com/user/profile/u-1?xsec_source=pc_search") + if err != nil || homepage.Kind != LinkHomepage || homepage.AccountKey != "u-1" { + t.Fatalf("homepage = %#v, err = %v", homepage, err) + } + share, err := ParseSourceURL("https://xhslink.com/a/abc") + if err != nil || share.Kind != LinkShare { + t.Fatalf("share = %#v, err = %v", share, err) + } + browser := &fakeBrowser{resolvedURL: "https://www.xiaohongshu.com/explore/n-1?xsec_token=tok-1&xsec_source=pc_search", postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"items":[{"note_id":"n-1","title":"详情","time":1710000000,"interact_info":{}}]}}`)}} + collector := &Collector{Browser: browser} + item, err := collector.GetNoteDetail(context.Background(), "https://xhslink.com/a/abc") + if err != nil || item.WorkKey != "n-1" || browser.resolveInput != "https://xhslink.com/a/abc" { + t.Fatalf("resolved detail = %#v, err = %v, input = %s", item, err, browser.resolveInput) + } +} + +func TestResponseErrorClassifiesRejectedPlatformPayload(t *testing.T) { + if err := responseError(Response{Status: 200, Body: []byte(`{"success":false,"code":"LOGIN_REQUIRED","msg":"请重新登录"}`)}, "works"); !errors.Is(err, creator.ErrConflict) { + t.Fatalf("expected conflict, got %v", err) + } + if err := responseError(Response{Status: 200, Body: []byte(`{"success":false,"code":"RATE_LIMIT","msg":"请求过于频繁"}`)}, "works"); !errors.Is(err, creator.ErrUnavailable) { + t.Fatalf("expected unavailable, got %v", err) + } +} + func TestSetWorkContextRejectsWrongHost(t *testing.T) { collector := &Collector{} if err := collector.SetWorkContext("n-1", "https://evil.example/explore/n-1"); !errors.Is(err, creator.ErrInvalid) { diff --git a/internal/xiaohongshu/detail.go b/internal/xiaohongshu/detail.go index b037922..65412b1 100644 --- a/internal/xiaohongshu/detail.go +++ b/internal/xiaohongshu/detail.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/url" - "strings" "git.ipao.vip/rogee/creator-hub/internal/creator" ) @@ -16,23 +14,16 @@ func (c *Collector) GetNoteDetail(ctx context.Context, originalURL string) (crea if c == nil || c.Browser == nil { return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail request", creator.ErrInvalid) } - parsed, err := url.Parse(originalURL) - if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" { - return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid) + link, err := c.resolveSourceURL(ctx, originalURL) + if err != nil { + return creator.WorkInput{}, err } - parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") - if len(parts) != 2 || parts[0] != "explore" || !keyPattern.MatchString(parts[1]) { - return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid) - } - access := accessContext{Token: parsed.Query().Get("xsec_token"), Source: parsed.Query().Get("xsec_source")} - if access.Source == "" { - access.Source = "pc_search" - } - if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) { - return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL context", creator.ErrInvalid) + if link.Kind != LinkNote { + return creator.WorkInput{}, fmt.Errorf("%w: xiaohongshu note URL is required", creator.ErrInvalid) } + access := accessContext{Token: link.Token, Source: link.Source} body, err := json.Marshal(map[string]any{ - "source_note_id": parts[1], + "source_note_id": link.WorkKey, "image_formats": []string{"jpg", "webp", "avif"}, "extra": map[string]string{"need_body_topic": "1"}, "xsec_source": access.Source, @@ -52,10 +43,10 @@ func (c *Collector) GetNoteDetail(ctx context.Context, originalURL string) (crea if !ok { return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail response", creator.ErrInvalid) } - if item.WorkKey != parts[1] { + if item.WorkKey != link.WorkKey { return creator.WorkInput{}, fmt.Errorf("%w: xiaohongshu detail returned another note", creator.ErrConflict) } - item.OriginalURL = originalURL + item.OriginalURL = link.CanonicalURL item.SourceType = c.SourceType item.SourceID = c.SourceID if item.SourceType == "" { diff --git a/web/src/CreatorCompetitorsPage.jsx b/web/src/CreatorCompetitorsPage.jsx index 1e24e27..5662a81 100644 --- a/web/src/CreatorCompetitorsPage.jsx +++ b/web/src/CreatorCompetitorsPage.jsx @@ -110,13 +110,19 @@ export function CreatorCompetitorsPage() { const parseHomepage = () => { try { const parsed = new URL(form.homepage_url); - const allowed = - form.platform === "douyin" - ? parsed.hostname === "www.douyin.com" - : parsed.hostname === "www.xiaohongshu.com" || - parsed.hostname === "xiaohongshu.com"; const parts = parsed.pathname.split("/").filter(Boolean); - const candidate = parts.at(-1) || ""; + const isXiaohongshu = form.platform === "xiaohongshu"; + const allowed = isXiaohongshu + ? parsed.protocol === "https:" && + parsed.hostname === "www.xiaohongshu.com" + : parsed.hostname === "www.douyin.com"; + let candidate = parts.at(-1) || ""; + if ( + isXiaohongshu && + !(parts.length === 3 && parts[0] === "user" && parts[1] === "profile") + ) { + candidate = ""; + } if ( !allowed || !candidate || diff --git a/web/src/CreatorPages.test.jsx b/web/src/CreatorPages.test.jsx index 67ecb42..7d8ac59 100644 --- a/web/src/CreatorPages.test.jsx +++ b/web/src/CreatorPages.test.jsx @@ -203,6 +203,36 @@ describe("creator pages", () => { ); }); + it("parses and confirms a Xiaohongshu profile URL", async () => { + const dataProvider = provider(); + renderPage(, dataProvider); + fireEvent.click(screen.getAllByRole("combobox")[0]); + fireEvent.click(screen.getByRole("option", { name: "小红书" })); + fireEvent.change(screen.getByLabelText("主页 URL", { exact: false }), { + target: { + value: "https://www.xiaohongshu.com/user/profile/xhs-b", + }, + }); + fireEvent.click(screen.getByRole("button", { name: "解析链接预览" })); + await waitFor(() => + expect( + screen.getAllByText("xhs-b", { exact: false }).length, + ).toBeGreaterThan(0), + ); + fireEvent.click(screen.getByRole("button", { name: "确认预览内容" })); + fireEvent.click(screen.getByRole("button", { name: "加入监测" })); + await waitFor(() => expect(dataProvider.create).toHaveBeenCalled()); + expect(dataProvider.create).toHaveBeenCalledWith( + expect.objectContaining({ + resource: "creator-competitors", + variables: expect.objectContaining({ + platform: "xiaohongshu", + platform_account_key: "xhs-b", + }), + }), + ); + }); + it("keeps account password out of the returned profile and exposes big-account action", async () => { const dataProvider = provider(); renderPage(, dataProvider);