rogee/creator-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: complete Xiaohongshu read-only flow

Browse Source
This commit is contained in:
rogee
2026-09-14 23:19:31 +08:00
parent 4c37d0c9bf
commit 5867aac20f
17 changed files with 857 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/control-plane/creator.go
+63 -3
View File
@@ -255,6 +255,9 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto
if err := decodeCreator(c, &input); err != nil {
return creatorError(c, err)
}
if err := validateXiaohongshuCompetitor(input); err != nil {
return creatorError(c, err)
}
item, err := store.CreateCompetitor(c.Context(), input)
if err != nil {
return creatorError(c, err)
@@ -296,6 +299,47 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto
return c.Status(fiber.StatusAccepted).JSON(report)
})
app.Post("/api/creator/xiaohongshu/search", func(c fiber.Ctx) error {
var input struct {
AccountID string `json:"account_id"`
Query string `json:"query"`
Page int `json:"page"`
}
if err := decodeCreator(c, &input); err != nil {
return creatorError(c, err)
}
if input.Page == 0 {
input.Page = 1
}
collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID)
if err != nil {
return creatorError(c, err)
}
page, err := collector.SearchNotes(c.Context(), input.Query, input.Page)
if err != nil {
return creatorError(c, err)
}
return c.JSON(page)
})
app.Post("/api/creator/xiaohongshu/detail", func(c fiber.Ctx) error {
var input struct {
AccountID string `json:"account_id"`
URL string `json:"url"`
}
if err := decodeCreator(c, &input); err != nil {
return creatorError(c, err)
}
collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID)
if err != nil {
return creatorError(c, err)
}
item, err := collector.GetNoteDetail(c.Context(), input.URL)
if err != nil {
return creatorError(c, err)
}
return c.JSON(item)
})
app.Get("/api/creator/works", func(c fiber.Ctx) error {
filter, err := workFilter(c)
if err != nil {
@@ -1128,6 +1172,11 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p
if competitor.Platform != creator.PlatformDouyin && competitor.Platform != creator.PlatformXiaohongshu {
return blocked(fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, competitor.Platform))
}
if competitor.Platform == creator.PlatformXiaohongshu {
if err := validateXiaohongshuSource(competitor.HomepageURL, competitor.PlatformAccountKey); err != nil {
return blocked(err)
}
}
account, err := phaseAStore.GetAccount(ctx, accountID)
if err != nil {
return blocked(err)
@@ -1153,7 +1202,7 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p
if err != nil {
return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err))
}
collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceCompetitor, competitor.ID)
collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, competitor.PlatformAccountKey, competitor.HomepageURL, creator.SourceCompetitor, competitor.ID)
if err != nil {
return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err))
}
@@ -1283,7 +1332,18 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS
if err != nil {
return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
}
collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, work.SourceType, work.SourceID)
targetAccountKey, homepageURL := account.PlatformAccountKey, ""
if work.SourceType == creator.SourceCompetitor {
competitor, competitorErr := store.GetCompetitor(ctx, work.SourceID)
if competitorErr != nil {
return competitorErr
}
if competitor.Platform != work.Platform {
return creator.ErrConflict
}
targetAccountKey, homepageURL = competitor.PlatformAccountKey, competitor.HomepageURL
}
collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, targetAccountKey, homepageURL, work.SourceType, work.SourceID)
if err != nil {
return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)
}
@@ -1356,7 +1416,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph
logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed")
}
}()
collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceOwned, account.ID)
collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, account.PlatformAccountKey, "", creator.SourceOwned, account.ID)
if err != nil {
blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays)
return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr)
cmd/control-plane/xiaohongshu.go
+83 -11
View File
@@ -14,6 +14,7 @@ import (
"git.ipao.vip/rogee/creator-hub/internal/creator"
"git.ipao.vip/rogee/creator-hub/internal/douyin"
"git.ipao.vip/rogee/creator-hub/internal/hub"
"git.ipao.vip/rogee/creator-hub/internal/phasea"
"git.ipao.vip/rogee/creator-hub/internal/xiaohongshu"
)
@@ -94,6 +95,26 @@ func (browser xiaohongshuGatewayBrowser) Identity(ctx context.Context, expectedK
return identity.UID, nil
}
func (browser xiaohongshuGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) {
request, err := browser.generation()
if err != nil {
return "", err
}
request["url"] = target
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/resolve", request, 30*time.Second)
if err != nil || status != http.StatusOK {
return "", errors.New("restricted Xiaohongshu share resolution failed")
}
var response struct {
URL string `json:"url"`
}
if err := json.Unmarshal(body, &response); err != nil || strings.TrimSpace(response.URL) == "" {
return "", errors.New("Xiaohongshu share resolution response omitted url")
}
return response.URL, nil
}
func (browser xiaohongshuGatewayBrowser) Media(ctx context.Context, target string) ([]byte, string, error) {
request, err := browser.generation()
if err != nil {
@@ -132,28 +153,79 @@ func decodeXiaohongshuResponse(body []byte) (xiaohongshu.Response, error) {
return xiaohongshu.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil
}
func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, accountKey, sourceType, sourceID string) (creator.PlatformCollector, string, error) {
func newXiaohongshuReadCollector(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, sourceType, sourceID string) (*xiaohongshu.Collector, error) {
if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" {
return nil, creator.ErrUnavailable
}
account, err := phaseAStore.GetAccount(ctx, accountID)
if err != nil {
return nil, err
}
if account.Platform != creator.PlatformXiaohongshu || account.AuthorizationStatus != "authorized" {
return nil, creator.ErrConflict
}
profile, err := store.GetAccountProfile(ctx, accountID)
if err != nil {
return nil, err
}
if profile.Platform != creator.PlatformXiaohongshu || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" {
return nil, creator.ErrConflict
}
environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID)
if err != nil {
return nil, fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err)
}
if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 {
return nil, fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable)
}
gateway, err := hubStore.GetGateway(ctx, environment.Gateway)
if err != nil {
return nil, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
}
browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}
if _, err := browser.Identity(ctx, account.PlatformAccountKey); err != nil {
return nil, fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)
}
return &xiaohongshu.Collector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: sourceType, SourceID: sourceID}, nil
}
func validateXiaohongshuSource(homepageURL, accountKey string) error {
return xiaohongshu.ValidateSourceURL(homepageURL, accountKey)
}
func validateXiaohongshuCompetitor(input creator.CompetitorInput) error {
if input.Platform != creator.PlatformXiaohongshu {
return nil
}
return validateXiaohongshuSource(input.HomepageURL, input.PlatformAccountKey)
}
func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, viewerAccountKey, targetAccountKey, homepageURL, sourceType, sourceID string) (creator.PlatformCollector, string, error) {
if strings.TrimSpace(viewerAccountKey) == "" || strings.TrimSpace(targetAccountKey) == "" {
return nil, "", fmt.Errorf("%w: creator collector account key is missing", creator.ErrInvalid)
}
switch platform {
case creator.PlatformDouyin:
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
uid, err := browser.Identity(ctx, accountKey)
if err != nil {
if _, err := browser.Identity(ctx, viewerAccountKey); err != nil {
return nil, "", err
}
collector := douyinCollector(browser, accountKey, sourceType, sourceID)
canonical, err := collector.CanonicalSecUID(ctx, uid)
if err != nil {
collector := douyinCollector(browser, targetAccountKey, sourceType, sourceID)
if _, err := collector.CanonicalSecUID(ctx, viewerAccountKey); err != nil {
return nil, "", err
}
collector.AccountKey = canonical
return &collector, canonical, nil
return &collector, targetAccountKey, nil
case creator.PlatformXiaohongshu:
if homepageURL != "" {
if err := xiaohongshu.ValidateSourceURL(homepageURL, targetAccountKey); err != nil {
return nil, "", err
}
}
browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}
uid, err := browser.Identity(ctx, accountKey)
if err != nil {
if _, err := browser.Identity(ctx, viewerAccountKey); err != nil {
return nil, "", err
}
return &xiaohongshu.Collector{Browser: browser, AccountKey: uid, SourceType: sourceType, SourceID: sourceID}, uid, nil
return &xiaohongshu.Collector{Browser: browser, AccountKey: targetAccountKey, HomepageURL: homepageURL, SourceType: sourceType, SourceID: sourceID}, targetAccountKey, nil
default:
return nil, "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform)
}
cmd/control-plane/xiaohongshu_test.go
+35
View File
@@ -7,7 +7,9 @@ import (
"net/http/httptest"
"testing"
"git.ipao.vip/rogee/creator-hub/internal/creator"
"git.ipao.vip/rogee/creator-hub/internal/hub"
"git.ipao.vip/rogee/creator-hub/internal/xiaohongshu"
)
const testXiaohongshuIdentityURL = "https://edith.xiaohongshu.com/api/sns/web/v2/user/me"
@@ -34,6 +36,39 @@ func TestXiaohongshuGatewayBrowserFencesAccountGeneration(t *testing.T) {
}
}
func TestXiaohongshuGatewayBrowserResolvesShareLinks(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/resolve" {
t.Fatalf("unexpected path: %s", request.URL.Path)
}
_ = json.NewEncoder(response).Encode(map[string]any{"url": "https://www.xiaohongshu.com/explore/n-1"})
}))
defer server.Close()
browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()}
resolved, err := browser.Resolve(context.Background(), "https://xhslink.com/a/abc")
if err != nil || resolved != "https://www.xiaohongshu.com/explore/n-1" {
t.Fatalf("resolved URL=%q err=%v", resolved, err)
}
}
func TestNewXiaohongshuCollectorKeepsViewerAndTargetSeparate(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/identity" {
t.Fatalf("unexpected path: %s", request.URL.Path)
}
_ = json.NewEncoder(response).Encode(map[string]any{"uid": "viewer-1"})
}))
defer server.Close()
collector, target, err := newCreatorCollector(context.Background(), creator.PlatformXiaohongshu, hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, readyDouyinEnvironment(), "viewer-1", "target-1", "https://www.xiaohongshu.com/user/profile/target-1?xsec_source=pc_search", creator.SourceCompetitor, "source-1")
if err != nil {
t.Fatalf("new collector: %v", err)
}
xhsCollector, ok := collector.(*xiaohongshu.Collector)
if !ok || xhsCollector.AccountKey != "target-1" || target != "target-1" || xhsCollector.HomepageURL == "" {
t.Fatalf("collector=%#v target=%q", collector, target)
}
}
func TestXiaohongshuGatewayBrowserPostCarriesJSONBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
var body map[string]any
cmd/docker_gateway/douyin.py
+91 -6
View File
@@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from datetime import datetime, timezone
from urllib.parse import urlsplit
from urllib.parse import parse_qs, urlsplit
import websocket
@@ -229,6 +229,7 @@ class DouyinBrowser:
origin: str = ORIGIN,
url_validator: Callable[[object], bool] | None = None,
media_validator: Callable[[object], bool] | None = None,
media_selector: str = "video",
) -> None:
self.endpoint = endpoint or (
lambda alias: f"http://creatorhub-browser-{alias}:9222"
@@ -236,6 +237,7 @@ class DouyinBrowser:
self.origin = origin
self.url_validator = url_validator or is_douyin_url
self.media_validator = media_validator or is_douyin_media_url
self.media_selector = media_selector
@contextmanager
def connection(self, alias: str):
@@ -396,8 +398,8 @@ class DouyinBrowser:
raise DouyinError("Douyin media page did not load")
result = cdp.evaluate(
f"""(async()=>{{
const video=document.querySelector('video');
const source=video?.currentSrc||video?.src||'';
const media=document.querySelector({json.dumps(self.media_selector)});
const source=media?.currentSrc||media?.src||'';
if(!source)return {{error:'media_source_unavailable'}};
const r=await fetch(source,{{credentials:'include',redirect:'error'}});
if(!r.body)return {{status:r.status,content_type:r.headers.get('content-type')||'',body:''}};
@@ -1289,7 +1291,9 @@ XHS_ORIGIN = "https://www.xiaohongshu.com"
XHS_API_ORIGIN = "https://edith.xiaohongshu.com"
XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com"
XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me"
XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"})
XHS_ALLOWED_HOSTS = frozenset(
{"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"}
)
class XiaohongshuBrowser(DouyinBrowser):
@@ -1299,6 +1303,7 @@ class XiaohongshuBrowser(DouyinBrowser):
origin=XHS_ORIGIN,
url_validator=is_xiaohongshu_url,
media_validator=is_xiaohongshu_media_url,
media_selector="video, img.note-slider-img",
)
def post(self, alias: str, target: str, body: bytes) -> BrowserResponse:
@@ -1334,7 +1339,29 @@ class XiaohongshuBrowser(DouyinBrowser):
response_body = result.get("body")
if not isinstance(response_body, str):
raise DouyinError("restricted Xiaohongshu POST returned invalid body")
return BrowserResponse(status, response_body, detect_challenge(status, response_body))
return BrowserResponse(
status, response_body, detect_challenge(status, response_body)
)
def resolve(self, alias: str, target: str) -> str:
if not is_xiaohongshu_share_url(target):
raise DouyinError("restricted Xiaohongshu share URL is invalid")
with self.connection(alias) as cdp:
if cdp.evaluate("location.origin") != self.origin:
raise DouyinError("restricted browser origin changed")
cdp.command("Page.navigate", {"url": target})
event = cdp.wait_event(
"Page.frameNavigated",
lambda params: _is_xiaohongshu_page_url(
params.get("frame", {}).get("url", "")
),
)
final_url = event.get("frame", {}).get("url")
if not isinstance(final_url, str) or not _is_xiaohongshu_page_url(final_url):
raise DouyinError(
"Xiaohongshu share URL did not resolve to a supported page"
)
return final_url
def identity(self, alias: str, expected_uid: str | None = None) -> dict:
response = self.get(alias, XHS_IDENTITY_URL)
@@ -1362,7 +1389,9 @@ class XiaohongshuBrowser(DouyinBrowser):
):
raise DouyinError("Xiaohongshu login is not valid")
if expected_uid and user_id != expected_uid:
raise DouyinError("Xiaohongshu identity does not match the expected account")
raise DouyinError(
"Xiaohongshu identity does not match the expected account"
)
return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""}
@@ -1403,6 +1432,62 @@ def is_xiaohongshu_url(value: object) -> bool:
)
def is_xiaohongshu_share_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
port = parsed.port
except (TypeError, ValueError):
return False
path = parsed.path.strip("/")
return (
parsed.scheme == "https"
and parsed.hostname in {"xhslink.com", "www.xhslink.com"}
and port is None
and parsed.username is None
and parsed.password is None
and parsed.fragment == ""
and bool(path)
and len(path) <= 256
and not parsed.query
)
def _is_xiaohongshu_page_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
port = parsed.port
query = parse_qs(parsed.query, keep_blank_values=True)
except (TypeError, ValueError):
return False
parts = parsed.path.strip("/").split("/")
if not (
parsed.scheme == "https"
and parsed.hostname == "www.xiaohongshu.com"
and port is None
and parsed.username is None
and parsed.password is None
and parsed.fragment == ""
and (
len(parts) == 2
and parts[0] == "explore"
or len(parts) == 3
and parts[:2] == ["user", "profile"]
)
):
return False
return all(
key in {"xsec_token", "xsec_source"}
and len(values) == 1
and len(values[0]) <= 2048
and not any(char in values[0] for char in "\r\n")
for key, values in query.items()
)
def notice_ids(event: dict) -> list[str]:
try:
payload = json.loads(event["payload"])
cmd/docker_gateway/gateway.py
+158 -26
View File
@@ -47,6 +47,7 @@ from .douyin import (
DouyinError,
SubscriptionManager,
XiaohongshuBrowser,
is_xiaohongshu_share_url,
)
from .proxy import ProxyExit, ProxyRegistry
@@ -99,7 +100,9 @@ class Gateway:
self.token = token
self.self_name = self_name
self.browser = browser or DouyinBrowser(self._browser_endpoint)
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint)
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(
self._browser_endpoint
)
self.proxies = ProxyRegistry()
self.reservations = AliasReservationManager(docker, self_name)
self.subscriptions = SubscriptionManager(self.browser)
@@ -685,9 +688,15 @@ class Gateway:
response = self.xiaohongshu_browser.get(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc))
LOG.warning(
"Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc)
)
raise RequestError("restricted Xiaohongshu operation failed") from exc
return {"status": response.status, "body": response.body, "challenge": response.challenge}
return {
"status": response.status,
"body": response.body,
"challenge": response.challenge,
}
def post_xiaohongshu(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
@@ -699,7 +708,9 @@ class Gateway:
):
raise RequestError("invalid restricted Xiaohongshu POST request", 400)
try:
encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
encoded = json.dumps(
body, ensure_ascii=False, separators=(",", ":")
).encode()
except (TypeError, ValueError) as exc:
raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc
with self._alias_lock(alias):
@@ -708,13 +719,47 @@ class Gateway:
response = self.xiaohongshu_browser.post(alias, target, encoded)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc))
LOG.warning(
"Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc)
)
raise RequestError("restricted Xiaohongshu operation failed") from exc
return {"status": response.status, "body": response.body, "challenge": response.challenge}
return {
"status": response.status,
"body": response.body,
"challenge": response.challenge,
}
def resolve_xiaohongshu(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_source_url(
target
):
raise RequestError("invalid restricted Xiaohongshu source URL", 400)
if not is_xiaohongshu_share_url(target):
return {"url": target}
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
try:
resolved = self.xiaohongshu_browser.resolve(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning(
"Xiaohongshu share resolution failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"restricted Xiaohongshu share resolution failed"
) from exc
if not valid_xiaohongshu_page_url(resolved):
raise RequestError("Xiaohongshu share resolved to an unsupported URL", 502)
return {"url": resolved}
def get_xiaohongshu_media(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target):
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(
target
):
raise RequestError("invalid restricted Xiaohongshu media request", 400)
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
@@ -722,9 +767,19 @@ class Gateway:
response = self.xiaohongshu_browser.get_media(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc))
raise RequestError("restricted Xiaohongshu media download failed") from exc
return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64}
LOG.warning(
"Xiaohongshu media download failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"restricted Xiaohongshu media download failed"
) from exc
return {
"status": response.status,
"content_type": response.content_type,
"body_base64": response.body_base64,
}
def xiaohongshu_identity(self, alias: str, input: dict) -> dict:
expected_account_key = input.get("expected_account_key", "")
@@ -739,10 +794,18 @@ class Gateway:
try:
identity = self.xiaohongshu_browser.identity(alias)
except DouyinError as exc:
LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc))
raise RequestError("Xiaohongshu login identity could not be verified") from exc
LOG.warning(
"Xiaohongshu identity verification failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"Xiaohongshu login identity could not be verified"
) from exc
if identity.get("uid") != expected_account_key:
raise RequestError("Xiaohongshu identity does not match the expected account", 409)
raise RequestError(
"Xiaohongshu identity does not match the expected account", 409
)
return identity
def douyin_action(self, alias: str, input: dict) -> dict:
@@ -1210,7 +1273,7 @@ class GatewayHandler(BaseHTTPRequestHandler):
gateway.restore_proxy(alias, body)
return None
match = re.fullmatch(
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)",
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity|resolve)",
path,
)
if match:
@@ -1221,6 +1284,8 @@ class GatewayHandler(BaseHTTPRequestHandler):
return gateway.post_xiaohongshu(alias, body)
if action == "media" and method == "POST":
return gateway.get_xiaohongshu_media(alias, body)
if action == "resolve" and method == "POST":
return gateway.resolve_xiaohongshu(alias, body)
if action == "identity" and method == "POST":
return gateway.xiaohongshu_identity(alias, body)
match = re.fullmatch(
@@ -1516,7 +1581,9 @@ def valid_xiaohongshu_generation(value: dict) -> bool:
return valid_douyin_generation(value)
def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool:
def valid_xhs_query(
query: object, allowed: set[str], required: set[str] | None = None
) -> bool:
if not isinstance(query, dict) or not isinstance(allowed, set):
return False
required = required or set()
@@ -1525,7 +1592,12 @@ def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None
for key, values in query.items():
if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1:
return False
if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]:
if (
not isinstance(values[0], str)
or len(values[0]) > 2048
or "\r" in values[0]
or "\n" in values[0]
):
return False
return True
@@ -1549,18 +1621,45 @@ def valid_xhs_url(raw: object) -> bool:
query = parse_qs(parsed.query, keep_blank_values=True)
except ValueError:
return False
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH:
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_IDENTITY_PATH
):
return not query
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH:
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_USER_POSTED_PATH
):
return (
valid_xhs_query(
query,
{
"user_id",
"cursor",
"num",
"image_formats",
"xsec_source",
"xsec_token",
},
{"user_id", "num"},
)
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0]))
and query["num"] == ["30"]
)
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_COMMENTS_PATH
):
return valid_xhs_query(
query,
{"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"},
{"user_id", "num"},
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"]
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH:
return valid_xhs_query(
query,
{"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"},
{
"note_id",
"cursor",
"top_comment_id",
"image_formats",
"xsec_source",
"xsec_token",
},
{"note_id", "cursor", "top_comment_id"},
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0]))
return False
@@ -1570,6 +1669,34 @@ def valid_xiaohongshu_url(raw: object) -> bool:
return valid_xhs_url(raw)
def valid_xiaohongshu_page_url(raw: object) -> bool:
if not isinstance(raw, str):
return False
try:
parsed = urlsplit(raw)
query = parse_qs(parsed.query, keep_blank_values=True)
except ValueError:
return False
if not _valid_xhs_host(parsed, "www.xiaohongshu.com") or not valid_xhs_query(
query, {"xsec_source", "xsec_token"}
):
return False
parts = parsed.path.strip("/").split("/")
return (
len(parts) == 2
and parts[0] == "explore"
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1]))
) or (
len(parts) == 3
and parts[:2] == ["user", "profile"]
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[2]))
)
def valid_xiaohongshu_source_url(raw: object) -> bool:
return valid_xiaohongshu_page_url(raw) or is_xiaohongshu_share_url(raw)
def valid_xhs_post_url(raw: object) -> bool:
if not isinstance(raw, str):
return False
@@ -1600,7 +1727,12 @@ def valid_xiaohongshu_media_url(raw: object) -> bool:
if not _valid_xhs_host(parsed, "www.xiaohongshu.com"):
return False
parts = parsed.path.strip("/").split("/")
return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"})
return (
len(parts) == 2
and parts[0] == "explore"
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1]))
and valid_xhs_query(query, {"xsec_source", "xsec_token"})
)
def valid_douyin_url(raw: object) -> bool:
cmd/docker_gateway/test_xiaohongshu.py
+77 -14
View File
@@ -13,31 +13,70 @@ valid_xhs_url = gateway_module.valid_xhs_url
valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url
valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url
valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation
valid_xiaohongshu_page_url = gateway_module.valid_xiaohongshu_page_url
valid_xiaohongshu_source_url = gateway_module.valid_xiaohongshu_source_url
class XiaohongshuValidationTests(unittest.TestCase):
def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None:
self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
self.assertTrue(
valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")
)
self.assertTrue(
valid_xiaohongshu_url(
"https://edith.xiaohongshu.com/api/sns/web/v2/user/me"
)
)
self.assertTrue(
valid_xiaohongshu_page_url("https://www.xiaohongshu.com/user/profile/u-1")
)
self.assertTrue(valid_xiaohongshu_source_url("https://xhslink.com/a/abc"))
self.assertTrue(
valid_xhs_url(
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user"
)
)
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"))
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me"))
self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes"))
self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed"))
self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"))
self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment"))
self.assertFalse(
valid_xhs_url(
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"
)
)
self.assertFalse(
valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me")
)
self.assertTrue(
valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes")
)
self.assertTrue(
valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed")
)
self.assertTrue(
valid_xiaohongshu_media_url(
"https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"
)
)
self.assertFalse(
valid_xiaohongshu_media_url(
"https://www.xiaohongshu.com/explore/n-1#fragment"
)
)
def test_generation_shape_matches_existing_browser_fence(self) -> None:
self.assertTrue(
valid_xiaohongshu_generation(
{"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""}
{
"binding_version": 1,
"runtime_id": "a" * 64,
"network_id": "network",
"network_exit_id": "",
}
)
)
self.assertFalse(
valid_xiaohongshu_generation(
{"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}
)
)
self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}))
class XiaohongshuRouteTests(unittest.TestCase):
@@ -48,18 +87,42 @@ class XiaohongshuRouteTests(unittest.TestCase):
gateway.post_xiaohongshu.return_value = {"status": 200}
gateway.get_xiaohongshu_media.return_value = {"status": 200}
gateway.xiaohongshu_identity.return_value = {"uid": "u-1"}
gateway.resolve_xiaohongshu.return_value = {
"url": "https://www.xiaohongshu.com/explore/n-1"
}
server = Mock()
server.gateway = gateway
cast(Any, handler).server = server
cast(Any, handler).server_as_gateway = lambda: server
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"})
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route(
"POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}
),
{"uid": "u-1"},
)
self.assertEqual(
handler._route(
"POST", "/v1/browsers/account-a/xiaohongshu/resolve", {}, {}
),
{"url": "https://www.xiaohongshu.com/explore/n-1"},
)
with self.assertRaises(RequestError):
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {})
gateway.get_xiaohongshu.assert_called_once_with("account-a", {})
gateway.post_xiaohongshu.assert_called_once_with("account-a", {})
gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {})
gateway.xiaohongshu_identity.assert_called_once_with("account-a", {})
gateway.resolve_xiaohongshu.assert_called_once_with("account-a", {})
cmd/docker_gateway/xiaohongshu.py
+2
View File
@@ -12,6 +12,7 @@ from .douyin import (
XHS_SEARCH_ORIGIN,
XiaohongshuBrowser,
is_xiaohongshu_media_url,
is_xiaohongshu_share_url,
is_xiaohongshu_url,
)
@@ -23,5 +24,6 @@ __all__ = [
"XHS_SEARCH_ORIGIN",
"XiaohongshuBrowser",
"is_xiaohongshu_media_url",
"is_xiaohongshu_share_url",
"is_xiaohongshu_url",
]
docs/research/xhs-all-in-one.md
+2 -1
View File
@@ -22,7 +22,8 @@
## 1.1 当前实现边界
- CreatorHub 当前已接入小红书只读 collector、详情/搜索/作品/一级评论读取、受限 gateway 路由和控制面平台分派；写操作与事件监听仍未接入。
- CreatorHub 当前已接入小红书只读 collector、详情/搜索/作品/一级评论读取、分享链接解析、原始 payload 保存、受限 gateway 路由和控制面平台分派；写操作与事件监听仍未接入。
- 小红书竞品主页输入会校验稳定账号标识并保留主页中的 `xsec_token/xsec_source`；作品详情可通过受限浏览器解析官方分享短链。
- `internal/creator/collection.go` 的分页、窗口、checkpoint 和 lease 模型可作为后续适配的复用边界，但不能证明小红书平台能力。
**未完成的真实能力验收：**私有接口签名是否能由浏览器当前会话完成、真实 UID/作品/评论分页、媒体下载、写操作和事件监听仍需真实小红书环境分别验证。HTTP 200、离线 fixture 和本地单元测试不能替代这些证据。
internal/creator/content.go
+29 -10
View File
@@ -12,6 +12,8 @@ import (
"unicode/utf8"
)
const maxRawPayloadBytes = 4 << 20
func validateHomepage(value string) error {
parsed, err := url.Parse(strings.TrimSpace(value))
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil {
@@ -218,7 +220,8 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time)
(input.SourceType != SourceOwned && input.SourceType != SourceCompetitor) ||
utf8.RuneCountInString(input.WorkKey) > 255 || utf8.RuneCountInString(input.Title) > 1000 ||
utf8.RuneCountInString(input.Body) > 100000 || utf8.RuneCountInString(input.OriginalURL) > 2000 ||
utf8.RuneCountInString(input.CoverURL) > 2000 {
utf8.RuneCountInString(input.CoverURL) > 2000 || len(input.RawPayload) > maxRawPayloadBytes ||
input.RawPayload != "" && !json.Valid([]byte(input.RawPayload)) {
return Work{}, false, ErrInvalid
}
if err := s.validateWorkSource(ctx, input.Platform, input.SourceType, input.SourceID); err != nil {
@@ -257,8 +260,8 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time)
var inserted bool
err = tx.QueryRowContext(ctx, `
INSERT INTO creator_work (id, platform, work_key, source_type, source_id, author_name, title, body,
published_at, published_at_status, original_url, cover_url, likes, comments_count, shares)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (platform, work_key) DO UPDATE SET
author_name = CASE WHEN EXCLUDED.author_name = '' THEN creator_work.author_name ELSE EXCLUDED.author_name END,
title = CASE WHEN EXCLUDED.title = '' THEN creator_work.title ELSE EXCLUDED.title END,
@@ -267,12 +270,13 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time)
published_at_status = CASE WHEN EXCLUDED.published_at IS NULL THEN creator_work.published_at_status ELSE EXCLUDED.published_at_status END,
original_url = CASE WHEN EXCLUDED.original_url = '' THEN creator_work.original_url ELSE EXCLUDED.original_url END,
cover_url = CASE WHEN EXCLUDED.cover_url = '' THEN creator_work.cover_url ELSE EXCLUDED.cover_url END,
raw_payload = COALESCE(EXCLUDED.raw_payload, creator_work.raw_payload),
likes = COALESCE(EXCLUDED.likes, creator_work.likes),
comments_count = COALESCE(EXCLUDED.comments_count, creator_work.comments_count),
shares = COALESCE(EXCLUDED.shares, creator_work.shares), updated_at = now()
RETURNING id, (xmax = 0)`, id, input.Platform, input.WorkKey, input.SourceType, input.SourceID,
input.AuthorName, input.Title, input.Body, input.PublishedAt, status, input.OriginalURL, input.CoverURL,
input.Likes, input.CommentsCount, input.Shares).Scan(&returnedID, &inserted)
nullableRawPayload(input.RawPayload), input.Likes, input.CommentsCount, input.Shares).Scan(&returnedID, &inserted)
if err != nil {
return Work{}, false, databaseError(err)
}
@@ -297,24 +301,35 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time)
func ptrTime(value time.Time) *time.Time { return &value }
func nullableRawPayload(value string) any {
if value == "" {
return nil
}
return value
}
func scanWork(scanner interface{ Scan(...any) error }) (Work, error) {
var result Work
var publishedAt, latestAt, nextAt sql.NullTime
var likes, commentsCount, shares sql.NullInt64
var rawPayload sql.NullString
if err := scanner.Scan(&result.ID, &result.Platform, &result.WorkKey, &result.SourceType, &result.SourceID,
&result.AuthorName, &result.Title, &result.Body, &publishedAt, &result.PublishedAtStatus,
&result.OriginalURL, &result.CoverURL, &likes, &commentsCount, &shares, &latestAt, &nextAt,
&result.OriginalURL, &result.CoverURL, &rawPayload, &likes, &commentsCount, &shares, &latestAt, &nextAt,
&result.MetricStopReason, &result.CreatedAt, &result.UpdatedAt); err != nil {
return Work{}, err
}
result.PublishedAt = nullableTime(publishedAt)
if rawPayload.Valid {
result.RawPayload = rawPayload.String
}
result.Likes, result.CommentsCount, result.Shares = nullableInt64(likes), nullableInt64(commentsCount), nullableInt64(shares)
result.LatestMetricsAt, result.NextMetricAt = nullableTime(latestAt), nullableTime(nextAt)
return result, nil
}
const workSelect = `SELECT id, platform, work_key, source_type, source_id, author_name, title, body,
published_at, published_at_status, original_url, cover_url, likes, comments_count, shares,
published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares,
latest_metrics_at, next_metric_at, metric_stop_reason, created_at, updated_at FROM creator_work`
func (s *Store) loadWorkSources(ctx context.Context, work *Work) error {
@@ -629,7 +644,7 @@ func (s *Store) SaveRewrite(ctx context.Context, workID, title, script string) (
func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, bool, error) {
input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.CommentType = strings.TrimSpace(input.Platform), strings.TrimSpace(input.CommentKey), strings.TrimSpace(input.WorkID), strings.TrimSpace(input.AuthorUID), strings.TrimSpace(input.AuthorName), strings.TrimSpace(input.CommentType)
if !ValidatePlatform(input.Platform) || input.CommentKey == "" || input.WorkID == "" || strings.TrimSpace(input.Content) == "" || (input.CommentType != "top_level" && input.CommentType != "unknown") || utf8.RuneCountInString(input.Content) > 10000 {
if !ValidatePlatform(input.Platform) || input.CommentKey == "" || input.WorkID == "" || strings.TrimSpace(input.Content) == "" || (input.CommentType != "top_level" && input.CommentType != "unknown") || utf8.RuneCountInString(input.Content) > 10000 || len(input.RawPayload) > maxRawPayloadBytes || input.RawPayload != "" && !json.Valid([]byte(input.RawPayload)) {
return Comment{}, false, ErrInvalid
}
work, err := s.GetWork(ctx, input.WorkID)
@@ -642,7 +657,7 @@ func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, b
id := newID("comment")
var returnedID string
var inserted bool
err = s.db.QueryRowContext(ctx, `INSERT INTO creator_comment (id, platform, comment_key, work_id, author_uid, author_name, content, published_at, comment_type) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (platform, comment_key) DO UPDATE SET work_id = EXCLUDED.work_id, author_uid = EXCLUDED.author_uid, author_name = EXCLUDED.author_name, content = EXCLUDED.content, published_at = EXCLUDED.published_at, comment_type = EXCLUDED.comment_type RETURNING id, (xmax = 0)`, id, input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.Content, input.PublishedAt, input.CommentType).Scan(&returnedID, &inserted)
err = s.db.QueryRowContext(ctx, `INSERT INTO creator_comment (id, platform, comment_key, work_id, author_uid, author_name, content, raw_payload, published_at, comment_type) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (platform, comment_key) DO UPDATE SET work_id = EXCLUDED.work_id, author_uid = EXCLUDED.author_uid, author_name = EXCLUDED.author_name, content = EXCLUDED.content, raw_payload = COALESCE(EXCLUDED.raw_payload, creator_comment.raw_payload), published_at = EXCLUDED.published_at, comment_type = EXCLUDED.comment_type RETURNING id, (xmax = 0)`, id, input.Platform, input.CommentKey, input.WorkID, input.AuthorUID, input.AuthorName, input.Content, nullableRawPayload(input.RawPayload), input.PublishedAt, input.CommentType).Scan(&returnedID, &inserted)
if err != nil {
return Comment{}, false, databaseError(err)
}
@@ -653,15 +668,19 @@ func (s *Store) SaveComment(ctx context.Context, input CommentInput) (Comment, b
func scanComment(scanner interface{ Scan(...any) error }) (Comment, error) {
var result Comment
var publishedAt sql.NullTime
if err := scanner.Scan(&result.ID, &result.Platform, &result.CommentKey, &result.WorkID, &result.AuthorUID, &result.AuthorName, &result.Content, &publishedAt, &result.CollectedAt, &result.CommentType); err != nil {
var rawPayload sql.NullString
if err := scanner.Scan(&result.ID, &result.Platform, &result.CommentKey, &result.WorkID, &result.AuthorUID, &result.AuthorName, &result.Content, &rawPayload, &publishedAt, &result.CollectedAt, &result.CommentType); err != nil {
return Comment{}, err
}
result.PublishedAt = nullableTime(publishedAt)
if rawPayload.Valid {
result.RawPayload = rawPayload.String
}
result.CollectedAt = result.CollectedAt.UTC()
return result, nil
}
const commentSelect = `SELECT id, platform, comment_key, work_id, author_uid, author_name, content, published_at, collected_at, comment_type FROM creator_comment`
const commentSelect = `SELECT id, platform, comment_key, work_id, author_uid, author_name, content, raw_payload, published_at, collected_at, comment_type FROM creator_comment`
func (s *Store) GetComment(ctx context.Context, id string) (Comment, error) {
result, err := scanComment(s.db.QueryRowContext(ctx, commentSelect+` WHERE id = $1`, id))
internal/creator/migrations/031_xhs_raw_payloads.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE creator_work ADD COLUMN IF NOT EXISTS raw_payload text;
ALTER TABLE creator_comment ADD COLUMN IF NOT EXISTS raw_payload text;
internal/creator/models.go
+4
View File
@@ -156,6 +156,7 @@ type Work struct {
MetricStopReason string `json:"metric_stop_reason,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
RawPayload string `json:"-"`
}
type WorkInput struct {
@@ -173,6 +174,7 @@ type WorkInput struct {
Likes *int64 `json:"likes"`
CommentsCount *int64 `json:"comments_count"`
Shares *int64 `json:"shares"`
RawPayload string `json:"-"`
}
type WorkFilter struct {
@@ -232,6 +234,7 @@ type Comment struct {
PublishedAt *time.Time `json:"published_at,omitempty"`
CollectedAt time.Time `json:"collected_at"`
CommentType string `json:"comment_type"`
RawPayload string `json:"-"`
}
type CommentInput struct {
@@ -243,6 +246,7 @@ type CommentInput struct {
Content string `json:"content"`
PublishedAt *time.Time `json:"published_at"`
CommentType string `json:"comment_type"`
RawPayload string `json:"-"`
}
type LeadRule struct {
internal/creator/store.go
+4
View File
@@ -58,6 +58,9 @@ var migration029 string
//go:embed migrations/030_creator_event_gateway_time.sql
var migration030 string
//go:embed migrations/031_xhs_raw_payloads.sql
var migration031 string
type SecretReference struct {
ID string
Provider string
@@ -135,6 +138,7 @@ func (s *Store) migrate(ctx context.Context) error {
{version: 28, sql: migration028},
{version: 29, sql: migration029},
{version: 30, sql: migration030},
{version: 31, sql: migration031},
}
for _, migration := range migrations {
var applied bool
internal/xiaohongshu/collector.go
+204 -44
View File
@@ -46,6 +46,27 @@ type Identity struct {
Nickname string
}
type LinkKind string
const (
LinkHomepage LinkKind = "homepage"
LinkNote LinkKind = "note"
LinkShare LinkKind = "share"
)
type LinkContext struct {
Kind LinkKind
AccountKey string
WorkKey string
Token string
Source string
CanonicalURL string
}
type ShareResolver interface {
Resolve(context.Context, string) (string, error)
}
type Collector struct {
Browser Browser
AccountKey string
@@ -84,6 +105,98 @@ func (c *Collector) Identity(ctx context.Context, expectedKey string) (Identity,
return identity, nil
}
func ParseSourceURL(raw string) (LinkContext, error) {
raw = strings.TrimSpace(raw)
parsed, err := url.Parse(raw)
if err != nil || parsed.Scheme != "https" || parsed.User != nil || parsed.Fragment != "" {
return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu source URL", creator.ErrInvalid)
}
if parsed.Port() != "" {
return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu source URL", creator.ErrInvalid)
}
query, err := parseAccessQuery(parsed)
if err != nil {
return LinkContext{}, err
}
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
context := LinkContext{Token: query.Token, Source: query.Source, CanonicalURL: parsed.String()}
switch parsed.Hostname() {
case "www.xiaohongshu.com":
switch {
case len(parts) == 2 && parts[0] == "explore" && keyPattern.MatchString(parts[1]):
context.Kind, context.WorkKey = LinkNote, parts[1]
case len(parts) == 3 && parts[0] == "user" && parts[1] == "profile" && keyPattern.MatchString(parts[2]):
context.Kind, context.AccountKey = LinkHomepage, parts[2]
default:
return LinkContext{}, fmt.Errorf("%w: unsupported xiaohongshu source path", creator.ErrInvalid)
}
case "xhslink.com", "www.xhslink.com":
if len(parts) < 1 || len(parts) > 4 || parsed.RawQuery != "" {
return LinkContext{}, fmt.Errorf("%w: invalid xiaohongshu share URL", creator.ErrInvalid)
}
context.Kind = LinkShare
default:
return LinkContext{}, fmt.Errorf("%w: unsupported xiaohongshu source host", creator.ErrInvalid)
}
return context, nil
}
type accessQuery struct {
Token string
Source string
}
func parseAccessQuery(parsed *url.URL) (accessQuery, error) {
values, err := url.ParseQuery(parsed.RawQuery)
if err != nil {
return accessQuery{}, fmt.Errorf("%w: invalid xiaohongshu access query", creator.ErrInvalid)
}
for key, items := range values {
if key != "xsec_token" && key != "xsec_source" || len(items) != 1 || strings.ContainsAny(items[0], "\r\n") || len(items[0]) > 2048 {
return accessQuery{}, fmt.Errorf("%w: unsupported xiaohongshu access query", creator.ErrInvalid)
}
}
result := accessQuery{Token: values.Get("xsec_token"), Source: values.Get("xsec_source")}
if result.Source != "" && !keyPattern.MatchString(result.Source) {
return accessQuery{}, fmt.Errorf("%w: invalid xiaohongshu access source", creator.ErrInvalid)
}
return result, nil
}
func ValidateSourceURL(raw, expectedAccountKey string) error {
link, err := ParseSourceURL(raw)
if err != nil {
return err
}
if link.Kind != LinkHomepage || link.AccountKey != expectedAccountKey {
return fmt.Errorf("%w: xiaohongshu homepage does not match the account", creator.ErrConflict)
}
return nil
}
func (c *Collector) resolveSourceURL(ctx context.Context, raw string) (LinkContext, error) {
link, err := ParseSourceURL(raw)
if err != nil {
return LinkContext{}, err
}
if link.Kind != LinkShare {
return link, nil
}
resolver, ok := c.Browser.(ShareResolver)
if !ok {
return LinkContext{}, fmt.Errorf("%w: xiaohongshu share URL resolver is unavailable", creator.ErrUnavailable)
}
canonical, err := resolver.Resolve(ctx, link.CanonicalURL)
if err != nil {
return LinkContext{}, err
}
resolved, err := ParseSourceURL(canonical)
if err != nil || resolved.Kind != LinkNote {
return LinkContext{}, fmt.Errorf("%w: xiaohongshu share URL did not resolve to a note", creator.ErrInvalid)
}
return resolved, nil
}
func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (creator.WorkPage, error) {
if c == nil {
return creator.WorkPage{}, fmt.Errorf("%w: xiaohongshu collector is nil", creator.ErrUnavailable)
@@ -94,14 +207,21 @@ func (c *Collector) ListWorks(ctx context.Context, accountKey, cursor string) (c
if c.Browser == nil || !keyPattern.MatchString(accountKey) || !validCursor(cursor) {
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu work collection request", creator.ErrInvalid)
}
access := c.defaultContext()
access, err := c.defaultContext()
if err != nil {
return creator.WorkPage{}, err
}
query := url.Values{
"user_id": {accountKey},
"cursor": {cursor},
"num": {"30"},
"image_formats": {"jpg,webp,avif"},
"xsec_source": {access.Source},
"xsec_token": {access.Token},
}
if access.Source != "" {
query.Set("xsec_source", access.Source)
}
if access.Token != "" {
query.Set("xsec_token", access.Token)
}
response, err := c.Browser.Get(ctx, APIOrigin+UserPostedPath+"?"+query.Encode())
if err != nil {
@@ -121,14 +241,21 @@ func (c *Collector) ListTopLevelComments(ctx context.Context, workKey, cursor st
if c == nil || c.Browser == nil || !keyPattern.MatchString(workKey) || !validCursor(cursor) {
return creator.CommentPage{}, fmt.Errorf("%w: invalid xiaohongshu comment collection request", creator.ErrInvalid)
}
access := c.contextFor(workKey)
access, err := c.contextFor(workKey)
if err != nil {
return creator.CommentPage{}, err
}
query := url.Values{
"note_id": {workKey},
"cursor": {cursor},
"top_comment_id": {""},
"image_formats": {"jpg,webp,avif"},
"xsec_source": {access.Source},
"xsec_token": {access.Token},
}
if access.Source != "" {
query.Set("xsec_source", access.Source)
}
if access.Token != "" {
query.Set("xsec_token", access.Token)
}
response, err := c.Browser.Get(ctx, APIOrigin+CommentsPath+"?"+query.Encode())
if err != nil {
@@ -178,7 +305,11 @@ func (c *Collector) SearchNotes(ctx context.Context, queryText string, page int)
if err := responseError(response, "search"); err != nil {
return creator.WorkPage{}, err
}
items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, c.defaultContext())
fallback, err := c.defaultContext()
if err != nil {
return creator.WorkPage{}, err
}
items, nextCursor, hasMore, ok := parseWorksPage(response.Body, c, fallback)
if !ok {
return creator.WorkPage{}, fmt.Errorf("%w: invalid xiaohongshu search response", creator.ErrInvalid)
}
@@ -193,51 +324,47 @@ func randomID() (string, error) {
return hex.EncodeToString(value), nil
}
func (c *Collector) defaultContext() accessContext {
access := accessContext{Source: "pc_user"}
if c != nil && strings.TrimSpace(c.HomepageURL) != "" {
parsed, err := url.Parse(c.HomepageURL)
if err == nil {
if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" {
access.Token = token
}
if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" {
access.Source = source
}
}
func (c *Collector) defaultContext() (accessContext, error) {
if c == nil || strings.TrimSpace(c.HomepageURL) == "" {
return accessContext{}, nil
}
return access
link, err := ParseSourceURL(c.HomepageURL)
if err != nil {
return accessContext{}, err
}
if link.Kind != LinkHomepage {
return accessContext{}, fmt.Errorf("%w: xiaohongshu homepage URL is required", creator.ErrInvalid)
}
return accessContext{Token: link.Token, Source: link.Source}, nil
}
func (c *Collector) SetWorkContext(workKey, originalURL string) error {
if c == nil || !keyPattern.MatchString(workKey) {
return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid)
}
access := c.defaultContext()
access, err := c.defaultContext()
if err != nil {
return err
}
if strings.TrimSpace(originalURL) != "" {
parsed, err := url.Parse(originalURL)
if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" || !strings.Contains(parsed.Path, workKey) {
link, parseErr := ParseSourceURL(originalURL)
if parseErr != nil || link.Kind != LinkNote || link.WorkKey != workKey {
return fmt.Errorf("%w: invalid xiaohongshu work URL", creator.ErrInvalid)
}
if token := strings.TrimSpace(parsed.Query().Get("xsec_token")); token != "" {
access.Token = token
}
if source := strings.TrimSpace(parsed.Query().Get("xsec_source")); source != "" {
access.Source = source
}
access.Token, access.Source = link.Token, link.Source
}
if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) {
if len(access.Token) > 2048 || access.Source != "" && !keyPattern.MatchString(access.Source) {
return fmt.Errorf("%w: invalid xiaohongshu work context", creator.ErrInvalid)
}
c.contexts.Store(workKey, access)
return nil
}
func (c *Collector) contextFor(workKey string) accessContext {
func (c *Collector) contextFor(workKey string) (accessContext, error) {
if c != nil {
if value, ok := c.contexts.Load(workKey); ok {
if access, ok := value.(accessContext); ok {
return access
return access, nil
}
}
}
@@ -339,6 +466,7 @@ func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput,
if !ok {
return creator.WorkInput{}, accessContext{}, false
}
outer := object
if nested := firstObject(object, "note_card", "noteCard"); nested != nil {
object = nested
}
@@ -376,17 +504,32 @@ func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput,
return creator.WorkInput{}, accessContext{}, false
}
access := fallback
if token := firstString(outer, "xsec_token"); token != "" {
access.Token = token
}
if source := firstString(outer, "xsec_source"); source != "" {
access.Source = source
}
if token := firstString(object, "xsec_token"); token != "" {
access.Token = token
}
if source := firstString(object, "xsec_source"); source != "" {
access.Source = source
}
if utf8.RuneCountInString(access.Token) > 2048 || !keyPattern.MatchString(access.Source) {
if utf8.RuneCountInString(access.Token) > 2048 || access.Source != "" && !keyPattern.MatchString(access.Source) {
return creator.WorkInput{}, accessContext{}, false
}
originalURL := firstString(object, "original_url", "note_url", "url")
if !validOriginalURL(originalURL, id) {
if originalURL == "" {
originalURL = firstString(outer, "original_url", "note_url", "url")
}
if originalURL != "" {
link, err := ParseSourceURL(originalURL)
if err != nil || link.Kind != LinkNote || link.WorkKey != id {
return creator.WorkInput{}, accessContext{}, false
}
originalURL = link.CanonicalURL
} else {
originalURL = noteURL(id, access)
}
cover := coverURL(object)
@@ -396,6 +539,7 @@ func parseWork(raw json.RawMessage, fallback accessContext) (creator.WorkInput,
}
return creator.WorkInput{
Platform: creator.PlatformXiaohongshu,
RawPayload: string(raw),
WorkKey: id,
AuthorName: authorName,
Title: title,
@@ -457,6 +601,7 @@ func parseCommentsPage(body []byte, workKey string) ([]creator.CommentInput, str
}
items = append(items, creator.CommentInput{
Platform: creator.PlatformXiaohongshu,
RawPayload: string(raw),
CommentKey: id,
WorkID: workKey,
AuthorUID: firstString(user, "user_id", "uid", "id"),
@@ -478,7 +623,7 @@ func responseError(response Response, resource string) error {
return fmt.Errorf("%w: xiaohongshu %s challenge %s", creator.ErrUnavailable, resource, response.Challenge)
}
if response.Status >= 200 && response.Status < 300 {
return nil
return platformFailure(response.Body, resource)
}
switch response.Status {
case 401, 403, 406, 461:
@@ -490,6 +635,29 @@ func responseError(response Response, resource string) error {
}
}
func platformFailure(body []byte, resource string) error {
var envelope struct {
Success *bool `json:"success"`
Code string `json:"code"`
Msg string `json:"msg"`
Message string `json:"message"`
}
if json.Unmarshal(body, &envelope) != nil || envelope.Success == nil || *envelope.Success {
return nil
}
reason := strings.TrimSpace(envelope.Msg)
if reason == "" {
reason = strings.TrimSpace(envelope.Message)
}
if len(reason) > 512 {
reason = reason[:512]
}
if strings.Contains(strings.ToLower(reason), "rate") || strings.Contains(reason, "频繁") || strings.Contains(reason, "验证") {
return fmt.Errorf("%w: xiaohongshu %s rejected request code=%s message=%s", creator.ErrUnavailable, resource, envelope.Code, reason)
}
return fmt.Errorf("%w: xiaohongshu %s rejected request code=%s message=%s", creator.ErrConflict, resource, envelope.Code, reason)
}
func validCursor(cursor string) bool {
return cursor == "" || len(cursor) <= 512 && !strings.ContainsAny(cursor, "\r\n")
}
@@ -582,7 +750,7 @@ func optionalTimestamp(object map[string]json.RawMessage, names ...string) (*tim
}
parsed, err := strconv.ParseInt(number.String(), 10, 64)
if err != nil || parsed <= 0 {
return nil, true
return nil, false
}
if parsed > 1_000_000_000_000 {
parsed /= 1000
@@ -633,14 +801,6 @@ func coverURL(object map[string]json.RawMessage) string {
return firstString(object, "cover_url", "cover")
}
func validOriginalURL(raw, id string) bool {
if raw == "" {
return false
}
parsed, err := url.Parse(raw)
return err == nil && parsed.Scheme == "https" && (parsed.Host == "www.xiaohongshu.com" || parsed.Host == "xhslink.com") && strings.Contains(parsed.Path, id)
}
func noteURL(id string, access accessContext) string {
query := url.Values{}
if access.Token != "" {
internal/xiaohongshu/collector_test.go
+52 -8
View File
@@ -10,12 +10,14 @@ import (
)
type fakeBrowser struct {
getURL string
postURL string
getBody []byte
postBody []byte
getResp Response
postResp Response
getURL string
postURL string
getBody []byte
postBody []byte
getResp Response
postResp Response
resolvedURL string
resolveInput string
}
func (f *fakeBrowser) Get(_ context.Context, target string) (Response, error) {
@@ -26,6 +28,10 @@ func (f *fakeBrowser) Post(_ context.Context, target string, body []byte) (Respo
f.postURL, f.postBody = target, body
return f.postResp, nil
}
func (f *fakeBrowser) Resolve(_ context.Context, target string) (string, error) {
f.resolveInput = target
return f.resolvedURL, nil
}
func TestIdentityRequiresMatchingUser(t *testing.T) {
browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"user_id":"u-1","nickname":"作者"}}`)}}
@@ -43,7 +49,7 @@ func TestListWorksCapturesContextForComments(t *testing.T) {
browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"next","has_more":true,"notes":[{"note_id":"n-1","title":"标题","desc":"正文","time":1710000000,"user":{"user_id":"u-1","nickname":"作者"},"interact_info":{"liked_count":"2","comment_count":3,"shared_count":4},"xsec_token":"token"}]}}`)}}
collector := &Collector{Browser: browser, AccountKey: "u-1", SourceType: creator.SourceOwned, SourceID: "source-1"}
page, err := collector.ListWorks(context.Background(), "ignored", "")
if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || !page.HasMore {
if err != nil || len(page.Items) != 1 || page.NextCursor != "next" || !page.HasMore || page.Items[0].RawPayload == "" {
t.Fatalf("page = %#v, err = %v", page, err)
}
parsed, err := url.Parse(browser.getURL)
@@ -52,7 +58,7 @@ func TestListWorksCapturesContextForComments(t *testing.T) {
}
browser.getResp = Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"comments":[{"id":"c-1","content":"评论","create_time":1710000000,"user_info":{"user_id":"u-2","nickname":"读者"}}]}}`)}
comments, err := collector.ListTopLevelComments(context.Background(), "n-1", "")
if err != nil || len(comments.Items) != 1 || comments.Items[0].CommentType != "top_level" {
if err != nil || len(comments.Items) != 1 || comments.Items[0].CommentType != "top_level" || comments.Items[0].RawPayload == "" {
t.Fatalf("comments = %#v, err = %v", comments, err)
}
commentURL, err := url.Parse(browser.getURL)
@@ -74,6 +80,44 @@ func TestSearchUsesBoundedPostAndRejectsMalformedPage(t *testing.T) {
}
}
func TestHomepageContextIsUsedWithoutInventingDefaults(t *testing.T) {
browser := &fakeBrowser{getResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"cursor":"","has_more":false,"notes":[]}}`)}}
collector := &Collector{Browser: browser, AccountKey: "u-1", HomepageURL: "https://www.xiaohongshu.com/user/profile/u-1?xsec_token=home-token&xsec_source=pc_search"}
if _, err := collector.ListWorks(context.Background(), "ignored", ""); err != nil {
t.Fatalf("list works: %v", err)
}
requestURL, err := url.Parse(browser.getURL)
if err != nil || requestURL.Query().Get("xsec_token") != "home-token" || requestURL.Query().Get("xsec_source") != "pc_search" {
t.Fatalf("homepage context was not forwarded: %s", browser.getURL)
}
}
func TestSourceURLParsingAndShareResolution(t *testing.T) {
homepage, err := ParseSourceURL("https://www.xiaohongshu.com/user/profile/u-1?xsec_source=pc_search")
if err != nil || homepage.Kind != LinkHomepage || homepage.AccountKey != "u-1" {
t.Fatalf("homepage = %#v, err = %v", homepage, err)
}
share, err := ParseSourceURL("https://xhslink.com/a/abc")
if err != nil || share.Kind != LinkShare {
t.Fatalf("share = %#v, err = %v", share, err)
}
browser := &fakeBrowser{resolvedURL: "https://www.xiaohongshu.com/explore/n-1?xsec_token=tok-1&xsec_source=pc_search", postResp: Response{Status: 200, Body: []byte(`{"success":true,"data":{"items":[{"note_id":"n-1","title":"详情","time":1710000000,"interact_info":{}}]}}`)}}
collector := &Collector{Browser: browser}
item, err := collector.GetNoteDetail(context.Background(), "https://xhslink.com/a/abc")
if err != nil || item.WorkKey != "n-1" || browser.resolveInput != "https://xhslink.com/a/abc" {
t.Fatalf("resolved detail = %#v, err = %v, input = %s", item, err, browser.resolveInput)
}
}
func TestResponseErrorClassifiesRejectedPlatformPayload(t *testing.T) {
if err := responseError(Response{Status: 200, Body: []byte(`{"success":false,"code":"LOGIN_REQUIRED","msg":"请重新登录"}`)}, "works"); !errors.Is(err, creator.ErrConflict) {
t.Fatalf("expected conflict, got %v", err)
}
if err := responseError(Response{Status: 200, Body: []byte(`{"success":false,"code":"RATE_LIMIT","msg":"请求过于频繁"}`)}, "works"); !errors.Is(err, creator.ErrUnavailable) {
t.Fatalf("expected unavailable, got %v", err)
}
}
func TestSetWorkContextRejectsWrongHost(t *testing.T) {
collector := &Collector{}
if err := collector.SetWorkContext("n-1", "https://evil.example/explore/n-1"); !errors.Is(err, creator.ErrInvalid) {
internal/xiaohongshu/detail.go
+9 -18
View File
@@ -4,8 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"net/url"
"strings"
"git.ipao.vip/rogee/creator-hub/internal/creator"
)
@@ -16,23 +14,16 @@ func (c *Collector) GetNoteDetail(ctx context.Context, originalURL string) (crea
if c == nil || c.Browser == nil {
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail request", creator.ErrInvalid)
}
parsed, err := url.Parse(originalURL)
if err != nil || parsed.Scheme != "https" || parsed.Host != "www.xiaohongshu.com" {
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid)
link, err := c.resolveSourceURL(ctx, originalURL)
if err != nil {
return creator.WorkInput{}, err
}
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
if len(parts) != 2 || parts[0] != "explore" || !keyPattern.MatchString(parts[1]) {
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL", creator.ErrInvalid)
}
access := accessContext{Token: parsed.Query().Get("xsec_token"), Source: parsed.Query().Get("xsec_source")}
if access.Source == "" {
access.Source = "pc_search"
}
if len(access.Token) > 2048 || !keyPattern.MatchString(access.Source) {
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu note URL context", creator.ErrInvalid)
if link.Kind != LinkNote {
return creator.WorkInput{}, fmt.Errorf("%w: xiaohongshu note URL is required", creator.ErrInvalid)
}
access := accessContext{Token: link.Token, Source: link.Source}
body, err := json.Marshal(map[string]any{
"source_note_id": parts[1],
"source_note_id": link.WorkKey,
"image_formats": []string{"jpg", "webp", "avif"},
"extra": map[string]string{"need_body_topic": "1"},
"xsec_source": access.Source,
@@ -52,10 +43,10 @@ func (c *Collector) GetNoteDetail(ctx context.Context, originalURL string) (crea
if !ok {
return creator.WorkInput{}, fmt.Errorf("%w: invalid xiaohongshu detail response", creator.ErrInvalid)
}
if item.WorkKey != parts[1] {
if item.WorkKey != link.WorkKey {
return creator.WorkInput{}, fmt.Errorf("%w: xiaohongshu detail returned another note", creator.ErrConflict)
}
item.OriginalURL = originalURL
item.OriginalURL = link.CanonicalURL
item.SourceType = c.SourceType
item.SourceID = c.SourceID
if item.SourceType == "" {
web/src/CreatorCompetitorsPage.jsx
+12 -6
View File
@@ -110,13 +110,19 @@ export function CreatorCompetitorsPage() {
const parseHomepage = () => {
try {
const parsed = new URL(form.homepage_url);
const allowed =
form.platform === "douyin"
? parsed.hostname === "www.douyin.com"
: parsed.hostname === "www.xiaohongshu.com" ||
parsed.hostname === "xiaohongshu.com";
const parts = parsed.pathname.split("/").filter(Boolean);
const candidate = parts.at(-1) || "";
const isXiaohongshu = form.platform === "xiaohongshu";
const allowed = isXiaohongshu
? parsed.protocol === "https:" &&
parsed.hostname === "www.xiaohongshu.com"
: parsed.hostname === "www.douyin.com";
let candidate = parts.at(-1) || "";
if (
isXiaohongshu &&
!(parts.length === 3 && parts[0] === "user" && parts[1] === "profile")
) {
candidate = "";
}
if (
!allowed ||
!candidate ||
web/src/CreatorPages.test.jsx
+30
View File
@@ -203,6 +203,36 @@ describe("creator pages", () => {
);
});
it("parses and confirms a Xiaohongshu profile URL", async () => {
const dataProvider = provider();
renderPage(<CreatorCompetitorsPage />, dataProvider);
fireEvent.click(screen.getAllByRole("combobox")[0]);
fireEvent.click(screen.getByRole("option", { name: "小红书" }));
fireEvent.change(screen.getByLabelText("主页 URL", { exact: false }), {
target: {
value: "https://www.xiaohongshu.com/user/profile/xhs-b",
},
});
fireEvent.click(screen.getByRole("button", { name: "解析链接预览" }));
await waitFor(() =>
expect(
screen.getAllByText("xhs-b", { exact: false }).length,
).toBeGreaterThan(0),
);
fireEvent.click(screen.getByRole("button", { name: "确认预览内容" }));
fireEvent.click(screen.getByRole("button", { name: "加入监测" }));
await waitFor(() => expect(dataProvider.create).toHaveBeenCalled());
expect(dataProvider.create).toHaveBeenCalledWith(
expect.objectContaining({
resource: "creator-competitors",
variables: expect.objectContaining({
platform: "xiaohongshu",
platform_account_key: "xhs-b",
}),
}),
);
});
it("keeps account password out of the returned profile and exposes big-account action", async () => {
const dataProvider = provider();
renderPage(<CreatorAccountsPage />, dataProvider);