rogee/creator-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: complete Xiaohongshu read-only flow

Browse Source
This commit is contained in:
rogee
2026-09-14 23:19:31 +08:00
parent 4c37d0c9bf
commit 5867aac20f
17 changed files with 857 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/control-plane/creator.go
+63 -3
View File
@@ -255,6 +255,9 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto
if err := decodeCreator(c, &input); err != nil {
return creatorError(c, err)
}
if err := validateXiaohongshuCompetitor(input); err != nil {
return creatorError(c, err)
}
item, err := store.CreateCompetitor(c.Context(), input)
if err != nil {
return creatorError(c, err)
@@ -296,6 +299,47 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto
return c.Status(fiber.StatusAccepted).JSON(report)
})
app.Post("/api/creator/xiaohongshu/search", func(c fiber.Ctx) error {
var input struct {
AccountID string `json:"account_id"`
Query string `json:"query"`
Page int `json:"page"`
}
if err := decodeCreator(c, &input); err != nil {
return creatorError(c, err)
}
if input.Page == 0 {
input.Page = 1
}
collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID)
if err != nil {
return creatorError(c, err)
}
page, err := collector.SearchNotes(c.Context(), input.Query, input.Page)
if err != nil {
return creatorError(c, err)
}
return c.JSON(page)
})
app.Post("/api/creator/xiaohongshu/detail", func(c fiber.Ctx) error {
var input struct {
AccountID string `json:"account_id"`
URL string `json:"url"`
}
if err := decodeCreator(c, &input); err != nil {
return creatorError(c, err)
}
collector, err := newXiaohongshuReadCollector(c.Context(), store, phaseAStore, hubStore, input.AccountID, creator.SourceOwned, input.AccountID)
if err != nil {
return creatorError(c, err)
}
item, err := collector.GetNoteDetail(c.Context(), input.URL)
if err != nil {
return creatorError(c, err)
}
return c.JSON(item)
})
app.Get("/api/creator/works", func(c fiber.Ctx) error {
filter, err := workFilter(c)
if err != nil {
@@ -1128,6 +1172,11 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p
if competitor.Platform != creator.PlatformDouyin && competitor.Platform != creator.PlatformXiaohongshu {
return blocked(fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, competitor.Platform))
}
if competitor.Platform == creator.PlatformXiaohongshu {
if err := validateXiaohongshuSource(competitor.HomepageURL, competitor.PlatformAccountKey); err != nil {
return blocked(err)
}
}
account, err := phaseAStore.GetAccount(ctx, accountID)
if err != nil {
return blocked(err)
@@ -1153,7 +1202,7 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p
if err != nil {
return blocked(fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err))
}
collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceCompetitor, competitor.ID)
collector, _, err := newCreatorCollector(ctx, competitor.Platform, gateway, environment, account.PlatformAccountKey, competitor.PlatformAccountKey, competitor.HomepageURL, creator.SourceCompetitor, competitor.ID)
if err != nil {
return blocked(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err))
}
@@ -1283,7 +1332,18 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS
if err != nil {
return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
}
collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, work.SourceType, work.SourceID)
targetAccountKey, homepageURL := account.PlatformAccountKey, ""
if work.SourceType == creator.SourceCompetitor {
competitor, competitorErr := store.GetCompetitor(ctx, work.SourceID)
if competitorErr != nil {
return competitorErr
}
if competitor.Platform != work.Platform {
return creator.ErrConflict
}
targetAccountKey, homepageURL = competitor.PlatformAccountKey, competitor.HomepageURL
}
collector, collectionKey, err := newCreatorCollector(ctx, work.Platform, gateway, environment, account.PlatformAccountKey, targetAccountKey, homepageURL, work.SourceType, work.SourceID)
if err != nil {
return fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)
}
@@ -1356,7 +1416,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph
logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed")
}
}()
collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, creator.SourceOwned, account.ID)
collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, account.PlatformAccountKey, "", creator.SourceOwned, account.ID)
if err != nil {
blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays)
return errors.Join(fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err), blockErr)
cmd/control-plane/xiaohongshu.go
+83 -11
View File
@@ -14,6 +14,7 @@ import (
"git.ipao.vip/rogee/creator-hub/internal/creator"
"git.ipao.vip/rogee/creator-hub/internal/douyin"
"git.ipao.vip/rogee/creator-hub/internal/hub"
"git.ipao.vip/rogee/creator-hub/internal/phasea"
"git.ipao.vip/rogee/creator-hub/internal/xiaohongshu"
)
@@ -94,6 +95,26 @@ func (browser xiaohongshuGatewayBrowser) Identity(ctx context.Context, expectedK
return identity.UID, nil
}
func (browser xiaohongshuGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) {
request, err := browser.generation()
if err != nil {
return "", err
}
request["url"] = target
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/xiaohongshu/resolve", request, 30*time.Second)
if err != nil || status != http.StatusOK {
return "", errors.New("restricted Xiaohongshu share resolution failed")
}
var response struct {
URL string `json:"url"`
}
if err := json.Unmarshal(body, &response); err != nil || strings.TrimSpace(response.URL) == "" {
return "", errors.New("Xiaohongshu share resolution response omitted url")
}
return response.URL, nil
}
func (browser xiaohongshuGatewayBrowser) Media(ctx context.Context, target string) ([]byte, string, error) {
request, err := browser.generation()
if err != nil {
@@ -132,28 +153,79 @@ func decodeXiaohongshuResponse(body []byte) (xiaohongshu.Response, error) {
return xiaohongshu.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil
}
func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, accountKey, sourceType, sourceID string) (creator.PlatformCollector, string, error) {
func newXiaohongshuReadCollector(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, sourceType, sourceID string) (*xiaohongshu.Collector, error) {
if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" {
return nil, creator.ErrUnavailable
}
account, err := phaseAStore.GetAccount(ctx, accountID)
if err != nil {
return nil, err
}
if account.Platform != creator.PlatformXiaohongshu || account.AuthorizationStatus != "authorized" {
return nil, creator.ErrConflict
}
profile, err := store.GetAccountProfile(ctx, accountID)
if err != nil {
return nil, err
}
if profile.Platform != creator.PlatformXiaohongshu || profile.BusinessStatus != "normal" || profile.LoginStatus != "logged_in" {
return nil, creator.ErrConflict
}
environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID)
if err != nil {
return nil, fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err)
}
if environment.RuntimeID == "" || environment.RuntimeNetworkID == "" || environment.BindingVersion <= 0 {
return nil, fmt.Errorf("%w: account runtime is not running", creator.ErrUnavailable)
}
gateway, err := hubStore.GetGateway(ctx, environment.Gateway)
if err != nil {
return nil, fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err)
}
browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}
if _, err := browser.Identity(ctx, account.PlatformAccountKey); err != nil {
return nil, fmt.Errorf("%w: account identity verification failed: %v", creator.ErrConflict, err)
}
return &xiaohongshu.Collector{Browser: browser, AccountKey: account.PlatformAccountKey, SourceType: sourceType, SourceID: sourceID}, nil
}
func validateXiaohongshuSource(homepageURL, accountKey string) error {
return xiaohongshu.ValidateSourceURL(homepageURL, accountKey)
}
func validateXiaohongshuCompetitor(input creator.CompetitorInput) error {
if input.Platform != creator.PlatformXiaohongshu {
return nil
}
return validateXiaohongshuSource(input.HomepageURL, input.PlatformAccountKey)
}
func newCreatorCollector(ctx context.Context, platform string, gateway hub.Gateway, environment hub.EnvironmentContext, viewerAccountKey, targetAccountKey, homepageURL, sourceType, sourceID string) (creator.PlatformCollector, string, error) {
if strings.TrimSpace(viewerAccountKey) == "" || strings.TrimSpace(targetAccountKey) == "" {
return nil, "", fmt.Errorf("%w: creator collector account key is missing", creator.ErrInvalid)
}
switch platform {
case creator.PlatformDouyin:
browser := creatorGatewayBrowser{gateway: gateway, environment: environment}
uid, err := browser.Identity(ctx, accountKey)
if err != nil {
if _, err := browser.Identity(ctx, viewerAccountKey); err != nil {
return nil, "", err
}
collector := douyinCollector(browser, accountKey, sourceType, sourceID)
canonical, err := collector.CanonicalSecUID(ctx, uid)
if err != nil {
collector := douyinCollector(browser, targetAccountKey, sourceType, sourceID)
if _, err := collector.CanonicalSecUID(ctx, viewerAccountKey); err != nil {
return nil, "", err
}
collector.AccountKey = canonical
return &collector, canonical, nil
return &collector, targetAccountKey, nil
case creator.PlatformXiaohongshu:
if homepageURL != "" {
if err := xiaohongshu.ValidateSourceURL(homepageURL, targetAccountKey); err != nil {
return nil, "", err
}
}
browser := xiaohongshuGatewayBrowser{gateway: gateway, environment: environment}
uid, err := browser.Identity(ctx, accountKey)
if err != nil {
if _, err := browser.Identity(ctx, viewerAccountKey); err != nil {
return nil, "", err
}
return &xiaohongshu.Collector{Browser: browser, AccountKey: uid, SourceType: sourceType, SourceID: sourceID}, uid, nil
return &xiaohongshu.Collector{Browser: browser, AccountKey: targetAccountKey, HomepageURL: homepageURL, SourceType: sourceType, SourceID: sourceID}, targetAccountKey, nil
default:
return nil, "", fmt.Errorf("%w: unsupported creator platform %s", creator.ErrUnavailable, platform)
}
cmd/control-plane/xiaohongshu_test.go
+35
View File
@@ -7,7 +7,9 @@ import (
"net/http/httptest"
"testing"
"git.ipao.vip/rogee/creator-hub/internal/creator"
"git.ipao.vip/rogee/creator-hub/internal/hub"
"git.ipao.vip/rogee/creator-hub/internal/xiaohongshu"
)
const testXiaohongshuIdentityURL = "https://edith.xiaohongshu.com/api/sns/web/v2/user/me"
@@ -34,6 +36,39 @@ func TestXiaohongshuGatewayBrowserFencesAccountGeneration(t *testing.T) {
}
}
func TestXiaohongshuGatewayBrowserResolvesShareLinks(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/resolve" {
t.Fatalf("unexpected path: %s", request.URL.Path)
}
_ = json.NewEncoder(response).Encode(map[string]any{"url": "https://www.xiaohongshu.com/explore/n-1"})
}))
defer server.Close()
browser := xiaohongshuGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()}
resolved, err := browser.Resolve(context.Background(), "https://xhslink.com/a/abc")
if err != nil || resolved != "https://www.xiaohongshu.com/explore/n-1" {
t.Fatalf("resolved URL=%q err=%v", resolved, err)
}
}
func TestNewXiaohongshuCollectorKeepsViewerAndTargetSeparate(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/v1/browsers/account-a/xiaohongshu/identity" {
t.Fatalf("unexpected path: %s", request.URL.Path)
}
_ = json.NewEncoder(response).Encode(map[string]any{"uid": "viewer-1"})
}))
defer server.Close()
collector, target, err := newCreatorCollector(context.Background(), creator.PlatformXiaohongshu, hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, readyDouyinEnvironment(), "viewer-1", "target-1", "https://www.xiaohongshu.com/user/profile/target-1?xsec_source=pc_search", creator.SourceCompetitor, "source-1")
if err != nil {
t.Fatalf("new collector: %v", err)
}
xhsCollector, ok := collector.(*xiaohongshu.Collector)
if !ok || xhsCollector.AccountKey != "target-1" || target != "target-1" || xhsCollector.HomepageURL == "" {
t.Fatalf("collector=%#v target=%q", collector, target)
}
}
func TestXiaohongshuGatewayBrowserPostCarriesJSONBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
var body map[string]any
cmd/docker_gateway/douyin.py
+91 -6
View File
@@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from datetime import datetime, timezone
from urllib.parse import urlsplit
from urllib.parse import parse_qs, urlsplit
import websocket
@@ -229,6 +229,7 @@ class DouyinBrowser:
origin: str = ORIGIN,
url_validator: Callable[[object], bool] | None = None,
media_validator: Callable[[object], bool] | None = None,
media_selector: str = "video",
) -> None:
self.endpoint = endpoint or (
lambda alias: f"http://creatorhub-browser-{alias}:9222"
@@ -236,6 +237,7 @@ class DouyinBrowser:
self.origin = origin
self.url_validator = url_validator or is_douyin_url
self.media_validator = media_validator or is_douyin_media_url
self.media_selector = media_selector
@contextmanager
def connection(self, alias: str):
@@ -396,8 +398,8 @@ class DouyinBrowser:
raise DouyinError("Douyin media page did not load")
result = cdp.evaluate(
f"""(async()=>{{
const video=document.querySelector('video');
const source=video?.currentSrc||video?.src||'';
const media=document.querySelector({json.dumps(self.media_selector)});
const source=media?.currentSrc||media?.src||'';
if(!source)return {{error:'media_source_unavailable'}};
const r=await fetch(source,{{credentials:'include',redirect:'error'}});
if(!r.body)return {{status:r.status,content_type:r.headers.get('content-type')||'',body:''}};
@@ -1289,7 +1291,9 @@ XHS_ORIGIN = "https://www.xiaohongshu.com"
XHS_API_ORIGIN = "https://edith.xiaohongshu.com"
XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com"
XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me"
XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"})
XHS_ALLOWED_HOSTS = frozenset(
{"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"}
)
class XiaohongshuBrowser(DouyinBrowser):
@@ -1299,6 +1303,7 @@ class XiaohongshuBrowser(DouyinBrowser):
origin=XHS_ORIGIN,
url_validator=is_xiaohongshu_url,
media_validator=is_xiaohongshu_media_url,
media_selector="video, img.note-slider-img",
)
def post(self, alias: str, target: str, body: bytes) -> BrowserResponse:
@@ -1334,7 +1339,29 @@ class XiaohongshuBrowser(DouyinBrowser):
response_body = result.get("body")
if not isinstance(response_body, str):
raise DouyinError("restricted Xiaohongshu POST returned invalid body")
return BrowserResponse(status, response_body, detect_challenge(status, response_body))
return BrowserResponse(
status, response_body, detect_challenge(status, response_body)
)
def resolve(self, alias: str, target: str) -> str:
if not is_xiaohongshu_share_url(target):
raise DouyinError("restricted Xiaohongshu share URL is invalid")
with self.connection(alias) as cdp:
if cdp.evaluate("location.origin") != self.origin:
raise DouyinError("restricted browser origin changed")
cdp.command("Page.navigate", {"url": target})
event = cdp.wait_event(
"Page.frameNavigated",
lambda params: _is_xiaohongshu_page_url(
params.get("frame", {}).get("url", "")
),
)
final_url = event.get("frame", {}).get("url")
if not isinstance(final_url, str) or not _is_xiaohongshu_page_url(final_url):
raise DouyinError(
"Xiaohongshu share URL did not resolve to a supported page"
)
return final_url
def identity(self, alias: str, expected_uid: str | None = None) -> dict:
response = self.get(alias, XHS_IDENTITY_URL)
@@ -1362,7 +1389,9 @@ class XiaohongshuBrowser(DouyinBrowser):
):
raise DouyinError("Xiaohongshu login is not valid")
if expected_uid and user_id != expected_uid:
raise DouyinError("Xiaohongshu identity does not match the expected account")
raise DouyinError(
"Xiaohongshu identity does not match the expected account"
)
return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""}
@@ -1403,6 +1432,62 @@ def is_xiaohongshu_url(value: object) -> bool:
)
def is_xiaohongshu_share_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
port = parsed.port
except (TypeError, ValueError):
return False
path = parsed.path.strip("/")
return (
parsed.scheme == "https"
and parsed.hostname in {"xhslink.com", "www.xhslink.com"}
and port is None
and parsed.username is None
and parsed.password is None
and parsed.fragment == ""
and bool(path)
and len(path) <= 256
and not parsed.query
)
def _is_xiaohongshu_page_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
port = parsed.port
query = parse_qs(parsed.query, keep_blank_values=True)
except (TypeError, ValueError):
return False
parts = parsed.path.strip("/").split("/")
if not (
parsed.scheme == "https"
and parsed.hostname == "www.xiaohongshu.com"
and port is None
and parsed.username is None
and parsed.password is None
and parsed.fragment == ""
and (
len(parts) == 2
and parts[0] == "explore"
or len(parts) == 3
and parts[:2] == ["user", "profile"]
)
):
return False
return all(
key in {"xsec_token", "xsec_source"}
and len(values) == 1
and len(values[0]) <= 2048
and not any(char in values[0] for char in "\r\n")
for key, values in query.items()
)
def notice_ids(event: dict) -> list[str]:
try:
payload = json.loads(event["payload"])
cmd/docker_gateway/gateway.py
+158 -26
View File
@@ -47,6 +47,7 @@ from .douyin import (
DouyinError,
SubscriptionManager,
XiaohongshuBrowser,
is_xiaohongshu_share_url,
)
from .proxy import ProxyExit, ProxyRegistry
@@ -99,7 +100,9 @@ class Gateway:
self.token = token
self.self_name = self_name
self.browser = browser or DouyinBrowser(self._browser_endpoint)
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint)
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(
self._browser_endpoint
)
self.proxies = ProxyRegistry()
self.reservations = AliasReservationManager(docker, self_name)
self.subscriptions = SubscriptionManager(self.browser)
@@ -685,9 +688,15 @@ class Gateway:
response = self.xiaohongshu_browser.get(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc))
LOG.warning(
"Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc)
)
raise RequestError("restricted Xiaohongshu operation failed") from exc
return {"status": response.status, "body": response.body, "challenge": response.challenge}
return {
"status": response.status,
"body": response.body,
"challenge": response.challenge,
}
def post_xiaohongshu(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
@@ -699,7 +708,9 @@ class Gateway:
):
raise RequestError("invalid restricted Xiaohongshu POST request", 400)
try:
encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
encoded = json.dumps(
body, ensure_ascii=False, separators=(",", ":")
).encode()
except (TypeError, ValueError) as exc:
raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc
with self._alias_lock(alias):
@@ -708,13 +719,47 @@ class Gateway:
response = self.xiaohongshu_browser.post(alias, target, encoded)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc))
LOG.warning(
"Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc)
)
raise RequestError("restricted Xiaohongshu operation failed") from exc
return {"status": response.status, "body": response.body, "challenge": response.challenge}
return {
"status": response.status,
"body": response.body,
"challenge": response.challenge,
}
def resolve_xiaohongshu(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_source_url(
target
):
raise RequestError("invalid restricted Xiaohongshu source URL", 400)
if not is_xiaohongshu_share_url(target):
return {"url": target}
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
try:
resolved = self.xiaohongshu_browser.resolve(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning(
"Xiaohongshu share resolution failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"restricted Xiaohongshu share resolution failed"
) from exc
if not valid_xiaohongshu_page_url(resolved):
raise RequestError("Xiaohongshu share resolved to an unsupported URL", 502)
return {"url": resolved}
def get_xiaohongshu_media(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target):
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(
target
):
raise RequestError("invalid restricted Xiaohongshu media request", 400)
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
@@ -722,9 +767,19 @@ class Gateway:
response = self.xiaohongshu_browser.get_media(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc))
raise RequestError("restricted Xiaohongshu media download failed") from exc
return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64}
LOG.warning(
"Xiaohongshu media download failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"restricted Xiaohongshu media download failed"
) from exc
return {
"status": response.status,
"content_type": response.content_type,
"body_base64": response.body_base64,
}
def xiaohongshu_identity(self, alias: str, input: dict) -> dict:
expected_account_key = input.get("expected_account_key", "")
@@ -739,10 +794,18 @@ class Gateway:
try:
identity = self.xiaohongshu_browser.identity(alias)
except DouyinError as exc:
LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc))
raise RequestError("Xiaohongshu login identity could not be verified") from exc
LOG.warning(
"Xiaohongshu identity verification failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"Xiaohongshu login identity could not be verified"
) from exc
if identity.get("uid") != expected_account_key:
raise RequestError("Xiaohongshu identity does not match the expected account", 409)
raise RequestError(
"Xiaohongshu identity does not match the expected account", 409
)
return identity
def douyin_action(self, alias: str, input: dict) -> dict:
@@ -1210,7 +1273,7 @@ class GatewayHandler(BaseHTTPRequestHandler):
gateway.restore_proxy(alias, body)
return None
match = re.fullmatch(
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)",
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity|resolve)",
path,
)
if match:
@@ -1221,6 +1284,8 @@ class GatewayHandler(BaseHTTPRequestHandler):
return gateway.post_xiaohongshu(alias, body)
if action == "media" and method == "POST":
return gateway.get_xiaohongshu_media(alias, body)
if action == "resolve" and method == "POST":
return gateway.resolve_xiaohongshu(alias, body)
if action == "identity" and method == "POST":
return gateway.xiaohongshu_identity(alias, body)
match = re.fullmatch(
@@ -1516,7 +1581,9 @@ def valid_xiaohongshu_generation(value: dict) -> bool:
return valid_douyin_generation(value)
def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool:
def valid_xhs_query(
query: object, allowed: set[str], required: set[str] | None = None
) -> bool:
if not isinstance(query, dict) or not isinstance(allowed, set):
return False
required = required or set()
@@ -1525,7 +1592,12 @@ def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None
for key, values in query.items():
if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1:
return False
if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]:
if (
not isinstance(values[0], str)
or len(values[0]) > 2048
or "\r" in values[0]
or "\n" in values[0]
):
return False
return True
@@ -1549,18 +1621,45 @@ def valid_xhs_url(raw: object) -> bool:
query = parse_qs(parsed.query, keep_blank_values=True)
except ValueError:
return False
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH:
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_IDENTITY_PATH
):
return not query
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH:
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_USER_POSTED_PATH
):
return (
valid_xhs_query(
query,
{
"user_id",
"cursor",
"num",
"image_formats",
"xsec_source",
"xsec_token",
},
{"user_id", "num"},
)
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0]))
and query["num"] == ["30"]
)
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_COMMENTS_PATH
):
return valid_xhs_query(
query,
{"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"},
{"user_id", "num"},
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"]
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH:
return valid_xhs_query(
query,
{"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"},
{
"note_id",
"cursor",
"top_comment_id",
"image_formats",
"xsec_source",
"xsec_token",
},
{"note_id", "cursor", "top_comment_id"},
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0]))
return False
@@ -1570,6 +1669,34 @@ def valid_xiaohongshu_url(raw: object) -> bool:
return valid_xhs_url(raw)
def valid_xiaohongshu_page_url(raw: object) -> bool:
if not isinstance(raw, str):
return False
try:
parsed = urlsplit(raw)
query = parse_qs(parsed.query, keep_blank_values=True)
except ValueError:
return False
if not _valid_xhs_host(parsed, "www.xiaohongshu.com") or not valid_xhs_query(
query, {"xsec_source", "xsec_token"}
):
return False
parts = parsed.path.strip("/").split("/")
return (
len(parts) == 2
and parts[0] == "explore"
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1]))
) or (
len(parts) == 3
and parts[:2] == ["user", "profile"]
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[2]))
)
def valid_xiaohongshu_source_url(raw: object) -> bool:
return valid_xiaohongshu_page_url(raw) or is_xiaohongshu_share_url(raw)
def valid_xhs_post_url(raw: object) -> bool:
if not isinstance(raw, str):
return False
@@ -1600,7 +1727,12 @@ def valid_xiaohongshu_media_url(raw: object) -> bool:
if not _valid_xhs_host(parsed, "www.xiaohongshu.com"):
return False
parts = parsed.path.strip("/").split("/")
return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"})
return (
len(parts) == 2
and parts[0] == "explore"
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1]))
and valid_xhs_query(query, {"xsec_source", "xsec_token"})
)
def valid_douyin_url(raw: object) -> bool:
cmd/docker_gateway/test_xiaohongshu.py
+77 -14
View File
@@ -13,31 +13,70 @@ valid_xhs_url = gateway_module.valid_xhs_url
valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url
valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url
valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation
valid_xiaohongshu_page_url = gateway_module.valid_xiaohongshu_page_url
valid_xiaohongshu_source_url = gateway_module.valid_xiaohongshu_source_url
class XiaohongshuValidationTests(unittest.TestCase):
def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None:
self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
self.assertTrue(
valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")
)
self.assertTrue(
valid_xiaohongshu_url(
"https://edith.xiaohongshu.com/api/sns/web/v2/user/me"
)
)
self.assertTrue(
valid_xiaohongshu_page_url("https://www.xiaohongshu.com/user/profile/u-1")
)
self.assertTrue(valid_xiaohongshu_source_url("https://xhslink.com/a/abc"))
self.assertTrue(
valid_xhs_url(
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user"
)
)
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"))
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me"))
self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes"))
self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed"))
self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"))
self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment"))
self.assertFalse(
valid_xhs_url(
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"
)
)
self.assertFalse(
valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me")
)
self.assertTrue(
valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes")
)
self.assertTrue(
valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed")
)
self.assertTrue(
valid_xiaohongshu_media_url(
"https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"
)
)
self.assertFalse(
valid_xiaohongshu_media_url(
"https://www.xiaohongshu.com/explore/n-1#fragment"
)
)
def test_generation_shape_matches_existing_browser_fence(self) -> None:
self.assertTrue(
valid_xiaohongshu_generation(
{"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""}
{
"binding_version": 1,
"runtime_id": "a" * 64,
"network_id": "network",
"network_exit_id": "",
}
)
)
self.assertFalse(
valid_xiaohongshu_generation(
{"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}
)
)
self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}))
class XiaohongshuRouteTests(unittest.TestCase):
@@ -48,18 +87,42 @@ class XiaohongshuRouteTests(unittest.TestCase):
gateway.post_xiaohongshu.return_value = {"status": 200}
gateway.get_xiaohongshu_media.return_value = {"status": 200}
gateway.xiaohongshu_identity.return_value = {"uid": "u-1"}
gateway.resolve_xiaohongshu.return_value = {
"url": "https://www.xiaohongshu.com/explore/n-1"
}
server = Mock()
server.gateway = gateway
cast(Any, handler).server = server
cast(Any, handler).server_as_gateway = lambda: server
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"})
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route(
"POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}
),
{"uid": "u-1"},
)
self.assertEqual(
handler._route(
"POST", "/v1/browsers/account-a/xiaohongshu/resolve", {}, {}
),
{"url": "https://www.xiaohongshu.com/explore/n-1"},
)
with self.assertRaises(RequestError):
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {})
gateway.get_xiaohongshu.assert_called_once_with("account-a", {})
gateway.post_xiaohongshu.assert_called_once_with("account-a", {})
gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {})
gateway.xiaohongshu_identity.assert_called_once_with("account-a", {})
gateway.resolve_xiaohongshu.assert_called_once_with("account-a", {})
cmd/docker_gateway/xiaohongshu.py
+2
View File
@@ -12,6 +12,7 @@ from .douyin import (
XHS_SEARCH_ORIGIN,
XiaohongshuBrowser,
is_xiaohongshu_media_url,
is_xiaohongshu_share_url,
is_xiaohongshu_url,
)
@@ -23,5 +24,6 @@ __all__ = [
"XHS_SEARCH_ORIGIN",
"XiaohongshuBrowser",
"is_xiaohongshu_media_url",
"is_xiaohongshu_share_url",
"is_xiaohongshu_url",
]