rogee/creator-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: complete Xiaohongshu read-only flow

Browse Source
This commit is contained in:
rogee
2026-09-14 23:19:31 +08:00
parent 4c37d0c9bf
commit 5867aac20f
17 changed files with 857 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/docker_gateway/douyin.py
+91 -6
View File
@@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from datetime import datetime, timezone
from urllib.parse import urlsplit
from urllib.parse import parse_qs, urlsplit
import websocket
@@ -229,6 +229,7 @@ class DouyinBrowser:
origin: str = ORIGIN,
url_validator: Callable[[object], bool] | None = None,
media_validator: Callable[[object], bool] | None = None,
media_selector: str = "video",
) -> None:
self.endpoint = endpoint or (
lambda alias: f"http://creatorhub-browser-{alias}:9222"
@@ -236,6 +237,7 @@ class DouyinBrowser:
self.origin = origin
self.url_validator = url_validator or is_douyin_url
self.media_validator = media_validator or is_douyin_media_url
self.media_selector = media_selector
@contextmanager
def connection(self, alias: str):
@@ -396,8 +398,8 @@ class DouyinBrowser:
raise DouyinError("Douyin media page did not load")
result = cdp.evaluate(
f"""(async()=>{{
const video=document.querySelector('video');
const source=video?.currentSrc||video?.src||'';
const media=document.querySelector({json.dumps(self.media_selector)});
const source=media?.currentSrc||media?.src||'';
if(!source)return {{error:'media_source_unavailable'}};
const r=await fetch(source,{{credentials:'include',redirect:'error'}});
if(!r.body)return {{status:r.status,content_type:r.headers.get('content-type')||'',body:''}};
@@ -1289,7 +1291,9 @@ XHS_ORIGIN = "https://www.xiaohongshu.com"
XHS_API_ORIGIN = "https://edith.xiaohongshu.com"
XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com"
XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me"
XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"})
XHS_ALLOWED_HOSTS = frozenset(
{"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"}
)
class XiaohongshuBrowser(DouyinBrowser):
@@ -1299,6 +1303,7 @@ class XiaohongshuBrowser(DouyinBrowser):
origin=XHS_ORIGIN,
url_validator=is_xiaohongshu_url,
media_validator=is_xiaohongshu_media_url,
media_selector="video, img.note-slider-img",
)
def post(self, alias: str, target: str, body: bytes) -> BrowserResponse:
@@ -1334,7 +1339,29 @@ class XiaohongshuBrowser(DouyinBrowser):
response_body = result.get("body")
if not isinstance(response_body, str):
raise DouyinError("restricted Xiaohongshu POST returned invalid body")
return BrowserResponse(status, response_body, detect_challenge(status, response_body))
return BrowserResponse(
status, response_body, detect_challenge(status, response_body)
)
def resolve(self, alias: str, target: str) -> str:
if not is_xiaohongshu_share_url(target):
raise DouyinError("restricted Xiaohongshu share URL is invalid")
with self.connection(alias) as cdp:
if cdp.evaluate("location.origin") != self.origin:
raise DouyinError("restricted browser origin changed")
cdp.command("Page.navigate", {"url": target})
event = cdp.wait_event(
"Page.frameNavigated",
lambda params: _is_xiaohongshu_page_url(
params.get("frame", {}).get("url", "")
),
)
final_url = event.get("frame", {}).get("url")
if not isinstance(final_url, str) or not _is_xiaohongshu_page_url(final_url):
raise DouyinError(
"Xiaohongshu share URL did not resolve to a supported page"
)
return final_url
def identity(self, alias: str, expected_uid: str | None = None) -> dict:
response = self.get(alias, XHS_IDENTITY_URL)
@@ -1362,7 +1389,9 @@ class XiaohongshuBrowser(DouyinBrowser):
):
raise DouyinError("Xiaohongshu login is not valid")
if expected_uid and user_id != expected_uid:
raise DouyinError("Xiaohongshu identity does not match the expected account")
raise DouyinError(
"Xiaohongshu identity does not match the expected account"
)
return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""}
@@ -1403,6 +1432,62 @@ def is_xiaohongshu_url(value: object) -> bool:
)
def is_xiaohongshu_share_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
port = parsed.port
except (TypeError, ValueError):
return False
path = parsed.path.strip("/")
return (
parsed.scheme == "https"
and parsed.hostname in {"xhslink.com", "www.xhslink.com"}
and port is None
and parsed.username is None
and parsed.password is None
and parsed.fragment == ""
and bool(path)
and len(path) <= 256
and not parsed.query
)
def _is_xiaohongshu_page_url(value: object) -> bool:
if not isinstance(value, str):
return False
try:
parsed = urlsplit(value)
port = parsed.port
query = parse_qs(parsed.query, keep_blank_values=True)
except (TypeError, ValueError):
return False
parts = parsed.path.strip("/").split("/")
if not (
parsed.scheme == "https"
and parsed.hostname == "www.xiaohongshu.com"
and port is None
and parsed.username is None
and parsed.password is None
and parsed.fragment == ""
and (
len(parts) == 2
and parts[0] == "explore"
or len(parts) == 3
and parts[:2] == ["user", "profile"]
)
):
return False
return all(
key in {"xsec_token", "xsec_source"}
and len(values) == 1
and len(values[0]) <= 2048
and not any(char in values[0] for char in "\r\n")
for key, values in query.items()
)
def notice_ids(event: dict) -> list[str]:
try:
payload = json.loads(event["payload"])
cmd/docker_gateway/gateway.py
+158 -26
View File
@@ -47,6 +47,7 @@ from .douyin import (
DouyinError,
SubscriptionManager,
XiaohongshuBrowser,
is_xiaohongshu_share_url,
)
from .proxy import ProxyExit, ProxyRegistry
@@ -99,7 +100,9 @@ class Gateway:
self.token = token
self.self_name = self_name
self.browser = browser or DouyinBrowser(self._browser_endpoint)
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint)
self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(
self._browser_endpoint
)
self.proxies = ProxyRegistry()
self.reservations = AliasReservationManager(docker, self_name)
self.subscriptions = SubscriptionManager(self.browser)
@@ -685,9 +688,15 @@ class Gateway:
response = self.xiaohongshu_browser.get(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc))
LOG.warning(
"Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc)
)
raise RequestError("restricted Xiaohongshu operation failed") from exc
return {"status": response.status, "body": response.body, "challenge": response.challenge}
return {
"status": response.status,
"body": response.body,
"challenge": response.challenge,
}
def post_xiaohongshu(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
@@ -699,7 +708,9 @@ class Gateway:
):
raise RequestError("invalid restricted Xiaohongshu POST request", 400)
try:
encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
encoded = json.dumps(
body, ensure_ascii=False, separators=(",", ":")
).encode()
except (TypeError, ValueError) as exc:
raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc
with self._alias_lock(alias):
@@ -708,13 +719,47 @@ class Gateway:
response = self.xiaohongshu_browser.post(alias, target, encoded)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc))
LOG.warning(
"Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc)
)
raise RequestError("restricted Xiaohongshu operation failed") from exc
return {"status": response.status, "body": response.body, "challenge": response.challenge}
return {
"status": response.status,
"body": response.body,
"challenge": response.challenge,
}
def resolve_xiaohongshu(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_source_url(
target
):
raise RequestError("invalid restricted Xiaohongshu source URL", 400)
if not is_xiaohongshu_share_url(target):
return {"url": target}
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
try:
resolved = self.xiaohongshu_browser.resolve(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning(
"Xiaohongshu share resolution failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"restricted Xiaohongshu share resolution failed"
) from exc
if not valid_xiaohongshu_page_url(resolved):
raise RequestError("Xiaohongshu share resolved to an unsupported URL", 502)
return {"url": resolved}
def get_xiaohongshu_media(self, alias: str, input: dict) -> dict:
target = input.get("url", "")
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target):
if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(
target
):
raise RequestError("invalid restricted Xiaohongshu media request", 400)
with self._alias_lock(alias):
self._require_douyin_generation(alias, input)
@@ -722,9 +767,19 @@ class Gateway:
response = self.xiaohongshu_browser.get_media(alias, target)
self._require_douyin_generation(alias, input)
except DouyinError as exc:
LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc))
raise RequestError("restricted Xiaohongshu media download failed") from exc
return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64}
LOG.warning(
"Xiaohongshu media download failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"restricted Xiaohongshu media download failed"
) from exc
return {
"status": response.status,
"content_type": response.content_type,
"body_base64": response.body_base64,
}
def xiaohongshu_identity(self, alias: str, input: dict) -> dict:
expected_account_key = input.get("expected_account_key", "")
@@ -739,10 +794,18 @@ class Gateway:
try:
identity = self.xiaohongshu_browser.identity(alias)
except DouyinError as exc:
LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc))
raise RequestError("Xiaohongshu login identity could not be verified") from exc
LOG.warning(
"Xiaohongshu identity verification failed alias=%s reason=%s",
alias,
str(exc),
)
raise RequestError(
"Xiaohongshu login identity could not be verified"
) from exc
if identity.get("uid") != expected_account_key:
raise RequestError("Xiaohongshu identity does not match the expected account", 409)
raise RequestError(
"Xiaohongshu identity does not match the expected account", 409
)
return identity
def douyin_action(self, alias: str, input: dict) -> dict:
@@ -1210,7 +1273,7 @@ class GatewayHandler(BaseHTTPRequestHandler):
gateway.restore_proxy(alias, body)
return None
match = re.fullmatch(
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)",
r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity|resolve)",
path,
)
if match:
@@ -1221,6 +1284,8 @@ class GatewayHandler(BaseHTTPRequestHandler):
return gateway.post_xiaohongshu(alias, body)
if action == "media" and method == "POST":
return gateway.get_xiaohongshu_media(alias, body)
if action == "resolve" and method == "POST":
return gateway.resolve_xiaohongshu(alias, body)
if action == "identity" and method == "POST":
return gateway.xiaohongshu_identity(alias, body)
match = re.fullmatch(
@@ -1516,7 +1581,9 @@ def valid_xiaohongshu_generation(value: dict) -> bool:
return valid_douyin_generation(value)
def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool:
def valid_xhs_query(
query: object, allowed: set[str], required: set[str] | None = None
) -> bool:
if not isinstance(query, dict) or not isinstance(allowed, set):
return False
required = required or set()
@@ -1525,7 +1592,12 @@ def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None
for key, values in query.items():
if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1:
return False
if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]:
if (
not isinstance(values[0], str)
or len(values[0]) > 2048
or "\r" in values[0]
or "\n" in values[0]
):
return False
return True
@@ -1549,18 +1621,45 @@ def valid_xhs_url(raw: object) -> bool:
query = parse_qs(parsed.query, keep_blank_values=True)
except ValueError:
return False
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH:
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_IDENTITY_PATH
):
return not query
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH:
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_USER_POSTED_PATH
):
return (
valid_xhs_query(
query,
{
"user_id",
"cursor",
"num",
"image_formats",
"xsec_source",
"xsec_token",
},
{"user_id", "num"},
)
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0]))
and query["num"] == ["30"]
)
if (
_valid_xhs_host(parsed, "edith.xiaohongshu.com")
and parsed.path == XHS_COMMENTS_PATH
):
return valid_xhs_query(
query,
{"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"},
{"user_id", "num"},
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"]
if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH:
return valid_xhs_query(
query,
{"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"},
{
"note_id",
"cursor",
"top_comment_id",
"image_formats",
"xsec_source",
"xsec_token",
},
{"note_id", "cursor", "top_comment_id"},
) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0]))
return False
@@ -1570,6 +1669,34 @@ def valid_xiaohongshu_url(raw: object) -> bool:
return valid_xhs_url(raw)
def valid_xiaohongshu_page_url(raw: object) -> bool:
if not isinstance(raw, str):
return False
try:
parsed = urlsplit(raw)
query = parse_qs(parsed.query, keep_blank_values=True)
except ValueError:
return False
if not _valid_xhs_host(parsed, "www.xiaohongshu.com") or not valid_xhs_query(
query, {"xsec_source", "xsec_token"}
):
return False
parts = parsed.path.strip("/").split("/")
return (
len(parts) == 2
and parts[0] == "explore"
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1]))
) or (
len(parts) == 3
and parts[:2] == ["user", "profile"]
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[2]))
)
def valid_xiaohongshu_source_url(raw: object) -> bool:
return valid_xiaohongshu_page_url(raw) or is_xiaohongshu_share_url(raw)
def valid_xhs_post_url(raw: object) -> bool:
if not isinstance(raw, str):
return False
@@ -1600,7 +1727,12 @@ def valid_xiaohongshu_media_url(raw: object) -> bool:
if not _valid_xhs_host(parsed, "www.xiaohongshu.com"):
return False
parts = parsed.path.strip("/").split("/")
return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"})
return (
len(parts) == 2
and parts[0] == "explore"
and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1]))
and valid_xhs_query(query, {"xsec_source", "xsec_token"})
)
def valid_douyin_url(raw: object) -> bool:
cmd/docker_gateway/test_xiaohongshu.py
+77 -14
View File
@@ -13,31 +13,70 @@ valid_xhs_url = gateway_module.valid_xhs_url
valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url
valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url
valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation
valid_xiaohongshu_page_url = gateway_module.valid_xiaohongshu_page_url
valid_xiaohongshu_source_url = gateway_module.valid_xiaohongshu_source_url
class XiaohongshuValidationTests(unittest.TestCase):
def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None:
self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
self.assertTrue(
valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me")
)
self.assertTrue(
valid_xiaohongshu_url(
"https://edith.xiaohongshu.com/api/sns/web/v2/user/me"
)
)
self.assertTrue(
valid_xiaohongshu_page_url("https://www.xiaohongshu.com/user/profile/u-1")
)
self.assertTrue(valid_xiaohongshu_source_url("https://xhslink.com/a/abc"))
self.assertTrue(
valid_xhs_url(
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user"
)
)
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"))
self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me"))
self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes"))
self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed"))
self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"))
self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment"))
self.assertFalse(
valid_xhs_url(
"https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"
)
)
self.assertFalse(
valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me")
)
self.assertTrue(
valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes")
)
self.assertTrue(
valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed")
)
self.assertTrue(
valid_xiaohongshu_media_url(
"https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"
)
)
self.assertFalse(
valid_xiaohongshu_media_url(
"https://www.xiaohongshu.com/explore/n-1#fragment"
)
)
def test_generation_shape_matches_existing_browser_fence(self) -> None:
self.assertTrue(
valid_xiaohongshu_generation(
{"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""}
{
"binding_version": 1,
"runtime_id": "a" * 64,
"network_id": "network",
"network_exit_id": "",
}
)
)
self.assertFalse(
valid_xiaohongshu_generation(
{"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}
)
)
self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}))
class XiaohongshuRouteTests(unittest.TestCase):
@@ -48,18 +87,42 @@ class XiaohongshuRouteTests(unittest.TestCase):
gateway.post_xiaohongshu.return_value = {"status": 200}
gateway.get_xiaohongshu_media.return_value = {"status": 200}
gateway.xiaohongshu_identity.return_value = {"uid": "u-1"}
gateway.resolve_xiaohongshu.return_value = {
"url": "https://www.xiaohongshu.com/explore/n-1"
}
server = Mock()
server.gateway = gateway
cast(Any, handler).server = server
cast(Any, handler).server_as_gateway = lambda: server
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200})
self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"})
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}),
{"status": 200},
)
self.assertEqual(
handler._route(
"POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}
),
{"uid": "u-1"},
)
self.assertEqual(
handler._route(
"POST", "/v1/browsers/account-a/xiaohongshu/resolve", {}, {}
),
{"url": "https://www.xiaohongshu.com/explore/n-1"},
)
with self.assertRaises(RequestError):
handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {})
gateway.get_xiaohongshu.assert_called_once_with("account-a", {})
gateway.post_xiaohongshu.assert_called_once_with("account-a", {})
gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {})
gateway.xiaohongshu_identity.assert_called_once_with("account-a", {})
gateway.resolve_xiaohongshu.assert_called_once_with("account-a", {})
cmd/docker_gateway/xiaohongshu.py
+2
View File
@@ -12,6 +12,7 @@ from .douyin import (
XHS_SEARCH_ORIGIN,
XiaohongshuBrowser,
is_xiaohongshu_media_url,
is_xiaohongshu_share_url,
is_xiaohongshu_url,
)
@@ -23,5 +24,6 @@ __all__ = [
"XHS_SEARCH_ORIGIN",
"XiaohongshuBrowser",
"is_xiaohongshu_media_url",
"is_xiaohongshu_share_url",
"is_xiaohongshu_url",
]