feat: restore Xiaohongshu read-only collection

cmd/docker_gateway/douyin.py

@@ -1285,6 +1285,124 @@ def detect_challenge(status: int, body: str) -> str:
     return ""
 
 
+XHS_ORIGIN = "https://www.xiaohongshu.com"
+XHS_API_ORIGIN = "https://edith.xiaohongshu.com"
+XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com"
+XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me"
+XHS_ALLOWED_HOSTS = frozenset({"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"})
+
+
+class XiaohongshuBrowser(DouyinBrowser):
+    def __init__(self, endpoint=None) -> None:
+        super().__init__(
+            endpoint,
+            origin=XHS_ORIGIN,
+            url_validator=is_xiaohongshu_url,
+            media_validator=is_xiaohongshu_media_url,
+        )
+
+    def post(self, alias: str, target: str, body: bytes) -> BrowserResponse:
+        if not is_xiaohongshu_url(target) or len(body) > RESPONSE_LIMIT:
+            raise DouyinError("restricted Xiaohongshu POST request is invalid")
+        try:
+            body_text = body.decode("utf-8")
+        except UnicodeDecodeError as exc:
+            raise DouyinError("restricted Xiaohongshu POST body is not UTF-8") from exc
+        with self.connection(alias) as cdp:
+            if cdp.evaluate("location.origin") != self.origin:
+                raise DouyinError("restricted browser origin changed")
+            expression = f"""(async()=>{{
+              const r=await fetch({json.dumps(target)},{{method:'POST',headers:{{'content-type':'application/json'}},body:{json.dumps(body_text)},credentials:'include',redirect:'error'}});
+              if(!r.body)return {{status:r.status,body:'',too_large:false}};
+              const reader=r.body.getReader(), decoder=new TextDecoder(); let size=0, responseBody='';
+              for(;;){{const item=await reader.read();if(item.done)break;
+                if(size+item.value.byteLength>={RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}}
+                size+=item.value.byteLength;responseBody+=decoder.decode(item.value,{{stream:true}});
+              }}
+              responseBody+=decoder.decode();return {{status:r.status,body:responseBody,too_large:false}};
+            }})()"""
+            result = cdp.evaluate(expression)
+        if (
+            not isinstance(result, dict)
+            or result.get("too_large")
+            or not isinstance(result.get("status"), int)
+        ):
+            raise DouyinError("restricted Xiaohongshu POST failed")
+        status = result["status"]
+        if 300 <= status < 400:
+            raise DouyinError("restricted Xiaohongshu POST redirected")
+        response_body = result.get("body")
+        if not isinstance(response_body, str):
+            raise DouyinError("restricted Xiaohongshu POST returned invalid body")
+        return BrowserResponse(status, response_body, detect_challenge(status, response_body))
+
+    def identity(self, alias: str, expected_uid: str | None = None) -> dict:
+        response = self.get(alias, XHS_IDENTITY_URL)
+        try:
+            payload = json.loads(response.body)
+        except json.JSONDecodeError as exc:
+            raise DouyinError("Xiaohongshu identity response is invalid") from exc
+        data = payload.get("data") if isinstance(payload, dict) else None
+        user_info = data.get("user_info") if isinstance(data, dict) else None
+        user_id = data.get("user_id", "") if isinstance(data, dict) else ""
+        nickname = data.get("nickname", "") if isinstance(data, dict) else ""
+        if isinstance(user_info, dict):
+            user_id = user_id or user_info.get("user_id", "")
+            nickname = nickname or user_info.get("nickname", "")
+        success = payload.get("success") if isinstance(payload, dict) else None
+        if (
+            response.status != 200
+            or not isinstance(payload, dict)
+            or not isinstance(success, bool)
+            or not success
+            or not isinstance(user_id, str)
+            or not ACCOUNT_KEY_RE.fullmatch(user_id)
+            or nickname is not None
+            and not isinstance(nickname, str)
+        ):
+            raise DouyinError("Xiaohongshu login is not valid")
+        if expected_uid and user_id != expected_uid:
+            raise DouyinError("Xiaohongshu identity does not match the expected account")
+        return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""}
+
+
+def is_xiaohongshu_media_url(value: object) -> bool:
+    if not isinstance(value, str):
+        return False
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except (TypeError, ValueError):
+        return False
+    return (
+        parsed.scheme == "https"
+        and parsed.hostname == "www.xiaohongshu.com"
+        and port is None
+        and parsed.username is None
+        and parsed.password is None
+        and parsed.fragment == ""
+        and parsed.path.startswith("/explore/")
+    )
+
+
+def is_xiaohongshu_url(value: object) -> bool:
+    if not isinstance(value, str):
+        return False
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except (TypeError, ValueError):
+        return False
+    return (
+        parsed.scheme == "https"
+        and parsed.hostname in XHS_ALLOWED_HOSTS
+        and port is None
+        and parsed.username is None
+        and parsed.password is None
+        and parsed.fragment == ""
+    )
+
+
 def notice_ids(event: dict) -> list[str]:
     try:
         payload = json.loads(event["payload"])

cmd/docker_gateway/gateway.py

@@ -46,6 +46,7 @@ from .douyin import (
     DouyinBrowser,
     DouyinError,
     SubscriptionManager,
+    XiaohongshuBrowser,
 )
 from .proxy import ProxyExit, ProxyRegistry
 
@@ -64,6 +65,12 @@ DOUYIN_IDENTITY_PATH = "/aweme/v1/web/user/profile/self/"
 DOUYIN_IDENTITY_URL = IDENTITY_URL
 DOUYIN_WORKS_PATH = WORKS_PATH
 DOUYIN_COMMENTS_PATH = COMMENTS_PATH
+XHS_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$")
+XHS_IDENTITY_PATH = "/api/sns/web/v2/user/me"
+XHS_USER_POSTED_PATH = "/api/sns/web/v1/user_posted"
+XHS_COMMENTS_PATH = "/api/sns/web/v2/comment/page"
+XHS_SEARCH_PATH = "/api/sns/web/v2/search/notes"
+XHS_FEED_PATH = "/api/sns/web/v1/feed"
 
 
 def _noop() -> None:
@@ -85,12 +92,14 @@ class Gateway:
         token: str,
         self_name: str,
         browser: DouyinBrowser | None = None,
+        xiaohongshu_browser: XiaohongshuBrowser | None = None,
     ) -> None:
         self.docker = docker
         self.network = network
         self.token = token
         self.self_name = self_name
         self.browser = browser or DouyinBrowser(self._browser_endpoint)
+        self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser(self._browser_endpoint)
         self.proxies = ProxyRegistry()
         self.reservations = AliasReservationManager(docker, self_name)
         self.subscriptions = SubscriptionManager(self.browser)
@@ -666,6 +675,76 @@ class Gateway:
                 )
         return identity
 
+    def get_xiaohongshu(self, alias: str, input: dict) -> dict:
+        target = input.get("url", "")
+        if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_url(target):
+            raise RequestError("invalid restricted Xiaohongshu request", 400)
+        with self._alias_lock(alias):
+            self._require_douyin_generation(alias, input)
+            try:
+                response = self.xiaohongshu_browser.get(alias, target)
+                self._require_douyin_generation(alias, input)
+            except DouyinError as exc:
+                LOG.warning("Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc))
+                raise RequestError("restricted Xiaohongshu operation failed") from exc
+        return {"status": response.status, "body": response.body, "challenge": response.challenge}
+
+    def post_xiaohongshu(self, alias: str, input: dict) -> dict:
+        target = input.get("url", "")
+        body = input.get("body")
+        if (
+            not valid_xiaohongshu_generation(input)
+            or not valid_xhs_post_url(target)
+            or not isinstance(body, dict)
+        ):
+            raise RequestError("invalid restricted Xiaohongshu POST request", 400)
+        try:
+            encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
+        except (TypeError, ValueError) as exc:
+            raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc
+        with self._alias_lock(alias):
+            self._require_douyin_generation(alias, input)
+            try:
+                response = self.xiaohongshu_browser.post(alias, target, encoded)
+                self._require_douyin_generation(alias, input)
+            except DouyinError as exc:
+                LOG.warning("Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc))
+                raise RequestError("restricted Xiaohongshu operation failed") from exc
+        return {"status": response.status, "body": response.body, "challenge": response.challenge}
+
+    def get_xiaohongshu_media(self, alias: str, input: dict) -> dict:
+        target = input.get("url", "")
+        if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url(target):
+            raise RequestError("invalid restricted Xiaohongshu media request", 400)
+        with self._alias_lock(alias):
+            self._require_douyin_generation(alias, input)
+            try:
+                response = self.xiaohongshu_browser.get_media(alias, target)
+                self._require_douyin_generation(alias, input)
+            except DouyinError as exc:
+                LOG.warning("Xiaohongshu media download failed alias=%s reason=%s", alias, str(exc))
+                raise RequestError("restricted Xiaohongshu media download failed") from exc
+        return {"status": response.status, "content_type": response.content_type, "body_base64": response.body_base64}
+
+    def xiaohongshu_identity(self, alias: str, input: dict) -> dict:
+        expected_account_key = input.get("expected_account_key", "")
+        if (
+            not valid_xiaohongshu_generation(input)
+            or not isinstance(expected_account_key, str)
+            or not XHS_ACCOUNT_KEY_RE.fullmatch(expected_account_key)
+        ):
+            raise RequestError("invalid Xiaohongshu identity request", 400)
+        with self._alias_lock(alias):
+            self._require_douyin_generation(alias, input)
+            try:
+                identity = self.xiaohongshu_browser.identity(alias)
+            except DouyinError as exc:
+                LOG.warning("Xiaohongshu identity verification failed alias=%s reason=%s", alias, str(exc))
+                raise RequestError("Xiaohongshu login identity could not be verified") from exc
+            if identity.get("uid") != expected_account_key:
+                raise RequestError("Xiaohongshu identity does not match the expected account", 409)
+        return identity
+
     def douyin_action(self, alias: str, input: dict) -> dict:
         expected_uid = input.get("expected_uid", "")
         action = input.get("action", "")
@@ -1130,6 +1209,20 @@ class GatewayHandler(BaseHTTPRequestHandler):
             if method == "POST" and action == "proxy":
                 gateway.restore_proxy(alias, body)
                 return None
+        match = re.fullmatch(
+            r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity)",
+            path,
+        )
+        if match:
+            alias, action = match.groups()
+            if action == "get" and method == "POST":
+                return gateway.get_xiaohongshu(alias, body)
+            if action == "post" and method == "POST":
+                return gateway.post_xiaohongshu(alias, body)
+            if action == "media" and method == "POST":
+                return gateway.get_xiaohongshu_media(alias, body)
+            if action == "identity" and method == "POST":
+                return gateway.xiaohongshu_identity(alias, body)
         match = re.fullmatch(
             r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|action|events)",
             path,
@@ -1419,6 +1512,97 @@ def valid_douyin_generation(value: dict) -> bool:
     )
 
 
+def valid_xiaohongshu_generation(value: dict) -> bool:
+    return valid_douyin_generation(value)
+
+
+def valid_xhs_query(query: object, allowed: set[str], required: set[str] | None = None) -> bool:
+    if not isinstance(query, dict) or not isinstance(allowed, set):
+        return False
+    required = required or set()
+    if not required.issubset(query) or not set(query).issubset(allowed):
+        return False
+    for key, values in query.items():
+        if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1:
+            return False
+        if not isinstance(values[0], str) or len(values[0]) > 2048 or "\r" in values[0] or "\n" in values[0]:
+            return False
+    return True
+
+
+def _valid_xhs_host(parsed: object, host: str) -> bool:
+    return (
+        getattr(parsed, "scheme", "") == "https"
+        and getattr(parsed, "hostname", None) == host
+        and getattr(parsed, "port", None) is None
+        and getattr(parsed, "username", None) is None
+        and getattr(parsed, "password", None) is None
+        and getattr(parsed, "fragment", "") == ""
+    )
+
+
+def valid_xhs_url(raw: object) -> bool:
+    if not isinstance(raw, str):
+        return False
+    try:
+        parsed = urlsplit(raw)
+        query = parse_qs(parsed.query, keep_blank_values=True)
+    except ValueError:
+        return False
+    if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_IDENTITY_PATH:
+        return not query
+    if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_USER_POSTED_PATH:
+        return valid_xhs_query(
+            query,
+            {"user_id", "cursor", "num", "image_formats", "xsec_source", "xsec_token"},
+            {"user_id", "num"},
+        ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) and query["num"] == ["30"]
+    if _valid_xhs_host(parsed, "edith.xiaohongshu.com") and parsed.path == XHS_COMMENTS_PATH:
+        return valid_xhs_query(
+            query,
+            {"note_id", "cursor", "top_comment_id", "image_formats", "xsec_source", "xsec_token"},
+            {"note_id", "cursor", "top_comment_id"},
+        ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0]))
+    return False
+
+
+def valid_xiaohongshu_url(raw: object) -> bool:
+    return valid_xhs_url(raw)
+
+
+def valid_xhs_post_url(raw: object) -> bool:
+    if not isinstance(raw, str):
+        return False
+    try:
+        parsed = urlsplit(raw)
+        query = parse_qs(parsed.query, keep_blank_values=True)
+    except ValueError:
+        return False
+    return (
+        _valid_xhs_host(parsed, "so.xiaohongshu.com")
+        and parsed.path == XHS_SEARCH_PATH
+        and not query
+    ) or (
+        _valid_xhs_host(parsed, "edith.xiaohongshu.com")
+        and parsed.path == XHS_FEED_PATH
+        and not query
+    )
+
+
+def valid_xiaohongshu_media_url(raw: object) -> bool:
+    if not isinstance(raw, str):
+        return False
+    try:
+        parsed = urlsplit(raw)
+        query = parse_qs(parsed.query, keep_blank_values=True)
+    except ValueError:
+        return False
+    if not _valid_xhs_host(parsed, "www.xiaohongshu.com"):
+        return False
+    parts = parsed.path.strip("/").split("/")
+    return len(parts) == 2 and parts[0] == "explore" and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) and valid_xhs_query(query, {"xsec_source", "xsec_token"})
+
+
 def valid_douyin_url(raw: object) -> bool:
     if not isinstance(raw, str):
         return False

cmd/docker_gateway/test_xiaohongshu.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import unittest
+from typing import Any, cast
+from unittest.mock import Mock
+
+from . import gateway as gateway_module
+
+Gateway = gateway_module.Gateway
+RequestError = gateway_module.RequestError
+valid_xhs_post_url = gateway_module.valid_xhs_post_url
+valid_xhs_url = gateway_module.valid_xhs_url
+valid_xiaohongshu_url = gateway_module.valid_xiaohongshu_url
+valid_xiaohongshu_media_url = gateway_module.valid_xiaohongshu_media_url
+valid_xiaohongshu_generation = gateway_module.valid_xiaohongshu_generation
+
+
+class XiaohongshuValidationTests(unittest.TestCase):
+    def test_read_urls_use_explicit_host_path_and_query_allowlist(self) -> None:
+        self.assertTrue(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
+        self.assertTrue(valid_xiaohongshu_url("https://edith.xiaohongshu.com/api/sns/web/v2/user/me"))
+        self.assertTrue(
+            valid_xhs_url(
+                "https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&cursor=&num=30&xsec_source=pc_user"
+            )
+        )
+        self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?user_id=u-1&num=10"))
+        self.assertFalse(valid_xhs_url("https://edith.xiaohongshu.com.evil/api/sns/web/v2/user/me"))
+        self.assertTrue(valid_xhs_post_url("https://so.xiaohongshu.com/api/sns/web/v2/search/notes"))
+        self.assertTrue(valid_xhs_post_url("https://edith.xiaohongshu.com/api/sns/web/v1/feed"))
+        self.assertTrue(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1?xsec_source=pc_search"))
+        self.assertFalse(valid_xiaohongshu_media_url("https://www.xiaohongshu.com/explore/n-1#fragment"))
+
+    def test_generation_shape_matches_existing_browser_fence(self) -> None:
+        self.assertTrue(
+            valid_xiaohongshu_generation(
+                {"binding_version": 1, "runtime_id": "a" * 64, "network_id": "network", "network_exit_id": ""}
+            )
+        )
+        self.assertFalse(valid_xiaohongshu_generation({"binding_version": 1, "runtime_id": "runtime", "network_id": "network"}))
+
+
+class XiaohongshuRouteTests(unittest.TestCase):
+    def test_read_only_routes_dispatch_without_action_or_event_routes(self) -> None:
+        handler = gateway_module.GatewayHandler.__new__(gateway_module.GatewayHandler)
+        gateway = Mock()
+        gateway.get_xiaohongshu.return_value = {"status": 200}
+        gateway.post_xiaohongshu.return_value = {"status": 200}
+        gateway.get_xiaohongshu_media.return_value = {"status": 200}
+        gateway.xiaohongshu_identity.return_value = {"uid": "u-1"}
+        server = Mock()
+        server.gateway = gateway
+        cast(Any, handler).server = server
+        cast(Any, handler).server_as_gateway = lambda: server
+
+        self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/get", {}, {}), {"status": 200})
+        self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/post", {}, {}), {"status": 200})
+        self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/media", {}, {}), {"status": 200})
+        self.assertEqual(handler._route("POST", "/v1/browsers/account-a/xiaohongshu/identity", {}, {}), {"uid": "u-1"})
+        with self.assertRaises(RequestError):
+            handler._route("POST", "/v1/browsers/account-a/xiaohongshu/action", {}, {})
+        gateway.get_xiaohongshu.assert_called_once_with("account-a", {})
+        gateway.post_xiaohongshu.assert_called_once_with("account-a", {})
+        gateway.get_xiaohongshu_media.assert_called_once_with("account-a", {})
+        gateway.xiaohongshu_identity.assert_called_once_with("account-a", {})

cmd/docker_gateway/xiaohongshu.py

@@ -0,0 +1,27 @@
+"""Xiaohongshu gateway facade.
+
+The browser implementation lives next to the existing Douyin browser so both
+platforms share the CDP transport and response limits without duplicating it.
+"""
+
+from .douyin import (
+    XHS_ALLOWED_HOSTS,
+    XHS_API_ORIGIN,
+    XHS_IDENTITY_URL,
+    XHS_ORIGIN,
+    XHS_SEARCH_ORIGIN,
+    XiaohongshuBrowser,
+    is_xiaohongshu_media_url,
+    is_xiaohongshu_url,
+)
+
+__all__ = [
+    "XHS_ALLOWED_HOSTS",
+    "XHS_API_ORIGIN",
+    "XHS_IDENTITY_URL",
+    "XHS_ORIGIN",
+    "XHS_SEARCH_ORIGIN",
+    "XiaohongshuBrowser",
+    "is_xiaohongshu_media_url",
+    "is_xiaohongshu_url",
+]