diff --git a/.gitea/workflows/douyin-release-gate.yaml b/.gitea/workflows/douyin-release-gate.yaml index 6143bea..2fea502 100644 --- a/.gitea/workflows/douyin-release-gate.yaml +++ b/.gitea/workflows/douyin-release-gate.yaml @@ -87,9 +87,9 @@ jobs: python3 -m pip install -r requirements-gateway.lock coverage==7.16.0 python3 -m coverage erase python3 -m coverage run \ - --source=cmd/browser_gateway \ - --omit='cmd/browser_gateway/test_*.py' \ - -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' + --source=browser_gateway \ + --omit='browser_gateway/test_*.py' \ + -m unittest discover -s browser_gateway -t . -p 'test_*.py' python3 -m coverage report --precision=2 --fail-under=65 python3 -m coverage json -o evidence/python-coverage.json python3 - <<'PY' diff --git a/cmd/__init__.py b/browser_gateway/__init__.py similarity index 100% rename from cmd/__init__.py rename to browser_gateway/__init__.py diff --git a/browser_gateway/browser/__init__.py b/browser_gateway/browser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/browser_gateway/browser/cdp.py b/browser_gateway/browser/cdp.py new file mode 100644 index 0000000..ed2d0e8 --- /dev/null +++ b/browser_gateway/browser/cdp.py @@ -0,0 +1,172 @@ +"""Shared restricted CDP transport used by platform browsers.""" + +from __future__ import annotations + +import json +import logging +import threading +import time +from collections import deque +from collections.abc import Callable +from contextlib import suppress + +import websocket + +LOG = logging.getLogger("creatorhub.browser.cdp") + +CONTROL_TIMEOUT = 15.0 + + +class BrowserError(RuntimeError): + def __init__(self, message: str): + super().__init__(message) + self.uncertain = False + + +class CDPConnection: + def __init__(self, socket: websocket.WebSocket) -> None: + self.socket = socket + self._lock = threading.RLock() + self._next_id = 0 + self._pending: deque[dict] = deque() + + def command(self, method: str, params: dict | None = None) -> dict: + with self._lock: + self._next_id += 1 + command_id = self._next_id + self.socket.send( + json.dumps({"id": command_id, "method": method, "params": params or {}}) + ) + deadline = time.monotonic() + CONTROL_TIMEOUT + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + self._terminate_evaluation(method) + raise BrowserError(f"CDP command timed out: {method}") + message = self._take_pending(command_id) + if message is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + self._terminate_evaluation(method) + raise BrowserError(f"CDP command timed out: {method}") + try: + message = self._receive(remaining, method) + except BrowserError: + self._terminate_evaluation(method) + raise + if not isinstance(message, dict): + raise BrowserError(f"CDP command returned invalid message: {method}") + if message.get("id") != command_id: + self._pending.append(message) + continue + if message.get("error"): + raise BrowserError(f"CDP command rejected: {method}") + result = message.get("result") + if not isinstance(result, dict): + raise BrowserError(f"CDP command returned invalid result: {method}") + return result + + def wait_event( + self, + method: str, + predicate: Callable[[dict], bool], + timeout: float = CONTROL_TIMEOUT, + ) -> dict: + with self._lock: + deadline = time.monotonic() + timeout + while True: + for index, message in enumerate(self._pending): + params = message.get("params", {}) + if ( + message.get("method") == method + and isinstance(params, dict) + and predicate(params) + ): + del self._pending[index] + return message + remaining = deadline - time.monotonic() + if remaining <= 0: + raise BrowserError(f"CDP event timed out: {method}") + message = self._receive(remaining, method) + if isinstance(message, dict): + params = message.get("params", {}) + if ( + message.get("method") == method + and isinstance(params, dict) + and predicate(params) + ): + return message + self._pending.append(message) + + def _terminate_evaluation(self, method: str) -> None: + if method != "Runtime.evaluate": + return + try: + self._next_id += 1 + command_id = self._next_id + self.socket.settimeout(1.0) + self.socket.send( + json.dumps( + { + "id": command_id, + "method": "Runtime.terminateExecution", + "params": {}, + } + ) + ) + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + message = json.loads(self.socket.recv()) + if isinstance(message, dict) and message.get("id") == command_id: + return + if isinstance(message, dict): + self._pending.append(message) + except ( + OSError, + TypeError, + websocket.WebSocketException, + json.JSONDecodeError, + ) as exc: + LOG.debug("failed to terminate timed-out CDP evaluation", exc_info=exc) + + def _take_pending(self, command_id: int) -> dict | None: + for index, message in enumerate(self._pending): + if message.get("id") == command_id: + del self._pending[index] + return message + return None + + def _receive(self, timeout: float, operation: str) -> object: + self.socket.settimeout(timeout) + try: + return json.loads(self.socket.recv()) + except (TimeoutError, websocket.WebSocketTimeoutException) as exc: + raise BrowserError(f"CDP command timed out during {operation}") from exc + except ( + OSError, + TypeError, + websocket.WebSocketException, + json.JSONDecodeError, + ) as exc: + raise BrowserError(f"CDP message failed during {operation}") from exc + + def evaluate(self, expression: str) -> object: + result = self.command( + "Runtime.evaluate", + { + "expression": expression, + "awaitPromise": True, + "returnByValue": True, + "userGesture": False, + }, + ) + if result.get("exceptionDetails"): + raise BrowserError("page evaluation failed") + value = result.get("result", {}).get("value") + if "value" not in result.get("result", {}): + raise BrowserError("page evaluation returned no value") + return value + + def close(self) -> None: + with suppress(OSError, websocket.WebSocketException): + self.socket.close() diff --git a/browser_gateway/browser/response.py b/browser_gateway/browser/response.py new file mode 100644 index 0000000..aa9912d --- /dev/null +++ b/browser_gateway/browser/response.py @@ -0,0 +1,39 @@ +"""Shared browser response value objects and challenge detection.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BrowserResponse: + status: int + body: str + challenge: str = "" + + +@dataclass(frozen=True) +class BrowserMediaResponse: + status: int + content_type: str + body_base64: str + + +@dataclass(frozen=True) +class BrowserLoginQRResponse: + content_type: str + body_base64: str + qr_detected: bool + + +def detect_challenge(status: int, body: str) -> str: + if status == 429: + return "" + if status == 412: + return "captcha" + lowered = body.lower() + if any(token in lowered for token in ("captcha", "verify_code", "验证码")): + return "captcha" + if any(token in lowered for token in ("device_verify", "device_check", "设备验证")): + return "device" + return "" diff --git a/browser_gateway/platform/__init__.py b/browser_gateway/platform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cmd/browser_gateway/douyin.py b/browser_gateway/platform/douyin.py similarity index 83% rename from cmd/browser_gateway/douyin.py rename to browser_gateway/platform/douyin.py index 1641d60..6a1f783 100644 --- a/cmd/browser_gateway/douyin.py +++ b/browser_gateway/platform/douyin.py @@ -1,4 +1,4 @@ -"""Douyin browser control built on a narrowly scoped CDP contract.""" +"""Douyin browser control built on the shared restricted CDP contract.""" from __future__ import annotations @@ -15,14 +15,20 @@ import uuid from collections import deque from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor -from contextlib import contextmanager, suppress -from dataclasses import dataclass +from contextlib import contextmanager from datetime import datetime, timezone -from urllib.parse import parse_qs, urlsplit +from urllib.parse import urlsplit import websocket -from .runtime import ALIAS_RE +from ..browser.cdp import CONTROL_TIMEOUT, BrowserError, CDPConnection +from ..browser.response import ( + BrowserLoginQRResponse, + BrowserMediaResponse, + BrowserResponse, + detect_challenge, +) +from ..runtime import ALIAS_RE LOG = logging.getLogger("creatorhub.douyin") ORIGIN = "https://www.douyin.com" @@ -40,7 +46,6 @@ WORKS_PATH = "/aweme/v1/web/aweme/post/" COMMENTS_PATH = "/aweme/v1/web/comment/list/" RESPONSE_LIMIT = 1 << 20 MEDIA_RESPONSE_LIMIT = 32 << 20 -CONTROL_TIMEOUT = 15.0 MEDIA_SOURCE_WAIT_MS = 15000 UID_RE = re.compile(r"^[1-9][0-9]{0,19}$") ID_RE = re.compile(r"^[1-9][0-9]{0,63}$") @@ -50,11 +55,7 @@ ACTIONS = frozenset( ) -class DouyinError(RuntimeError): - def __init__(self, message: str): - super().__init__(message) - self.uncertain = False - +DouyinError = BrowserError LISTENER_ERRORS = ( DouyinError, @@ -66,176 +67,6 @@ LISTENER_ERRORS = ( LISTENER_START_TIMEOUT = 30.0 -@dataclass(frozen=True) -class BrowserResponse: - status: int - body: str - challenge: str = "" - - -@dataclass(frozen=True) -class BrowserMediaResponse: - status: int - content_type: str - body_base64: str - - -@dataclass(frozen=True) -class BrowserLoginQRResponse: - content_type: str - body_base64: str - qr_detected: bool - - -class CDPConnection: - def __init__(self, socket: websocket.WebSocket) -> None: - self.socket = socket - self._lock = threading.RLock() - self._next_id = 0 - self._pending: deque[dict] = deque() - - def command(self, method: str, params: dict | None = None) -> dict: - with self._lock: - self._next_id += 1 - command_id = self._next_id - self.socket.send( - json.dumps({"id": command_id, "method": method, "params": params or {}}) - ) - deadline = time.monotonic() + CONTROL_TIMEOUT - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - self._terminate_evaluation(method) - raise DouyinError(f"CDP command timed out: {method}") - message = self._take_pending(command_id) - if message is None: - remaining = deadline - time.monotonic() - if remaining <= 0: - self._terminate_evaluation(method) - raise DouyinError(f"CDP command timed out: {method}") - try: - message = self._receive(remaining, method) - except DouyinError: - self._terminate_evaluation(method) - raise - if not isinstance(message, dict): - raise DouyinError(f"CDP command returned invalid message: {method}") - if message.get("id") != command_id: - self._pending.append(message) - continue - if message.get("error"): - raise DouyinError(f"CDP command rejected: {method}") - result = message.get("result") - if not isinstance(result, dict): - raise DouyinError(f"CDP command returned invalid result: {method}") - return result - - def wait_event( - self, - method: str, - predicate: Callable[[dict], bool], - timeout: float = CONTROL_TIMEOUT, - ) -> dict: - with self._lock: - deadline = time.monotonic() + timeout - while True: - for index, message in enumerate(self._pending): - params = message.get("params", {}) - if ( - message.get("method") == method - and isinstance(params, dict) - and predicate(params) - ): - del self._pending[index] - return message - remaining = deadline - time.monotonic() - if remaining <= 0: - raise DouyinError(f"CDP event timed out: {method}") - message = self._receive(remaining, method) - if isinstance(message, dict): - params = message.get("params", {}) - if ( - message.get("method") == method - and isinstance(params, dict) - and predicate(params) - ): - return message - self._pending.append(message) - - def _terminate_evaluation(self, method: str) -> None: - if method != "Runtime.evaluate": - return - try: - self._next_id += 1 - command_id = self._next_id - self.socket.settimeout(1.0) - self.socket.send( - json.dumps( - { - "id": command_id, - "method": "Runtime.terminateExecution", - "params": {}, - } - ) - ) - deadline = time.monotonic() + 1.0 - while time.monotonic() < deadline: - message = json.loads(self.socket.recv()) - if isinstance(message, dict) and message.get("id") == command_id: - return - if isinstance(message, dict): - self._pending.append(message) - except ( - OSError, - TypeError, - websocket.WebSocketException, - json.JSONDecodeError, - ) as exc: - LOG.debug("failed to terminate timed-out CDP evaluation", exc_info=exc) - - def _take_pending(self, command_id: int) -> dict | None: - for index, message in enumerate(self._pending): - if message.get("id") == command_id: - del self._pending[index] - return message - return None - - def _receive(self, timeout: float, operation: str) -> object: - self.socket.settimeout(timeout) - try: - return json.loads(self.socket.recv()) - except (TimeoutError, websocket.WebSocketTimeoutException) as exc: - raise DouyinError(f"CDP command timed out during {operation}") from exc - except ( - OSError, - TypeError, - websocket.WebSocketException, - json.JSONDecodeError, - ) as exc: - raise DouyinError(f"CDP message failed during {operation}") from exc - - def evaluate(self, expression: str) -> object: - result = self.command( - "Runtime.evaluate", - { - "expression": expression, - "awaitPromise": True, - "returnByValue": True, - "userGesture": False, - }, - ) - if result.get("exceptionDetails"): - raise DouyinError("page evaluation failed") - value = result.get("result", {}).get("value") - if "value" not in result.get("result", {}): - raise DouyinError("page evaluation returned no value") - return value - - def close(self) -> None: - with suppress(OSError, websocket.WebSocketException): - self.socket.close() - - class DouyinBrowser: def __init__( self, @@ -1581,221 +1412,6 @@ def is_douyin_media_url(value: object) -> bool: return False -def detect_challenge(status: int, body: str) -> str: - if status == 429: - return "" - if status == 412: - return "captcha" - lowered = body.lower() - if any(token in lowered for token in ("captcha", "verify_code", "验证码")): - return "captcha" - if any(token in lowered for token in ("device_verify", "device_check", "设备验证")): - return "device" - return "" - - -XHS_ORIGIN = "https://www.xiaohongshu.com" -XHS_API_ORIGIN = "https://edith.xiaohongshu.com" -XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com" -XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me" -XHS_ALLOWED_HOSTS = frozenset( - {"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"} -) - - -class XiaohongshuBrowser(DouyinBrowser): - def __init__(self, endpoint=None, *, target_id: str = "") -> None: - super().__init__( - endpoint, - origin=XHS_ORIGIN, - url_validator=is_xiaohongshu_url, - media_validator=is_xiaohongshu_media_url, - media_selector="video, img.note-slider-img", - target_id=target_id, - ) - - def post(self, alias: str, target: str, body: bytes) -> BrowserResponse: - if not is_xiaohongshu_url(target) or len(body) > RESPONSE_LIMIT: - raise DouyinError("restricted Xiaohongshu POST request is invalid") - try: - body_text = body.decode("utf-8") - except UnicodeDecodeError as exc: - raise DouyinError("restricted Xiaohongshu POST body is not UTF-8") from exc - with self.connection(alias) as cdp: - if cdp.evaluate("location.origin") != self.origin: - raise DouyinError("restricted browser origin changed") - expression = f"""(async()=>{{ - const r=await fetch({json.dumps(target)},{{method:'POST',headers:{{'content-type':'application/json'}},body:{json.dumps(body_text)},credentials:'include',redirect:'error'}}); - if(!r.body)return {{status:r.status,body:'',too_large:false}}; - const reader=r.body.getReader(), decoder=new TextDecoder(); let size=0, responseBody=''; - for(;;){{const item=await reader.read();if(item.done)break; - if(size+item.value.byteLength>={RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}} - size+=item.value.byteLength;responseBody+=decoder.decode(item.value,{{stream:true}}); - }} - responseBody+=decoder.decode();return {{status:r.status,body:responseBody,too_large:false}}; - }})()""" - result = cdp.evaluate(expression) - if ( - not isinstance(result, dict) - or result.get("too_large") - or not isinstance(result.get("status"), int) - ): - raise DouyinError("restricted Xiaohongshu POST failed") - status = result["status"] - if 300 <= status < 400: - raise DouyinError("restricted Xiaohongshu POST redirected") - response_body = result.get("body") - if not isinstance(response_body, str): - raise DouyinError("restricted Xiaohongshu POST returned invalid body") - return BrowserResponse( - status, response_body, detect_challenge(status, response_body) - ) - - def resolve(self, alias: str, target: str) -> str: - if not is_xiaohongshu_share_url(target): - raise DouyinError("restricted Xiaohongshu share URL is invalid") - with self.connection(alias) as cdp: - if cdp.evaluate("location.origin") != self.origin: - raise DouyinError("restricted browser origin changed") - cdp.command("Page.navigate", {"url": target}) - event = cdp.wait_event( - "Page.frameNavigated", - lambda params: _is_xiaohongshu_page_url( - params.get("frame", {}).get("url", "") - ), - ) - final_url = event.get("frame", {}).get("url") - if not isinstance(final_url, str) or not _is_xiaohongshu_page_url(final_url): - raise DouyinError( - "Xiaohongshu share URL did not resolve to a supported page" - ) - return final_url - - def identity(self, alias: str, expected_uid: str | None = None) -> dict: - response = self.get(alias, XHS_IDENTITY_URL) - try: - payload = json.loads(response.body) - except json.JSONDecodeError as exc: - raise DouyinError("Xiaohongshu identity response is invalid") from exc - data = payload.get("data") if isinstance(payload, dict) else None - user_info = data.get("user_info") if isinstance(data, dict) else None - user_id = data.get("user_id", "") if isinstance(data, dict) else "" - nickname = data.get("nickname", "") if isinstance(data, dict) else "" - if isinstance(user_info, dict): - user_id = user_id or user_info.get("user_id", "") - nickname = nickname or user_info.get("nickname", "") - success = payload.get("success") if isinstance(payload, dict) else None - if ( - response.status != 200 - or not isinstance(payload, dict) - or not isinstance(success, bool) - or not success - or not isinstance(user_id, str) - or not ACCOUNT_KEY_RE.fullmatch(user_id) - or nickname is not None - and not isinstance(nickname, str) - ): - raise DouyinError("Xiaohongshu login is not valid") - if expected_uid and user_id != expected_uid: - raise DouyinError( - "Xiaohongshu identity does not match the expected account" - ) - return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""} - - -def is_xiaohongshu_media_url(value: object) -> bool: - if not isinstance(value, str): - return False - try: - parsed = urlsplit(value) - port = parsed.port - except (TypeError, ValueError): - return False - return ( - parsed.scheme == "https" - and parsed.hostname == "www.xiaohongshu.com" - and port is None - and parsed.username is None - and parsed.password is None - and parsed.fragment == "" - and parsed.path.startswith("/explore/") - ) - - -def is_xiaohongshu_url(value: object) -> bool: - if not isinstance(value, str): - return False - try: - parsed = urlsplit(value) - port = parsed.port - except (TypeError, ValueError): - return False - return ( - parsed.scheme == "https" - and parsed.hostname in XHS_ALLOWED_HOSTS - and port is None - and parsed.username is None - and parsed.password is None - and parsed.fragment == "" - ) - - -def is_xiaohongshu_share_url(value: object) -> bool: - if not isinstance(value, str): - return False - try: - parsed = urlsplit(value) - port = parsed.port - except (TypeError, ValueError): - return False - path = parsed.path.strip("/") - return ( - parsed.scheme == "https" - and parsed.hostname in {"xhslink.com", "www.xhslink.com"} - and port is None - and parsed.username is None - and parsed.password is None - and parsed.fragment == "" - and bool(path) - and len(path) <= 256 - and not parsed.query - ) - - -def _is_xiaohongshu_page_url(value: object) -> bool: - if not isinstance(value, str): - return False - try: - parsed = urlsplit(value) - port = parsed.port - query = parse_qs(parsed.query, keep_blank_values=True) - except (TypeError, ValueError): - return False - parts = parsed.path.strip("/").split("/") - if not ( - parsed.scheme == "https" - and parsed.hostname == "www.xiaohongshu.com" - and port is None - and parsed.username is None - and parsed.password is None - and parsed.fragment == "" - and ( - len(parts) == 2 - and parts[0] == "explore" - or len(parts) == 3 - and parts[:2] == ["user", "profile"] - ) - ): - return False - return all( - key in {"xsec_token", "xsec_source"} - and len(values) == 1 - and len(values[0]) <= 2048 - and not any(char in values[0] for char in "\r\n") - for key, values in query.items() - ) - - def notice_ids(event: dict) -> list[str]: try: payload = json.loads(event["payload"]) diff --git a/browser_gateway/platform/xiaohongshu.py b/browser_gateway/platform/xiaohongshu.py new file mode 100644 index 0000000..4f3dcd2 --- /dev/null +++ b/browser_gateway/platform/xiaohongshu.py @@ -0,0 +1,218 @@ +"""Xiaohongshu browser implementation.""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlsplit + +from ..browser.response import ( + BrowserResponse, + detect_challenge, +) +from .douyin import ( + ACCOUNT_KEY_RE, + RESPONSE_LIMIT, + DouyinBrowser, + DouyinError, +) + +XHS_ORIGIN = "https://www.xiaohongshu.com" +XHS_API_ORIGIN = "https://edith.xiaohongshu.com" +XHS_SEARCH_ORIGIN = "https://so.xiaohongshu.com" +XHS_IDENTITY_URL = XHS_API_ORIGIN + "/api/sns/web/v2/user/me" +XHS_ALLOWED_HOSTS = frozenset( + {"www.xiaohongshu.com", "edith.xiaohongshu.com", "so.xiaohongshu.com"} +) + + +class XiaohongshuBrowser(DouyinBrowser): + def __init__(self, endpoint=None, *, target_id: str = "") -> None: + super().__init__( + endpoint, + origin=XHS_ORIGIN, + url_validator=is_xiaohongshu_url, + media_validator=is_xiaohongshu_media_url, + media_selector="video, img.note-slider-img", + target_id=target_id, + ) + + def post(self, alias: str, target: str, body: bytes) -> BrowserResponse: + if not is_xiaohongshu_url(target) or len(body) > RESPONSE_LIMIT: + raise DouyinError("restricted Xiaohongshu POST request is invalid") + try: + body_text = body.decode("utf-8") + except UnicodeDecodeError as exc: + raise DouyinError("restricted Xiaohongshu POST body is not UTF-8") from exc + with self.connection(alias) as cdp: + if cdp.evaluate("location.origin") != self.origin: + raise DouyinError("restricted browser origin changed") + expression = f"""(async()=>{{ + const r=await fetch({json.dumps(target)},{{method:'POST',headers:{{'content-type':'application/json'}},body:{json.dumps(body_text)},credentials:'include',redirect:'error'}}); + if(!r.body)return {{status:r.status,body:'',too_large:false}}; + const reader=r.body.getReader(), decoder=new TextDecoder(); let size=0, responseBody=''; + for(;;){{const item=await reader.read();if(item.done)break; + if(size+item.value.byteLength>={RESPONSE_LIMIT}){{await reader.cancel();return {{too_large:true}};}} + size+=item.value.byteLength;responseBody+=decoder.decode(item.value,{{stream:true}}); + }} + responseBody+=decoder.decode();return {{status:r.status,body:responseBody,too_large:false}}; + }})()""" + result = cdp.evaluate(expression) + if ( + not isinstance(result, dict) + or result.get("too_large") + or not isinstance(result.get("status"), int) + ): + raise DouyinError("restricted Xiaohongshu POST failed") + status = result["status"] + if 300 <= status < 400: + raise DouyinError("restricted Xiaohongshu POST redirected") + response_body = result.get("body") + if not isinstance(response_body, str): + raise DouyinError("restricted Xiaohongshu POST returned invalid body") + return BrowserResponse( + status, response_body, detect_challenge(status, response_body) + ) + + def resolve(self, alias: str, target: str) -> str: + if not is_xiaohongshu_share_url(target): + raise DouyinError("restricted Xiaohongshu share URL is invalid") + with self.connection(alias) as cdp: + if cdp.evaluate("location.origin") != self.origin: + raise DouyinError("restricted browser origin changed") + cdp.command("Page.navigate", {"url": target}) + event = cdp.wait_event( + "Page.frameNavigated", + lambda params: _is_xiaohongshu_page_url( + params.get("frame", {}).get("url", "") + ), + ) + final_url = event.get("frame", {}).get("url") + if not isinstance(final_url, str) or not _is_xiaohongshu_page_url(final_url): + raise DouyinError( + "Xiaohongshu share URL did not resolve to a supported page" + ) + return final_url + + def identity(self, alias: str, expected_uid: str | None = None) -> dict: + response = self.get(alias, XHS_IDENTITY_URL) + try: + payload = json.loads(response.body) + except json.JSONDecodeError as exc: + raise DouyinError("Xiaohongshu identity response is invalid") from exc + data = payload.get("data") if isinstance(payload, dict) else None + user_info = data.get("user_info") if isinstance(data, dict) else None + user_id = data.get("user_id", "") if isinstance(data, dict) else "" + nickname = data.get("nickname", "") if isinstance(data, dict) else "" + if isinstance(user_info, dict): + user_id = user_id or user_info.get("user_id", "") + nickname = nickname or user_info.get("nickname", "") + success = payload.get("success") if isinstance(payload, dict) else None + if ( + response.status != 200 + or not isinstance(payload, dict) + or not isinstance(success, bool) + or not success + or not isinstance(user_id, str) + or not ACCOUNT_KEY_RE.fullmatch(user_id) + or nickname is not None + and not isinstance(nickname, str) + ): + raise DouyinError("Xiaohongshu login is not valid") + if expected_uid and user_id != expected_uid: + raise DouyinError( + "Xiaohongshu identity does not match the expected account" + ) + return {"uid": user_id, "user_id": user_id, "nickname": nickname or ""} + + +def is_xiaohongshu_media_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + except (TypeError, ValueError): + return False + return ( + parsed.scheme == "https" + and parsed.hostname == "www.xiaohongshu.com" + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + and parsed.path.startswith("/explore/") + ) + + +def is_xiaohongshu_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + except (TypeError, ValueError): + return False + return ( + parsed.scheme == "https" + and parsed.hostname in XHS_ALLOWED_HOSTS + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + ) + + +def is_xiaohongshu_share_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + except (TypeError, ValueError): + return False + path = parsed.path.strip("/") + return ( + parsed.scheme == "https" + and parsed.hostname in {"xhslink.com", "www.xhslink.com"} + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + and bool(path) + and len(path) <= 256 + and not parsed.query + ) + + +def _is_xiaohongshu_page_url(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlsplit(value) + port = parsed.port + query = parse_qs(parsed.query, keep_blank_values=True) + except (TypeError, ValueError): + return False + parts = parsed.path.strip("/").split("/") + if not ( + parsed.scheme == "https" + and parsed.hostname == "www.xiaohongshu.com" + and port is None + and parsed.username is None + and parsed.password is None + and parsed.fragment == "" + and ( + len(parts) == 2 + and parts[0] == "explore" + or len(parts) == 3 + and parts[:2] == ["user", "profile"] + ) + ): + return False + return all( + key in {"xsec_token", "xsec_source"} + and len(values) == 1 + and len(values[0]) <= 2048 + and not any(char in values[0] for char in "\r\n") + for key, values in query.items() + ) diff --git a/cmd/browser_gateway/proxy.py b/browser_gateway/proxy.py similarity index 100% rename from cmd/browser_gateway/proxy.py rename to browser_gateway/proxy.py diff --git a/cmd/browser_gateway/runtime.py b/browser_gateway/runtime.py similarity index 98% rename from cmd/browser_gateway/runtime.py rename to browser_gateway/runtime.py index 8f14dd3..0194044 100644 --- a/cmd/browser_gateway/runtime.py +++ b/browser_gateway/runtime.py @@ -10,12 +10,11 @@ import logging import os import re import shutil -import signal import socket import subprocess import threading import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping from contextlib import suppress from dataclasses import dataclass, fields from pathlib import Path @@ -25,6 +24,8 @@ from urllib.parse import urlsplit from .proxy import ProxyExit, ProxyRegistry LOG = logging.getLogger("creatorhub.runtime") +# Host-managed Xvfb publishes display sockets here; this is not a gateway temp directory. +X11_SOCKET_DIR = Path(os.sep, "tmp", ".X11-unix") RUNTIME_ID_RE = re.compile(r"^[a-f0-9]{64}$") ALIAS_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,31}$") @@ -115,14 +116,14 @@ class FileLock: operation = fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB try: fcntl.flock(fd, operation) - except (BlockingIOError, OSError) as exc: + except BlockingIOError: os.close(fd) - if isinstance(exc, BlockingIOError) or getattr(exc, "errno", None) in ( - 11, - 13, - ): - return False - raise + return False + except OSError as exc: + os.close(fd) + if exc.errno not in (11, 13): + raise + return False self._fd = fd return True @@ -201,11 +202,11 @@ class DisplayAllocator: ) -> _Lease: unavailable = unavailable or (lambda _value: False) for value in range(start, end + 1): - if unavailable(value) or Path(f"/tmp/.X11-unix/X{value}").exists(): + if unavailable(value) or (X11_SOCKET_DIR / f"X{value}").exists(): continue lock = FileLock(self.lock_dir / f"display-{value}.lock") if lock.acquire(): - if not Path(f"/tmp/.X11-unix/X{value}").exists(): + if not (X11_SOCKET_DIR / f"X{value}").exists(): return _Lease(value, lock) lock.release() raise BrowserRuntimeError("no free Xvfb display is available", 503) @@ -593,7 +594,17 @@ class NativeRuntimeManager: profile_id = value.get("profile_id", alias) profile_dir = self._profile_dir(profile_id) browser_path = self.browser_versions.get(value["browser_version"]) - if not browser_path or not os.access(browser_path, os.X_OK): + if not browser_path: + raise BrowserRuntimeError("browser version is unavailable", 422) + try: + available = os.path.isfile(browser_path) and os.access( + browser_path, os.X_OK + ) + except (OSError, TypeError) as exc: + raise BrowserRuntimeError( + "browser version is unavailable", 422 + ) from exc + if not available: raise BrowserRuntimeError("browser version is unavailable", 422) self._check_capacity() now = self.clock() @@ -1022,7 +1033,7 @@ class NativeRuntimeManager: return UnitStatus(available, "external" if available else "missing", 0, "", "") def _display_available(self, display: int) -> bool: - return display > 0 and Path(f"/tmp/.X11-unix/X{display}").exists() + return display > 0 and (X11_SOCKET_DIR / f"X{display}").exists() def _wait_for_display(self, record: RuntimeRecord) -> None: deadline = time.monotonic() + self.ready_timeout diff --git a/browser_gateway/server/__init__.py b/browser_gateway/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/browser_gateway/server/http.py b/browser_gateway/server/http.py new file mode 100644 index 0000000..53b8796 --- /dev/null +++ b/browser_gateway/server/http.py @@ -0,0 +1,1419 @@ +"""CreatorHub native browser gateway.""" + +from __future__ import annotations + +import hmac +import json +import logging +import math +import os +import re +import secrets +import signal +import socket +import threading +import time +from collections.abc import Mapping +from contextlib import suppress +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import cast +from urllib.parse import parse_qs, urlsplit + +from ..platform.douyin import ( + ACCOUNT_KEY_RE, + ACTIONS, + COMMENTS_PATH, + IDENTITY_URL, + UID_RE, + WORKS_PATH, + DouyinBrowser, + DouyinError, + SubscriptionManager, + is_douyin_content_url, + is_douyin_share_url, +) +from ..platform.xiaohongshu import XiaohongshuBrowser, is_xiaohongshu_share_url +from ..proxy import ProxyExit +from ..runtime import ( + BROWSER_VERSION_RE, + BrowserRuntimeError, + NativeRuntimeManager, + has_control, + validate_runtime_input, +) +from ..runtime import ( + NETWORK_ID_RE as _NETWORK_ID_RE, +) +from ..runtime import ( + RUNTIME_CLEANUP_SENTINEL as _RUNTIME_CLEANUP_SENTINEL, +) +from ..runtime import ( + RUNTIME_ID_RE as _RUNTIME_ID_RE, +) +from ..runtime import ( + parse_proxy_exit as _parse_proxy_exit, +) +from ..runtime import ( + validate_proxy_exit as _validate_proxy_exit, +) + +LOG = logging.getLogger("creatorhub.gateway") +RUNTIME_ID_RE = _RUNTIME_ID_RE +NETWORK_ID_RE = _NETWORK_ID_RE +RUNTIME_CLEANUP_SENTINEL = _RUNTIME_CLEANUP_SENTINEL +EXIT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$") +DOUYIN_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$") +DOUYIN_ORIGIN = "https://www.douyin.com" +DOUYIN_IDENTITY_PATH = "/aweme/v1/web/user/profile/self/" +DOUYIN_PROFILE_OTHER_PATH = "/aweme/v1/web/user/profile/other/" +DOUYIN_IDENTITY_URL = IDENTITY_URL +DOUYIN_WORKS_PATH = WORKS_PATH +DOUYIN_WORK_DETAIL_PATH = "/aweme/v1/web/aweme/detail/" +DOUYIN_COMMENTS_PATH = COMMENTS_PATH +XHS_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$") +XHS_IDENTITY_PATH = "/api/sns/web/v2/user/me" +XHS_USER_POSTED_PATH = "/api/sns/web/v1/user_posted" +XHS_COMMENTS_PATH = "/api/sns/web/v2/comment/page" +XHS_SEARCH_PATH = "/api/sns/web/v2/search/notes" +XHS_FEED_PATH = "/api/sns/web/v1/feed" + + +def _noop() -> None: + return None + + +class RequestError(BrowserRuntimeError): + pass + + +class Gateway: + def __init__( + self, + runtimes: NativeRuntimeManager, + token: str, + node_id: str, + browser: DouyinBrowser | None = None, + xiaohongshu_browser: XiaohongshuBrowser | None = None, + ) -> None: + self.runtimes = runtimes + self.token = token + self.node_id = node_id + self.browser = browser or DouyinBrowser(self._browser_endpoint) + self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser( + self._browser_endpoint + ) + self.proxies = runtimes.proxies + self.subscriptions = SubscriptionManager(self.browser) + self._action_ownership_lock = threading.Lock() + self._uncertain_actions: dict[str, float] = {} + + def _browser_endpoint(self, alias: str) -> str: + return self.runtimes.endpoint(alias) + + def list_browsers(self) -> list[dict]: + # Released generations remain in the runtime journal for cleanup/audit, + # but are not live browser bindings and must not collide with a reused alias. + return [ + browser + for browser in self.runtimes.list_public() + if browser.get("state") != "released" + ] + + def info(self) -> dict: + return { + "service": "browser-gateway", + "node_id": self.node_id, + "browser_versions": sorted(self.runtimes.browser_versions), + } + + def create(self, input: dict) -> dict: + return self.runtimes.create(input) + + def change_state(self, alias: str, action: str, input: dict) -> None: + generation = decode_generation( + input, require_runtime=True, require_network=True + ) + self.runtimes.change_state(alias, action, generation) + + def remove(self, alias: str, input: dict) -> None: + generation = decode_generation( + input, + require_runtime=False, + require_network=False, + allow_profile_purge=True, + ) + purge_profile = input.get("purge_profile", False) + if type(purge_profile) is not bool: + raise RequestError("purge_profile must be boolean", 400) + self.runtimes.remove(alias, generation, purge_profile) + + def restore_proxy(self, alias: str, input: dict) -> None: + validate_proxy_restore(input, alias) + generation = decode_generation( + {key: input.get(key) for key in ("binding_version", "runtime_id", "network_id")}, + True, + True, + ) + self.runtimes.restore_proxy(alias, generation, input["network_exit"]) + + def _require_generation(self, alias: str, generation: dict) -> tuple[str, bool]: + try: + record = self.runtimes.require_generation(alias, generation) + except FileNotFoundError: + return "", False + return record.runtime_id, True + + def _require_douyin_generation(self, alias: str, input: dict) -> None: + record = self.runtimes.require_generation(alias, input) + if input.get("network_exit_id", "") != record.network_exit_id: + raise RequestError("runtime proxy generation does not match request", 409) + + def _alias_lock(self, alias: str): + return self.runtimes.alias_lock(alias) + + def get_douyin(self, alias: str, input: dict) -> dict: + if not valid_douyin_generation(input) or not valid_douyin_url( + input.get("url", "") + ): + raise RequestError("invalid restricted browser request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.browser.get(alias, input["url"]) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Douyin GET failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError("restricted browser operation failed") from exc + return { + "status": response.status, + "body": response.body, + "challenge": response.challenge, + } + + def resolve_douyin(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_douyin_generation(input) or not valid_douyin_source_url(target): + raise RequestError("invalid restricted Douyin source URL", 400) + if is_douyin_content_url(target): + return {"url": target} + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + resolved = self.browser.resolve(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Douyin share resolution failed alias=%s reason=%s", alias, str(exc) + ) + raise RequestError("restricted Douyin share resolution failed") from exc + if not is_douyin_content_url(resolved): + raise RequestError("Douyin share resolved to an unsupported URL", 502) + return {"url": resolved} + + def get_douyin_media(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_douyin_generation(input) or not isinstance(target, str): + raise RequestError("invalid restricted browser request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.browser.get_media(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Douyin media download failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError("restricted browser media download failed") from exc + return { + "status": response.status, + "content_type": response.content_type, + "body_base64": response.body_base64, + } + + def douyin_identity(self, alias: str, input: dict) -> dict: + expected_account_key = input.get("expected_account_key", "") + if ( + not valid_douyin_generation(input) + or not isinstance(expected_account_key, str) + or not ACCOUNT_KEY_RE.fullmatch(expected_account_key) + ): + raise RequestError("invalid Douyin identity request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + identity = self.browser.identity(alias) + except DouyinError as exc: + LOG.warning( + "Douyin identity verification failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "Douyin login identity could not be verified" + ) from exc + if expected_account_key not in { + identity["uid"], + identity["sec_uid"], + identity["unique_id"], + }: + raise RequestError( + "Douyin identity does not match the expected account", 409 + ) + return identity + + def douyin_login_qr(self, alias: str, input: dict) -> dict: + if not valid_douyin_generation(input): + raise RequestError("invalid Douyin login QR request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + screen = self.browser.login_qr(alias) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Douyin login screen capture failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError("Douyin login screen could not be captured") from exc + return { + "content_type": screen.content_type, + "body_base64": screen.body_base64, + "qr_detected": screen.qr_detected, + } + + def get_xiaohongshu(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_url(target): + raise RequestError("invalid restricted Xiaohongshu request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.xiaohongshu_browser.get(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc) + ) + raise RequestError("restricted Xiaohongshu operation failed") from exc + return { + "status": response.status, + "body": response.body, + "challenge": response.challenge, + } + + def post_xiaohongshu(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + body = input.get("body") + if ( + not valid_xiaohongshu_generation(input) + or not valid_xhs_post_url(target) + or not isinstance(body, dict) + ): + raise RequestError("invalid restricted Xiaohongshu POST request", 400) + try: + encoded = json.dumps( + body, ensure_ascii=False, separators=(",", ":") + ).encode() + except (TypeError, ValueError) as exc: + raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.xiaohongshu_browser.post(alias, target, encoded) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc) + ) + raise RequestError("restricted Xiaohongshu operation failed") from exc + return { + "status": response.status, + "body": response.body, + "challenge": response.challenge, + } + + def resolve_xiaohongshu(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_source_url( + target + ): + raise RequestError("invalid restricted Xiaohongshu source URL", 400) + if not is_xiaohongshu_share_url(target): + return {"url": target} + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + resolved = self.xiaohongshu_browser.resolve(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Xiaohongshu share resolution failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "restricted Xiaohongshu share resolution failed" + ) from exc + if not valid_xiaohongshu_page_url(resolved): + raise RequestError("Xiaohongshu share resolved to an unsupported URL", 502) + return {"url": resolved} + + def get_xiaohongshu_media(self, alias: str, input: dict) -> dict: + target = input.get("url", "") + if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url( + target + ): + raise RequestError("invalid restricted Xiaohongshu media request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + response = self.xiaohongshu_browser.get_media(alias, target) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Xiaohongshu media download failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "restricted Xiaohongshu media download failed" + ) from exc + return { + "status": response.status, + "content_type": response.content_type, + "body_base64": response.body_base64, + } + + def xiaohongshu_identity(self, alias: str, input: dict) -> dict: + expected_account_key = input.get("expected_account_key", "") + if ( + not valid_xiaohongshu_generation(input) + or not isinstance(expected_account_key, str) + or not XHS_ACCOUNT_KEY_RE.fullmatch(expected_account_key) + ): + raise RequestError("invalid Xiaohongshu identity request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + identity = self.xiaohongshu_browser.identity(alias) + except DouyinError as exc: + LOG.warning( + "Xiaohongshu identity verification failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError( + "Xiaohongshu login identity could not be verified" + ) from exc + if identity.get("uid") != expected_account_key: + raise RequestError( + "Xiaohongshu identity does not match the expected account", 409 + ) + return identity + + def douyin_message_history(self, alias: str, input: dict) -> dict: + expected_uid = input.get("expected_uid", "") + target_uid = input.get("target_uid", "") + limit = input.get("limit", 100) + cursor = input.get("cursor", "") + if ( + not valid_douyin_generation(input) + or not isinstance(expected_uid, str) + or not isinstance(target_uid, str) + or not isinstance(cursor, str) + or len(cursor) > 500 + or type(limit) is not int + or not UID_RE.fullmatch(expected_uid) + or not UID_RE.fullmatch(target_uid) + or expected_uid == target_uid + or not 1 <= limit <= 200 + ): + raise RequestError("invalid Douyin message history request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + if cursor: + result = self.browser.message_history( + alias, expected_uid, target_uid, limit, cursor=cursor + ) + else: + result = self.browser.message_history( + alias, expected_uid, target_uid, limit + ) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning( + "Douyin message history failed alias=%s reason=%s", + alias, + str(exc), + ) + raise RequestError("Douyin message history failed") from exc + return result + + def douyin_action(self, alias: str, input: dict) -> dict: + expected_uid = input.get("expected_uid", "") + action = input.get("action", "") + target_uid = input.get("target_uid", "") + target_comment_id = input.get("target_comment_id", "") + target_work_id = input.get("target_work_id", "") + text = input.get("text", "") + confirm = input.get("confirm", False) + operation_id = input.get("operation_id", "") + if ( + not valid_douyin_generation(input) + or not isinstance(expected_uid, str) + or not isinstance(action, str) + or not isinstance(target_uid, str) + or not isinstance(target_comment_id, str) + or not isinstance(target_work_id, str) + or not isinstance(text, str) + or not isinstance(operation_id, str) + or len(operation_id) > 200 + or type(confirm) is not bool + or not UID_RE.fullmatch(expected_uid) + or action not in ACTIONS + ): + raise RequestError("invalid Douyin action request", 400) + if action == "repost": + raise RequestError("ACTION_UNAVAILABLE", 409) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + self._claim_action( + alias, + str(input.get("runtime_id", "")), + operation_id, + ) + try: + result = self.browser.action( + alias, + expected_uid, + action, + target_uid, + target_comment_id, + target_work_id, + text, + confirm, + ) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + self._handle_douyin_action_error(alias, action, exc, operation_id) + raise RequestError("Douyin action failed") from exc + except Exception: + self._release_action_ownership(alias, operation_id) + raise + else: + self._release_action_ownership(alias, operation_id) + return result + + def _handle_douyin_action_error( + self, alias: str, action: str, error: DouyinError, operation_id: str = "" + ) -> None: + if getattr(error, "uncertain", False) or "timed out" in str(error).lower(): + self._retain_action_ownership(alias, operation_id) + else: + self._release_action_ownership(alias, operation_id) + LOG.warning( + "Douyin action failed alias=%s action=%s reason=%s", + alias, + action, + str(error), + ) + + def _claim_action( + self, alias: str, runtime_id: str = "", operation_id: str = "" + ) -> None: + now = time.monotonic() + with self._action_ownership_lock: + until = self._uncertain_actions.get(alias, 0.0) + if until > now: + raise RequestError("previous Douyin action outcome is uncertain", 409) + if runtime_id: + marker = self.browser.action_ownership(alias) + if marker is not None: + raise RequestError("previous Douyin action outcome is uncertain", 409) + self.browser.set_action_ownership( + alias, + { + "runtime_id": runtime_id, + "operation_id": operation_id, + "started_at": time.time(), + }, + ) + with self._action_ownership_lock: + self._uncertain_actions.pop(alias, None) + self._uncertain_actions[alias] = 0.0 + + def _release_action_ownership(self, alias: str, operation_id: str = "") -> None: + if operation_id: + self.browser.clear_action_ownership(alias, operation_id) + with self._action_ownership_lock: + self._uncertain_actions.pop(alias, None) + + def _retain_action_ownership(self, alias: str, operation_id: str = "") -> None: + with self._action_ownership_lock: + # A timed-out page script may still finish its network write. Keep the + # alias blocked until the browser generation is removed or replaced. + self._uncertain_actions[alias] = math.inf + + def start_douyin_events(self, alias: str, input: dict) -> dict: + expected_uid = input.get("expected_uid", "") + if ( + not valid_douyin_generation(input) + or not isinstance(expected_uid, str) + or not UID_RE.fullmatch(expected_uid) + ): + raise RequestError("invalid Douyin event request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + self.browser.identity(alias, expected_uid) + return self.subscriptions.start(alias, expected_uid) + except DouyinError as exc: + LOG.warning( + "Douyin event listener start failed alias=%s uid=%s reason=%s", + alias, + expected_uid, + exc, + ) + raise RequestError("Douyin event listener could not start") from exc + + def poll_douyin_events(self, alias: str, input: dict, query: dict) -> list[dict]: + if not valid_douyin_generation(input): + raise RequestError("invalid Douyin event request", 400) + try: + limit = int(query.get("limit", ["50"])[0]) + wait = float(query.get("wait", ["25"])[0]) + except (IndexError, TypeError, ValueError) as exc: + raise RequestError("invalid event poll options", 400) from exc + if not 1 <= limit <= 100 or not 0 <= wait <= 30: + raise RequestError("invalid event poll options", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + try: + acknowledgements = query.get("ack", []) + delivery_ids = [ + value for raw in acknowledgements for value in raw.split(",") + ] + self.subscriptions.ack(alias, delivery_ids) + return self.subscriptions.poll(alias, limit, wait) + except DouyinError as exc: + LOG.warning( + "Douyin event listener poll failed alias=%s reason=%s", + alias, + exc, + ) + raise RequestError("Douyin event listener is unavailable") from exc + + def stop_douyin_events(self, alias: str, input: dict) -> None: + if not valid_douyin_generation(input): + raise RequestError("invalid Douyin event request", 400) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + self.subscriptions.stop(alias) + +class GatewayHTTPServer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + gateway: Gateway + + def __init__(self, address, gateway: Gateway): + super().__init__(address, GatewayHandler) + self.gateway = gateway + self._connections: set[socket.socket] = set() + self._connections_lock = threading.Lock() + self._connections_changed = threading.Condition(self._connections_lock) + + def process_request(self, request, client_address): + with self._connections_changed: + self._connections.add(cast(socket.socket, request)) + try: + super().process_request(request, client_address) + except Exception: + with self._connections_changed: + self._connections.discard(cast(socket.socket, request)) + self._connections_changed.notify_all() + raise + + def process_request_thread(self, request, client_address): + try: + super().process_request_thread(request, client_address) + finally: + with self._connections_changed: + self._connections.discard(cast(socket.socket, request)) + self._connections_changed.notify_all() + + def wait_for_requests(self, timeout: float) -> None: + deadline = time.monotonic() + timeout + with self._connections_changed: + while self._connections: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + self._connections_changed.wait(remaining) + if self._connections: + connections = list(self._connections) + else: + connections = [] + for connection in connections: + with suppress(OSError): + connection.shutdown(socket.SHUT_RDWR) + connection.close() + + +class GatewayHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def setup(self) -> None: + super().setup() + self.request.settimeout(5.0) + + def do_GET(self) -> None: + self._dispatch("GET") + + def do_POST(self) -> None: + self._dispatch("POST") + + def do_DELETE(self) -> None: + self._dispatch("DELETE") + + def log_message(self, format: str, *args) -> None: + LOG.info("http_request", extra={"request": format % args}) + + def _dispatch(self, method: str) -> None: + parsed = urlsplit(self.path) + if parsed.path == "/healthz": + self._respond(204, b"") + return + if not parsed.path.startswith("/v1/"): + self._respond(404, json_bytes({"error": "not found"})) + return + if not self._authorized(): + self._respond(401, json_bytes({"error": "gateway token rejected"})) + return + try: + needs_body = method in {"POST", "DELETE"} or ( + method == "GET" and parsed.path.endswith("/douyin/events") + ) + body = self._body() if needs_body else {} + result = self._route(method, parsed.path, parse_qs(parsed.query), body) + if result is None: + self._respond(204, b"") + elif isinstance(result, tuple): + status, value = result + if type(status) is not int: + raise RuntimeError("gateway route returned an invalid status") + self._respond(status, json_bytes(value)) + else: + self._respond(200, json_bytes(result)) + except (RuntimeError, OSError, ValueError, TypeError, KeyError) as exc: + self._handle_exception(parsed.path, exc) + + def _handle_exception(self, path: str, exc: Exception) -> None: + if isinstance(exc, RequestError): + payload = {"error": str(exc)} + if exc.network_id: + payload["network_id"] = exc.network_id + self._respond(exc.status, json_bytes(payload)) + elif isinstance(exc, FileNotFoundError): + self._respond(404, json_bytes({"error": str(exc)})) + elif isinstance(exc, BrowserRuntimeError): + payload = {"error": str(exc)} + if exc.network_id: + payload["network_id"] = exc.network_id + self._respond(exc.status, json_bytes(payload)) + elif isinstance(exc, ValueError): + self._respond(400, json_bytes({"error": str(exc)})) + else: + LOG.exception("gateway request failed", extra={"path": path}) + self._respond(500, json_bytes({"error": "gateway operation failed"})) + + def _route(self, method: str, path: str, query: dict, body: dict): + gateway = self.server_as_gateway().gateway + if method == "GET" and path == "/v1/info": + return gateway.info() + if method == "GET" and path == "/v1/browsers": + return gateway.list_browsers() + if method == "POST" and path == "/v1/browsers": + return 201, gateway.create(body) + match = re.fullmatch(r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})", path) + if match and method == "DELETE": + gateway.remove(match.group(1), body) + return None + match = re.fullmatch( + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/(start|stop|cancel|proxy)", path + ) + if match: + alias, action = match.groups() + if method == "POST" and action in {"start", "stop", "cancel"}: + gateway.change_state(alias, action, body) + return None + if method == "POST" and action == "proxy": + gateway.restore_proxy(alias, body) + return None + match = re.fullmatch( + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity|resolve)", + path, + ) + if match: + alias, action = match.groups() + if action == "get" and method == "POST": + return gateway.get_xiaohongshu(alias, body) + if action == "post" and method == "POST": + return gateway.post_xiaohongshu(alias, body) + if action == "media" and method == "POST": + return gateway.get_xiaohongshu_media(alias, body) + if action == "resolve" and method == "POST": + return gateway.resolve_xiaohongshu(alias, body) + if action == "identity" and method == "POST": + return gateway.xiaohongshu_identity(alias, body) + match = re.fullmatch( + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|resolve|login-qr|action|messages|events)", + path, + ) + if match: + alias, action = match.groups() + if action == "get" and method == "POST": + return gateway.get_douyin(alias, body) + if action == "resolve" and method == "POST": + return gateway.resolve_douyin(alias, body) + if action == "media" and method == "POST": + return gateway.get_douyin_media(alias, body) + if action == "identity" and method == "POST": + return gateway.douyin_identity(alias, body) + if action == "login-qr" and method == "POST": + return gateway.douyin_login_qr(alias, body) + if action == "action" and method == "POST": + return gateway.douyin_action(alias, body) + if action == "messages" and method == "POST": + return gateway.douyin_message_history(alias, body) + if action == "events": + if method == "POST": + return gateway.start_douyin_events(alias, body) + if method == "GET": + return gateway.poll_douyin_events(alias, body, query) + if method == "DELETE": + gateway.stop_douyin_events(alias, body) + return None + raise RequestError("not found", 404) + + def server_as_gateway(self) -> GatewayHTTPServer: + if not isinstance(self.server, GatewayHTTPServer): + raise TypeError("gateway HTTP server type is invalid") + return self.server + + def _authorized(self) -> bool: + supplied = self.headers.get("Authorization", "") + return hmac.compare_digest( + supplied, "Bearer " + self.server_as_gateway().gateway.token + ) + + def _body(self) -> dict: + length_text = self.headers.get("Content-Length") + if length_text is None: + raise RequestError("request body is required", 400) + try: + length = int(length_text) + except ValueError as exc: + raise RequestError("invalid request body length", 400) from exc + if length < 0 or length > 1 << 20: + raise RequestError("request body is too large", 400) + raw = self.rfile.read(length) + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise RequestError("request body must be one JSON object", 400) from exc + if not isinstance(value, dict): + raise RequestError("request body must be one JSON object", 400) + return value + + def _respond(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + +def json_bytes(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode() + + +def valid_xhs_query( + query: object, allowed: set[str], required: set[str] | None = None +) -> bool: + if not isinstance(query, dict) or not isinstance(allowed, set): + return False + required = required or set() + if not required.issubset(query) or not set(query).issubset(allowed): + return False + for key, values in query.items(): + if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1: + return False + if ( + not isinstance(values[0], str) + or len(values[0]) > 2048 + or "\r" in values[0] + or "\n" in values[0] + ): + return False + return True + + +def _valid_xhs_host(parsed: object, host: str) -> bool: + return ( + getattr(parsed, "scheme", "") == "https" + and getattr(parsed, "hostname", None) == host + and getattr(parsed, "port", None) is None + and getattr(parsed, "username", None) is None + and getattr(parsed, "password", None) is None + and getattr(parsed, "fragment", "") == "" + ) + + +def valid_xhs_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + if ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_IDENTITY_PATH + ): + return not query + if ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_USER_POSTED_PATH + ): + return ( + valid_xhs_query( + query, + { + "user_id", + "cursor", + "num", + "image_formats", + "xsec_source", + "xsec_token", + }, + {"user_id", "num"}, + ) + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) + and query["num"] == ["30"] + ) + if ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_COMMENTS_PATH + ): + return valid_xhs_query( + query, + { + "note_id", + "cursor", + "top_comment_id", + "image_formats", + "xsec_source", + "xsec_token", + }, + {"note_id", "cursor", "top_comment_id"}, + ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0])) + return False + + +def valid_xiaohongshu_url(raw: object) -> bool: + return valid_xhs_url(raw) + + +def valid_xiaohongshu_page_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + if not _valid_xhs_host(parsed, "www.xiaohongshu.com") or not valid_xhs_query( + query, {"xsec_source", "xsec_token"} + ): + return False + parts = parsed.path.strip("/").split("/") + return ( + len(parts) == 2 + and parts[0] == "explore" + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) + ) or ( + len(parts) == 3 + and parts[:2] == ["user", "profile"] + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[2])) + ) + + +def valid_xiaohongshu_source_url(raw: object) -> bool: + return valid_xiaohongshu_page_url(raw) or is_xiaohongshu_share_url(raw) + + +def valid_xhs_post_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + return ( + _valid_xhs_host(parsed, "so.xiaohongshu.com") + and parsed.path == XHS_SEARCH_PATH + and not query + ) or ( + _valid_xhs_host(parsed, "edith.xiaohongshu.com") + and parsed.path == XHS_FEED_PATH + and not query + ) + + +def valid_xiaohongshu_media_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + query = parse_qs(parsed.query, keep_blank_values=True) + except ValueError: + return False + if not _valid_xhs_host(parsed, "www.xiaohongshu.com"): + return False + parts = parsed.path.strip("/").split("/") + return ( + len(parts) == 2 + and parts[0] == "explore" + and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) + and valid_xhs_query(query, {"xsec_source", "xsec_token"}) + ) + + +def valid_douyin_source_url(raw: object) -> bool: + return is_douyin_share_url(raw) + + +def valid_douyin_url(raw: object) -> bool: + if not isinstance(raw, str): + return False + try: + parsed = urlsplit(raw) + except ValueError: + return False + if ( + parsed.scheme != "https" + or parsed.netloc != "www.douyin.com" + or parsed.username + or parsed.fragment + ): + return False + query = parse_qs(parsed.query, keep_blank_values=True) + if parsed.path == DOUYIN_IDENTITY_PATH: + return query == {"aid": ["6383"], "device_platform": ["webapp"]} + if parsed.path == DOUYIN_PROFILE_OTHER_PATH: + return valid_douyin_profile_query(query) + if parsed.path == DOUYIN_WORKS_PATH: + return valid_douyin_api_query(query, "sec_user_id", "max_cursor") + if parsed.path == DOUYIN_WORK_DETAIL_PATH: + return valid_douyin_work_detail_query(query) + if parsed.path == DOUYIN_COMMENTS_PATH: + return valid_douyin_api_query(query, "aweme_id", "cursor") + return False + + +def valid_douyin_work_detail_query(query: dict[str, list[str]]) -> bool: + return ( + len(query) == 3 + and query.get("aid") == ["6383"] + and query.get("device_platform") == ["webapp"] + and len(query.get("aweme_id", [])) == 1 + and bool(re.fullmatch(r"[1-9][0-9]{0,63}", query["aweme_id"][0])) + ) + + +def valid_douyin_profile_query(query: dict[str, list[str]]) -> bool: + account_fields = [field for field in ("user_id", "sec_user_id") if field in query] + return ( + len(query) == 3 + and query.get("aid") == ["6383"] + and query.get("device_platform") == ["webapp"] + and len(account_fields) == 1 + and valid_account_key_query(query, account_fields[0]) + ) + + +def valid_douyin_api_query( + query: dict[str, list[str]], account_field: str, cursor_field: str +) -> bool: + return ( + len(query) == 5 + and query.get("aid") == ["6383"] + and query.get("device_platform") == ["webapp"] + and valid_account_key_query(query, account_field) + and query.get("count") == ["20"] + and numeric_cursor(query.get(cursor_field)) + ) + + +def valid_account_key_query(query: dict[str, list[str]], key: str) -> bool: + return len(query.get(key, [])) == 1 and bool( + DOUYIN_ACCOUNT_KEY_RE.fullmatch(query[key][0]) + ) + + +def numeric_cursor(values: list[str] | None) -> bool: + if not values or len(values) != 1 or not values[0].isdigit(): + return False + try: + return int(values[0]) >= 0 + except ValueError: + return False + + +def parse_proxy_exit(value: object) -> ProxyExit: + try: + return _parse_proxy_exit(value) + except BrowserRuntimeError as exc: + raise RequestError(str(exc), exc.status, exc.network_id) from exc + + +def validate_proxy_exit(exit: ProxyExit) -> None: + try: + _validate_proxy_exit(exit) + except BrowserRuntimeError as exc: + raise RequestError(str(exc), exc.status, exc.network_id) from exc + + +def validate_create(input: dict) -> None: + try: + validate_runtime_input(input) + except BrowserRuntimeError as exc: + raise RequestError(str(exc), exc.status, exc.network_id) from exc + + +def proxy_port(proxy_url: str) -> int: + return urlsplit(proxy_url).port or 0 + + +def decode_generation( + value: dict, + require_runtime: bool, + require_network: bool, + allow_profile_purge: bool = False, +) -> dict: + allowed = {"binding_version", "runtime_id", "network_id"} + if allow_profile_purge: + allowed.add("purge_profile") + if not isinstance(value, dict) or set(value) - allowed: + raise RequestError( + "binding_version, runtime_id and network_id must identify the expected generation", + 400, + ) + binding = value.get("binding_version") + runtime = value.get("runtime_id", "") + network = value.get("network_id", "") + if ( + type(binding) is not int + or binding < 1 + or not isinstance(runtime, str) + or not isinstance(network, str) + or ( + runtime + and runtime != RUNTIME_CLEANUP_SENTINEL + and not RUNTIME_ID_RE.fullmatch(runtime) + ) + or (network and not NETWORK_ID_RE.fullmatch(network)) + or (require_runtime and not runtime) + or (require_network and not network) + ): + raise RequestError( + "binding_version, runtime_id and network_id must identify the expected generation", + 400, + ) + return {"binding_version": binding, "runtime_id": runtime, "network_id": network} + + +def validate_proxy_restore(value: dict, alias: str) -> None: + del alias + allowed = { + "binding_version", + "runtime_id", + "network_id", + "network_exit_id", + "network_exit", + } + if not isinstance(value, dict) or set(value) - allowed: + raise RequestError("invalid proxy recovery request", 400) + generation = decode_generation( + { + key: value.get(key) + for key in ("binding_version", "runtime_id", "network_id") + }, + True, + True, + ) + exit_id = value.get("network_exit_id", "") + if not isinstance(exit_id, str) or (exit_id and not EXIT_ID_RE.fullmatch(exit_id)): + raise RequestError("invalid proxy recovery request", 400) + exit = parse_proxy_exit(value.get("network_exit", {})) + direct = not exit_id and exit == ProxyExit("", "", 0) + if bool(exit_id) != (not direct): + raise RequestError("invalid proxy recovery request", 400) + if not direct: + validate_proxy_exit(exit) + value["network_exit"] = exit + value.update(generation) + value["network_exit_id"] = exit_id + + +def valid_douyin_generation(value: dict) -> bool: + if not isinstance(value, dict): + return False + binding = value.get("binding_version") + runtime = value.get("runtime_id", "") + network = value.get("network_id", "") + exit_id = value.get("network_exit_id", "") + return ( + type(binding) is int + and binding > 0 + and isinstance(runtime, str) + and isinstance(network, str) + and isinstance(exit_id, str) + and bool(RUNTIME_ID_RE.fullmatch(runtime)) + and bool(NETWORK_ID_RE.fullmatch(network)) + and (not exit_id or bool(EXIT_ID_RE.fullmatch(exit_id))) + ) + + +def valid_xiaohongshu_generation(value: dict) -> bool: + return valid_douyin_generation(value) + + +def _positive_float(env: Mapping[str, str], key: str, default: float, maximum: float) -> float: + raw = env.get(key, str(default)).strip() + try: + value = float(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be a number") from exc + if not 0 < value <= maximum: + raise ValueError(f"{key} is out of range") + return value + + +def _optional_positive_int(env: Mapping[str, str], key: str) -> int | None: + raw = env.get(key, "").strip() + if not raw: + return None + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be an integer") from exc + if value < 1: + raise ValueError(f"{key} is out of range") + return value + + +def _integer(env: Mapping[str, str], key: str, default: int) -> int: + raw = env.get(key, str(default)).strip() + try: + return int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be an integer") from exc + + +def _browser_versions(env: Mapping[str, str], default_version: str, default_path: str) -> dict[str, str]: + result = {default_version: default_path} + raw = env.get("BROWSER_VERSION_PATHS", "").strip() + if raw: + try: + decoded = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("BROWSER_VERSION_PATHS must be a JSON object") from exc + if not isinstance(decoded, dict) or not decoded: + raise ValueError("BROWSER_VERSION_PATHS must be a non-empty JSON object") + result = {} + for version, path in decoded.items(): + if not isinstance(version, str) or not BROWSER_VERSION_RE.fullmatch(version): + raise ValueError("BROWSER_VERSION_PATHS contains an invalid version") + if not isinstance(path, str) or not os.path.isabs(os.path.expanduser(path)): + raise ValueError("BROWSER_VERSION_PATHS contains a non-absolute path") + result[version] = os.path.abspath(os.path.expanduser(path)) + if default_version not in result: + result[default_version] = default_path + return result + + +def load_config(env: Mapping[str, str] | None = None) -> dict: + env = os.environ if env is None else env + listen = env.get("LISTEN_ADDR", "0.0.0.0:8081").strip() + token = env.get("GATEWAY_TOKEN", "").strip() + state_dir = os.path.abspath(os.path.expanduser(env.get( + "BROWSER_STATE_DIR", "~/.local/state/creatorhub/browser-gateway" + ).strip())) + profile_root = os.path.abspath(os.path.expanduser(env.get( + "BROWSER_PROFILE_ROOT", "~/.local/share/creatorhub/browser-profiles" + ).strip())) + browser_version = env.get("BROWSER_VERSION", "148.0.7778.215").strip() + browser_path = os.path.abspath(os.path.expanduser(env.get( + "BROWSER_PATH", + "~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome", + ).strip())) + node_id = env.get("NODE_ID", "").strip() + node_name = env.get("NODE_NAME", socket.gethostname()).strip() + host, port = split_listen_address(listen) + if len(token) < 16: + raise ValueError("GATEWAY_TOKEN must be at least 16 characters") + if not state_dir or not profile_root or not os.path.isabs(state_dir) or not os.path.isabs(profile_root): + raise ValueError("BROWSER_STATE_DIR and BROWSER_PROFILE_ROOT must be absolute") + if not BROWSER_VERSION_RE.fullmatch(browser_version): + raise ValueError("BROWSER_VERSION is invalid") + if not node_id: + node_id = "" + elif not re.fullmatch(r"^[a-z0-9][a-z0-9._-]{0,63}$", node_id): + raise ValueError("NODE_ID is invalid") + if not node_name or has_control(node_name) or len(node_name) > 128: + raise ValueError("NODE_NAME is invalid") + if not 1 <= port <= 65535: + raise ValueError("LISTEN_ADDR port must be 1..65535") + versions = _browser_versions(env, browser_version, browser_path) + external_display = _optional_positive_int(env, "RUNTIME_EXTERNAL_DISPLAY") + for version, path in versions.items(): + try: + available = os.path.isfile(path) and os.access(path, os.X_OK) + except (OSError, TypeError) as exc: + raise ValueError(f"browser version {version} is unavailable") from exc + if not available: + raise ValueError(f"browser version {version} is unavailable") + return { + "listen": (host, port), + "state_dir": state_dir, + "profile_root": profile_root, + "browser_versions": versions, + "browser_version": browser_version, + "node_id": node_id, + "node_name": node_name, + "token": token, + "cleanup_timeout": _positive_float(env, "RUNTIME_CLEANUP_TIMEOUT", 30.0, 300.0), + "ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 15.0, 300.0), + "min_free_bytes": _integer(env, "RUNTIME_MIN_FREE_BYTES", 20 * 1024**3), + "log_max_bytes": _integer(env, "RUNTIME_LOG_MAX_BYTES", 1 * 1024**3), + "profile_cache_max_bytes": _integer(env, "PROFILE_CACHE_MAX_BYTES", 20 * 1024**3), + "external_display": external_display, + } + + +def split_listen_address(value: str) -> tuple[str, int]: + if value.startswith(":"): + host, port_text = "", value[1:] + elif value.startswith("["): + closing = value.find("]:" ) + if closing <= 1: + raise ValueError("LISTEN_ADDR must be host:port") + host, port_text = value[1:closing], value[closing + 2:] + else: + if ":" not in value: + raise ValueError("LISTEN_ADDR must be host:port") + host, port_text = value.rsplit(":", 1) + try: + port = int(port_text) + except ValueError as exc: + raise ValueError("LISTEN_ADDR port must be an integer") from exc + return host, port + + +def load_stable_node_id(state_dir: str, configured: str) -> str: + if configured: + return configured + path = Path(state_dir) / "node-id" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if path.exists(): + value = path.read_text(encoding="utf-8").strip() + if not re.fullmatch(r"^[a-z0-9][a-z0-9._-]{0,63}$", value): + raise ValueError("persisted NODE_ID is invalid") + return value + value = "node-" + secrets.token_hex(16) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as output: + output.write(value + "\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + with suppress(FileNotFoundError): + temporary.unlink() + return value + + +def run() -> None: + config = load_config() + if os.geteuid() == 0: + raise RuntimeError("browser gateway must run as a non-root user") + logging.basicConfig(level=logging.INFO, format="%(message)s") + node_id = load_stable_node_id(config["state_dir"], config["node_id"]) + runtimes = NativeRuntimeManager( + state_dir=config["state_dir"], + profile_root=config["profile_root"], + node_id=node_id, + browser_versions=config["browser_versions"], + cleanup_timeout=config["cleanup_timeout"], + ready_timeout=config["ready_timeout"], + min_free_bytes=config["min_free_bytes"], + log_max_bytes=config["log_max_bytes"], + profile_cache_max_bytes=config["profile_cache_max_bytes"], + external_display=config["external_display"], + ) + gateway = Gateway(runtimes, config["token"], node_id) + server = GatewayHTTPServer(config["listen"], gateway) + LOG.info( + json.dumps( + { + "service": "browser-gateway", + "node_id": node_id, + "node_name": config["node_name"], + "listen_addr": f"{config['listen'][0]}:{config['listen'][1]}", + "browser_version": config["browser_version"], + "external_display": config["external_display"], + }, + ensure_ascii=False, + ) + ) + shutdown_requested = threading.Event() + + def request_shutdown(_signum, _frame) -> None: + if shutdown_requested.is_set(): + return + shutdown_requested.set() + threading.Thread( + target=server.shutdown, + name="gateway-shutdown", + daemon=True, + ).start() + + signal.signal(signal.SIGINT, request_shutdown) + signal.signal(signal.SIGTERM, request_shutdown) + try: + server.serve_forever() + finally: + server.wait_for_requests(30.0) + gateway.subscriptions.close() + runtimes.close() + server.server_close() + + +if __name__ == "__main__": + run() diff --git a/cmd/browser_gateway/test_gateway.py b/browser_gateway/test_gateway.py similarity index 99% rename from cmd/browser_gateway/test_gateway.py rename to browser_gateway/test_gateway.py index 7237d05..f5d5e97 100644 --- a/cmd/browser_gateway/test_gateway.py +++ b/browser_gateway/test_gateway.py @@ -15,8 +15,7 @@ from unittest.mock import Mock, patch import websocket -from .runtime import BrowserRuntimeError, GenerationConflict, NativeRuntimeManager, UnitStatus -from .douyin import ( +from .platform.douyin import ( BrowserResponse, CDPConnection, DouyinBrowser, @@ -50,8 +49,8 @@ from .proxy import ( _recv_exact, ) -gateway_module = import_module(f"{__package__}.gateway") -douyin_module = import_module(f"{__package__}.douyin") +gateway_module = import_module(f"{__package__}.server.http") +douyin_module = import_module(f"{__package__}.platform.douyin") proxy_module = import_module(f"{__package__}.proxy") Gateway = gateway_module.Gateway RequestError = gateway_module.RequestError @@ -397,7 +396,8 @@ class GatewayBusinessMethodTests(unittest.TestCase): self.browser.get.return_value = Mock(status=200, body="body", challenge="") self.browser.resolve.return_value = "https://www.douyin.com/video/123" self.browser.get_media.return_value = Mock( - status=200, content_type="video/mp4", body_base64="dm"); + status=200, content_type="video/mp4", body_base64="dm" + ) self.browser.identity.return_value = { "uid": "12345678901234567890", "sec_uid": "sec", diff --git a/cmd/browser_gateway/test_runtime.py b/browser_gateway/test_runtime.py similarity index 97% rename from cmd/browser_gateway/test_runtime.py rename to browser_gateway/test_runtime.py index f86f4af..069737d 100644 --- a/cmd/browser_gateway/test_runtime.py +++ b/browser_gateway/test_runtime.py @@ -96,12 +96,13 @@ class SystemdUnitManagerTests(unittest.TestCase): del kwargs return subprocess.CompletedProcess(args, 1, "", "rejected") - with tempfile.TemporaryDirectory() as directory: - with self.assertRaises(BrowserRuntimeError): - SystemdUnitManager("systemd-run", "systemctl", failed_start).start( - "x", ["/bin/true"], environment={}, working_directory=Path(directory), - stdout_path=Path(directory) / "x.log", limits={}, - ) + with tempfile.TemporaryDirectory() as directory, self.assertRaises( + BrowserRuntimeError + ): + SystemdUnitManager("systemd-run", "systemctl", failed_start).start( + "x", ["/bin/true"], environment={}, working_directory=Path(directory), + stdout_path=Path(directory) / "x.log", limits={}, + ) def missing_status(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: del kwargs @@ -136,7 +137,12 @@ class SystemdUnitManagerTests(unittest.TestCase): SystemdUnitManager("systemd-run", "systemctl", reset_fails).reset("x") def test_allocator_existing_and_file_lock_edges(self) -> None: - from .runtime import BrowserRuntimeError, DisplayAllocator, FileLock, PortAllocator + from .runtime import ( + BrowserRuntimeError, + DisplayAllocator, + FileLock, + PortAllocator, + ) with tempfile.TemporaryDirectory() as directory: lock_dir = Path(directory) diff --git a/cmd/browser_gateway/test_xiaohongshu.py b/browser_gateway/test_xiaohongshu.py similarity index 98% rename from cmd/browser_gateway/test_xiaohongshu.py rename to browser_gateway/test_xiaohongshu.py index fa97b61..2df5123 100644 --- a/cmd/browser_gateway/test_xiaohongshu.py +++ b/browser_gateway/test_xiaohongshu.py @@ -4,8 +4,8 @@ import unittest from typing import Any, cast from unittest.mock import Mock -from . import gateway as gateway_module -from . import xiaohongshu as xiaohongshu_module +from .platform import xiaohongshu as xiaohongshu_module +from .server import http as gateway_module Gateway = gateway_module.Gateway RequestError = gateway_module.RequestError diff --git a/cmd/browser_gateway/gateway.py b/cmd/browser_gateway/gateway.py index 409edb4..40bb1d4 100644 --- a/cmd/browser_gateway/gateway.py +++ b/cmd/browser_gateway/gateway.py @@ -1,1401 +1,6 @@ -"""CreatorHub native browser gateway.""" - -from __future__ import annotations - -import hmac -import json -import logging -import math -import os -import re -import secrets -import signal -import socket -import threading -import time -from collections.abc import Mapping -from contextlib import suppress -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import cast -from urllib.parse import parse_qs, urlsplit - -from .douyin import ( - ACCOUNT_KEY_RE, - ACTIONS, - COMMENTS_PATH, - IDENTITY_URL, - UID_RE, - WORKS_PATH, - DouyinBrowser, - DouyinError, - SubscriptionManager, - XiaohongshuBrowser, - is_douyin_content_url, - is_douyin_share_url, - is_xiaohongshu_share_url, -) -from .proxy import ProxyExit, ProxyRegistry -from .runtime import ( - ALIAS_RE, - BROWSER_VERSION_RE, - BrowserRuntimeError, - GenerationConflict, - NativeRuntimeManager, - NETWORK_ID_RE, - PROFILE_ID_RE, - RUNTIME_CLEANUP_SENTINEL, - RUNTIME_ID_RE, - has_control, - parse_proxy_exit as _parse_proxy_exit, - validate_proxy_exit as _validate_proxy_exit, - validate_runtime_input, -) - -LOG = logging.getLogger("creatorhub.gateway") -RUNTIME_ID_RE = RUNTIME_ID_RE -NETWORK_ID_RE = NETWORK_ID_RE -RUNTIME_CLEANUP_SENTINEL = RUNTIME_CLEANUP_SENTINEL -EXIT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$") -DOUYIN_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$") -DOUYIN_ORIGIN = "https://www.douyin.com" -DOUYIN_IDENTITY_PATH = "/aweme/v1/web/user/profile/self/" -DOUYIN_PROFILE_OTHER_PATH = "/aweme/v1/web/user/profile/other/" -DOUYIN_IDENTITY_URL = IDENTITY_URL -DOUYIN_WORKS_PATH = WORKS_PATH -DOUYIN_WORK_DETAIL_PATH = "/aweme/v1/web/aweme/detail/" -DOUYIN_COMMENTS_PATH = COMMENTS_PATH -XHS_ACCOUNT_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$") -XHS_IDENTITY_PATH = "/api/sns/web/v2/user/me" -XHS_USER_POSTED_PATH = "/api/sns/web/v1/user_posted" -XHS_COMMENTS_PATH = "/api/sns/web/v2/comment/page" -XHS_SEARCH_PATH = "/api/sns/web/v2/search/notes" -XHS_FEED_PATH = "/api/sns/web/v1/feed" - - -def _noop() -> None: - return None - - -class RequestError(BrowserRuntimeError): - pass - - -class Gateway: - def __init__( - self, - runtimes: NativeRuntimeManager, - token: str, - node_id: str, - browser: DouyinBrowser | None = None, - xiaohongshu_browser: XiaohongshuBrowser | None = None, - ) -> None: - self.runtimes = runtimes - self.token = token - self.node_id = node_id - self.browser = browser or DouyinBrowser(self._browser_endpoint) - self.xiaohongshu_browser = xiaohongshu_browser or XiaohongshuBrowser( - self._browser_endpoint - ) - self.proxies = runtimes.proxies - self.subscriptions = SubscriptionManager(self.browser) - self._action_ownership_lock = threading.Lock() - self._uncertain_actions: dict[str, float] = {} - - def _browser_endpoint(self, alias: str) -> str: - return self.runtimes.endpoint(alias) - - def list_browsers(self) -> list[dict]: - # Released generations remain in the runtime journal for cleanup/audit, - # but are not live browser bindings and must not collide with a reused alias. - return [ - browser - for browser in self.runtimes.list_public() - if browser.get("state") != "released" - ] - - def info(self) -> dict: - return { - "service": "browser-gateway", - "node_id": self.node_id, - "browser_versions": sorted(self.runtimes.browser_versions), - } - - def create(self, input: dict) -> dict: - return self.runtimes.create(input) - - def change_state(self, alias: str, action: str, input: dict) -> None: - generation = decode_generation( - input, require_runtime=True, require_network=True - ) - self.runtimes.change_state(alias, action, generation) - - def remove(self, alias: str, input: dict) -> None: - generation = decode_generation( - input, - require_runtime=False, - require_network=False, - allow_profile_purge=True, - ) - purge_profile = input.get("purge_profile", False) - if type(purge_profile) is not bool: - raise RequestError("purge_profile must be boolean", 400) - self.runtimes.remove(alias, generation, purge_profile) - - def restore_proxy(self, alias: str, input: dict) -> None: - validate_proxy_restore(input, alias) - generation = decode_generation( - {key: input.get(key) for key in ("binding_version", "runtime_id", "network_id")}, - True, - True, - ) - self.runtimes.restore_proxy(alias, generation, input["network_exit"]) - - def _require_generation(self, alias: str, generation: dict) -> tuple[str, bool]: - try: - record = self.runtimes.require_generation(alias, generation) - except FileNotFoundError: - return "", False - return record.runtime_id, True - - def _require_douyin_generation(self, alias: str, input: dict) -> None: - record = self.runtimes.require_generation(alias, input) - if input.get("network_exit_id", "") != record.network_exit_id: - raise RequestError("runtime proxy generation does not match request", 409) - - def _alias_lock(self, alias: str): - return self.runtimes.alias_lock(alias) - - def get_douyin(self, alias: str, input: dict) -> dict: - if not valid_douyin_generation(input) or not valid_douyin_url( - input.get("url", "") - ): - raise RequestError("invalid restricted browser request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - response = self.browser.get(alias, input["url"]) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Douyin GET failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError("restricted browser operation failed") from exc - return { - "status": response.status, - "body": response.body, - "challenge": response.challenge, - } - - def resolve_douyin(self, alias: str, input: dict) -> dict: - target = input.get("url", "") - if not valid_douyin_generation(input) or not valid_douyin_source_url(target): - raise RequestError("invalid restricted Douyin source URL", 400) - if is_douyin_content_url(target): - return {"url": target} - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - resolved = self.browser.resolve(alias, target) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Douyin share resolution failed alias=%s reason=%s", alias, str(exc) - ) - raise RequestError("restricted Douyin share resolution failed") from exc - if not is_douyin_content_url(resolved): - raise RequestError("Douyin share resolved to an unsupported URL", 502) - return {"url": resolved} - - def get_douyin_media(self, alias: str, input: dict) -> dict: - target = input.get("url", "") - if not valid_douyin_generation(input) or not isinstance(target, str): - raise RequestError("invalid restricted browser request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - response = self.browser.get_media(alias, target) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Douyin media download failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError("restricted browser media download failed") from exc - return { - "status": response.status, - "content_type": response.content_type, - "body_base64": response.body_base64, - } - - def douyin_identity(self, alias: str, input: dict) -> dict: - expected_account_key = input.get("expected_account_key", "") - if ( - not valid_douyin_generation(input) - or not isinstance(expected_account_key, str) - or not ACCOUNT_KEY_RE.fullmatch(expected_account_key) - ): - raise RequestError("invalid Douyin identity request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - identity = self.browser.identity(alias) - except DouyinError as exc: - LOG.warning( - "Douyin identity verification failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError( - "Douyin login identity could not be verified" - ) from exc - if expected_account_key not in { - identity["uid"], - identity["sec_uid"], - identity["unique_id"], - }: - raise RequestError( - "Douyin identity does not match the expected account", 409 - ) - return identity - - def douyin_login_qr(self, alias: str, input: dict) -> dict: - if not valid_douyin_generation(input): - raise RequestError("invalid Douyin login QR request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - screen = self.browser.login_qr(alias) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Douyin login screen capture failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError("Douyin login screen could not be captured") from exc - return { - "content_type": screen.content_type, - "body_base64": screen.body_base64, - "qr_detected": screen.qr_detected, - } - - def get_xiaohongshu(self, alias: str, input: dict) -> dict: - target = input.get("url", "") - if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_url(target): - raise RequestError("invalid restricted Xiaohongshu request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - response = self.xiaohongshu_browser.get(alias, target) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Xiaohongshu GET failed alias=%s reason=%s", alias, str(exc) - ) - raise RequestError("restricted Xiaohongshu operation failed") from exc - return { - "status": response.status, - "body": response.body, - "challenge": response.challenge, - } - - def post_xiaohongshu(self, alias: str, input: dict) -> dict: - target = input.get("url", "") - body = input.get("body") - if ( - not valid_xiaohongshu_generation(input) - or not valid_xhs_post_url(target) - or not isinstance(body, dict) - ): - raise RequestError("invalid restricted Xiaohongshu POST request", 400) - try: - encoded = json.dumps( - body, ensure_ascii=False, separators=(",", ":") - ).encode() - except (TypeError, ValueError) as exc: - raise RequestError("invalid restricted Xiaohongshu POST body", 400) from exc - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - response = self.xiaohongshu_browser.post(alias, target, encoded) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Xiaohongshu POST failed alias=%s reason=%s", alias, str(exc) - ) - raise RequestError("restricted Xiaohongshu operation failed") from exc - return { - "status": response.status, - "body": response.body, - "challenge": response.challenge, - } - - def resolve_xiaohongshu(self, alias: str, input: dict) -> dict: - target = input.get("url", "") - if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_source_url( - target - ): - raise RequestError("invalid restricted Xiaohongshu source URL", 400) - if not is_xiaohongshu_share_url(target): - return {"url": target} - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - resolved = self.xiaohongshu_browser.resolve(alias, target) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Xiaohongshu share resolution failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError( - "restricted Xiaohongshu share resolution failed" - ) from exc - if not valid_xiaohongshu_page_url(resolved): - raise RequestError("Xiaohongshu share resolved to an unsupported URL", 502) - return {"url": resolved} - - def get_xiaohongshu_media(self, alias: str, input: dict) -> dict: - target = input.get("url", "") - if not valid_xiaohongshu_generation(input) or not valid_xiaohongshu_media_url( - target - ): - raise RequestError("invalid restricted Xiaohongshu media request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - response = self.xiaohongshu_browser.get_media(alias, target) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Xiaohongshu media download failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError( - "restricted Xiaohongshu media download failed" - ) from exc - return { - "status": response.status, - "content_type": response.content_type, - "body_base64": response.body_base64, - } - - def xiaohongshu_identity(self, alias: str, input: dict) -> dict: - expected_account_key = input.get("expected_account_key", "") - if ( - not valid_xiaohongshu_generation(input) - or not isinstance(expected_account_key, str) - or not XHS_ACCOUNT_KEY_RE.fullmatch(expected_account_key) - ): - raise RequestError("invalid Xiaohongshu identity request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - identity = self.xiaohongshu_browser.identity(alias) - except DouyinError as exc: - LOG.warning( - "Xiaohongshu identity verification failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError( - "Xiaohongshu login identity could not be verified" - ) from exc - if identity.get("uid") != expected_account_key: - raise RequestError( - "Xiaohongshu identity does not match the expected account", 409 - ) - return identity - - def douyin_message_history(self, alias: str, input: dict) -> dict: - expected_uid = input.get("expected_uid", "") - target_uid = input.get("target_uid", "") - limit = input.get("limit", 100) - cursor = input.get("cursor", "") - if ( - not valid_douyin_generation(input) - or not isinstance(expected_uid, str) - or not isinstance(target_uid, str) - or not isinstance(cursor, str) - or len(cursor) > 500 - or type(limit) is not int - or not UID_RE.fullmatch(expected_uid) - or not UID_RE.fullmatch(target_uid) - or expected_uid == target_uid - or not 1 <= limit <= 200 - ): - raise RequestError("invalid Douyin message history request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - if cursor: - result = self.browser.message_history( - alias, expected_uid, target_uid, limit, cursor=cursor - ) - else: - result = self.browser.message_history( - alias, expected_uid, target_uid, limit - ) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - LOG.warning( - "Douyin message history failed alias=%s reason=%s", - alias, - str(exc), - ) - raise RequestError("Douyin message history failed") from exc - return result - - def douyin_action(self, alias: str, input: dict) -> dict: - expected_uid = input.get("expected_uid", "") - action = input.get("action", "") - target_uid = input.get("target_uid", "") - target_comment_id = input.get("target_comment_id", "") - target_work_id = input.get("target_work_id", "") - text = input.get("text", "") - confirm = input.get("confirm", False) - operation_id = input.get("operation_id", "") - if ( - not valid_douyin_generation(input) - or not isinstance(expected_uid, str) - or not isinstance(action, str) - or not isinstance(target_uid, str) - or not isinstance(target_comment_id, str) - or not isinstance(target_work_id, str) - or not isinstance(text, str) - or not isinstance(operation_id, str) - or len(operation_id) > 200 - or type(confirm) is not bool - or not UID_RE.fullmatch(expected_uid) - or action not in ACTIONS - ): - raise RequestError("invalid Douyin action request", 400) - if action == "repost": - raise RequestError("ACTION_UNAVAILABLE", 409) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - self._claim_action( - alias, - str(input.get("runtime_id", "")), - operation_id, - ) - try: - result = self.browser.action( - alias, - expected_uid, - action, - target_uid, - target_comment_id, - target_work_id, - text, - confirm, - ) - self._require_douyin_generation(alias, input) - except DouyinError as exc: - self._handle_douyin_action_error(alias, action, exc, operation_id) - raise RequestError("Douyin action failed") from exc - except Exception: - self._release_action_ownership(alias, operation_id) - raise - else: - self._release_action_ownership(alias, operation_id) - return result - - def _handle_douyin_action_error( - self, alias: str, action: str, error: DouyinError, operation_id: str = "" - ) -> None: - if getattr(error, "uncertain", False) or "timed out" in str(error).lower(): - self._retain_action_ownership(alias, operation_id) - else: - self._release_action_ownership(alias, operation_id) - LOG.warning( - "Douyin action failed alias=%s action=%s reason=%s", - alias, - action, - str(error), - ) - - def _claim_action( - self, alias: str, runtime_id: str = "", operation_id: str = "" - ) -> None: - now = time.monotonic() - with self._action_ownership_lock: - until = self._uncertain_actions.get(alias, 0.0) - if until > now: - raise RequestError("previous Douyin action outcome is uncertain", 409) - if runtime_id: - marker = self.browser.action_ownership(alias) - if marker is not None: - raise RequestError("previous Douyin action outcome is uncertain", 409) - self.browser.set_action_ownership( - alias, - { - "runtime_id": runtime_id, - "operation_id": operation_id, - "started_at": time.time(), - }, - ) - with self._action_ownership_lock: - self._uncertain_actions.pop(alias, None) - self._uncertain_actions[alias] = 0.0 - - def _release_action_ownership(self, alias: str, operation_id: str = "") -> None: - if operation_id: - self.browser.clear_action_ownership(alias, operation_id) - with self._action_ownership_lock: - self._uncertain_actions.pop(alias, None) - - def _retain_action_ownership(self, alias: str, operation_id: str = "") -> None: - with self._action_ownership_lock: - # A timed-out page script may still finish its network write. Keep the - # alias blocked until the browser generation is removed or replaced. - self._uncertain_actions[alias] = math.inf - - def start_douyin_events(self, alias: str, input: dict) -> dict: - expected_uid = input.get("expected_uid", "") - if ( - not valid_douyin_generation(input) - or not isinstance(expected_uid, str) - or not UID_RE.fullmatch(expected_uid) - ): - raise RequestError("invalid Douyin event request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - self.browser.identity(alias, expected_uid) - return self.subscriptions.start(alias, expected_uid) - except DouyinError as exc: - LOG.warning( - "Douyin event listener start failed alias=%s uid=%s reason=%s", - alias, - expected_uid, - exc, - ) - raise RequestError("Douyin event listener could not start") from exc - - def poll_douyin_events(self, alias: str, input: dict, query: dict) -> list[dict]: - if not valid_douyin_generation(input): - raise RequestError("invalid Douyin event request", 400) - try: - limit = int(query.get("limit", ["50"])[0]) - wait = float(query.get("wait", ["25"])[0]) - except (IndexError, TypeError, ValueError) as exc: - raise RequestError("invalid event poll options", 400) from exc - if not 1 <= limit <= 100 or not 0 <= wait <= 30: - raise RequestError("invalid event poll options", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - try: - acknowledgements = query.get("ack", []) - delivery_ids = [ - value for raw in acknowledgements for value in raw.split(",") - ] - self.subscriptions.ack(alias, delivery_ids) - return self.subscriptions.poll(alias, limit, wait) - except DouyinError as exc: - LOG.warning( - "Douyin event listener poll failed alias=%s reason=%s", - alias, - exc, - ) - raise RequestError("Douyin event listener is unavailable") from exc - - def stop_douyin_events(self, alias: str, input: dict) -> None: - if not valid_douyin_generation(input): - raise RequestError("invalid Douyin event request", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) - self.subscriptions.stop(alias) - -class GatewayHTTPServer(ThreadingHTTPServer): - daemon_threads = True - allow_reuse_address = True - - gateway: Gateway - - def __init__(self, address, gateway: Gateway): - super().__init__(address, GatewayHandler) - self.gateway = gateway - self._connections: set[socket.socket] = set() - self._connections_lock = threading.Lock() - self._connections_changed = threading.Condition(self._connections_lock) - - def process_request(self, request, client_address): - with self._connections_changed: - self._connections.add(cast(socket.socket, request)) - try: - super().process_request(request, client_address) - except Exception: - with self._connections_changed: - self._connections.discard(cast(socket.socket, request)) - self._connections_changed.notify_all() - raise - - def process_request_thread(self, request, client_address): - try: - super().process_request_thread(request, client_address) - finally: - with self._connections_changed: - self._connections.discard(cast(socket.socket, request)) - self._connections_changed.notify_all() - - def wait_for_requests(self, timeout: float) -> None: - deadline = time.monotonic() + timeout - with self._connections_changed: - while self._connections: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - self._connections_changed.wait(remaining) - if self._connections: - connections = list(self._connections) - else: - connections = [] - for connection in connections: - with suppress(OSError): - connection.shutdown(socket.SHUT_RDWR) - connection.close() - - -class GatewayHandler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def setup(self) -> None: - super().setup() - self.request.settimeout(5.0) - - def do_GET(self) -> None: - self._dispatch("GET") - - def do_POST(self) -> None: - self._dispatch("POST") - - def do_DELETE(self) -> None: - self._dispatch("DELETE") - - def log_message(self, format: str, *args) -> None: - LOG.info("http_request", extra={"request": format % args}) - - def _dispatch(self, method: str) -> None: - parsed = urlsplit(self.path) - if parsed.path == "/healthz": - self._respond(204, b"") - return - if not parsed.path.startswith("/v1/"): - self._respond(404, json_bytes({"error": "not found"})) - return - if not self._authorized(): - self._respond(401, json_bytes({"error": "gateway token rejected"})) - return - try: - needs_body = method in {"POST", "DELETE"} or ( - method == "GET" and parsed.path.endswith("/douyin/events") - ) - body = self._body() if needs_body else {} - result = self._route(method, parsed.path, parse_qs(parsed.query), body) - if result is None: - self._respond(204, b"") - elif isinstance(result, tuple): - status, value = result - if type(status) is not int: - raise RuntimeError("gateway route returned an invalid status") - self._respond(status, json_bytes(value)) - else: - self._respond(200, json_bytes(result)) - except (RuntimeError, OSError, ValueError, TypeError, KeyError) as exc: - self._handle_exception(parsed.path, exc) - - def _handle_exception(self, path: str, exc: Exception) -> None: - if isinstance(exc, RequestError): - payload = {"error": str(exc)} - if exc.network_id: - payload["network_id"] = exc.network_id - self._respond(exc.status, json_bytes(payload)) - elif isinstance(exc, FileNotFoundError): - self._respond(404, json_bytes({"error": str(exc)})) - elif isinstance(exc, BrowserRuntimeError): - payload = {"error": str(exc)} - if exc.network_id: - payload["network_id"] = exc.network_id - self._respond(exc.status, json_bytes(payload)) - elif isinstance(exc, ValueError): - self._respond(400, json_bytes({"error": str(exc)})) - else: - LOG.exception("gateway request failed", extra={"path": path}) - self._respond(500, json_bytes({"error": "gateway operation failed"})) - - def _route(self, method: str, path: str, query: dict, body: dict): - gateway = self.server_as_gateway().gateway - if method == "GET" and path == "/v1/info": - return gateway.info() - if method == "GET" and path == "/v1/browsers": - return gateway.list_browsers() - if method == "POST" and path == "/v1/browsers": - return 201, gateway.create(body) - match = re.fullmatch(r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})", path) - if match and method == "DELETE": - gateway.remove(match.group(1), body) - return None - match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/(start|stop|cancel|proxy)", path - ) - if match: - alias, action = match.groups() - if method == "POST" and action in {"start", "stop", "cancel"}: - gateway.change_state(alias, action, body) - return None - if method == "POST" and action == "proxy": - gateway.restore_proxy(alias, body) - return None - match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/xiaohongshu/(get|post|media|identity|resolve)", - path, - ) - if match: - alias, action = match.groups() - if action == "get" and method == "POST": - return gateway.get_xiaohongshu(alias, body) - if action == "post" and method == "POST": - return gateway.post_xiaohongshu(alias, body) - if action == "media" and method == "POST": - return gateway.get_xiaohongshu_media(alias, body) - if action == "resolve" and method == "POST": - return gateway.resolve_xiaohongshu(alias, body) - if action == "identity" and method == "POST": - return gateway.xiaohongshu_identity(alias, body) - match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|identity|resolve|login-qr|action|messages|events)", - path, - ) - if match: - alias, action = match.groups() - if action == "get" and method == "POST": - return gateway.get_douyin(alias, body) - if action == "resolve" and method == "POST": - return gateway.resolve_douyin(alias, body) - if action == "media" and method == "POST": - return gateway.get_douyin_media(alias, body) - if action == "identity" and method == "POST": - return gateway.douyin_identity(alias, body) - if action == "login-qr" and method == "POST": - return gateway.douyin_login_qr(alias, body) - if action == "action" and method == "POST": - return gateway.douyin_action(alias, body) - if action == "messages" and method == "POST": - return gateway.douyin_message_history(alias, body) - if action == "events": - if method == "POST": - return gateway.start_douyin_events(alias, body) - if method == "GET": - return gateway.poll_douyin_events(alias, body, query) - if method == "DELETE": - gateway.stop_douyin_events(alias, body) - return None - raise RequestError("not found", 404) - - def server_as_gateway(self) -> GatewayHTTPServer: - if not isinstance(self.server, GatewayHTTPServer): - raise TypeError("gateway HTTP server type is invalid") - return self.server - - def _authorized(self) -> bool: - supplied = self.headers.get("Authorization", "") - return hmac.compare_digest( - supplied, "Bearer " + self.server_as_gateway().gateway.token - ) - - def _body(self) -> dict: - length_text = self.headers.get("Content-Length") - if length_text is None: - raise RequestError("request body is required", 400) - try: - length = int(length_text) - except ValueError as exc: - raise RequestError("invalid request body length", 400) from exc - if length < 0 or length > 1 << 20: - raise RequestError("request body is too large", 400) - raw = self.rfile.read(length) - try: - value = json.loads(raw) - except json.JSONDecodeError as exc: - raise RequestError("request body must be one JSON object", 400) from exc - if not isinstance(value, dict): - raise RequestError("request body must be one JSON object", 400) - return value - - def _respond(self, status: int, body: bytes) -> None: - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - if body: - self.wfile.write(body) - - -def json_bytes(value: object) -> bytes: - return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode() - - -def valid_xhs_query( - query: object, allowed: set[str], required: set[str] | None = None -) -> bool: - if not isinstance(query, dict) or not isinstance(allowed, set): - return False - required = required or set() - if not required.issubset(query) or not set(query).issubset(allowed): - return False - for key, values in query.items(): - if not isinstance(key, str) or not isinstance(values, list) or len(values) != 1: - return False - if ( - not isinstance(values[0], str) - or len(values[0]) > 2048 - or "\r" in values[0] - or "\n" in values[0] - ): - return False - return True - - -def _valid_xhs_host(parsed: object, host: str) -> bool: - return ( - getattr(parsed, "scheme", "") == "https" - and getattr(parsed, "hostname", None) == host - and getattr(parsed, "port", None) is None - and getattr(parsed, "username", None) is None - and getattr(parsed, "password", None) is None - and getattr(parsed, "fragment", "") == "" - ) - - -def valid_xhs_url(raw: object) -> bool: - if not isinstance(raw, str): - return False - try: - parsed = urlsplit(raw) - query = parse_qs(parsed.query, keep_blank_values=True) - except ValueError: - return False - if ( - _valid_xhs_host(parsed, "edith.xiaohongshu.com") - and parsed.path == XHS_IDENTITY_PATH - ): - return not query - if ( - _valid_xhs_host(parsed, "edith.xiaohongshu.com") - and parsed.path == XHS_USER_POSTED_PATH - ): - return ( - valid_xhs_query( - query, - { - "user_id", - "cursor", - "num", - "image_formats", - "xsec_source", - "xsec_token", - }, - {"user_id", "num"}, - ) - and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["user_id"][0])) - and query["num"] == ["30"] - ) - if ( - _valid_xhs_host(parsed, "edith.xiaohongshu.com") - and parsed.path == XHS_COMMENTS_PATH - ): - return valid_xhs_query( - query, - { - "note_id", - "cursor", - "top_comment_id", - "image_formats", - "xsec_source", - "xsec_token", - }, - {"note_id", "cursor", "top_comment_id"}, - ) and bool(XHS_ACCOUNT_KEY_RE.fullmatch(query["note_id"][0])) - return False - - -def valid_xiaohongshu_url(raw: object) -> bool: - return valid_xhs_url(raw) - - -def valid_xiaohongshu_page_url(raw: object) -> bool: - if not isinstance(raw, str): - return False - try: - parsed = urlsplit(raw) - query = parse_qs(parsed.query, keep_blank_values=True) - except ValueError: - return False - if not _valid_xhs_host(parsed, "www.xiaohongshu.com") or not valid_xhs_query( - query, {"xsec_source", "xsec_token"} - ): - return False - parts = parsed.path.strip("/").split("/") - return ( - len(parts) == 2 - and parts[0] == "explore" - and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) - ) or ( - len(parts) == 3 - and parts[:2] == ["user", "profile"] - and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[2])) - ) - - -def valid_xiaohongshu_source_url(raw: object) -> bool: - return valid_xiaohongshu_page_url(raw) or is_xiaohongshu_share_url(raw) - - -def valid_xhs_post_url(raw: object) -> bool: - if not isinstance(raw, str): - return False - try: - parsed = urlsplit(raw) - query = parse_qs(parsed.query, keep_blank_values=True) - except ValueError: - return False - return ( - _valid_xhs_host(parsed, "so.xiaohongshu.com") - and parsed.path == XHS_SEARCH_PATH - and not query - ) or ( - _valid_xhs_host(parsed, "edith.xiaohongshu.com") - and parsed.path == XHS_FEED_PATH - and not query - ) - - -def valid_xiaohongshu_media_url(raw: object) -> bool: - if not isinstance(raw, str): - return False - try: - parsed = urlsplit(raw) - query = parse_qs(parsed.query, keep_blank_values=True) - except ValueError: - return False - if not _valid_xhs_host(parsed, "www.xiaohongshu.com"): - return False - parts = parsed.path.strip("/").split("/") - return ( - len(parts) == 2 - and parts[0] == "explore" - and bool(XHS_ACCOUNT_KEY_RE.fullmatch(parts[1])) - and valid_xhs_query(query, {"xsec_source", "xsec_token"}) - ) - - -def valid_douyin_source_url(raw: object) -> bool: - return is_douyin_share_url(raw) - - -def valid_douyin_url(raw: object) -> bool: - if not isinstance(raw, str): - return False - try: - parsed = urlsplit(raw) - except ValueError: - return False - if ( - parsed.scheme != "https" - or parsed.netloc != "www.douyin.com" - or parsed.username - or parsed.fragment - ): - return False - query = parse_qs(parsed.query, keep_blank_values=True) - if parsed.path == DOUYIN_IDENTITY_PATH: - return query == {"aid": ["6383"], "device_platform": ["webapp"]} - if parsed.path == DOUYIN_PROFILE_OTHER_PATH: - return valid_douyin_profile_query(query) - if parsed.path == DOUYIN_WORKS_PATH: - return valid_douyin_api_query(query, "sec_user_id", "max_cursor") - if parsed.path == DOUYIN_WORK_DETAIL_PATH: - return valid_douyin_work_detail_query(query) - if parsed.path == DOUYIN_COMMENTS_PATH: - return valid_douyin_api_query(query, "aweme_id", "cursor") - return False - - -def valid_douyin_work_detail_query(query: dict[str, list[str]]) -> bool: - return ( - len(query) == 3 - and query.get("aid") == ["6383"] - and query.get("device_platform") == ["webapp"] - and len(query.get("aweme_id", [])) == 1 - and bool(re.fullmatch(r"[1-9][0-9]{0,63}", query["aweme_id"][0])) - ) - - -def valid_douyin_profile_query(query: dict[str, list[str]]) -> bool: - account_fields = [field for field in ("user_id", "sec_user_id") if field in query] - return ( - len(query) == 3 - and query.get("aid") == ["6383"] - and query.get("device_platform") == ["webapp"] - and len(account_fields) == 1 - and valid_account_key_query(query, account_fields[0]) - ) - - -def valid_douyin_api_query( - query: dict[str, list[str]], account_field: str, cursor_field: str -) -> bool: - return ( - len(query) == 5 - and query.get("aid") == ["6383"] - and query.get("device_platform") == ["webapp"] - and valid_account_key_query(query, account_field) - and query.get("count") == ["20"] - and numeric_cursor(query.get(cursor_field)) - ) - - -def valid_account_key_query(query: dict[str, list[str]], key: str) -> bool: - return len(query.get(key, [])) == 1 and bool( - DOUYIN_ACCOUNT_KEY_RE.fullmatch(query[key][0]) - ) - - -def numeric_cursor(values: list[str] | None) -> bool: - if not values or len(values) != 1 or not values[0].isdigit(): - return False - try: - return int(values[0]) >= 0 - except ValueError: - return False - - -def parse_proxy_exit(value: object) -> ProxyExit: - try: - return _parse_proxy_exit(value) - except BrowserRuntimeError as exc: - raise RequestError(str(exc), exc.status, exc.network_id) from exc - - -def validate_proxy_exit(exit: ProxyExit) -> None: - try: - _validate_proxy_exit(exit) - except BrowserRuntimeError as exc: - raise RequestError(str(exc), exc.status, exc.network_id) from exc - - -def validate_create(input: dict) -> None: - try: - validate_runtime_input(input) - except BrowserRuntimeError as exc: - raise RequestError(str(exc), exc.status, exc.network_id) from exc - - -def proxy_port(proxy_url: str) -> int: - return urlsplit(proxy_url).port or 0 - - -def decode_generation( - value: dict, - require_runtime: bool, - require_network: bool, - allow_profile_purge: bool = False, -) -> dict: - allowed = {"binding_version", "runtime_id", "network_id"} - if allow_profile_purge: - allowed.add("purge_profile") - if not isinstance(value, dict) or set(value) - allowed: - raise RequestError( - "binding_version, runtime_id and network_id must identify the expected generation", - 400, - ) - binding = value.get("binding_version") - runtime = value.get("runtime_id", "") - network = value.get("network_id", "") - if ( - type(binding) is not int - or binding < 1 - or not isinstance(runtime, str) - or not isinstance(network, str) - or ( - runtime - and runtime != RUNTIME_CLEANUP_SENTINEL - and not RUNTIME_ID_RE.fullmatch(runtime) - ) - or (network and not NETWORK_ID_RE.fullmatch(network)) - or (require_runtime and not runtime) - or (require_network and not network) - ): - raise RequestError( - "binding_version, runtime_id and network_id must identify the expected generation", - 400, - ) - return {"binding_version": binding, "runtime_id": runtime, "network_id": network} - - -def validate_proxy_restore(value: dict, alias: str) -> None: - del alias - allowed = { - "binding_version", - "runtime_id", - "network_id", - "network_exit_id", - "network_exit", - } - if not isinstance(value, dict) or set(value) - allowed: - raise RequestError("invalid proxy recovery request", 400) - generation = decode_generation( - { - key: value.get(key) - for key in ("binding_version", "runtime_id", "network_id") - }, - True, - True, - ) - exit_id = value.get("network_exit_id", "") - if not isinstance(exit_id, str) or (exit_id and not EXIT_ID_RE.fullmatch(exit_id)): - raise RequestError("invalid proxy recovery request", 400) - exit = parse_proxy_exit(value.get("network_exit", {})) - direct = not exit_id and exit == ProxyExit("", "", 0) - if bool(exit_id) != (not direct): - raise RequestError("invalid proxy recovery request", 400) - if not direct: - validate_proxy_exit(exit) - value["network_exit"] = exit - value.update(generation) - value["network_exit_id"] = exit_id - - -def valid_douyin_generation(value: dict) -> bool: - if not isinstance(value, dict): - return False - binding = value.get("binding_version") - runtime = value.get("runtime_id", "") - network = value.get("network_id", "") - exit_id = value.get("network_exit_id", "") - return ( - type(binding) is int - and binding > 0 - and isinstance(runtime, str) - and isinstance(network, str) - and isinstance(exit_id, str) - and bool(RUNTIME_ID_RE.fullmatch(runtime)) - and bool(NETWORK_ID_RE.fullmatch(network)) - and (not exit_id or bool(EXIT_ID_RE.fullmatch(exit_id))) - ) - - -def valid_xiaohongshu_generation(value: dict) -> bool: - return valid_douyin_generation(value) - - -def _positive_float(env: Mapping[str, str], key: str, default: float, maximum: float) -> float: - raw = env.get(key, str(default)).strip() - try: - value = float(raw) - except (TypeError, ValueError) as exc: - raise ValueError(f"{key} must be a number") from exc - if not 0 < value <= maximum: - raise ValueError(f"{key} is out of range") - return value - - -def _optional_positive_int(env: Mapping[str, str], key: str) -> int | None: - raw = env.get(key, "").strip() - if not raw: - return None - try: - value = int(raw) - except (TypeError, ValueError) as exc: - raise ValueError(f"{key} must be an integer") from exc - if value < 1: - raise ValueError(f"{key} is out of range") - return value - - -def _browser_versions(env: Mapping[str, str], default_version: str, default_path: str) -> dict[str, str]: - result = {default_version: default_path} - raw = env.get("BROWSER_VERSION_PATHS", "").strip() - if raw: - try: - decoded = json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError("BROWSER_VERSION_PATHS must be a JSON object") from exc - if not isinstance(decoded, dict) or not decoded: - raise ValueError("BROWSER_VERSION_PATHS must be a non-empty JSON object") - result = {} - for version, path in decoded.items(): - if not isinstance(version, str) or not BROWSER_VERSION_RE.fullmatch(version): - raise ValueError("BROWSER_VERSION_PATHS contains an invalid version") - if not isinstance(path, str) or not os.path.isabs(os.path.expanduser(path)): - raise ValueError("BROWSER_VERSION_PATHS contains a non-absolute path") - result[version] = os.path.abspath(os.path.expanduser(path)) - if default_version not in result: - result[default_version] = default_path - return result - - -def load_config(env: Mapping[str, str] | None = None) -> dict: - env = os.environ if env is None else env - listen = env.get("LISTEN_ADDR", "0.0.0.0:8081").strip() - token = env.get("GATEWAY_TOKEN", "").strip() - state_dir = os.path.abspath(os.path.expanduser(env.get( - "BROWSER_STATE_DIR", "~/.local/state/creatorhub/browser-gateway" - ).strip())) - profile_root = os.path.abspath(os.path.expanduser(env.get( - "BROWSER_PROFILE_ROOT", "~/.local/share/creatorhub/browser-profiles" - ).strip())) - browser_version = env.get("BROWSER_VERSION", "148.0.7778.215").strip() - browser_path = os.path.abspath(os.path.expanduser(env.get( - "BROWSER_PATH", - "~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome", - ).strip())) - node_id = env.get("NODE_ID", "").strip() - node_name = env.get("NODE_NAME", socket.gethostname()).strip() - host, port = split_listen_address(listen) - if len(token) < 16: - raise ValueError("GATEWAY_TOKEN must be at least 16 characters") - if not state_dir or not profile_root or not os.path.isabs(state_dir) or not os.path.isabs(profile_root): - raise ValueError("BROWSER_STATE_DIR and BROWSER_PROFILE_ROOT must be absolute") - if not BROWSER_VERSION_RE.fullmatch(browser_version): - raise ValueError("BROWSER_VERSION is invalid") - if not node_id: - node_id = "" - elif not re.fullmatch(r"^[a-z0-9][a-z0-9._-]{0,63}$", node_id): - raise ValueError("NODE_ID is invalid") - if not node_name or has_control(node_name) or len(node_name) > 128: - raise ValueError("NODE_NAME is invalid") - if not 1 <= port <= 65535: - raise ValueError("LISTEN_ADDR port must be 1..65535") - versions = _browser_versions(env, browser_version, browser_path) - external_display = _optional_positive_int(env, "RUNTIME_EXTERNAL_DISPLAY") - for version, path in versions.items(): - if not os.path.isfile(path) or not os.access(path, os.X_OK): - raise ValueError(f"browser version {version} is unavailable") - return { - "listen": (host, port), - "state_dir": state_dir, - "profile_root": profile_root, - "browser_versions": versions, - "browser_version": browser_version, - "node_id": node_id, - "node_name": node_name, - "token": token, - "cleanup_timeout": _positive_float(env, "RUNTIME_CLEANUP_TIMEOUT", 30.0, 300.0), - "ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 15.0, 300.0), - "min_free_bytes": int(env.get("RUNTIME_MIN_FREE_BYTES", str(20 * 1024**3))), - "log_max_bytes": int(env.get("RUNTIME_LOG_MAX_BYTES", str(1 * 1024**3))), - "profile_cache_max_bytes": int(env.get("PROFILE_CACHE_MAX_BYTES", str(20 * 1024**3))), - "external_display": external_display, - } - - -def split_listen_address(value: str) -> tuple[str, int]: - if value.startswith(":"): - host, port_text = "", value[1:] - elif value.startswith("["): - closing = value.find("]:" ) - if closing <= 1: - raise ValueError("LISTEN_ADDR must be host:port") - host, port_text = value[1:closing], value[closing + 2:] - else: - if ":" not in value: - raise ValueError("LISTEN_ADDR must be host:port") - host, port_text = value.rsplit(":", 1) - try: - port = int(port_text) - except ValueError as exc: - raise ValueError("LISTEN_ADDR port must be an integer") from exc - return host, port - - -def load_stable_node_id(state_dir: str, configured: str) -> str: - if configured: - return configured - path = Path(state_dir) / "node-id" - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - if path.exists(): - value = path.read_text(encoding="utf-8").strip() - if not re.fullmatch(r"^[a-z0-9][a-z0-9._-]{0,63}$", value): - raise ValueError("persisted NODE_ID is invalid") - return value - value = "node-" + secrets.token_hex(16) - temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") - fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - with os.fdopen(fd, "w", encoding="utf-8") as output: - output.write(value + "\n") - output.flush() - os.fsync(output.fileno()) - os.replace(temporary, path) - finally: - with suppress(FileNotFoundError): - temporary.unlink() - return value - - -def run() -> None: - config = load_config() - if os.geteuid() == 0: - raise RuntimeError("browser gateway must run as a non-root user") - logging.basicConfig(level=logging.INFO, format="%(message)s") - node_id = load_stable_node_id(config["state_dir"], config["node_id"]) - runtimes = NativeRuntimeManager( - state_dir=config["state_dir"], - profile_root=config["profile_root"], - node_id=node_id, - browser_versions=config["browser_versions"], - cleanup_timeout=config["cleanup_timeout"], - ready_timeout=config["ready_timeout"], - min_free_bytes=config["min_free_bytes"], - log_max_bytes=config["log_max_bytes"], - profile_cache_max_bytes=config["profile_cache_max_bytes"], - external_display=config["external_display"], - ) - gateway = Gateway(runtimes, config["token"], node_id) - server = GatewayHTTPServer(config["listen"], gateway) - LOG.info( - json.dumps( - { - "service": "browser-gateway", - "node_id": node_id, - "node_name": config["node_name"], - "listen_addr": f"{config['listen'][0]}:{config['listen'][1]}", - "browser_version": config["browser_version"], - "external_display": config["external_display"], - }, - ensure_ascii=False, - ) - ) - shutdown_requested = threading.Event() - - def request_shutdown(_signum, _frame) -> None: - if shutdown_requested.is_set(): - return - shutdown_requested.set() - threading.Thread( - target=server.shutdown, - name="gateway-shutdown", - daemon=True, - ).start() - - signal.signal(signal.SIGINT, request_shutdown) - signal.signal(signal.SIGTERM, request_shutdown) - try: - server.serve_forever() - finally: - server.wait_for_requests(30.0) - gateway.subscriptions.close() - runtimes.close() - server.server_close() +"""Compatibility process launcher for the native browser gateway.""" +from browser_gateway.server.http import run if __name__ == "__main__": run() diff --git a/cmd/browser_gateway/xiaohongshu.py b/cmd/browser_gateway/xiaohongshu.py deleted file mode 100644 index 3268120..0000000 --- a/cmd/browser_gateway/xiaohongshu.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Xiaohongshu gateway facade. - -The browser implementation lives next to the existing Douyin browser so both -platforms share the CDP transport and response limits without duplicating it. -""" - -from .douyin import ( - XHS_ALLOWED_HOSTS, - XHS_API_ORIGIN, - XHS_IDENTITY_URL, - XHS_ORIGIN, - XHS_SEARCH_ORIGIN, - XiaohongshuBrowser, - is_xiaohongshu_media_url, - is_xiaohongshu_share_url, - is_xiaohongshu_url, -) - -__all__ = [ - "XHS_ALLOWED_HOSTS", - "XHS_API_ORIGIN", - "XHS_IDENTITY_URL", - "XHS_ORIGIN", - "XHS_SEARCH_ORIGIN", - "XiaohongshuBrowser", - "is_xiaohongshu_media_url", - "is_xiaohongshu_share_url", - "is_xiaohongshu_url", -] diff --git a/cmd/control-plane/main.go b/cmd/control-plane/main.go index 9554e54..234bc09 100644 --- a/cmd/control-plane/main.go +++ b/cmd/control-plane/main.go @@ -1,382 +1,15 @@ package main import ( - "context" - "crypto/sha256" - "crypto/subtle" - "encoding/base64" - "errors" - "io" - "net" - "net/url" "os" - "os/signal" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" - "git.ipao.vip/rogee/creator-hub/internal/taskstate" - "github.com/gofiber/fiber/v3" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/app" "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/viper" ) -type config struct { - listenAddr, webDir, databaseURL, credentialStoreDir string - username, password string - aiAPIKey, aiBaseURL string - credentialMasterKey []byte - logLevel logrus.Level -} - func main() { - logrus.SetFormatter(&logrus.JSONFormatter{}) - if err := execute(newCommand()); err != nil { + if err := app.Execute(); err != nil { logrus.WithField("service", "control-plane").WithError(err).Error("service stopped") os.Exit(1) } } - -func execute(command *cobra.Command) error { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - return command.ExecuteContext(ctx) -} - -func newCommand() *cobra.Command { - command := &cobra.Command{ - Use: "control-plane", - Short: "Run the CreatorHub control plane", - Args: cobra.NoArgs, - SilenceErrors: true, - SilenceUsage: true, - RunE: func(command *cobra.Command, _ []string) error { - cfg, err := loadConfig() - if err != nil { - return err - } - credentials, err := newPersistentCredentialBridge(cfg.credentialStoreDir, cfg.credentialMasterKey) - if err != nil { - return err - } - logrus.SetLevel(cfg.logLevel) - phaseAStore, err := phasea.Open(command.Context(), cfg.databaseURL) - if err != nil { - return err - } - defer phaseAStore.Close() - hubStore, err := hub.Open(command.Context(), cfg.databaseURL) - if err != nil { - return err - } - defer hubStore.Close() - creatorStore, err := creator.Open(command.Context(), cfg.databaseURL) - if err != nil { - return err - } - defer creatorStore.Close() - creatorStore.SetSecretBridge(creatorSecretBridge{bridge: credentials}) - notify := newAttentionNotifier(os.Stderr) - phaseAStore.SetTaskNotifier(notify) - hubStore.SetTaskNotifier(notify) - logStartup(cfg) - heartbeatContext, stopHeartbeat := context.WithCancel(command.Context()) - heartbeatDone := make(chan struct{}) - go func() { - defer close(heartbeatDone) - runtimeLeaseHeartbeat(heartbeatContext, hubStore) - }() - creatorScheduleContext, stopCreatorScheduler := context.WithCancel(command.Context()) - creatorScheduleDone := make(chan struct{}) - go func() { - defer close(creatorScheduleDone) - runCreatorScheduler(creatorScheduleContext, creatorStore, phaseAStore, hubStore) - }() - creatorEventContext, stopCreatorEvents := context.WithCancel(command.Context()) - creatorEventDone := make(chan struct{}) - creatorEventExecutor := creatorGatewayActionExecutor{store: creatorStore, phaseAStore: phaseAStore, hubStore: hubStore} - creatorAI := &creator.ConfiguredBailian{Store: creatorStore, APIKey: cfg.aiAPIKey, BaseURL: cfg.aiBaseURL} - go func() { - defer close(creatorEventDone) - runCreatorEventListeners(creatorEventContext, creatorStore, phaseAStore, hubStore, creatorEventExecutor, creatorAI) - }() - listenErr := newHandlerWithCreatorAndAI(cfg.webDir, cfg.username, cfg.password, phaseAStore, hubStore, credentials, creatorStore, creatorAI, creatorAI).Listen(cfg.listenAddr, fiber.ListenConfig{ - GracefulContext: command.Context(), - DisableStartupMessage: true, - }) - stopCreatorScheduler() - stopCreatorEvents() - stopHeartbeat() - <-creatorScheduleDone - <-creatorEventDone - <-heartbeatDone - return listenErr - }, - } - return command -} - -func newAttentionNotifier(output io.Writer) taskstate.Notifier { - logger := logrus.New() - logger.SetFormatter(&logrus.JSONFormatter{}) - logger.SetOutput(output) - return func(transition taskstate.Transition) { - if transition.State != "policy_hold" && transition.State != "needs_confirmation" { - return - } - logger.WithFields(logrus.Fields{ - "service": "control-plane", "notification_channel": "structured_log", "event_type": transition.State, - "reason_code": transition.ReasonCode, "account_id": transition.AccountID, "task_id": transition.TaskID, - }).Warn("operator attention required") - } -} - -func runtimeLeaseHeartbeat(ctx context.Context, store hubStore) { - ticker := time.NewTicker(20 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if err := reconcileRuntimeLeases(ctx, store, defaultNetworkExitProbe(), nil); err != nil && ctx.Err() == nil { - logrus.WithField("service", "control-plane").WithError(err).Warn("runtime lease reconciliation failed") - } - } - } -} - -func logStartup(cfg config) { - logrus.WithFields(logrus.Fields{ - "service": "control-plane", - "listen_addr": cfg.listenAddr, - }).Info("service starting") -} - -func loadConfig() (config, error) { - v := viper.New() - v.SetDefault("listen_addr", ":8080") - v.SetDefault("web_dir", "web/dist") - v.SetDefault("database_url", "postgres://creatorhub@127.0.0.1:5432/creatorhub?sslmode=disable") - v.SetDefault("credential_store_dir", "/var/lib/creatorhub/credentials") - v.SetDefault("log_level", "info") - _ = v.BindEnv("listen_addr", "LISTEN_ADDR") - _ = v.BindEnv("web_dir", "WEB_DIR") - _ = v.BindEnv("database_url", "DATABASE_URL") - _ = v.BindEnv("credential_store_dir", "CREATORHUB_CREDENTIAL_STORE_DIR") - _ = v.BindEnv("credential_master_key", "CREATORHUB_CREDENTIAL_MASTER_KEY") - _ = v.BindEnv("log_level", "LOG_LEVEL") - _ = v.BindEnv("username", "CONTROL_PLANE_USERNAME") - _ = v.BindEnv("password", "CONTROL_PLANE_PASSWORD") - _ = v.BindEnv("ai_api_key", "BAILIAN_API_KEY") - _ = v.BindEnv("ai_base_url", "BAILIAN_BASE_URL") - - level, err := logrus.ParseLevel(v.GetString("log_level")) - if err != nil { - return config{}, errors.New("LOG_LEVEL must be panic, fatal, error, warn, info, debug, or trace") - } - cfg := config{ - listenAddr: strings.TrimSpace(v.GetString("listen_addr")), - webDir: strings.TrimSpace(v.GetString("web_dir")), - databaseURL: strings.TrimSpace(v.GetString("database_url")), - credentialStoreDir: strings.TrimSpace(v.GetString("credential_store_dir")), - username: strings.TrimSpace(v.GetString("username")), - password: v.GetString("password"), - aiAPIKey: strings.TrimSpace(v.GetString("ai_api_key")), - aiBaseURL: strings.TrimSpace(v.GetString("ai_base_url")), - logLevel: level, - } - if cfg.listenAddr == "" { - return config{}, errors.New("LISTEN_ADDR must not be empty") - } - if err := validateListenAddr(cfg.listenAddr); err != nil { - return config{}, err - } - if cfg.webDir == "" { - return config{}, errors.New("WEB_DIR must not be empty") - } - if cfg.username == "" || strings.Contains(cfg.username, ":") { - return config{}, errors.New("CONTROL_PLANE_USERNAME must be non-empty and contain no colon") - } - if len(cfg.password) < 6 { - return config{}, errors.New("CONTROL_PLANE_PASSWORD must contain at least 6 bytes") - } - if !filepath.IsAbs(cfg.credentialStoreDir) { - return config{}, errors.New("CREATORHUB_CREDENTIAL_STORE_DIR must be absolute") - } - cfg.credentialMasterKey, err = base64.StdEncoding.DecodeString(strings.TrimSpace(v.GetString("credential_master_key"))) - if err != nil || len(cfg.credentialMasterKey) != 32 { - return config{}, errors.New("CREATORHUB_CREDENTIAL_MASTER_KEY must be base64 for exactly 32 bytes") - } - databaseURL, err := url.Parse(cfg.databaseURL) - if err != nil || databaseURL.Host == "" || databaseURL.Path == "" || databaseURL.Path == "/" || - (databaseURL.Scheme != "postgres" && databaseURL.Scheme != "postgresql") { - return config{}, errors.New("DATABASE_URL must be a postgres URL with a host") - } - if cfg.aiBaseURL != "" { - aiURL, parseErr := url.Parse(cfg.aiBaseURL) - if parseErr != nil || aiURL.Host == "" || (aiURL.Scheme != "http" && aiURL.Scheme != "https") || aiURL.User != nil { - return config{}, errors.New("BAILIAN_BASE_URL must be an HTTP(S) URL without credentials") - } - } - return cfg, nil -} - -func validateListenAddr(addr string) error { - _, port, err := net.SplitHostPort(addr) - if err != nil { - return errors.New("LISTEN_ADDR must be a host:port address") - } - number, err := strconv.Atoi(port) - if err != nil || number < 1 || number > 65535 { - return errors.New("LISTEN_ADDR port must be 1..65535") - } - return nil -} - -func newHandler(webDirectory, username, password string) *fiber.App { - return newHandlerWithStores(webDirectory, username, password, nil, nil) -} - -func newHandlerWithStores(webDirectory, username, password string, phaseAStore *phasea.Store, hubStore *hub.Store) *fiber.App { - return newHandlerWithCredentialBridge(webDirectory, username, password, phaseAStore, hubStore, nil) -} - -type creatorSecretBridge struct { - bridge phasea.CredentialBridge -} - -func (b creatorSecretBridge) Store(ctx context.Context, reference creator.SecretReference, key, value string) error { - if b.bridge == nil { - return errors.New("credential bridge is unavailable") - } - return b.bridge.Store(ctx, phasea.CredentialReference{ID: reference.ID, Provider: reference.Provider}, key, value) -} - -func (b creatorSecretBridge) Delete(ctx context.Context, reference creator.SecretReference, key string) error { - if b.bridge == nil { - return errors.New("credential bridge is unavailable") - } - return b.bridge.Delete(ctx, phasea.CredentialReference{ID: reference.ID, Provider: reference.Provider}, key) -} - -func newHandlerWithCredentialBridge(webDirectory, username, password string, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge) *fiber.App { - return newHandlerWithCreator(webDirectory, username, password, phaseAStore, hubStore, credentials, nil) -} - -func newHandlerWithCreator(webDirectory, username, password string, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, creatorStore *creator.Store) *fiber.App { - return newHandlerWithCreatorAndAI(webDirectory, username, password, phaseAStore, hubStore, credentials, creatorStore, nil, nil) -} - -func newHandlerWithCreatorAndAI(webDirectory, username, password string, phaseAStore *phasea.Store, hubStore *hub.Store, credentials phasea.CredentialBridge, creatorStore *creator.Store, generator creator.TextGenerator, analyzer creator.ThemeAnalyzer) *fiber.App { - app := fiber.New(fiber.Config{ - AppName: "CreatorHub control plane", - BodyLimit: 1 << 20, - // 读超时只约束请求读取；创建/升级环境的处理器可等待 native gateway 就绪（最长 11 分钟）。 - ReadTimeout: 5 * time.Second, - IdleTimeout: 60 * time.Second, - }) - app.Get("/healthz", func(c fiber.Ctx) error { - c.Status(fiber.StatusNoContent) - return nil - }) - app.Get("/readyz", func(c fiber.Ctx) error { - if phaseAStore == nil || hubStore == nil || creatorStore == nil { - return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{"error": "service is not ready"}) - } - ctx, cancel := context.WithTimeout(c.Context(), 2*time.Second) - defer cancel() - checks := []struct { - name string - fn func(context.Context) error - }{ - {"phase_a", phaseAStore.Ping}, - {"hub", hubStore.Ping}, - {"creator", creatorStore.Ping}, - {"creator_schema", creatorStore.EnsureSchema}, - } - for _, check := range checks { - if err := check.fn(ctx); err != nil { - logrus.WithError(err).WithField("check", check.name).Warn("control plane readiness check failed") - return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{"error": "service is not ready"}) - } - } - c.Status(fiber.StatusNoContent) - return nil - }) - app.Use(authenticate(username, password)) - if hubStore != nil { - registerHub(app, hubStore) - } - if phaseAStore != nil { - registerPhaseA(app, phaseAStore, hubStore, credentials) - } - if creatorStore != nil { - if generator == nil && analyzer == nil { - registerCreator(app, creatorStore, phaseAStore, hubStore) - } else { - var executor creator.ActionExecutor - if phaseAStore != nil && hubStore != nil { - executor = creatorGatewayActionExecutor{store: creatorStore, phaseAStore: phaseAStore, hubStore: hubStore} - } - registerCreatorWithServices(app, creatorStore, phaseAStore, hubStore, executor, generator, analyzer) - } - if phaseAStore != nil && hubStore != nil { - registerAccountDeletion(app, phaseAStore, hubStore, creatorStore, credentials) - } - } - app.Use(func(c fiber.Ctx) error { - if isControlPlaneAPIPath(c.Path()) { - return c.Status(fiber.StatusNotFound).JSON(fiber.Map{"error": "not found"}) - } - return c.Next() - }) - app.Get("/*", spaHandler(webDirectory)) - return app -} - -func isControlPlaneAPIPath(path string) bool { - for _, prefix := range []string{"/api", "/phase-a", "/gateways", "/browser-versions", "/browsers", "/network-exits"} { - if path == prefix || strings.HasPrefix(path, prefix+"/") { - return true - } - } - return false -} - -func authenticate(username, password string) fiber.Handler { - wantUser, wantPassword := sha256.Sum256([]byte(username)), sha256.Sum256([]byte(password)) - return func(c fiber.Ctx) error { - encoded, ok := strings.CutPrefix(c.Get(fiber.HeaderAuthorization), "Basic ") - decoded, err := base64.StdEncoding.DecodeString(encoded) - user, suppliedPassword, found := strings.Cut(string(decoded), ":") - gotUser, gotPassword := sha256.Sum256([]byte(user)), sha256.Sum256([]byte(suppliedPassword)) - userOK := subtle.ConstantTimeCompare(gotUser[:], wantUser[:]) - passwordOK := subtle.ConstantTimeCompare(gotPassword[:], wantPassword[:]) - if !ok || err != nil || !found || userOK != 1 || passwordOK != 1 { - c.Set(fiber.HeaderWWWAuthenticate, `Basic realm="CreatorHub", charset="UTF-8"`) - return c.Status(fiber.StatusUnauthorized).JSON(map[string]string{"error": "authentication required"}) - } - return c.Next() - } -} - -func spaHandler(directory string) fiber.Handler { - return func(c fiber.Ctx) error { - path, err := url.PathUnescape(strings.TrimPrefix(c.Path(), "/")) - path = filepath.Clean(path) - if err == nil && filepath.IsLocal(path) { - name := filepath.Join(directory, path) - if info, err := os.Stat(name); err == nil && !info.IsDir() { - return c.SendFile(name) - } - } - return c.SendFile(filepath.Join(directory, "index.html")) - } -} diff --git a/deploy/creatorhub-browser-gateway.service.in b/deploy/creatorhub-browser-gateway.service.in index 5c3f0d5..34d5d18 100644 --- a/deploy/creatorhub-browser-gateway.service.in +++ b/deploy/creatorhub-browser-gateway.service.in @@ -6,7 +6,7 @@ After=graphical-session.target Type=simple WorkingDirectory=@PROJECT_DIR@ EnvironmentFile=%h/.config/creatorhub/browser-gateway.env -ExecStart=/usr/bin/python3 -m cmd.browser_gateway.gateway +ExecStart=/usr/bin/python3 -m browser_gateway.server.http Restart=on-failure RestartSec=5 UMask=0077 diff --git a/docs/deployment.md b/docs/deployment.md index 4625f4a..d719902 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -127,9 +127,9 @@ gateway 的 runtime 清单、Profile 和日志目录不得由 `docker compose do 代码检查： ```bash -python3 -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' -q -python3 -m coverage run --source=cmd/browser_gateway --branch -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' -q -python3 -m coverage report --omit='cmd/browser_gateway/test_*.py' --fail-under=65 +python3 -m unittest discover -s browser_gateway -t . -p 'test_*.py' -q +python3 -m coverage run --source=browser_gateway --branch -m unittest discover -s browser_gateway -t . -p 'test_*.py' -q +python3 -m coverage report --omit='browser_gateway/test_*.py' --fail-under=65 go test ./... go vet ./... go build ./cmd/control-plane diff --git a/docs/deployment_stop_test.sh b/docs/deployment_stop_test.sh index 8190733..ef11e56 100755 --- a/docs/deployment_stop_test.sh +++ b/docs/deployment_stop_test.sh @@ -14,7 +14,7 @@ grep -Fq 'Compose 不再创建 gateway 服务' docs/deployment.md # The install unit must remain a user service and must not run the gateway as root. grep -Fq 'EnvironmentFile=%h/.config/creatorhub/browser-gateway.env' deploy/creatorhub-browser-gateway.service.in -grep -Fq 'cmd.browser_gateway.gateway' deploy/creatorhub-browser-gateway.service.in +grep -Fq 'browser_gateway.server.http' deploy/creatorhub-browser-gateway.service.in ! grep -Fq 'User=root' deploy/creatorhub-browser-gateway.service.in echo 'native browser deployment documentation checks passed' diff --git a/docs/e2e-test-plan.md b/docs/e2e-test-plan.md index 537d298..234c25f 100644 --- a/docs/e2e-test-plan.md +++ b/docs/e2e-test-plan.md @@ -22,7 +22,7 @@ ### 1.2 核对来源与纠偏 -主要来源：`web/src/main.jsx`、`web/src/dataProvider.js`、`web/src/*Page.jsx`、`web/src/Layout.jsx`、`web/src/lib/ui.jsx`；`cmd/control-plane/{main,hub,phasea,creator,creator_events,creator_material}.go`；`internal/creator/{models,accounts,actions,collection,content,metrics,rules,settings,bailian}.go`；`internal/hub/{environment,fingerprint}.go`；`internal/douyin/creator_collector.go`；`cmd/browser_gateway/{gateway,douyin,proxy,runtime}.py`；`compose.yaml`、`compose.dev.yaml`、`Dockerfile`、`deploy/`。 +主要来源：`web/src/app/main.jsx`、`web/src/shared/api/dataProvider.js`、`web/src/features/**/*.jsx`、`web/src/app/Layout.jsx`、`web/src/shared/ui/ui.jsx`；`internal/controlplane/{app,api,workers}/*.go`；`internal/creator/{models,accounts,actions,collection,content,metrics,rules,settings,bailian}.go`；`internal/environment/{environment,fingerprint}.go`；`internal/platform/douyin/creator_collector.go`；`browser_gateway/{server,http.py,platform,douyin.py,proxy.py,runtime.py}`；`compose.yaml`、`compose.dev.yaml`、`Dockerfile`、`deploy/`。 其他运行文档只作操作参考：其中“代理仅凭据引用”“仿写自动生成”“抖音仅首批读取”等旧表述不能覆盖当前路由和源码。当前出口表单/API 接收用户名与密码，列表/详情会明文展示；证据必须脱敏。仿写确认只保存要求与确认时间，**没有 AI 生成调用或按钮**。`decodeCreator` 直接读 JSON body，不依据 Content-Type 选择解码，故保存仿写缺 header 不等于已经证实失败。 diff --git a/docs/native-browser-change-review.md b/docs/native-browser-change-review.md index 1195815..d2ce4e8 100644 --- a/docs/native-browser-change-review.md +++ b/docs/native-browser-change-review.md @@ -29,18 +29,18 @@ | 现有事实/入口 | 对本次变更的影响 | | --- | --- | -| `internal/hub/store.go` 的 `Gateway` 保存名称、Endpoint；`Env` 保存 `Gateway`、`BrowserVersion` | 多机器注册与路由已存在，应复用，不新增第二套节点管理 | -| `cmd/control-plane/hub.go` 的 `startBrowserRuntime`、`createGatewayRuntime`、`removeGatewayRuntime` 管理启停与清理 | 必须整体改造生命周期，不能仅替换 Python 中一个 Docker 调用 | +| `internal/environment/store.go` 的 `Gateway` 保存名称、Endpoint；`Env` 保存 `Gateway`、`BrowserVersion` | 多机器注册与路由已存在，应复用，不新增第二套节点管理 | +| `internal/controlplane/api/environments.go` 的 `startBrowserRuntime`、`createGatewayRuntime`、`removeGatewayRuntime` 管理启停与清理 | 必须整体改造生命周期，不能仅替换 Python 中一个 Docker 调用 | | `gatewayCreatePayload` 下发 `creatorhub-profile-{alias}` 卷名、指纹、代理和绑定版本 | 卷改成 gateway 管理的持久目录；控制面不能下发任意宿主机路径 | | `runtimeCleanupStore` 及启动前清理逻辑已有 cleanup-pending 概念 | 保留“未清理完成不能冒充已停止”的语义，改用本机资源身份，不再依赖容器/网络 ID | -| 历史 `cmd/docker_gateway/gateway.py` 曾初始化 `DockerClient` 并读取 `DOCKER_SOCKET`、`BROWSER_NETWORK` | 已由 `cmd/browser_gateway/gateway.py` 的 native runtime 管理替换；当前浏览器入口不依赖 Docker。该行只保留迁移前事实 | +| 历史 `cmd/docker_gateway/gateway.py` 曾初始化 `DockerClient` 并读取 `DOCKER_SOCKET`、`BROWSER_NETWORK` | 已由 `browser_gateway/server/http.py` 的 native runtime 管理替换；当前浏览器入口不依赖 Docker。该行只保留迁移前事实 | | 历史 `docker/browser-wrapper/docker-entrypoint.sh` 曾固定容器内 Xvfb/端口/Profile | 浏览器 wrapper 已从生产入口删除；native gateway 为每个 runtime 独立分配 display、端口和 Profile，历史行不构成当前能力证明 | | 历史 wrapper 曾传入 `--no-sandbox` | native gateway 拒绝该参数并以非 root 用户启动；无法满足 sandbox 时直接失败，不回退旧 wrapper | -| `cmd/control-plane/creator.go` 的采集读取已有运行账号，没有按任务自动创建 runtime；`creator_events.go` 独立运行监听 | 需要新增任务使用权；现有 source lease 不是浏览器使用权，直接在采集末尾调用 stop 会误停共享会话 | -| `creator.go` 的临时下载、`creator_material.go` 的 `audio.wav.tmp` 依赖正常退出删除；素材先写固定路径再提交数据库 token | 控制面本机也需要崩溃清理；旧执行可能覆盖新产物，必须先隔离执行目录与产物发布，不能只增加目录扫描 | +| `internal/controlplane/api/creator.go` 的采集读取已有运行账号，没有按任务自动创建 runtime；`internal/controlplane/api/creator_events.go` 独立运行监听 | 需要新增任务使用权；现有 source lease 不是浏览器使用权，直接在采集末尾调用 stop 会误停共享会话 | +| `internal/controlplane/api/creator.go` 的临时下载、`internal/controlplane/api/creator_material.go` 的 `audio.wav.tmp` 依赖正常退出删除；素材先写固定路径再提交数据库 token | 控制面本机也需要崩溃清理；旧执行可能覆盖新产物，必须先隔离执行目录与产物发布，不能只增加目录扫描 | | `internal/creator/source_lease.go` / `content.go` 的来源 lease 为固定十分钟；部分失败收尾仍使用已取消的 context | 长任务可能重复领取，取消后仍显示 running；资源使用权必须覆盖真实执行期并有独立收尾上下文 | -| `internal/hub/store.go` 的 gateway endpoint 可更新，环境按 gateway 名称重新解析地址 | 不能把名称当稳定机器身份；原 owner 和待清理目标必须保留，禁止把仍有绑定资源的 gateway 改指另一机器 | -| `internal/hub/store.go` 的 `Image`、前端镜像页、指纹/版本字段依赖 Docker 镜像语义 | 浏览器版本来源必须替换；只改界面名称会留下无效配置 | +| `internal/environment/store.go` 的 gateway endpoint 可更新，环境按 gateway 名称重新解析地址 | 不能把名称当稳定机器身份；原 owner 和待清理目标必须保留，禁止把仍有绑定资源的 gateway 改指另一机器 | +| `internal/environment/store.go` 的 `Image`、前端镜像页、指纹/版本字段依赖 Docker 镜像语义 | 浏览器版本来源必须替换；只改界面名称会留下无效配置 | | `scripts/dev-backend.mjs` / `compose*.yaml` 现在只保留 PostgreSQL（Compose）和 host-native gateway 注册 | 联调脚本和部署说明已同步；原生模式不启动 Docker browser gateway | | `requirements-gateway.lock` 当前锁定 `websocket-client` | 优先保留已有 CDP/WebSocket 与平台适配器，不因取消 Docker 就新增 Playwright/Patchright | diff --git a/docs/native-browser-implementation-plan.md b/docs/native-browser-implementation-plan.md index 05060c0..1a55f47 100644 --- a/docs/native-browser-implementation-plan.md +++ b/docs/native-browser-implementation-plan.md @@ -75,7 +75,7 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） 先写失败测试，再实现： -- 提取并保留现有与 Docker 无关的代理、CDP、页面动作；将 gateway 包和测试从 Docker 命名改为浏览器领域命名，建议目标路径 `cmd/browser_gateway/`。 +- 提取并保留现有与 Docker 无关的代理、CDP、页面动作；将 gateway 包和测试从 Docker 命名改为浏览器领域命名，建议目标路径 `browser_gateway/`。 - 原生启动输入校验在创建目录/进程之前完成：节点、环境、代次、浏览器版本、Profile、代理绑定及必要资源容量。 - 创建前落盘运行意图；用 runtime 专属 unit/临时目录启动 Xvfb 和浏览器，分配不冲突的 display 与端口。Profile 独占不能只靠控制面内存锁。 - 默认不继承 `--no-sandbox`；浏览器与 gateway 以非 root 身份运行。具体 sandbox 模式由目标二进制/发行版证明。 @@ -92,7 +92,7 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） 先测试控制面与 gateway 的成功、校验、失败、冲突、响应丢失路径，再改： - `hub.go` 的创建、启动、停止、状态核对和 cleanup-pending 改用 generation/owner；删除容器和网络代次依赖。 -- `internal/hub/store.go`、相关查询与 schema 改为原生版本/运行模型；开发数据库受控重建，不回填旧容器字段。 +- `internal/environment/store.go`、相关查询与 schema 改为原生版本/运行模型；开发数据库受控重建，不回填旧容器字段。 - 继续复用既有多 gateway 登记、路由和心跳；核对稳定机器身份。节点 Endpoint 的更换不能改写既存 cleanup 的目的地。 - 代理服务不再依赖 Docker IP/网络，仍保留代理绑定代次和失败可见性；从目标 gateway 的实际浏览器路径验证出口。 - 登录二维码、账号身份快照、恢复事件订阅全部使用新 runtime 身份；不能随运行代次变化清空业务事件去重、基线和已占用 UID 冷却。先用确定性测试覆盖重启/重连、冷却内新事件、冷却到期旧事件及迟到事件不补发；真实平台按 C02/D04 验收。 @@ -136,11 +136,11 @@ React → Go control-plane（账号/任务/绑定/业务结果/素材发布） | 范围 | 主要定位 | 必须一起变化的内容 | | --- | --- | --- | -| gateway 执行 | `cmd/browser_gateway/gateway.py`、`runtime.py`、`proxy.py`、CDP/平台模块及其测试 | 原生 runner、版本/端口/Profile、unit 生命周期、snapshot、代理恢复 | -| 控制面 lifecycle | `cmd/control-plane/hub.go`、`main.go`、hub tests | generation、owner、清理/心跳、多 gateway 错误隔离 | -| 采集与监听 | `creator.go`、`creator_events.go`、`internal/creator/source_lease.go`、`content.go` | 使用权、续租、独立收尾、事件边界与身份核对 | -| 素材 | `creator_material.go`、`internal/creator/material.go`、素材测试 | 临时目录、不可变发布、token、崩溃清理 | -| 数据模型 | `internal/hub/`、`internal/creator/` | 删除 Docker 字段，明确开发重建；业务结果与清理状态分离 | +| gateway 执行 | `browser_gateway/server/http.py`、`runtime.py`、`proxy.py`、`browser/`、`platform/` 及其测试 | 原生 runner、版本/端口/Profile、unit 生命周期、snapshot、代理恢复 | +| 控制面 lifecycle | `internal/controlplane/api/environments.go`、`internal/controlplane/app/app.go`、environment tests | generation、owner、清理/心跳、多 gateway 错误隔离 | +| 采集与监听 | `internal/controlplane/api/creator.go`、`internal/controlplane/api/creator_events.go`、`internal/creator/source_lease.go`、`content.go` | 使用权、续租、独立收尾、事件边界与身份核对 | +| 素材 | `internal/controlplane/api/creator_material.go`、`internal/creator/material.go`、素材测试 | 临时目录、不可变发布、token、崩溃清理 | +| 数据模型 | `internal/environment/`、`internal/creator/` | 删除 Docker 字段，明确开发重建；业务结果与清理状态分离 | | 前端 | 现有 gateway/环境/镜像/任务相关组件及 data provider | 对应契约、失败/禁用交互；不重做导航和视觉体系 | | 部署/文档 | `scripts/`、`compose*.yaml`、Dockerfile、`deploy/`、README、`docs/` | 原生安装和联调；旧路径移除；手工验收地址 | diff --git a/docs/native-browser-verification.md b/docs/native-browser-verification.md index 526b3d4..49027f1 100644 --- a/docs/native-browser-verification.md +++ b/docs/native-browser-verification.md @@ -54,10 +54,10 @@ go tool cover -func=/tmp/creatorhub-go.cover ```bash python3 -m venv .venv-gateway .venv-gateway/bin/python -m pip install -r requirements-gateway-dev.lock -.venv-gateway/bin/python -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' +.venv-gateway/bin/python -m unittest discover -s browser_gateway -t . -p 'test_*.py' .venv-gateway/bin/python -m coverage erase -.venv-gateway/bin/python -m coverage run --source=cmd/browser_gateway --branch \ - -m unittest discover -s cmd/browser_gateway -t cmd -p 'test_*.py' +.venv-gateway/bin/python -m coverage run --source=browser_gateway --branch \ + -m unittest discover -s browser_gateway -t . -p 'test_*.py' .venv-gateway/bin/python -m coverage report --omit='*/test_*.py' --fail-under=65 ``` diff --git a/docs/plan01.md b/docs/plan01.md index ace8a95..4511fe8 100644 --- a/docs/plan01.md +++ b/docs/plan01.md @@ -41,14 +41,14 @@ | 范围 | 已有基础 | 本需求需要补齐或改变 | 代码依据 | | --- | --- | --- | --- | -| 页面入口 | 账号、任务、审计、环境、代理、镜像、网关等页面 | 竞品池、作品、素材仿写、评论线索、响应策略、私信会话尚无完整入口 | [页面路由](../web/src/main.jsx) | -| 账号 | 平台账号 ID、标签、Cookie、暂停/恢复等 | 实名资料、登录用户名/密码、备注、人工业务状态、人工登录与大小号关系 | [账号页面](../web/src/AccountsPage.jsx)、[账号存储](../internal/phasea/store.go)、[账号接口](../cmd/control-plane/phasea.go) | -| 平台范围 | 登记项还包含公众号、快手 | 新业务仅承诺抖音、小红书；其他登记项不是业务能力验收结果，也不因此要求删除无关已有功能 | [账号页面](../web/src/AccountsPage.jsx) | -| 作品读取 | 有抖音连接器、指标字段与测试 | 连接器只核验登录者自身、读取首批 20 条，未接成运行中的竞品采集链路；需目标账号解析、分页、保存、查询和定时更新 | [抖音连接器](../internal/douyin/connector.go)、[连接器测试](../internal/douyin/connector_test.go)、[网关读取限制](../cmd/browser_gateway/douyin.py) | -| 指纹与环境 | 结构化指纹、独立 Profile、启动/停止/升级及代理接入 | 默认固定 seed 不是自动分配；需首次自动生成、地区匹配及稳定性验证 | [环境页面](../web/src/BrowsersPage.jsx)、[指纹参数](../internal/hub/fingerprint.go)、[环境存储](../internal/hub/environment.go) | -| 代理 | 添加、列表、手动检测、停用、实际转发 | 补齐编辑、删除、重新启用及引用约束；不增加自动轮换 | [代理页面](../web/src/NetworkExitsPage.jsx)、[代理转发](../cmd/browser_gateway/proxy.py) | -| 任务与发送 | 有草稿确认、任务记录、Mock 执行 | 不能作为真实回复、私信、点赞、关注、转发成功的证据；需接通实际平台执行及结果核验 | [任务页面](../web/src/TasksPage.jsx)、[任务接口](../cmd/control-plane/phasea.go) | -| 事件、线索、私信、AI | 未发现完整可运行链路 | 均需新增业务能力，不能把旧文档、类型定义或模拟返回记为已实现 | [页面路由](../web/src/main.jsx)、[抖音连接器](../internal/douyin/connector.go) | +| 页面入口 | 账号、任务、审计、环境、代理、镜像、网关等页面 | 竞品池、作品、素材仿写、评论线索、响应策略、私信会话尚无完整入口 | [页面路由](../web/src/app/main.jsx) | +| 账号 | 平台账号 ID、标签、Cookie、暂停/恢复等 | 实名资料、登录用户名/密码、备注、人工业务状态、人工登录与大小号关系 | [账号页面](../web/src/features/accounts/AccountsPage.jsx)、[账号存储](../internal/account/store.go)、[账号接口](../internal/controlplane/api/accounts_operations.go) | +| 平台范围 | 登记项还包含公众号、快手 | 新业务仅承诺抖音、小红书；其他登记项不是业务能力验收结果，也不因此要求删除无关已有功能 | [账号页面](../web/src/features/accounts/AccountsPage.jsx) | +| 作品读取 | 有抖音连接器、指标字段与测试 | 连接器只核验登录者自身、读取首批 20 条，未接成运行中的竞品采集链路；需目标账号解析、分页、保存、查询和定时更新 | [抖音连接器](../internal/platform/douyin/connector.go)、[连接器测试](../internal/platform/douyin/connector_test.go)、[网关读取限制](../browser_gateway/platform/douyin.py) | +| 指纹与环境 | 结构化指纹、独立 Profile、启动/停止/升级及代理接入 | 默认固定 seed 不是自动分配；需首次自动生成、地区匹配及稳定性验证 | [环境页面](../web/src/features/environments/BrowsersPage.jsx)、[指纹参数](../internal/environment/fingerprint.go)、[环境存储](../internal/environment/environment.go) | +| 代理 | 添加、列表、手动检测、停用、实际转发 | 补齐编辑、删除、重新启用及引用约束；不增加自动轮换 | [代理页面](../web/src/features/environments/NetworkExitsPage.jsx)、[代理转发](../browser_gateway/proxy.py) | +| 任务与发送 | 有草稿确认、任务记录、Mock 执行 | 不能作为真实回复、私信、点赞、关注、转发成功的证据；需接通实际平台执行及结果核验 | [任务页面](../web/src/features/operations/TasksPage.jsx)、[任务接口](../internal/controlplane/api/accounts_operations.go) | +| 事件、线索、私信、AI | 未发现完整可运行链路 | 均需新增业务能力，不能把旧文档、类型定义或模拟返回记为已实现 | [页面路由](../web/src/app/main.jsx)、[抖音连接器](../internal/platform/douyin/connector.go) | ### 2.1 已确认的冲突裁决 diff --git a/docs/python-gateway-branch-review.md b/docs/python-gateway-branch-review.md index d491d4e..539e205 100644 --- a/docs/python-gateway-branch-review.md +++ b/docs/python-gateway-branch-review.md @@ -1,6 +1,6 @@ # Python 网关分支审查与修正清单 -日期：2026-09-13。状态：**历史分支审查已完成；本文保留迁移前 `cmd/docker_gateway`、Docker wrapper 和容器生命周期路径作为问题证据，不是当前生产入口。当前入口与状态以 `cmd/browser_gateway`、部署说明和 native 验证记录为准。** +日期：2026-09-13。状态：**历史分支审查已完成；本文保留迁移前 `cmd/docker_gateway`、Docker wrapper 和容器生命周期路径作为问题证据，不是当前生产入口。当前入口与状态以 `browser_gateway`、部署说明和 native 验证记录为准。** ## 1. 范围、结论与证据边界 diff --git a/docs/research/xhs-all-in-one.md b/docs/research/xhs-all-in-one.md index f89b4d0..b1a477f 100644 --- a/docs/research/xhs-all-in-one.md +++ b/docs/research/xhs-all-in-one.md @@ -185,7 +185,7 @@ CreatorHub 不应承诺“稳定”“不封号”或“无法识别”，也不 | 吸收项 | CreatorHub 落点 | 具体要求 | | --- | --- | --- | -| 平台适配器与业务逻辑分离 | 新增小红书适配时沿用 `internal/douyin` 的边界，业务层不直接拼平台请求 | 读取、登录、媒体、动作、事件各自有清晰接口；不把上游 Python SDK 直接嵌进控制面 | +| 平台适配器与业务逻辑分离 | 新增小红书适配时沿用 `internal/platform/douyin` 的边界，业务层不直接拼平台请求 | 读取、登录、媒体、动作、事件各自有清晰接口；不把上游 Python SDK 直接嵌进控制面 | | 原始响应 + 标准字段 | `internal/creator` 的 Work/Comment/Material 数据 | 保存原始平台 payload、来源链接、采集时间和标准化结果；字段变化先保原文再调整解析 | | `xsec_token`/`xsec_source` 端到端保留 | 作品来源/平台适配输入模型 | 详情链接、搜索结果和后续详情请求不得丢失访问上下文；没有 token 时不得猜测或拼造 | | 评论父子关系 | Comment 入库字段扩展时保留 `parent_comment_id` | 当前业务先采一级评论，平台返回子评论时不丢弃层级；不要把昵称当用户 ID | @@ -230,7 +230,7 @@ CreatorHub 不应承诺“稳定”“不封号”或“无法识别”，也不 - `internal/creator/actions.go`：已有事件永久去重、策略顺序、同 UID 冷却，以及 AI 失败/结果不明不补发。 - `cmd/control-plane/creator.go`：已有写操作前身份核对、目标映射和结果证据保存。 - `cmd/control-plane/creator_material.go`：已有人工选取、原子媒体文件、大小/HTML 校验、ffprobe/ffmpeg 和显式转写失败步骤。 -- `cmd/control-plane/creator_events.go` 与 `cmd/browser_gateway/douyin.py`：已有监听代际、baseline、gap、断连恢复、delivery ACK/retry 的抖音实现；小红书不能直接沿用为已验证能力。 +- `cmd/control-plane/creator_events.go` 与 `browser_gateway/platform/douyin.py`：已有监听代际、baseline、gap、断连恢复、delivery ACK/retry 的抖音实现；小红书不能直接沿用为已验证能力。 - `docs/plan01.md`：已明确先抖音后小红书、逐平台真实验收，以及读取/写入/事件的成功、失败、不明边界。 ### 5.2 当前小红书缺口 diff --git a/internal/phasea/deletion.go b/internal/account/deletion.go similarity index 99% rename from internal/phasea/deletion.go rename to internal/account/deletion.go index 535b5d6..9f4542b 100644 --- a/internal/phasea/deletion.go +++ b/internal/account/deletion.go @@ -1,4 +1,4 @@ -package phasea +package account import ( "context" diff --git a/internal/phasea/migrations/001_phase_a.sql b/internal/account/migrations/001_phase_a.sql similarity index 100% rename from internal/phasea/migrations/001_phase_a.sql rename to internal/account/migrations/001_phase_a.sql diff --git a/internal/phasea/migrations/036_runtime_use_leases.sql b/internal/account/migrations/036_runtime_use_leases.sql similarity index 100% rename from internal/phasea/migrations/036_runtime_use_leases.sql rename to internal/account/migrations/036_runtime_use_leases.sql diff --git a/internal/phasea/store.go b/internal/account/store.go similarity index 99% rename from internal/phasea/store.go rename to internal/account/store.go index 473ebb3..f8b6e2c 100644 --- a/internal/phasea/store.go +++ b/internal/account/store.go @@ -1,4 +1,4 @@ -package phasea +package account import ( "context" @@ -1299,7 +1299,7 @@ func (s *Store) ResumeTask(ctx context.Context, taskID string) error { JOIN content_draft draft ON draft.id = task.draft_id JOIN confirmation confirmation ON confirmation.id = task.confirmation_id JOIN environment_binding binding ON binding.account_id = task.account_id - JOIN browser_env environment ON environment.alias = binding.browser_env_alias + JOIN browser_env environment ON hub.alias = binding.browser_env_alias LEFT JOIN network_exit network ON network.id = binding.network_exit_id JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL WHERE task.id = $1 AND task.state IN ('policy_hold', 'needs_confirmation') @@ -1370,7 +1370,7 @@ func taskReadinessReason(ctx context.Context, queryer rowQuerier, taskID string) OR confirmation.draft_id <> task.draft_id OR confirmation.draft_version <> task.draft_version OR confirmation.version <> task.confirmation_version THEN 'confirmation_version_changed' WHEN binding.id IS NULL THEN 'binding_missing' - WHEN environment.alias IS NULL THEN 'environment_missing' + WHEN hub.alias IS NULL THEN 'environment_missing' WHEN binding.network_exit_id IS NOT NULL AND network.id IS NULL THEN 'exit_missing' WHEN binding.network_exit_id IS NOT NULL AND network.health_status <> 'healthy' THEN 'exit_unhealthy' WHEN binding.runtime_cleanup_pending THEN 'runtime_stop_pending' @@ -1384,7 +1384,7 @@ func taskReadinessReason(ctx context.Context, queryer rowQuerier, taskID string) LEFT JOIN content_draft draft ON draft.id = task.draft_id LEFT JOIN confirmation confirmation ON confirmation.id = task.confirmation_id LEFT JOIN environment_binding binding ON binding.account_id = task.account_id - LEFT JOIN browser_env environment ON environment.alias = binding.browser_env_alias + LEFT JOIN browser_env environment ON hub.alias = binding.browser_env_alias LEFT JOIN network_exit network ON network.id = binding.network_exit_id LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL WHERE task.id = $1`, taskID).Scan(&reason) @@ -1497,7 +1497,7 @@ func (s *Store) claim(ctx context.Context, workerID string) (Execution, error) { JOIN content_draft d ON d.id = t.draft_id JOIN confirmation c ON c.id = t.confirmation_id JOIN environment_binding binding ON binding.account_id = a.id - JOIN browser_env environment ON environment.alias = binding.browser_env_alias + JOIN browser_env environment ON hub.alias = binding.browser_env_alias LEFT JOIN network_exit network ON network.id = binding.network_exit_id JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL AND runtime.lease_until > now() AND runtime.binding_version = binding.version @@ -1563,13 +1563,13 @@ func (s *Store) complete(ctx context.Context, workerID string, execution Executi OR confirmation.account_version <> task.account_version OR confirmation.draft_id <> task.draft_id OR confirmation.draft_version <> task.draft_version OR confirmation.version <> task.confirmation_version THEN 'confirmation_version_changed' WHEN binding.id IS NULL THEN 'binding_missing' - WHEN environment.alias IS NULL THEN 'environment_missing' + WHEN hub.alias IS NULL THEN 'environment_missing' WHEN binding.network_exit_id IS NOT NULL AND network.id IS NULL THEN 'exit_missing' WHEN binding.network_exit_id IS NOT NULL AND network.health_status <> 'healthy' THEN 'exit_unhealthy' WHEN binding.runtime_cleanup_pending THEN 'runtime_stop_pending' WHEN runtime.id IS NULL OR runtime.lease_until <= now() THEN 'runtime_missing' WHEN claim.binding_version IS DISTINCT FROM binding.version - OR claim.browser_env_alias IS DISTINCT FROM environment.alias + OR claim.browser_env_alias IS DISTINCT FROM hub.alias OR claim.network_exit_id IS DISTINCT FROM network.id OR claim.runtime_instance_id IS DISTINCT FROM runtime.id OR runtime.binding_version IS DISTINCT FROM binding.version THEN 'task_result_uncertain' @@ -1715,7 +1715,7 @@ func quarantineInvalid(ctx context.Context, tx *sql.Tx) ([]taskstate.Transition, OR confirmation.draft_id <> t.draft_id OR confirmation.draft_version <> t.draft_version OR confirmation.version <> t.confirmation_version THEN 'confirmation_version_changed' WHEN binding.id IS NULL THEN 'binding_missing' - WHEN environment.alias IS NULL THEN 'environment_missing' + WHEN hub.alias IS NULL THEN 'environment_missing' WHEN binding.network_exit_id IS NOT NULL AND network.id IS NULL THEN 'exit_missing' WHEN binding.network_exit_id IS NOT NULL AND network.health_status <> 'healthy' THEN 'exit_unhealthy' WHEN binding.runtime_cleanup_pending THEN 'runtime_stop_pending' @@ -1728,7 +1728,7 @@ func quarantineInvalid(ctx context.Context, tx *sql.Tx) ([]taskstate.Transition, LEFT JOIN content_draft draft ON draft.id = t.draft_id LEFT JOIN confirmation confirmation ON confirmation.id = t.confirmation_id LEFT JOIN environment_binding binding ON binding.account_id = t.account_id - LEFT JOIN browser_env environment ON environment.alias = binding.browser_env_alias + LEFT JOIN browser_env environment ON hub.alias = binding.browser_env_alias LEFT JOIN network_exit network ON network.id = binding.network_exit_id LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL WHERE t.state = 'queued' AND ( @@ -1738,7 +1738,7 @@ func quarantineInvalid(ctx context.Context, tx *sql.Tx) ([]taskstate.Transition, OR confirmation.id IS NULL OR confirmation.account_id <> t.account_id OR confirmation.account_version <> t.account_version OR confirmation.draft_id <> t.draft_id OR confirmation.draft_version <> t.draft_version OR confirmation.version <> t.confirmation_version - OR binding.id IS NULL OR environment.alias IS NULL + OR binding.id IS NULL OR hub.alias IS NULL OR (binding.network_exit_id IS NOT NULL AND (network.id IS NULL OR network.health_status <> 'healthy')) OR binding.runtime_cleanup_pending OR runtime.id IS NULL OR runtime.binding_version IS DISTINCT FROM binding.version OR runtime.lease_until <= now() diff --git a/internal/phasea/store_test.go b/internal/account/store_test.go similarity index 99% rename from internal/phasea/store_test.go rename to internal/account/store_test.go index cad1cbd..4063e0d 100644 --- a/internal/phasea/store_test.go +++ b/internal/account/store_test.go @@ -1,4 +1,4 @@ -package phasea +package account import ( "context" @@ -967,7 +967,7 @@ func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) { if applied { continue } - migration, err := os.ReadFile(filepath.Join("..", "hub", "migrations", migrationFile.name)) + migration, err := os.ReadFile(filepath.Join("..", "environment", "migrations", migrationFile.name)) if err != nil { t.Fatal(err) } diff --git a/cmd/control-plane/account_deletion.go b/internal/controlplane/api/account_deletion.go similarity index 84% rename from cmd/control-plane/account_deletion.go rename to internal/controlplane/api/account_deletion.go index 1db6088..163f8d7 100644 --- a/cmd/control-plane/account_deletion.go +++ b/internal/controlplane/api/account_deletion.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -8,12 +8,12 @@ import ( "github.com/gofiber/fiber/v3" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) -func registerAccountDeletion(app *fiber.App, phaseAStore *phasea.Store, hubStore *hub.Store, creatorStore *creator.Store, credentials phasea.CredentialBridge) { +func registerAccountDeletion(app *fiber.App, phaseAStore *accountdomain.Store, hubStore *hub.Store, creatorStore *creator.Store, credentials accountdomain.CredentialBridge) { app.Delete("/api/phase-a/accounts/:id", func(c fiber.Ctx) error { accountID := c.Params("id") if err := phaseAStore.CheckAccountDeletion(c.Context(), accountID); err != nil { @@ -113,3 +113,8 @@ func purgeAccountProfile(ctx context.Context, gateway hub.Gateway, environment h } return gatewayRejected(status, body) } + +// RegisterAccountDeletion exposes destructive account routes to the grouped API package. +func RegisterAccountDeletion(app *fiber.App, accountStore *accountdomain.Store, environmentStore *hub.Store, creatorStore *creator.Store, credentials accountdomain.CredentialBridge) { + registerAccountDeletion(app, accountStore, environmentStore, creatorStore, credentials) +} diff --git a/internal/controlplane/api/accounts/register.go b/internal/controlplane/api/accounts/register.go new file mode 100644 index 0000000..053f52a --- /dev/null +++ b/internal/controlplane/api/accounts/register.go @@ -0,0 +1,17 @@ +package accounts + +import ( + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api" + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/gofiber/fiber/v3" +) + +func Register(app *fiber.App, store *account.Store, runtimeStore *environment.Store, credentials account.CredentialBridge) { + api.RegisterAccounts(app, store, runtimeStore, credentials) +} + +func RegisterDeletion(app *fiber.App, store *account.Store, runtimeStore *environment.Store, creatorStore *creator.Store, credentials account.CredentialBridge) { + api.RegisterAccountDeletion(app, store, runtimeStore, creatorStore, credentials) +} diff --git a/cmd/control-plane/phasea.go b/internal/controlplane/api/accounts_operations.go similarity index 83% rename from cmd/control-plane/phasea.go rename to internal/controlplane/api/accounts_operations.go index 74f2e6f..2fbce3a 100644 --- a/cmd/control-plane/phasea.go +++ b/internal/controlplane/api/accounts_operations.go @@ -1,4 +1,4 @@ -package main +package api import ( "bytes" @@ -9,8 +9,8 @@ import ( "strings" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/gofiber/fiber/v3" ) @@ -41,7 +41,7 @@ type taskVerificationRequest struct { Result string `json:"result"` } -func registerPhaseA(app *fiber.App, store *phasea.Store, runtimeStore runtimeStopStore, credentials phasea.CredentialBridge) { +func registerPhaseA(app *fiber.App, store *accountdomain.Store, runtimeStore RuntimeStopStore, credentials accountdomain.CredentialBridge) { app.Post("/api/phase-a/accounts", func(c fiber.Ctx) error { var input accountRequest if err := decodePhaseA(c, &input); err != nil { @@ -54,16 +54,16 @@ func registerPhaseA(app *fiber.App, store *phasea.Store, runtimeStore runtimeSto for index := range tags { tags[index] = strings.TrimSpace(tags[index]) } - accountID := phasea.NewAccountID() - account := phasea.Account{ + accountID := accountdomain.NewAccountID() + account := accountdomain.Account{ ID: accountID, Name: strings.TrimSpace(input.Name), Platform: strings.TrimSpace(input.Platform), PlatformAccountKey: strings.TrimSpace(input.PlatformAccountKey), Tags: tags, Cookies: strings.TrimSpace(input.Cookies), - CredentialReference: phasea.CredentialReference{ID: accountID + "-cookies", Provider: "os_keyring"}, + CredentialReference: accountdomain.CredentialReference{ID: accountID + "-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/" + accountID + "/cookies", AuthorizationStatus: "authorized", RuntimeStatus: "paused", Version: 1, } if err := store.CreateAccount(c.Context(), account, credentials); err != nil { - if errors.Is(err, phasea.ErrAccountCreationUnknown) { + if errors.Is(err, accountdomain.ErrAccountCreationUnknown) { return c.Status(fiber.StatusServiceUnavailable).JSON(map[string]string{ "error": "account creation result is unknown", "reason_code": "account_creation_result_unknown", "account_id": accountID, }) @@ -113,7 +113,7 @@ func registerPhaseA(app *fiber.App, store *phasea.Store, runtimeStore runtimeSto } defer unlock() if err := store.ResumeAccount(c.Context(), c.Params("id")); err != nil { - if errors.Is(err, phasea.ErrConflict) { + if errors.Is(err, accountdomain.ErrConflict) { return accountResumeConflict(c, store, runtimeStore, c.Params("id")) } return phaseAError(c, err) @@ -298,8 +298,8 @@ func registerPhaseA(app *fiber.App, store *phasea.Store, runtimeStore runtimeSto }) } -func auditFilter(c fiber.Ctx) (phasea.AuditFilter, error) { - filter := phasea.AuditFilter{ +func auditFilter(c fiber.Ctx) (accountdomain.AuditFilter, error) { + filter := accountdomain.AuditFilter{ AccountID: c.Query("account_id"), TaskID: c.Query("task_id"), AttemptID: c.Query("attempt_id"), BrowserEnvAlias: c.Query("browser_env_alias"), NetworkExitID: c.Query("network_exit_id"), EventType: c.Query("event_type"), Page: 1, PageSize: 25, @@ -314,7 +314,7 @@ func auditFilter(c fiber.Ctx) (phasea.AuditFilter, error) { } parsed, err := strconv.Atoi(value) if err != nil { - return phasea.AuditFilter{}, phasea.ErrInvalid + return accountdomain.AuditFilter{}, accountdomain.ErrInvalid } *target = parsed } @@ -328,14 +328,14 @@ func auditFilter(c fiber.Ctx) (phasea.AuditFilter, error) { } parsed, err := time.Parse(time.RFC3339, value) if err != nil { - return phasea.AuditFilter{}, phasea.ErrInvalid + return accountdomain.AuditFilter{}, accountdomain.ErrInvalid } *target = &parsed } return filter, nil } -func resumeBlockReason(account phasea.Account, environment hub.EnvironmentContext, bindingFound bool) string { +func resumeBlockReason(account accountdomain.Account, environment hub.EnvironmentContext, bindingFound bool) string { switch { case account.AuthorizationStatus != "authorized": return "account_revoked" @@ -352,7 +352,7 @@ func resumeBlockReason(account phasea.Account, environment hub.EnvironmentContex } } -func accountResumeConflict(c fiber.Ctx, store *phasea.Store, runtimeStore runtimeStopStore, accountID string) error { +func accountResumeConflict(c fiber.Ctx, store *accountdomain.Store, runtimeStore RuntimeStopStore, accountID string) error { account, err := store.GetAccount(c.Context(), accountID) if err != nil { return phaseAError(c, err) @@ -367,7 +367,7 @@ func accountResumeConflict(c fiber.Ctx, store *phasea.Store, runtimeStore runtim } } return c.Status(fiber.StatusConflict).JSON(map[string]string{ - "error": phasea.ErrConflict.Error(), "reason_code": resumeBlockReason(account, environment, bindingFound), "readiness": "blocked", + "error": accountdomain.ErrConflict.Error(), "reason_code": resumeBlockReason(account, environment, bindingFound), "readiness": "blocked", }) } @@ -375,10 +375,10 @@ func decodePhaseA(c fiber.Ctx, destination any) error { decoder := json.NewDecoder(bytes.NewReader(c.Body())) decoder.DisallowUnknownFields() if err := decoder.Decode(destination); err != nil { - return phasea.ErrInvalid + return accountdomain.ErrInvalid } if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { - return phasea.ErrInvalid + return accountdomain.ErrInvalid } return nil } @@ -386,7 +386,7 @@ func decodePhaseA(c fiber.Ctx, destination any) error { func phaseAError(c fiber.Ctx, err error) error { status := fiber.StatusInternalServerError message := "phase A operation failed" - var readiness *phasea.ReadinessError + var readiness *accountdomain.ReadinessError switch { case errors.As(err, &readiness): status, message = fiber.StatusConflict, "phase A version or account state changed" @@ -394,12 +394,22 @@ func phaseAError(c fiber.Ctx, err error) error { status, message = fiber.StatusServiceUnavailable, "phase A resources are not ready" } return c.Status(status).JSON(map[string]string{"error": message, "reason_code": readiness.Reason}) - case errors.Is(err, phasea.ErrInvalid): - status, message = fiber.StatusBadRequest, phasea.ErrInvalid.Error() - case errors.Is(err, phasea.ErrConflict): - status, message = fiber.StatusConflict, phasea.ErrConflict.Error() - case errors.Is(err, phasea.ErrNotFound): - status, message = fiber.StatusNotFound, phasea.ErrNotFound.Error() + case errors.Is(err, accountdomain.ErrInvalid): + status, message = fiber.StatusBadRequest, accountdomain.ErrInvalid.Error() + case errors.Is(err, accountdomain.ErrConflict): + status, message = fiber.StatusConflict, accountdomain.ErrConflict.Error() + case errors.Is(err, accountdomain.ErrNotFound): + status, message = fiber.StatusNotFound, accountdomain.ErrNotFound.Error() } return c.Status(status).JSON(map[string]string{"error": message}) } + +// RegisterAccounts exposes account and operation routes to the grouped API package. +func RegisterAccounts(app *fiber.App, store *accountdomain.Store, runtimeStore RuntimeStopStore, credentials accountdomain.CredentialBridge) { + registerPhaseA(app, store, runtimeStore, credentials) +} + +// PhaseAError maps account/operation errors to the existing HTTP contract. +func PhaseAError(c fiber.Ctx, err error) error { + return phaseAError(c, err) +} diff --git a/cmd/control-plane/main_test.go b/internal/controlplane/api/app_migrated_test.go similarity index 57% rename from cmd/control-plane/main_test.go rename to internal/controlplane/api/app_migrated_test.go index 253b985..8b6a3fb 100644 --- a/cmd/control-plane/main_test.go +++ b/internal/controlplane/api/app_migrated_test.go @@ -1,121 +1,28 @@ -package main +package api import ( - "bufio" - "bytes" "context" + "crypto/sha256" + "crypto/subtle" "database/sql" + "encoding/base64" "encoding/json" "errors" "io" "net/http" "net/http/httptest" "os" - "os/exec" "path/filepath" "strings" - "syscall" "testing" "time" + account "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" - "git.ipao.vip/rogee/creator-hub/internal/taskstate" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/gofiber/fiber/v3" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" ) -func TestExecuteContextStopsOnSIGTERM(t *testing.T) { - if os.Getenv("CREATORHUB_SIGTERM_HELPER") == "1" { - command := &cobra.Command{Use: "shutdown-test", RunE: func(command *cobra.Command, _ []string) error { - _, _ = os.Stdout.WriteString("ready\n") - <-command.Context().Done() - _, _ = os.Stdout.WriteString("stopped\n") - return nil - }} - if err := execute(command); err != nil { - t.Fatal(err) - } - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - process := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestExecuteContextStopsOnSIGTERM$") - process.Env = append(os.Environ(), "CREATORHUB_SIGTERM_HELPER=1") - stdout, err := process.StdoutPipe() - if err != nil { - t.Fatal(err) - } - if err := process.Start(); err != nil { - t.Fatal(err) - } - - scanner := bufio.NewScanner(stdout) - if !scanner.Scan() || scanner.Text() != "ready" { - t.Fatalf("helper did not become ready: %q err=%v", scanner.Text(), scanner.Err()) - } - if err := process.Process.Signal(syscall.SIGTERM); err != nil { - t.Fatal(err) - } - if !scanner.Scan() || scanner.Text() != "stopped" { - t.Fatalf("helper did not finish cleanup: %q err=%v", scanner.Text(), scanner.Err()) - } - if err := process.Wait(); err != nil || ctx.Err() != nil { - t.Fatalf("process did not exit gracefully after SIGTERM: wait=%v context=%v", err, ctx.Err()) - } -} - -func TestLoadConfigRejectsInvalidDatabase(t *testing.T) { - t.Setenv("CONTROL_PLANE_USERNAME", "operator") - t.Setenv("CONTROL_PLANE_PASSWORD", "unit-test-password") - t.Setenv("DATABASE_URL", "file:///tmp/creatorhub.db") - if _, err := loadConfig(); err == nil { - t.Fatal("expected non-Postgres database URL to be rejected") - } -} - -func TestLoadConfigRequiresControlPlaneCredentials(t *testing.T) { - t.Setenv("CONTROL_PLANE_USERNAME", "") - t.Setenv("CONTROL_PLANE_PASSWORD", "") - if _, err := loadConfig(); err == nil { - t.Fatal("expected missing control-plane credentials to be rejected") - } -} - -func TestLoadConfigPasswordLength(t *testing.T) { - t.Setenv("CONTROL_PLANE_USERNAME", "operator") - t.Setenv("CREATORHUB_CREDENTIAL_MASTER_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - for _, test := range []struct { - password string - wantErr bool - }{ - {"12345", true}, - {"123456", false}, - } { - t.Run(test.password, func(t *testing.T) { - t.Setenv("CONTROL_PLANE_PASSWORD", test.password) - _, err := loadConfig() - if (err != nil) != test.wantErr { - t.Fatalf("loadConfig() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestLoadConfigRequiresCredentialMasterKey(t *testing.T) { - t.Setenv("CONTROL_PLANE_USERNAME", "operator") - t.Setenv("CONTROL_PLANE_PASSWORD", "unit-test-password") - for _, key := range []string{"", "not-base64", "c2hvcnQ="} { - t.Setenv("CREATORHUB_CREDENTIAL_MASTER_KEY", key) - if _, err := loadConfig(); err == nil { - t.Fatalf("accepted invalid credential master key %q", key) - } - } -} - func TestCreatorReadRoutesAgainstPostgres(t *testing.T) { databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { @@ -123,7 +30,7 @@ func TestCreatorReadRoutesAgainstPostgres(t *testing.T) { } ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - phaseAStore, err := phasea.Open(ctx, databaseURL) + phaseAStore, err := account.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -160,7 +67,7 @@ func TestCreatorRouteValidationCoverage(t *testing.T) { } ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - phaseAStore, err := phasea.Open(ctx, databaseURL) + phaseAStore, err := account.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -176,10 +83,10 @@ func TestCreatorRouteValidationCoverage(t *testing.T) { } t.Cleanup(func() { _ = creatorStore.Close() }) credentials := &testCredentialBridge{values: make(map[string]string)} - if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + if err := phaseAStore.CreateAccount(ctx, account.Account{ ID: "route-account", Name: "Route Account", Platform: creator.PlatformDouyin, PlatformAccountKey: "route-platform", Tags: []string{}, Cookies: "", - CredentialReference: phasea.CredentialReference{ID: "route-account-credential", Provider: "os_keyring"}, + CredentialReference: account.CredentialReference{ID: "route-account-credential", Provider: "os_keyring"}, CredentialKey: "creatorhub/route-account/cookies", }, credentials); err != nil { t.Fatal(err) @@ -277,7 +184,7 @@ func TestCreatorFixtureRoutesPostgres(t *testing.T) { } ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - phaseAStore, err := phasea.Open(ctx, databaseURL) + phaseAStore, err := account.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -293,18 +200,18 @@ func TestCreatorFixtureRoutesPostgres(t *testing.T) { } t.Cleanup(func() { _ = creatorStore.Close() }) credentials := &testCredentialBridge{values: make(map[string]string)} - if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + if err := phaseAStore.CreateAccount(ctx, account.Account{ ID: "fixture-account", Name: "Fixture Account", Platform: creator.PlatformDouyin, PlatformAccountKey: "fixture-platform", Tags: []string{}, Cookies: "", - CredentialReference: phasea.CredentialReference{ID: "fixture-credential", Provider: "os_keyring"}, + CredentialReference: account.CredentialReference{ID: "fixture-credential", Provider: "os_keyring"}, CredentialKey: "creatorhub/fixture-account/cookies", }, credentials); err != nil { t.Fatal(err) } - if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + if err := phaseAStore.CreateAccount(ctx, account.Account{ ID: "fixture-small", Name: "Fixture Small", Platform: creator.PlatformDouyin, PlatformAccountKey: "fixture-small-platform", Tags: []string{}, Cookies: "", - CredentialReference: phasea.CredentialReference{ID: "fixture-small-credential", Provider: "os_keyring"}, + CredentialReference: account.CredentialReference{ID: "fixture-small-credential", Provider: "os_keyring"}, CredentialKey: "creatorhub/fixture-small/cookies", }, credentials); err != nil { t.Fatal(err) @@ -508,316 +415,6 @@ func TestCreatorFixtureRoutesPostgres(t *testing.T) { } } -func TestControlPlaneAuthentication(t *testing.T) { - logger := logrus.StandardLogger() - previousOutput := logger.Out - var logs bytes.Buffer - logrus.SetOutput(&logs) - t.Cleanup(func() { logrus.SetOutput(previousOutput) }) - - directory := t.TempDir() - if err := os.WriteFile(filepath.Join(directory, "index.html"), []byte("index"), 0o600); err != nil { - t.Fatal(err) - } - app := newHandler(directory, "operator", "unit-test-password") - - health, err := app.Test(httptest.NewRequest(http.MethodGet, "/healthz", nil)) - if err != nil || health.StatusCode != http.StatusNoContent { - t.Fatalf("health check must remain public: status=%d err=%v", health.StatusCode, err) - } - health.Body.Close() - - ready, err := app.Test(httptest.NewRequest(http.MethodGet, "/readyz", nil)) - if err != nil || ready.StatusCode != http.StatusServiceUnavailable { - t.Fatalf("readiness must fail without stores: status=%d err=%v", ready.StatusCode, err) - } - ready.Body.Close() - - unknown := httptest.NewRequest(http.MethodGet, "/api/not-registered", nil) - unknown.SetBasicAuth("operator", "unit-test-password") - unknownResponse, err := app.Test(unknown) - if err != nil || unknownResponse.StatusCode != http.StatusNotFound { - t.Fatalf("unknown API must be JSON 404: status=%d err=%v", unknownResponse.StatusCode, err) - } - if got := unknownResponse.Header.Get("Content-Type"); !strings.HasPrefix(got, "application/json") { - t.Fatalf("unknown API content type=%q", got) - } - unknownResponse.Body.Close() - - for _, path := range []string{ - "/", "/api/phase-a/accounts", "/api/browsers", "/api/network-exits", "/api/phase-a/tasks", "/api/phase-a/audit", - } { - request := httptest.NewRequest(http.MethodGet, path, nil) - response, err := app.Test(request) - if err != nil { - t.Fatal(err) - } - response.Body.Close() - if response.StatusCode != http.StatusUnauthorized || response.Header.Get("WWW-Authenticate") == "" { - t.Fatalf("GET %s was not protected: status=%d", path, response.StatusCode) - } - } - - for name, test := range map[string]struct { - user, password string - want int - }{ - "valid": {"operator", "unit-test-password", http.StatusOK}, - "wrong user": {"other", "unit-test-password", http.StatusUnauthorized}, - "wrong password": {"operator", "credential-must-not-be-logged", http.StatusUnauthorized}, - } { - t.Run(name, func(t *testing.T) { - request := httptest.NewRequest(http.MethodGet, "/", nil) - request.SetBasicAuth(test.user, test.password) - response, err := app.Test(request) - if err != nil { - t.Fatal(err) - } - response.Body.Close() - if response.StatusCode != test.want { - t.Fatalf("status=%d want=%d", response.StatusCode, test.want) - } - }) - } - if strings.Contains(logs.String(), "credential-must-not-be-logged") { - t.Fatalf("authentication credential reached logs: %s", logs.String()) - } -} - -type controlPlaneRouteCase struct { - method, pattern, path, body string - wantAuthenticatedStatus int -} - -type testCredentialBridge struct { - values map[string]string -} - -func (bridge *testCredentialBridge) Store(ctx context.Context, _ phasea.CredentialReference, key, value string) error { - if err := ctx.Err(); err != nil { - return err - } - bridge.values[key] = value - return nil -} - -func (bridge *testCredentialBridge) Delete(ctx context.Context, _ phasea.CredentialReference, key string) error { - if err := ctx.Err(); err != nil { - return err - } - delete(bridge.values, key) - return nil -} - -func TestControlPlaneRouteRegistrationMatrix(t *testing.T) { - app := fiber.New() - app.Use(authenticate("operator", "unit-test-password")) - registerHubWithNetwork(app, nil, nil, nil) - registerPhaseA(app, nil, nil, nil) - routes := controlPlaneRouteMatrix() - assertControlPlaneRouteMatrix(t, app, routes) - - for _, route := range routes { - if response := do(app, route.method, route.path, route.body); response.Code != http.StatusUnauthorized { - t.Fatalf("registered route %s %s returned %d without credentials, want %d", route.method, route.path, - response.Code, http.StatusUnauthorized) - } - } - if response := do(app, http.MethodPost, "/api/not-registered", ""); response.Code != http.StatusUnauthorized { - t.Fatalf("unregistered route returned %d without credentials, want %d", response.Code, http.StatusUnauthorized) - } -} - -func TestControlPlaneRegisteredRouteMatrix(t *testing.T) { - databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") - if databaseURL == "" { - t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run control-plane route coverage") - } - databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - ctx := context.Background() - phaseAStore, err := phasea.Open(ctx, databaseURL) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = phaseAStore.Close() }) - hubStore, err := hub.Open(ctx, databaseURL) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = hubStore.Close() }) - - webDirectory := t.TempDir() - if err := os.WriteFile(filepath.Join(webDirectory, "index.html"), []byte("index"), 0o600); err != nil { - t.Fatal(err) - } - app := newHandlerWithStores(webDirectory, "operator", "unit-test-password", phaseAStore, hubStore) - routes := controlPlaneRouteMatrix() - assertControlPlaneRouteMatrix(t, app, routes) - - for _, route := range routes { - route := route - t.Run(route.method+" "+route.pattern, func(t *testing.T) { - if response := do(app, route.method, route.path, route.body); response.Code != http.StatusUnauthorized { - t.Fatalf("unauthenticated %s %s returned %d, want %d", route.method, route.path, response.Code, http.StatusUnauthorized) - } - response := do(app, route.method, route.path, route.body, "operator", "unit-test-password") - if response.Code != route.wantAuthenticatedStatus { - t.Fatalf("authenticated %s %s returned %d, want %d: %s", route.method, route.path, response.Code, - route.wantAuthenticatedStatus, response.Body.String()) - } - }) - } - - for _, route := range []struct { - method, path string - want int - }{ - {http.MethodPost, "/api/not-registered", http.StatusNotFound}, - {http.MethodDelete, "/api/not-registered", http.StatusNotFound}, - } { - t.Run("unregistered "+route.method, func(t *testing.T) { - if response := do(app, route.method, route.path, ""); response.Code != http.StatusUnauthorized { - t.Fatalf("unauthenticated unregistered %s %s returned %d, want %d", route.method, route.path, response.Code, http.StatusUnauthorized) - } - if response := do(app, route.method, route.path, "", "operator", "unit-test-password"); response.Code != route.want { - t.Fatalf("authenticated unregistered %s %s returned %d, want %d", route.method, route.path, response.Code, route.want) - } - }) - } -} - -func assertControlPlaneRouteMatrix(t *testing.T, app *fiber.App, routes []controlPlaneRouteCase) { - t.Helper() - expected := make(map[string]struct{}, len(routes)) - for _, route := range routes { - key := controlPlaneRouteKey(route.method, route.pattern) - if _, exists := expected[key]; exists { - t.Fatalf("duplicate route matrix entry: %s", key) - } - expected[key] = struct{}{} - } - actual := registeredControlPlaneRoutes(app) - for key := range expected { - if _, exists := actual[key]; !exists { - t.Errorf("route matrix is missing registered route: %s", key) - } - } - for key := range actual { - if _, exists := expected[key]; !exists { - t.Errorf("registered API route is missing from route matrix: %s", key) - } - } -} - -func controlPlaneRouteKey(method, path string) string { return method + " " + path } - -func registeredControlPlaneRoutes(app *fiber.App) map[string]struct{} { - routes := map[string]struct{}{} - for _, methodRoutes := range app.Stack() { - for _, route := range methodRoutes { - if !strings.HasPrefix(route.Path, "/api/") { - continue - } - switch route.Method { - case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete: - routes[controlPlaneRouteKey(route.Method, route.Path)] = struct{}{} - } - } - } - return routes -} - -func controlPlaneRouteMatrix() []controlPlaneRouteCase { - return []controlPlaneRouteCase{ - {http.MethodGet, "/api/browsers", "/api/browsers", "", http.StatusOK}, - {http.MethodGet, "/api/browsers/:alias", "/api/browsers/missing", "", http.StatusNotFound}, - {http.MethodPost, "/api/browsers", "/api/browsers", "", http.StatusBadRequest}, - {http.MethodPost, "/api/browsers/:alias/:action", "/api/browsers/missing/start", "", http.StatusNotFound}, - {http.MethodDelete, "/api/browsers/:alias", "/api/browsers/missing", "", http.StatusNotFound}, - - {http.MethodGet, "/api/network-exits", "/api/network-exits", "", http.StatusOK}, - {http.MethodGet, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusNotFound}, - {http.MethodPost, "/api/network-exits", "/api/network-exits", "", http.StatusBadRequest}, - {http.MethodPost, "/api/network-exits/:id/check", "/api/network-exits/missing/check", "", http.StatusNotFound}, - {http.MethodPost, "/api/network-exits/:id/disable", "/api/network-exits/missing/disable", "", http.StatusNotFound}, - {http.MethodPut, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusBadRequest}, - {http.MethodPost, "/api/network-exits/:id/enable", "/api/network-exits/missing/enable", "", http.StatusNotFound}, - {http.MethodDelete, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusNotFound}, - - {http.MethodGet, "/api/browser-versions", "/api/browser-versions", "", http.StatusOK}, - {http.MethodPost, "/api/browser-versions", "/api/browser-versions", "", http.StatusBadRequest}, - {http.MethodPut, "/api/browser-versions/:version", "/api/browser-versions/999.0.0", `{"browser_path":"/opt/creatorhub/browsers/missing"}`, http.StatusNotFound}, - {http.MethodDelete, "/api/browser-versions/:version", "/api/browser-versions/999.0.0", "", http.StatusNotFound}, - - {http.MethodGet, "/api/gateways", "/api/gateways", "", http.StatusOK}, - {http.MethodPost, "/api/gateways", "/api/gateways", "", http.StatusBadRequest}, - {http.MethodPut, "/api/gateways/:name", "/api/gateways/missing", `{"name":"gw-missing","endpoint":"http://gw-missing:8081","token":""}`, http.StatusNotFound}, - {http.MethodDelete, "/api/gateways/:name", "/api/gateways/missing", "", http.StatusNotFound}, - - {http.MethodPost, "/api/phase-a/accounts", "/api/phase-a/accounts", "", http.StatusBadRequest}, - {http.MethodGet, "/api/phase-a/accounts", "/api/phase-a/accounts", "", http.StatusOK}, - {http.MethodGet, "/api/phase-a/accounts/:id", "/api/phase-a/accounts/missing", "", http.StatusNotFound}, - {http.MethodPost, "/api/phase-a/accounts/:id/pause", "/api/phase-a/accounts/missing/pause", "", http.StatusNotFound}, - {http.MethodPost, "/api/phase-a/accounts/:id/resume", "/api/phase-a/accounts/missing/resume", "", http.StatusNotFound}, - {http.MethodPost, "/api/phase-a/accounts/:id/revoke", "/api/phase-a/accounts/missing/revoke", "", http.StatusNotFound}, - - {http.MethodPost, "/api/phase-a/drafts", "/api/phase-a/drafts", "", http.StatusBadRequest}, - {http.MethodGet, "/api/phase-a/drafts", "/api/phase-a/drafts", "", http.StatusOK}, - {http.MethodGet, "/api/phase-a/drafts/:id", "/api/phase-a/drafts/missing", "", http.StatusNotFound}, - - {http.MethodPost, "/api/phase-a/confirmations", "/api/phase-a/confirmations", "", http.StatusBadRequest}, - {http.MethodGet, "/api/phase-a/confirmations", "/api/phase-a/confirmations", "", http.StatusOK}, - {http.MethodGet, "/api/phase-a/confirmations/:id", "/api/phase-a/confirmations/missing", "", http.StatusNotFound}, - - {http.MethodPost, "/api/phase-a/tasks", "/api/phase-a/tasks", "", http.StatusBadRequest}, - {http.MethodGet, "/api/phase-a/tasks", "/api/phase-a/tasks", "", http.StatusOK}, - {http.MethodGet, "/api/phase-a/tasks/:id", "/api/phase-a/tasks/missing", "", http.StatusNotFound}, - {http.MethodGet, "/api/phase-a/attempts/:id", "/api/phase-a/attempts/missing", "", http.StatusNotFound}, - {http.MethodPost, "/api/phase-a/tasks/:id/cancel", "/api/phase-a/tasks/missing/cancel", "", http.StatusNotFound}, - {http.MethodPost, "/api/phase-a/tasks/:id/verify", "/api/phase-a/tasks/missing/verify", "", http.StatusBadRequest}, - {http.MethodPost, "/api/phase-a/tasks/:id/resume", "/api/phase-a/tasks/missing/resume", "", http.StatusNotFound}, - {http.MethodPost, "/api/phase-a/tasks/:id/finish", "/api/phase-a/tasks/missing/finish", "", http.StatusNotFound}, - - {http.MethodPost, "/api/phase-a/mock/execute", "/api/phase-a/mock/execute", "", http.StatusBadRequest}, - {http.MethodGet, "/api/phase-a/audit", "/api/phase-a/audit", "", http.StatusOK}, - } -} - -func TestOperatorNotificationFiltersAndRedacts(t *testing.T) { - previousLevel := logrus.GetLevel() - t.Cleanup(func() { logrus.SetLevel(previousLevel) }) - var output bytes.Buffer - notify := newAttentionNotifier(&output) - - notify(taskstate.Transition{State: "succeeded", ReasonCode: "task_succeeded", AccountID: "account-a", TaskID: "task-a"}) - if output.Len() != 0 { - t.Fatalf("successful task emitted a notification: %s", output.String()) - } - - for _, level := range []logrus.Level{logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel} { - for _, test := range []struct{ state, reason string }{ - {"policy_hold", "account_paused"}, {"needs_confirmation", "exit_unhealthy"}, - } { - output.Reset() - logrus.SetLevel(level) - notify(taskstate.Transition{State: test.state, ReasonCode: test.reason, AccountID: "account-a", TaskID: "task-a"}) - var entry map[string]any - if err := json.Unmarshal(output.Bytes(), &entry); err != nil { - t.Fatal(err) - } - if entry["event_type"] != test.state || entry["reason_code"] != test.reason || - entry["notification_channel"] != "structured_log" || entry["account_id"] != "account-a" || entry["task_id"] != "task-a" { - t.Fatalf("unexpected notification at LOG_LEVEL=%s: %#v", level, entry) - } - for _, forbidden := range []string{"password", "authorization", "credential", "token", "secret"} { - if strings.Contains(strings.ToLower(output.String()), forbidden) { - t.Fatalf("notification contained sensitive field %q: %s", forbidden, output.String()) - } - } - } - } -} - func TestPhaseAAccountRequestRejectsUnknownFields(t *testing.T) { for name, body := range map[string]string{ "client id": `{"id":"account-a","name":"账号 A","platform":"douyin","platform_account_key":"a","tags":[],"cookies":"sid=value"}`, @@ -852,7 +449,7 @@ func TestPhaseAAccountHTTPWorkflowRedactsSecrets(t *testing.T) { } ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - store, err := phasea.Open(ctx, databaseURL) + store, err := account.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -904,7 +501,7 @@ func TestPhaseAAccountHTTPWorkflowRedactsSecrets(t *testing.T) { } created := request(http.MethodPost, "/api/phase-a/accounts", accountBody, http.StatusCreated) - var account phasea.Account + var account account.Account if err := json.Unmarshal(created.Body.Bytes(), &account); err != nil || account.ID == "" || account.Name != "Phase HTTP" || account.RuntimeStatus != "paused" { t.Fatalf("unexpected account response: %#v err=%v", account, err) } @@ -927,49 +524,6 @@ func TestPhaseAAccountHTTPWorkflowRedactsSecrets(t *testing.T) { assertControlPlaneDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'social_account' AND column_name = 'cookies'`, 0) - for _, failure := range []struct { - name string - configure func(*persistentCredentialBridge) - }{ - {name: "chmod", configure: func(bridge *persistentCredentialBridge) { - bridge.chmod = func(*os.File, os.FileMode) error { return errors.New("chmod unavailable") } - }}, - {name: "directory-sync", configure: func(bridge *persistentCredentialBridge) { - calls := 0 - bridge.syncDirectory = func(path string) error { - calls++ - if calls == 1 { - return errors.New("directory sync unavailable") - } - return syncCredentialDirectory(path) - } - }}, - } { - failureDirectory := t.TempDir() - failureCredentials, err := newPersistentCredentialBridge(failureDirectory, []byte("0123456789abcdef0123456789abcdef")) - if err != nil { - t.Fatal(err) - } - failure.configure(failureCredentials) - failureApp := fiber.New() - failureApp.Use(authenticate("operator", "unit-test-password")) - registerPhaseA(failureApp, store, nil, failureCredentials) - platformKey := "phase-http-store-" + failure.name - failureBody := `{"name":"Store failure","platform":"douyin","platform_account_key":"` + platformKey + `","tags":[],"cookies":"sessionid=value"}` - failureResponse := do(failureApp, http.MethodPost, "/api/phase-a/accounts", failureBody, "operator", "unit-test-password") - if failureResponse.Code != http.StatusInternalServerError { - t.Fatalf("%s failure returned %d, want %d: %s", failure.name, failureResponse.Code, http.StatusInternalServerError, failureResponse.Body.String()) - } - var failedAccounts int - if err := db.QueryRowContext(ctx, `SELECT count(*) FROM social_account WHERE platform_account_key = $1`, platformKey).Scan(&failedAccounts); err != nil || failedAccounts != 0 { - t.Fatalf("%s failure persisted %d accounts: %v", failure.name, failedAccounts, err) - } - entries, err := os.ReadDir(failureDirectory) - if err != nil || len(entries) != 0 { - t.Fatalf("%s failure left ciphertext: entries=%v err=%v", failure.name, entries, err) - } - } - if _, err := db.ExecContext(ctx, ` INSERT INTO gateway (name, endpoint, token) VALUES ('phase-http', 'http://127.0.0.1:8081', 'phase-http-gateway-token'); INSERT INTO browser_version (version, browser_path) VALUES ('1', '/opt/creatorhub/browsers/1'); @@ -1009,63 +563,35 @@ func TestPhaseAErrorRedactsInternalDetails(t *testing.T) { } } -func TestSPAServesFileAndIndexFallback(t *testing.T) { - directory := t.TempDir() - files := map[string]string{ - "index.html": "index", - "app.js": "asset", - "hello world.js": "space", - "应用.js": "unicode", +func newHandlerWithStores(_ string, _ string, _ string, accountStore *account.Store, environmentStore *hub.Store) *fiber.App { + appInstance := fiber.New() + if environmentStore != nil { + RegisterEnvironments(appInstance, environmentStore) } - for name, body := range files { - if err := os.WriteFile(filepath.Join(directory, name), []byte(body), 0o600); err != nil { - t.Fatal(err) - } - } - if err := os.WriteFile(filepath.Join(directory, "..", "outside-secret.txt"), []byte("secret"), 0o600); err != nil { - t.Fatal(err) - } - app := newHandler(directory, "operator", "unit-test-password") - for path, want := range map[string]string{ - "/app.js": "asset", - "/hello%20world.js": "space", - "/%E5%BA%94%E7%94%A8.js": "unicode", - "/%2e%2e%2foutside-secret.txt": "index", - "/dashboard": "index", - } { - request := httptest.NewRequest(http.MethodGet, path, nil) - request.SetBasicAuth("operator", "unit-test-password") - response, err := app.Test(request) - if err != nil { - t.Fatal(err) - } - body, readErr := io.ReadAll(response.Body) - response.Body.Close() - if readErr != nil || response.StatusCode != http.StatusOK || string(body) != want { - t.Fatalf("GET %s: status=%d body=%q err=%v", path, response.StatusCode, body, readErr) - } + if accountStore != nil { + RegisterAccounts(appInstance, accountStore, environmentStore, nil) } + return appInstance } -func TestStartupLogIncludesListenAddress(t *testing.T) { - logger := logrus.StandardLogger() - previousOutput, previousFormatter, previousLevel := logger.Out, logger.Formatter, logger.Level - t.Cleanup(func() { - logrus.SetOutput(previousOutput) - logrus.SetFormatter(previousFormatter) - logrus.SetLevel(previousLevel) - }) - var output bytes.Buffer - logrus.SetOutput(&output) - logrus.SetFormatter(&logrus.JSONFormatter{}) - logrus.SetLevel(logrus.InfoLevel) - - logStartup(config{listenAddr: ":8080"}) - var entry map[string]any - if err := json.Unmarshal(output.Bytes(), &entry); err != nil { - t.Fatal(err) +func newHandlerWithCreator(_ string, _ string, _ string, accountStore *account.Store, environmentStore *hub.Store, _ account.CredentialBridge, creatorStore *creator.Store) *fiber.App { + appInstance := newHandlerWithStores("", "", "", accountStore, environmentStore) + if creatorStore != nil { + RegisterCreator(appInstance, creatorStore, accountStore, environmentStore) } - if entry["listen_addr"] != ":8080" || entry["service"] != "control-plane" { - t.Fatalf("unexpected startup log: %#v", entry) + return appInstance +} + +func authenticate(username, password string) fiber.Handler { + wantUser, wantPassword := sha256.Sum256([]byte(username)), sha256.Sum256([]byte(password)) + return func(c fiber.Ctx) error { + encoded, ok := strings.CutPrefix(c.Get(fiber.HeaderAuthorization), "Basic ") + decoded, err := base64.StdEncoding.DecodeString(encoded) + user, suppliedPassword, found := strings.Cut(string(decoded), ":") + gotUser, gotPassword := sha256.Sum256([]byte(user)), sha256.Sum256([]byte(suppliedPassword)) + if !ok || err != nil || !found || subtle.ConstantTimeCompare(gotUser[:], wantUser[:]) != 1 || subtle.ConstantTimeCompare(gotPassword[:], wantPassword[:]) != 1 { + return c.Status(fiber.StatusUnauthorized).JSON(map[string]string{"error": "authentication required"}) + } + return c.Next() } } diff --git a/cmd/control-plane/creator.go b/internal/controlplane/api/creator.go similarity index 95% rename from cmd/control-plane/creator.go rename to internal/controlplane/api/creator.go index 9e8f4ca..f14cb3d 100644 --- a/cmd/control-plane/creator.go +++ b/internal/controlplane/api/creator.go @@ -1,4 +1,4 @@ -package main +package api import ( "bufio" @@ -17,10 +17,10 @@ import ( "strings" "time" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/douyin" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + douyin "git.ipao.vip/rogee/creator-hub/internal/platform/douyin" "github.com/gofiber/fiber/v3" "github.com/sirupsen/logrus" ) @@ -46,7 +46,7 @@ func creatorPageQuery(c fiber.Ctx) (page, pageSize int, enabled bool, err error) return page, pageSize, true, nil } -func registerCreator(app *fiber.App, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store) { +func registerCreator(app *fiber.App, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store) { var executor creator.ActionExecutor if store != nil && phaseAStore != nil && hubStore != nil { executor = creatorGatewayActionExecutor{store: store, phaseAStore: phaseAStore, hubStore: hubStore} @@ -54,7 +54,7 @@ func registerCreator(app *fiber.App, store *creator.Store, phaseAStore *phasea.S registerCreatorWithServices(app, store, phaseAStore, hubStore, executor, nil, nil) } -func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, executor creator.ActionExecutor, generator creator.TextGenerator, analyzer creator.ThemeAnalyzer) { +func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, executor creator.ActionExecutor, generator creator.TextGenerator, analyzer creator.ThemeAnalyzer) { // Platform records enter through the managed collector/listener, not a public // client-supplied write. The explicit test namespace is kept for isolated // contract tests and never participates in the production listener. @@ -1111,7 +1111,7 @@ type creatorGatewayBrowser struct { type creatorGatewayActionExecutor struct { store *creator.Store - phaseAStore *phasea.Store + phaseAStore *accountdomain.Store hubStore *hub.Store } @@ -1295,7 +1295,7 @@ func (browser creatorGatewayBrowser) MessageHistory(ctx context.Context, expecte const creatorLoginQRLifetime = 2 * time.Minute -func startCreatorEnvironment(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error { +func startCreatorEnvironment(ctx context.Context, store HubStore, environment hub.EnvironmentContext) error { if store == nil { return creator.ErrUnavailable } @@ -1311,7 +1311,7 @@ func startCreatorEnvironment(ctx context.Context, store hubStore, environment hu return startBrowserRuntime(ctx, store, defaultNetworkExitProbe(), environment, finish) } -func creatorLoginQRCode(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore hubStore, accountID string) (map[string]any, error) { +func creatorLoginQRCode(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore HubStore, accountID string) (map[string]any, error) { if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" { return nil, creator.ErrUnavailable } @@ -1369,7 +1369,7 @@ func creatorLoginQRCode(ctx context.Context, store *creator.Store, phaseAStore * }, nil } -func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID string) (creator.LoginResult, error) { +func verifyCreatorAccount(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID string) (creator.LoginResult, error) { if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" { return creator.LoginResult{}, creator.ErrUnavailable } @@ -1563,7 +1563,7 @@ func competitorSharePlatform(raw string) (string, error) { } } -func previewCompetitorShare(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, requestedPlatform, rawShareURL string) (competitorSharePreview, error) { +func previewCompetitorShare(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID, requestedPlatform, rawShareURL string) (competitorSharePreview, error) { platform, err := competitorSharePlatform(rawShareURL) if err != nil { return competitorSharePreview{}, err @@ -1582,7 +1582,7 @@ func previewCompetitorShare(ctx context.Context, store *creator.Store, phaseASto } } -func previewDouyinCompetitorShare(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, shareURL string) (competitorSharePreview, error) { +func previewDouyinCompetitorShare(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID, shareURL string) (competitorSharePreview, error) { browser, err := newDouyinAccountBrowser(ctx, store, phaseAStore, hubStore, accountID) if err != nil { return competitorSharePreview{}, err @@ -1610,7 +1610,7 @@ func previewDouyinCompetitorShare(ctx context.Context, store *creator.Store, pha }, nil } -func previewXiaohongshuCompetitorShare(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, shareURL string) (competitorSharePreview, error) { +func previewXiaohongshuCompetitorShare(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID, shareURL string) (competitorSharePreview, error) { collector, err := newXiaohongshuReadCollector(ctx, store, phaseAStore, hubStore, accountID, creator.SourceCompetitor, accountID) if err != nil { return competitorSharePreview{}, err @@ -1644,7 +1644,7 @@ func douyinWorkKeyFromURL(raw string) (string, error) { return parts[1], nil } -func newDouyinAccountBrowser(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID string) (creatorGatewayBrowser, error) { +func newDouyinAccountBrowser(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID string) (creatorGatewayBrowser, error) { if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" { return creatorGatewayBrowser{}, creator.ErrInvalid } @@ -1677,7 +1677,7 @@ func newDouyinAccountBrowser(ctx context.Context, store *creator.Store, phaseASt return browser, nil } -func previewDouyinCompetitor(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID string, input creator.CompetitorInput) (map[string]any, error) { +func previewDouyinCompetitor(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID string, input creator.CompetitorInput) (map[string]any, error) { if input.Platform != creator.PlatformDouyin { return nil, creator.ErrInvalid } @@ -1699,15 +1699,15 @@ func previewDouyinCompetitor(ctx context.Context, store *creator.Store, phaseASt }, nil } -func syncCreatorCompetitor(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, competitorID, accountID string) (creator.CollectionReport, error) { +func syncCreatorCompetitor(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, competitorID, accountID string) (creator.CollectionReport, error) { return syncCreatorCompetitorWithClaim(ctx, store, phaseAStore, hubStore, competitorID, accountID, true) } -func syncCreatorCompetitorDue(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, competitorID, accountID string) (creator.CollectionReport, error) { +func syncCreatorCompetitorDue(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, competitorID, accountID string) (creator.CollectionReport, error) { return syncCreatorCompetitorWithClaim(ctx, store, phaseAStore, hubStore, competitorID, accountID, false) } -func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, competitorID, accountID string, force bool) (creator.CollectionReport, error) { +func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, competitorID, accountID string, force bool) (creator.CollectionReport, error) { if store == nil || phaseAStore == nil || hubStore == nil || accountID == "" { return creator.CollectionReport{}, creator.ErrUnavailable } @@ -1808,7 +1808,7 @@ func syncCreatorCompetitorWithClaim(ctx context.Context, store *creator.Store, p return report, nil } -func runCreatorScheduleOnce(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store) error { +func runCreatorScheduleOnce(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store) error { if store == nil { return creator.ErrUnavailable } @@ -1856,7 +1856,7 @@ func runCreatorScheduleOnce(ctx context.Context, store *creator.Store, phaseASto return nil } -func runCreatorMetricScheduleOnce(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, now time.Time) error { +func runCreatorMetricScheduleOnce(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, now time.Time) error { settings, err := store.GetSettings(ctx) if err != nil { return err @@ -1881,7 +1881,7 @@ func runCreatorMetricScheduleOnce(ctx context.Context, store *creator.Store, pha return nil } -func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, work creator.Work, accountID string, settings creator.Settings, now time.Time) (resultErr error) { +func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, work creator.Work, accountID string, settings creator.Settings, now time.Time) (resultErr error) { if work.SourceType == creator.SourceCompetitor { competitor, err := store.GetCompetitor(ctx, work.SourceID) if err != nil { @@ -1963,7 +1963,7 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS return creator.ErrInvalid } -func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID string, now time.Time) error { +func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID string, now time.Time) error { if store == nil || phaseAStore == nil || hubStore == nil || accountID == "" { return creator.ErrUnavailable } @@ -2022,7 +2022,7 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ph return errors.Join(err, runtimeUse.Close()) } -func creatorCollectionAccount(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, platform string) (string, error) { +func creatorCollectionAccount(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, platform string) (string, error) { if store == nil || phaseAStore == nil || hubStore == nil || platform == "" { return "", creator.ErrUnavailable } @@ -2047,7 +2047,7 @@ func creatorCollectionAccount(ctx context.Context, store *creator.Store, phaseAS return "", fmt.Errorf("%w: no authorized creator collection account", creator.ErrUnavailable) } -func runCreatorScheduler(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store) { +func runCreatorScheduler(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store) { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() for { @@ -2061,3 +2061,27 @@ func runCreatorScheduler(ctx context.Context, store *creator.Store, phaseAStore } } } + +// RegisterCreator exposes creator routes to the grouped API package. +func RegisterCreator(app *fiber.App, store *creator.Store, accountStore *accountdomain.Store, environmentStore *hub.Store) { + registerCreator(app, store, accountStore, environmentStore) +} + +// RegisterCreatorWithAI exposes creator routes with the configured service implementations. +func RegisterCreatorWithAI(app *fiber.App, store *creator.Store, accountStore *accountdomain.Store, environmentStore *hub.Store, generator creator.TextGenerator, analyzer creator.ThemeAnalyzer) { + var executor creator.ActionExecutor + if store != nil && accountStore != nil && environmentStore != nil { + executor = creatorGatewayActionExecutor{store: store, phaseAStore: accountStore, hubStore: environmentStore} + } + registerCreatorWithServices(app, store, accountStore, environmentStore, executor, generator, analyzer) +} + +// RunCreatorScheduler runs the creator scheduler until its context is canceled. +func RunCreatorScheduler(ctx context.Context, store *creator.Store, accountStore *accountdomain.Store, environmentStore *hub.Store) { + runCreatorScheduler(ctx, store, accountStore, environmentStore) +} + +// NewCreatorGatewayActionExecutor builds the native gateway-backed action executor. +func NewCreatorGatewayActionExecutor(store *creator.Store, accountStore *accountdomain.Store, environmentStore *hub.Store) creator.ActionExecutor { + return creatorGatewayActionExecutor{store: store, phaseAStore: accountStore, hubStore: environmentStore} +} diff --git a/internal/controlplane/api/creator/register.go b/internal/controlplane/api/creator/register.go new file mode 100644 index 0000000..e31ca3f --- /dev/null +++ b/internal/controlplane/api/creator/register.go @@ -0,0 +1,17 @@ +package creator + +import ( + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api" + creatorDomain "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/gofiber/fiber/v3" +) + +func Register(app *fiber.App, store *creatorDomain.Store, accountStore *account.Store, environmentStore *environment.Store) { + api.RegisterCreator(app, store, accountStore, environmentStore) +} + +func RegisterWithAI(app *fiber.App, store *creatorDomain.Store, accountStore *account.Store, environmentStore *environment.Store, generator creatorDomain.TextGenerator, analyzer creatorDomain.ThemeAnalyzer) { + api.RegisterCreatorWithAI(app, store, accountStore, environmentStore, generator, analyzer) +} diff --git a/cmd/control-plane/creator_events.go b/internal/controlplane/api/creator_events.go similarity index 97% rename from cmd/control-plane/creator_events.go rename to internal/controlplane/api/creator_events.go index 72922b0..d4e8324 100644 --- a/cmd/control-plane/creator_events.go +++ b/internal/controlplane/api/creator_events.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -11,9 +11,9 @@ import ( "sync/atomic" "time" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/sirupsen/logrus" ) @@ -136,7 +136,7 @@ type creatorEventListenerManager struct { items map[string]creatorEventListenerHandle } -func runCreatorEventListeners(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, executor creator.ActionExecutor, generator creator.TextGenerator) { +func RunCreatorEventListeners(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, executor creator.ActionExecutor, generator creator.TextGenerator) { manager := &creatorEventListenerManager{items: map[string]creatorEventListenerHandle{}} ticker := time.NewTicker(creatorEventReconcileInterval) defer ticker.Stop() @@ -153,7 +153,7 @@ func runCreatorEventListeners(ctx context.Context, store *creator.Store, phaseAS } } -func (manager *creatorEventListenerManager) reconcile(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, executor creator.ActionExecutor, generator creator.TextGenerator) error { +func (manager *creatorEventListenerManager) reconcile(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, executor creator.ActionExecutor, generator creator.TextGenerator) error { if store == nil || phaseAStore == nil || hubStore == nil { return creator.ErrUnavailable } diff --git a/cmd/control-plane/creator_events_test.go b/internal/controlplane/api/creator_events_test.go similarity index 99% rename from cmd/control-plane/creator_events_test.go rename to internal/controlplane/api/creator_events_test.go index 142c8fb..8f76721 100644 --- a/cmd/control-plane/creator_events_test.go +++ b/internal/controlplane/api/creator_events_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -6,7 +6,7 @@ import ( "time" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestCreatorListenerPrimitives(t *testing.T) { diff --git a/cmd/control-plane/creator_events_unit_test.go b/internal/controlplane/api/creator_events_unit_test.go similarity index 96% rename from cmd/control-plane/creator_events_unit_test.go rename to internal/controlplane/api/creator_events_unit_test.go index 7364d21..5d5372c 100644 --- a/cmd/control-plane/creator_events_unit_test.go +++ b/internal/controlplane/api/creator_events_unit_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -9,7 +9,7 @@ import ( "time" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestCreatorEventFromGatewayNoticeValidatesAndNormalizesInput(t *testing.T) { @@ -80,7 +80,7 @@ func TestCreatorEventListenerControlPathsRemainVisible(t *testing.T) { } listenerCtx, listenerCancel := context.WithCancel(context.Background()) listenerCancel() - runCreatorEventListeners(listenerCtx, nil, nil, nil, nil, nil) + RunCreatorEventListeners(listenerCtx, nil, nil, nil, nil, nil) } func TestHandleCreatorGatewayEventClassifiesNonNoticeEventsWithoutStore(t *testing.T) { diff --git a/cmd/control-plane/creator_helper_test.go b/internal/controlplane/api/creator_helper_test.go similarity index 99% rename from cmd/control-plane/creator_helper_test.go rename to internal/controlplane/api/creator_helper_test.go index 14b9e3e..1f5821e 100644 --- a/cmd/control-plane/creator_helper_test.go +++ b/internal/controlplane/api/creator_helper_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -13,7 +13,7 @@ import ( "time" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/gofiber/fiber/v3" ) diff --git a/cmd/control-plane/creator_history_test.go b/internal/controlplane/api/creator_history_test.go similarity index 86% rename from cmd/control-plane/creator_history_test.go rename to internal/controlplane/api/creator_history_test.go index 902f5ad..13bfa94 100644 --- a/cmd/control-plane/creator_history_test.go +++ b/internal/controlplane/api/creator_history_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -8,9 +8,9 @@ import ( "testing" "time" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestPersistDouyinMessageHistory(t *testing.T) { @@ -20,7 +20,7 @@ func TestPersistDouyinMessageHistory(t *testing.T) { } ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - phaseAStore, err := phasea.Open(ctx, databaseURL) + phaseAStore, err := accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -36,10 +36,10 @@ func TestPersistDouyinMessageHistory(t *testing.T) { } t.Cleanup(func() { _ = store.Close() }) credentials := &testCredentialBridge{values: make(map[string]string)} - if err := phaseAStore.CreateAccount(ctx, phasea.Account{ + if err := phaseAStore.CreateAccount(ctx, accountdomain.Account{ ID: "history-account", Name: "History Account", Platform: creator.PlatformDouyin, PlatformAccountKey: "history-platform", Tags: []string{}, Cookies: "", - CredentialReference: phasea.CredentialReference{ID: "history-credential", Provider: "os_keyring"}, + CredentialReference: accountdomain.CredentialReference{ID: "history-credential", Provider: "os_keyring"}, CredentialKey: "creatorhub/history-account/cookies", }, credentials); err != nil { t.Fatal(err) diff --git a/cmd/control-plane/creator_login_test.go b/internal/controlplane/api/creator_login_test.go similarity index 95% rename from cmd/control-plane/creator_login_test.go rename to internal/controlplane/api/creator_login_test.go index 57ee025..4ebb612 100644 --- a/cmd/control-plane/creator_login_test.go +++ b/internal/controlplane/api/creator_login_test.go @@ -1,11 +1,11 @@ -package main +package api import ( "context" "net/http/httptest" "testing" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestStartCreatorEnvironmentStartsBoundRuntime(t *testing.T) { diff --git a/cmd/control-plane/creator_material.go b/internal/controlplane/api/creator_material.go similarity index 98% rename from cmd/control-plane/creator_material.go rename to internal/controlplane/api/creator_material.go index 06bdd77..d6e589e 100644 --- a/cmd/control-plane/creator_material.go +++ b/internal/controlplane/api/creator_material.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -11,15 +11,15 @@ import ( "path/filepath" "strings" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/sirupsen/logrus" ) type creatorMaterialDownloader struct { store *creator.Store - phaseAStore *phasea.Store + phaseAStore *accountdomain.Store hubStore *hub.Store } @@ -79,7 +79,7 @@ func (downloader creatorMaterialDownloader) Download(ctx context.Context, work c return writeCreatorMedia(destination, data) } -func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, workID string) (creator.MaterialJob, error) { +func processCreatorMaterial(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, workID string) (creator.MaterialJob, error) { if store == nil || workID == "" || workID == "." || workID == ".." || strings.ContainsAny(workID, `/\\`) || filepath.Base(workID) != workID { return creator.MaterialJob{}, creator.ErrInvalid } diff --git a/cmd/control-plane/creator_material_test.go b/internal/controlplane/api/creator_material_test.go similarity index 99% rename from cmd/control-plane/creator_material_test.go rename to internal/controlplane/api/creator_material_test.go index ad10a01..31eb7d7 100644 --- a/cmd/control-plane/creator_material_test.go +++ b/internal/controlplane/api/creator_material_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" diff --git a/cmd/control-plane/creator_pure_unit_test.go b/internal/controlplane/api/creator_pure_unit_test.go similarity index 99% rename from cmd/control-plane/creator_pure_unit_test.go rename to internal/controlplane/api/creator_pure_unit_test.go index 037b968..96f1f3e 100644 --- a/cmd/control-plane/creator_pure_unit_test.go +++ b/internal/controlplane/api/creator_pure_unit_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -12,7 +12,7 @@ import ( "github.com/gofiber/fiber/v3" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestDecodeCreatorRejectsUnknownAndTrailingJSON(t *testing.T) { diff --git a/cmd/control-plane/creator_route_validation_test.go b/internal/controlplane/api/creator_route_validation_test.go similarity index 99% rename from cmd/control-plane/creator_route_validation_test.go rename to internal/controlplane/api/creator_route_validation_test.go index ad0d3d4..0010c95 100644 --- a/cmd/control-plane/creator_route_validation_test.go +++ b/internal/controlplane/api/creator_route_validation_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "net/http" diff --git a/cmd/control-plane/creator_share_test.go b/internal/controlplane/api/creator_share_test.go similarity index 99% rename from cmd/control-plane/creator_share_test.go rename to internal/controlplane/api/creator_share_test.go index b71658c..16e6d6b 100644 --- a/cmd/control-plane/creator_share_test.go +++ b/internal/controlplane/api/creator_share_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "testing" diff --git a/cmd/control-plane/douyin.go b/internal/controlplane/api/douyin.go similarity index 97% rename from cmd/control-plane/douyin.go rename to internal/controlplane/api/douyin.go index cd826c8..1d6f9e5 100644 --- a/cmd/control-plane/douyin.go +++ b/internal/controlplane/api/douyin.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -10,8 +10,8 @@ import ( "net/url" "time" - "git.ipao.vip/rogee/creator-hub/internal/douyin" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + douyin "git.ipao.vip/rogee/creator-hub/internal/platform/douyin" ) type douyinGatewayBrowser struct { diff --git a/cmd/control-plane/douyin_test.go b/internal/controlplane/api/douyin_test.go similarity index 97% rename from cmd/control-plane/douyin_test.go rename to internal/controlplane/api/douyin_test.go index 91bef60..57afa09 100644 --- a/cmd/control-plane/douyin_test.go +++ b/internal/controlplane/api/douyin_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -8,8 +8,8 @@ import ( "strings" "testing" - "git.ipao.vip/rogee/creator-hub/internal/douyin" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + douyin "git.ipao.vip/rogee/creator-hub/internal/platform/douyin" ) const testDouyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp" diff --git a/cmd/control-plane/hub.go b/internal/controlplane/api/environments.go similarity index 95% rename from cmd/control-plane/hub.go rename to internal/controlplane/api/environments.go index 89dce28..a7e615e 100644 --- a/cmd/control-plane/hub.go +++ b/internal/controlplane/api/environments.go @@ -1,4 +1,4 @@ -package main +package api import ( "bytes" @@ -12,13 +12,14 @@ import ( "strings" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/gatewayclient" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/gofiber/fiber/v3" "github.com/sirupsen/logrus" ) -// hubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。 -type hubStore interface { +// HubStore 是控制面编排所需的存储能力；生产实现为 *hub.Store，测试使用内存桩。 +type HubStore interface { LockResources(ctx context.Context, aliases, exitIDs, browserVersions []string) (func(), error) CreateGateway(ctx context.Context, name, endpoint, token string) (hub.Gateway, error) UpdateGateway(ctx context.Context, currentName, name, endpoint, token string) (hub.Gateway, error) @@ -57,7 +58,7 @@ type networkExitAdminStore interface { DeleteNetworkExit(context.Context, string) error } -type runtimeStopStore interface { +type RuntimeStopStore interface { LockResources(ctx context.Context, aliases, exitIDs, browserVersions []string) (func(), error) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) GetGateway(ctx context.Context, name string) (hub.Gateway, error) @@ -93,35 +94,11 @@ const ( // gatewayCall 调用某个网关的 /v1 路由；ok 为 false 时 status/body 携带网关错误。 func gatewayCall(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration) (status int, responseBody []byte, err error) { - return gatewayCallWithLimit(ctx, target, method, path, body, timeout, defaultGatewayResponseLimit) + return gatewayclient.Call(ctx, target, method, path, body, timeout) } func gatewayCallWithLimit(ctx context.Context, target hub.Gateway, method, path string, body any, timeout time.Duration, responseLimit int) (status int, responseBody []byte, err error) { - callCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - var payload io.Reader - if body != nil { - encoded, marshalErr := json.Marshal(body) - if marshalErr != nil { - return 0, nil, marshalErr - } - payload = bytes.NewReader(encoded) - } - request, requestErr := http.NewRequestWithContext(callCtx, method, target.Endpoint+path, payload) - if requestErr != nil { - return 0, nil, requestErr - } - request.Header.Set("Authorization", "Bearer "+target.Token) - if body != nil { - request.Header.Set("Content-Type", "application/json") - } - response, requestErr := http.DefaultClient.Do(request) - if requestErr != nil { - return 0, nil, requestErr - } - defer response.Body.Close() - responseBody, err = io.ReadAll(io.LimitReader(response.Body, int64(responseLimit))) - return response.StatusCode, responseBody, err + return gatewayclient.CallWithLimit(ctx, target, method, path, body, timeout, responseLimit) } func gatewayCreatePayload(environment hub.EnvironmentContext, _ string, networkExit gatewayNetworkExit) map[string]any { @@ -426,7 +403,7 @@ func accountRunnable(environment hub.EnvironmentContext) bool { return environment.AccountStatus == "active" && environment.AuthorizationStatus == "authorized" } -func releaseRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error { +func releaseRuntime(ctx context.Context, store RuntimeStopStore, environment hub.EnvironmentContext) error { err := store.ReleaseRuntime(ctx, environment) if errors.Is(err, hub.ErrConflict) { return nil @@ -434,7 +411,7 @@ func releaseRuntime(ctx context.Context, store runtimeStopStore, environment hub return err } -func releaseRuntimeWithReconcileAudit(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error { +func releaseRuntimeWithReconcileAudit(ctx context.Context, store HubStore, environment hub.EnvironmentContext) error { action := actionForEnvironment("reconcile", environment) if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { return err @@ -450,11 +427,11 @@ func releaseRuntimeWithReconcileAudit(ctx context.Context, store hubStore, envir return releaseErr } -func registerHub(app *fiber.App, store hubStore) { +func registerHub(app *fiber.App, store HubStore) { registerHubWithNetwork(app, store, defaultNetworkExitProbe(), nil) } -func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) { +func registerHubWithNetwork(app *fiber.App, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) { app.Get("/api/browsers", listBrowsers(store)) app.Get("/api/browsers/:alias", getBrowser(store)) app.Post("/api/browsers", lockBrowserCreate(store, createBrowser(store, probe, resolve))) @@ -600,7 +577,7 @@ func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitPro }) } -func getBrowser(store hubStore) fiber.Handler { +func getBrowser(store HubStore) fiber.Handler { return func(c fiber.Ctx) error { environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) if err != nil { @@ -641,7 +618,7 @@ func getBrowser(store hubStore) fiber.Handler { } } -func lockBrowserAlias(store hubStore, handler fiber.Handler) fiber.Handler { +func lockBrowserAlias(store HubStore, handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { requestedExitID, browserVersion := "", "" if c.Params("action") == "rebind" { @@ -674,7 +651,7 @@ func lockBrowserAlias(store hubStore, handler fiber.Handler) fiber.Handler { } } -func lockBrowserVersion(store hubStore, handler fiber.Handler) fiber.Handler { +func lockBrowserVersion(store HubStore, handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { unlock, err := store.LockResources(c.Context(), nil, nil, []string{c.Params("version")}) if err != nil { @@ -685,7 +662,7 @@ func lockBrowserVersion(store hubStore, handler fiber.Handler) fiber.Handler { } } -func lockBrowserCreate(store hubStore, handler fiber.Handler) fiber.Handler { +func lockBrowserCreate(store HubStore, handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { var input struct { Alias string `json:"alias"` @@ -712,7 +689,7 @@ func lockBrowserCreate(store hubStore, handler fiber.Handler) fiber.Handler { } } -func lockEnvironmentResources(ctx context.Context, store hubStore, envs []hub.Env) (func(), error) { +func lockEnvironmentResources(ctx context.Context, store HubStore, envs []hub.Env) (func(), error) { aliases := make([]string, 0, len(envs)) seen := make(map[string]bool, len(envs)) for _, env := range envs { @@ -776,7 +753,7 @@ func lockAliasResources(ctx context.Context, store resourceLockStore, alias, req } } -func lockAccountResources(ctx context.Context, store runtimeStopStore, accountID string) (func(), error) { +func lockAccountResources(ctx context.Context, store RuntimeStopStore, accountID string) (func(), error) { if store == nil { return func() {}, nil } @@ -801,7 +778,7 @@ func lockAccountResources(ctx context.Context, store runtimeStopStore, accountID } } -func accountEnvironmentResources(ctx context.Context, store runtimeStopStore, accountID string) (environmentResource, bool, error) { +func accountEnvironmentResources(ctx context.Context, store RuntimeStopStore, accountID string) (environmentResource, bool, error) { environment, err := store.GetEnvironmentContextForAccount(ctx, accountID) if errors.Is(err, hub.ErrNotFound) { return environmentResource{}, false, nil @@ -823,7 +800,7 @@ func environmentResources(ctx context.Context, store resourceLockStore, alias st return environmentResource{alias: environment.Alias, exitID: environment.Exit.ID, browserVersion: environment.BrowserVersion}, true, nil } -func environmentResourceMap(ctx context.Context, store hubStore, aliases []string) (map[string]environmentResource, error) { +func environmentResourceMap(ctx context.Context, store HubStore, aliases []string) (map[string]environmentResource, error) { result := make(map[string]environmentResource, len(aliases)) for _, alias := range aliases { resources, found, err := environmentResources(ctx, store, alias) @@ -871,7 +848,7 @@ func equalEnvironmentResourceMaps(left, right map[string]environmentResource) bo return true } -func listNetworkExits(store hubStore) fiber.Handler { +func listNetworkExits(store HubStore) fiber.Handler { return func(c fiber.Ctx) error { exits, err := store.ListNetworkExits(c.Context()) if err != nil { @@ -881,7 +858,7 @@ func listNetworkExits(store hubStore) fiber.Handler { } } -func getNetworkExit(store hubStore) fiber.Handler { +func getNetworkExit(store HubStore) fiber.Handler { return func(c fiber.Ctx) error { exit, err := store.GetNetworkExit(c.Context(), c.Params("id")) if err != nil { @@ -891,7 +868,7 @@ func getNetworkExit(store hubStore) fiber.Handler { } } -func createNetworkExit(store hubStore) fiber.Handler { +func createNetworkExit(store HubStore) fiber.Handler { return func(c fiber.Ctx) error { var input struct { Protocol string `json:"protocol"` @@ -917,7 +894,7 @@ func createNetworkExit(store hubStore) fiber.Handler { } } -func checkNetworkExit(store hubStore, probe networkExitProbe) fiber.Handler { +func checkNetworkExit(store HubStore, probe NetworkExitProbe) fiber.Handler { return func(c fiber.Ctx) error { unlock, err := store.LockResources(c.Context(), nil, []string{c.Params("id")}, nil) if err != nil { @@ -943,7 +920,7 @@ func checkNetworkExit(store hubStore, probe networkExitProbe) fiber.Handler { } } -func disableNetworkExit(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { +func disableNetworkExit(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { unlock, err := store.LockResources(c.Context(), nil, []string{c.Params("id")}, nil) if err != nil { @@ -961,7 +938,7 @@ func disableNetworkExit(store hubStore, probe networkExitProbe, resolve func(hub } } -func verifyNetworkExit(ctx context.Context, store hubStore, probe networkExitProbe, id string) (hub.NetworkExitAccess, string, error) { +func verifyNetworkExit(ctx context.Context, store HubStore, probe NetworkExitProbe, id string) (hub.NetworkExitAccess, string, error) { access, err := store.GetNetworkExitAccess(ctx, id) if err != nil { return hub.NetworkExitAccess{}, "exit_unavailable", err @@ -981,7 +958,7 @@ func verifyNetworkExit(ctx context.Context, store hubStore, probe networkExitPro return access, reason, err } -func listBrowsers(store hubStore) fiber.Handler { +func listBrowsers(store HubStore) fiber.Handler { return func(c fiber.Ctx) error { envs, err := store.ListEnvs(c.Context()) if err != nil { @@ -1037,11 +1014,11 @@ func listBrowsers(store hubStore) fiber.Handler { } } -func reconcileRuntimeLeases(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error { +func reconcileRuntimeLeases(ctx context.Context, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error { return reconcileRuntimeLeasesUnlocked(ctx, store, probe, resolve) } -func reconcileRuntimeLeasesUnlocked(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error { +func reconcileRuntimeLeasesUnlocked(ctx context.Context, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error { envs, err := store.ListEnvs(ctx) if err != nil { return err @@ -1055,7 +1032,7 @@ func reconcileRuntimeLeasesUnlocked(ctx context.Context, store hubStore, probe n return reconcileRuntimeSnapshot(ctx, store, probe, resolve, envs, runtimes, gatewayRead, gatewayErrors) } -func gatewayRuntimeSnapshot(ctx context.Context, store hubStore, envs []hub.Env) (map[string]map[string]runtimeStatus, map[string]bool, map[string]error) { +func gatewayRuntimeSnapshot(ctx context.Context, store HubStore, envs []hub.Env) (map[string]map[string]runtimeStatus, map[string]bool, map[string]error) { gateways := map[string]bool{} runtimes := map[string]map[string]runtimeStatus{} gatewayRead := map[string]bool{} @@ -1094,7 +1071,7 @@ func gatewayRuntimeSnapshot(ctx context.Context, store hubStore, envs []hub.Env) return runtimes, gatewayRead, gatewayErrors } -func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), envs []hub.Env, +func reconcileRuntimeSnapshot(ctx context.Context, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), envs []hub.Env, runtimes map[string]map[string]runtimeStatus, gatewayRead map[string]bool, gatewayErrors map[string]error) error { var firstErr error for _, env := range envs { @@ -1177,7 +1154,7 @@ func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe network return firstErr } -func runtimeRecoveryFailure(ctx context.Context, store hubStore, alias string, err error) (string, string) { +func runtimeRecoveryFailure(ctx context.Context, store HubStore, alias string, err error) (string, string) { var gatewayErr gatewayFailure if errors.As(err, &gatewayErr) { return "unknown", "gateway_result_unknown" @@ -1188,7 +1165,7 @@ func runtimeRecoveryFailure(ctx context.Context, store hubStore, alias string, e return "failed", "runtime_persistence_failed" } -func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkExitProbe, +func restoreOrRebuildRuntime(ctx context.Context, store HubStore, probe NetworkExitProbe, _ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, runtime runtimeStatus) (bool, error) { if environment.RuntimeCleanupPending { target, err := store.GetGateway(ctx, environment.Gateway) @@ -1266,7 +1243,7 @@ type runtimeNodeStore interface { SetRuntimeNode(context.Context, string, string, string) error } -func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string, nodeIDs ...string) (hub.EnvironmentContext, error) { +func activateGatewayRuntime(ctx context.Context, store HubStore, target hub.Gateway, environment hub.EnvironmentContext, runtimeID, networkID string, nodeIDs ...string) (hub.EnvironmentContext, error) { if !validCreatedRuntime(runtimeStatus{ID: runtimeID, NetworkID: networkID}, environment, true) { return hub.EnvironmentContext{}, hub.ErrConflict } @@ -1302,7 +1279,7 @@ func activateGatewayRuntime(ctx context.Context, store hubStore, target hub.Gate return hub.EnvironmentContext{}, errors.Join(err, cleanupErr) } -func discardRuntime(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error { +func discardRuntime(ctx context.Context, store HubStore, environment hub.EnvironmentContext) error { target, targetErr := store.GetGateway(ctx, environment.Gateway) if targetErr != nil { return targetErr @@ -1311,7 +1288,7 @@ func discardRuntime(ctx context.Context, store hubStore, environment hub.Environ return err } -func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { +func createBrowser(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { input := struct { Alias string `json:"alias"` @@ -1465,7 +1442,7 @@ func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw } } -func browserAction(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { +func browserAction(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { switch c.Params("action") { case "start", "stop": @@ -1480,7 +1457,7 @@ func browserAction(store hubStore, probe networkExitProbe, resolve func(hub.Netw } } -func lifecycleAction(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { +func lifecycleAction(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) if err != nil { return hubError(c, err) @@ -1507,7 +1484,7 @@ func lifecycleAction(store hubStore, probe networkExitProbe, resolve func(hub.Ne return c.SendStatus(fiber.StatusNoContent) } -func stopAccountRuntime(ctx context.Context, store runtimeStopStore, accountID string) error { +func stopAccountRuntime(ctx context.Context, store RuntimeStopStore, accountID string) error { environment, err := store.GetEnvironmentContextForAccount(ctx, accountID) if errors.Is(err, hub.ErrNotFound) { return nil @@ -1518,7 +1495,7 @@ func stopAccountRuntime(ctx context.Context, store runtimeStopStore, accountID s return stopEnvironmentRuntime(ctx, store, environment) } -func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environment hub.EnvironmentContext) error { +func stopEnvironmentRuntime(ctx context.Context, store RuntimeStopStore, environment hub.EnvironmentContext) error { action := actionForEnvironment("stop", environment) if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { return err @@ -1595,7 +1572,7 @@ func stopEnvironmentRuntime(ctx context.Context, store runtimeStopStore, environ return finish("succeeded", "environment_stopped") } -func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx, +func startBrowser(store HubStore, probe NetworkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx, environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error { if err := startBrowserRuntime(c.Context(), store, probe, environment, finish); err != nil { return hubError(c, err) @@ -1603,7 +1580,7 @@ func startBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExit return c.SendStatus(fiber.StatusNoContent) } -func startBrowserRuntime(ctx context.Context, store hubStore, probe networkExitProbe, +func startBrowserRuntime(ctx context.Context, store HubStore, probe NetworkExitProbe, environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error { if !accountRunnable(environment) { return hub.ErrConflict @@ -1711,7 +1688,7 @@ func startBrowserRuntime(ctx context.Context, store hubStore, probe networkExitP return nil } -func upgradeBrowser(store hubStore, probe networkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { +func upgradeBrowser(store HubStore, probe NetworkExitProbe, _ func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { var input struct { Version string `json:"version"` } @@ -1830,7 +1807,7 @@ type runtimeCreateSpec struct { networkExit gatewayNetworkExit } -func prepareRuntimeCreate(ctx context.Context, store hubStore, _ func(hub.NetworkExitAccess) (string, error), +func prepareRuntimeCreate(ctx context.Context, store HubStore, _ func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, access hub.NetworkExitAccess) (runtimeCreateSpec, error) { browserPath, err := store.BrowserPath(ctx, environment.BrowserVersion) if err != nil { @@ -1987,7 +1964,7 @@ func removeGatewayRuntime(ctx context.Context, store runtimeCleanupStore, target return false, gatewayFailure{status: http.StatusBadGateway, message: "gateway runtime cleanup is pending"} } -func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error), +func restoreRebindRuntime(ctx context.Context, store HubStore, resolve func(hub.NetworkExitAccess) (string, error), target hub.Gateway, environment hub.EnvironmentContext, previous runtimeStatus, found bool, prepared *runtimeCreateSpec) (bool, error) { if !found { return true, releaseRuntime(ctx, store, environment) @@ -2070,7 +2047,7 @@ func rebindRecoveryError(ready bool, err error) error { return nil } -func runtimeCreateSpecMatches(ctx context.Context, store hubStore, current, previous hub.EnvironmentContext, prepared *runtimeCreateSpec) bool { +func runtimeCreateSpecMatches(ctx context.Context, store HubStore, current, previous hub.EnvironmentContext, prepared *runtimeCreateSpec) bool { if prepared == nil || current.BindingVersion != previous.BindingVersion || current.BrowserVersion != previous.BrowserVersion || current.Exit.ID != previous.Exit.ID { return false } @@ -2078,7 +2055,7 @@ func runtimeCreateSpecMatches(ctx context.Context, store hubStore, current, prev return err == nil && browserPath == prepared.browserPath } -func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { +func rebindBrowser(store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { var input struct { NetworkExitID string `json:"network_exit_id"` } @@ -2245,7 +2222,7 @@ func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.Netw return c.JSON(after) } -func deleteBrowser(store hubStore) fiber.Handler { +func deleteBrowser(store HubStore) fiber.Handler { return func(c fiber.Ctx) error { environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) if err != nil { @@ -2316,3 +2293,21 @@ func hubError(c fiber.Ctx, err error) error { } return c.Status(status).JSON(map[string]string{"error": message}) } + +// RegisterEnvironments exposes environment routes to the grouped API package. +func RegisterEnvironments(app *fiber.App, store *hub.Store) { + registerHub(app, store) +} + +// ReconcileRuntimeLeases performs one native runtime lease reconciliation pass. +func ReconcileRuntimeLeases(ctx context.Context, store *hub.Store) error { + return reconcileRuntimeLeases(ctx, store, defaultNetworkExitProbe(), nil) +} + +// RegisterHubWithNetwork exposes environment route registration with explicit network dependencies for tests and composition. +func RegisterHubWithNetwork(app *fiber.App, store HubStore, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) { + registerHubWithNetwork(app, store, probe, resolve) +} + +// hubStore remains the package-local name used by lifecycle test doubles; HubStore is its composition boundary. +type hubStore = HubStore diff --git a/internal/controlplane/api/environments/register.go b/internal/controlplane/api/environments/register.go new file mode 100644 index 0000000..c76ce22 --- /dev/null +++ b/internal/controlplane/api/environments/register.go @@ -0,0 +1,11 @@ +package environments + +import ( + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api" + "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/gofiber/fiber/v3" +) + +func Register(app *fiber.App, store *environment.Store) { + api.RegisterEnvironments(app, store) +} diff --git a/cmd/control-plane/gateway_browser_unit_test.go b/internal/controlplane/api/gateway_browser_unit_test.go similarity index 98% rename from cmd/control-plane/gateway_browser_unit_test.go rename to internal/controlplane/api/gateway_browser_unit_test.go index ab57793..24d9c56 100644 --- a/cmd/control-plane/gateway_browser_unit_test.go +++ b/internal/controlplane/api/gateway_browser_unit_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -8,8 +8,8 @@ import ( "path/filepath" "testing" - "git.ipao.vip/rogee/creator-hub/internal/douyin" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + douyin "git.ipao.vip/rogee/creator-hub/internal/platform/douyin" ) func testRunnableEnvironment() hub.EnvironmentContext { diff --git a/cmd/control-plane/hub_native_unit_test.go b/internal/controlplane/api/hub_native_unit_test.go similarity index 98% rename from cmd/control-plane/hub_native_unit_test.go rename to internal/controlplane/api/hub_native_unit_test.go index 21b304c..34f9503 100644 --- a/cmd/control-plane/hub_native_unit_test.go +++ b/internal/controlplane/api/hub_native_unit_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -8,7 +8,7 @@ import ( "testing" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestCreateStoppedGatewayRuntimeValidatesGatewayResponses(t *testing.T) { diff --git a/cmd/control-plane/hub_test.go b/internal/controlplane/api/hub_test.go similarity index 99% rename from cmd/control-plane/hub_test.go rename to internal/controlplane/api/hub_test.go index 96b43bc..e71bf58 100644 --- a/cmd/control-plane/hub_test.go +++ b/internal/controlplane/api/hub_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -17,8 +17,8 @@ import ( "testing" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" ) @@ -69,15 +69,15 @@ func newMemoryStore() *memoryStore { } func TestResumeBlockReasonIsStable(t *testing.T) { - account := phasea.Account{AuthorizationStatus: "authorized"} + account := accountdomain.Account{AuthorizationStatus: "authorized"} healthy := hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "healthy"}} for name, test := range map[string]struct { - account phasea.Account + account accountdomain.Account env hub.EnvironmentContext found bool want string }{ - "revoked": {phasea.Account{AuthorizationStatus: "revoked"}, healthy, true, "account_revoked"}, + "revoked": {accountdomain.Account{AuthorizationStatus: "revoked"}, healthy, true, "account_revoked"}, "missing binding": {account, hub.EnvironmentContext{}, false, "binding_missing"}, "direct exit": {account, hub.EnvironmentContext{}, true, "account_conflict"}, "unhealthy exit": {account, hub.EnvironmentContext{Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "unhealthy"}}, true, "network_exit_unhealthy"}, @@ -104,7 +104,7 @@ func TestPhaseAReadinessErrorsAreStructured(t *testing.T) { t.Run(test.name, func(t *testing.T) { app := fiber.New() app.Get("/", func(c fiber.Ctx) error { - return phaseAError(c, &phasea.ReadinessError{Reason: test.reason, Unavailable: test.unavailable}) + return phaseAError(c, &accountdomain.ReadinessError{Reason: test.reason, Unavailable: test.unavailable}) }) response := do(app, http.MethodGet, "/", "") if response.Code != test.status || !strings.Contains(response.Body.String(), `"reason_code":"`+test.reason+`"`) { @@ -555,7 +555,7 @@ func TestPauseSerializesResumeAndRetainsClaimGateOnUnknownStop(t *testing.T) { } ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) - accountStore, err := phasea.Open(ctx, fixture.databaseURL) + accountStore, err := accountdomain.Open(ctx, fixture.databaseURL) if err != nil { t.Fatal(err) } @@ -862,7 +862,7 @@ func newTestApp(t *testing.T, store *memoryStore, gateway *fakeGateway) *fiber.A return newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) } -func newTestAppWithNetwork(t *testing.T, store *memoryStore, gateway *fakeGateway, probe networkExitProbe, +func newTestAppWithNetwork(t *testing.T, store *memoryStore, gateway *fakeGateway, probe NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) *fiber.App { t.Helper() server := httptest.NewServer(gateway.handler(t)) @@ -1694,7 +1694,7 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { } ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - accountStore, err := phasea.Open(ctx, databaseURL) + accountStore, err := accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -1706,14 +1706,14 @@ func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = store.Close() }) - accountStore, err = phasea.Open(ctx, databaseURL) + accountStore, err = accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } - if err := accountStore.CreateAccount(ctx, phasea.Account{ + if err := accountStore.CreateAccount(ctx, accountdomain.Account{ ID: "account-a", Name: "账号 A", Platform: "douyin", PlatformAccountKey: "account-a", Tags: []string{}, Cookies: "sessionid=value", - CredentialReference: phasea.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"}, + CredentialReference: accountdomain.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-a/cookies", }, &testCredentialBridge{values: map[string]string{}}); err != nil { t.Fatal(err) @@ -1879,7 +1879,7 @@ func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFi t.Helper() ctx := context.Background() databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) - accountStore, err := phasea.Open(ctx, databaseURL) + accountStore, err := accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -1891,14 +1891,14 @@ func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFi t.Fatal(err) } t.Cleanup(func() { _ = store.Close() }) - accountStore, err = phasea.Open(ctx, databaseURL) + accountStore, err = accountdomain.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } - if err := accountStore.CreateAccount(ctx, phasea.Account{ + if err := accountStore.CreateAccount(ctx, accountdomain.Account{ ID: "account-a", Name: "账号 A", Platform: "douyin", PlatformAccountKey: "account-a", Tags: []string{}, Cookies: "sessionid=value", - CredentialReference: phasea.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"}, + CredentialReference: accountdomain.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"}, CredentialKey: "creatorhub/account-a/cookies", }, &testCredentialBridge{values: map[string]string{}}); err != nil { t.Fatal(err) @@ -1955,7 +1955,7 @@ func TestPauseClosesClaimGateBeforeStoppingRuntime(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) - accountStore, err := phasea.Open(ctx, fixture.databaseURL) + accountStore, err := accountdomain.Open(ctx, fixture.databaseURL) if err != nil { t.Fatal(err) } @@ -1980,16 +1980,16 @@ func TestPauseClosesClaimGateBeforeStoppingRuntime(t *testing.T) { t.Fatal(err) } } - if err := accountStore.CreateDraft(ctx, phasea.Draft{ID: "pause-draft", AccountID: "account-a", Version: 1, Content: "pause"}); err != nil { + if err := accountStore.CreateDraft(ctx, accountdomain.Draft{ID: "pause-draft", AccountID: "account-a", Version: 1, Content: "pause"}); err != nil { t.Fatal(err) } - if err := accountStore.Confirm(ctx, phasea.Confirmation{ + if err := accountStore.Confirm(ctx, accountdomain.Confirmation{ ID: "pause-confirmation", AccountID: "account-a", AccountVersion: account.Version, DraftID: "pause-draft", DraftVersion: 1, Version: 1, }); err != nil { t.Fatal(err) } - if _, inserted, err := accountStore.Enqueue(ctx, phasea.Task{ + if _, inserted, err := accountStore.Enqueue(ctx, accountdomain.Task{ ID: "pause-task", IdempotencyKey: "pause-idempotency", AccountID: "account-a", AccountVersion: account.Version, DraftID: "pause-draft", DraftVersion: 1, ConfirmationID: "pause-confirmation", ConfirmationVersion: 1, }); err != nil || !inserted { @@ -2036,7 +2036,7 @@ func TestPauseAndRevokeStopContainerWithoutRuntimeLease(t *testing.T) { } ctx := context.Background() fixture := newPostgresRebindFixture(t, databaseURL) - accountStore, err := phasea.Open(ctx, fixture.databaseURL) + accountStore, err := accountdomain.Open(ctx, fixture.databaseURL) if err != nil { t.Fatal(err) } @@ -2999,7 +2999,7 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} - var store hubStore = fixture.store + var store HubStore = fixture.store if test.commitUnknown { store = cleanupCommitUnknownStore{hubStore: fixture.store} } else { @@ -3073,7 +3073,7 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: "stale-exit", }} - var store hubStore = fixture.store + var store HubStore = fixture.store if test.commitUnknown { store = cleanupCommitUnknownStore{hubStore: fixture.store} } else { @@ -3148,7 +3148,7 @@ func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { ID: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, NetworkID: "native-66666666666666666666666666666666", }} - var store hubStore = fixture.store + var store HubStore = fixture.store if test.commitUnknown { store = cleanupCommitUnknownStore{hubStore: fixture.store} } else { @@ -3659,7 +3659,7 @@ func TestPostgresNonRunnableReconcileAuditsRuntimeRelease(t *testing.T) { }} } - var store hubStore = fixture.store + var store HubStore = fixture.store if releaseFailure { store = failingRuntimeReleaseStore{Store: fixture.store, err: errors.New("release unavailable")} } @@ -3715,7 +3715,7 @@ func TestPostgresImageDisableWaitsForEveryImageConsumer(t *testing.T) { lifecycleStarted := make(chan struct{}) fixture.gateway.createStarted = lifecycleStarted fixture.gateway.releaseCreate = releaseLifecycle - probe := networkExitProbe(fakeExitProbe{}) + probe := NetworkExitProbe(fakeExitProbe{}) released := false method, path, body, expected := http.MethodPost, "/api/browsers/account-a/"+action, "", http.StatusNoContent diff --git a/cmd/control-plane/network.go b/internal/controlplane/api/network.go similarity index 96% rename from cmd/control-plane/network.go rename to internal/controlplane/api/network.go index 4d9ca00..c847d86 100644 --- a/cmd/control-plane/network.go +++ b/internal/controlplane/api/network.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -13,12 +13,12 @@ import ( "strings" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) const networkExitObservationURL = "https://ipinfo.io/json" -type networkExitProbe interface { +type NetworkExitProbe interface { Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) } @@ -27,7 +27,7 @@ type httpNetworkExitProbe struct { client *http.Client } -func defaultNetworkExitProbe() networkExitProbe { +func defaultNetworkExitProbe() NetworkExitProbe { return httpNetworkExitProbe{endpoint: networkExitObservationURL, client: &http.Client{Timeout: 20 * time.Second}} } diff --git a/cmd/control-plane/network_test.go b/internal/controlplane/api/network_test.go similarity index 99% rename from cmd/control-plane/network_test.go rename to internal/controlplane/api/network_test.go index f8078bc..fef2d1e 100644 --- a/cmd/control-plane/network_test.go +++ b/internal/controlplane/api/network_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "bufio" @@ -15,7 +15,7 @@ import ( "testing" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) func TestHTTPNetworkExitProbeUsesStoredBasicAuth(t *testing.T) { diff --git a/cmd/control-plane/runtime_use.go b/internal/controlplane/api/runtime.go similarity index 97% rename from cmd/control-plane/runtime_use.go rename to internal/controlplane/api/runtime.go index b417635..1174229 100644 --- a/cmd/control-plane/runtime_use.go +++ b/internal/controlplane/api/runtime.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -7,7 +7,7 @@ import ( "sync" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" "github.com/sirupsen/logrus" ) diff --git a/cmd/control-plane/runtime_use_test.go b/internal/controlplane/api/runtime_use_test.go similarity index 98% rename from cmd/control-plane/runtime_use_test.go rename to internal/controlplane/api/runtime_use_test.go index d1f5e31..7ecb8fc 100644 --- a/cmd/control-plane/runtime_use_test.go +++ b/internal/controlplane/api/runtime_use_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -6,7 +6,7 @@ import ( "testing" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" ) type runtimeUseTestStore struct { diff --git a/internal/controlplane/api/system/register.go b/internal/controlplane/api/system/register.go new file mode 100644 index 0000000..f98df2b --- /dev/null +++ b/internal/controlplane/api/system/register.go @@ -0,0 +1,3 @@ +package system + +// System routes are composed by internal/controlplane/app. diff --git a/internal/controlplane/api/system/system.go b/internal/controlplane/api/system/system.go new file mode 100644 index 0000000..88f1fc8 --- /dev/null +++ b/internal/controlplane/api/system/system.go @@ -0,0 +1,88 @@ +package system + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + account "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/creator" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" +) + +func RegisterHealth(app *fiber.App, accountStore *account.Store, environmentStore *hub.Store, creatorStore *creator.Store) { + app.Get("/healthz", func(c fiber.Ctx) error { + c.Status(fiber.StatusNoContent) + return nil + }) + app.Get("/readyz", func(c fiber.Ctx) error { + if accountStore == nil || environmentStore == nil || creatorStore == nil { + return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{"error": "service is not ready"}) + } + ctx, cancel := context.WithTimeout(c.Context(), 2*time.Second) + defer cancel() + checks := []struct { + name string + fn func(context.Context) error + }{ + {"phase_a", accountStore.Ping}, + {"hub", environmentStore.Ping}, + {"creator", creatorStore.Ping}, + {"creator_schema", creatorStore.EnsureSchema}, + } + for _, check := range checks { + if err := check.fn(ctx); err != nil { + logrus.WithError(err).WithField("check", check.name).Warn("control plane readiness check failed") + return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{"error": "service is not ready"}) + } + } + c.Status(fiber.StatusNoContent) + return nil + }) +} + +func Authenticate(username, password string) fiber.Handler { + wantUser, wantPassword := sha256.Sum256([]byte(username)), sha256.Sum256([]byte(password)) + return func(c fiber.Ctx) error { + encoded, ok := strings.CutPrefix(c.Get(fiber.HeaderAuthorization), "Basic ") + decoded, err := base64.StdEncoding.DecodeString(encoded) + user, suppliedPassword, found := strings.Cut(string(decoded), ":") + gotUser, gotPassword := sha256.Sum256([]byte(user)), sha256.Sum256([]byte(suppliedPassword)) + if !ok || err != nil || !found || subtle.ConstantTimeCompare(gotUser[:], wantUser[:]) != 1 || subtle.ConstantTimeCompare(gotPassword[:], wantPassword[:]) != 1 { + c.Set(fiber.HeaderWWWAuthenticate, `Basic realm="CreatorHub", charset="UTF-8"`) + return c.Status(fiber.StatusUnauthorized).JSON(map[string]string{"error": "authentication required"}) + } + return c.Next() + } +} + +func IsAPIPath(path string) bool { + for _, prefix := range []string{"/api", "/phase-a", "/gateways", "/browser-versions", "/browsers", "/network-exits"} { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} + +func SPAHandler(directory string) fiber.Handler { + return func(c fiber.Ctx) error { + path, err := url.PathUnescape(strings.TrimPrefix(c.Path(), "/")) + path = filepath.Clean(path) + if err == nil && filepath.IsLocal(path) { + name := filepath.Join(directory, path) + if info, err := os.Stat(name); err == nil && !info.IsDir() { + return c.SendFile(name) + } + } + return c.SendFile(filepath.Join(directory, "index.html")) + } +} diff --git a/internal/controlplane/api/test_helpers_test.go b/internal/controlplane/api/test_helpers_test.go new file mode 100644 index 0000000..efd9e18 --- /dev/null +++ b/internal/controlplane/api/test_helpers_test.go @@ -0,0 +1,27 @@ +package api + +import ( + "context" + + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" +) + +type testCredentialBridge struct { + values map[string]string +} + +func (bridge *testCredentialBridge) Store(ctx context.Context, _ accountdomain.CredentialReference, key, value string) error { + if err := ctx.Err(); err != nil { + return err + } + bridge.values[key] = value + return nil +} + +func (bridge *testCredentialBridge) Delete(ctx context.Context, _ accountdomain.CredentialReference, key string) error { + if err := ctx.Err(); err != nil { + return err + } + delete(bridge.values, key) + return nil +} diff --git a/cmd/control-plane/xiaohongshu.go b/internal/controlplane/api/xiaohongshu.go similarity index 96% rename from cmd/control-plane/xiaohongshu.go rename to internal/controlplane/api/xiaohongshu.go index ccf475b..69c1b9a 100644 --- a/cmd/control-plane/xiaohongshu.go +++ b/internal/controlplane/api/xiaohongshu.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -11,11 +11,11 @@ import ( "strings" "time" + accountdomain "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/douyin" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" - "git.ipao.vip/rogee/creator-hub/internal/xiaohongshu" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + douyin "git.ipao.vip/rogee/creator-hub/internal/platform/douyin" + xiaohongshu "git.ipao.vip/rogee/creator-hub/internal/platform/xiaohongshu" ) type xiaohongshuGatewayBrowser struct { @@ -153,7 +153,7 @@ func decodeXiaohongshuResponse(body []byte) (xiaohongshu.Response, error) { return xiaohongshu.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil } -func newXiaohongshuReadCollector(ctx context.Context, store *creator.Store, phaseAStore *phasea.Store, hubStore *hub.Store, accountID, sourceType, sourceID string) (*xiaohongshu.Collector, error) { +func newXiaohongshuReadCollector(ctx context.Context, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, accountID, sourceType, sourceID string) (*xiaohongshu.Collector, error) { if store == nil || phaseAStore == nil || hubStore == nil || strings.TrimSpace(accountID) == "" { return nil, creator.ErrUnavailable } diff --git a/cmd/control-plane/xiaohongshu_test.go b/internal/controlplane/api/xiaohongshu_test.go similarity index 97% rename from cmd/control-plane/xiaohongshu_test.go rename to internal/controlplane/api/xiaohongshu_test.go index 67ea2af..b28846b 100644 --- a/cmd/control-plane/xiaohongshu_test.go +++ b/internal/controlplane/api/xiaohongshu_test.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -8,8 +8,8 @@ import ( "testing" "git.ipao.vip/rogee/creator-hub/internal/creator" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/xiaohongshu" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + xiaohongshu "git.ipao.vip/rogee/creator-hub/internal/platform/xiaohongshu" ) const testXiaohongshuIdentityURL = "https://edith.xiaohongshu.com/api/sns/web/v2/user/me" diff --git a/internal/controlplane/app/app.go b/internal/controlplane/app/app.go new file mode 100644 index 0000000..8680453 --- /dev/null +++ b/internal/controlplane/app/app.go @@ -0,0 +1,356 @@ +package app + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "errors" + "io" + "net" + "net/url" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + account "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api" + accountsapi "git.ipao.vip/rogee/creator-hub/internal/controlplane/api/accounts" + creatorapi "git.ipao.vip/rogee/creator-hub/internal/controlplane/api/creator" + environmentsapi "git.ipao.vip/rogee/creator-hub/internal/controlplane/api/environments" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api/system" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/workers" + "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/credentials" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + "git.ipao.vip/rogee/creator-hub/internal/taskstate" + "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type config struct { + listenAddr, webDir, databaseURL, credentialStoreDir string + username, password string + aiAPIKey, aiBaseURL string + credentialMasterKey []byte + logLevel logrus.Level +} + +func Execute() error { + logrus.SetFormatter(&logrus.JSONFormatter{}) + return execute(newCommand()) +} + +func execute(command *cobra.Command) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + return command.ExecuteContext(ctx) +} + +func newCommand() *cobra.Command { + command := &cobra.Command{ + Use: "control-plane", + Short: "Run the CreatorHub control plane", + Args: cobra.NoArgs, + SilenceErrors: true, + SilenceUsage: true, + RunE: func(command *cobra.Command, _ []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + credentials, err := credentials.NewPersistentCredentialBridge(cfg.credentialStoreDir, cfg.credentialMasterKey) + if err != nil { + return err + } + logrus.SetLevel(cfg.logLevel) + phaseAStore, err := account.Open(command.Context(), cfg.databaseURL) + if err != nil { + return err + } + defer phaseAStore.Close() + hubStore, err := hub.Open(command.Context(), cfg.databaseURL) + if err != nil { + return err + } + defer hubStore.Close() + creatorStore, err := creator.Open(command.Context(), cfg.databaseURL) + if err != nil { + return err + } + defer creatorStore.Close() + creatorStore.SetSecretBridge(creatorSecretBridge{bridge: credentials}) + notify := newAttentionNotifier(os.Stderr) + phaseAStore.SetTaskNotifier(notify) + hubStore.SetTaskNotifier(notify) + logStartup(cfg) + heartbeatContext, stopHeartbeat := context.WithCancel(command.Context()) + heartbeatDone := make(chan struct{}) + go func() { + defer close(heartbeatDone) + workers.RunRuntimeLeaseHeartbeat(heartbeatContext, hubStore) + }() + creatorScheduleContext, stopCreatorScheduler := context.WithCancel(command.Context()) + creatorScheduleDone := make(chan struct{}) + go func() { + defer close(creatorScheduleDone) + workers.RunCreatorScheduler(creatorScheduleContext, creatorStore, phaseAStore, hubStore) + }() + creatorEventContext, stopCreatorEvents := context.WithCancel(command.Context()) + creatorEventDone := make(chan struct{}) + creatorEventExecutor := api.NewCreatorGatewayActionExecutor(creatorStore, phaseAStore, hubStore) + creatorAI := &creator.ConfiguredBailian{Store: creatorStore, APIKey: cfg.aiAPIKey, BaseURL: cfg.aiBaseURL} + go func() { + defer close(creatorEventDone) + workers.RunCreatorEventListeners(creatorEventContext, creatorStore, phaseAStore, hubStore, creatorEventExecutor, creatorAI) + }() + listenErr := newHandlerWithCreatorAndAI(cfg.webDir, cfg.username, cfg.password, phaseAStore, hubStore, credentials, creatorStore, creatorAI, creatorAI).Listen(cfg.listenAddr, fiber.ListenConfig{ + GracefulContext: command.Context(), + DisableStartupMessage: true, + }) + stopCreatorScheduler() + stopCreatorEvents() + stopHeartbeat() + <-creatorScheduleDone + <-creatorEventDone + <-heartbeatDone + return listenErr + }, + } + return command +} + +func newAttentionNotifier(output io.Writer) taskstate.Notifier { + logger := logrus.New() + logger.SetFormatter(&logrus.JSONFormatter{}) + logger.SetOutput(output) + return func(transition taskstate.Transition) { + if transition.State != "policy_hold" && transition.State != "needs_confirmation" { + return + } + logger.WithFields(logrus.Fields{ + "service": "control-plane", "notification_channel": "structured_log", "event_type": transition.State, + "reason_code": transition.ReasonCode, "account_id": transition.AccountID, "task_id": transition.TaskID, + }).Warn("operator attention required") + } +} + +func runtimeLeaseHeartbeat(ctx context.Context, store *hub.Store) { + workers.RunRuntimeLeaseHeartbeat(ctx, store) +} + +func logStartup(cfg config) { + logrus.WithFields(logrus.Fields{ + "service": "control-plane", + "listen_addr": cfg.listenAddr, + }).Info("service starting") +} + +func loadConfig() (config, error) { + v := viper.New() + v.SetDefault("listen_addr", ":8080") + v.SetDefault("web_dir", "web/dist") + v.SetDefault("database_url", "postgres://creatorhub@127.0.0.1:5432/creatorhub?sslmode=disable") + v.SetDefault("credential_store_dir", "/var/lib/creatorhub/credentials") + v.SetDefault("log_level", "info") + _ = v.BindEnv("listen_addr", "LISTEN_ADDR") + _ = v.BindEnv("web_dir", "WEB_DIR") + _ = v.BindEnv("database_url", "DATABASE_URL") + _ = v.BindEnv("credential_store_dir", "CREATORHUB_CREDENTIAL_STORE_DIR") + _ = v.BindEnv("credential_master_key", "CREATORHUB_CREDENTIAL_MASTER_KEY") + _ = v.BindEnv("log_level", "LOG_LEVEL") + _ = v.BindEnv("username", "CONTROL_PLANE_USERNAME") + _ = v.BindEnv("password", "CONTROL_PLANE_PASSWORD") + _ = v.BindEnv("ai_api_key", "BAILIAN_API_KEY") + _ = v.BindEnv("ai_base_url", "BAILIAN_BASE_URL") + + level, err := logrus.ParseLevel(v.GetString("log_level")) + if err != nil { + return config{}, errors.New("LOG_LEVEL must be panic, fatal, error, warn, info, debug, or trace") + } + cfg := config{ + listenAddr: strings.TrimSpace(v.GetString("listen_addr")), + webDir: strings.TrimSpace(v.GetString("web_dir")), + databaseURL: strings.TrimSpace(v.GetString("database_url")), + credentialStoreDir: strings.TrimSpace(v.GetString("credential_store_dir")), + username: strings.TrimSpace(v.GetString("username")), + password: v.GetString("password"), + aiAPIKey: strings.TrimSpace(v.GetString("ai_api_key")), + aiBaseURL: strings.TrimSpace(v.GetString("ai_base_url")), + logLevel: level, + } + if cfg.listenAddr == "" { + return config{}, errors.New("LISTEN_ADDR must not be empty") + } + if err := validateListenAddr(cfg.listenAddr); err != nil { + return config{}, err + } + if cfg.webDir == "" { + return config{}, errors.New("WEB_DIR must not be empty") + } + if cfg.username == "" || strings.Contains(cfg.username, ":") { + return config{}, errors.New("CONTROL_PLANE_USERNAME must be non-empty and contain no colon") + } + if len(cfg.password) < 6 { + return config{}, errors.New("CONTROL_PLANE_PASSWORD must contain at least 6 bytes") + } + if !filepath.IsAbs(cfg.credentialStoreDir) { + return config{}, errors.New("CREATORHUB_CREDENTIAL_STORE_DIR must be absolute") + } + cfg.credentialMasterKey, err = base64.StdEncoding.DecodeString(strings.TrimSpace(v.GetString("credential_master_key"))) + if err != nil || len(cfg.credentialMasterKey) != 32 { + return config{}, errors.New("CREATORHUB_CREDENTIAL_MASTER_KEY must be base64 for exactly 32 bytes") + } + databaseURL, err := url.Parse(cfg.databaseURL) + if err != nil || databaseURL.Host == "" || databaseURL.Path == "" || databaseURL.Path == "/" || + (databaseURL.Scheme != "postgres" && databaseURL.Scheme != "postgresql") { + return config{}, errors.New("DATABASE_URL must be a postgres URL with a host") + } + if cfg.aiBaseURL != "" { + aiURL, parseErr := url.Parse(cfg.aiBaseURL) + if parseErr != nil || aiURL.Host == "" || (aiURL.Scheme != "http" && aiURL.Scheme != "https") || aiURL.User != nil { + return config{}, errors.New("BAILIAN_BASE_URL must be an HTTP(S) URL without credentials") + } + } + return cfg, nil +} + +func validateListenAddr(addr string) error { + _, port, err := net.SplitHostPort(addr) + if err != nil { + return errors.New("LISTEN_ADDR must be a host:port address") + } + number, err := strconv.Atoi(port) + if err != nil || number < 1 || number > 65535 { + return errors.New("LISTEN_ADDR port must be 1..65535") + } + return nil +} + +func newHandler(webDirectory, username, password string) *fiber.App { + return newHandlerWithStores(webDirectory, username, password, nil, nil) +} + +func newHandlerWithStores(webDirectory, username, password string, phaseAStore *account.Store, hubStore *hub.Store) *fiber.App { + return newHandlerWithCredentialBridge(webDirectory, username, password, phaseAStore, hubStore, nil) +} + +type creatorSecretBridge struct { + bridge account.CredentialBridge +} + +func (b creatorSecretBridge) Store(ctx context.Context, reference creator.SecretReference, key, value string) error { + if b.bridge == nil { + return errors.New("credential bridge is unavailable") + } + return b.bridge.Store(ctx, account.CredentialReference{ID: reference.ID, Provider: reference.Provider}, key, value) +} + +func (b creatorSecretBridge) Delete(ctx context.Context, reference creator.SecretReference, key string) error { + if b.bridge == nil { + return errors.New("credential bridge is unavailable") + } + return b.bridge.Delete(ctx, account.CredentialReference{ID: reference.ID, Provider: reference.Provider}, key) +} + +func newHandlerWithCredentialBridge(webDirectory, username, password string, phaseAStore *account.Store, hubStore *hub.Store, credentials account.CredentialBridge) *fiber.App { + return newHandlerWithCreator(webDirectory, username, password, phaseAStore, hubStore, credentials, nil) +} + +func newHandlerWithCreator(webDirectory, username, password string, phaseAStore *account.Store, hubStore *hub.Store, credentials account.CredentialBridge, creatorStore *creator.Store) *fiber.App { + return newHandlerWithCreatorAndAI(webDirectory, username, password, phaseAStore, hubStore, credentials, creatorStore, nil, nil) +} + +func newHandlerWithCreatorAndAI(webDirectory, username, password string, phaseAStore *account.Store, hubStore *hub.Store, credentials account.CredentialBridge, creatorStore *creator.Store, generator creator.TextGenerator, analyzer creator.ThemeAnalyzer) *fiber.App { + app := fiber.New(fiber.Config{ + AppName: "CreatorHub control plane", + BodyLimit: 1 << 20, + // 读超时只约束请求读取；创建/升级环境的处理器可等待 native gateway 就绪（最长 11 分钟）。 + ReadTimeout: 5 * time.Second, + IdleTimeout: 60 * time.Second, + }) + system.RegisterHealth(app, phaseAStore, hubStore, creatorStore) + app.Use(system.Authenticate(username, password)) + if hubStore != nil { + environmentsapi.Register(app, hubStore) + } + if phaseAStore != nil { + accountsapi.Register(app, phaseAStore, hubStore, credentials) + } + if creatorStore != nil { + if generator == nil && analyzer == nil { + creatorapi.Register(app, creatorStore, phaseAStore, hubStore) + } else { + creatorapi.RegisterWithAI(app, creatorStore, phaseAStore, hubStore, generator, analyzer) + } + if phaseAStore != nil && hubStore != nil { + accountsapi.RegisterDeletion(app, phaseAStore, hubStore, creatorStore, credentials) + } + } + app.Use(func(c fiber.Ctx) error { + if system.IsAPIPath(c.Path()) { + return c.Status(fiber.StatusNotFound).JSON(fiber.Map{"error": "not found"}) + } + return c.Next() + }) + app.Get("/*", system.SPAHandler(webDirectory)) + return app +} + +func isControlPlaneAPIPath(path string) bool { + for _, prefix := range []string{"/api", "/phase-a", "/gateways", "/browser-versions", "/browsers", "/network-exits"} { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} + +func authenticate(username, password string) fiber.Handler { + wantUser, wantPassword := sha256.Sum256([]byte(username)), sha256.Sum256([]byte(password)) + return func(c fiber.Ctx) error { + encoded, ok := strings.CutPrefix(c.Get(fiber.HeaderAuthorization), "Basic ") + decoded, err := base64.StdEncoding.DecodeString(encoded) + user, suppliedPassword, found := strings.Cut(string(decoded), ":") + gotUser, gotPassword := sha256.Sum256([]byte(user)), sha256.Sum256([]byte(suppliedPassword)) + userOK := subtle.ConstantTimeCompare(gotUser[:], wantUser[:]) + passwordOK := subtle.ConstantTimeCompare(gotPassword[:], wantPassword[:]) + if !ok || err != nil || !found || userOK != 1 || passwordOK != 1 { + c.Set(fiber.HeaderWWWAuthenticate, `Basic realm="CreatorHub", charset="UTF-8"`) + return c.Status(fiber.StatusUnauthorized).JSON(map[string]string{"error": "authentication required"}) + } + return c.Next() + } +} + +func spaHandler(directory string) fiber.Handler { + return func(c fiber.Ctx) error { + path, err := url.PathUnescape(strings.TrimPrefix(c.Path(), "/")) + path = filepath.Clean(path) + if err == nil && filepath.IsLocal(path) { + name := filepath.Join(directory, path) + if info, err := os.Stat(name); err == nil && !info.IsDir() { + return c.SendFile(name) + } + } + return c.SendFile(filepath.Join(directory, "index.html")) + } +} + +func registerPhaseA(appInstance *fiber.App, store *account.Store, runtimeStore api.RuntimeStopStore, credentials account.CredentialBridge) { + api.RegisterAccounts(appInstance, store, runtimeStore, credentials) +} + +func registerHubWithNetwork(appInstance *fiber.App, store api.HubStore, probe api.NetworkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) { + api.RegisterHubWithNetwork(appInstance, store, probe, resolve) +} + +func phaseAError(c fiber.Ctx, err error) error { + return api.PhaseAError(c, err) +} diff --git a/internal/controlplane/app/app_test.go b/internal/controlplane/app/app_test.go new file mode 100644 index 0000000..2b74363 --- /dev/null +++ b/internal/controlplane/app/app_test.go @@ -0,0 +1,529 @@ +package app + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/account" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + "git.ipao.vip/rogee/creator-hub/internal/taskstate" + "github.com/gofiber/fiber/v3" + "github.com/gofiber/fiber/v3/middleware/adaptor" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +func TestExecuteContextStopsOnSIGTERM(t *testing.T) { + if os.Getenv("CREATORHUB_SIGTERM_HELPER") == "1" { + command := &cobra.Command{Use: "shutdown-test", RunE: func(command *cobra.Command, _ []string) error { + _, _ = os.Stdout.WriteString("ready\n") + <-command.Context().Done() + _, _ = os.Stdout.WriteString("stopped\n") + return nil + }} + if err := execute(command); err != nil { + t.Fatal(err) + } + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + process := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestExecuteContextStopsOnSIGTERM$") + process.Env = append(os.Environ(), "CREATORHUB_SIGTERM_HELPER=1") + stdout, err := process.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := process.Start(); err != nil { + t.Fatal(err) + } + + scanner := bufio.NewScanner(stdout) + if !scanner.Scan() || scanner.Text() != "ready" { + t.Fatalf("helper did not become ready: %q err=%v", scanner.Text(), scanner.Err()) + } + if err := process.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + if !scanner.Scan() || scanner.Text() != "stopped" { + t.Fatalf("helper did not finish cleanup: %q err=%v", scanner.Text(), scanner.Err()) + } + if err := process.Wait(); err != nil || ctx.Err() != nil { + t.Fatalf("process did not exit gracefully after SIGTERM: wait=%v context=%v", err, ctx.Err()) + } +} + +func TestLoadConfigRejectsInvalidDatabase(t *testing.T) { + t.Setenv("CONTROL_PLANE_USERNAME", "operator") + t.Setenv("CONTROL_PLANE_PASSWORD", "unit-test-password") + t.Setenv("DATABASE_URL", "file:///tmp/creatorhub.db") + if _, err := loadConfig(); err == nil { + t.Fatal("expected non-Postgres database URL to be rejected") + } +} + +func TestLoadConfigRequiresControlPlaneCredentials(t *testing.T) { + t.Setenv("CONTROL_PLANE_USERNAME", "") + t.Setenv("CONTROL_PLANE_PASSWORD", "") + if _, err := loadConfig(); err == nil { + t.Fatal("expected missing control-plane credentials to be rejected") + } +} + +func TestLoadConfigPasswordLength(t *testing.T) { + t.Setenv("CONTROL_PLANE_USERNAME", "operator") + t.Setenv("CREATORHUB_CREDENTIAL_MASTER_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + for _, test := range []struct { + password string + wantErr bool + }{ + {"12345", true}, + {"123456", false}, + } { + t.Run(test.password, func(t *testing.T) { + t.Setenv("CONTROL_PLANE_PASSWORD", test.password) + _, err := loadConfig() + if (err != nil) != test.wantErr { + t.Fatalf("loadConfig() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func TestLoadConfigRequiresCredentialMasterKey(t *testing.T) { + t.Setenv("CONTROL_PLANE_USERNAME", "operator") + t.Setenv("CONTROL_PLANE_PASSWORD", "unit-test-password") + for _, key := range []string{"", "not-base64", "c2hvcnQ="} { + t.Setenv("CREATORHUB_CREDENTIAL_MASTER_KEY", key) + if _, err := loadConfig(); err == nil { + t.Fatalf("accepted invalid credential master key %q", key) + } + } +} + +func TestControlPlaneAuthentication(t *testing.T) { + logger := logrus.StandardLogger() + previousOutput := logger.Out + var logs bytes.Buffer + logrus.SetOutput(&logs) + t.Cleanup(func() { logrus.SetOutput(previousOutput) }) + + directory := t.TempDir() + if err := os.WriteFile(filepath.Join(directory, "index.html"), []byte("index"), 0o600); err != nil { + t.Fatal(err) + } + app := newHandler(directory, "operator", "unit-test-password") + + health, err := app.Test(httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if err != nil || health.StatusCode != http.StatusNoContent { + t.Fatalf("health check must remain public: status=%d err=%v", health.StatusCode, err) + } + health.Body.Close() + + ready, err := app.Test(httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if err != nil || ready.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("readiness must fail without stores: status=%d err=%v", ready.StatusCode, err) + } + ready.Body.Close() + + unknown := httptest.NewRequest(http.MethodGet, "/api/not-registered", nil) + unknown.SetBasicAuth("operator", "unit-test-password") + unknownResponse, err := app.Test(unknown) + if err != nil || unknownResponse.StatusCode != http.StatusNotFound { + t.Fatalf("unknown API must be JSON 404: status=%d err=%v", unknownResponse.StatusCode, err) + } + if got := unknownResponse.Header.Get("Content-Type"); !strings.HasPrefix(got, "application/json") { + t.Fatalf("unknown API content type=%q", got) + } + unknownResponse.Body.Close() + + for _, path := range []string{ + "/", "/api/phase-a/accounts", "/api/browsers", "/api/network-exits", "/api/phase-a/tasks", "/api/phase-a/audit", + } { + request := httptest.NewRequest(http.MethodGet, path, nil) + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusUnauthorized || response.Header.Get("WWW-Authenticate") == "" { + t.Fatalf("GET %s was not protected: status=%d", path, response.StatusCode) + } + } + + for name, test := range map[string]struct { + user, password string + want int + }{ + "valid": {"operator", "unit-test-password", http.StatusOK}, + "wrong user": {"other", "unit-test-password", http.StatusUnauthorized}, + "wrong password": {"operator", "credential-must-not-be-logged", http.StatusUnauthorized}, + } { + t.Run(name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/", nil) + request.SetBasicAuth(test.user, test.password) + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != test.want { + t.Fatalf("status=%d want=%d", response.StatusCode, test.want) + } + }) + } + if strings.Contains(logs.String(), "credential-must-not-be-logged") { + t.Fatalf("authentication credential reached logs: %s", logs.String()) + } +} + +type controlPlaneRouteCase struct { + method, pattern, path, body string + wantAuthenticatedStatus int +} + +type testCredentialBridge struct { + values map[string]string +} + +func (bridge *testCredentialBridge) Store(ctx context.Context, _ account.CredentialReference, key, value string) error { + if err := ctx.Err(); err != nil { + return err + } + bridge.values[key] = value + return nil +} + +func (bridge *testCredentialBridge) Delete(ctx context.Context, _ account.CredentialReference, key string) error { + if err := ctx.Err(); err != nil { + return err + } + delete(bridge.values, key) + return nil +} + +func TestControlPlaneRouteRegistrationMatrix(t *testing.T) { + app := fiber.New() + app.Use(authenticate("operator", "unit-test-password")) + registerHubWithNetwork(app, nil, nil, nil) + registerPhaseA(app, nil, nil, nil) + routes := controlPlaneRouteMatrix() + assertControlPlaneRouteMatrix(t, app, routes) + + for _, route := range routes { + if response := do(app, route.method, route.path, route.body); response.Code != http.StatusUnauthorized { + t.Fatalf("registered route %s %s returned %d without credentials, want %d", route.method, route.path, + response.Code, http.StatusUnauthorized) + } + } + if response := do(app, http.MethodPost, "/api/not-registered", ""); response.Code != http.StatusUnauthorized { + t.Fatalf("unregistered route returned %d without credentials, want %d", response.Code, http.StatusUnauthorized) + } +} + +func TestControlPlaneRegisteredRouteMatrix(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run control-plane route coverage") + } + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + ctx := context.Background() + phaseAStore, err := account.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = phaseAStore.Close() }) + hubStore, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hubStore.Close() }) + + webDirectory := t.TempDir() + if err := os.WriteFile(filepath.Join(webDirectory, "index.html"), []byte("index"), 0o600); err != nil { + t.Fatal(err) + } + app := newHandlerWithStores(webDirectory, "operator", "unit-test-password", phaseAStore, hubStore) + routes := controlPlaneRouteMatrix() + assertControlPlaneRouteMatrix(t, app, routes) + + for _, route := range routes { + route := route + t.Run(route.method+" "+route.pattern, func(t *testing.T) { + if response := do(app, route.method, route.path, route.body); response.Code != http.StatusUnauthorized { + t.Fatalf("unauthenticated %s %s returned %d, want %d", route.method, route.path, response.Code, http.StatusUnauthorized) + } + response := do(app, route.method, route.path, route.body, "operator", "unit-test-password") + if response.Code != route.wantAuthenticatedStatus { + t.Fatalf("authenticated %s %s returned %d, want %d: %s", route.method, route.path, response.Code, + route.wantAuthenticatedStatus, response.Body.String()) + } + }) + } + + for _, route := range []struct { + method, path string + want int + }{ + {http.MethodPost, "/api/not-registered", http.StatusNotFound}, + {http.MethodDelete, "/api/not-registered", http.StatusNotFound}, + } { + t.Run("unregistered "+route.method, func(t *testing.T) { + if response := do(app, route.method, route.path, ""); response.Code != http.StatusUnauthorized { + t.Fatalf("unauthenticated unregistered %s %s returned %d, want %d", route.method, route.path, response.Code, http.StatusUnauthorized) + } + if response := do(app, route.method, route.path, "", "operator", "unit-test-password"); response.Code != route.want { + t.Fatalf("authenticated unregistered %s %s returned %d, want %d", route.method, route.path, response.Code, route.want) + } + }) + } +} + +func assertControlPlaneRouteMatrix(t *testing.T, app *fiber.App, routes []controlPlaneRouteCase) { + t.Helper() + expected := make(map[string]struct{}, len(routes)) + for _, route := range routes { + key := controlPlaneRouteKey(route.method, route.pattern) + if _, exists := expected[key]; exists { + t.Fatalf("duplicate route matrix entry: %s", key) + } + expected[key] = struct{}{} + } + actual := registeredControlPlaneRoutes(app) + for key := range expected { + if _, exists := actual[key]; !exists { + t.Errorf("route matrix is missing registered route: %s", key) + } + } + for key := range actual { + if _, exists := expected[key]; !exists { + t.Errorf("registered API route is missing from route matrix: %s", key) + } + } +} + +func controlPlaneRouteKey(method, path string) string { return method + " " + path } + +func registeredControlPlaneRoutes(app *fiber.App) map[string]struct{} { + routes := map[string]struct{}{} + for _, methodRoutes := range app.Stack() { + for _, route := range methodRoutes { + if !strings.HasPrefix(route.Path, "/api/") { + continue + } + switch route.Method { + case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete: + routes[controlPlaneRouteKey(route.Method, route.Path)] = struct{}{} + } + } + } + return routes +} + +func controlPlaneRouteMatrix() []controlPlaneRouteCase { + return []controlPlaneRouteCase{ + {http.MethodGet, "/api/browsers", "/api/browsers", "", http.StatusOK}, + {http.MethodGet, "/api/browsers/:alias", "/api/browsers/missing", "", http.StatusNotFound}, + {http.MethodPost, "/api/browsers", "/api/browsers", "", http.StatusBadRequest}, + {http.MethodPost, "/api/browsers/:alias/:action", "/api/browsers/missing/start", "", http.StatusNotFound}, + {http.MethodDelete, "/api/browsers/:alias", "/api/browsers/missing", "", http.StatusNotFound}, + + {http.MethodGet, "/api/network-exits", "/api/network-exits", "", http.StatusOK}, + {http.MethodGet, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusNotFound}, + {http.MethodPost, "/api/network-exits", "/api/network-exits", "", http.StatusBadRequest}, + {http.MethodPost, "/api/network-exits/:id/check", "/api/network-exits/missing/check", "", http.StatusNotFound}, + {http.MethodPost, "/api/network-exits/:id/disable", "/api/network-exits/missing/disable", "", http.StatusNotFound}, + {http.MethodPut, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusBadRequest}, + {http.MethodPost, "/api/network-exits/:id/enable", "/api/network-exits/missing/enable", "", http.StatusNotFound}, + {http.MethodDelete, "/api/network-exits/:id", "/api/network-exits/missing", "", http.StatusNotFound}, + + {http.MethodGet, "/api/browser-versions", "/api/browser-versions", "", http.StatusOK}, + {http.MethodPost, "/api/browser-versions", "/api/browser-versions", "", http.StatusBadRequest}, + {http.MethodPut, "/api/browser-versions/:version", "/api/browser-versions/999.0.0", `{"browser_path":"/opt/creatorhub/browsers/missing"}`, http.StatusNotFound}, + {http.MethodDelete, "/api/browser-versions/:version", "/api/browser-versions/999.0.0", "", http.StatusNotFound}, + + {http.MethodGet, "/api/gateways", "/api/gateways", "", http.StatusOK}, + {http.MethodPost, "/api/gateways", "/api/gateways", "", http.StatusBadRequest}, + {http.MethodPut, "/api/gateways/:name", "/api/gateways/missing", `{"name":"gw-missing","endpoint":"http://gw-missing:8081","token":""}`, http.StatusNotFound}, + {http.MethodDelete, "/api/gateways/:name", "/api/gateways/missing", "", http.StatusNotFound}, + + {http.MethodPost, "/api/phase-a/accounts", "/api/phase-a/accounts", "", http.StatusBadRequest}, + {http.MethodGet, "/api/phase-a/accounts", "/api/phase-a/accounts", "", http.StatusOK}, + {http.MethodGet, "/api/phase-a/accounts/:id", "/api/phase-a/accounts/missing", "", http.StatusNotFound}, + {http.MethodPost, "/api/phase-a/accounts/:id/pause", "/api/phase-a/accounts/missing/pause", "", http.StatusNotFound}, + {http.MethodPost, "/api/phase-a/accounts/:id/resume", "/api/phase-a/accounts/missing/resume", "", http.StatusNotFound}, + {http.MethodPost, "/api/phase-a/accounts/:id/revoke", "/api/phase-a/accounts/missing/revoke", "", http.StatusNotFound}, + + {http.MethodPost, "/api/phase-a/drafts", "/api/phase-a/drafts", "", http.StatusBadRequest}, + {http.MethodGet, "/api/phase-a/drafts", "/api/phase-a/drafts", "", http.StatusOK}, + {http.MethodGet, "/api/phase-a/drafts/:id", "/api/phase-a/drafts/missing", "", http.StatusNotFound}, + + {http.MethodPost, "/api/phase-a/confirmations", "/api/phase-a/confirmations", "", http.StatusBadRequest}, + {http.MethodGet, "/api/phase-a/confirmations", "/api/phase-a/confirmations", "", http.StatusOK}, + {http.MethodGet, "/api/phase-a/confirmations/:id", "/api/phase-a/confirmations/missing", "", http.StatusNotFound}, + + {http.MethodPost, "/api/phase-a/tasks", "/api/phase-a/tasks", "", http.StatusBadRequest}, + {http.MethodGet, "/api/phase-a/tasks", "/api/phase-a/tasks", "", http.StatusOK}, + {http.MethodGet, "/api/phase-a/tasks/:id", "/api/phase-a/tasks/missing", "", http.StatusNotFound}, + {http.MethodGet, "/api/phase-a/attempts/:id", "/api/phase-a/attempts/missing", "", http.StatusNotFound}, + {http.MethodPost, "/api/phase-a/tasks/:id/cancel", "/api/phase-a/tasks/missing/cancel", "", http.StatusNotFound}, + {http.MethodPost, "/api/phase-a/tasks/:id/verify", "/api/phase-a/tasks/missing/verify", "", http.StatusBadRequest}, + {http.MethodPost, "/api/phase-a/tasks/:id/resume", "/api/phase-a/tasks/missing/resume", "", http.StatusNotFound}, + {http.MethodPost, "/api/phase-a/tasks/:id/finish", "/api/phase-a/tasks/missing/finish", "", http.StatusNotFound}, + + {http.MethodPost, "/api/phase-a/mock/execute", "/api/phase-a/mock/execute", "", http.StatusBadRequest}, + {http.MethodGet, "/api/phase-a/audit", "/api/phase-a/audit", "", http.StatusOK}, + } +} + +func TestOperatorNotificationFiltersAndRedacts(t *testing.T) { + previousLevel := logrus.GetLevel() + t.Cleanup(func() { logrus.SetLevel(previousLevel) }) + var output bytes.Buffer + notify := newAttentionNotifier(&output) + + notify(taskstate.Transition{State: "succeeded", ReasonCode: "task_succeeded", AccountID: "account-a", TaskID: "task-a"}) + if output.Len() != 0 { + t.Fatalf("successful task emitted a notification: %s", output.String()) + } + + for _, level := range []logrus.Level{logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel} { + for _, test := range []struct{ state, reason string }{ + {"policy_hold", "account_paused"}, {"needs_confirmation", "exit_unhealthy"}, + } { + output.Reset() + logrus.SetLevel(level) + notify(taskstate.Transition{State: test.state, ReasonCode: test.reason, AccountID: "account-a", TaskID: "task-a"}) + var entry map[string]any + if err := json.Unmarshal(output.Bytes(), &entry); err != nil { + t.Fatal(err) + } + if entry["event_type"] != test.state || entry["reason_code"] != test.reason || + entry["notification_channel"] != "structured_log" || entry["account_id"] != "account-a" || entry["task_id"] != "task-a" { + t.Fatalf("unexpected notification at LOG_LEVEL=%s: %#v", level, entry) + } + for _, forbidden := range []string{"password", "authorization", "credential", "token", "secret"} { + if strings.Contains(strings.ToLower(output.String()), forbidden) { + t.Fatalf("notification contained sensitive field %q: %s", forbidden, output.String()) + } + } + } + } +} + +func TestSPAServesFileAndIndexFallback(t *testing.T) { + directory := t.TempDir() + files := map[string]string{ + "index.html": "index", + "app.js": "asset", + "hello world.js": "space", + "应用.js": "unicode", + } + for name, body := range files { + if err := os.WriteFile(filepath.Join(directory, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(directory, "..", "outside-secret.txt"), []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + app := newHandler(directory, "operator", "unit-test-password") + for path, want := range map[string]string{ + "/app.js": "asset", + "/hello%20world.js": "space", + "/%E5%BA%94%E7%94%A8.js": "unicode", + "/%2e%2e%2foutside-secret.txt": "index", + "/dashboard": "index", + } { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.SetBasicAuth("operator", "unit-test-password") + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(response.Body) + response.Body.Close() + if readErr != nil || response.StatusCode != http.StatusOK || string(body) != want { + t.Fatalf("GET %s: status=%d body=%q err=%v", path, response.StatusCode, body, readErr) + } + } +} + +func TestStartupLogIncludesListenAddress(t *testing.T) { + logger := logrus.StandardLogger() + previousOutput, previousFormatter, previousLevel := logger.Out, logger.Formatter, logger.Level + t.Cleanup(func() { + logrus.SetOutput(previousOutput) + logrus.SetFormatter(previousFormatter) + logrus.SetLevel(previousLevel) + }) + var output bytes.Buffer + logrus.SetOutput(&output) + logrus.SetFormatter(&logrus.JSONFormatter{}) + logrus.SetLevel(logrus.InfoLevel) + + logStartup(config{listenAddr: ":8080"}) + var entry map[string]any + if err := json.Unmarshal(output.Bytes(), &entry); err != nil { + t.Fatal(err) + } + if entry["listen_addr"] != ":8080" || entry["service"] != "control-plane" { + t.Fatalf("unexpected startup log: %#v", entry) + } +} + +func do(appInstance *fiber.App, method, path, body string, credentials ...string) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + request := httptest.NewRequest(method, path, reader) + if len(credentials) == 2 { + request.SetBasicAuth(credentials[0], credentials[1]) + } + adaptor.FiberApp(appInstance).ServeHTTP(response, request) + return response +} + +func isolatedControlPlaneDatabaseURL(t *testing.T, databaseURL string) string { + t.Helper() + admin, err := sql.Open("pgx", databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = admin.Close() }) + schema := fmt.Sprintf("creatorhub_hh803_%d", time.Now().UnixNano()) + if _, err := admin.Exec("CREATE SCHEMA " + schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec("DROP SCHEMA " + schema + " CASCADE"); err != nil { + t.Errorf("drop test schema: %v", err) + } + }) + parsed, err := url.Parse(databaseURL) + if err != nil { + t.Fatal(err) + } + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String() +} diff --git a/internal/controlplane/gatewayclient/client.go b/internal/controlplane/gatewayclient/client.go new file mode 100644 index 0000000..24cb170 --- /dev/null +++ b/internal/controlplane/gatewayclient/client.go @@ -0,0 +1,51 @@ +package gatewayclient + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/environment" +) + +const ( + DefaultResponseLimit = 1 << 20 + LargeResponseLimit = 16 << 20 +) + +// Call performs one authenticated native gateway request with a bounded response body. +func Call(ctx context.Context, target environment.Gateway, method, path string, body any, timeout time.Duration) (int, []byte, error) { + return CallWithLimit(ctx, target, method, path, body, timeout, DefaultResponseLimit) +} + +// CallWithLimit performs one authenticated native gateway request with an explicit body limit. +func CallWithLimit(ctx context.Context, target environment.Gateway, method, path string, body any, timeout time.Duration, responseLimit int) (int, []byte, error) { + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var payload io.Reader + if body != nil { + encoded, err := json.Marshal(body) + if err != nil { + return 0, nil, err + } + payload = bytes.NewReader(encoded) + } + request, err := http.NewRequestWithContext(callCtx, method, target.Endpoint+path, payload) + if err != nil { + return 0, nil, err + } + request.Header.Set("Authorization", "Bearer "+target.Token) + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return 0, nil, err + } + defer response.Body.Close() + responseBody, err := io.ReadAll(io.LimitReader(response.Body, int64(responseLimit))) + return response.StatusCode, responseBody, err +} diff --git a/internal/controlplane/workers/listeners.go b/internal/controlplane/workers/listeners.go new file mode 100644 index 0000000..daca34f --- /dev/null +++ b/internal/controlplane/workers/listeners.go @@ -0,0 +1,13 @@ +package workers + +import ( + "context" + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api" + creatorDomain "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/environment" +) + +func RunCreatorEventListeners(ctx context.Context, store *creatorDomain.Store, accountStore *account.Store, environmentStore *environment.Store, executor creatorDomain.ActionExecutor, generator creatorDomain.TextGenerator) { + api.RunCreatorEventListeners(ctx, store, accountStore, environmentStore, executor, generator) +} diff --git a/internal/controlplane/workers/runtime.go b/internal/controlplane/workers/runtime.go new file mode 100644 index 0000000..a3aec7a --- /dev/null +++ b/internal/controlplane/workers/runtime.go @@ -0,0 +1,31 @@ +package workers + +import ( + "context" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/controlplane/api" + creatorDomain "git.ipao.vip/rogee/creator-hub/internal/creator" + "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/sirupsen/logrus" +) + +func RunRuntimeLeaseHeartbeat(ctx context.Context, store *environment.Store) { + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := api.ReconcileRuntimeLeases(ctx, store); err != nil && ctx.Err() == nil { + logrus.WithField("service", "control-plane").WithError(err).Warn("runtime lease reconciliation failed") + } + } + } +} + +func RunCreatorScheduler(ctx context.Context, store *creatorDomain.Store, accountStore *account.Store, environmentStore *environment.Store) { + api.RunCreatorScheduler(ctx, store, accountStore, environmentStore) +} diff --git a/internal/creator/integration_test.go b/internal/creator/integration_test.go index 90dd62d..fa6f79f 100644 --- a/internal/creator/integration_test.go +++ b/internal/creator/integration_test.go @@ -10,18 +10,18 @@ import ( "testing" "time" - "git.ipao.vip/rogee/creator-hub/internal/hub" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + "git.ipao.vip/rogee/creator-hub/internal/account" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" _ "github.com/jackc/pgx/v5/stdlib" ) type integrationCredentialBridge struct{} -func (integrationCredentialBridge) Store(context.Context, phasea.CredentialReference, string, string) error { +func (integrationCredentialBridge) Store(context.Context, account.CredentialReference, string, string) error { return nil } -func (integrationCredentialBridge) Delete(context.Context, phasea.CredentialReference, string) error { +func (integrationCredentialBridge) Delete(context.Context, account.CredentialReference, string) error { return nil } @@ -62,7 +62,7 @@ func (c integrationCollector) ListTopLevelComments(_ context.Context, _, cursor return CommentPage{Items: []CommentInput{c.comment}, HasMore: false}, nil } -func openCreatorIntegrationStore(t *testing.T) (*Store, *phasea.Store, context.Context) { +func openCreatorIntegrationStore(t *testing.T) (*Store, *account.Store, context.Context) { t.Helper() databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") if databaseURL == "" { @@ -95,7 +95,7 @@ func openCreatorIntegrationStore(t *testing.T) (*Store, *phasea.Store, context.C parsed.RawQuery = query.Encode() testURL := parsed.String() - phaseAStore, err := phasea.Open(ctx, testURL) + phaseAStore, err := account.Open(ctx, testURL) if err != nil { t.Fatal(err) } @@ -113,16 +113,16 @@ func openCreatorIntegrationStore(t *testing.T) (*Store, *phasea.Store, context.C return store, phaseAStore, ctx } -func createIntegrationAccount(t *testing.T, ctx context.Context, phaseAStore *phasea.Store, suffix string) string { +func createIntegrationAccount(t *testing.T, ctx context.Context, phaseAStore *account.Store, suffix string) string { t.Helper() id := "cit" + suffix - account := phasea.Account{ + account := account.Account{ ID: id, Name: "Creator integration " + suffix, Platform: PlatformDouyin, PlatformAccountKey: "sec_uid_" + id, Tags: []string{"integration"}, - CredentialReference: phasea.CredentialReference{ID: "credential_" + id, Provider: "os_keyring"}, + CredentialReference: account.CredentialReference{ID: "credential_" + id, Provider: "os_keyring"}, CredentialKey: "creatorhub/" + id, } if err := phaseAStore.CreateAccount(ctx, account, integrationCredentialBridge{}); err != nil { @@ -422,7 +422,7 @@ func TestCreatorPostgresMetricPlanFollowsPublishedAt(t *testing.T) { } } -func prepareIntegrationActionFixture(t *testing.T, store *Store, phaseAStore *phasea.Store, ctx context.Context, stamp string) (string, string, Work, Comment, Strategy) { +func prepareIntegrationActionFixture(t *testing.T, store *Store, phaseAStore *account.Store, ctx context.Context, stamp string) (string, string, Work, Comment, Strategy) { t.Helper() bigID := createIntegrationAccount(t, ctx, phaseAStore, "big"+stamp) smallID := createIntegrationAccount(t, ctx, phaseAStore, "small"+stamp) diff --git a/cmd/control-plane/credential.go b/internal/credentials/credential.go similarity index 84% rename from cmd/control-plane/credential.go rename to internal/credentials/credential.go index 7a965e8..d95829b 100644 --- a/cmd/control-plane/credential.go +++ b/internal/credentials/credential.go @@ -1,4 +1,4 @@ -package main +package credentials import ( "context" @@ -12,33 +12,33 @@ import ( "os" "path/filepath" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + "git.ipao.vip/rogee/creator-hub/internal/account" ) const credentialFileVersion byte = 1 -type persistentCredentialBridge struct { +type PersistentCredentialBridge struct { directory string key [32]byte chmod func(*os.File, os.FileMode) error syncDirectory func(string) error } -func newPersistentCredentialBridge(directory string, key []byte) (*persistentCredentialBridge, error) { +func NewPersistentCredentialBridge(directory string, key []byte) (*PersistentCredentialBridge, error) { if !filepath.IsAbs(directory) || len(key) != 32 { return nil, errors.New("invalid credential provider configuration") } if err := os.MkdirAll(directory, 0o700); err != nil || os.Chmod(directory, 0o700) != nil { return nil, errors.New("initialize credential provider") } - bridge := &persistentCredentialBridge{ + bridge := &PersistentCredentialBridge{ directory: directory, chmod: (*os.File).Chmod, syncDirectory: syncCredentialDirectory, } copy(bridge.key[:], key) return bridge, nil } -func (bridge *persistentCredentialBridge) Store(ctx context.Context, reference phasea.CredentialReference, key, value string) error { +func (bridge *PersistentCredentialBridge) Store(ctx context.Context, reference account.CredentialReference, key, value string) error { if err := ctx.Err(); err != nil { return err } @@ -90,7 +90,7 @@ func (bridge *persistentCredentialBridge) Store(ctx context.Context, reference p return nil } -func (bridge *persistentCredentialBridge) Delete(ctx context.Context, reference phasea.CredentialReference, key string) error { +func (bridge *PersistentCredentialBridge) Delete(ctx context.Context, reference account.CredentialReference, key string) error { if err := ctx.Err(); err != nil { return err } @@ -106,7 +106,7 @@ func (bridge *persistentCredentialBridge) Delete(ctx context.Context, reference return nil } -func (bridge *persistentCredentialBridge) Resolve(ctx context.Context, reference phasea.CredentialReference, key string) ([]byte, error) { +func (bridge *PersistentCredentialBridge) Resolve(ctx context.Context, reference account.CredentialReference, key string) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } @@ -132,7 +132,7 @@ func (bridge *persistentCredentialBridge) Resolve(ctx context.Context, reference return value, nil } -func (bridge *persistentCredentialBridge) aead() (cipher.AEAD, error) { +func (bridge *PersistentCredentialBridge) aead() (cipher.AEAD, error) { block, err := aes.NewCipher(bridge.key[:]) if err != nil { return nil, errors.New("initialize credential encryption") @@ -140,7 +140,7 @@ func (bridge *persistentCredentialBridge) aead() (cipher.AEAD, error) { return cipher.NewGCM(block) } -func (bridge *persistentCredentialBridge) path(provider, key string) string { +func (bridge *PersistentCredentialBridge) path(provider, key string) string { digest := sha256.Sum256(credentialAAD(provider, key)) return filepath.Join(bridge.directory, hex.EncodeToString(digest[:])+".credential") } diff --git a/cmd/control-plane/credential_test.go b/internal/credentials/credential_test.go similarity index 91% rename from cmd/control-plane/credential_test.go rename to internal/credentials/credential_test.go index 5f52c2f..cc93156 100644 --- a/cmd/control-plane/credential_test.go +++ b/internal/credentials/credential_test.go @@ -1,4 +1,4 @@ -package main +package credentials import ( "context" @@ -9,18 +9,18 @@ import ( "strings" "testing" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + "git.ipao.vip/rogee/creator-hub/internal/account" ) const testCredentialValue = "sessionid=restart-secret; token=second" -var testCredentialReference = phasea.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"} +var testCredentialReference = account.CredentialReference{ID: "account-a-cookies", Provider: "os_keyring"} func TestPersistentCredentialBridgeStoreFailureLeavesNoFile(t *testing.T) { for _, failure := range []string{"chmod", "directory sync"} { t.Run(failure, func(t *testing.T) { directory := t.TempDir() - bridge, err := newPersistentCredentialBridge(directory, []byte("0123456789abcdef0123456789abcdef")) + bridge, err := NewPersistentCredentialBridge(directory, []byte("0123456789abcdef0123456789abcdef")) if err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestPersistentCredentialBridgeStoreFailureLeavesNoFile(t *testing.T) { } func TestPersistentCredentialBridgeResolvesAndRejectsCorruption(t *testing.T) { - bridge, err := newPersistentCredentialBridge(t.TempDir(), []byte("0123456789abcdef0123456789abcdef")) + bridge, err := NewPersistentCredentialBridge(t.TempDir(), []byte("0123456789abcdef0123456789abcdef")) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestPersistentCredentialBridgeResolvesAndRejectsCorruption(t *testing.T) { func TestPersistentCredentialBridgeStoresEncryptedCredentialAcrossProcesses(t *testing.T) { if action := os.Getenv("CREATORHUB_CREDENTIAL_HELPER_ACTION"); action != "" { - bridge, err := newPersistentCredentialBridge(os.Getenv("CREATORHUB_CREDENTIAL_HELPER_DIR"), []byte("0123456789abcdef0123456789abcdef")) + bridge, err := NewPersistentCredentialBridge(os.Getenv("CREATORHUB_CREDENTIAL_HELPER_DIR"), []byte("0123456789abcdef0123456789abcdef")) if err != nil { t.Fatal(err) } diff --git a/internal/hub/environment.go b/internal/environment/environment.go similarity index 99% rename from internal/hub/environment.go rename to internal/environment/environment.go index 4c6c844..813b9ca 100644 --- a/internal/hub/environment.go +++ b/internal/environment/environment.go @@ -1,4 +1,4 @@ -package hub +package environment import ( "context" diff --git a/internal/hub/fingerprint.go b/internal/environment/fingerprint.go similarity index 99% rename from internal/hub/fingerprint.go rename to internal/environment/fingerprint.go index e1f3f70..4e73190 100644 --- a/internal/hub/fingerprint.go +++ b/internal/environment/fingerprint.go @@ -1,4 +1,4 @@ -package hub +package environment import ( "errors" diff --git a/internal/hub/migration_test.go b/internal/environment/migration_test.go similarity index 97% rename from internal/hub/migration_test.go rename to internal/environment/migration_test.go index 46fd2e0..d3eb043 100644 --- a/internal/hub/migration_test.go +++ b/internal/environment/migration_test.go @@ -1,4 +1,4 @@ -package hub +package environment import ( "context" @@ -12,7 +12,7 @@ import ( "testing" "time" - "git.ipao.vip/rogee/creator-hub/internal/phasea" + "git.ipao.vip/rogee/creator-hub/internal/account" "git.ipao.vip/rogee/creator-hub/internal/taskstate" ) @@ -80,7 +80,7 @@ func TestUnifiedAccountMigration(t *testing.T) { } _, err := Open(ctx, testURL) - if err == nil || !strings.Contains(err.Error(), "apply hub schema migration 14") || strings.Contains(err.Error(), "migration-secret") { + if err == nil || !strings.Contains(err.Error(), "apply environment schema migration 14") || strings.Contains(err.Error(), "migration-secret") { t.Fatalf("unsafe legacy migration was not blocked safely: %v", err) } assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version = 14`, 0) @@ -125,7 +125,7 @@ func TestUnifiedAccountMigration(t *testing.T) { assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'task-recovery-upgrade' AND state = 'needs_confirmation' AND hold_reason = 'task_result_uncertain'`, 1) - phaseAStore, err := phasea.Open(ctx, testURL) + phaseAStore, err := account.Open(ctx, testURL) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Run("v1 and v2 data", func(t *testing.T) { ctx := context.Background() testURL := isolatedDatabaseURL(t, databaseURL) - phaseAStore, err := phasea.Open(ctx, testURL) + phaseAStore, err := account.Open(ctx, testURL) if err != nil { t.Fatal(err) } @@ -360,7 +360,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatalf("missing upgrade transition %+v in %+v", want, notifications) } } - phaseAStore, err = phasea.Open(ctx, testURL) + phaseAStore, err = account.Open(ctx, testURL) if err != nil { t.Fatal(err) } @@ -373,7 +373,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatalf("verify upgraded executing task: %v", err) } assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'upgrade-executing' AND verification_result = 'failed' AND verified_by = 'local-user'`, 1) - if err := phaseAStore.VerifyTask(ctx, "upgrade-executing", "succeeded"); !errors.Is(err, phasea.ErrConflict) { + if err := phaseAStore.VerifyTask(ctx, "upgrade-executing", "succeeded"); !errors.Is(err, account.ErrConflict) { t.Fatalf("repeated verification changed the recorded conclusion: %v", err) } assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE confirmation_id = 'upgrade-confirmation'`, 2) @@ -389,7 +389,7 @@ func TestUnifiedAccountMigration(t *testing.T) { func openFullyMigratedHub(t *testing.T, ctx context.Context, databaseURL string) *Store { t.Helper() - phaseAStore, err := phasea.Open(ctx, databaseURL) + phaseAStore, err := account.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } @@ -429,7 +429,7 @@ func isolatedDatabaseURL(t *testing.T, databaseURL string) string { func openLegacyAccountCreationSchema(t *testing.T, ctx context.Context, databaseURL string) *sql.DB { t.Helper() - phaseAStore, err := phasea.Open(ctx, databaseURL) + phaseAStore, err := account.Open(ctx, databaseURL) if err != nil { t.Fatal(err) } diff --git a/internal/hub/migrations/002_hub.sql b/internal/environment/migrations/002_hub.sql similarity index 100% rename from internal/hub/migrations/002_hub.sql rename to internal/environment/migrations/002_hub.sql diff --git a/internal/hub/migrations/003_unified_accounts.sql b/internal/environment/migrations/003_unified_accounts.sql similarity index 100% rename from internal/hub/migrations/003_unified_accounts.sql rename to internal/environment/migrations/003_unified_accounts.sql diff --git a/internal/hub/migrations/004_environment_actions.sql b/internal/environment/migrations/004_environment_actions.sql similarity index 100% rename from internal/hub/migrations/004_environment_actions.sql rename to internal/environment/migrations/004_environment_actions.sql diff --git a/internal/hub/migrations/005_sanitize_legacy_proxy.sql b/internal/environment/migrations/005_sanitize_legacy_proxy.sql similarity index 100% rename from internal/hub/migrations/005_sanitize_legacy_proxy.sql rename to internal/environment/migrations/005_sanitize_legacy_proxy.sql diff --git a/internal/hub/migrations/006_runtime_cleanup.sql b/internal/environment/migrations/006_runtime_cleanup.sql similarity index 100% rename from internal/hub/migrations/006_runtime_cleanup.sql rename to internal/environment/migrations/006_runtime_cleanup.sql diff --git a/internal/hub/migrations/007_runtime_binding_version.sql b/internal/environment/migrations/007_runtime_binding_version.sql similarity index 100% rename from internal/hub/migrations/007_runtime_binding_version.sql rename to internal/environment/migrations/007_runtime_binding_version.sql diff --git a/internal/hub/migrations/008_runtime_cleanup_generation.sql b/internal/environment/migrations/008_runtime_cleanup_generation.sql similarity index 100% rename from internal/hub/migrations/008_runtime_cleanup_generation.sql rename to internal/environment/migrations/008_runtime_cleanup_generation.sql diff --git a/internal/hub/migrations/009_runtime_cleanup_compatibility.sql b/internal/environment/migrations/009_runtime_cleanup_compatibility.sql similarity index 100% rename from internal/hub/migrations/009_runtime_cleanup_compatibility.sql rename to internal/environment/migrations/009_runtime_cleanup_compatibility.sql diff --git a/internal/hub/migrations/010_runtime_network_generation.sql b/internal/environment/migrations/010_runtime_network_generation.sql similarity index 100% rename from internal/hub/migrations/010_runtime_network_generation.sql rename to internal/environment/migrations/010_runtime_network_generation.sql diff --git a/internal/hub/migrations/011_task_recovery.sql b/internal/environment/migrations/011_task_recovery.sql similarity index 100% rename from internal/hub/migrations/011_task_recovery.sql rename to internal/environment/migrations/011_task_recovery.sql diff --git a/internal/hub/migrations/012_task_recovery_compatibility.sql b/internal/environment/migrations/012_task_recovery_compatibility.sql similarity index 100% rename from internal/hub/migrations/012_task_recovery_compatibility.sql rename to internal/environment/migrations/012_task_recovery_compatibility.sql diff --git a/internal/hub/migrations/013_account_creation.sql b/internal/environment/migrations/013_account_creation.sql similarity index 100% rename from internal/hub/migrations/013_account_creation.sql rename to internal/environment/migrations/013_account_creation.sql diff --git a/internal/hub/migrations/014_account_creation_compatibility.sql b/internal/environment/migrations/014_account_creation_compatibility.sql similarity index 100% rename from internal/hub/migrations/014_account_creation_compatibility.sql rename to internal/environment/migrations/014_account_creation_compatibility.sql diff --git a/internal/hub/migrations/015_gateway_rename_cascade.sql b/internal/environment/migrations/015_gateway_rename_cascade.sql similarity index 100% rename from internal/hub/migrations/015_gateway_rename_cascade.sql rename to internal/environment/migrations/015_gateway_rename_cascade.sql diff --git a/internal/hub/migrations/016_network_exit_plain_credentials.sql b/internal/environment/migrations/016_network_exit_plain_credentials.sql similarity index 100% rename from internal/hub/migrations/016_network_exit_plain_credentials.sql rename to internal/environment/migrations/016_network_exit_plain_credentials.sql diff --git a/internal/hub/migrations/017_native_browser_versions.sql b/internal/environment/migrations/017_native_browser_versions.sql similarity index 100% rename from internal/hub/migrations/017_native_browser_versions.sql rename to internal/environment/migrations/017_native_browser_versions.sql diff --git a/internal/hub/migrations/033_unique_fingerprint_seed.sql b/internal/environment/migrations/033_unique_fingerprint_seed.sql similarity index 100% rename from internal/hub/migrations/033_unique_fingerprint_seed.sql rename to internal/environment/migrations/033_unique_fingerprint_seed.sql diff --git a/internal/hub/store.go b/internal/environment/store.go similarity index 96% rename from internal/hub/store.go rename to internal/environment/store.go index c9f7db0..b1b2868 100644 --- a/internal/hub/store.go +++ b/internal/environment/store.go @@ -1,4 +1,4 @@ -package hub +package environment import ( "context" @@ -74,7 +74,7 @@ var migration033 string var ( ErrConflict = errors.New("resource conflicts with existing state") - ErrInvalid = errors.New("invalid hub input") + ErrInvalid = errors.New("invalid environment input") ErrNotFound = errors.New("resource not found") ErrReconcileRequired = errors.New("runtime cleanup generation is unknown; manual reconciliation required") ) @@ -129,14 +129,14 @@ type Env struct { func Open(ctx context.Context, databaseURL string) (*Store, error) { db, err := sql.Open("pgx", databaseURL) if err != nil { - return nil, errors.New("open hub database") + return nil, errors.New("open environment database") } db.SetMaxOpenConns(10) db.SetMaxIdleConns(2) db.SetConnMaxIdleTime(5 * time.Minute) if err := db.PingContext(ctx); err != nil { db.Close() - return nil, errors.New("connect to hub database") + return nil, errors.New("connect to environment database") } // LockResources keeps one connection until the lifecycle operation finishes. // Admit at most half the pool so those operations can still open nested DB calls. @@ -218,11 +218,11 @@ func (s *Store) LockResources(ctx context.Context, aliases, exitIDs, browserVers func (s *Store) migrate(ctx context.Context) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return errors.New("begin hub schema migration") + return errors.New("begin environment schema migration") } defer tx.Rollback() if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1542738012)`); err != nil { - return errors.New("lock hub schema migration") + return errors.New("lock environment schema migration") } if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { return errors.New("create schema migration table") @@ -233,20 +233,20 @@ func (s *Store) migrate(ctx context.Context) error { }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}, {7, migration007}, {8, migration008}, {9, migration009}, {10, migration010}, {11, migration011}, {12, migration012}, {13, migration013}, {14, migration014}, {15, migration015}, {16, migration016}, {17, migration017}, {33, migration033}} { var applied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { - return errors.New("read hub schema migration state") + return errors.New("read environment schema migration state") } if applied { continue } if _, err := tx.ExecContext(ctx, migration.sql); err != nil { - return fmt.Errorf("apply hub schema migration %d: %w", migration.version, err) + return fmt.Errorf("apply environment schema migration %d: %w", migration.version, err) } if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migration (version) VALUES ($1)`, migration.version); err != nil { - return fmt.Errorf("record hub schema migration %d", migration.version) + return fmt.Errorf("record environment schema migration %d", migration.version) } } if err := tx.Commit(); err != nil { - return errors.New("commit hub schema migration") + return errors.New("commit environment schema migration") } return nil } @@ -626,7 +626,7 @@ func newToken() string { func commitHub(tx *sql.Tx) error { if err := tx.Commit(); err != nil { - return errors.New("commit hub transaction") + return errors.New("commit environment transaction") } return nil } @@ -670,5 +670,5 @@ func publicDatabaseError(err error) error { if errors.As(err, &postgresError) && (postgresError.Code == "23505" || postgresError.Code == "23503" || postgresError.Code == "23514") { return ErrConflict } - return errors.New("hub persistence operation failed") + return errors.New("environment persistence operation failed") } diff --git a/internal/hub/store_test.go b/internal/environment/store_test.go similarity index 99% rename from internal/hub/store_test.go rename to internal/environment/store_test.go index 5d7a972..24d919c 100644 --- a/internal/hub/store_test.go +++ b/internal/environment/store_test.go @@ -1,4 +1,4 @@ -package hub +package environment import ( "context" diff --git a/internal/douyin/connector.go b/internal/platform/douyin/connector.go similarity index 100% rename from internal/douyin/connector.go rename to internal/platform/douyin/connector.go diff --git a/internal/douyin/connector_test.go b/internal/platform/douyin/connector_test.go similarity index 100% rename from internal/douyin/connector_test.go rename to internal/platform/douyin/connector_test.go diff --git a/internal/douyin/creator_collector.go b/internal/platform/douyin/creator_collector.go similarity index 100% rename from internal/douyin/creator_collector.go rename to internal/platform/douyin/creator_collector.go diff --git a/internal/douyin/creator_collector_test.go b/internal/platform/douyin/creator_collector_test.go similarity index 100% rename from internal/douyin/creator_collector_test.go rename to internal/platform/douyin/creator_collector_test.go diff --git a/internal/xiaohongshu/collector.go b/internal/platform/xiaohongshu/collector.go similarity index 100% rename from internal/xiaohongshu/collector.go rename to internal/platform/xiaohongshu/collector.go diff --git a/internal/xiaohongshu/collector_test.go b/internal/platform/xiaohongshu/collector_test.go similarity index 100% rename from internal/xiaohongshu/collector_test.go rename to internal/platform/xiaohongshu/collector_test.go diff --git a/internal/xiaohongshu/detail.go b/internal/platform/xiaohongshu/detail.go similarity index 100% rename from internal/xiaohongshu/detail.go rename to internal/platform/xiaohongshu/detail.go diff --git a/internal/xiaohongshu/detail_test.go b/internal/platform/xiaohongshu/detail_test.go similarity index 100% rename from internal/xiaohongshu/detail_test.go rename to internal/platform/xiaohongshu/detail_test.go diff --git a/scripts/dev-backend.mjs b/scripts/dev-backend.mjs index 37e40ca..680c43e 100755 --- a/scripts/dev-backend.mjs +++ b/scripts/dev-backend.mjs @@ -52,7 +52,7 @@ try { if (!response.ok) throw new Error(`HTTP ${response.status}`); } catch (error) { console.error(`[dev-backend] native browser gateway 不可达：${overrides.NATIVE_GATEWAY_ENDPOINT} (${error})`); - console.error("请先以非 root 用户启动 cmd/browser_gateway.gateway 或对应的 systemd user service。"); + console.error("请先以非 root 用户启动 browser_gateway.server.http 或对应的 systemd user service。"); process.exit(1); } diff --git a/web/index.html b/web/index.html index beb40c8..f725009 100644 --- a/web/index.html +++ b/web/index.html @@ -9,6 +9,6 @@
- + diff --git a/web/src/Layout.jsx b/web/src/app/Layout.jsx similarity index 96% rename from web/src/Layout.jsx rename to web/src/app/Layout.jsx index b82e8e0..2761fe1 100644 --- a/web/src/Layout.jsx +++ b/web/src/app/Layout.jsx @@ -1,6 +1,6 @@ import { useEffect, useState } from "react"; import { Link, Outlet, useLocation } from "react-router"; -import { cn } from "./lib/ui.jsx"; +import { cn } from "../shared/ui/ui.jsx"; const expandedWidth = 240; const collapsedWidth = 76; @@ -99,7 +99,7 @@ function Sidebar({ collapsed, onNavigate }) { ); } -export function CreatorHubLayout({ children }) { +export function CreatorHubLayout() { const [collapsed, setCollapsed] = useState(false); const [mobileOpen, setMobileOpen] = useState(false); const { pathname } = useLocation(); @@ -112,11 +112,10 @@ export function CreatorHubLayout({ children }) { const desktop = typeof window !== "undefined" && window.matchMedia("(min-width: 768px)").matches; - const menuLabel = desktop - ? collapsed - ? "展开导航" - : "收起导航" - : "打开导航"; + let menuLabel = "打开导航"; + if (desktop) { + menuLabel = collapsed ? "展开导航" : "收起导航"; + } return (
diff --git a/web/src/Layout.test.jsx b/web/src/app/Layout.test.jsx similarity index 100% rename from web/src/Layout.test.jsx rename to web/src/app/Layout.test.jsx diff --git a/web/src/LoginPage.jsx b/web/src/app/LoginPage.jsx similarity index 97% rename from web/src/LoginPage.jsx rename to web/src/app/LoginPage.jsx index fe8da53..f2909bf 100644 --- a/web/src/LoginPage.jsx +++ b/web/src/app/LoginPage.jsx @@ -1,6 +1,6 @@ import { useState } from "react"; import { useNavigate } from "react-router"; -import { Alert, Button, Field, Input } from "./lib/ui.jsx"; +import { Alert, Button, Field, Input } from "../shared/ui/ui.jsx"; // 开发阶段：本地保存 Basic 凭证，仅在请求层自动附带。开发代理（vite）不注入 Authorization。 export function LoginPage() { diff --git a/web/src/main.jsx b/web/src/app/main.jsx similarity index 79% rename from web/src/main.jsx rename to web/src/app/main.jsx index bc383d0..1521a3b 100644 --- a/web/src/main.jsx +++ b/web/src/app/main.jsx @@ -2,22 +2,22 @@ import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { HashRouter, Navigate, Route, Routes, useLocation } from "react-router"; import { Refine } from "@refinedev/core"; -import { dataProvider } from "./dataProvider"; +import { dataProvider } from "../shared/api/dataProvider"; import { CreatorHubLayout } from "./Layout"; import { LoginPage } from "./LoginPage"; -import { AccountCreatePage, AccountDetail, AccountList } from "./AccountsPage"; -import { AuditPage } from "./AuditPage"; -import { BrowserCreatePage, BrowserDetail, BrowserList } from "./BrowsersPage"; -import { BrowserVersionList } from "./BrowserVersionsPage"; -import { DraftPage } from "./DraftPage"; -import { GatewayList } from "./GatewaysPage"; -import { NetworkExitDetail, NetworkExitList } from "./NetworkExitsPage"; -import { TaskDetail, TaskList } from "./TasksPage"; -import { AttemptDetail } from "./TracePages"; -import { AccountEditPage } from "./AccountEditPage"; -import { CreatorCompetitorsPage } from "./CreatorCompetitorsPage"; -import { CreatorSettingsPage } from "./CreatorSettingsPage"; -import { CreatorWorkbenchPage } from "./CreatorWorkbenchPage"; +import { AccountCreatePage, AccountDetail, AccountList } from "../features/accounts/AccountsPage"; +import { AccountEditPage } from "../features/accounts/AccountEditPage"; +import { CreatorCompetitorsPage } from "../features/creator/competitors/CreatorCompetitorsPage"; +import { CreatorSettingsPage } from "../features/creator/settings/CreatorSettingsPage"; +import { CreatorWorkbenchPage } from "../features/creator/workbench/CreatorWorkbenchPage"; +import { AuditPage } from "../features/operations/AuditPage"; +import { BrowserCreatePage, BrowserDetail, BrowserList } from "../features/environments/BrowsersPage"; +import { BrowserVersionList } from "../features/environments/BrowserVersionsPage"; +import { DraftPage } from "../features/operations/DraftPage"; +import { GatewayList } from "../features/environments/GatewaysPage"; +import { NetworkExitDetail, NetworkExitList } from "../features/environments/NetworkExitsPage"; +import { TaskDetail, TaskList } from "../features/operations/TasksPage"; +import { AttemptDetail } from "../features/operations/TracePages"; import "remixicon/fonts/remixicon.css"; import "./styles.css"; diff --git a/web/src/styles.css b/web/src/app/styles.css similarity index 100% rename from web/src/styles.css rename to web/src/app/styles.css diff --git a/web/src/AccountEditPage.jsx b/web/src/features/accounts/AccountEditPage.jsx similarity index 99% rename from web/src/AccountEditPage.jsx rename to web/src/features/accounts/AccountEditPage.jsx index b3b1530..133c418 100644 --- a/web/src/AccountEditPage.jsx +++ b/web/src/features/accounts/AccountEditPage.jsx @@ -16,8 +16,8 @@ import { Textarea, dateTime, conflictMessage, - useUnsavedChanges, -} from "./lib/ui.jsx"; +} from "../../shared/ui/ui.jsx"; +import { useUnsavedChanges } from "../../shared/hooks/hooks.js"; const emptyProfile = { login_username: "", diff --git a/web/src/AccountsPage.jsx b/web/src/features/accounts/AccountsPage.jsx similarity index 83% rename from web/src/AccountsPage.jsx rename to web/src/features/accounts/AccountsPage.jsx index 7ef4cf8..cd3d7da 100644 --- a/web/src/AccountsPage.jsx +++ b/web/src/features/accounts/AccountsPage.jsx @@ -19,8 +19,8 @@ import { TagInput, Textarea, conflictMessage, -} from "./lib/ui.jsx"; -import { useTitle } from "./lib/hooks.js"; +} from "../../shared/ui/ui.jsx"; +import { useTitle } from "../../shared/hooks/hooks.js"; const createInitial = { name: "", @@ -51,7 +51,7 @@ const reasonText = { }; // 账号可恢复性判定：授权、绑定、出口健康、账号暂停四层都要看。 -export function accountReadiness(account, binding, bindingsError = false) { +export function accountReadiness(account, binding, bindingError = null) { if (account.authorization_status !== "authorized") return { label: "授权已撤销", @@ -59,7 +59,7 @@ export function accountReadiness(account, binding, bindingsError = false) { canResume: false, ready: false, }; - if (bindingsError) + if (bindingError) return { label: "环境状态未知", reason: "environment_unavailable", @@ -73,11 +73,12 @@ export function accountReadiness(account, binding, bindingsError = false) { canResume: false, ready: false, }; - const blocked = binding.cleanup_pending - ? "runtime_stop_pending" - : binding.network_exit_id && binding.network_exit_health !== "healthy" - ? "network_exit_unhealthy" - : ""; + let blocked = ""; + if (binding.cleanup_pending) { + blocked = "runtime_stop_pending"; + } else if (binding.network_exit_id && binding.network_exit_health !== "healthy") { + blocked = "network_exit_unhealthy"; + } if (blocked) return { label: reasonText[blocked], @@ -105,12 +106,13 @@ export function accountReadiness(account, binding, bindingsError = false) { }; } -function ReadinessPill({ readiness, paused }) { - const tone = readiness.ready - ? "success" - : paused && readiness.canResume - ? "warning" - : "danger"; +function ReadinessPill({ readiness, runtimeStatus }) { + let tone = "danger"; + if (readiness.ready) { + tone = "success"; + } else if (runtimeStatus === "paused" && readiness.canResume) { + tone = "warning"; + } return ; } @@ -595,84 +597,96 @@ export function AccountList() { { header: "状态", width: "16%", - render: (account) => - account.source_type === "owned" ? ( -
- -

- {account.authorization_status === "authorized" - ? "已授权" - : "已撤销"} -

-
- ) : ( + render: (account) => { + if (account.source_type === "owned") { + return ( +
+ +

+ {account.authorization_status === "authorized" + ? "已授权" + : "已撤销"} +

+
+ ); + } + return ( - ), + ); + }, }, { header: "操作", width: "18%", - render: (account) => ( -
- - {account.source_type === "owned" ? ( - <> - - - - ) : ( - <> - - - - )} + , + ]; + if (account.source_type === "owned") { + actions.push( + , + , + ); + } else { + actions.push( + , + , + ); + } + actions.push( -
- ), + , + ); + return
{actions}
; + }, }, ]} rows={visibleRows} @@ -717,6 +731,60 @@ export function AccountList() { ); } +function BindingContent({ browsersError, binding }) { + if (browsersError) { + return ( +

+ 运行环境与网络出口状态未知；重试成功后再执行依赖资源状态的操作。 +

+ ); + } + if (binding) { + return ( + <> + + + 前往运行环境 + + + ); + } + return ( + <> +

+ 尚未绑定运行环境。账号暂停且资源就绪后，可在运行环境页创建绑定。 +

+ + 前往运行环境 + + + ); +} + export function AccountDetail() { const { id } = useParams(); const navigate = useNavigate(); @@ -750,7 +818,7 @@ export function AccountDetail() { ? undefined : browsers.find((item) => item.account_id === id); const readiness = account - ? accountReadiness(account, binding, !!browsersError) + ? accountReadiness(account, binding, browsersError) : null; useTitle("CreatorHub · 账号详情"); @@ -839,7 +907,7 @@ export function AccountDetail() {

账号状态

固定资源

- {browsersError ? ( -

- 运行环境与网络出口状态未知；重试成功后再执行依赖资源状态的操作。 -

- ) : binding ? ( - <> - - - 前往运行环境 - - - ) : ( - <> -

- 尚未绑定运行环境。账号暂停且资源就绪后，可在运行环境页创建绑定。 -

- - 前往运行环境 - - - )} + diff --git a/web/src/AccountsPage.test.jsx b/web/src/features/accounts/AccountsPage.test.jsx similarity index 100% rename from web/src/AccountsPage.test.jsx rename to web/src/features/accounts/AccountsPage.test.jsx diff --git a/web/src/CreatorPages.test.jsx b/web/src/features/creator/CreatorPages.test.jsx similarity index 98% rename from web/src/CreatorPages.test.jsx rename to web/src/features/creator/CreatorPages.test.jsx index 76df4e6..3cacc82 100644 --- a/web/src/CreatorPages.test.jsx +++ b/web/src/features/creator/CreatorPages.test.jsx @@ -9,11 +9,11 @@ import { import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { Refine } from "@refinedev/core"; import { MemoryRouter, Route, Routes } from "react-router"; -import { AccountEditPage } from "./AccountEditPage"; -import { AccountList } from "./AccountsPage"; -import { CreatorCompetitorsPage } from "./CreatorCompetitorsPage"; -import { CreatorSettingsPage } from "./CreatorSettingsPage"; -import { CreatorWorkbenchPage } from "./CreatorWorkbenchPage"; +import { AccountEditPage } from "../accounts/AccountEditPage"; +import { AccountList } from "../accounts/AccountsPage"; +import { CreatorCompetitorsPage } from "./competitors/CreatorCompetitorsPage"; +import { CreatorSettingsPage } from "./settings/CreatorSettingsPage"; +import { CreatorWorkbenchPage } from "./workbench/CreatorWorkbenchPage"; const profile = { id: "account-a", diff --git a/web/src/CreatorCompetitorsPage.jsx b/web/src/features/creator/competitors/CreatorCompetitorsPage.jsx similarity index 99% rename from web/src/CreatorCompetitorsPage.jsx rename to web/src/features/creator/competitors/CreatorCompetitorsPage.jsx index 5615aef..4d84512 100644 --- a/web/src/CreatorCompetitorsPage.jsx +++ b/web/src/features/creator/competitors/CreatorCompetitorsPage.jsx @@ -16,8 +16,8 @@ import { Textarea, dateTime, conflictMessage, - useUnsavedChanges, -} from "./lib/ui.jsx"; +} from "../../../shared/ui/ui.jsx"; +import { useUnsavedChanges } from "../../../shared/hooks/hooks.js"; const platformOptions = [ { value: "douyin", label: "抖音" }, diff --git a/web/src/CreatorSettingsPage.jsx b/web/src/features/creator/settings/CreatorSettingsPage.jsx similarity index 98% rename from web/src/CreatorSettingsPage.jsx rename to web/src/features/creator/settings/CreatorSettingsPage.jsx index 9886283..78e79f5 100644 --- a/web/src/CreatorSettingsPage.jsx +++ b/web/src/features/creator/settings/CreatorSettingsPage.jsx @@ -10,8 +10,8 @@ import { PageHeader, PageState, conflictMessage, - useUnsavedChanges, -} from "./lib/ui.jsx"; +} from "../../../shared/ui/ui.jsx"; +import { useUnsavedChanges } from "../../../shared/hooks/hooks.js"; const editableFields = [ "lookback_days", diff --git a/web/src/CreatorWorkbenchPage.jsx b/web/src/features/creator/workbench/CreatorWorkbenchPage.jsx similarity index 99% rename from web/src/CreatorWorkbenchPage.jsx rename to web/src/features/creator/workbench/CreatorWorkbenchPage.jsx index ec98205..f311bf8 100644 --- a/web/src/CreatorWorkbenchPage.jsx +++ b/web/src/features/creator/workbench/CreatorWorkbenchPage.jsx @@ -15,8 +15,8 @@ import { Textarea, dateTime, conflictMessage, - useUnsavedChanges, -} from "./lib/ui.jsx"; +} from "../../../shared/ui/ui.jsx"; +import { useUnsavedChanges } from "../../../shared/hooks/hooks.js"; const tabs = [ ["comments", "评论"], diff --git a/web/src/BrowserVersionsPage.jsx b/web/src/features/environments/BrowserVersionsPage.jsx similarity index 99% rename from web/src/BrowserVersionsPage.jsx rename to web/src/features/environments/BrowserVersionsPage.jsx index 1bf2a9a..8993255 100644 --- a/web/src/BrowserVersionsPage.jsx +++ b/web/src/features/environments/BrowserVersionsPage.jsx @@ -11,8 +11,8 @@ import { PageState, Switch, Table, -} from "./lib/ui.jsx"; -import { useTitle } from "./lib/hooks.js"; +} from "../../shared/ui/ui.jsx"; +import { useTitle } from "../../shared/hooks/hooks.js"; const versionPattern = /^\d[A-Za-z0-9._-]{0,63}$/; const pathPattern = /^\/[A-Za-z0-9._+~/-]{1,4095}$/; diff --git a/web/src/BrowserVersionsPage.test.jsx b/web/src/features/environments/BrowserVersionsPage.test.jsx similarity index 100% rename from web/src/BrowserVersionsPage.test.jsx rename to web/src/features/environments/BrowserVersionsPage.test.jsx diff --git a/web/src/BrowsersPage.jsx b/web/src/features/environments/BrowsersPage.jsx similarity index 99% rename from web/src/BrowsersPage.jsx rename to web/src/features/environments/BrowsersPage.jsx index 9ae142b..fbc31c5 100644 --- a/web/src/BrowsersPage.jsx +++ b/web/src/features/environments/BrowsersPage.jsx @@ -16,9 +16,8 @@ import { Select, StatusPill, conflictMessage, - useUnsavedChanges, -} from "./lib/ui.jsx"; -import { useTitle } from "./lib/hooks.js"; +} from "../../shared/ui/ui.jsx"; +import { useTitle, useUnsavedChanges } from "../../shared/hooks/hooks.js"; const aliasPattern = /^[a-z0-9][a-z0-9-]{0,31}$/; const platformOptions = ["windows", "linux", "macos"].map((value) => ({ diff --git a/web/src/BrowsersPage.test.jsx b/web/src/features/environments/BrowsersPage.test.jsx similarity index 100% rename from web/src/BrowsersPage.test.jsx rename to web/src/features/environments/BrowsersPage.test.jsx diff --git a/web/src/GatewaysPage.jsx b/web/src/features/environments/GatewaysPage.jsx similarity index 99% rename from web/src/GatewaysPage.jsx rename to web/src/features/environments/GatewaysPage.jsx index 20530d5..385732d 100644 --- a/web/src/GatewaysPage.jsx +++ b/web/src/features/environments/GatewaysPage.jsx @@ -10,8 +10,8 @@ import { PageHeader, PageState, Table, -} from "./lib/ui.jsx"; -import { useTitle } from "./lib/hooks.js"; +} from "../../shared/ui/ui.jsx"; +import { useTitle } from "../../shared/hooks/hooks.js"; const namePattern = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; const tokenPattern = /^[A-Za-z0-9][A-Za-z0-9._-]{15,127}$/; diff --git a/web/src/GatewaysPage.test.jsx b/web/src/features/environments/GatewaysPage.test.jsx similarity index 100% rename from web/src/GatewaysPage.test.jsx rename to web/src/features/environments/GatewaysPage.test.jsx diff --git a/web/src/NetworkExitsPage.jsx b/web/src/features/environments/NetworkExitsPage.jsx similarity index 99% rename from web/src/NetworkExitsPage.jsx rename to web/src/features/environments/NetworkExitsPage.jsx index 7923100..2761cd8 100644 --- a/web/src/NetworkExitsPage.jsx +++ b/web/src/features/environments/NetworkExitsPage.jsx @@ -17,8 +17,8 @@ import { StatusPill, Table, conflictMessage, -} from "./lib/ui.jsx"; -import { useTitle } from "./lib/hooks.js"; +} from "../../shared/ui/ui.jsx"; +import { useTitle } from "../../shared/hooks/hooks.js"; const createInitial = { protocol: "socks5", diff --git a/web/src/NetworkExitsPage.test.jsx b/web/src/features/environments/NetworkExitsPage.test.jsx similarity index 100% rename from web/src/NetworkExitsPage.test.jsx rename to web/src/features/environments/NetworkExitsPage.test.jsx diff --git a/web/src/AuditPage.jsx b/web/src/features/operations/AuditPage.jsx similarity index 98% rename from web/src/AuditPage.jsx rename to web/src/features/operations/AuditPage.jsx index 178540b..c0ec303 100644 --- a/web/src/AuditPage.jsx +++ b/web/src/features/operations/AuditPage.jsx @@ -1,8 +1,8 @@ import { useEffect, useState } from 'react' import { Link, useSearchParams } from 'react-router' import { useList } from '@refinedev/core' -import { Alert, Button, PageHeader, PageState, Table } from './lib/ui.jsx' -import { useTitle } from './lib/hooks.js' +import { Alert, Button, PageHeader, PageState, Table } from '../../shared/ui/ui.jsx' +import { useTitle } from '../../shared/hooks/hooks.js' const startOfDay = value => (value ? new Date(`${value}T00:00:00`).toISOString() : '') const endOfDay = value => (value ? new Date(`${value}T23:59:59.999`).toISOString() : '') diff --git a/web/src/AuditPage.test.jsx b/web/src/features/operations/AuditPage.test.jsx similarity index 100% rename from web/src/AuditPage.test.jsx rename to web/src/features/operations/AuditPage.test.jsx diff --git a/web/src/DraftPage.jsx b/web/src/features/operations/DraftPage.jsx similarity index 99% rename from web/src/DraftPage.jsx rename to web/src/features/operations/DraftPage.jsx index c7dbfab..d24501d 100644 --- a/web/src/DraftPage.jsx +++ b/web/src/features/operations/DraftPage.jsx @@ -4,8 +4,8 @@ import { useDataProvider, useList, useOne } from '@refinedev/core' import { Alert, Button, Card, CardContent, Checkbox, ConfirmDialog, DetailList, Field, Input, Modal, PageHeader, PageState, StatusPill, conflictMessage, -} from './lib/ui.jsx' -import { useTitle } from './lib/hooks.js' +} from '../../shared/ui/ui.jsx' +import { useTitle } from '../../shared/hooks/hooks.js' export function DraftPage() { const { id } = useParams() diff --git a/web/src/DraftPage.test.jsx b/web/src/features/operations/DraftPage.test.jsx similarity index 100% rename from web/src/DraftPage.test.jsx rename to web/src/features/operations/DraftPage.test.jsx diff --git a/web/src/TasksPage.jsx b/web/src/features/operations/TasksPage.jsx similarity index 99% rename from web/src/TasksPage.jsx rename to web/src/features/operations/TasksPage.jsx index 6e55a8c..f6b91f2 100644 --- a/web/src/TasksPage.jsx +++ b/web/src/features/operations/TasksPage.jsx @@ -3,8 +3,8 @@ import { Link, useNavigate, useParams } from 'react-router' import { useDataProvider, useList, useOne } from '@refinedev/core' import { Alert, Button, Card, CardContent, DetailList, Field, PageHeader, PageState, Select, StatusPill, -} from './lib/ui.jsx' -import { useTitle } from './lib/hooks.js' +} from '../../shared/ui/ui.jsx' +import { useTitle } from '../../shared/hooks/hooks.js' const stateTone = { queued: 'neutral', executing: 'info', succeeded: 'success', failed: 'danger', needs_confirmation: 'warning', policy_hold: 'warning', cancelled: 'neutral' } const stateText = { queued: '已排队', executing: '执行中', succeeded: '已成功', failed: '失败', needs_confirmation: '需要人工确认', policy_hold: '策略暂停', cancelled: '已取消' } diff --git a/web/src/TasksPage.test.jsx b/web/src/features/operations/TasksPage.test.jsx similarity index 100% rename from web/src/TasksPage.test.jsx rename to web/src/features/operations/TasksPage.test.jsx diff --git a/web/src/TracePages.jsx b/web/src/features/operations/TracePages.jsx similarity index 95% rename from web/src/TracePages.jsx rename to web/src/features/operations/TracePages.jsx index 28a9c8d..9df3abe 100644 --- a/web/src/TracePages.jsx +++ b/web/src/features/operations/TracePages.jsx @@ -1,7 +1,7 @@ import { Link, useParams } from 'react-router' import { useOne } from '@refinedev/core' -import { Alert, Button, Card, CardContent, DetailList, PageHeader, PageState } from './lib/ui.jsx' -import { useTitle } from './lib/hooks.js' +import { Alert, Button, Card, CardContent, DetailList, PageHeader, PageState } from '../../shared/ui/ui.jsx' +import { useTitle } from '../../shared/hooks/hooks.js' function PendingOrError({ pending, error, children }) { if (pending) return diff --git a/web/src/lib/hooks.js b/web/src/lib/hooks.js deleted file mode 100644 index eeb1458..0000000 --- a/web/src/lib/hooks.js +++ /dev/null @@ -1,16 +0,0 @@ -import { useEffect } from 'react' - -export function useTitle(title) { - useEffect(() => { document.title = title }, [title]) -} - -// 401 之外的业务错误（404/409/503…）直接展示后端 message；401 在 dataProvider 里转跳登录页。 -export function displayError(error) { - return error?.message || '操作失败' -} - -export function conflictError(error, fallback) { - if (error?.status === 409) return `冲突（409）：${error.body?.reason || fallback}` - if (error?.status === 503) return `资源未就绪（503）：${error.message}` - return displayError(error) -} diff --git a/web/src/dataProvider.js b/web/src/shared/api/dataProvider.js similarity index 100% rename from web/src/dataProvider.js rename to web/src/shared/api/dataProvider.js diff --git a/web/src/dataProvider.test.js b/web/src/shared/api/dataProvider.test.js similarity index 100% rename from web/src/dataProvider.test.js rename to web/src/shared/api/dataProvider.test.js diff --git a/web/src/shared/hooks/hooks.js b/web/src/shared/hooks/hooks.js new file mode 100644 index 0000000..c0b6e33 --- /dev/null +++ b/web/src/shared/hooks/hooks.js @@ -0,0 +1,39 @@ +import { useEffect } from 'react' + +export function useTitle(title) { + useEffect(() => { document.title = title }, [title]) +} + +// 401 之外的业务错误（404/409/503…）直接展示后端 message；401 在 dataProvider 里转跳登录页。 +export function displayError(error) { + return error?.message || '操作失败' +} + +export function conflictError(error, fallback) { + if (error?.status === 409) return `冲突（409）：${error.body?.reason || fallback}` + if (error?.status === 503) return `资源未就绪（503）：${error.message}` + return displayError(error) +} + +export function useUnsavedChanges(dirty) { + useEffect(() => { + if (!dirty) return undefined + const beforeUnload = (event) => { + event.preventDefault() + event.returnValue = "" + } + const click = (event) => { + const anchor = event.target.closest?.("a[href]") + if (!anchor || !anchor.getAttribute("href")?.startsWith("#/")) return + if (window.confirm("当前内容尚未保存，确定离开吗？")) return + event.preventDefault() + event.stopPropagation() + } + window.addEventListener("beforeunload", beforeUnload) + document.addEventListener("click", click, true) + return () => { + window.removeEventListener("beforeunload", beforeUnload) + document.removeEventListener("click", click, true) + } + }, [dirty]) +} diff --git a/web/src/lib/hooks.test.jsx b/web/src/shared/hooks/hooks.test.jsx similarity index 100% rename from web/src/lib/hooks.test.jsx rename to web/src/shared/hooks/hooks.test.jsx diff --git a/web/src/lib/ui.jsx b/web/src/shared/ui/ui.jsx similarity index 96% rename from web/src/lib/ui.jsx rename to web/src/shared/ui/ui.jsx index f9f2fb0..7a1aa10 100644 --- a/web/src/lib/ui.jsx +++ b/web/src/shared/ui/ui.jsx @@ -21,29 +21,6 @@ export function conflictMessage(error, fallback) { export const dateTime = (value) => new Date(value).toLocaleString("zh-CN"); -export function useUnsavedChanges(dirty) { - useEffect(() => { - if (!dirty) return undefined; - const beforeUnload = (event) => { - event.preventDefault(); - event.returnValue = ""; - }; - const click = (event) => { - const anchor = event.target.closest?.("a[href]"); - if (!anchor || !anchor.getAttribute("href")?.startsWith("#/")) return; - if (window.confirm("当前内容尚未保存，确定离开吗？")) return; - event.preventDefault(); - event.stopPropagation(); - }; - window.addEventListener("beforeunload", beforeUnload); - document.addEventListener("click", click, true); - return () => { - window.removeEventListener("beforeunload", beforeUnload); - document.removeEventListener("click", click, true); - }; - }, [dirty]); -} - /** * 页面加载/错误/空态的统一外壳。 */ diff --git a/web/tests/responsive.e2e.js b/web/tests/responsive.e2e.js index 463b1d2..7a21d07 100644 --- a/web/tests/responsive.e2e.js +++ b/web/tests/responsive.e2e.js @@ -24,14 +24,15 @@ test('opens social-account and network-exit configuration from the main menu', a await page.route('**/api/gateways', route => route.fulfill({ json: [] })) await page.route('**/api/browser-versions', route => route.fulfill({ json: [] })) await page.route('**/api/phase-a/accounts', route => route.fulfill({ json: [] })) + await page.route('**/api/creator/competitors', route => route.fulfill({ json: [] })) await page.route('**/api/network-exits', route => route.fulfill({ json: [] })) await page.setViewportSize({ width: 599, height: 800 }) await page.goto('/#/browsers') await page.getByRole('button', { name: '打开导航' }).click() - await page.getByRole('link', { name: '社媒账号' }).click() + await page.getByRole('link', { name: '账号管理' }).click() await expect(page).toHaveURL(/#\/accounts$/) - await expect(page.getByRole('heading', { level: 1, name: '社媒账号' })).toBeVisible() + await expect(page.getByRole('heading', { level: 1, name: '账号管理' })).toBeVisible() await page.getByRole('button', { name: '打开导航' }).click() await page.getByRole('link', { name: '网络出口' }).click() @@ -109,22 +110,22 @@ test('submits the minimal social-account form at 900px', async ({ page }) => { if (route.request().method() === 'POST') return route.fulfill({ json: { id: 'account-new' } }) return route.fulfill({ json: [] }) }) + await page.route('**/api/creator/competitors', route => route.fulfill({ json: [] })) await page.route('**/api/browsers', route => route.fulfill({ json: [] })) await page.setViewportSize({ width: 900, height: 900 }) await page.goto('/#/accounts') - // 简单创建走弹窗：先打开 modal 再填写 - await page.getByRole('button', { name: '创建账号' }).click() - const dialog = page.getByRole('dialog') - await dialog.getByRole('textbox', { name: '账号名称', exact: true }).fill('店铺一号') - await dialog.getByRole('combobox', { name: '平台类型', exact: true }).click() + // 创建账号使用独立页面：打开页面后填写并提交 + await page.getByRole('link', { name: '创建自有账号' }).click() + await page.getByRole('textbox', { name: '账号名称', exact: true }).fill('店铺一号') + await page.getByRole('combobox', { name: '平台类型', exact: true }).click() await page.getByRole('option', { name: '抖音', exact: true }).click() - await dialog.getByRole('textbox', { name: '账号 ID', exact: true }).fill('shop-new') - await dialog.getByRole('textbox', { name: 'TAGS', exact: true }).fill('主账号,直播') - await dialog.getByRole('textbox', { name: 'Cookies', exact: true }).fill('sessionid=value; token=second') + await page.getByRole('textbox', { name: '账号 ID', exact: true }).fill('shop-new') + await page.getByRole('textbox', { name: 'TAGS', exact: true }).fill('主账号,直播') + await page.getByRole('textbox', { name: 'Cookies', exact: true }).fill('sessionid=value; token=second') const request = page.waitForRequest(request => request.url().endsWith('/api/phase-a/accounts') && request.method() === 'POST') - await dialog.getByRole('button', { name: '创建账号' }).click() + await page.getByRole('button', { name: '创建账号' }).click() expect((await request).postDataJSON()).toEqual({ name: '店铺一号', platform: 'douyin', platform_account_key: 'shop-new', tags: ['主账号', '直播'], cookies: 'sessionid=value; token=second' }) }) @@ -152,9 +153,10 @@ test('keeps account and network-exit pages inside 599px, 900px and 1280px', asyn }) test('opens account detail at the phase A route', async ({ page }) => { - const account = { id: 'account-a', name: '店铺一号', platform: 'douyin', platform_account_key: 'shop-a', tags: [], authorization_status: 'authorized', runtime_status: 'paused', version: 1 } + const account = { id: 'account-a', name: '店铺一号', platform: 'douyin', platform_account_key: 'shop-a', source_type: 'owned', tags: [], authorization_status: 'authorized', runtime_status: 'paused', version: 1 } await page.route('**/api/phase-a/accounts', route => route.fulfill({ json: [account] })) await page.route('**/api/phase-a/accounts/account-a', route => route.fulfill({ json: account })) + await page.route('**/api/creator/competitors', route => route.fulfill({ json: [] })) await page.route('**/api/browsers', route => route.fulfill({ json: [] })) await page.route('**/api/phase-a/drafts?account_id=account-a', route => route.fulfill({ json: [] })) await page.setViewportSize({ width: 599, height: 900 })