diff --git a/.gitignore b/.gitignore index cf94366..520365d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ dist/ build/ # 测试报告与缓存 +.coverage +.coverage.* web/coverage/ web/test-results/ .pytest_cache/ diff --git a/AGENTS.md b/AGENTS.md index c8dbcc1..68dd197 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,16 +40,28 @@ 产品方向：平台当前仅支持抖音（douyin）；小红书等其他平台的业务与代码已全部移除，不得重新引入。 -环境先行：创建时只创建抖音浏览器环境，不填写昵称、UID 或 Cookie，不创建占位账号。未登录环境与已登录账号在“我的账号”同一列表展示，不设独立区域；缺失资料使用前端默认值，不创建占位账号，不写回数据库。待登录行仅保留“登录并同步账号”，不提供账号详情或其他账号操作。统一列表按共用的浏览器环境数据库 ID 倒序排列，新建项在前；登录绑定保留环境 ID，不改变行的相对位置。首次浏览器身份核验成功后，同一事务创建/关联账号并同步真实 UID、昵称、头像、抖音号与 secUID；同一 UID 只能绑定一个账号，已有绑定禁止换绑或自动覆盖。后续同 UID 核验更新平台资料，不覆盖本地备注和业务设置。浏览器 profile_id 与指纹 seed 在环境创建时确定，账号绑定不得改变它们；旧环境保持原 profile_id 和 seed，不重建浏览器或清空 Cookie。指纹 seed 由环境独立序列分配，不再依赖账号 ID。 +环境先行：创建时只创建抖音浏览器环境，不填写昵称、UID 或 Cookie，不创建占位账号。未登录环境与已登录账号在“我的账号”同一列表展示，不设独立区域；缺失资料使用前端默认值，不创建占位账号，不写回数据库。待登录行仅保留“登录并同步账号”，不提供账号详情或其他账号操作。统一列表按共用的浏览器环境数据库 ID 倒序排列，新建项在前；登录绑定保留环境 ID，不改变行的相对位置。首次浏览器身份核验成功后，同一事务创建/关联账号并同步真实 UID、昵称、头像、抖音号与 secUID；同一 UID 只能绑定一个账号，已有绑定禁止换绑或自动覆盖。后续同 UID 核验更新平台资料，不覆盖本地备注和业务设置。身份核验成功在同一事务解除仅由未登录造成的采集阻塞，恢复为待采集；保留原采集窗口、游标、累计进度和完成时间，不清除其他错误、不标记采集完成。浏览器 profile_id 与指纹 seed 在环境创建时确定，账号绑定不得改变它们；旧环境保持原 profile_id 和 seed，不重建浏览器或清空 Cookie。指纹 seed 由环境独立序列分配，不再依赖账号 ID。 + +浏览器内存：每个环境独立配置 `memory_limit_mb`，默认 2048 MB，允许 512–65536 MB；创建与编辑均可设置。运行中保存不重启浏览器、不改变指纹、profile_id 或账号绑定，下次启动生效；启动时必须将保存值传给网关并施加真实内存上限。 作品封面：自有账号与监测账号采集后都下载封面，图片保存为静态文件 `<账号 UID>/<作品 ID>.<实际图片扩展名>`，不把图片字节存入数据库。目录由 `CREATOR_COVER_DIR` 配置，开发默认 `.data/covers`，容器使用持久卷；作品封面接口只读取本地文件，不使用远程图片兜底。下载与文件错误必须可追踪并反映到同步结果；同一来源每次同步最多回填 40 张历史缺失封面。 账号创建交互：在“我的账号”列表通过 Modal 创建，网关必选，浏览器指纹为默认折叠的可选配置；创建成功关闭弹窗并刷新列表，不保留独立新增页面。未登录环境及空昵称仅在前端显示“待登录”，不将此文案写入数据库。 +我的账号操作：列表只保留一个状态感知主操作和“更多”；未绑定环境仍仅显示“登录并同步账号”。已绑定账号暂停时显示“恢复运行”，已知需要登录时显示“重新登录”，实时浏览器未运行时显示“启动环境”，浏览器运行且最近核验登录正常时显示“同步资料”，状态无法确认时显示“检查状态”。操作前读取当前账号与真实网关运行状态，不用保存的 runtime_id 冒充运行；登录、启动、同步资料不会偷偷恢复已暂停账号。正常登录直接核验并同步平台资料，只有明确 awaiting_login 才展示二维码；资料同步不重新采集作品。编辑标签、编辑账号、启动环境、暂停账号、停止浏览器、删除收纳到“更多”；暂停、停止、删除均明确确认。暂停账号会暂停账号调度并停止浏览器；仅停止浏览器不暂停账号。状态读取失败必须明确提示，不假装正常。 + 自有账号作品：作品总数使用抖音最新非空 `aweme_count`，已保存 `work_count` 仅表示采集进度，不得互相替代；未知总数与零作品必须区分。自有账号作品库保存平台分页返回的全部历史作品，不受采集回看天数限制；回看天数仍限制监控账号作品和评论采集。详情分页浏览已采集作品，并明确展示采集进度与未完整状态，不能将采集成功等同于已采齐。 +我的账号状态：列表明确区分“调度状态”“业务状态”“采集状态”；调度可运行及业务正常不代表作品或评论采集正常。列表的采集状态仅展示一个 Tag，不展示时间、不区分作品和评论；任一采集异常优先于采集中或完成，异常原因只在悬停 Tag 时展示并去重。详情页仍分别展示作品、评论结果和真实上次完成时间。浏览器需要登录时明确中文提示，未登录环境只显示“待登录” Tag。缺少状态及读取失败不能冒充待采集或采集正常；没有完成时间不虚构时间。采集成功只表示上次任务完成，不表示作品已采齐。资源问题造成的采集受阻不能永久退出调度；仅调度开启、已登录且业务正常或禁言的自有账号可自动重试。重试保留原窗口、游标、累计进度和上次完成时间；选取重试任务不清除错误，只有实际执行采集才更新结果，不以同步资料冒充采集成功。 + 作品分析：左侧原“竞品分析”统一为“作品分析”，页面通过“竞品账号”“我的账号”两个 TAB 严格区分作品来源；各 TAB 独立保留筛选与分页。支持所属账号、发布时间和互动阈值筛选，以及发布时间、点赞、评论、分享、收藏、24h 点赞增量升降序；自有作品额外支持观看量，不展示“时间核验状态”筛选。默认最新发布在前，条件变化回到第一页，筛选与排序作用于全部已采集作品后再分页；缺失指标显示“—”，排序放末尾、阈值筛选不将其当作 0。未采集作品不参与分析。 +账号事件监听：仅支持抖音，自有账号默认关闭，在“我的账号”的“监听状态”列逐个开启，未登录环境不能开启。互动通知使用网页通知列表只读轮询，不再依赖私有 WebSocket SDK；每页 50 条，is_mark_read=0，不标记已读，原始 JSON 在 Python 解码以保留 64 位 ID。只记录点赞、评论、关注、转发，私信在独立收件箱同步与展示，不并入事件聚合，不自动互动。监听依赖已登录且运行中的浏览器；按账号固定页面并核验 UID，支持多标签页。分组读取位置、开启边界和事件在同一事务保存，提交成功后确认，重复投递去重；开关代次拒绝关闭期间的旧投递；首次开启、重新开启和服务重启均补齐平台仍可返回的全部历史和漏收记录，包括关闭期间通知，正常运行约每 5 分钟完整核对一次。开启时间只用于历史标记，不作为丢弃依据，补入保留平台原时间；不按头部位置或 100 页上限截断完整核对。开启设置不等于接收正常，状态应展示实际读取结果、最后成功时间及错误或可能遗漏。左侧“事件聚合”只展示当前已开启账号的已接收互动通知，按接收时间倒序，支持账号、类型、接收时间范围筛选和分页，每 5 秒刷新；关闭后保留历史但隐藏，重新开启后可查看。缺失发生时间不能用接收时间冒充；真实验收必须核对平台、数据库和页面同一通知，不把读取成功或平台合并后的重复互动当作新事件通过。 + +事件聚合资料：互动用户展示真实昵称与 UID，以平台返回的 secUID 链接主页，不使用数字 UID 猜测主页；发生时间在前、接收时间紧邻其后，所属账号及筛选选项仅显示昵称。对应作品展示 48px 小封面并链接抖音作品详情，图文使用 note 地址；缺封面保留作品入口，缺资料明确标示。通知中的作品封面复用 `<作者 UID>/<作品 ID>.<图片扩展名>` 本地缓存，不创建占位账号或作品，不回退展示远程图片。历史核对可补齐已保存事件的用户与作品资料，但只更新同一互动 UID、同一作品 ID 的资料，不改原事件时间、内容、历史标记或去重规则；封面下载失败单独记录和展示，不能阻断通知保存，也不能伪装成已缓存。 + +私信管理：仅支持已开启监听的自有抖音账号，独立聊天标签共享原浏览器资料，不干扰采集或主页面登录。展示聊天客户端已加载会话的最近 50 条消息，不宣称完整历史；只支持手动发送文字，不自动回复、不群发。聊天界面采用 Ant Design X Conversations、Bubble.List 和 Sender：左侧合并联系人并明确所属账号，同一联系人在不同账号下为独立会话，草稿按账号与联系人隔离。专用标签从已登录的个人中心“消息”入口进入，沿用现有登录，不增加私信扫码；直接 /chat 的二维码不能作为账号未登录的依据。仅以聊天客户端的实际 UID 与已绑定 UID 一致确认私信身份，不创建账号或改变绑定。发送请求先持久化并按请求 ID 去重；超时、重启中断及 SDK 网络错误 1008 标记“结果未确认”，不自动重发，不伪造送达或已读。同步记录须持久化后推进检查点；关闭监听阻止旧代次写入。 + 评论聚合：独立页面通过“我的作品评论”“竞品作品评论”两个 TAB 严格区分来源，各 TAB 保留筛选与页码。列表展示评论发布时间、评论者、内容、所属账号名称及对应作品的小封面；所属账号列与筛选下拉均不展示 UID，缺名仅显示前端“未命名账号”，评论者显示不变；作品使用本地封面，点击在新标签页打开抖音作品页面，缺图明确提示，不展示作品标题、不使用远程图片兜底。支持所属账号筛选，以及最近 1/6/12 小时、1/3/5/7 天筛选，默认最近 1 天。按评论发布时间计算范围，最新评论在前，同时间按评论 ID 倒序；未记录发布时间、未来时间及未采集评论不参与。筛选先作用于全部已采集评论再分页，条件或每页数量变化回到第一页；只读展示，不新增采集机制。 前端框架：Umi Max 4.7 + React 19； diff --git a/browser_gateway/platform/douyin.py b/browser_gateway/platform/douyin.py index 0134df5..07dd670 100644 --- a/browser_gateway/platform/douyin.py +++ b/browser_gateway/platform/douyin.py @@ -12,12 +12,10 @@ import re import threading import time import uuid -from collections import deque from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from datetime import datetime, timezone -from urllib.parse import urlsplit +from urllib.parse import quote, urlsplit import websocket @@ -40,6 +38,22 @@ ORIGIN = "https://www.douyin.com" ORIGIN_URL = ORIGIN + "/" # 扫码登录默认打开个人中心页：未登录访问会引导登录，登录后即停在账号个人中心。 LOGIN_PAGE_URL = ORIGIN + "/user/self" +CHAT_PAGE_URL = LOGIN_PAGE_URL +CHAT_PANEL_SCRIPT = r'''(()=>{ + if(location.origin!=="https://www.douyin.com")throw Error("private-message origin changed"); + const panel=document.querySelector("#imSaasContainerId");const rect=panel?.getBoundingClientRect(); + return {ready:rect?.width>0&&rect?.height>0}; +})()''' +CHAT_ENTRY_SCRIPT = r'''(()=>{ + if(location.origin!=="https://www.douyin.com")throw Error("private-message origin changed"); + const ready=()=>{const rect=document.querySelector("#imSaasContainerId")?.getBoundingClientRect();return rect?.width>0&&rect?.height>0}; + if(ready())return {ready:true}; + const entry=document.querySelector('[data-e2e="im-entry"] [data-e2e="something-button"]'); + if(entry)entry.click(); + return {ready:ready(),clicked:!!entry}; +})()''' +CHAT_PANEL_TIMEOUT = 6.0 +CHAT_PAGE_NAME = "__creatorhub_private_messages" LOGIN_ORIGINS = frozenset( {ORIGIN, "https://sso.douyin.com", "https://verify.snssdk.com", "https://verify.bytedance.com"} ) @@ -119,6 +133,8 @@ class DouyinBrowser: self.media_selector = media_selector self.image_validator = image_validator or is_douyin_image_url self.target_id = target_id.strip() + self._chat_targets: dict[str, str] = {} + self._chat_lock = threading.Lock() @contextmanager def connection(self, alias: str): @@ -128,7 +144,7 @@ class DouyinBrowser: finally: connection.close() - def _connect(self, alias: str) -> CDPConnection: + def _target_list(self, alias: str): if not ALIAS_RE.fullmatch(alias): raise DouyinError("browser alias is invalid") if self.endpoint is None: @@ -172,11 +188,20 @@ class DouyinBrowser: raise DouyinError("browser target discovery response is invalid") from exc if not isinstance(targets, list): raise DouyinError("browser target discovery response is invalid") + return parsed, port, targets + + def _connect(self, alias: str) -> CDPConnection: + parsed, port, targets = self._target_list(alias) page_targets = [ target for target in targets if isinstance(target, dict) and target.get("type") == "page" ] + if not self.target_id: + page_targets = [ + target for target in page_targets + if not self._is_chat_target(alias, target) + ] if self.target_id: matching_targets = [ target for target in page_targets if target.get("id") == self.target_id @@ -189,12 +214,19 @@ class DouyinBrowser: for target in page_targets if self.url_validator(target.get("url", "")) ] - if len(matching_targets) > 1 or ( - not matching_targets and len(page_targets) > 1 - ): - raise DouyinError("browser has more than one page target") + if not matching_targets and len(page_targets) > 1: + LOG.warning( + "browser page selection failed alias=%s eligible_pages=%s", + alias, len(page_targets), + ) + raise DouyinError("browser page target is unavailable; open Douyin in this browser") + # Tabs in one profile share login cookies. Match notification polling's + # stable selection rather than treating ordinary extra tabs as an error. target = ( - matching_targets[0] + min(matching_targets, key=lambda target: ( + target.get("url", "").split("?", 1)[0] != self.origin + "/user/self", + target.get("id", ""), + )) if matching_targets else page_targets[0] if page_targets @@ -202,6 +234,7 @@ class DouyinBrowser: ) if target is None: raise DouyinError("browser page target is unavailable") + LOG.info("browser request target selected alias=%s target=%s pinned=%s", alias, target.get("id", ""), bool(self.target_id)) websocket_url = target.get("webSocketDebuggerUrl") try: parsed_ws = urlsplit( @@ -224,6 +257,10 @@ class DouyinBrowser: ): raise DouyinError("browser target websocket host is invalid") page_target = parsed_ws.geturl() + LOG.info( + "browser page selected alias=%s target=%s eligible_pages=%s matching_pages=%s pinned=%s", + alias, target.get("id"), len(page_targets), len(matching_targets), bool(self.target_id), + ) try: socket = websocket.create_connection( page_target, @@ -235,6 +272,106 @@ class DouyinBrowser: raise DouyinError("browser CDP connection failed") from exc return CDPConnection(socket) + def _notification_browser(self, alias: str): + # Every page belongs to this alias's profile. Pin one authenticated-origin + # page so multiple ordinary tabs cannot make polling ambiguous. + _, _, targets = self._target_list(alias) + candidates = [t for t in targets if isinstance(t, dict) + and t.get("type") == "page" + and isinstance(t.get("id"), str) + and isinstance(t.get("url"), str) + and self.url_validator(t["url"])] + if not candidates: + raise DouyinError("notification page is unavailable; open Douyin in this browser") + chosen = min(candidates, key=lambda t: (t["url"].split("?", 1)[0] != self.origin + "/user/self", t["id"])) + LOG.info("notification page selected alias=%s target=%s", alias, chosen["id"]) + return self._browser_for_target(chosen["id"]) + + def _browser_for_target(self, target_id: str): + return DouyinBrowser(self.endpoint, origin=self.origin, url_validator=self.url_validator, target_id=target_id) + + def _is_chat_target(self, alias: str, target: dict) -> bool: + target_id = target.get("id", "") + if target_id in self._chat_targets.values(): + return True + if target.get("url", "") != CHAT_PAGE_URL + "#" + CHAT_PAGE_NAME: + return False + with self._browser_for_target(target_id).connection(alias) as cdp: + return cdp.evaluate("window.name") == CHAT_PAGE_NAME + + def _create_chat_target(self, alias: str, parsed, port: int) -> str: + connection = http.client.HTTPConnection(parsed.hostname, port, timeout=CONTROL_TIMEOUT) + try: + connection.request("PUT", "/json/new?" + quote(CHAT_PAGE_URL + "#" + CHAT_PAGE_NAME, safe="")) + response = connection.getresponse() + if response.status != 200: + raise DouyinError("private-message page could not be created") + target = json.loads(response.read(64 * 1024 + 1)) + except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc: + raise DouyinError("private-message page could not be created") from exc + finally: + connection.close() + target_id = target.get("id") if isinstance(target, dict) else None + if not isinstance(target_id, str) or not target_id: + raise DouyinError("private-message page target is invalid") + # Reserve the role before initialization so ordinary collection cannot + # accidentally select this new page while it is still loading. + self._chat_targets[alias] = target_id + with self._browser_for_target(target_id).connection(alias) as cdp: + cdp.command("Runtime.runIfWaitingForDebugger") + deadline = time.monotonic() + LOGIN_RENDER_TIMEOUT + while cdp.evaluate("location.origin") != self.origin: + if time.monotonic() >= deadline: + raise DouyinError("private-message page did not finish loading") + time.sleep(0.2) + cdp.evaluate(f"(() => {{ window.name = {json.dumps(CHAT_PAGE_NAME)}; return true; }})()") + LOG.info("private-message page created alias=%s target=%s", alias, target_id) + return target_id + + def _chat_browser(self, alias: str): + with self._chat_lock: + parsed, port, targets = self._target_list(alias) + pages = [target for target in targets if isinstance(target, dict) and target.get("type") == "page"] + target_id = self._chat_targets.get(alias) + if target_id not in {target.get("id") for target in pages}: + marked = [target for target in pages if self._is_chat_target(alias, target)] + if len(marked) > 1: + raise DouyinError("browser has multiple private-message pages") + target_id = marked[0]["id"] if marked else self._create_chat_target(alias, parsed, port) + self._chat_targets[alias] = target_id + browser = self._browser_for_target(target_id) + with browser.connection(alias) as cdp: + # Only our marked tab is normalized; the user's/collector's main + # page is never navigated or used as a private-message fallback. + if cdp.evaluate("location.pathname") != "/user/self": + cdp.command("Page.enable") + cdp.command("Page.navigate", {"url": CHAT_PAGE_URL}) + cdp.wait_event("Page.domContentEventFired", lambda params: True, timeout=5) + # Background tabs throttle JS timers. Poll short, synchronous CDP + # reads from Python instead of keeping Runtime.evaluate pending. + for attempt in range(2): + deadline = time.monotonic() + CHAT_PANEL_TIMEOUT + clicked = False + while time.monotonic() < deadline: + prepared = cdp.evaluate(CHAT_PANEL_SCRIPT if clicked else CHAT_ENTRY_SCRIPT) + if not isinstance(prepared, dict) or not isinstance(prepared.get("ready"), bool): + raise DouyinError("private-message entry response is invalid") + if prepared.get("ready") is True: + if attempt: + LOG.info("private-message panel recovered alias=%s target=%s", alias, target_id) + return browser + clicked = clicked or prepared.get("clicked") is True + time.sleep(0.2) + LOG.warning("private-message panel unavailable alias=%s target=%s clicked=%s refreshed=%s", alias, target_id, clicked, bool(attempt)) + if attempt == 0: + # A tab opened before login can retain a guest UI even + # after the shared profile is authenticated. Reload only + # our chat tab; SDK identity is still verified afterward. + cdp.command("Page.enable") + cdp.command("Page.reload") + cdp.wait_event("Page.domContentEventFired", lambda params: True, timeout=5) + raise DouyinError("private-message panel did not open after refreshing the chat page; confirm the browser account is logged in") + def _navigate_to_origin(self, cdp: CDPConnection) -> None: """把浏览器带到抖音首页。 @@ -398,6 +535,10 @@ class DouyinBrowser: ): raise DouyinError("Douyin login page navigation failed") self._wait_for_login_render(cdp) + return self._capture_login_qr(alias) + + def _capture_login_qr(self, alias: str) -> BrowserLoginQRResponse: + with self.connection(alias) as cdp: opened = cdp.evaluate( """(() => { const text = value => String(value || '').replace(/\\s+/g, ''); @@ -559,6 +700,7 @@ class DouyinBrowser: raise DouyinError("Douyin identity response is invalid") from exc user = payload.get("user") if isinstance(payload, dict) else None uid = str(user.get("uid", "")) if isinstance(user, dict) else "" + LOG.info("browser identity response alias=%s target=%s http_status=%s status_code=%s uid_present=%s expected_uid_set=%s", alias, self.target_id or "auto", response.status, payload.get("status_code") if isinstance(payload, dict) else "invalid", bool(uid), bool(expected_uid)) if response.status == 200 and isinstance(payload, dict) and ( payload.get("status_code") == 8 or (payload.get("status_code") == 0 and isinstance(user, dict) and uid == "0") @@ -616,7 +758,7 @@ class DouyinBrowser: def set_action_ownership(self, alias: str, marker: dict) -> None: if not isinstance(marker, dict): raise DouyinError("browser action ownership marker is invalid") - expression = f"localStorage.setItem('__creatorhub_action_ownership_v1', {json.dumps(json.dumps(marker, separators=(',', ':')))})" + expression = f"(() => {{ localStorage.setItem('__creatorhub_action_ownership_v1', {json.dumps(json.dumps(marker, separators=(',', ':')))}); return true; }})()" self._evaluate(alias, expression) def clear_action_ownership(self, alias: str, operation_id: str = "") -> None: @@ -657,7 +799,15 @@ class DouyinBrowser: not text.strip() or len(text) > 1000 ): raise DouyinError("action text is invalid") - identity = self.identity(alias, expected_uid) + action_browser = self._chat_browser(alias) if action == "dm" else self + if action == "dm": + identity = action_browser._evaluate(alias, private_identity_expression(expected_uid)) + if not isinstance(identity, dict): + raise DouyinError("private-message identity response is invalid") + if identity.get("status") != "logged_in": + return {"status": "failed", "code": identity.get("code", "LOGIN_REQUIRED")} + else: + identity = action_browser.identity(alias, expected_uid) if action == "follow": params = { "expected": expected_uid, @@ -685,7 +835,7 @@ class DouyinBrowser: "cursor": "9223372036854775807", "action": "send", } - value = self._confirmed_evaluate(alias, im_expression(params, expected_uid)) + value = action_browser._confirmed_evaluate(alias, im_expression(params, expected_uid)) else: params = { "action": action, @@ -723,6 +873,14 @@ class DouyinBrowser: exc.uncertain = True raise + def inbox(self, alias: str, expected_uid: str, checkpoints: dict | None = None) -> dict: + if not UID_RE.fullmatch(expected_uid): + raise DouyinError("inbox UID is invalid") + value = self._chat_browser(alias)._evaluate(alias, inbox_expression(expected_uid, checkpoints)) + if not isinstance(value, dict): + raise DouyinError("Douyin inbox response is invalid") + return value + def message_history( self, alias: str, @@ -747,533 +905,8 @@ class DouyinBrowser: return value -class DouyinSubscription: - def __init__( - self, - browser: DouyinBrowser, - alias: str, - uid: str, - pending: list[dict] | None = None, - ) -> None: - self.browser = browser - self.alias = alias - self.uid = uid - # A stable key lets a fresh wrapper dispose a stale listener left on the - # same page. Recovery disposes the old state before installing a new one. - self.key = "__creatorhub_notice_sub_" + alias - self._connection_lock = threading.RLock() - self.queue: deque[dict] = deque(pending or []) - self.condition = threading.Condition() - self.stopped = threading.Event() - self._epoch = 0 - self._initial_boundary_pending = True - self._browser_inflight: set[str] = set() - self._browser_inflight_lock = threading.Lock() - self._recovery_lock = threading.Lock() - self._detail_pool = ThreadPoolExecutor( - max_workers=4, thread_name_prefix=f"creatorhub-notice-details-{alias}" - ) - try: - self.connection, boundary_at = self._open_listener_until_ready() - except Exception: - self._detail_pool.shutdown(wait=False, cancel_futures=True) - raise - # Events observed before the consumer establishes its baseline boundary - # are explicitly classified as baseline and must never trigger writes. - self._put( - { - "kind": "baseline", - "reason": "listener_start", - "uid": self.uid, - "boundary_at": boundary_at, - "boundary_source": "douyin_identity_extra_now", - } - ) - self.thread = threading.Thread( - target=self._run, name=f"creatorhub-notices-{alias}", daemon=True - ) - self.thread.start() - - def _open_listener_until_ready(self) -> tuple[CDPConnection, str]: - deadline = time.monotonic() + LISTENER_START_TIMEOUT - delay = 0.5 - while True: - try: - return self._open_listener() - except LISTENER_ERRORS as exc: - if time.monotonic() >= deadline: - raise - LOG.info( - "Douyin notification listener is not ready; retrying", - extra={"alias": self.alias, "uid": self.uid, "reason": str(exc)}, - ) - if self.stopped.wait(delay): - raise DouyinError("Douyin notification listener stopped") from exc - delay = min(delay * 2, 5.0) - - def _open_listener(self) -> tuple[CDPConnection, str]: - connection = self.browser._connect(self.alias) - try: - result = connection.evaluate(install_expression(self.key, self.uid)) - if not isinstance(result, str): - raise DouyinError("notification listener returned invalid state") - state = json.loads(result) - if not isinstance(state, dict) or not state.get("connected"): - raise DouyinError("Douyin notification connection is not ready") - identity = self.browser.identity(self.alias, self.uid) - boundary_at = identity.get("platform_now") - if not isinstance(boundary_at, str) or not boundary_at: - raise DouyinError("Douyin platform event boundary is unavailable") - return connection, boundary_at - except LISTENER_ERRORS: - try: - connection.evaluate(dispose_expression(self.key)) - except LISTENER_ERRORS: - LOG.debug( - "failed to dispose a partially installed Douyin listener", - extra={"alias": self.alias, "uid": self.uid}, - exc_info=True, - ) - finally: - connection.close() - raise - - def _get_connection(self) -> CDPConnection: - with self._connection_lock: - return self.connection - - def _run(self) -> None: - wait = wait_expression(self.key) - while not self.stopped.is_set(): - try: - raw = self._get_connection().evaluate(wait) - events = json.loads(raw) if isinstance(raw, str) else raw - if not isinstance(events, list): - raise DouyinError("notification listener returned invalid events") - initial_boundary = self._initial_boundary_pending - self._initial_boundary_pending = False - for event in events: - if ( - initial_boundary - and isinstance(event, dict) - and event.get("kind") == "push" - ): - event = dict(event) - event["baseline"] = True - self._handle(event) - except ( - DouyinError, - OSError, - TypeError, - ValueError, - websocket.WebSocketException, - ) as exc: - if self.stopped.is_set(): - break - self._put({"kind": "error", "reason": str(exc)}) - self._request_recovery() - self._dispose_current() - - def _request_recovery(self) -> None: - if not self._recovery_lock.acquire(blocking=False): - return - try: - self._recover() - finally: - self._recovery_lock.release() - - def _recover(self) -> None: - # Dispose the old same-page handlers before installing a new state with - # the stable key; disposing afterward would remove the fresh handlers. - with self._connection_lock: - old = self.connection - try: - old.evaluate(dispose_expression(self.key)) - except (DouyinError, OSError, websocket.WebSocketException): - LOG.debug( - "old notification listener disposal was not available", - exc_info=True, - ) - finally: - old.close() - self._initial_boundary_pending = True - delay = 0.5 - while not self.stopped.is_set(): - if self.stopped.wait(delay): - return - try: - connection, boundary_at = self._open_listener() - except ( - DouyinError, - OSError, - TypeError, - ValueError, - websocket.WebSocketException, - ) as exc: - self._put({"kind": "error", "reason": str(exc)}) - delay = min(delay * 2, 30.0) - continue - with self._connection_lock: - self.connection = connection - self._epoch += 1 - self._put({"kind": "reconnected", "uid": self.uid}) - # The boundary is a separate event so the control plane never treats - # a transport reconnect itself as proof that new notices are safe. - self._put( - { - "kind": "baseline", - "reason": "listener_reconnected", - "uid": self.uid, - "boundary_at": boundary_at, - "boundary_source": "douyin_identity_extra_now", - } - ) - return - - def _dispose_current(self) -> None: - connection = self._get_connection() - try: - connection.evaluate(dispose_expression(self.key)) - except (DouyinError, OSError, websocket.WebSocketException): - LOG.debug("notification listener disposal was not available", exc_info=True) - finally: - connection.close() - - def _handle(self, event: object) -> None: - if not isinstance(event, dict): - raise DouyinError("notification event is invalid") - kind = event.get("kind") - delivery_id = event.get("delivery_id") - if kind in {"open", "close"}: - self._put({"kind": kind}) - if delivery_id and hasattr(self, "_connection_lock"): - self._ack_browser_event(delivery_id, self._get_connection()) - if kind == "close": - raise DouyinError("Douyin notification connection closed") - return - if kind == "error": - self._put( - { - "kind": "error", - "reason": str(event.get("reason", "notification continuity gap")), - "continuity": "gap", - } - ) - if delivery_id and hasattr(self, "_connection_lock"): - self._ack_browser_event(delivery_id, self._get_connection()) - return - if kind != "push": - raise DouyinError("notification event kind is invalid") - event = dict(event) - event.setdefault("gateway_received_at", datetime.now(timezone.utc).isoformat()) - if not hasattr(self, "_browser_inflight_lock"): - self._browser_inflight_lock = threading.Lock() - if not hasattr(self, "_browser_inflight"): - self._browser_inflight = set() - if isinstance(delivery_id, str) and delivery_id: - with self._browser_inflight_lock: - if delivery_id in self._browser_inflight: - return - self._browser_inflight.add(delivery_id) - epoch = getattr(self, "_epoch", 0) - baseline = bool(event.get("baseline")) - if hasattr(self, "_detail_pool"): - self._detail_pool.submit(self._process_push_async, event, epoch, baseline) - else: - success = self._process_push(event, self._get_connection(), baseline) - if delivery_id: - if success: - self._ack_browser_event(delivery_id, self._get_connection()) - else: - if not self._retry_browser_event( - delivery_id, self._get_connection() - ): - self._request_recovery() - if delivery_id: - with self._browser_inflight_lock: - self._browser_inflight.discard(delivery_id) - - def _handle_push_error( - self, - error: Exception, - delivery_id: object, - connection: CDPConnection | None, - ) -> None: - self._put({"kind": "error", "reason": str(error), "continuity": "gap"}) - if not isinstance(delivery_id, str): - return - if connection is None or not self._retry_browser_event(delivery_id, connection): - self._request_recovery() - - def _process_push( - self, event: dict, connection: CDPConnection, baseline: bool - ) -> bool: - try: - ids = notice_ids(event) - except ( - DouyinError, - KeyError, - TypeError, - ValueError, - websocket.WebSocketException, - ) as exc: - self._put({"kind": "error", "reason": str(exc), "continuity": "gap"}) - return False - # Detail lookup is isolated per platform notification. A malformed or - # temporarily unavailable detail must not discard its siblings. - complete = True - for notice_id in ids: - try: - for notice in self._details([notice_id], connection): - normalized = normalize_notice( - notice, event.get("gateway_received_at") - ) - if normalized is not None: - self._put( - { - "kind": "notice", - "notice": normalized, - "baseline": baseline, - } - ) - except ( - DouyinError, - OSError, - TypeError, - ValueError, - websocket.WebSocketException, - ) as exc: - self._put( - { - "kind": "error", - "reason": str(exc), - "event_key": notice_id, - "continuity": "gap", - } - ) - complete = False - return complete - - def _process_push_async(self, event: dict, epoch: int, baseline: bool) -> None: - connection: CDPConnection | None = None - success = False - delivery_id = event.get("delivery_id") - try: - connection = self.browser._connect(self.alias) - success = self._process_push( - event, - connection, - baseline or epoch != getattr(self, "_epoch", 0), - ) - if isinstance(delivery_id, str): - if success: - self._ack_browser_event(delivery_id, connection) - else: - # A malformed detail is a continuity gap, not a retryable - # delivery. Drop the bad delivery, record the gap, and - # establish a new baseline before accepting later events. - self._ack_browser_event(delivery_id, connection) - self._request_recovery() - except LISTENER_ERRORS as exc: - self._handle_push_error(exc, delivery_id, connection) - finally: - if connection is not None: - connection.close() - if isinstance(delivery_id, str): - with self._browser_inflight_lock: - self._browser_inflight.discard(delivery_id) - - def _ack_browser_event( - self, delivery_id: object, connection: CDPConnection - ) -> None: - if not isinstance(delivery_id, str) or not delivery_id: - return - try: - result = connection.evaluate(ack_expression(self.key, [delivery_id])) - if result not in (True, "true"): - raise DouyinError("notification acknowledgement failed") - except ( - DouyinError, - OSError, - TypeError, - ValueError, - websocket.WebSocketException, - ): - LOG.warning("notification acknowledgement failed", exc_info=True) - - def _retry_browser_event( - self, delivery_id: object, connection: CDPConnection - ) -> bool: - if not isinstance(delivery_id, str) or not delivery_id: - return False - try: - result = connection.evaluate(retry_expression(self.key, [delivery_id])) - if result not in (True, "true"): - raise DouyinError("notification retry acknowledgement failed") - return True - except ( - DouyinError, - OSError, - TypeError, - ValueError, - websocket.WebSocketException, - ): - LOG.warning("notification retry marker failed", exc_info=True) - return False - - def pending(self) -> list[dict]: - with self.condition: - return list(self.queue) - - def _details( - self, ids: list[str], connection: CDPConnection | None = None - ) -> list[dict]: - expected = set(ids) - connection = connection or self._get_connection() - for attempt in range(2): - raw = connection.evaluate(details_expression(ids)) - if not isinstance(raw, str): - raise DouyinError("notification details returned invalid data") - try: - response = json.loads(raw) - body = json.loads(response["body"]) - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: - raise DouyinError("notification details could not be parsed") from exc - if ( - not isinstance(response, dict) - or not isinstance(body, dict) - or response.get("status") != 200 - or body.get("status_code") != 0 - ): - raise DouyinError("notification details request failed") - notices = body.get("notice_list_v2") - if not isinstance(notices, list): - raise DouyinError("notification details list is invalid") - normalized: list[dict] = [] - found: set[str] = set() - for notice in notices: - if ( - not isinstance(notice, dict) - or str(notice.get("user_id")) != self.uid - ): - raise DouyinError("notification identity changed") - nid = str(notice.get("nid_str") or notice.get("nid") or "") - if not nid.isascii() or not nid.isdecimal(): - raise DouyinError("notification ID is invalid") - found.add(nid) - normalized.append(notice) - if found == expected: - return normalized - if attempt == 0: - time.sleep(0.2) - continue - raise DouyinError("notification details are incomplete") - raise DouyinError("notification details are incomplete") - - def _put(self, item: dict) -> None: - item = dict(item) - item.setdefault("delivery_id", uuid.uuid4().hex) - if item.get("kind") == "notice": - notice = item.get("notice") - if isinstance(notice, dict): - notice.setdefault( - "gateway_received_at", datetime.now(timezone.utc).isoformat() - ) - with self.condition: - if len(self.queue) >= 1000: - # Keep the backlog visible; replace only one oldest item with a - # gap marker instead of silently dropping the entire queue. - dropped = getattr(self, "_overflow_count", 0) + 1 - self._overflow_count = dropped - self.queue[0] = { - "kind": "error", - "reason": "notification queue overflow", - "continuity": "gap", - "dropped": dropped, - "delivery_id": uuid.uuid4().hex, - } - else: - self.queue.append(item) - self.condition.notify_all() - - def poll(self, limit: int, wait_seconds: float) -> list[dict]: - deadline = time.monotonic() + wait_seconds - with self.condition: - while not self.queue and not self.stopped.is_set(): - remaining = deadline - time.monotonic() - if remaining <= 0: - break - self.condition.wait(remaining) - return list(self.queue)[:limit] - - def ack(self, delivery_ids: list[str]) -> None: - ids = {value for value in delivery_ids if isinstance(value, str) and value} - if not ids: - return - with self.condition: - self.queue = deque( - item for item in self.queue if item.get("delivery_id") not in ids - ) - - def stop(self) -> None: - self.stopped.set() - with self._connection_lock: - connection = self.connection - connection.close() - with self.condition: - self.condition.notify_all() - self.thread.join(timeout=2.0) - if self.thread.is_alive(): - LOG.error( - "notification listener did not stop within timeout", - extra={"alias": self.alias}, - ) - pool = getattr(self, "_detail_pool", None) - if pool is not None: - pool.shutdown(wait=True, cancel_futures=True) -class SubscriptionManager: - def __init__(self, browser: DouyinBrowser) -> None: - self.browser = browser - self._lock = threading.RLock() - self._items: dict[str, DouyinSubscription] = {} - - def start(self, alias: str, uid: str) -> dict: - with self._lock: - previous = self._items.pop(alias, None) - pending = previous.pending() if previous else [] - if previous: - previous.stop() - item = DouyinSubscription(self.browser, alias, uid, pending=pending) - self._items[alias] = item - return {"connected": True, "alias": alias, "uid": uid} - - def poll(self, alias: str, limit: int, wait_seconds: float) -> list[dict]: - with self._lock: - item = self._items.get(alias) - if not item: - raise DouyinError("notification listener is not running") - return item.poll(limit, wait_seconds) - - def ack(self, alias: str, delivery_ids: list[str]) -> None: - with self._lock: - item = self._items.get(alias) - if not item: - raise DouyinError("notification listener is not running") - item.ack(delivery_ids) - - def stop(self, alias: str) -> None: - with self._lock: - item = self._items.pop(alias, None) - if item: - item.stop() - - def close(self) -> None: - with self._lock: - aliases = list(self._items) - for alias in aliases: - self.stop(alias) def _notice_id(value: object) -> str: @@ -1284,41 +917,49 @@ def _notice_id(value: object) -> str: return "" -def _direct_message_type(detail: dict, message: object) -> str: - raw_type = ( - detail.get("message_type") or detail.get("msg_type") or detail.get("type") - ) - if isinstance(message, dict): - raw_type = raw_type or message.get("type") or message.get("message_type") - if isinstance(raw_type, int) and not isinstance(raw_type, bool): - raw_type = {1: "text", 2: "image", 3: "voice", 4: "video", 5: "sticker"}.get( - raw_type - ) - if isinstance(raw_type, str): - normalized = raw_type.strip().lower() - aliases = { - "text": "text", - "txt": "text", - "image": "image", - "img": "image", - "picture": "image", - "voice": "voice", - "audio": "voice", - "video": "video", - "sticker": "sticker", - "emoji": "sticker", - } - if normalized in aliases: - return aliases[normalized] - if any(detail.get(key) for key in ("voice", "audio", "audio_url")): - return "voice" - if any(detail.get(key) for key in ("image", "picture", "image_url")): - return "image" - if any(detail.get(key) for key in ("video", "video_url")): - return "video" - if isinstance(message, str) and message: - return "text" - return "non_text" + + +def _notice_display_details(users: list, work: dict, uid: str, work_id: str) -> dict: + def text(value, label): + if value is None: + return "" + if not isinstance(value, str) or len(value) > 8192: + raise DouyinError(f"notification {label} is invalid") + return value + + user = next((u for u in users if isinstance(u, dict) and uid and + _notice_id(u.get("uid") or u.get("user_id")) == uid), {}) + video = work.get("video") or {} + if not isinstance(video, dict): + raise DouyinError("notification video is invalid") + cover = video.get("cover") or {} + if not isinstance(cover, dict): + raise DouyinError("notification cover is invalid") + urls = cover.get("url_list") or [] + images = work.get("images") or [] + if not isinstance(images, list): + raise DouyinError("notification images are invalid") + if not urls and images: + image = images[0] + if not isinstance(image, dict): + raise DouyinError("notification image is invalid") + display = image.get("display_image") or image + if not isinstance(display, dict): + raise DouyinError("notification image is invalid") + urls = display.get("url_list") or [] + if not isinstance(urls, list): + raise DouyinError("notification cover URLs are invalid") + author = work.get("author") or {} + if not isinstance(author, dict): + raise DouyinError("notification work author is invalid") + route = "note" if work.get("aweme_type") == 68 else "video" + return { + "interactor_name": text(user.get("nickname"), "user nickname"), + "interactor_sec_uid": text(user.get("sec_uid"), "user secUID"), + "work_author_uid": _notice_id(author.get("uid") or author.get("user_id")), + "work_cover_url": text(urls[0], "cover URL") if urls else "", + "work_url": f"{ORIGIN_URL.rstrip('/')}/{route}/{work_id}" if work_id else "", + } def normalize_notice( @@ -1334,74 +975,11 @@ def normalize_notice( kind, event_type = "digg", "like" elif notice.get("share"): kind, event_type = "share", "repost" - elif ( - notice.get("dm") - or notice.get("message") - or notice.get("im") - or notice.get("chat") - ): - kind, event_type = next( - (candidate, "dm") - for candidate in ("dm", "message", "im", "chat") - if notice.get(candidate) - ) else: return None detail = notice.get(kind) if not isinstance(detail, dict): raise DouyinError("notification detail payload is invalid") - if event_type == "dm": - sender = ( - detail.get("from_user") or detail.get("sender") or detail.get("user") or {} - ) - if isinstance(sender, list): - sender = sender[0] if sender else {} - if not isinstance(sender, dict): - raise DouyinError("direct-message sender is invalid") - message = ( - detail.get("text") - or detail.get("content") - or detail.get("message") - or notice.get("text") - or "" - ) - message_type = _direct_message_type(detail, message) - if isinstance(message, dict): - message = message.get("text") or message.get("content") or "" - if not isinstance(message, str) or len(message) > 100000: - raise DouyinError("direct-message text is invalid") - event_key = _notice_id( - notice.get("message_id") - or notice.get("msg_id") - or detail.get("message_id") - or detail.get("msg_id") - or notice.get("nid_str") - or notice.get("nid") - ) - if not event_key: - raise DouyinError("direct-message ID is invalid") - result = { - "event_key": event_key, - "event_type": "dm", - "interactor_uid": _notice_id(sender.get("uid") or sender.get("user_id")), - "comment_id": "", - "work_id": "", - "message_type": message_type, - "message_text": message, - } - create_time = notice.get("create_time") - if isinstance(create_time, (int, float)) and not isinstance(create_time, bool): - try: - timestamp = float(create_time) - if math.isfinite(timestamp) and timestamp > 0: - result["platform_event_at"] = datetime.fromtimestamp( - timestamp, timezone.utc - ).isoformat() - except (OverflowError, OSError, ValueError) as exc: - raise DouyinError("notification timestamp is invalid") from exc - if isinstance(gateway_received_at, str) and gateway_received_at: - result["gateway_received_at"] = gateway_received_at - return result users = detail.get("from_user") or [] if isinstance(users, dict): users = [users] @@ -1458,6 +1036,14 @@ def normalize_notice( "comment_id": _notice_id(comment_id), "work_id": _notice_id(work_id), } + result.update(_notice_display_details(users, work, result["interactor_uid"], result["work_id"])) + if event_type == "comment": + comment_text = comment.get("text", "") + if not isinstance(comment_text, str) or len(comment_text) > 100000: + raise DouyinError("notification comment text is invalid") + if comment_text: + result["message_text"] = comment_text + result["message_type"] = "text" create_time = notice.get("create_time") if isinstance(create_time, (int, float)) and not isinstance(create_time, bool): try: @@ -1655,31 +1241,14 @@ def notice_ids(event: dict) -> list[str]: return list(dict.fromkeys(ids)) -def install_expression(key: str, uid: str) -> str: - return INSTALL_SCRIPT.replace("KEY_VALUE", json.dumps(key)).replace( - "UID_VALUE", json.dumps(uid) - ) -def wait_expression(key: str) -> str: - return WAIT_SCRIPT.replace("KEY_VALUE", json.dumps(key)) def dispose_expression(key: str) -> str: return f"(() => {{ const key={json.dumps(key)}; window[key]?.dispose(); delete window[key]; return true; }})()" -def ack_expression(key: str, ids: list[str]) -> str: - if ( - not isinstance(key, str) - or not key - or not ids - or any(not isinstance(value, str) or not value for value in ids) - ): - raise ValueError("notification delivery IDs are invalid") - return ACK_SCRIPT.replace("KEY_VALUE", json.dumps(key)).replace( - "IDS_VALUE", json.dumps(ids) - ) def retry_expression(key: str, ids: list[str]) -> str: @@ -1714,6 +1283,14 @@ def im_expression(params: dict, expected_uid: str) -> str: ) +def inbox_expression(expected_uid: str, checkpoints: dict | None = None) -> str: + return INBOX_SCRIPT.replace("EXPECTED_UID_VALUE", json.dumps(expected_uid)).replace("CHECKPOINTS_VALUE", json.dumps(checkpoints or {})).replace("INBOX_MODE_VALUE", '"inbox"') + + +def private_identity_expression(expected_uid: str) -> str: + return INBOX_SCRIPT.replace("EXPECTED_UID_VALUE", json.dumps(expected_uid)).replace("CHECKPOINTS_VALUE", "{}").replace("INBOX_MODE_VALUE", '"identity"') + + def message_history_expression(params: dict, expected_uid: str) -> str: return MESSAGE_HISTORY_SCRIPT.replace( "EXPECTED_UID_VALUE", json.dumps(expected_uid) @@ -1724,20 +1301,6 @@ def action_expression(params: dict) -> str: return ACTION_SCRIPT.replace("PARAMS_VALUE", json.dumps(params, ensure_ascii=True)) -INSTALL_SCRIPT = r"""(async()=>{try{ - if(location.origin!=="https://www.douyin.com")throw Error("WRONG_ORIGIN"); - const key=KEY_VALUE,uid=UID_VALUE,chunks=window.webpackChunkdouyin_web;if(!chunks)throw Error("RUNTIME_MISSING");const old=window[key],carry=Array.isArray(old?.queue)?old.queue.filter(e=>e&&e.delivery_id):[];old?.dispose?.();delete window[key];let req; - chunks.push([["creatorhub-notice-"+Date.now()],{},r=>{req=r;}]);chunks.pop(); - const entries=Object.entries(req.m||{});const entry=entries.find(([,f])=>String(f).includes("NOTICE_PUSH_EVENT_NAMES:function")); - const codec=entries.find(([,f])=>{const s=String(f);return s.includes(".decodedFrame=")&&s.includes(".encodeFrame=");}); - if(!entry||!codec)throw Error("SDK_CHANGED");const C=req(entry[0]).NoticeFrontier,decode=req(codec[0]).decodedFrame,f=C.frontierInstance; - if(!f||String(f._options.deviceID)!==uid)throw Error("SOCKET_NOT_READY"); - const state={queue:carry,wake:null,C,f,uid,dropped:0};const delivery=()=>"browser-"+crypto.randomUUID();const emit=e=>{const item={...e,delivered:false,delivery_id:delivery()};if(state.queue.length>=1000){state.queue.shift();state.dropped++;state.queue.push({kind:"error",reason:"QUEUE_OVERFLOW",continuity:"gap",dropped:state.dropped,delivered:false,delivery_id:delivery()});}else state.queue.push(item);if(state.wake)state.wake();}; - const message=e=>{try{const frame=decode(new Uint8Array(e.data));if(frame.service===20313||frame.service===20003)emit({kind:"push",service:frame.service,payload:new TextDecoder().decode(frame.payload)});}catch(_){emit({kind:"error",reason:"FRAME_DECODE_FAILED",continuity:"gap"});}}; - const open=()=>emit({kind:"open"}),close=()=>emit({kind:"close"});f.addEventListener("message",message);f.addEventListener("open",open);f.addEventListener("close",close); - state.dispose=()=>{f.removeEventListener("message",message);f.removeEventListener("open",open);f.removeEventListener("close",close);if(state.wake)state.wake();};window[key]=state; - return JSON.stringify({connected:f.readyState===f.OPEN}); -}catch(e){return JSON.stringify({bridge_error:String(e.message||e)});}})()""" WAIT_SCRIPT = r"""(async()=>{const s=window[KEY_VALUE];if(!s||s.C.frontierInstance!==s.f||String(s.f._options.deviceID)!==s.uid)throw Error("LISTENER_INVALID"); @@ -1762,4 +1325,47 @@ ACTION_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;let sent=false;const fail=(c MESSAGE_HISTORY_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;const fail=code=>{throw Error(code);};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");const response=await fetch("/aweme/v1/web/user/profile/self/?device_platform=webapp&aid=6383",{credentials:"include",signal:AbortSignal.timeout(15000)});if(!response.ok)fail("LOGIN_CHECK_FAILED");const profile=await response.json();if(profile.status_code!==0||String(profile.user?.uid)!==EXPECTED_UID_VALUE)fail("IDENTITY_MISMATCH");if(String(profile.user.uid)===p.uid)fail("SELF_TARGET");let service;for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){const chunks=window[key];if(!Array.isArray(chunks))continue;let req;chunks.push([["creatorhub-history-"+Date.now()],{},r=>{req=r;}]);chunks.pop();for(const [id,module] of Object.entries(req?.c||{})){if(!String(req.m?.[id]||"").includes("getOrCreatePrivateConversationByUid"))continue;for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService;}if(service)break;}if(!service)fail("IM_SDK_NOT_READY");const sdk=service.imSdkManager.getImSdkInstance();if(!sdk)fail("IM_SDK_NOT_READY");const conversation=sdk.getConversationList().find(c=>c.type===1&&String(c.toParticipantUserId)===p.uid);if(!conversation)fail("CONVERSATION_NOT_FOUND");const meta=c=>({id:String(c.id),short_id:String(c.shortId),uid:String(c.toParticipantUserId),type:c.type});let cursor=p.cursor?String(p.cursor):undefined,previousCursor=cursor??"",hasMore=false;for(let page=0;page<20;page++){const request={conversation,limit:Math.min(50,p.limit)};if(cursor!==undefined)request.cursor=cursor;const result=await sdk.getMessagesByConversation(request);if(!result||!Array.isArray(result.messages))fail("MESSAGE_HISTORY_INVALID");hasMore=Boolean(result.hasMore);if(!hasMore)break;const next=result.cursor,key=String(next?.toString?.()??next??"");if(!key||key===previousCursor)fail("MESSAGE_HISTORY_CURSOR_INVALID");previousCursor=key;cursor=next;}const messages=conversation.getMessageList();if(!Array.isArray(messages))fail("MESSAGE_HISTORY_INVALID");const pack=m=>{const ext=m.ext||{};const rawTime=ext["s:server_message_create_time"]||"";return {server_id:String(m.serverId||""),sender_uid:String(m.sender||""),message_type:String(m.type??""),content:typeof m.content==="string"?m.content.slice(0,100000):m.content,created_at:/^[0-9]+$/.test(String(rawTime))?String(rawTime):null,server_status:m.serverStatus??null};};return {status:"succeeded",action:"history",history_source:"im_sdk_pull",history_cursor:hasMore?String(cursor??""):"",history_has_more:hasMore,account_uid:String(profile.user.uid),conversation:meta(conversation),messages:messages.slice(-p.limit).map(pack)};}catch(e){const known=["WRONG_ORIGIN","LOGIN_CHECK_FAILED","IDENTITY_MISMATCH","SELF_TARGET","IM_SDK_NOT_READY","CONVERSATION_NOT_FOUND","MESSAGE_HISTORY_INVALID","MESSAGE_HISTORY_CURSOR_INVALID"];return {status:"failed",code:known.includes(e.message)?e.message:"SDK_REQUEST_FAILED"};}})()""" -IM_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;let sent=false;const fail=(code,definitive=false)=>{const e=Error(code);e.definitive=definitive;throw e;};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");const response=await fetch("/aweme/v1/web/user/profile/self/?device_platform=webapp&aid=6383",{credentials:"include",signal:AbortSignal.timeout(15000)});if(!response.ok)fail("LOGIN_CHECK_FAILED");const profile=await response.json();if(profile.status_code!==0||String(profile.user?.uid)!==EXPECTED_UID_VALUE)fail("IDENTITY_MISMATCH");if(String(profile.user.uid)===p.uid)fail("SELF_TARGET");let service;for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){const chunks=window[key];if(!Array.isArray(chunks))continue;let req;chunks.push([["creatorhub-im-"+Date.now()],{},r=>{req=r;}]);for(const [id,module] of Object.entries(req?.c||{})){if(!String(req.m[id]).includes("getOrCreatePrivateConversationByUid"))continue;for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService;}}if(!service)fail("IM_SDK_NOT_READY");const sdk=service.imSdkManager.getImSdkInstance();if(!sdk)fail("IM_SDK_NOT_READY");const meta=c=>({id:String(c.id),short_id:String(c.shortId),uid:String(c.toParticipantUserId),type:c.type});const pack=m=>({server_id:String(m.serverId||""),client_id:m.clientId||null,sender:String(m.sender),type:m.type,content:m.content,created_at:m.createdAt,server_status:m.serverStatus});let conversation=sdk.getConversationList().find(c=>c.type===1&&String(c.toParticipantUserId)===p.uid);if(!conversation&&p.action==="send")conversation=await service.conversationManager.getOrCreatePrivateConversationByUid(p.uid);if(!conversation)fail("CONVERSATION_NOT_FOUND");if(conversation.type!==1||String(conversation.toParticipantUserId)!==p.uid)fail("TARGET_MISMATCH");if(p.action!=="send"||!p.confirm)return {status:"preview",action:"preview",sender_uid:String(profile.user.uid),uid:p.uid,text:p.text,conversation:meta(conversation)};const message=await sdk.createMessage({conversation,type:7,content:JSON.stringify({aweType:700,type:0,richTextInfos:[],text:p.text})});if(!message||typeof message.sendFunc!=="function")fail("MESSAGE_BUILD_FAILED");sent=true;const result=await Promise.race([sdk.sendMessage({message}),new Promise((_,reject)=>setTimeout(()=>reject(Error("MESSAGE_UNCONFIRMED")),10000))]);if(result?.success===false)fail("MESSAGE_REJECTED",true);if(result?.success!==true)fail("MESSAGE_UNCONFIRMED");const packed=pack(message);if(!packed.server_id&&!packed.client_id)fail("MESSAGE_UNCONFIRMED");return {status:"succeeded",action:"send",success:true,status_code:result?.statusCode??null,check_code:String(result?.checkCode??""),conversation:meta(conversation),message:packed,evidence:{conversation_id:String(conversation.id),message_server_id:packed.server_id,message_client_id:String(packed.client_id??"")}};}catch(e){const known=["WRONG_ORIGIN","LOGIN_CHECK_FAILED","IDENTITY_MISMATCH","SELF_TARGET","IM_SDK_NOT_READY","CONVERSATION_NOT_FOUND","TARGET_MISMATCH","MESSAGE_BUILD_FAILED","MESSAGE_REJECTED","MESSAGE_UNCONFIRMED"];return {status:sent?(e.definitive?"failed":"unknown"):"failed",code:known.includes(e.message)?e.message:"SDK_REQUEST_FAILED"};}})()""" +INBOX_SCRIPT = r"""(async()=>{ + const expected=EXPECTED_UID_VALUE, checkpoints=CHECKPOINTS_VALUE, mode=INBOX_MODE_VALUE; + const fail=code=>{throw Error(code);}; + try { + if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN"); + let service; + for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){ + const chunks=window[key];if(!Array.isArray(chunks))continue;let req; + chunks.push([["creatorhub-inbox-"+crypto.randomUUID()],{},r=>{req=r;}]);chunks.pop(); + for(const [id,module] of Object.entries(req?.c||{})){ + if(!String(req.m?.[id]||"").includes("getOrCreatePrivateConversationByUid"))continue; + for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService; + } + if(service)break; + } + const sdk=service?.imSdkManager?.getImSdkInstance();if(!sdk||sdk.disposed)fail("IM_SDK_NOT_READY"); + if(String(sdk.ctx?.option?.userId||"")!==expected)fail("ACCOUNT_IDENTITY_MISMATCH"); + if(mode==="identity")return {status:"logged_in",uid:expected}; + const conversations=sdk.getConversationList().filter(c=>c.type===1); + if(conversations.length>100)fail("IM_INBOX_LIMIT_EXCEEDED"); + const messages=[],next={}; + for(const conversation of conversations){ + const peer=String(conversation.toParticipantUserId||""); + if(!/^[1-9][0-9]{0,19}$/.test(peer)||peer===expected)fail("CONVERSATION_IDENTITY_INVALID"); + const latest=String(conversation.lastMessage?.serverId||"");next[peer]=latest; + if(latest&&checkpoints[peer]===latest)continue; + const list=conversation.getMessageList();if(!Array.isArray(list))fail("MESSAGE_HISTORY_INVALID"); + for(const m of list.slice(-50)){ + if(m.type===1||m.type===2)continue; + const server=String(m.serverId||"");if(!/^[1-9][0-9]{0,19}$/.test(server))continue; + const sender=String(m.sender||m.ext?.["s:sender_uid"]||""); + if(sender!==peer&&sender!==expected)fail("MESSAGE_IDENTITY_INVALID"); + const rawTime=m.ext?.["s:server_message_create_time"]; + messages.push({peer_uid:peer,peer_name:String(conversation.coreInfo?.name||""),server_id:server,client_id:String(m.clientId||""),sender_uid:sender,message_type:String(m.type),content:m.content,created_at:/^[0-9]+$/.test(String(rawTime||""))?String(rawTime):null}); + } + } + return {status:"succeeded",account_uid:expected,messages,checkpoints:next}; + }catch(e){ + if(mode==="identity"&&e.message==="IM_SDK_NOT_READY")return {status:"manual_login",reason:"awaiting_login"}; + return {status:"failed",code:String(e.message||"IM_INBOX_FAILED")}; + } +})()""" + +IM_SCRIPT = r"""(async()=>{const p=PARAMS_VALUE;let sent=false;const fail=(code,definitive=false)=>{const e=Error(code);e.definitive=definitive;throw e;};try{if(location.origin!=="https://www.douyin.com")fail("WRONG_ORIGIN");let service;for(const key of Object.keys(window).filter(k=>k.startsWith("@pc-im/im:"))){const chunks=window[key];if(!Array.isArray(chunks))continue;let req;chunks.push([["creatorhub-im-"+Date.now()],{},r=>{req=r;}]);for(const [id,module] of Object.entries(req?.c||{})){if(!String(req.m[id]).includes("getOrCreatePrivateConversationByUid"))continue;for(const exported of Object.values(module.exports||{}))if(exported?.instance?.imSdkService)service=exported.instance.imSdkService;}}if(!service)fail("IM_SDK_NOT_READY");const sdk=service.imSdkManager.getImSdkInstance();if(!sdk||sdk.disposed)fail("IM_SDK_NOT_READY");const sender=String(sdk.ctx?.option?.userId||"");if(sender!==EXPECTED_UID_VALUE)fail("IDENTITY_MISMATCH");if(sender===p.uid)fail("SELF_TARGET");const meta=c=>({id:String(c.id),short_id:String(c.shortId),uid:String(c.toParticipantUserId),type:c.type});const pack=m=>({server_id:String(m.serverId||""),client_id:m.clientId||null,sender:String(m.sender),type:m.type,content:m.content,created_at:m.createdAt,server_status:m.serverStatus});let conversation=sdk.getConversationList().find(c=>c.type===1&&String(c.toParticipantUserId)===p.uid);if(!conversation&&p.action==="send")conversation=await service.conversationManager.getOrCreatePrivateConversationByUid(p.uid);if(!conversation)fail("CONVERSATION_NOT_FOUND");if(conversation.type!==1||String(conversation.toParticipantUserId)!==p.uid)fail("TARGET_MISMATCH");if(p.action!=="send"||!p.confirm)return {status:"preview",action:"preview",sender_uid:sender,uid:p.uid,text:p.text,conversation:meta(conversation)};const message=await sdk.createMessage({conversation,type:7,content:JSON.stringify({aweType:700,type:0,richTextInfos:[],text:p.text})});if(!message||typeof message.sendFunc!=="function")fail("MESSAGE_BUILD_FAILED");sent=true;const result=await Promise.race([sdk.sendMessage({message}),new Promise((_,reject)=>setTimeout(()=>reject(Error("MESSAGE_UNCONFIRMED")),10000))]);if(result?.success===false){const network=Number(result.statusCode)===1008;const definitive=!network&&result.checkCode!=null&&String(result.checkCode)!=="0"&&String(result.checkCode)!=="";const e=Error(network?"SDK_NETWORK_ERROR":definitive?"MESSAGE_REJECTED":"SDK_SEND_UNCONFIRMED");e.definitive=definitive;e.detail={status_code:result.statusCode??null,check_code:String(result.checkCode??""),check_message:String(result.checkMessage??result.msg??""),send_status:result.status??null};throw e;}if(result?.success!==true)fail("MESSAGE_UNCONFIRMED");const packed=pack(message);if(!packed.server_id&&!packed.client_id)fail("MESSAGE_UNCONFIRMED");return {status:"succeeded",action:"send",success:true,status_code:result?.statusCode??null,check_code:String(result?.checkCode??""),conversation:meta(conversation),message:packed,evidence:{conversation_id:String(conversation.id),message_server_id:packed.server_id,message_client_id:String(packed.client_id??"")}};}catch(e){const known=["WRONG_ORIGIN","LOGIN_CHECK_FAILED","IDENTITY_MISMATCH","SELF_TARGET","IM_SDK_NOT_READY","CONVERSATION_NOT_FOUND","TARGET_MISMATCH","MESSAGE_BUILD_FAILED","MESSAGE_REJECTED","SDK_NETWORK_ERROR","SDK_SEND_UNCONFIRMED","MESSAGE_UNCONFIRMED"];return {status:sent?(e.definitive?"failed":"unknown"):"failed",code:known.includes(e.message)?e.message:"SDK_REQUEST_FAILED",...(e.detail||{})};}})()""" diff --git a/browser_gateway/platform/notice_polling.py b/browser_gateway/platform/notice_polling.py new file mode 100644 index 0000000..570c5db --- /dev/null +++ b/browser_gateway/platform/notice_polling.py @@ -0,0 +1,209 @@ +"""Read-only notification-list polling. Checkpoints advance only after DB ACK.""" +from collections import Counter, deque +from datetime import datetime, timezone +import json +import logging +import math +import threading +import time +import uuid +from urllib.parse import urlencode + +from .douyin import DouyinError, _notice_id, normalize_notice + +LOG = logging.getLogger('creatorhub.gateway.notices') +GROUPS = ('700', '960', '961') +HISTORY_RECONCILE_SECONDS = 300 + + +def boundary_time(value): + try: + result = datetime.fromisoformat(value.replace('Z', '+00:00')) + if result.tzinfo is None or result.timestamp() <= 0: + raise ValueError('timezone or timestamp') + return result + except (AttributeError, TypeError, ValueError, OverflowError) as exc: + raise DouyinError('notification boundary is invalid') from exc + + +def validate_checkpoint(boundary_at, checkpoints): + if boundary_at is not None: + boundary_time(boundary_at) + if not isinstance(checkpoints, dict) or (checkpoints and boundary_at is None): + raise DouyinError('notification checkpoints require a persisted boundary') + for group, nid in checkpoints.items(): + if group not in GROUPS or not isinstance(nid, str) or not _notice_id(nid): + raise DouyinError('notification checkpoint is invalid') + + +class NoticePollingSession: + def __init__(self, browser, alias, uid, boundary_at=None, checkpoints=None): + checkpoints = {} if checkpoints is None else checkpoints + validate_checkpoint(boundary_at, checkpoints) + self.browser, self.alias, self.uid = browser, alias, uid + identity = browser.identity(alias, uid) + self.boundary_at = boundary_at or identity.get('platform_now') + self.boundary = boundary_time(self.boundary_at) + self.initial = boundary_at is None + self.checkpoints = dict(checkpoints) + self.queue = deque() + self.lock = threading.RLock() + self.stopped = False + self.last_full_scan_at = None + + def _page(self, group, cursor): + url = 'https://www.douyin.com/aweme/v1/web/notice/?' + urlencode({ + 'device_platform': 'webapp', 'aid': 6383, 'channel': 'channel_pc_web', + 'is_new_notice': 1, 'is_mark_read': 0, 'notice_group': group, + 'count': 50, 'min_time': 0, 'max_time': cursor, + }) + response = self.browser.get(self.alias, url) + try: + # Raw text, never JSON.parse in the browser: IDs exceed JS safe integers. + body = json.loads(response.body) + except (TypeError, ValueError) as exc: + raise DouyinError(f'notification group {group} response is invalid JSON') from exc + if response.status != 200 or not isinstance(body, dict) or body.get('status_code') != 0: + raise DouyinError(f'notification group {group} request failed HTTP {response.status}: {body.get("status_code") if isinstance(body, dict) else "invalid body"}') + if not isinstance(body.get('notice_list_v2'), list) or type(body.get('has_more')) is not int or body['has_more'] not in (0, 1): + raise DouyinError(f'notification group {group} list or pagination is invalid') + return body + + def _scan(self): + self.browser.identity(self.alias, self.uid) + received_at = datetime.now(timezone.utc).isoformat() + next_checkpoints = dict(self.checkpoints) + notices = {} + gaps, ignored = [], Counter() + pages = 0 + full_scan = self.last_full_scan_at is None or time.monotonic() - self.last_full_scan_at >= HISTORY_RECONCILE_SECONDS + for group in GROUPS: + cursor, found, head = 0, False, None + while True: + body = self._page(group, cursor) + pages += 1 + for raw in body['notice_list_v2']: + if not isinstance(raw, dict) or _notice_id(raw.get('user_id')) != self.uid: + raise DouyinError(f'notification group {group} identity changed') + nid = _notice_id(raw.get('nid_str') or raw.get('nid')) + if not nid: + raise DouyinError('notification ID is invalid') + if head is None: + head = nid + if nid == self.checkpoints.get(group): + found = True + # Private messages belong exclusively to the IM inbox. + if not any(raw.get(k) for k in ('comment', 'follow', 'digg', 'share')): + ignored[str(raw.get('type') or next((k for k in ('dm', 'favorite', 'collect') if raw.get(k)), 'unknown'))] += 1 + continue + timestamp = raw.get('create_time') + if not isinstance(timestamp, (int, float)) or isinstance(timestamp, bool) or not math.isfinite(timestamp) or timestamp <= 0: + raise DouyinError('notification timestamp is invalid') + normalized = normalize_notice(raw, received_at) + # The start time labels history; it never excludes messages. + notices[(normalized['event_type'], nid)] = (normalized, timestamp <= self.boundary.timestamp()) + # Full scans recover older holes, including records behind the head. + # Incremental scans still process the entire checkpoint page. + if not body['has_more'] or (found and not full_scan): + break + following = body.get('max_time') + if type(following) is not int or following <= 0 or (cursor and following >= cursor): + raise DouyinError(f'notification group {group} pagination cursor did not advance') + cursor = following + if self.checkpoints.get(group) and not found: + gaps.append(group) + if head: + next_checkpoints[group] = head + # No partial scans enter the queue. A failed group cannot advance another. + result = [] + if self.initial: + result.append({'kind': 'baseline', 'boundary_at': self.boundary_at}) + result.extend({'kind': 'notice', 'notice': n, 'baseline': baseline} + for n, baseline in notices.values()) + reason = ('通知列表未找到已保存的边界，历史完整性无法确认；分组：' + ','.join(gaps)) if gaps else '' + checked_at = datetime.now(timezone.utc).isoformat() + result.append({'kind': 'checkpoint', 'checkpoints': next_checkpoints, + 'boundary_at': self.boundary_at, 'checked_at': checked_at, + 'reason': reason, 'ignored_types': dict(ignored), + 'history_reconciled': full_scan}) + for item in result: + item['delivery_id'] = uuid.uuid4().hex + LOG.info('notification scan alias=%s uid=%s mode=%s pages=%s events=%s ignored=%s gap_groups=%s', + self.alias, self.uid, 'history_reconcile' if full_scan else 'incremental', + pages, len(notices), dict(ignored), gaps) + self.queue.extend(result) + + def poll(self, limit, wait_seconds): + # Pacing is owned by the control plane, outside the browser alias lock. + with self.lock: + if self.stopped: + raise DouyinError('notification polling stopped') + if not self.queue: + self._scan() + return list(self.queue)[:limit] + + def pending(self): + with self.lock: + return list(self.queue) + + def ack(self, delivery_ids): + ids = set(delivery_ids) + with self.lock: + for item in self.queue: + if item['delivery_id'] in ids and item['kind'] == 'checkpoint': + if any(d['delivery_id'] not in ids for d in self.queue): + raise DouyinError('notification checkpoint acknowledged before its notices') + self.checkpoints = dict(item['checkpoints']) + self.initial = False + if item['history_reconciled']: + self.last_full_scan_at = time.monotonic() + self.queue = deque(item for item in self.queue if item['delivery_id'] not in ids) + + def stop(self): + with self.lock: + self.stopped = True + self.queue.clear() + + +class SubscriptionManager: + def __init__(self, browser): + self.browser = browser + self._lock = threading.RLock() + self._items = {} + + def start(self, alias, uid, boundary_at=None, checkpoints=None): + with self._lock: + previous = self._items.get(alias) + if previous: + previous.stop() + self._items.pop(alias) + browser = self.browser._notification_browser(alias) + item = NoticePollingSession(browser, alias, uid, boundary_at, checkpoints) + self._items[alias] = item + return {'connected': True, 'alias': alias, 'uid': uid, 'mode': 'notice_polling'} + + def _get(self, alias): + with self._lock: + item = self._items.get(alias) + if item is None: + raise DouyinError('notification polling is not running') + return item + + def poll(self, alias, limit, wait_seconds): + return self._get(alias).poll(limit, wait_seconds) + + def ack(self, alias, delivery_ids): + self._get(alias).ack(delivery_ids) + + def stop(self, alias): + with self._lock: + item = self._items.get(alias) + if item: + item.stop() + self._items.pop(alias) + + def close(self): + with self._lock: + aliases = list(self._items) + for alias in aliases: + self.stop(alias) diff --git a/browser_gateway/runtime.py b/browser_gateway/runtime.py index f166c61..a105112 100644 --- a/browser_gateway/runtime.py +++ b/browser_gateway/runtime.py @@ -419,6 +419,7 @@ class RuntimeRecord: profile_dir: str browser_path: str command: list[str] + memory_limit_mb: int = 2048 network_exit_id: str = "" proxy_port: int = 0 display: int = 0 @@ -473,6 +474,7 @@ class RuntimeRecord: profile_dir=value["profile_dir"], browser_path=value["browser_path"], command=command, + memory_limit_mb=value.get("memory_limit_mb", 2048), network_exit_id=value.get("network_exit_id", ""), proxy_port=value.get("proxy_port", 0), display=value.get("display", 0), @@ -493,6 +495,7 @@ class RuntimeRecord: ) except (KeyError, TypeError, ValueError) as exc: raise BrowserRuntimeError("runtime metadata is invalid", 500) from exc + validate_memory_limit(record.memory_limit_mb) if ( not isinstance(record.alias, str) or not ALIAS_RE.fullmatch(record.alias) @@ -650,6 +653,7 @@ class NativeRuntimeManager: profile_dir=str(profile_dir), browser_path=browser_path, command=list(value["cmd"]), + memory_limit_mb=value["memory_limit_mb"], network_exit_id=value.get("network_exit_id", ""), node_id=self.node_id, owner=self.node_id, @@ -674,7 +678,9 @@ class NativeRuntimeManager: finally: self._cancel_events.pop(record.runtime_id, None) - def change_state(self, alias: str, action: str, generation: Mapping[str, Any]) -> None: + def change_state(self, alias: str, action: str, generation: Mapping[str, Any], memory_limit_mb: int | None = None) -> None: + if memory_limit_mb is not None: + validate_memory_limit(memory_limit_mb) if action == "cancel": self.cancel(alias, generation) return @@ -689,6 +695,8 @@ class NativeRuntimeManager: return if record.state != "stopped": raise BrowserRuntimeError("runtime cannot be started in its current state", 409, record.network_id) + if memory_limit_mb is not None: + record.memory_limit_mb = memory_limit_mb self._start_locked(record, ProxyExit("", "", 0)) return if record.state == "stopped": @@ -916,7 +924,7 @@ class NativeRuntimeManager: environment={"DISPLAY": f":{record.display}"}, working_directory=runtime_dir, stdout_path=browser_log, - limits={"MemoryMax": "1G", "TasksMax": "512", "CPUQuota": "200%", "TimeoutStopSec": "10s"}, + limits={"MemoryMax": f"{record.memory_limit_mb}M", "TasksMax": "512", "CPUQuota": "200%", "TimeoutStopSec": "10s"}, ) self._release_startup_locks(record) record.browser_pid = browser_status.pid @@ -1327,6 +1335,7 @@ class NativeRuntimeManager: "ready": ready, "endpoint": endpoint, "binding_version": record.binding_version, + "memory_limit_mb": record.memory_limit_mb, "network_exit_id": record.network_exit_id, "network_id": record.network_id, "node_id": record.node_id, @@ -1448,6 +1457,11 @@ class NativeRuntimeManager: lock.release() +def validate_memory_limit(value: object) -> None: + if type(value) is not int or not 512 <= value <= 65536: + raise BrowserRuntimeError("memory_limit_mb must be an integer between 512 and 65536", 400) + + def validate_runtime_input(value: Mapping[str, Any]) -> None: if not isinstance(value, Mapping): raise BrowserRuntimeError("request body must be one JSON object", 400) @@ -1460,6 +1474,7 @@ def validate_runtime_input(value: Mapping[str, Any]) -> None: "network_exit_id", "network_exit", "stopped", + "memory_limit_mb", } if set(value) - allowed: raise BrowserRuntimeError("body contains unknown fields", 400) @@ -1470,6 +1485,8 @@ def validate_runtime_input(value: Mapping[str, Any]) -> None: binding = value.get("binding_version") exit_id = value.get("network_exit_id", "") stopped = value.get("stopped", False) + memory_limit_mb = value.get("memory_limit_mb", 2048) + validate_memory_limit(memory_limit_mb) if not isinstance(alias, str) or not ALIAS_RE.fullmatch(alias): raise BrowserRuntimeError("alias must match [a-z0-9][a-z0-9-]{0,31}", 400) if not isinstance(name, str) or not 1 <= len(name) <= 64 or has_control(name): @@ -1520,6 +1537,7 @@ def validate_runtime_input(value: Mapping[str, Any]) -> None: value["profile_id"] = profile_id value["network_exit"] = exit_value value["network_exit_id"] = exit_id + value["memory_limit_mb"] = memory_limit_mb def parse_proxy_exit(value: object) -> ProxyExit: diff --git a/browser_gateway/server/http.py b/browser_gateway/server/http.py index c1e0129..acff5eb 100644 --- a/browser_gateway/server/http.py +++ b/browser_gateway/server/http.py @@ -13,6 +13,7 @@ import secrets import signal import socket import threading +from contextlib import contextmanager import time from collections.abc import Mapping from contextlib import suppress @@ -30,10 +31,10 @@ from ..platform.douyin import ( DouyinBrowser, DouyinError, DouyinLoginPending, - SubscriptionManager, is_douyin_content_url, is_douyin_share_url, ) +from ..platform.notice_polling import SubscriptionManager, validate_checkpoint from ..proxy import ProxyExit from ..runtime import ( NETWORK_ID_RE as _NETWORK_ID_RE, @@ -49,6 +50,7 @@ from ..runtime import ( NativeRuntimeManager, has_control, validate_runtime_input, + validate_memory_limit, ) from ..runtime import ( parse_proxy_exit as _parse_proxy_exit, @@ -97,6 +99,8 @@ class Gateway: self.subscriptions = SubscriptionManager(self.browser) self._action_ownership_lock = threading.Lock() self._uncertain_actions: dict[str, float] = {} + self._operation_locks_guard = threading.Lock() + self._operation_locks: dict[str, threading.RLock] = {} def _browser_endpoint(self, alias: str) -> str: return self.runtimes.endpoint(alias) @@ -120,10 +124,17 @@ class Gateway: return self.runtimes.create(input) def change_state(self, alias: str, action: str, input: dict) -> None: - generation = decode_generation( - input, require_runtime=True, require_network=True - ) - self.runtimes.change_state(alias, action, generation) + generation_input = input + memory_limit_mb = None + if action == "start" and isinstance(input, dict) and "memory_limit_mb" in input: + generation_input = dict(input) + memory_limit_mb = generation_input.pop("memory_limit_mb") + validate_memory_limit(memory_limit_mb) + generation = decode_generation(generation_input, require_runtime=True, require_network=True) + if memory_limit_mb is None: + self.runtimes.change_state(alias, action, generation) + else: + self.runtimes.change_state(alias, action, generation, memory_limit_mb=memory_limit_mb) def remove(self, alias: str, input: dict) -> None: generation = decode_generation( @@ -154,12 +165,33 @@ class Gateway: return record.runtime_id, True def _require_douyin_generation(self, alias: str, input: dict) -> None: - record = self.runtimes.require_generation(alias, input) + try: + record = self.runtimes.require_generation(alias, input) + except BrowserRuntimeError as exc: + LOG.warning("douyin_generation_rejected alias=%s binding_version=%s runtime_id=%s network_id=%s reason=%s", alias, input.get("binding_version"), input.get("runtime_id"), input.get("network_id"), str(exc)) + raise if input.get("network_exit_id", "") != record.network_exit_id: + LOG.warning("douyin_exit_generation_rejected alias=%s expected_exit=%s observed_exit=%s", alias, input.get("network_exit_id", ""), record.network_exit_id) raise RequestError("runtime proxy generation does not match request", 409) + @contextmanager def _alias_lock(self, alias: str): - return self.runtimes.alias_lock(alias) + # Polling, QR capture and manual actions share one browser. Queue local + # requests before taking the native cross-process/profile fence. + with self._operation_locks_guard: + lock = self._operation_locks.setdefault(alias, threading.RLock()) + started = time.monotonic() + if not lock.acquire(timeout=10): + LOG.warning("browser_operation_queue_timeout alias=%s waited_seconds=10", alias) + raise BrowserRuntimeError("browser operation queue timed out", 503) + try: + waited = time.monotonic() - started + if waited >= 0.1: + LOG.info("browser_operation_queued alias=%s waited_seconds=%.3f", alias, waited) + with self.runtimes.alias_lock(alias): + yield + finally: + lock.release() def get_douyin(self, alias: str, input: dict) -> dict: if not valid_douyin_generation(input) or not valid_douyin_url( @@ -299,6 +331,26 @@ class Gateway: "qr_detected": screen.qr_detected, } + def douyin_inbox(self, alias: str, input: dict) -> dict: + expected_uid = input.get("expected_uid", "") + checkpoints = input.get("checkpoints", {}) + if (not valid_douyin_generation(input) or not isinstance(expected_uid, str) + or not UID_RE.fullmatch(expected_uid) or not isinstance(checkpoints, dict) + or len(checkpoints) > 100 or any(not isinstance(k, str) or not UID_RE.fullmatch(k) + or not isinstance(v, str) or len(v) > 20 for k, v in checkpoints.items())): + raise RequestError("invalid Douyin inbox request", 400) + # Inbox reads do not mutate the browser or claim action ownership. + # A notification subscription can take time to connect, so its alias + # lock must not prevent private-message reception. + self._require_douyin_generation(alias, input) + try: + result = self.browser.inbox(alias, expected_uid, checkpoints) + self._require_douyin_generation(alias, input) + except DouyinError as exc: + LOG.warning("Douyin inbox failed alias=%s reason=%s", alias, str(exc)) + raise RequestError("Douyin inbox could not be read") from exc + return result + def douyin_message_history(self, alias: str, input: dict) -> dict: expected_uid = input.get("expected_uid", "") target_uid = input.get("target_uid", "") @@ -451,46 +503,49 @@ class Gateway: or not UID_RE.fullmatch(expected_uid) ): raise RequestError("invalid Douyin event request", 400) - with self._alias_lock(alias): + try: + validate_checkpoint(input.get("boundary_at"), input.get("notice_checkpoints", {})) + except DouyinError as exc: + raise RequestError(str(exc), 400) from exc + try: + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + # Polling-session initialization performs read-only page requests. + # It must not occupy the manual-send slot while waiting on them. + result = self.subscriptions.start(alias, expected_uid, input.get("boundary_at"), input.get("notice_checkpoints", {})) self._require_douyin_generation(alias, input) - try: - self.browser.identity(alias, expected_uid) - return self.subscriptions.start(alias, expected_uid) - except DouyinError as exc: - LOG.warning( - "Douyin event listener start failed alias=%s uid=%s reason=%s", - alias, - expected_uid, - exc, - ) - raise RequestError("Douyin event listener could not start") from exc + return result + except DouyinError as exc: + LOG.warning( + "Douyin event listener start failed alias=%s uid=%s reason=%s", + alias, expected_uid, exc, + ) + raise RequestError(f"Douyin notification polling could not start: {exc}") from exc def poll_douyin_events(self, alias: str, input: dict, query: dict) -> list[dict]: if not valid_douyin_generation(input): raise RequestError("invalid Douyin event request", 400) try: limit = int(query.get("limit", ["50"])[0]) - wait = float(query.get("wait", ["25"])[0]) + wait = float(query.get("wait", ["0"])[0]) except (IndexError, TypeError, ValueError) as exc: raise RequestError("invalid event poll options", 400) from exc if not 1 <= limit <= 100 or not 0 <= wait <= 30: raise RequestError("invalid event poll options", 400) - with self._alias_lock(alias): - self._require_douyin_generation(alias, input) try: - acknowledgements = query.get("ack", []) - delivery_ids = [ - value for raw in acknowledgements for value in raw.split(",") - ] - self.subscriptions.ack(alias, delivery_ids) - return self.subscriptions.poll(alias, limit, wait) + with self._alias_lock(alias): + self._require_douyin_generation(alias, input) + acknowledgements = query.get("ack", []) + delivery_ids = [value for raw in acknowledgements for value in raw.split(",")] + self.subscriptions.ack(alias, delivery_ids) + return self.subscriptions.poll(alias, limit, 0) except DouyinError as exc: LOG.warning( "Douyin event listener poll failed alias=%s reason=%s", alias, exc, ) - raise RequestError("Douyin event listener is unavailable") from exc + raise RequestError(f"Douyin notification polling failed: {exc}") from exc def stop_douyin_events(self, alias: str, input: dict) -> None: if not valid_douyin_generation(input): @@ -640,7 +695,7 @@ class GatewayHandler(BaseHTTPRequestHandler): gateway.restore_proxy(alias, body) return None match = re.fullmatch( - r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|image|identity|resolve|login-qr|action|messages|events)", + r"/v1/browsers/([a-z0-9][a-z0-9-]{0,31})/douyin/(get|media|image|identity|resolve|login-qr|action|messages|inbox|events)", path, ) if match: @@ -659,6 +714,8 @@ class GatewayHandler(BaseHTTPRequestHandler): return gateway.douyin_login_qr(alias, body) if action == "action" and method == "POST": return gateway.douyin_action(alias, body) + if action == "inbox" and method == "POST": + return gateway.douyin_inbox(alias, body) if action == "messages" and method == "POST": return gateway.douyin_message_history(alias, body) if action == "events": diff --git a/browser_gateway/test_browser_target_selection.py b/browser_gateway/test_browser_target_selection.py new file mode 100644 index 0000000..b73ce10 --- /dev/null +++ b/browser_gateway/test_browser_target_selection.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import json +import unittest +from contextlib import contextmanager +from importlib import import_module +from unittest.mock import Mock, patch + +from .platform.douyin import CHAT_PAGE_NAME, CHAT_PAGE_URL, DouyinBrowser, DouyinError, ORIGIN + + +douyin_module = import_module(f"{__package__}.platform.douyin") + + +class BrowserTargetSelectionTests(unittest.TestCase): + @staticmethod + def page(target_id: str, url: str = ORIGIN) -> dict: + return { + "id": target_id, + "type": "page", + "url": url, + "webSocketDebuggerUrl": f"ws://127.0.0.1:9222/devtools/page/{target_id}", + } + + @contextmanager + def targets(self, pages: list, browser: DouyinBrowser | None = None): + browser = browser or DouyinBrowser(lambda _: "http://browser:9222") + response = Mock(status=200) + response.read.return_value = json.dumps(pages).encode() + http = Mock() + http.getresponse.return_value = response + cdp = Mock() + with ( + patch.object(douyin_module.http.client, "HTTPConnection", return_value=http), + patch.object(douyin_module.websocket, "create_connection") as connect, + patch.object(douyin_module, "CDPConnection", return_value=cdp), + ): + yield browser, connect, cdp + + def assert_selected(self, connect: Mock, target_id: str) -> None: + connect.assert_called_once_with( + f"ws://browser:9222/devtools/page/{target_id}", + timeout=douyin_module.CONTROL_TIMEOUT, + suppress_origin=True, + enable_multithread=True, + ) + + def test_multiple_douyin_pages_prefer_self_profile_in_any_order(self) -> None: + pages = [ + self.page("A", ORIGIN + "/video/123"), + self.page("Z", ORIGIN + "/user/self?showTab=post"), + self.page("B", ORIGIN + "/"), + ] + for ordered in (pages, list(reversed(pages))): + with self.subTest(order=[page["id"] for page in ordered]): + with self.targets(ordered) as (browser, connect, cdp): + browser._connect("safe") + self.assert_selected(connect, "Z") + cdp.command.assert_not_called() + cdp.notify.assert_not_called() + + def test_multiple_profile_pages_use_stable_target_id(self) -> None: + pages = [ + self.page("Z", ORIGIN + "/user/self"), + self.page("A", ORIGIN + "/user/self?showTab=like"), + ] + with self.targets(pages) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "A") + + def test_multiple_douyin_pages_without_profile_use_stable_target_id(self) -> None: + pages = [self.page("Z", ORIGIN + "/video/123"), self.page("A")] + for ordered in (pages, list(reversed(pages))): + with self.subTest(order=[page["id"] for page in ordered]): + with self.targets(ordered) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "A") + + def test_unrelated_pages_and_non_page_targets_are_not_selected(self) -> None: + pages = [ + self.page("A", "https://example.com/"), + {"id": "worker", "type": "service_worker", "url": ORIGIN}, + None, + self.page("Z", ORIGIN + "/video/123"), + self.page("B", ORIGIN + "/user/self"), + ] + with self.targets(pages) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "B") + + def test_dedicated_chat_target_is_not_used_for_regular_operations(self) -> None: + browser = DouyinBrowser(lambda _: "http://browser:9222") + browser._chat_targets["safe"] = "A" + pages = [self.page("A", CHAT_PAGE_URL), self.page("Z"), self.page("B")] + with self.targets(pages, browser) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "B") + self.assertEqual(browser._chat_targets["safe"], "A") + + def test_dedicated_chat_page_is_identified_after_gateway_restart(self) -> None: + pages = [self.page("chat", CHAT_PAGE_URL + "#" + CHAT_PAGE_NAME), self.page("B"), self.page("A")] + with self.targets(pages) as (browser, connect, cdp): + cdp.evaluate.return_value = CHAT_PAGE_NAME + browser._connect("safe") + self.assertEqual(connect.call_count, 2) + self.assertEqual(connect.call_args.args[0], "ws://browser:9222/devtools/page/A") + cdp.close.assert_called_once() + cdp.command.assert_not_called() + cdp.notify.assert_not_called() + + def test_only_dedicated_chat_page_after_restart_is_not_reused(self) -> None: + with self.targets([self.page("chat", CHAT_PAGE_URL + "#" + CHAT_PAGE_NAME)]) as (browser, connect, cdp): + cdp.evaluate.return_value = CHAT_PAGE_NAME + with self.assertRaisesRegex(DouyinError, "page target is unavailable"): + browser._connect("safe") + # Only the read-only page-role probe connects; no account request follows. + connect.assert_called_once() + cdp.close.assert_called_once() + cdp.command.assert_not_called() + cdp.notify.assert_not_called() + + def test_only_dedicated_chat_page_is_not_reused(self) -> None: + browser = DouyinBrowser(lambda _: "http://browser:9222") + browser._chat_targets["safe"] = "chat" + with self.targets([self.page("chat", CHAT_PAGE_URL)], browser) as (browser, connect, _): + with self.assertRaisesRegex(DouyinError, "page target is unavailable"): + browser._connect("safe") + connect.assert_not_called() + + def test_chat_page_with_blank_regular_page_preserves_initial_login(self) -> None: + browser = DouyinBrowser(lambda _: "http://browser:9222") + browser._chat_targets["safe"] = "chat" + pages = [self.page("chat", CHAT_PAGE_URL), self.page("blank", "about:blank")] + with self.targets(pages, browser) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "blank") + + def test_explicit_target_remains_pinned_even_for_private_chat(self) -> None: + browser = DouyinBrowser(lambda _: "http://browser:9222", target_id="chat") + browser._chat_targets["safe"] = "chat" + pages = [self.page("main", ORIGIN + "/user/self"), self.page("chat", CHAT_PAGE_URL)] + with self.targets(pages, browser) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "chat") + + def test_missing_explicit_target_does_not_choose_another_page(self) -> None: + browser = DouyinBrowser(lambda _: "http://browser:9222", target_id="missing") + with self.targets([self.page("A"), self.page("B")], browser) as (browser, connect, _): + with self.assertRaisesRegex(DouyinError, "configured browser page target is unavailable"): + browser._connect("safe") + connect.assert_not_called() + + def test_multiple_unrelated_pages_are_not_navigated_arbitrarily(self) -> None: + pages = [self.page("A", "https://example.com/"), self.page("B", "about:blank")] + with self.targets(pages) as (browser, connect, _): + with self.assertRaises(DouyinError): + browser._connect("safe") + connect.assert_not_called() + + def test_single_blank_page_still_supports_initial_login(self) -> None: + with self.targets([self.page("blank", "about:blank")]) as (browser, connect, _): + browser._connect("safe") + self.assert_selected(connect, "blank") + + def test_multi_page_identity_verifies_bound_uid_and_closes_connection(self) -> None: + payload = {"status_code": 0, "user": {"uid": "1004291301", "sec_uid": "safe_sec_uid"}} + pages = [self.page("A", ORIGIN + "/video/123"), self.page("B", ORIGIN + "/user/self")] + with self.targets(pages) as (browser, connect, cdp): + cdp.evaluate.side_effect = [ORIGIN, {"status": 200, "body": json.dumps(payload)}] + identity = browser.identity("safe", "1004291301") + self.assertEqual(identity["uid"], "1004291301") + self.assert_selected(connect, "B") + cdp.close.assert_called_once() + cdp.notify.assert_not_called() + + def test_multi_page_identity_still_rejects_wrong_uid(self) -> None: + payload = {"status_code": 0, "user": {"uid": "987654321", "sec_uid": "safe_sec_uid"}} + with self.targets([self.page("A"), self.page("B")]) as (browser, connect, cdp): + cdp.evaluate.side_effect = [ORIGIN, {"status": 200, "body": json.dumps(payload)}] + with self.assertRaisesRegex(DouyinError, "does not match the expected account"): + browser.identity("safe", "1004291301") + self.assert_selected(connect, "A") + cdp.close.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/browser_gateway/test_event_details.py b/browser_gateway/test_event_details.py new file mode 100644 index 0000000..bb9c4f4 --- /dev/null +++ b/browser_gateway/test_event_details.py @@ -0,0 +1,44 @@ +import unittest + +from .platform.douyin import DouyinError, normalize_notice + + +class EventDetailsTests(unittest.TestCase): + def test_notification_preserves_user_profile_and_work_cover(self): + result = normalize_notice({'nid':'9007199254740993', 'comment': { + 'from_user':{'uid':'456','nickname':'肉宝儿','sec_uid':'MS4wLjABAAAAprofile'}, + 'comment':{'cid':'789','text':'你好'}, + 'aweme':{'aweme_id':'987','aweme_type':0,'author':{'uid':'123'}, + 'video':{'cover':{'url_list':['https://p3.douyinpic.com/cover.jpg']}}}, + }}) + self.assertEqual(result['interactor_name'],'肉宝儿') + self.assertEqual(result['interactor_sec_uid'],'MS4wLjABAAAAprofile') + self.assertEqual(result['work_author_uid'],'123') + self.assertEqual(result['work_cover_url'],'https://p3.douyinpic.com/cover.jpg') + self.assertEqual(result['work_url'],'https://www.douyin.com/video/987') + + def test_photo_and_comment_user_without_from_user(self): + result = normalize_notice({'nid':'1','comment':{ + 'comment':{'user':{'uid':'2','nickname':'作者','sec_uid':'MS4wLjABAAAAother'}}, + 'aweme':{'aweme_id':'3','aweme_type':68,'author':{'uid':'4'}, + 'images':[{'url_list':['https://p3.douyinpic.com/photo.jpg']}]}, + }}) + self.assertEqual(result['interactor_name'],'作者') + self.assertEqual(result['work_url'],'https://www.douyin.com/note/3') + self.assertEqual(result['work_cover_url'],'https://p3.douyinpic.com/photo.jpg') + + def test_unknown_or_merged_users_do_not_get_fabricated_profiles(self): + result = normalize_notice({'nid':'4','follow':{'from_user':[{'uid':'2'},{'uid':'3'}]}}) + self.assertEqual(result['interactor_name'],'') + self.assertEqual(result['interactor_sec_uid'],'') + self.assertEqual(result['work_cover_url'],'') + self.assertEqual(result['work_url'],'') + + def test_invalid_profile_fields_fail_clearly(self): + for field in ('nickname','sec_uid'): + with self.subTest(field=field), self.assertRaises(DouyinError): + normalize_notice({'nid':'5','follow':{'from_user':{'uid':'2',field:123}}}) + + +if __name__ == '__main__': + unittest.main() diff --git a/browser_gateway/test_event_listener.py b/browser_gateway/test_event_listener.py new file mode 100644 index 0000000..081ab36 --- /dev/null +++ b/browser_gateway/test_event_listener.py @@ -0,0 +1,78 @@ +import threading +import unittest +from unittest.mock import Mock + +from browser_gateway.platform.douyin import ( + DouyinError, + normalize_notice, +) +from browser_gateway.platform.notice_polling import SubscriptionManager +from browser_gateway.server.http import GatewayHandler, GatewayHTTPServer + + +class EventListenerTests(unittest.TestCase): + def test_comment_text_is_preserved(self): + notice = { + "nid_str": "100", + "comment": { + "from_user": [{"uid": "200"}], + "comment": {"cid_str": "300", "text": "测试评论"}, + }, + } + result = normalize_notice(notice) + self.assertEqual(result["message_text"], "测试评论") + self.assertEqual(result["message_type"], "text") + + def test_invalid_comment_text_is_not_silently_ignored(self): + with self.assertRaises(DouyinError): + normalize_notice({ + "nid_str": "100", + "comment": { + "from_user": [{"uid": "200"}], + "comment": {"text": ["invalid"]}, + }, + }) + + + + def test_failed_stop_keeps_subscription_for_retry(self): + manager = SubscriptionManager(Mock()) + subscription = Mock() + subscription.stop.side_effect = DouyinError("stop timed out") + manager._items["browser"] = subscription + with self.assertRaises(DouyinError): + manager.stop("browser") + self.assertIs(manager._items["browser"], subscription) + subscription.stop.side_effect = None + manager.stop("browser") + self.assertNotIn("browser", manager._items) + + def test_failed_restart_keeps_subscription_for_retry(self): + manager = SubscriptionManager(Mock()) + subscription = Mock() + subscription.stop.side_effect = DouyinError("stop timed out") + manager._items["browser"] = subscription + with self.assertRaises(DouyinError): + manager.start("browser", "123") + self.assertIs(manager._items["browser"], subscription) + + def test_gateway_listener_contract_uses_post_get_delete(self): + gateway = Mock() + gateway.start_douyin_events.return_value = {"connected": True, "alias": "browser", "uid": "123"} + gateway.poll_douyin_events.return_value = [{"kind": "open", "delivery_id": "1"}] + gateway.stop_douyin_events.return_value = None + server = GatewayHTTPServer(("127.0.0.1", 0), gateway) + self.addCleanup(server.server_close) + handler = GatewayHandler.__new__(GatewayHandler) + handler.server = server + route = "/v1/browsers/browser/douyin/events" + payload = {"expected_uid": "123", "binding_version": 7, "runtime_id": "runtime", "network_id": "network", "network_exit_id": ""} + self.assertTrue(handler._route("POST", route, {}, payload)["connected"]) + query = {"ack": ["1"], "wait": ["5"], "limit": ["100"]} + self.assertIsInstance(handler._route("GET", route, query, payload), list) + gateway.poll_douyin_events.assert_called_once_with("browser", payload, query) + self.assertIsNone(handler._route("DELETE", route, {}, payload)) + + +if __name__ == "__main__": + unittest.main() diff --git a/browser_gateway/test_gateway.py b/browser_gateway/test_gateway.py index 36944b7..956e8a8 100644 --- a/browser_gateway/test_gateway.py +++ b/browser_gateway/test_gateway.py @@ -22,21 +22,16 @@ from .platform.douyin import ( CDPConnection, DouyinBrowser, DouyinError, - DouyinSubscription, LOGIN_PAGE_URL, ORIGIN_URL, - SubscriptionManager, - ack_expression, action_expression, detect_challenge, im_expression, - install_expression, is_douyin_content_url, is_douyin_share_url, is_douyin_url, normalize_notice, notice_ids, - wait_expression, ) from .proxy import ( MemoryProxy, @@ -634,68 +629,20 @@ class CDPTests(unittest.TestCase): "event_key": "9007199254740993", "event_type": "comment", "interactor_uid": "7654321", + "interactor_name": "", + "interactor_sec_uid": "", "comment_id": "987654", "work_id": "123456", + "work_author_uid": "", + "work_cover_url": "", + "work_url": "https://www.douyin.com/video/123456", "platform_event_at": "2023-11-14T22:13:20+00:00", }, ) favorite = {"nid_str": "7", "favorite": {"from_user": [{"uid": "1"}]}} self.assertIsNone(normalize_notice(favorite)) - def test_notification_detail_retries_partial_response(self) -> None: - details = [ - {"nid_str": "1", "user_id": "123"}, - {"nid_str": "2", "user_id": "123"}, - ] - payload = json.dumps({"status_code": 0, "notice_list_v2": details}) - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.uid = "123" - subscription.connection = cast( - CDPConnection, - FakeConnection( - [ - json.dumps( - { - "status": 200, - "body": json.dumps( - {"status_code": 0, "notice_list_v2": details[:1]} - ), - } - ), - json.dumps({"status": 200, "body": payload}), - ] - ), - ) - subscription._get_connection = lambda: subscription.connection - with patch.object(douyin_module.time, "sleep"): - self.assertEqual(len(subscription._details(["1", "2"])), 2) - def test_notification_detail_rejects_unexpected_id(self) -> None: - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.uid = "123" - subscription.connection = cast( - CDPConnection, - FakeConnection( - [ - json.dumps( - { - "status": 200, - "body": json.dumps( - { - "status_code": 0, - "notice_list_v2": [ - {"nid_str": "9", "user_id": "123"} - ], - } - ), - } - ) - ] - ), - ) - subscription._get_connection = lambda: subscription.connection - with self.assertRaises(DouyinError): - subscription._details(["1"]) class BrowserCDP: @@ -1051,34 +998,12 @@ class BrowserTests(unittest.TestCase): ) self.assertIsInstance(connection, CDPConnection) self.assertEqual(http.requested[0], ("GET", "/json/list")) - expression = install_expression("safe", "123") - self.assertIn("safe", expression) - self.assertIn("old?.dispose?.()", expression) - self.assertEqual( - notice_ids( - { - "service": 20313, - "payload": json.dumps( - {"notices": [{"notice_id_str": "1", "effect_groups": [960]}]} - ), - } - ), - ["1"], - ) - self.assertEqual( - notice_ids( - { - "service": 20003, - "payload": json.dumps({"notice_type": 45, "notice_id_str": "2"}), - } - ), - ["2"], - ) browser = DouyinBrowser() cast(Any, browser).identity = lambda alias, expected: {"uid": expected} + cast(Any, browser)._chat_browser = lambda alias: browser cast(Any, browser)._evaluate = lambda *args: { - "status": "succeeded", + "status": "logged_in", "uid": "123", "action": "followed", } preview = browser.action( @@ -1096,153 +1021,11 @@ class BrowserTests(unittest.TestCase): with self.assertRaises(DouyinError): browser.action("safe", "123", "follow", "bad", confirm=False) - def test_browser_queue_is_retained_until_ack(self) -> None: - wait = wait_expression("alpha") - self.assertIn("delivered", wait) - self.assertNotIn("splice(0)", wait) - ack = ack_expression("__creatorhub_notice_sub_alpha", ["browser-1"]) - self.assertIn("browser-1", ack) - self.assertIn("__creatorhub_notice_sub_alpha", ack) - self.assertIn( - "crypto.randomUUID", douyin_module.install_expression("alpha", "123") - ) - def test_listener_start_retries_until_runtime_ready(self) -> None: - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.alias = "safe" - subscription.uid = "123" - subscription.stopped = threading.Event() - subscription.stopped.wait = Mock(return_value=False) - subscription._open_listener = Mock( - side_effect=[ - DouyinError("SDK_NOT_READY"), - (Mock(), "2026-09-14T18:00:00+00:00"), - ] - ) - with patch.object(douyin_module.time, "monotonic", side_effect=[0.0, 1.0]): - connection, boundary = subscription._open_listener_until_ready() - self.assertEqual(boundary, "2026-09-14T18:00:00+00:00") - self.assertIsNotNone(connection) - self.assertEqual(subscription._open_listener.call_count, 2) - subscription.stopped.wait.assert_called_once_with(0.5) - def test_subscription_receipts_are_replayed_until_ack(self) -> None: - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.uid = "123" - subscription.queue = deque() - subscription.condition = threading.Condition() - subscription.stopped = threading.Event() - subscription._put({"kind": "notice", "notice": {"event_key": "1"}}) - first = subscription.poll(10, 0) - second = subscription.poll(10, 0) - self.assertEqual(first, second) - self.assertTrue(first[0]["notice"]["gateway_received_at"].endswith("+00:00")) - subscription.ack([first[0]["delivery_id"]]) - self.assertEqual(subscription.poll(10, 0), []) - def test_subscription_connection_failure_requests_recovery(self) -> None: - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.alias = "safe" - subscription.browser = Mock() - subscription.browser._connect.side_effect = DouyinError("browser unavailable") - subscription._put = Mock() - subscription._request_recovery = Mock() - subscription._browser_inflight_lock = threading.Lock() - subscription._browser_inflight = set() - subscription._process_push_async({"delivery_id": "delivery-1"}, 0, False) - subscription._request_recovery.assert_called_once_with() - def test_subscription_detail_failure_does_not_discard_siblings(self) -> None: - bad = json.dumps( - { - "status": 200, - "body": json.dumps( - { - "status_code": 0, - "notice_list_v2": [{"nid_str": "9", "user_id": "123"}], - } - ), - } - ) - good = json.dumps( - { - "status": 200, - "body": json.dumps( - { - "status_code": 0, - "notice_list_v2": [ - { - "nid_str": "2", - "user_id": "123", - "follow": {"from_user": [{"uid": "7"}]}, - } - ], - } - ), - } - ) - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.uid = "123" - subscription.queue = deque() - subscription.condition = threading.Condition() - subscription.stopped = threading.Event() - subscription.connection = cast(CDPConnection, FakeConnection([bad, bad, good])) - subscription._get_connection = lambda: subscription.connection - subscription._handle( - { - "kind": "push", - "service": 20313, - "payload": json.dumps( - { - "notices": [ - {"notice_id_str": "1", "effect_groups": [960]}, - {"notice_id_str": "2", "effect_groups": [960]}, - ] - } - ), - } - ) - events = subscription.poll(10, 0) - self.assertEqual(events[0]["kind"], "error") - self.assertEqual(events[0]["event_key"], "1") - self.assertEqual(events[1]["notice"]["event_key"], "2") - def test_subscription_manager_and_queue(self) -> None: - event = {"kind": "open"} - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.uid = "123" - subscription.queue = deque() - subscription.condition = __import__("threading").Condition() - subscription.stopped = __import__("threading").Event() - subscription._handle(event) - first = subscription.poll(10, 0) - self.assertEqual(first[0]["kind"], "open") - self.assertTrue(first[0]["delivery_id"]) - subscription.queue = deque([{} for _ in range(1000)]) - subscription._put({"kind": "new"}) - self.assertEqual(subscription.queue[0]["kind"], "error") - subscription.stopped.set() - overflow = subscription.poll(10, 0) - self.assertEqual(overflow[0]["kind"], "error") - self.assertEqual(overflow[0]["reason"], "notification queue overflow") - self.assertTrue(overflow[0]["delivery_id"]) - self.assertEqual(len(overflow), 10) - self.assertEqual(len(subscription.queue), 1000) - subscription.ack([overflow[0]["delivery_id"]]) - self.assertEqual(len(subscription.queue), 999) - self.assertEqual(subscription.queue[0], {}) - - browser = DouyinBrowser() - fake = Mock() - fake.poll.return_value = [{"kind": "notice"}] - with patch.object(douyin_module, "DouyinSubscription", return_value=fake): - manager = SubscriptionManager(browser) - self.assertTrue(manager.start("safe", "123")["connected"]) - self.assertEqual(manager.poll("safe", 1, 0), [{"kind": "notice"}]) - manager.stop("safe") - manager.close() - with self.assertRaises(DouyinError): - manager.poll("safe", 1, 0) class ProxyTests(unittest.TestCase): @@ -1557,10 +1340,12 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): cast(Any, browser).identity = lambda alias, expected_uid=None: { "uid": expected_uid or "1" } - cast(Any, browser)._evaluate = lambda alias, expression: { - "status": 200, - "action": "sent", - } + cast(Any, browser)._evaluate = lambda alias, expression: ( + {"status": "logged_in", "uid": "1"} if 'mode="identity"' in expression else { + "status": 200, "action": "sent", + } + ) + cast(Any, browser)._chat_browser = lambda alias: browser bind(BrowserCDP(["https://www.douyin.com", {}])) self.assertEqual( browser.action("safe", "1", "follow", "2", confirm=True)["action"], "sent" @@ -1658,35 +1443,7 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): ) def test_direct_message_notice_and_uncertain_post_contract(self) -> None: - notice = normalize_notice( - { - "dm": { - "message_id": "123456", - "from_user": {"uid": "456789"}, - "text": "hello", - }, - "create_time": 1700000000, - } - ) - self.assertIsNotNone(notice) - assert notice is not None - self.assertEqual(notice["event_type"], "dm") - self.assertEqual(notice["interactor_uid"], "456789") - self.assertEqual(notice["message_type"], "text") - self.assertEqual(notice["message_text"], "hello") - non_text = normalize_notice( - { - "dm": { - "message_id": "123457", - "from_user": {"uid": "456789"}, - "message_type": "image", - "image_url": "https://example.invalid/image", - } - } - ) - assert non_text is not None - self.assertEqual(non_text["message_type"], "image") - self.assertEqual(non_text["message_text"], "") + self.assertIsNone(normalize_notice({"dm": {"message_id": "123456", "text": "hello"}})) script = action_expression({"alias": "safe", "action": "follow", "target": "2"}) self.assertIn("POST_UNCERTAIN", script) self.assertIn("BUSINESS_REJECTED", script) @@ -1796,59 +1553,6 @@ class AdditionalGatewayCoverageTests(unittest.TestCase): with self.assertRaisesRegex(DouyinError, "message history response"): browser.message_history("safe", "123", "456") - def test_listener_open_and_recovery_boundaries(self) -> None: - browser = Mock() - browser.identity.return_value = { - "uid": "123", - "platform_now": "2026-09-14T00:00:00+00:00", - } - connection = BrowserCDP([json.dumps({"connected": True})]) - browser._connect.return_value = connection - subscription = DouyinSubscription.__new__(DouyinSubscription) - subscription.browser = browser - subscription.alias = "safe" - subscription.uid = "123" - subscription.key = "__creatorhub_notice_sub_safe" - opened, boundary_at = subscription._open_listener() - self.assertIs(opened, connection) - self.assertEqual(boundary_at, "2026-09-14T00:00:00+00:00") - - constructor_connection = BrowserCDP([json.dumps({"connected": True})]) - browser._connect.return_value = constructor_connection - with patch.object(threading.Thread, "start"): - constructed = DouyinSubscription(browser, "safe", "123") - self.assertEqual(constructed.queue[0]["kind"], "baseline") - constructed.stopped.set() - constructed._detail_pool.shutdown(wait=True, cancel_futures=True) - - broken = BrowserCDP([json.dumps({"connected": True}), "{}"]) - browser._connect.return_value = broken - browser.identity.return_value = {"uid": "123"} - with self.assertRaisesRegex(DouyinError, "event boundary"): - subscription._open_listener() - self.assertTrue(broken.closed) - - old = Mock() - old.evaluate.side_effect = DouyinError("stale listener") - new = Mock() - subscription.connection = old - subscription._connection_lock = threading.RLock() - subscription._epoch = 0 - subscription._initial_boundary_pending = False - subscription.stopped = threading.Event() - subscription.stopped.wait = Mock(return_value=False) - subscription._open_listener = Mock( - return_value=(new, "2026-09-14T00:00:01+00:00") - ) - subscription._put = Mock() - subscription._recover() - old.close.assert_called_once_with() - self.assertIs(subscription.connection, new) - self.assertEqual(subscription._epoch, 1) - self.assertEqual( - [call.args[0]["kind"] for call in subscription._put.call_args_list], - ["reconnected", "baseline"], - ) @@ -1881,7 +1585,8 @@ class DouyinReleaseRemediationTests(unittest.TestCase): def test_notice_keeps_raw_gateway_receipt_time(self) -> None: notice = normalize_notice( - {"dm": {"message_id": "1", "from_user": {"uid": "2"}, "text": "hi"}}, + {"nid_str": "1", "follow": {"from_user": [{"uid": "2"}]}}, + "2026-09-15T00:00:00+00:00", ) assert notice is not None diff --git a/browser_gateway/test_memory_limits.py b/browser_gateway/test_memory_limits.py new file mode 100644 index 0000000..875a9e1 --- /dev/null +++ b/browser_gateway/test_memory_limits.py @@ -0,0 +1,78 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock +from .server.http import Gateway, RequestError + +from .runtime import BrowserRuntimeError, NativeRuntimeManager, RuntimeRecord, validate_runtime_input +from . import test_runtime as helpers + + +class MemoryUnits(helpers.FakeUnits): + def __init__(self): + super().__init__() + self.limits = {} + self.start_count = 0 + + def start(self, unit, args, **limits): + self.limits[unit] = limits['limits'] + self.start_count += 1 + return super().start(unit, args, **limits) + + +class EnvironmentMemoryTests(unittest.TestCase): + def test_start_route_forwards_memory_separately_from_generation(self): + gateway=object.__new__(Gateway);gateway.runtimes=Mock() + generation={'runtime_id':'a'*64,'network_id':'native-'+'b'*32,'binding_version':1} + gateway.change_state('account','start',dict(generation,memory_limit_mb=4096)) + gateway.runtimes.change_state.assert_called_once_with('account','start',generation,memory_limit_mb=4096) + with self.assertRaises(RequestError):gateway.change_state('account','stop',dict(generation,memory_limit_mb=4096)) + with self.assertRaises(BrowserRuntimeError):gateway.change_state('account','start',dict(generation,memory_limit_mb=0)) + + def manager(self, root, units): + manager=NativeRuntimeManager(state_dir=Path(root)/'state',profile_root=Path(root)/'profiles',node_id='node-a',browser_path='/bin/true',unit_manager=units,min_free_bytes=0,display_start=1000,display_end=1010) + manager._wait_for_display=lambda record:None + manager._wait_for_cdp=lambda record:None + manager._ready=lambda record:True + self.addCleanup(manager.close) + return manager + + def test_memory_default_and_input_validation(self): + value = helpers.NativeRuntimeManagerTests().payload() + validate_runtime_input(value) + self.assertEqual(value['memory_limit_mb'], 2048) + for limit in [512, 2048, 3072, 65536]: + value = helpers.NativeRuntimeManagerTests().payload();value['memory_limit_mb'] = limit + validate_runtime_input(value) + self.assertEqual(value['memory_limit_mb'], limit) + for limit in [0, -1, 511, 65537, True, '2048', 2.5]: + value = helpers.NativeRuntimeManagerTests().payload();value['memory_limit_mb'] = limit + with self.assertRaises(BrowserRuntimeError): validate_runtime_input(value) + + def test_custom_limit_reaches_systemd_and_metadata(self): + with tempfile.TemporaryDirectory() as root: + units=MemoryUnits();manager=self.manager(root,units) + value=helpers.NativeRuntimeManagerTests().payload();value['memory_limit_mb']=3072 + result=manager.create(value) + self.assertEqual(units.limits[manager._records()[0].browser_unit]['MemoryMax'],'3072M') + self.assertEqual(result['memory_limit_mb'],3072) + loaded=manager._records()[0] + self.assertEqual(loaded.memory_limit_mb,3072) + self.assertEqual(RuntimeRecord.from_dict(loaded.to_dict()).memory_limit_mb,3072) + + def test_updated_limit_applies_when_started_without_changing_profile(self): + with tempfile.TemporaryDirectory() as root: + units=MemoryUnits();manager=self.manager(root,units) + value=helpers.NativeRuntimeManagerTests().payload(stopped=True);result=manager.create(value) + generation={k:result[k] for k in ['runtime_id','binding_version','network_id']} + manager.change_state(value['alias'],'start',generation,memory_limit_mb=4096) + self.assertEqual(units.limits[manager._records()[0].browser_unit]['MemoryMax'],'4096M') + record=manager._records()[0] + self.assertEqual(record.profile_id,value['profile_id']) + self.assertEqual(record.command,value['cmd']) + before=units.start_count + current=manager._public(manager._records()[0], ready=True) + generation={k:current[k] for k in ['runtime_id','binding_version','network_id']} + manager.change_state(value['alias'],'start',generation,memory_limit_mb=8192) + self.assertEqual(units.start_count,before) + self.assertEqual(manager._records()[0].memory_limit_mb,4096) diff --git a/browser_gateway/test_notice_polling.py b/browser_gateway/test_notice_polling.py new file mode 100644 index 0000000..2f95a5b --- /dev/null +++ b/browser_gateway/test_notice_polling.py @@ -0,0 +1,215 @@ +import json +import unittest +from unittest.mock import Mock, patch +from urllib.parse import parse_qs, urlsplit + +from .platform.douyin import BrowserResponse, DouyinBrowser, DouyinError +from .platform.notice_polling import NoticePollingSession, SubscriptionManager + +NOW = '2026-10-06T10:00:00+00:00' +BOUNDARY = '2026-10-06T09:00:00+00:00' + + +def notice(nid='9007199254740993', timestamp=1791281100, uid='123', kind='follow'): + return {'nid': int(nid), 'user_id': int(uid), 'create_time': timestamp, + kind: {'from_user': [{'uid': 456}]}} + + +def page(items=(), more=0, cursor=0, status=0): + return BrowserResponse(200, json.dumps({'status_code': status, 'notice_list_v2': list(items), + 'has_more': more, 'max_time': cursor}), False) + + +class NoticePollingTests(unittest.TestCase): + def browser(self, response=None): + browser = Mock() + browser.identity.return_value = {'uid': '123', 'platform_now': NOW} + browser.get.return_value = response or page() + browser._notification_browser.return_value = browser + return browser + + def session(self, browser=None, checkpoints=None): + return NoticePollingSession(browser or self.browser(), 'safe', '123', BOUNDARY, checkpoints) + + def test_initial_scan_backfills_history_and_reports_boundary_only_after_success(self): + browser = self.browser(page([notice(timestamp=1)])) + session = NoticePollingSession(browser, 'safe', '123') + self.assertEqual(session.pending(), []) + result = session.poll(100, 0) + self.assertEqual([d['kind'] for d in result], ['baseline', 'notice', 'checkpoint']) + self.assertTrue(result[1]['baseline']) + self.assertEqual(result[1]['notice']['platform_event_at'], '1970-01-01T00:00:01+00:00') + self.assertEqual(result[0]['boundary_at'], NOW) + for call in browser.get.call_args_list: + params = parse_qs(urlsplit(call.args[1]).query) + self.assertEqual(params['is_mark_read'], ['0']) + self.assertEqual(params['min_time'], ['0']) + self.assertEqual(params['count'], ['50']) + self.assertEqual(set(result[-1]['checkpoints']), {'700', '960', '961'}) + + def test_big_ids_and_duplicates_survive_replay_until_checkpoint_ack(self): + session = self.session(self.browser(page([notice(), notice()]))) + first = session.poll(100, 0) + self.assertEqual(first, session.poll(100, 0)) + events = [d for d in first if d['kind'] == 'notice'] + self.assertEqual(len(events), 1) + self.assertEqual(events[0]['notice']['event_key'], '9007199254740993') + self.assertEqual(session.checkpoints, {}) + session.ack([d['delivery_id'] for d in first[:-1]]) + self.assertEqual(session.checkpoints, {}) + session.ack([first[-1]['delivery_id']]) + self.assertEqual(session.checkpoints['960'], '9007199254740993') + # Re-read the checkpoint page for out-of-order visibility; DB deduplicates it. + self.assertEqual(session.poll(100, 0)[-1]['kind'], 'checkpoint') + + def test_incremental_scan_reads_entire_boundary_page_and_out_of_order_notices(self): + browser = self.browser() + def fetch(alias, url): + cursor = parse_qs(urlsplit(url).query)['max_time'][0] + if cursor == '0': + return page([notice('21', 1791281300)], 1, 1791281200) + return page([notice('20', 1791281200), notice('19', 1791281190)], 1, 1791281100) + browser.get.side_effect = fetch + session = self.session(browser, {'700': '20', '960': '20', '961': '20'}) + session.last_full_scan_at = 1000 + with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1001): + result = session.poll(100, 0) + self.assertEqual({d['notice']['event_key'] for d in result if d['kind'] == 'notice'}, {'21', '20', '19'}) + self.assertEqual(browser.get.call_count, 6) + + def test_failure_never_advances_checkpoint_or_returns_partial_scan(self): + cases = [page(status=8), page([notice(uid='999')]), page([dict(notice(), nid=0)]), + page([dict(notice(), create_time=None)]), page([], 1, 0), + page([], 2), BrowserResponse(503, '{}', False), + BrowserResponse(200, 'not json', False), page([dict(notice(), follow='bad')])] + for bad in cases: + with self.subTest(body=bad.body): + session = self.session(self.browser(bad)) + with self.assertRaises(DouyinError): + session.poll(100, 0) + self.assertEqual(session.pending(), []) + self.assertEqual(session.checkpoints, {}) + + def test_cursor_loops_fail_visibly(self): + session = self.session(self.browser(page([notice()], 1, 100))) + with self.assertRaisesRegex(DouyinError, 'cursor'): + session.poll(100, 0) + self.assertEqual(session.pending(), []) + + def test_all_supported_interactions_have_distinct_types_and_accurate_actor(self): + raws = [notice(str(100 + i), kind=kind) for i, kind in enumerate(('follow','comment','digg','share'))] + result = self.session(self.browser(page(raws))).poll(100, 0) + events = [d['notice'] for d in result if d['kind'] == 'notice'] + self.assertEqual({e['event_type'] for e in events}, {'follow','comment','like','repost'}) + self.assertEqual({e['interactor_uid'] for e in events}, {'456'}) + self.assertEqual(len(events), 4) + + def test_restart_backfills_before_the_saved_head_and_before_enable_time(self): + browser = self.browser() + def fetch(alias, url): + cursor = parse_qs(urlsplit(url).query)['max_time'][0] + if cursor == '0': + return page([notice('21')], 1, 100) + return page([notice('20', 1)], 0) + browser.get.side_effect = fetch + session = self.session(browser, {'700':'21', '960':'21', '961':'21'}) + deliveries = session.poll(100, 0) + events = [d['notice'] for d in deliveries if d['kind'] == 'notice'] + self.assertEqual({e['event_key'] for e in events}, {'21','20'}) + self.assertTrue(next(d for d in deliveries if d.get('notice',{}).get('event_key')=='20')['baseline']) + + def test_periodic_full_reconciliation_recovers_old_holes_behind_the_head(self): + browser = self.browser() + missing_visible = False + def fetch(alias, url): + cursor = parse_qs(urlsplit(url).query)['max_time'][0] + if cursor == '0': + return page([notice('21')], 1, 100) + return page([notice('20', 1)] if missing_visible else [], 0) + browser.get.side_effect = fetch + session = self.session(browser) + with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1000): + first = session.poll(100,0) + session.ack([d['delivery_id'] for d in first]) + missing_visible = True + with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1001): + incremental = session.poll(100,0) + self.assertNotIn('20', {d['notice']['event_key'] for d in incremental if d['kind']=='notice'}) + session.ack([d['delivery_id'] for d in incremental]) + with patch('browser_gateway.platform.notice_polling.time.monotonic', return_value=1301): + complete = session.poll(100,0) + self.assertIn('20', {d['notice']['event_key'] for d in complete if d['kind']=='notice'}) + + def test_history_scan_is_not_truncated_at_one_hundred_pages(self): + browser = self.browser() + def fetch(alias,url): + cursor = int(parse_qs(urlsplit(url).query)['max_time'][0]) + current = cursor or 2000 + return page([notice(str(current),1)], int(current>1899), current-1) + browser.get.side_effect = fetch + events = [d for d in self.session(browser).poll(1000,0) if d['kind']=='notice'] + self.assertEqual(len(events), 102) + self.assertGreater(browser.get.call_count,300) + + def test_private_messages_and_unrecognized_categories_are_not_aggregated(self): + session = self.session(self.browser(page([notice(kind='dm'), notice('2', kind='favorite')]))) + result = session.poll(100, 0) + self.assertEqual([d['kind'] for d in result], ['checkpoint']) + self.assertEqual(result[-1]['ignored_types'], {'dm': 3, 'favorite': 3}) + + def test_checkpoint_missing_is_a_visible_gap_not_silent_continuity(self): + session = self.session(self.browser(page([notice()])), {'960': '404'}) + result = session.poll(100, 0) + self.assertEqual(result[-1]['kind'], 'checkpoint') + self.assertIn('960', result[-1]['reason']) + + def test_wrong_uid_and_invalid_restart_checkpoint_are_rejected(self): + browser = self.browser() + browser.identity.side_effect = DouyinError('identity mismatch') + with self.assertRaisesRegex(DouyinError, 'identity'): + self.session(browser) + for boundary, checkpoints in [('bad', {}), (BOUNDARY, {'bad': '1'}), (BOUNDARY, {'960': 'bad'}), (None, {'960': '1'})]: + with self.subTest(boundary=boundary, checkpoints=checkpoints), self.assertRaises(DouyinError): + NoticePollingSession(self.browser(), 'safe', '123', boundary, checkpoints) + + def test_stop_prevents_further_reads_and_manager_restarts_from_persisted_state(self): + browser = self.browser() + manager = SubscriptionManager(browser) + self.assertTrue(manager.start('safe', '123', BOUNDARY, {'960': '9'})['connected']) + browser._notification_browser.assert_called_once_with('safe') + old = manager._items['safe'] + manager.start('safe', '123', BOUNDARY, {'960': '10'}) + with self.assertRaisesRegex(DouyinError, 'stopped'): + old.poll(100, 0) + self.assertEqual(manager._items['safe'].checkpoints, {'960': '10'}) + manager.poll('safe', 100, 0) + manager.ack('safe', [d['delivery_id'] for d in manager._items['safe'].pending()]) + manager.close() + manager.stop('safe') + with self.assertRaises(DouyinError): + manager.poll('safe', 100, 0) + with self.assertRaises(DouyinError): + manager.ack('safe', []) + + def test_notifications_pin_one_page_when_multiple_tabs_exist(self): + browser = DouyinBrowser(lambda _: 'http://127.0.0.1:19000') + browser._target_list = Mock(return_value=(None, 19000, [ + {'type':'page', 'id':'profile', 'url':'https://www.douyin.com/user/other'}, + {'type':'page', 'id':'chat', 'url':'https://www.douyin.com/chat'}, + {'type':'page', 'id':'self', 'url':'https://www.douyin.com/user/self'}, + ])) + self.assertEqual(browser._notification_browser('safe').target_id, 'self') + browser._target_list.return_value = (None, 19000, [{'type':'page','id':'external','url':'https://example.com'}]) + with self.assertRaisesRegex(DouyinError, 'notification page is unavailable'): + browser._notification_browser('safe') + + def test_changed_identity_after_start_is_reported(self): + browser = self.browser() + session = self.session(browser) + browser.identity.side_effect = DouyinError('identity changed') + with self.assertRaisesRegex(DouyinError, 'identity'): + session.poll(100, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/browser_gateway/test_private_messages.py b/browser_gateway/test_private_messages.py new file mode 100644 index 0000000..020595b --- /dev/null +++ b/browser_gateway/test_private_messages.py @@ -0,0 +1,286 @@ +import unittest +import threading +import json +import subprocess +from contextlib import nullcontext, contextmanager +from unittest.mock import Mock, patch + +from .platform.douyin import DouyinBrowser, DouyinError, inbox_expression, im_expression, private_identity_expression +from .server.http import Gateway, GatewayHandler, RequestError + + +class PrivateMessageTests(unittest.TestCase): + def test_private_login_uses_sdk_uid_and_never_queries_profile(self): + script=''' +const vm=require('node:vm'),assert=require('node:assert/strict'); +const source=SOURCE; +const sdk={ctx:{option:{userId:'123'}},getConversationList:()=>{throw Error('login must not read messages')}}; +const req={m:{one:function(){/* getOrCreatePrivateConversationByUid */}},c:{one:{exports:{A:{instance:{imSdkService:{imSdkManager:{getImSdkInstance:()=>sdk}}}}}}}}; +const chunks=[];chunks.push=function(args){args[2](req)}; +const context={location:{origin:'https://www.douyin.com'},fetch:()=>{throw Error('profile is not chat authentication')},crypto:{randomUUID:()=> 'id'},window:{'@pc-im/im:version':chunks}}; +(async()=>{ + let result=await vm.runInNewContext(source,context);assert.equal(result.status,'logged_in');assert.equal(result.uid,'123'); + sdk.ctx.option.userId='999';result=await vm.runInNewContext(source,context);assert.equal(result.code,'ACCOUNT_IDENTITY_MISMATCH'); + context.window={};result=await vm.runInNewContext(source,context);assert.equal(result.status,'manual_login');assert.equal(result.reason,'awaiting_login'); +})().catch(e=>{console.error(e);process.exitCode=1}); +'''.replace('SOURCE',json.dumps(private_identity_expression('123'))) + result=subprocess.run(['node','-e',script],capture_output=True,text=True,timeout=10) + self.assertEqual(result.returncode,0,result.stderr) + + def test_private_message_entry_reuses_logged_in_profile_without_qr(self): + from .platform.douyin import CHAT_PAGE_URL, CHAT_ENTRY_SCRIPT + self.assertEqual(CHAT_PAGE_URL,'https://www.douyin.com/user/self') + self.assertIn('im-entry',CHAT_ENTRY_SCRIPT) + self.assertIn('something-button',CHAT_ENTRY_SCRIPT) + self.assertIn('imSaasContainerId',CHAT_ENTRY_SCRIPT) + self.assertNotIn('get_qrcode',CHAT_ENTRY_SCRIPT) + script=''' +const vm=require('node:vm'),assert=require('node:assert/strict');let opened=false,clicks=0; +const container={getBoundingClientRect:()=>({width:opened?700:0,height:opened?400:0})}; +const entry={click:()=>{clicks++;opened=true}}; +const document={querySelector:selector=>selector==='#imSaasContainerId'?container:entry}; +(async()=>{const context={document,location:{origin:'https://www.douyin.com'},Date,setTimeout}; +assert.equal((await vm.runInNewContext(SOURCE,context)).ready,true); +assert.equal((await vm.runInNewContext(SOURCE,context)).ready,true);assert.equal(clicks,1); +})().catch(e=>{console.error(e);process.exitCode=1}); +'''.replace('SOURCE',json.dumps(CHAT_ENTRY_SCRIPT)) + result=subprocess.run(['node','-e',script],capture_output=True,text=True,timeout=10) + self.assertEqual(result.returncode,0,result.stderr) + + def test_gateway_requests_queue_before_the_native_exclusive_lock(self): + active=threading.Lock();entered=threading.Event();release=threading.Event();second=threading.Event();errors=[] + @contextmanager + def native_lock(alias): + if not active.acquire(blocking=False):raise AssertionError('native lock overlapped') + try:yield + finally:active.release() + runtimes=Mock();runtimes.alias_lock.side_effect=native_lock + gateway=Gateway(runtimes,'token','node',Mock()) + def first(): + try: + with gateway._alias_lock('account'):entered.set();release.wait(3) + except BaseException as e:errors.append(e) + def following(): + try: + with gateway._alias_lock('account'):second.set() + except BaseException as e:errors.append(e) + t1=threading.Thread(target=first);t2=threading.Thread(target=following);t1.start();self.assertTrue(entered.wait(1));t2.start() + try:self.assertFalse(second.wait(.1));self.assertEqual(errors,[]) + finally:release.set();t1.join(3);t2.join(3) + self.assertTrue(second.is_set());self.assertEqual(errors,[]) + + def test_extra_private_login_routes_are_removed(self): + handler=object.__new__(GatewayHandler);gateway=Mock();handler.server_as_gateway=Mock(return_value=Mock(gateway=gateway)) + for action in ['private-login-qr','private-login-verify']: + with self.assertRaises(RequestError):handler._route('POST','/v1/browsers/account/douyin/'+action,{}, {'expected_uid':'123'}) + + def test_listener_pacing_does_not_wait_while_holding_alias_lock(self): + gateway = object.__new__(Gateway) + gateway._alias_lock = lambda alias: threading.Lock() + gateway._require_douyin_generation = Mock() + gateway.subscriptions = Mock() + gateway.subscriptions.poll.return_value = [] + payload = {'runtime_id':'a'*64, 'network_id':'native-'+'b'*32, 'binding_version':1} + self.assertEqual(gateway.poll_douyin_events('account', payload, {'wait':['25']}), []) + gateway.subscriptions.poll.assert_called_once_with('account', 50, 0) + + def test_listener_restart_validates_checkpoint_before_touching_browser(self): + gateway = object.__new__(Gateway) + gateway.subscriptions = Mock() + with self.assertRaises(RequestError): + gateway.start_douyin_events('account', {'expected_uid':'123','notice_checkpoints':{'960':'1'}}) + gateway.subscriptions.start.assert_not_called() + + def test_inbox_route_is_dispatched(self): + handler = object.__new__(GatewayHandler) + gateway = Mock() + server = Mock(gateway=gateway) + handler.server_as_gateway = Mock(return_value=server) + body = {'expected_uid':'123'} + handler._route('POST', '/v1/browsers/account/douyin/inbox', {}, body) + gateway.douyin_inbox.assert_called_once_with('account',body) + + def test_inbox_script_uses_identity_and_server_messages(self): + script = inbox_expression('123') + for text in ['getConversationList', 'getMessageList', 's:server_message_create_time', 's:sender_uid', 'serverId', 'ACCOUNT_IDENTITY_MISMATCH', 'IM_INBOX_LIMIT_EXCEEDED']: + self.assertIn(text, script) + self.assertNotIn('getMessagesByConversation', script) + self.assertNotIn('sendMessage(', script) + self.assertNotIn('Date.now()', script) + self.assertIn('123', script) + + def test_sdk_network_error_does_not_claim_definitive_rejection(self): + script=''' +const vm=require('node:vm'),assert=require('node:assert/strict'); +const source=SOURCE; +let result={success:false,statusCode:1008,status:1}; +const conversation={type:1,id:'conversation',shortId:'short',toParticipantUserId:'456'}; +const message={serverId:'100',clientId:'client',sender:'123',type:7,content:'{}',sendFunc(){}}; +const sdk={ctx:{option:{userId:'123'}},getConversationList:()=>[conversation],createMessage:async()=>message,sendMessage:async()=>result}; +const req={m:{one:function(){/* getOrCreatePrivateConversationByUid */}},c:{one:{exports:{A:{instance:{imSdkService:{imSdkManager:{getImSdkInstance:()=>sdk}}}}}}}}; +const chunks=[];chunks.push=function(args){args[2](req)}; +const context={location:{origin:'https://www.douyin.com'},fetch:async()=>({ok:true,json:async()=>({status_code:0,user:{uid:'123'}})}),AbortSignal:{timeout:()=>null},Date,setTimeout:()=>0,window:{'@pc-im/im:version':chunks}}; +(async()=>{ + let response=await vm.runInNewContext(source,context);assert.equal(response.status,'unknown');assert.equal(response.code,'SDK_NETWORK_ERROR');assert.equal(response.status_code,1008); + result={success:false,statusCode:0,checkCode:5,checkMessage:'unavailable'};response=await vm.runInNewContext(source,context);assert.equal(response.status,'failed');assert.equal(response.check_code,'5'); + result={success:false,statusCode:0};response=await vm.runInNewContext(source,context);assert.equal(response.status,'unknown'); +})().catch(e=>{console.error(e);process.exitCode=1}); +'''.replace('SOURCE',json.dumps(im_expression({'uid':'456','action':'send','text':'你好','confirm':True},'123'))) + result=subprocess.run(['node','-e',script],capture_output=True,text=True,timeout=10) + self.assertEqual(result.returncode,0,result.stderr) + + def test_inbox_reads_live_sdk_cache_and_honors_committed_checkpoints(self): + script = ''' +const vm=require('node:vm'),assert=require('node:assert/strict'); +const source=SOURCE; +let seen=0; +const message={type:7,serverId:'100',sender:'456',content:'{"aweType":700,"text":"你好"}',ext:{'s:server_message_create_time':'1700000000000'}}; +const conversation={type:1,toParticipantUserId:'456',lastMessage:message,coreInfo:{name:'好友'},getMessageList(){seen++;return [message]}}; +const sdk={ctx:{option:{userId:'123'}},getConversationList:()=>[conversation]}; +const req={m:{one:function(){/* getOrCreatePrivateConversationByUid */}},c:{one:{exports:{A:{instance:{imSdkService:{imSdkManager:{getImSdkInstance:()=>sdk}}}}}}}}; +const chunks=[];chunks.push=function(args){args[2](req)}; +const context={location:{origin:'https://www.douyin.com'},fetch:async()=>{throw Error('inbox polling must not make repeated profile requests')},AbortSignal:{timeout:()=>null},crypto:{randomUUID:()=> 'id'},window:{'@pc-im/im:version':chunks}}; +(async()=>{ + let result=await vm.runInNewContext(source,context); + assert.equal(result.status,'succeeded');assert.equal(result.messages[0].server_id,'100');assert.equal(result.checkpoints['456'],'100');assert.equal(seen,1); + result=await vm.runInNewContext(source.replace('const expected="123", checkpoints={}', 'const expected="123", checkpoints={"456":"100"}'),context); + assert.equal(result.messages.length,0);assert.equal(seen,1); + sdk.ctx.option.userId='999';result=await vm.runInNewContext(source,context);assert.equal(result.code,'ACCOUNT_IDENTITY_MISMATCH');sdk.ctx.option.userId='123'; + message.sender='777';result=await vm.runInNewContext(source,context);assert.equal(result.code,'MESSAGE_IDENTITY_INVALID'); + conversation.toParticipantUserId='bad';result=await vm.runInNewContext(source,context);assert.equal(result.code,'CONVERSATION_IDENTITY_INVALID'); + context.window={};result=await vm.runInNewContext(source,context);assert.equal(result.code,'IM_SDK_NOT_READY'); +})().catch(e=>{console.error(e);process.exitCode=1}); +'''.replace('SOURCE',json.dumps(inbox_expression('123'))) + result=subprocess.run(['node','-e',script],capture_output=True,text=True,timeout=10) + self.assertEqual(result.returncode,0,result.stderr) + + def test_action_ownership_write_returns_a_cdp_value(self): + browser=object.__new__(DouyinBrowser) + browser._evaluate=Mock() + browser.set_action_ownership('account',{'operation_id':'one'}) + self.assertIn('return true',browser._evaluate.call_args.args[1]) + + def test_manual_dm_uses_the_chat_page_not_the_primary_page(self): + browser=DouyinBrowser() + private=Mock();private._evaluate.return_value={'status':'logged_in','uid':'123'};private._confirmed_evaluate.return_value={'status':'succeeded'} + browser._chat_browser=Mock(return_value=private) + browser.identity=Mock(side_effect=AssertionError('primary page used for DM')) + result=browser.action('account','123','dm','456',text='你好',confirm=True) + self.assertEqual(result['status'],'succeeded') + private.identity.assert_not_called() + private._evaluate.assert_called_once_with('account',private_identity_expression('123')) + private._confirmed_evaluate.assert_called_once() + browser.identity.assert_not_called() + + def test_chat_page_is_created_once_and_never_navigates_the_primary_page(self): + browser=DouyinBrowser(endpoint=lambda alias:'http://127.0.0.1:19000') + from urllib.parse import urlsplit + primary={'id':'primary','type':'page','url':'https://www.douyin.com/user/self'} + chat={'id':'chat','type':'page','url':'https://www.douyin.com/user/self#__creatorhub_private_messages'} + browser._target_list=Mock(side_effect=[(urlsplit('http://127.0.0.1:19000'),19000,[primary]),(urlsplit('http://127.0.0.1:19000'),19000,[primary,chat])]) + browser._create_chat_target=Mock(return_value='chat') + dedicated=Mock();cdp=Mock();cdp.evaluate.side_effect=lambda expression:'/user/self' if expression=='location.pathname' else {'ready':True} + dedicated.connection=lambda alias:nullcontext(cdp);browser._browser_for_target=Mock(return_value=dedicated) + self.assertIs(browser._chat_browser('account'),dedicated) + self.assertIs(browser._chat_browser('account'),dedicated) + browser._create_chat_target.assert_called_once() + browser._browser_for_target.assert_called_with('chat') + + def test_chat_page_is_rediscovered_after_gateway_restart(self): + browser=DouyinBrowser(endpoint=lambda alias:'http://127.0.0.1:19000') + from urllib.parse import urlsplit + browser._target_list=Mock(return_value=(urlsplit('http://127.0.0.1:19000'),19000,[{'id':'primary','type':'page','url':'https://www.douyin.com/user/self'},{'id':'chat','type':'page','url':'https://www.douyin.com/user/self#__creatorhub_private_messages'}])) + browser._is_chat_target=Mock(side_effect=lambda alias,target:target['id']=='chat') + browser._create_chat_target=Mock() + dedicated=Mock();cdp=Mock();cdp.evaluate.side_effect=lambda expression:'/user/self' if expression=='location.pathname' else {'ready':True} + dedicated.connection=lambda alias:nullcontext(cdp);browser._browser_for_target=Mock(return_value=dedicated) + self.assertIs(browser._chat_browser('account'),dedicated) + browser._create_chat_target.assert_not_called() + self.assertEqual(browser._chat_targets['account'],'chat') + + def test_chat_entry_does_not_wait_on_background_page_timers(self): + from .platform.douyin import CHAT_ENTRY_SCRIPT + script = ''' +const vm=require('node:vm'),assert=require('node:assert/strict');let clicks=0; +const context={document:{querySelector:s=>s==='#imSaasContainerId'?null:{click:()=>clicks++}},location:{origin:'https://www.douyin.com'},Date,setTimeout:()=>{throw Error('background timers cannot drive readiness')}}; +const result=vm.runInNewContext(SOURCE,context); +assert.equal(typeof result.then,'undefined');assert.equal(result.ready,false);assert.equal(result.clicked,true);assert.equal(clicks,1); +'''.replace('SOURCE', json.dumps(CHAT_ENTRY_SCRIPT)) + result = subprocess.run(['node', '-e', script], capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + + def chat_setup(self, cdp): + from urllib.parse import urlsplit + browser = DouyinBrowser(endpoint=lambda alias: 'http://127.0.0.1:19000') + browser._target_list = Mock(return_value=(urlsplit('http://127.0.0.1:19000'), 19000, [{'id': 'chat', 'type': 'page', 'url': 'https://www.douyin.com/user/self#__creatorhub_private_messages'}])) + browser._chat_targets['account'] = 'chat' + dedicated = Mock() + dedicated.connection = lambda alias: nullcontext(cdp) + browser._browser_for_target = Mock(return_value=dedicated) + return browser, dedicated + + def test_stale_chat_page_is_refreshed_once_without_touching_the_primary_page(self): + from .platform.douyin import CHAT_ENTRY_SCRIPT + cdp = Mock(); refreshed = False + def evaluate(script): + if script == 'location.pathname': return '/user/self' + return {'ready': refreshed, 'clicked': not refreshed} + def call(method, *args, **kwargs): + nonlocal refreshed + if method == 'Page.reload': refreshed = True + cdp.evaluate.side_effect = evaluate; cdp.command.side_effect = call + browser, dedicated = self.chat_setup(cdp) + with patch('browser_gateway.platform.douyin.time.monotonic', side_effect=[0, 0, 13, 13, 13]), patch('browser_gateway.platform.douyin.time.sleep'): + self.assertIs(browser._chat_browser('account'), dedicated) + self.assertEqual([c.args[0] for c in cdp.command.call_args_list], ['Page.enable', 'Page.reload']) + cdp.wait_event.assert_called_once() + self.assertEqual(cdp.wait_event.call_args.args[0], 'Page.domContentEventFired') + self.assertEqual(sum(c.args[0] == CHAT_ENTRY_SCRIPT for c in cdp.evaluate.call_args_list), 2) + browser._browser_for_target.assert_called_once_with('chat') + + def test_chat_polling_clicks_entry_once_and_still_reports_failure_after_refresh(self): + from .platform.douyin import CHAT_ENTRY_SCRIPT + cdp = Mock() + cdp.evaluate.side_effect = lambda script: '/user/self' if script == 'location.pathname' else {'ready': False, 'clicked': True} + browser, _ = self.chat_setup(cdp) + with patch('browser_gateway.platform.douyin.time.monotonic', side_effect=[0, 0, 1, 13, 13, 13, 14, 26]), patch('browser_gateway.platform.douyin.time.sleep'): + with self.assertRaisesRegex(DouyinError, 'private-message panel.*refresh'): + browser._chat_browser('account') + self.assertEqual(sum(c.args[0] == CHAT_ENTRY_SCRIPT for c in cdp.evaluate.call_args_list), 2) + self.assertEqual([c.args[0] for c in cdp.command.call_args_list], ['Page.enable', 'Page.reload']) + + def test_chat_invalid_response_is_not_hidden_by_refresh(self): + cdp = Mock() + cdp.evaluate.side_effect = ['/user/self', {}] + browser, _ = self.chat_setup(cdp) + with self.assertRaisesRegex(DouyinError, 'response is invalid'): + browser._chat_browser('account') + cdp.command.assert_not_called() + + def test_browser_inbox_validation_and_response(self): + browser = object.__new__(DouyinBrowser) + browser.identity = Mock() + browser._evaluate = Mock(return_value={'status': 'succeeded', 'account_uid': '123', 'messages': []}) + browser._chat_browser = Mock(return_value=browser) + self.assertEqual(browser.inbox('account', '123')['status'], 'succeeded') + browser.identity.assert_not_called() + with self.assertRaises(DouyinError): browser.inbox('account', 'invalid') + browser._evaluate.return_value = [] + with self.assertRaises(DouyinError): browser.inbox('account', '123') + + def test_gateway_generation_and_response(self): + gateway = object.__new__(Gateway) + gateway.browser = Mock() + gateway.browser.inbox.return_value = {'status':'succeeded','account_uid':'123','messages':[]} + gateway._alias_lock = Mock(return_value=nullcontext()) + gateway._require_douyin_generation = Mock() + payload = {'runtime_id':'a'*64,'network_id':'native-'+'b'*32,'binding_version':1,'expected_uid':'123'} + self.assertEqual(gateway.douyin_inbox('alias', payload)['status'],'succeeded') + self.assertEqual(gateway._require_douyin_generation.call_count,2) + gateway.browser.inbox.assert_called_once_with('alias','123',{}) + for bad in [{},dict(payload,expected_uid='bad'),dict(payload,binding_version=0)]: + with self.assertRaises(RequestError):gateway.douyin_inbox('alias',bad) + gateway.browser.inbox.return_value = {'status':'failed','code':'IM_SDK_NOT_READY'} + self.assertEqual(gateway.douyin_inbox('alias',payload)['code'],'IM_SDK_NOT_READY') + gateway.browser.inbox.side_effect = DouyinError('broken') + with self.assertRaises(RequestError):gateway.douyin_inbox('alias',payload) diff --git a/docs/research/douyin-event-listener.md b/docs/research/douyin-event-listener.md new file mode 100644 index 0000000..eaafe54 --- /dev/null +++ b/docs/research/douyin-event-listener.md @@ -0,0 +1,175 @@ +# 抖音账号通知：实现、操作与验收 + +更新：2026-10-06。用户已修订规则：**互动通知必须补齐平台仍可返回的历史和漏收记录，包括关闭期间发生的通知。** 私信保留在独立收件箱，本次不改变其同步范围。 + +## 结果与范围 + +已真实验证：关注通知能够进入数据库并在聚合页显示；关闭时不保存，但重新开启或重新连接后会补齐可读历史。此前关闭期间未保存的关注通知 `7693513967616263205` 现已补入，原发生时间保持 `2026-10-06T11:23:29Z`，标记为历史记录。当前核验共保存 67 条互动通知，其中 66 条历史记录，最早的通知发生于 2024 年；已保存通知不会重复添加。 + +一次关注样本的平台时间至网关接收约 **5.6 秒**，至数据库保存约 **7.4 秒**。这不是延迟分布或性能承诺；产品仍按“先测量再定”确定延迟目标。点赞、评论、转发已覆盖解析单元测试，本轮没有分别制造这些类别的真实新事件，不把关注实测扩展解释为全部类别已实测。 + +本次回填证据：`.data/validation/account-notice-history-backfill-live.json`。此前初版记录保留在 `.data/validation/account-notice-polling-live.json`，其中“不补录关闭期间通知”的旧规则已被本次修改替代。 + +## 事件聚合资料与作品封面 + +互动用户显示真实昵称和 UID；昵称通过平台返回的 `sec_uid` 链接用户主页。发生时间在前、接收时间紧邻其后。所属账号与其筛选选项仅显示昵称。对应作品使用宽 48px 的本地小封面并链接作品详情，图文指向 `/note/`；资料缺失时明确提示，缺封面不取消作品链接，也不使用数字 UID 猜测主页。 + +通知解析提取用户与作品资料，migration 1056 保存新增字段。历史重复读取仅补齐相同互动 UID、相同作品 ID 的资料，保留原始事件时间、内容和历史标记。封面复用现有 `<作者 UID>/<作品 ID>.<扩展名>` 文件缓存，覆盖未入作品库的通知作品，不创建占位账号或作品；`GET /api/creator/events/:id/cover` 只读取本地文件，关闭账号不提供其事件封面。下载失败记录 `work_cover_error` 与关联日志，通知仍保存，下次收到同一通知时重试；实际文件读取或图片显示错误不能伪装为普通未缓存。 + +本次真实验证：67 条历史事件原有字段完全不变，59 条补齐昵称和主页标识，35 条关联本地封面，涉及 16 个不同作品；封面接口返回 HTTP 200、`image/jpeg`，真实页面显示 48×64px 图片和正确主页/作品链接，时间列相邻、所属账号无 UID。没有发送私信或制造新互动。两账号开关曾被其他来源关闭，经用户重新确认恢复开启；`1004291301` 接收正常，`肉宝儿` 等待抖音用户确认登录，其 8 条资料尚未补齐，不能宣称全部完成。确认登录后按已有历史核对机制继续补齐平台仍可获取的资料。 + +证据：`.data/validation/account-event-details-live.json`，页面截图 `.data/validation/account-event-details-live.png` 与 `.data/validation/account-event-details-covers-live.png`。275 项前端测试、29 项相关 Python 测试、Go 单元测试及 creator 模块全部 PostgreSQL 测试通过，creator 覆盖率 72.5%，封面缓存处理覆盖率 100%；完整 PostgreSQL 全包运行曾超时，不作为通过证据。 + +### 用户确认登录后的复验 + +用户确认 `肉宝儿` 已登录后，剩余 8 条资料已自动补齐：67 条事件均有真实昵称和 secUID 主页标识，原事件所有非资料字段完全不变。真实页面按 `肉宝儿` 筛选展示 8 条事件，8 个昵称均有主页链接，所属账号仅昵称，缺封面的评论仍有作品入口。35 条事件关联 16 个作品的本地封面；其余 4 个作品涉及 6 条事件，通知未提供封面和作者资料，保留缺失提示，不推测图片来源。 + +本次复验未修改开关或制造互动。`肉宝儿` 监听为 `ready`；`1004291301` 开关仍开启，但浏览器已停止，状态为 `error`，提示在“我的账号”启动浏览器，不能报告两个账号均在正常接收。新增证据 `.data/validation/account-event-details-login-confirmed.json` 与同名前缀截图。 + +## 使用方式 + +1. 在“我的账号”启动该账号对应浏览器并完成登录、同步。不要使用其他账号的旧浏览器代替。 +2. 开启该行监听。开启表示设置已保存，**不等于接收已成功**。 +3. 等待状态变为“接收正常”。状态提示可查看最近成功检查时间；异常时展示实际原因。 +4. 在“事件聚合”查看已开启账号的通知，支持账号、事件类型和最近时间范围筛选。时间范围按应用接收时间计算。 +5. 关闭后不再保存通知，该账号历史记录从聚合页隐藏，但数据库保留；再次开启后已有记录重新可见，并补齐平台仍可返回的历史和漏收通知，包括关闭期间的通知。 + +只记录，不自动点赞、评论、关注、发私信或回复。此轮真实关注操作是得到明确授权后的验证，不是后台功能。 + +## 当前接收链路 + +```text +账号开关：enabled + generation + → Go 账号工作循环 + → 网关初始化通知读取会话（传入数据库保存的边界、分组读取位置） + → UID 校验 + 选择并固定该账号浏览器的一张抖音页面 + → 只读 GET /aweme/v1/web/notice/（组 700、960、961） + → Python 从原始文本解析 64 位通知 ID、逐页读取并标准化 + → notices + checkpoint + → 同一数据库事务保存事件、分组读取位置、最后成功时间 + → 事务成功后 ACK；重复通知按账号、类别、通知 ID 去重 +``` + +私信仍走独立 IM 收件箱，不读取通知列表中的私信正文。旧 `INSTALL_SCRIPT`、webpack 特征匹配和 WebSocket bridge 生命周期已移除，不保留 SDK 失败后回退的双路径。 + +### 读取约定 + +- 请求 `is_mark_read=0`，不标记通知已读。 +- 每页 50 条。真实账号验证了 1/5/10/20/50 条可读，而 100 条返回业务码 5；50 条不是未经验证的猜测。 +- 每次读取前核验 UID；每条通知的接收账号 `user_id` 也必须匹配。 +- 使用原始响应文本在 Python 中 `json.loads`，不在浏览器里对通知 JSON 做数值转换，避免 JavaScript 舍入 64 位 ID。 +- 首次开启、重新开启、重新连接/重启均按 `has_more` / `max_time` 读完全部可见分页；不再以开启时间或已保存头部 ID 截断历史，也不以 100 页为上限。 +- 正常运行增量读取到上次保存的头部通知所在整页；每轮完整历史读取确认成功后，约每 5 分钟再次完整核对，补回较早分页中的漏收记录。整页处理，不假定页内完全按时间顺序排列。 +- 游标必须为正整数并严格递减；循环游标、无效响应、账号不一致和任一分组失败会明确报错，不提交部分分组的新读取位置。 +- 查不到上次保存的头部 ID 时，完整读取当前可见列表，报告“可能有遗漏”，并在日志记录分组；不能证明平台已删除或不再返回的通知不存在。 +- 关闭或重新开启会递增 generation 并清除本轮边界和读取位置；迟到的旧会话写入被拒绝。 +- 工作循环约每 5 秒读取一次，等待发生在 Go 侧，不占用浏览器操作锁。网络请求与手动操作仍按账号串行，忙碌或超时明确报错并重试。 + +首次身份读取的 `platform_now` 只用于区分历史记录和开启后的新通知，不再作为丢弃依据。历史与新通知都保留真实平台发生时间；原已保存通知的时间、内容和历史标记不会被重复读取覆盖。完整核对完成并确认之后才开始计算下一次历史核对间隔；读取跨越多个投递批次时，最后一批确认前不算完成。 + +列表接口是网页内部接口，没有公开稳定契约。周期完整核对可以补回仍在列表中的迟到或漏收记录；平台已经删除或不再返回的记录无法恢复。较大历史量会延长首次读取时间，不能把接口成功或部分批次保存说成全部历史已补齐。 + +## 本机只读验证脚本 + +交付： + +- `scripts/validate_douyin_event_listener.py` +- `scripts/test_validate_douyin_event_listener.py` + +脚本通过 `browser-harness` 连接明确指定的本机 CDP；不导航页面、不修改应用开关、不发互动、不安装页面监听器。检查登录身份、三个通知分组，再用实际生产解析代码完成一次读取、确认以及临时会话的读取位置恢复。临时读取位置只存在于脚本内存，不写应用数据库。 + +仅报告结构、通知类别与条数，不输出通知正文、通知 ID、Cookie 或请求凭据。账号 UID 用于本机身份核验。 + +### 准备 + +已安装 `browser-harness`，且其 Python 环境能够导入项目依赖 `websocket-client`。若出现 `No module named 'websocket'`，应为该工具实际使用的 Python 安装依赖，而不是忽略错误；例如当前安装方式： + +```bash +PATH="$HOME/.local/bin:$PATH" uv pip install \ + --python "$HOME/.local/share/uv/tools/browser-harness/bin/python" websocket-client +``` + +从该账号运行时或本机服务状态取得正在运行的 CDP 端口；端口可能随重启改变，不能把本次 19000/19002 永久写死。没有运行中的正确浏览器或没有登录时，由账号持有人完成启动、扫码及确认。 + +```bash +PATH="$HOME/.local/bin:$PATH" python3 scripts/validate_douyin_event_listener.py \ + --cdp-url http://127.0.0.1:<该账号的CDP端口> \ + --expected-uid <已绑定的抖音UID> \ + --harness-name <本次独立名称> + +python3 scripts/test_validate_douyin_event_listener.py +``` + +常见结果： + +| `outcome` | 含义 | +| --- | --- | +| `notification_polling_read_verified_delivery_unverified` | 接口、生产解析和临时读取位置恢复通过；**没有验证真实新事件入库** | +| `notification_api_readable_delivery_unverified` | 仅接口小样本可读 | +| `notification_polling_failed` | 生产读取失败，查看 `polling_read_probe.error` | +| `notification_polling_gap` | 读取成功，但发现无法确认完整性的边界缺口 | +| `notification_api_unavailable` | 分组读取、响应结构或身份不符合约定 | +| `no_matching_logged_in_page` | 未找到已登录且 UID 匹配的页面 | +| `multiple_matching_pages` | 多个匹配页面的诊断记录；生产会话会固定选择一个页面，并再次核验 UID | +| `probe_failed` | 工具依赖、CDP 或执行错误 | + +退出码 0 只表示脚本完整输出诊断。必须阅读 `outcome`，不能把退出码、HTTP 200 或“接收正常”当成真实事件验收。 + +## 单元与数据库验证 + +```bash +python3 -m unittest discover -s browser_gateway -t . -p 'test_*.py' +python3 scripts/test_validate_douyin_event_listener.py + +go test ./... + +CREATORHUB_POSTGRES_TEST_URL='<本机测试数据库连接>' go test \ + ./internal/creator ./internal/controlplane/api \ + -run 'TestNotificationHistoryBackfill|TestNotificationCheckpoint|Test.*Listener|TestAccountEvent|TestEventAggregationFilter' -count=1 + +cd web +node --test tests/event-aggregation*.test.cjs tests/account-management-unified-list.test.cjs +npx tsc --noEmit +antd lint src/pages/creator/events/index.tsx --format json +antd lint src/components/accounts/AccountManagementList.tsx --format json +``` + +数据库测试使用独立临时 schema，不把示例通知注入实际账号。普通测试不自动创建真实互动，也不运行 Playwright/Cypress 等 E2E 测试。 + +重点覆盖：首次历史补齐、重启时读取已保存头部 ID 之前的缺漏、周期完整核对、超过 100 页、超过单批 100 条、历史标记和原时间保留、重复、64 位 ID、未知类别、错误响应、游标循环、UID 变化、确认前重放、事务回滚、读取位置恢复、关闭代次拒绝及重新开启补入，以及多标签页明确选择。 + +## 真实验收方法 + +仅在另行取得授权后使用两个已有账号；未授权不发内容、不联系其他账号。 + +1. 核验两个浏览器的 UID，记录监听设置、关注关系及原始事件数量。 +2. 启用接收账号并确认最近成功检查；在另一账号产生一次允许的自然互动。 +3. 比对抖音通知 ID、数据库事件 ID 和页面记录，核对接收账号、互动 UID、类别与时间。 +4. 再读两轮、重启网关或控制面，确认已保存事件不重复，读取位置持续恢复。 +5. 关闭接收账号并等到 `stopped`；确认聚合隐藏历史，但数据库保留。 +6. 产生另一条互动，并在抖音端证明它是**独立的新通知**。仅“点击成功”或数据库数量未增加不够：平台可能合并或抑制重复关注。 +7. 关闭期间等待至少两轮正常周期，确认新通知 ID 尚未保存；重新开启后应补入这条通知，并保留平台原发生时间。再读取两轮和重启，确认不会重复。另验证已保存头部 ID 更早的漏收记录可通过周期完整核对补回。 +8. 恢复原设置、原关系并核验。每一种需要承诺支持的事件类别分别实测；禁止把关注通过说成全部类别通过。 + +## 初版真实记录（旧规则，已被替代） + +- 接收账号 A：`1004291301`（UID `99491952055`）；账号 B：`肉宝儿`（UID `3296761621001100`）。两者身份均通过核验。 +- A 开启期间：B 的关注产生通知 `7693511107768353829`；平台时间 `2026-10-06T11:12:24Z`，网关接收 `11:12:29.642467Z`，数据库保存 `11:12:31.363603Z`。页面和 API 均显示这一条关注，UID 与类型一致。 +- A 关闭后：聚合返回 0 条，数据库仍保留原记录。B 再次关注 A 未产生独立的新平台通知，**这次操作不作为停止接收证据**。 +- B 关闭并确认 `stopped` 后：A 的关注产生独立通知 `7693513967616263205`（平台时间 `2026-10-06T11:23:29Z`），与此前 B 收到的旧通知 `7682082397432644666` 不同。该新 ID 没有入库。B 重新开启后仍未补入，最终数据库仅原来的一条关注通知。 +- 真实验证开始时两账号开关均为开启；结束恢复这一状态，双方原有互相关注关系也已核验恢复。更早只读调研曾恢复关闭，这是不同时间点的快照，不以旧记录覆盖当前设置。 +- 实测中发现多普通标签页会使旧浏览器选页逻辑歧义；已增加通知专用固定选页，并有回归测试。 +- 实测中 A 浏览器被 systemd 的 1 GB 内存上限触发 OOM 终止，已根据系统日志定位，不归咎于通知 SDK。恢复时沿用原 profile 和登录信息，未清空 Cookie。本模块不调整浏览器内存策略；对应运行环境的资源限制需独立处理。 + +## 本次历史补齐记录 + +- 修改前数据库只有一条互动通知，已知关闭期间的 `7693513967616263205` 不在库中。 +- 新版本已补入该通知及较早历史：账号 A 保存 59 条（58 条历史），账号 B 保存 8 条（均为历史），总计 67 条；最早时间分别为 2025-07-14 和 2024-06-04。 +- 已有一条新通知没有被改写为历史，补入记录保留平台原时间；重复读取未发现重复记录。 +- 验证期间发现两账号被改为关闭，本次没有调用关闭接口，来源未确认。用户明确要求恢复开启后，两者均核验为 `ready`，聚合 API 可查看补入的记录。 +- 本次没有发送私信、产生公开内容或再制造互动；仅读取并补齐既有真实通知。 +- 15 项轮询单元测试和 5 项验证脚本测试通过，轮询代码覆盖率 99%；新增数据库测试验证初次历史、关闭时拒绝写入、重新开启补入、原时间和历史标记保留及去重。 + +## 历史原因 + +此前 WebSocket 方案依赖 `NOTICE_PUSH_EVENT_NAMES`、`NoticeFrontier` 和 codec 等私有 webpack 特征。2026-10-06 在已登录页面的 5893 个 factory 中，4 项所需特征均缺失，安装探针返回 `SDK_CHANGED`。参考 `douyin-pc` 的 `get_notifications.py` 已有 `is_mark_read=0`、`has_more/max_time` 分页和 Python 原文 JSON 解码约定；本实现沿用这些已验证模式，而不是继续维护失效的私有 SDK 桥接。 diff --git a/internal/controlplane/api/account_environment.go b/internal/controlplane/api/account_environment.go index 3e8a662..ced5374 100644 --- a/internal/controlplane/api/account_environment.go +++ b/internal/controlplane/api/account_environment.go @@ -28,7 +28,10 @@ func soleGateway(ctx context.Context, store HubStore) (hub.Gateway, error) { // ensureAccountEnvironment 幂等补建账号环境：已绑定（任意出口）原样返回； // 未绑定时以 alias=账号 ID、派生 seed、指定指纹、直连出口创建。 -func ensureAccountEnvironment(ctx context.Context, store HubStore, accountID string, fingerprint hub.Fingerprint) (hub.EnvironmentContext, bool, error) { +func ensureAccountEnvironment(ctx context.Context, store HubStore, accountID string, fingerprint hub.Fingerprint, memoryLimitMB int) (hub.EnvironmentContext, bool, error) { + if memoryLimitMB != 0 && (memoryLimitMB < 512 || memoryLimitMB > 65536) { + return hub.EnvironmentContext{}, false, hub.ErrInvalid + } environment, err := store.GetEnvironmentContextForAccount(ctx, accountID) if err == nil { return environment, false, nil @@ -42,7 +45,7 @@ func ensureAccountEnvironment(ctx context.Context, store HubStore, accountID str } // seed 由 CreateBoundEnv 从账号 bigint id + 1000 派生（数字主键）。 return store.CreateBoundEnv(ctx, hub.Env{ - Alias: accountID, Name: accountID, Gateway: gateway.Name, Fingerprint: fingerprint, + Alias: accountID, Name: accountID, Gateway: gateway.Name, Fingerprint: fingerprint, MemoryLimitMB: memoryLimitMB, }, accountID, "") } @@ -51,7 +54,7 @@ func startAccountEnvironment(ctx context.Context, store HubStore, accountID stri environment, err := store.GetEnvironmentContextForAccount(ctx, accountID) if errors.Is(err, hub.ErrNotFound) { // start 自愈补建时创建表单不可得，零值指纹 = 派生 seed + 浏览器默认参数。 - environment, _, err = ensureAccountEnvironment(ctx, store, accountID, hub.Fingerprint{}) + environment, _, err = ensureAccountEnvironment(ctx, store, accountID, hub.Fingerprint{}, 0) } if err != nil { return err diff --git a/internal/controlplane/api/account_environment_unit_test.go b/internal/controlplane/api/account_environment_unit_test.go index a8c5346..17ac018 100644 --- a/internal/controlplane/api/account_environment_unit_test.go +++ b/internal/controlplane/api/account_environment_unit_test.go @@ -54,7 +54,7 @@ func TestAccountEnvironmentAutoBindingAndStart(t *testing.T) { if _, err := hubStore.CreateGateway(ctx, "gw-main", gatewayServer.URL, gateway.token); err != nil { t.Fatal(err) } - response := do(app, http.MethodPost, "/api/phase-a/accounts/"+accountID+"/environment", "") + response := do(app, http.MethodPost, "/api/phase-a/accounts/"+accountID+"/environment", `{"memory_limit_mb":8192}`) if response.Code != http.StatusOK { t.Fatalf("expected 200 environment rebind, got %d: %s", response.Code, response.Body.String()) } @@ -71,7 +71,7 @@ func TestAccountEnvironmentAutoBindingAndStart(t *testing.T) { t.Fatal(err) } environment, err := hubStore.GetEnvironmentContext(ctx, accountID) - if err != nil || environment.Fingerprint.Seed != accountRowID+1000 || environment.Exit.ID != "" { + if err != nil || environment.MemoryLimitMB != 8192 || environment.Fingerprint.Seed != accountRowID+1000 || environment.Exit.ID != "" { t.Fatalf("auto-bound environment: %#v err=%v (account row id %d)", environment, err, accountRowID) } // 幂等：重复补建返回既有环境 @@ -149,7 +149,7 @@ func TestAccountEnvironmentDeletionLifecycle(t *testing.T) { } accountID := maintenanceAccountFixture(t, ctx, accountStore) - if _, _, err := ensureAccountEnvironment(ctx, hubStore, accountID, hub.Fingerprint{}); err != nil { + if _, _, err := ensureAccountEnvironment(ctx, hubStore, accountID, hub.Fingerprint{}, 0); err != nil { t.Fatal(err) } if _, err := hubStore.GetEnvironmentContext(ctx, accountID); err != nil { diff --git a/internal/controlplane/api/account_fingerprint_test.go b/internal/controlplane/api/account_fingerprint_test.go index 76c7aef..b6a8a63 100644 --- a/internal/controlplane/api/account_fingerprint_test.go +++ b/internal/controlplane/api/account_fingerprint_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "net/http" "os" "testing" @@ -63,7 +64,7 @@ func TestEnvironmentCreationDoesNotCreatePlatformAccount(t *testing.T) { if err := json.Unmarshal(response.Body.Bytes(), &env); err != nil { t.Fatal(err) } - if env.AccountID != "" || env.ProfileID != env.Alias || env.Fingerprint.Seed < 1001 || env.Fingerprint.Lang != "zh-CN" { + if env.MemoryLimitMB != 2048 || env.AccountID != "" || env.ProfileID != env.Alias || env.Fingerprint.Seed < 1001 || env.Fingerprint.Lang != "zh-CN" { t.Fatalf("unbound environment: %#v", env) } var count int @@ -74,23 +75,73 @@ func TestEnvironmentCreationDoesNotCreatePlatformAccount(t *testing.T) { if err != nil || len(pending) != 1 { t.Fatalf("pending: %v %v", pending, err) } - for _, body := range []string{`{"gateway":"gw-main","name":"手填昵称"}`, `{"gateway":"gw-main","platform_account_key":"123"}`, `{"gateway":"gw-main","fingerprint":{"platform":"android"}}`, `{"gateway":"gw-main","fingerprint":{"seed":42}}`, `{"gateway":"gw-main","fingerprint":{"proxy_server":"http://invalid:8080"}}`} { + for _, body := range []string{`{"gateway":"gw-main","memory_limit_mb":511}`, `{"gateway":"gw-main","memory_limit_mb":-1}`, `{"gateway":"gw-main","memory_limit_mb":65537}`, `{"gateway":"gw-main","memory_limit_mb":512.5}`, `{"gateway":"gw-main","memory_limit_mb":"4096"}`, `{"gateway":"gw-main","name":"手填昵称"}`, `{"gateway":"gw-main","platform_account_key":"123"}`, `{"gateway":"gw-main","fingerprint":{"platform":"android"}}`, `{"gateway":"gw-main","fingerprint":{"seed":42}}`, `{"gateway":"gw-main","fingerprint":{"proxy_server":"http://invalid:8080"}}`} { if response := do(app, http.MethodPost, "/api/creator/environments", body); response.Code != http.StatusBadRequest { t.Fatalf("invalid input accepted: %s %d %s", body, response.Code, response.Body.String()) } } + for _, limit := range []int{512, 65536, 4096} { + response := do(app, http.MethodPut, "/api/creator/environments/"+env.Alias+"/fingerprint", fmt.Sprintf(`{"memory_limit_mb":%d}`, limit)) + if response.Code != http.StatusOK { + t.Fatalf("pending memory edit: %d %s", response.Code, response.Body.String()) + } + var updated hub.EnvironmentContext + if err := json.Unmarshal(response.Body.Bytes(), &updated); err != nil { + t.Fatal(err) + } + if updated.MemoryLimitMB != limit || updated.Fingerprint != env.Fingerprint || updated.ProfileID != env.ProfileID { + t.Fatalf("pending identity changed: %+v", updated) + } + } + response = do(app, http.MethodPost, "/api/creator/environments", `{"gateway":"gw-main","memory_limit_mb":8192}`) + if response.Code != http.StatusCreated { + t.Fatalf("custom create: %d %s", response.Code, response.Body.String()) + } + var custom hub.EnvironmentContext + if err := json.Unmarshal(response.Body.Bytes(), &custom); err != nil || custom.MemoryLimitMB != 8192 { + t.Fatalf("custom memory: %+v %v", custom, err) + } + if err := hs.DeleteEnv(ctx, custom.Alias); err != nil { + t.Fatal(err) + } result, err := cs.RecordVerifiedEnvironmentLogin(ctx, env.Alias, creator.PlatformIdentity{UID: "99491952055", Nickname: "同步昵称", DouyinNumber: "1004291301", AvatarURL: "https://example.com/avatar.jpg"}) if err != nil { t.Fatal(err) } bound, err := hs.GetEnvironmentContext(ctx, env.Alias) - if err != nil || bound.AccountID != result.AccountID || bound.ProfileID != env.ProfileID || bound.Fingerprint.Seed != env.Fingerprint.Seed { + if err != nil || bound.MemoryLimitMB != 4096 || bound.AccountID != result.AccountID || bound.ProfileID != env.ProfileID || bound.Fingerprint.Seed != env.Fingerprint.Seed { t.Fatalf("binding replaced browser: %#v %v", bound, err) } pending, err = hs.ListPendingEnvironments(ctx) if err != nil || len(pending) != 0 { t.Fatalf("bound env still pending: %v %v", pending, err) } + if _, err := db.ExecContext(ctx, `UPDATE browser_env SET runtime_id='memory-running',runtime_lease_until=now()+interval '1 hour' WHERE alias=$1`, env.Alias); err != nil { + t.Fatal(err) + } + settingsApp := fiber.New() + registerCreator(settingsApp, cs, nil, hs) + response = do(settingsApp, http.MethodPut, "/api/creator/accounts/"+result.AccountID+"/fingerprint", `{"memory_limit_mb":8192}`) + if response.Code != http.StatusOK { + t.Fatalf("running memory edit: %d %s", response.Code, response.Body.String()) + } + var view map[string]any + if err := json.Unmarshal(response.Body.Bytes(), &view); err != nil { + t.Fatal(err) + } + if view["memory_limit_mb"] != float64(8192) || view["runtime_id"] != "memory-running" { + t.Fatalf("memory response: %+v", view) + } + for _, body := range []string{`{"memory_limit_mb":0}`, `{"memory_limit_mb":511}`, `{"memory_limit_mb":65537}`, `{"memory_limit_mb":null}`, `{"memory_limit_mb":8192,"platform":"android"}`} { + response = do(settingsApp, http.MethodPut, "/api/creator/accounts/"+result.AccountID+"/fingerprint", body) + if response.Code != http.StatusBadRequest { + t.Fatalf("invalid edit accepted: %s %d %s", body, response.Code, response.Body.String()) + } + } + updated, err := hs.GetEnvironmentContext(ctx, env.Alias) + if err != nil || updated.MemoryLimitMB != 8192 || updated.RuntimeID != "memory-running" || updated.ProfileID != env.ProfileID || updated.Fingerprint != env.Fingerprint || updated.BindingVersion != bound.BindingVersion { + t.Fatalf("running edit changed identity: %+v %v", updated, err) + } RegisterAccountRoutes(app, nil, nil, nil) if response := do(app, http.MethodPost, "/api/phase-a/accounts", `{"name":"废弃入口"}`); response.Code != http.StatusMethodNotAllowed { t.Fatalf("old creation path remains: %d", response.Code) diff --git a/internal/controlplane/api/accounts_operations.go b/internal/controlplane/api/accounts_operations.go index 0a36d62..933ff46 100644 --- a/internal/controlplane/api/accounts_operations.go +++ b/internal/controlplane/api/accounts_operations.go @@ -64,7 +64,8 @@ func RegisterAccountRoutes(app *fiber.App, store *accountdomain.Store, runtimeSt } // 幂等补建：空体保持零值指纹（seed 仍由账号派生）；可携带创建时未落库的指纹表单重试。 var rebind struct { - Fingerprint hub.Fingerprint `json:"fingerprint"` + Fingerprint hub.Fingerprint `json:"fingerprint"` + MemoryLimitMB int `json:"memory_limit_mb"` } if len(c.Body()) > 0 { if err := decodePhaseA(c, &rebind); err != nil { @@ -81,11 +82,11 @@ func RegisterAccountRoutes(app *fiber.App, store *accountdomain.Store, runtimeSt return hubError(c, err) } defer unlock() - environment, created, err := ensureAccountEnvironment(c.Context(), runtimeStore, c.Params("id"), rebind.Fingerprint) + environment, created, err := ensureAccountEnvironment(c.Context(), runtimeStore, c.Params("id"), rebind.Fingerprint, rebind.MemoryLimitMB) if err != nil { return hubError(c, err) } - return c.JSON(map[string]any{"alias": environment.Alias, "gateway": environment.Gateway, "created": created}) + return c.JSON(map[string]any{"alias": environment.Alias, "gateway": environment.Gateway, "created": created, "memory_limit_mb": environment.MemoryLimitMB}) }) // 账号即环境：start = resume + 启动环境；停止沿用 pause（已联动停环境）。 @@ -249,7 +250,8 @@ func accountEnvironmentView(environment hub.EnvironmentContext) map[string]any { "network_exit_id": environment.Exit.ID, "network_exit_health": environment.Exit.HealthStatus, "schedule_status": scheduleStatus, "schedule_block_reason": scheduleBlockReason, // 指纹回显：编辑页预填用；seed 属账号派生、代理属出口管理，均不外发。 - "fingerprint": environment.Fingerprint, + "fingerprint": environment.Fingerprint, + "memory_limit_mb": environment.MemoryLimitMB, } } diff --git a/internal/controlplane/api/creator.go b/internal/controlplane/api/creator.go index 4d7ce33..26cc45e 100644 --- a/internal/controlplane/api/creator.go +++ b/internal/controlplane/api/creator.go @@ -60,6 +60,8 @@ func registerCreator(app *fiber.App, store *creator.Store, phaseAStore *accountd func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseAStore *accountdomain.Store, hubStore *hub.Store, analyzer creator.ThemeAnalyzer) { registerEnvironmentLoginRoutes(app, store, hubStore) + registerAccountEventRoutes(app, store) + registerPrivateMessageRoutes(app, store, gatewayPrivateMessageSender(hubStore, store)) // Platform records enter through the managed collector/listener, not a public // client-supplied write. The explicit test namespace is kept for isolated // contract tests and never participates in the production listener. @@ -197,14 +199,14 @@ func registerCreatorWithServices(app *fiber.App, store *creator.Store, phaseASto }) // 编辑页指纹浏览器环境：单独保存；运行中的环境停后以新指纹重启，未运行仅落库下次启动生效。 app.Put("/api/creator/accounts/:id/fingerprint", func(c fiber.Ctx) error { - var input hub.Fingerprint - if err := decodeCreator(c, &input); err != nil { + fingerprint, memoryLimitMB, err := decodeEnvironmentSettings(c) + if err != nil { return creatorError(c, err) } if hubStore == nil { return c.Status(fiber.StatusServiceUnavailable).JSON(map[string]string{"error": "environment store unavailable"}) } - view, err := updateAccountEnvironmentFingerprint(c.Context(), hubStore, c.Params("id"), input) + view, err := updateAccountEnvironmentSettings(c.Context(), hubStore, c.Params("id"), fingerprint, memoryLimitMB) if err != nil { return creatorError(c, err) } @@ -698,7 +700,7 @@ func flattenActionEvidence(destination map[string]string, prefix string, value a } } -var errCreatorLoginPending = errors.New("browser is awaiting user login") +var errCreatorLoginPending = errors.New(creator.CollectionLoginRequiredReason) func (browser creatorGatewayBrowser) Identity(ctx context.Context, expectedKey string) (string, error) { identity, err := browser.IdentityProfile(ctx, expectedKey) @@ -776,10 +778,12 @@ func startCreatorEnvironment(ctx context.Context, store HubStore, environment hu return startBrowserRuntime(ctx, store, defaultNetworkExitProbe(), environment, finish) } -// updateAccountEnvironmentFingerprint 编辑页指纹浏览器环境单独保存： -// 运行中的环境先停后启以应用新指纹；未运行仅落库，下次启动自然生效。 -// seed 属账号派生、代理属出口管理，Store 层拒绝透传入参携带。 -func updateAccountEnvironmentFingerprint(ctx context.Context, hubStore HubStore, accountID string, input hub.Fingerprint) (map[string]any, error) { +// updateAccountEnvironmentSettings saves memory for the next start without restarting. +// Fingerprint changes retain the existing stop/start flow; seed and profile stay unchanged. +func updateAccountEnvironmentSettings(ctx context.Context, hubStore HubStore, accountID string, input *hub.Fingerprint, memoryLimitMB *int) (map[string]any, error) { + if (input != nil && (input.ProxyServer != "" || input.Validate() != nil)) || (memoryLimitMB != nil && (*memoryLimitMB < 512 || *memoryLimitMB > 65536)) { + return nil, creator.ErrInvalid + } unlock, err := lockAccountResources(ctx, hubStore, accountID) if err != nil { return nil, fmt.Errorf("%w: lock account environment: %v", creator.ErrUnavailable, err) @@ -793,13 +797,19 @@ func updateAccountEnvironmentFingerprint(ctx context.Context, hubStore HubStore, if err != nil { return nil, fmt.Errorf("%w: load account environment: %v", creator.ErrUnavailable, err) } - restart := environment.RuntimeID != "" + fingerprintChanged := false + if input != nil { + updated := *input + updated.Seed = environment.Fingerprint.Seed + fingerprintChanged = updated != environment.Fingerprint + } + restart := environment.RuntimeID != "" && fingerprintChanged if restart { if err := stopEnvironmentRuntime(ctx, hubStore, environment); err != nil { return nil, fmt.Errorf("%w: stop environment before fingerprint update: %v", creator.ErrUnavailable, err) } } - environment, err = hubStore.UpdateAccountFingerprint(ctx, accountID, input) + environment, err = hubStore.UpdateEnvironmentSettings(ctx, environment.Alias, input, memoryLimitMB) if errors.Is(err, hub.ErrInvalid) { return nil, creator.ErrInvalid } @@ -818,6 +828,7 @@ func updateAccountEnvironmentFingerprint(ctx context.Context, hubStore HubStore, return nil, fmt.Errorf("%w: reload environment after fingerprint restart: %v", creator.ErrUnavailable, err) } } + logrus.WithFields(logrus.Fields{"event_type": "environment_settings_saved", "alias": environment.Alias, "memory_limit_mb": environment.MemoryLimitMB, "fingerprint_changed": fingerprintChanged, "restarted": restart}).Info("browser environment settings saved") return accountEnvironmentView(environment), nil } @@ -1230,6 +1241,7 @@ func newAnonymousBrowser(ctx context.Context, store *hub.Store) (anonymousBrowse operationID := hub.NewOperationID() template.Alias = "anon-" + strings.TrimPrefix(operationID, "operation-") template.Name = "匿名竞品解析" + template.MemoryLimitMB = 2048 template.Gateway = gateway.Name template.Fingerprint.ProxyServer = "" template.Fingerprint.DisableNonProxiedUDP = false @@ -1666,6 +1678,11 @@ func refreshCreatorMetricWork(ctx context.Context, store *creator.Store, phaseAS if account.Platform != creator.PlatformDouyin || (profile.BusinessStatus != "normal" && profile.BusinessStatus != "muted") || profile.LoginStatus != "logged_in" { return creator.ErrConflict } + unlockResources, err := lockAccountResources(ctx, hubStore, accountID) + if err != nil { + return err + } + defer unlockResources() environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) if err != nil { return fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) @@ -1741,6 +1758,21 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ac if err != nil { return err } + syncLease, err := store.ClaimSourceSync(ctx, creator.SourceOwned, account.ID) + if err != nil { + return err + } + defer func() { + if releaseErr := store.ReleaseSourceSync(context.WithoutCancel(ctx), creator.SourceOwned, account.ID, syncLease); releaseErr != nil { + logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed") + } + }() + unlockResources, err := lockAccountResources(ctx, hubStore, account.ID) + if err != nil { + blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays) + return errors.Join(err, blockErr) + } + defer unlockResources() environment, err := hubStore.GetEnvironmentContextForAccount(ctx, accountID) if err != nil { return fmt.Errorf("%w: account environment unavailable: %v", creator.ErrUnavailable, err) @@ -1752,15 +1784,6 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ac if err != nil { return fmt.Errorf("%w: gateway unavailable: %v", creator.ErrUnavailable, err) } - syncLease, err := store.ClaimSourceSync(ctx, creator.SourceOwned, account.ID) - if err != nil { - return err - } - defer func() { - if releaseErr := store.ReleaseSourceSync(context.WithoutCancel(ctx), creator.SourceOwned, account.ID, syncLease); releaseErr != nil { - logrus.WithError(releaseErr).WithField("account_id", account.ID).Warn("creator source sync lease release failed") - } - }() collector, _, err := newCreatorCollector(ctx, account.Platform, gateway, environment, account.PlatformAccountKey, account.PlatformAccountKey, "", creator.SourceOwned, account.ID) if err != nil { blockErr := store.MarkCollectionBlocked(ctx, creator.SourceOwned, account.ID, err.Error(), now, settings.LookbackDays) @@ -1770,8 +1793,18 @@ func syncCreatorOwned(ctx context.Context, store *creator.Store, phaseAStore *ac if windowErr != nil { return windowErr } - _, err = store.CollectSource(ctx, account.Platform, creator.SourceOwned, account.ID, collector, collectionNow) - collectErr := err + logrus.WithFields(logrus.Fields{"account_id": account.ID, "window_end": collectionNow}).Info("creator owned collection started") + report, collectErr := store.CollectSource(ctx, account.Platform, creator.SourceOwned, account.ID, collector, collectionNow) + collectionLog := logrus.WithFields(logrus.Fields{ + "account_id": account.ID, "window_end": collectionNow, + "works_seen": report.WorksSeen, "comments_seen": report.CommentsSeen, + "public_works_seen": report.PublicWorksSeen, "pagination_complete": report.PaginationComplete, + }) + if collectErr != nil { + collectionLog.WithError(collectErr).Warn("creator owned collection failed") + } else { + collectionLog.Info("creator owned collection completed") + } // 作品采集成功后顺带拉 self profile 记账号画像快照（粉丝/关注/获赞/作品总数）。 // 失败不吞：join 进返回错误，由调度器日志可见，但不影响已入库的作品数据。 profileErr := error(nil) diff --git a/internal/controlplane/api/douyin.go b/internal/controlplane/api/douyin.go index 8058158..f73f7c0 100644 --- a/internal/controlplane/api/douyin.go +++ b/internal/controlplane/api/douyin.go @@ -45,6 +45,10 @@ func (browser douyinGatewayBrowser) LoginQR(ctx context.Context) (douyinLoginQRR if status != http.StatusOK { return douyinLoginQRResponse{}, fmt.Errorf("douyin login screen request rejected with HTTP %d: %s", status, string(body)) } + return decodeDouyinLoginQR(body) +} + +func decodeDouyinLoginQR(body []byte) (douyinLoginQRResponse, error) { var response douyinLoginQRResponse if err := json.Unmarshal(body, &response); err != nil { return douyinLoginQRResponse{}, fmt.Errorf("decode douyin login screen response: %w", err) diff --git a/internal/controlplane/api/environment_login.go b/internal/controlplane/api/environment_login.go index a32b46e..8f37aa8 100644 --- a/internal/controlplane/api/environment_login.go +++ b/internal/controlplane/api/environment_login.go @@ -22,20 +22,49 @@ func registerEnvironmentLoginRoutes(app *fiber.App, store *creator.Store, hubSto }) app.Post("/api/creator/environments", func(c fiber.Ctx) error { var input struct { - Gateway string `json:"gateway"` - Fingerprint hub.Fingerprint `json:"fingerprint"` + Gateway string `json:"gateway"` + Fingerprint hub.Fingerprint `json:"fingerprint"` + MemoryLimitMB int `json:"memory_limit_mb"` } if err := decodePhaseA(c, &input); err != nil { return phaseAError(c, err) } - env, err := hubStore.CreateStandaloneEnv(c.Context(), input.Gateway, input.Fingerprint) + env, err := hubStore.CreateStandaloneEnv(c.Context(), input.Gateway, input.Fingerprint, input.MemoryLimitMB) if err != nil { logrus.WithError(err).WithFields(logrus.Fields{"event_type": "login_environment_create_failed", "gateway": input.Gateway}).Error("browser environment creation failed") return hubError(c, err) } - logrus.WithFields(logrus.Fields{"event_type": "login_environment_created", "alias": env.Alias, "gateway": env.Gateway}).Info("browser environment created without platform account") + logrus.WithFields(logrus.Fields{"event_type": "login_environment_created", "alias": env.Alias, "gateway": env.Gateway, "memory_limit_mb": env.MemoryLimitMB}).Info("browser environment created without platform account") return c.Status(fiber.StatusCreated).JSON(env) }) + app.Put("/api/creator/environments/:alias/fingerprint", func(c fiber.Ctx) error { + fingerprint, memoryLimitMB, err := decodeEnvironmentSettings(c) + if err != nil { + return creatorError(c, err) + } + alias := c.Params("alias") + unlock, err := hubStore.LockResources(c.Context(), []string{alias}, nil) + if err != nil { + return hubError(c, err) + } + defer unlock() + env, err := hubStore.GetEnvironmentContext(c.Context(), alias) + if err != nil { + return hubError(c, err) + } + if fingerprint != nil { + fingerprint.Seed = env.Fingerprint.Seed + if env.RuntimeID != "" && *fingerprint != env.Fingerprint { + return hubError(c, hub.ErrConflict) + } + } + env, err = hubStore.UpdateEnvironmentSettings(c.Context(), alias, fingerprint, memoryLimitMB) + if err != nil { + return hubError(c, err) + } + logrus.WithFields(logrus.Fields{"event_type": "environment_settings_saved", "alias": env.Alias, "memory_limit_mb": env.MemoryLimitMB, "runtime_id": env.RuntimeID}).Info("browser environment settings saved without restart") + return c.JSON(env) + }) app.Post("/api/creator/environments/:alias/verify", func(c fiber.Ctx) error { result, err := verifyLoginEnvironment(c.Context(), store, hubStore, c.Params("alias")) if err != nil { diff --git a/internal/controlplane/api/environment_login_test.go b/internal/controlplane/api/environment_login_test.go index 7dc30b0..ff5e7f2 100644 --- a/internal/controlplane/api/environment_login_test.go +++ b/internal/controlplane/api/environment_login_test.go @@ -80,7 +80,7 @@ func TestPendingEnvironmentLoginRoutesKeepBrowserIdentity(t *testing.T) { if _, err := hs.CreateGateway(ctx, "login-gw", gateway.URL, "test-token-login"); err != nil { t.Fatal(err) } - env, err := hs.CreateStandaloneEnv(ctx, "login-gw", hub.Fingerprint{}) + env, err := hs.CreateStandaloneEnv(ctx, "login-gw", hub.Fingerprint{}, 0) if err != nil { t.Fatal(err) } diff --git a/internal/controlplane/api/environment_memory_test.go b/internal/controlplane/api/environment_memory_test.go new file mode 100644 index 0000000..38ce656 --- /dev/null +++ b/internal/controlplane/api/environment_memory_test.go @@ -0,0 +1,39 @@ +package api + +import ( + "context" + "testing" + + hub "git.ipao.vip/rogee/creator-hub/internal/environment" +) + +func TestGatewayMemoryLimitPayload(t *testing.T) { + env := hub.EnvironmentContext{Env: hub.Env{Alias: "memory-env", MemoryLimitMB: 8192}} + for _, payload := range []map[string]any{gatewayCreatePayload(env, "", gatewayNetworkExit{}), gatewayCreatePayloadForAccount(env, "", gatewayNetworkExit{})} { + if payload["memory_limit_mb"] != 8192 { + t.Fatalf("memory not passed: %+v", payload) + } + } +} + +func TestRunningEnvironmentMemoryOnlyDoesNotRestart(t *testing.T) { + s := newMemoryStore() + env := hub.EnvironmentContext{Env: hub.Env{Alias: "memory-env", Gateway: "missing-gateway", MemoryLimitMB: 2048, Fingerprint: hub.Fingerprint{Seed: 1001, Timezone: "Asia/Shanghai"}}, AccountID: "memory-owner", ProfileID: "same-profile", RuntimeID: "same-runtime", BindingVersion: 3} + s.bindings[env.Alias] = env + limit := 4096 + view, err := updateAccountEnvironmentSettings(context.Background(), s, env.AccountID, nil, &limit) + if err != nil { + t.Fatal(err) + } + if view["memory_limit_mb"] != limit { + t.Fatalf("view: %+v", view) + } + limit = 8192 + if _, err := updateAccountEnvironmentSettings(context.Background(), s, env.AccountID, &env.Fingerprint, &limit); err != nil { + t.Fatalf("unchanged fingerprint with memory must not restart: %v", err) + } + after, err := s.GetEnvironmentContextForAccount(context.Background(), env.AccountID) + if err != nil || after.MemoryLimitMB != limit || after.RuntimeID != env.RuntimeID || after.ProfileID != env.ProfileID || after.Fingerprint != env.Fingerprint || after.BindingVersion != env.BindingVersion { + t.Fatalf("identity changed: %+v %v", after, err) + } +} diff --git a/internal/controlplane/api/environment_settings.go b/internal/controlplane/api/environment_settings.go new file mode 100644 index 0000000..bda8b0a --- /dev/null +++ b/internal/controlplane/api/environment_settings.go @@ -0,0 +1,33 @@ +package api + +import ( + "encoding/json" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/gofiber/fiber/v3" +) + +// Existing fingerprint endpoints use a flat fingerprint object. Memory-only +// requests must not replace an omitted fingerprint with its zero value. +func decodeEnvironmentSettings(c fiber.Ctx) (*hub.Fingerprint, *int, error) { + var input struct { + hub.Fingerprint + MemoryLimitMB *int `json:"memory_limit_mb"` + } + if err := decodeCreator(c, &input); err != nil { + return nil, nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(c.Body(), &fields); err != nil { + return nil, nil, creator.ErrInvalid + } + if value, present := fields["memory_limit_mb"]; present && string(value) == "null" { + return nil, nil, creator.ErrInvalid + } + delete(fields, "memory_limit_mb") + if len(fields) == 0 && input.MemoryLimitMB != nil { + return nil, input.MemoryLimitMB, nil + } + return &input.Fingerprint, input.MemoryLimitMB, nil +} diff --git a/internal/controlplane/api/environments.go b/internal/controlplane/api/environments.go index 6bdf95e..b955c19 100644 --- a/internal/controlplane/api/environments.go +++ b/internal/controlplane/api/environments.go @@ -38,7 +38,7 @@ type HubStore interface { CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) GetEnvironmentContextForAccount(ctx context.Context, accountID string) (hub.EnvironmentContext, error) - UpdateAccountFingerprint(ctx context.Context, accountID string, fingerprint hub.Fingerprint) (hub.EnvironmentContext, error) + UpdateEnvironmentSettings(ctx context.Context, alias string, fingerprint *hub.Fingerprint, memoryLimitMB *int) (hub.EnvironmentContext, error) ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64, networkID ...string) (hub.EnvironmentContext, error) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string, networkID ...string) (hub.EnvironmentContext, error) @@ -106,6 +106,7 @@ func gatewayCreatePayload(environment hub.EnvironmentContext, _ string, networkE "alias": environment.Alias, "name": environment.Name, "profile_id": profileID, + "memory_limit_mb": environment.MemoryLimitMB, "cmd": append(fingerprint.Args(), "about:blank"), "binding_version": environment.BindingVersion, "network_exit_id": environment.Exit.ID, @@ -1200,6 +1201,7 @@ func createBrowser(store HubStore, probe NetworkExitProbe, resolve func(hub.Netw Name string `json:"name"` Gateway string `json:"gateway"` Fingerprint hub.Fingerprint `json:"fingerprint"` + MemoryLimitMB int `json:"memory_limit_mb"` AccountID string `json:"account_id"` NetworkExitID string `json:"network_exit_id"` }{} @@ -1210,7 +1212,7 @@ func createBrowser(store HubStore, probe NetworkExitProbe, resolve func(hub.Netw return hubError(c, hub.ErrInvalid) } input.Fingerprint.DisableNonProxiedUDP = false - env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, Fingerprint: input.Fingerprint} + env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, Fingerprint: input.Fingerprint, MemoryLimitMB: input.MemoryLimitMB} if err := env.Fingerprint.Validate(); err != nil { return c.Status(fiber.StatusBadRequest).JSON(map[string]string{"error": err.Error()}) } diff --git a/internal/controlplane/api/event_cover_cache.go b/internal/controlplane/api/event_cover_cache.go new file mode 100644 index 0000000..5cbac15 --- /dev/null +++ b/internal/controlplane/api/event_cover_cache.go @@ -0,0 +1,39 @@ +package api + +import ( + "context" + "errors" + + creator "git.ipao.vip/rogee/creator-hub/internal/creator" + "github.com/sirupsen/logrus" +) + +type eventCoverFiles interface { + GetPlatformWorkCover(string, string) (string, error) + SavePlatformWorkCover(string, string, string, []byte) error +} + +// Cover failures are per-event results: preserve the notification, expose the +// failure on its cover, and retry when the platform returns the notice again. +func cacheListenerWorkCovers(ctx context.Context, files eventCoverFiles, fetch func(context.Context, string) (string, []byte, error), items []creator.ListenerDelivery) { + for i := range items { + n := items[i].Notice + if n == nil || n.WorkID == "" || n.WorkAuthorUID == "" || n.WorkCoverURL == "" { + continue + } + _, err := files.GetPlatformWorkCover(n.WorkAuthorUID, n.WorkID) + if errors.Is(err, creator.ErrNotFound) { + var contentType string + var data []byte + contentType, data, err = fetch(ctx, n.WorkCoverURL) + if err == nil { + err = files.SavePlatformWorkCover(n.WorkAuthorUID, n.WorkID, contentType, data) + } + } + n.WorkCoverError = "" + if err != nil { + n.WorkCoverError = err.Error() + logrus.WithError(err).WithFields(logrus.Fields{"event_key": n.EventKey, "work_key": n.WorkID, "author_uid": n.WorkAuthorUID}).Error("notification cover cache failed") + } + } +} diff --git a/internal/controlplane/api/event_cover_cache_test.go b/internal/controlplane/api/event_cover_cache_test.go new file mode 100644 index 0000000..37f2ddf --- /dev/null +++ b/internal/controlplane/api/event_cover_cache_test.go @@ -0,0 +1,73 @@ +package api + +import ( + "context" + "errors" + "testing" + + creator "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +type eventCoverFilesStub struct { + files map[string]bool + readErr, saveErr error +} + +func (s *eventCoverFilesStub) GetPlatformWorkCover(uid, key string) (string, error) { + if s.readErr != nil { + return "", s.readErr + } + if s.files[uid+"/"+key] { + return "cached", nil + } + return "", creator.ErrNotFound +} +func (s *eventCoverFilesStub) SavePlatformWorkCover(uid, key, mime string, data []byte) error { + if s.saveErr != nil { + return s.saveErr + } + s.files[uid+"/"+key] = true + return nil +} +func coverNotice() creator.ListenerDelivery { + return creator.ListenerDelivery{Kind: "notice", Notice: &creator.ListenerNotice{EventKey: "1", WorkID: "2", WorkAuthorUID: "3", WorkCoverURL: "https://p3.douyinpic.com/a.jpg", MessageText: "原内容", WorkCoverError: "old failure"}} +} +func TestListenerWorkCoversReuseSharedFilesAndExposeFailures(t *testing.T) { + files := &eventCoverFilesStub{files: map[string]bool{}} + calls := 0 + fetch := func(context.Context, string) (string, []byte, error) { + calls++ + return "image/jpeg", []byte("image"), nil + } + items := []creator.ListenerDelivery{coverNotice(), coverNotice(), {Kind: "checkpoint"}} + cacheListenerWorkCovers(context.Background(), files, fetch, items) + if calls != 1 || items[0].Notice.WorkCoverError != "" || items[1].Notice.WorkCoverError != "" { + t.Fatalf("duplicate fetch/old errors: %d %+v", calls, items) + } + for _, failure := range []string{"read", "download", "save"} { + files := &eventCoverFilesStub{files: map[string]bool{}} + failed := errors.New(failure + " failed") + called := false + fetch := func(context.Context, string) (string, []byte, error) { + called = true + if failure == "download" { + return "", nil, failed + } + return "image/jpeg", []byte("image"), nil + } + if failure == "read" { + files.readErr = failed + } + if failure == "save" { + files.saveErr = failed + } + items := []creator.ListenerDelivery{coverNotice()} + cacheListenerWorkCovers(context.Background(), files, fetch, items) + if items[0].Notice.WorkCoverError != failed.Error() || items[0].Notice.MessageText != "原内容" { + t.Fatalf("failure hidden: %+v", items) + } + if failure == "read" && called { + t.Fatal("filesystem failure concealed by download") + } + } +} diff --git a/internal/controlplane/api/event_listener.go b/internal/controlplane/api/event_listener.go new file mode 100644 index 0000000..cc182ff --- /dev/null +++ b/internal/controlplane/api/event_listener.go @@ -0,0 +1,58 @@ +package api + +import ( + "git.ipao.vip/rogee/creator-hub/internal/creator" + "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" + "strconv" + "strings" +) + +func registerAccountEventRoutes(app *fiber.App, store *creator.Store) { + app.Get("/api/creator/events/:id/cover", func(c fiber.Ctx) error { + path, err := store.GetEventWorkCover(c.Context(), c.Params("id")) + if err != nil { + return creatorError(c, err) + } + return c.SendFile(path) + }) + app.Put("/api/creator/accounts/:id/listener", func(c fiber.Ctx) error { + var input struct { + Enabled *bool `json:"enabled"` + } + if err := c.Bind().Body(&input); err != nil || input.Enabled == nil { + return creatorError(c, creator.ErrInvalid) + } + state, err := store.SetListenerEnabled(c.Context(), c.Params("id"), *input.Enabled) + if err != nil { + return creatorError(c, err) + } + logrus.WithFields(logrus.Fields{"account_id": state.AccountID, "generation": state.Generation, "enabled": state.Enabled}).Info("account event listener setting changed") + return c.JSON(state) + }) + app.Get("/api/creator/events", func(c fiber.Ctx) error { + filter := creator.EventAggregationFilter{AccountID: strings.TrimSpace(c.Query("account_id")), EventType: strings.TrimSpace(c.Query("event_type"))} + if value := c.Query("hours"); value != "" { + hours, err := strconv.Atoi(value) + if err != nil { + return creatorError(c, creator.ErrInvalid) + } + filter.Hours = hours + } + if err := filter.Validate(); err != nil { + return creatorError(c, err) + } + page, pageSize, paged, err := creatorPageQuery(c) + if err != nil { + return creatorError(c, err) + } + if !paged { + page, pageSize = 1, 25 + } + result, err := store.ListAggregatedEventsPage(c.Context(), filter, page, pageSize) + if err != nil { + return creatorError(c, err) + } + return c.JSON(result) + }) +} diff --git a/internal/controlplane/api/event_listener_lifecycle_test.go b/internal/controlplane/api/event_listener_lifecycle_test.go new file mode 100644 index 0000000..044c2b9 --- /dev/null +++ b/internal/controlplane/api/event_listener_lifecycle_test.go @@ -0,0 +1,377 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" +) + +type listenerControlStub struct { + mu sync.Mutex + states []creator.ListenerState + profile creator.AccountProfile + profileErr, listErr, updateErr, recordErr error + records int + changes chan string +} + +func (s *listenerControlStub) GetAccountProfile(context.Context, string) (creator.AccountProfile, error) { + return s.profile, s.profileErr +} +func (s *listenerControlStub) ListListenerStates(context.Context) ([]creator.ListenerState, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]creator.ListenerState{}, s.states...), s.listErr +} +func (s *listenerControlStub) UpdateListenerStatus(_ context.Context, id, generation, status, reason string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.updateErr != nil { + return s.updateErr + } + for i := range s.states { + if s.states[i].AccountID == id && s.states[i].Generation == generation { + s.states[i].Status = status + s.states[i].Reason = reason + if s.changes != nil { + select { + case s.changes <- status: + default: + } + } + return nil + } + } + return creator.ErrConflict +} +func (s *listenerControlStub) RecordListenerDeliveries(_ context.Context, _ string, _ string, items []creator.ListenerDelivery) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.recordErr != nil { + return s.recordErr + } + s.records += len(items) + if s.changes != nil { + select { + case s.changes <- "recorded": + default: + } + } + return nil +} + +type listenerAccountsStub struct { + HubStore + environment hub.EnvironmentContext + gateway hub.Gateway + environmentErr, gatewayErr error + lockCalls chan []string +} + +func (s listenerAccountsStub) GetEnvironmentContextForAccount(context.Context, string) (hub.EnvironmentContext, error) { + return s.environment, s.environmentErr +} +func (s listenerAccountsStub) GetGateway(context.Context, string) (hub.Gateway, error) { + return s.gateway, s.gatewayErr +} +func (s listenerAccountsStub) LockResources(_ context.Context, aliases, _ []string, _ ...[]string) (func(), error) { + if s.lockCalls != nil { + s.lockCalls <- append([]string(nil), aliases...) + } + return func() {}, nil +} +func listenerFixture(endpoint string) (*listenerControlStub, listenerAccountsStub, creator.ListenerState) { + state := creator.ListenerState{AccountID: "account", Generation: "generation", Enabled: true, Status: "starting"} + store := &listenerControlStub{states: []creator.ListenerState{state}, profile: creator.AccountProfile{LoginStatus: "logged_in", PlatformAccountKey: "123"}, changes: make(chan string, 20)} + accounts := listenerAccountsStub{gateway: hub.Gateway{Endpoint: endpoint, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser", Gateway: "gateway"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 7}, lockCalls: make(chan []string, 20)} + return store, accounts, state +} +func awaitListenerChange(t *testing.T, store *listenerControlStub, value string) { + t.Helper() + timer := time.NewTimer(8 * time.Second) + defer timer.Stop() + for { + select { + case actual := <-store.changes: + if actual == value { + return + } + case <-timer.C: + t.Fatalf("listener did not reach %s", value) + } + } +} + +func TestGatewayListenerSessionProtocol(t *testing.T) { + calls := make(chan map[string]any, 3) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer token" || r.URL.Path != "/v1/browsers/browser/douyin/events" { + t.Errorf("request=%+v", r) + } + var payload map[string]any + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Error(err) + } + payload["method"] = r.Method + payload["query"] = r.URL.Query() + calls <- payload + w.Header().Set("Content-Type", "application/json") + switch r.Method { + case http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + case http.MethodPost: + w.Write([]byte(`{"connected":true,"alias":"browser","uid":"123"}`)) + case http.MethodGet: + w.Write([]byte(`[{"delivery_id":"1","kind":"open"}]`)) + default: + t.Errorf("invalid method: %s", r.Method) + } + })) + defer server.Close() + store, accounts, state := listenerFixture(server.URL) + session, err := listenerSessionForAccount(context.Background(), store, accounts, state) + if err != nil { + t.Fatal(err) + } + if items, err := session.Start(context.Background()); err != nil || len(items) != 0 { + t.Fatalf("start=%+v %v", items, err) + } + start := <-calls + if start["method"] != http.MethodPost || start["expected_uid"] != "123" || start["binding_version"] != float64(7) || start["runtime_id"] != "runtime" { + t.Fatalf("start=%+v", start) + } + session.pollInterval = 0 + if _, err := session.Poll(context.Background(), []string{"1"}); err != nil { + t.Fatal(err) + } + poll := <-calls + query := poll["query"].(url.Values) + if poll["method"] != http.MethodGet || len(query["ack"]) != 1 || query.Get("ack") != "1" || query.Get("wait") != "0" || query.Get("limit") != "100" { + t.Fatalf("poll=%+v", poll) + } + if err := session.Stop(context.Background()); err != nil { + t.Fatal(err) + } + stop := <-calls + if stop["method"] != http.MethodDelete { + t.Fatalf("stop=%+v", stop) + } + for i := 0; i < 3; i++ { + select { + case aliases := <-accounts.lockCalls: + if len(aliases) != 1 || aliases[0] != "browser" { + t.Fatalf("request %d locked aliases=%v", i+1, aliases) + } + default: + t.Fatalf("request %d did not lock the browser alias", i+1) + } + } + if _, exists := stop["generation"]; exists { + t.Fatal("stop should use runtime binding, not subscription generation") + } +} +func TestGatewayListenerRejectsInvalidResponse(t *testing.T) { + for _, body := range []string{`not-json`, `{}`, `{"items":[{"kind":"unknown","delivery_id":"1"}]}`, `{"items":null}`} { + t.Run(body, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(body)) })) + defer server.Close() + session := gatewayListenerSession{target: hub.Gateway{Endpoint: server.URL}, resources: listenerAccountsStub{}} + if _, err := session.Start(context.Background()); err == nil { + t.Fatal("accepted malformed gateway response") + } + }) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "not logged in", 409) })) + session := gatewayListenerSession{target: hub.Gateway{Endpoint: server.URL}, resources: listenerAccountsStub{}} + if _, err := session.Start(context.Background()); err == nil || !strings.Contains(err.Error(), "409") { + t.Fatal(err) + } + server.Close() + if _, err := session.Start(context.Background()); err == nil { + t.Fatal("ignored unreachable gateway") + } +} +func TestListenerRequiresLoggedInRunningBrowser(t *testing.T) { + cases := []struct { + name string + modify func(*listenerControlStub, *listenerAccountsStub) + }{ + {"profile", func(s *listenerControlStub, _ *listenerAccountsStub) { s.profileErr = errors.New("profile failed") }}, + {"environment", func(_ *listenerControlStub, a *listenerAccountsStub) { + a.environmentErr = errors.New("environment failed") + }}, + {"stopped", func(_ *listenerControlStub, a *listenerAccountsStub) { a.environment.RuntimeID = "" }}, + {"missing runtime", func(_ *listenerControlStub, a *listenerAccountsStub) { a.environment.RuntimeID = missingRuntimeID }}, + {"login", func(s *listenerControlStub, _ *listenerAccountsStub) { s.profile.LoginStatus = "not_logged_in" }}, + {"gateway", func(_ *listenerControlStub, a *listenerAccountsStub) { a.gatewayErr = errors.New("gateway failed") }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + store, accounts, state := listenerFixture("") + c.modify(store, &accounts) + if _, err := listenerSessionForAccount(context.Background(), store, accounts, state); err == nil { + t.Fatal("missing prerequisite ignored") + } + }) + } +} +func TestListenerOffStopsWithoutStarting(t *testing.T) { + for _, runtime := range []string{"", "runtime"} { + t.Run(runtime, func(t *testing.T) { + var starts, stops int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + starts++ + } + if r.Method == http.MethodDelete { + stops++ + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + store, accounts, state := listenerFixture(server.URL) + state.Enabled = false + store.states[0] = state + accounts.environment.RuntimeID = runtime + runAccountEventListener(context.Background(), store, accounts, state) + states, _ := store.ListListenerStates(context.Background()) + if starts != 0 || states[0].Status != "stopped" || (runtime != "" && stops != 1) { + t.Fatalf("starts=%d stops=%d state=%+v", starts, stops, states) + } + }) + } +} +func TestListenerFailuresAreVisibleAndCancelable(t *testing.T) { + store, accounts, state := listenerFixture("") + accounts.environmentErr = errors.New("browser unavailable") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); runAccountEventListener(ctx, store, accounts, state) }() + awaitListenerChange(t, store, "error") + cancel() + <-done + states, _ := store.ListListenerStates(context.Background()) + if states[0].Reason != "browser unavailable" { + t.Fatalf("state=%+v", states) + } + // Disabled listeners must also surface cleanup failures, without starting. + state.Enabled = false + store.states[0] = state + ctx, cancel = context.WithCancel(context.Background()) + done = make(chan struct{}) + go func() { defer close(done); runAccountEventListener(ctx, store, accounts, state) }() + awaitListenerChange(t, store, "error") + cancel() + <-done + // A status write failure is logged and cancellation still terminates. + store.updateErr = errors.New("database write failed") + ctx, cancel = context.WithCancel(context.Background()) + cancel() + runAccountEventListener(ctx, store, accounts, state) +} +func TestListenerReconcilerHonorsEnableDisableAndRestart(t *testing.T) { + var mu sync.Mutex + var starts, stops int + var pending string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost: + mu.Lock() + starts++ + pending = fmt.Sprintf("open-%d", starts) + mu.Unlock() + w.Write([]byte(`{"connected":true,"alias":"browser","uid":"123"}`)) + case r.Method == http.MethodDelete: + mu.Lock() + stops++ + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + select { + case <-r.Context().Done(): + return + case <-time.After(20 * time.Millisecond): + } + mu.Lock() + for _, ack := range r.URL.Query()["ack"] { + if ack == pending { + pending = "" + } + } + deliver := pending + mu.Unlock() + if deliver != "" { + json.NewEncoder(w).Encode([]creator.ListenerDelivery{{Kind: "open", DeliveryID: deliver}}) + } else { + w.Write([]byte(`[]`)) + } + } + })) + defer server.Close() + store, accounts, state := listenerFixture(server.URL) + store.states = append(store.states, creator.ListenerState{AccountID: "disabled", Generation: "off", Status: "stopped"}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); RunAccountEventListeners(ctx, store, accounts) }() + awaitListenerChange(t, store, "recorded") + cancel() + <-done + mu.Lock() + gotStarts, gotStops := starts, stops + mu.Unlock() + if gotStarts != 1 || gotStops != 1 { + t.Fatalf("startup starts=%d stops=%d", gotStarts, gotStops) + } + // Persisted enabled state resumes on a new service lifetime; stopped remains off. + ctx, cancel = context.WithCancel(context.Background()) + done = make(chan struct{}) + go func() { defer close(done); RunAccountEventListeners(ctx, store, accounts) }() + awaitListenerChange(t, store, "recorded") + store.mu.Lock() + store.states[0].Enabled = false + store.states[0].Generation = "off-generation" + store.states[0].Status = "stopping" + store.mu.Unlock() + awaitListenerChange(t, store, "stopped") + cancel() + <-done + mu.Lock() + gotStarts, gotStops = starts, stops + mu.Unlock() + if gotStarts != 2 || gotStops < 3 { + t.Fatalf("restart starts=%d stops=%d state=%+v", gotStarts, gotStops, state) + } +} +func TestListenerReconcilerStoreFailureDoesNotStart(t *testing.T) { + store, accounts, _ := listenerFixture("") + store.listErr = errors.New("database unavailable") + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + RunAccountEventListeners(ctx, store, accounts) + if store.records != 0 { + t.Fatal("received events without settings") + } +} +func TestListenerDatabaseFailureKeepsUnacknowledgedBatchUntilCancellation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + failure := errors.New("database unavailable") + store := &listenerWriterStub{fail: failure} + session := &listenerSessionStub{initial: []creator.ListenerDelivery{{DeliveryID: "1", Kind: "open"}}} + err := runListenerSession(ctx, store, "account", "generation", session) + if !errors.Is(err, failure) || !errors.Is(err, context.DeadlineExceeded) || session.polls != 0 || session.stops != 1 { + t.Fatalf("err=%v session=%+v", err, session) + } +} diff --git a/internal/controlplane/api/event_listener_recovery_test.go b/internal/controlplane/api/event_listener_recovery_test.go new file mode 100644 index 0000000..5c7d67d --- /dev/null +++ b/internal/controlplane/api/event_listener_recovery_test.go @@ -0,0 +1,46 @@ +package api + +import ( + "context" + "errors" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" +) + +type recoveringListenerWriter struct { + attempts int + statuses []string +} + +func (s *recoveringListenerWriter) RecordListenerDeliveries(context.Context, string, string, []creator.ListenerDelivery) error { + s.attempts++ + if s.attempts == 1 { + return errors.New("temporary write failure") + } + return nil +} +func (s *recoveringListenerWriter) UpdateListenerStatus(_ context.Context, _ string, _ string, status, reason string) error { + s.statuses = append(s.statuses, status) + return nil +} + +func TestListenerPersistenceRecoveryPreservesConnectionHealth(t *testing.T) { + for _, health := range []string{"ready", "gap"} { + t.Run(health, func(t *testing.T) { + now := time.Now().UTC() + items := []creator.ListenerDelivery{{DeliveryID: "boundary", Kind: "baseline", BoundaryAt: &now}} + if health == "gap" { + items = append(items, creator.ListenerDelivery{DeliveryID: "close", Kind: "close", Reason: "disconnected"}) + } + failure := errors.New("end test") + session := &listenerSessionStub{initial: items, pollErr: failure} + store := &recoveringListenerWriter{} + err := runListenerSession(context.Background(), store, "account", "generation", session) + if !errors.Is(err, failure) || store.attempts != 2 || len(session.ack) != len(items) || len(store.statuses) != 2 || store.statuses[0] != "error" || store.statuses[1] != health { + t.Fatalf("err=%v writer=%+v session=%+v", err, store, session) + } + }) + } +} diff --git a/internal/controlplane/api/event_listener_routes_integration_test.go b/internal/controlplane/api/event_listener_routes_integration_test.go new file mode 100644 index 0000000..f417004 --- /dev/null +++ b/internal/controlplane/api/event_listener_routes_integration_test.go @@ -0,0 +1,136 @@ +package api + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/creator" + "github.com/gofiber/fiber/v3" +) + +func TestAccountEventRoutesLifecycle(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL") + } + store, accounts, ctx := openCreatorIntegrationStoreForAPITest(t, databaseURL) + uid := fmt.Sprint(time.Now().UnixNano()) + account := account.Account{ID: "event-listener-account", Platform: creator.PlatformDouyin, Name: "监听账号", PlatformAccountKey: uid, + CredentialReference: account.CredentialReference{ID: "event-listener-credential", Provider: "os_keyring"}, + CredentialKey: "creatorhub/event-listener-account/cookies", + } + bridge := &testCredentialBridge{values: make(map[string]string)} + if err := accounts.CreateAccount(ctx, account, bridge); err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, account.ID, uid); err != nil { + t.Fatal(err) + } + app := fiber.New() + registerAccountEventRoutes(app, store) + change := func(enabled bool) creator.ListenerState { + t.Helper() + body := fmt.Sprintf(`{"enabled":%t}`, enabled) + request := httptest.NewRequest("PUT", "/api/creator/accounts/"+account.ID+"/listener", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != 200 { + t.Fatalf("setting response=%d", response.StatusCode) + } + var state creator.ListenerState + if err := json.NewDecoder(response.Body).Decode(&state); err != nil { + t.Fatal(err) + } + return state + } + state := change(true) + now := time.Now().UTC() + if err := store.RecordListenerDeliveries(ctx, account.ID, state.Generation, []creator.ListenerDelivery{ + {DeliveryID: "1", Kind: "baseline", BoundaryAt: &now}, + {DeliveryID: "2", Kind: "notice", Notice: &creator.ListenerNotice{EventKey: "100", EventType: "like", InteractorUID: "22"}}, + {DeliveryID: "3", Kind: "notice", Notice: &creator.ListenerNotice{EventKey: "100", EventType: "comment", InteractorUID: "22", MessageType: "text", MessageText: "你好"}}, + }); err != nil { + t.Fatal(err) + } + for _, target := range []string{"/api/creator/events", "/api/creator/events?page=1&page_size=1&event_type=like&hours=24&account_id=" + account.ID} { + response, err := app.Test(httptest.NewRequest("GET", target, nil)) + if err != nil { + t.Fatal(err) + } + var page creator.Page[creator.AggregatedEvent] + err = json.NewDecoder(response.Body).Decode(&page) + response.Body.Close() + if err != nil || response.StatusCode != 200 || len(page.Data) == 0 || page.Data[0].AccountUID != uid || page.Data[0].AccountName != "监听账号" { + t.Fatalf("target=%s page=%+v status=%d err=%v", target, page, response.StatusCode, err) + } + if target == "/api/creator/events" && page.Total != 2 { + t.Fatal("event-type namespaces must not deduplicate each other") + } + } + change(false) + response, err := app.Test(httptest.NewRequest("GET", "/api/creator/events", nil)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var page creator.Page[creator.AggregatedEvent] + if err := json.NewDecoder(response.Body).Decode(&page); err != nil { + t.Fatal(err) + } + if page.Total != 0 || page.Data == nil { + t.Fatalf("disabled account response=%+v", page) + } + change(true) + if err := store.DeleteOwnedAccountData(ctx, account.ID); err != nil { + t.Fatal(err) + } + if err := accounts.DeleteAccountData(ctx, account.ID); err != nil { + t.Fatal(err) + } + if err := accounts.DeleteAccount(ctx, account.ID, bridge); err != nil { + t.Fatal(err) + } + states, err := store.ListListenerStates(ctx) + if err != nil || len(states) != 0 { + t.Fatalf("deleted account listener remains: %+v %v", states, err) + } +} + +func TestAccountEventRoutesStoreErrors(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL") + } + store, _, _ := openCreatorIntegrationStoreForAPITest(t, databaseURL) + app := fiber.New() + registerAccountEventRoutes(app, store) + request := httptest.NewRequest("PUT", "/api/creator/accounts/missing/listener", strings.NewReader(`{"enabled":true}`)) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 404 { + t.Fatalf("status=%d", response.StatusCode) + } + store.Close() + response, err = app.Test(httptest.NewRequest("GET", "/api/creator/events", nil)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 500 { + t.Fatalf("status=%d", response.StatusCode) + } +} diff --git a/internal/controlplane/api/event_listener_test.go b/internal/controlplane/api/event_listener_test.go new file mode 100644 index 0000000..eabb275 --- /dev/null +++ b/internal/controlplane/api/event_listener_test.go @@ -0,0 +1,35 @@ +package api + +import ( + "github.com/gofiber/fiber/v3" + "net/http/httptest" + "strings" + "testing" +) + +func TestEventListenerRoutesValidateBeforeStore(t *testing.T) { + app := fiber.New() + registerAccountEventRoutes(app, nil) + for _, target := range []string{"/api/creator/events?event_type=unknown", "/api/creator/events?hours=2", "/api/creator/events?page=0", "/api/creator/events?page_size=501", "/api/creator/events?hours=bad"} { + response, err := app.Test(httptest.NewRequest("GET", target, nil)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 400 { + t.Fatalf("%s: %d", target, response.StatusCode) + } + } + for _, body := range []string{`{}`, `{"enabled":"yes"}`, `{"enabled":null}`, `broken`} { + request := httptest.NewRequest("PUT", "/api/creator/accounts/account/listener", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 400 { + t.Fatalf("%s: %d", body, response.StatusCode) + } + } +} diff --git a/internal/controlplane/api/event_listener_worker.go b/internal/controlplane/api/event_listener_worker.go new file mode 100644 index 0000000..b1b5bd4 --- /dev/null +++ b/internal/controlplane/api/event_listener_worker.go @@ -0,0 +1,400 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "sync" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" + "github.com/sirupsen/logrus" +) + +type listenerWriter interface { + RecordListenerDeliveries(context.Context, string, string, []creator.ListenerDelivery) error + UpdateListenerStatus(context.Context, string, string, string, string) error +} +type listenerStore interface { + listenerWriter + GetAccountProfile(context.Context, string) (creator.AccountProfile, error) + ListListenerStates(context.Context) ([]creator.ListenerState, error) +} + +type listenerSession interface { + Start(context.Context) ([]creator.ListenerDelivery, error) + Poll(context.Context, []string) ([]creator.ListenerDelivery, error) + Stop(context.Context) error +} + +type browserAliasResourceLocker interface { + LockResources(context.Context, []string, []string, ...[]string) (func(), error) +} + +type gatewayListenerSession struct { + target hub.Gateway + environment hub.EnvironmentContext + uid string + resources browserAliasResourceLocker + boundaryAt *time.Time + checkpoints map[string]string + pollInterval time.Duration + coverFiles eventCoverFiles +} + +func (s gatewayListenerSession) request(ctx context.Context, action string, ack []string) ([]creator.ListenerDelivery, error) { + if s.resources == nil { + return nil, errors.New("event listener browser alias resource locker unavailable") + } + unlock, err := s.resources.LockResources(ctx, []string{s.environment.Alias}, nil) + if err != nil { + return nil, fmt.Errorf("lock browser alias for event listener %s: %w", action, err) + } + defer unlock() + + payload := gatewayGenerationPayload(s.environment) + path := "/v1/browsers/" + url.PathEscape(s.environment.Alias) + "/douyin/events" + method, expectedStatus := http.MethodPost, http.StatusOK + switch action { + case "start": + payload["expected_uid"] = s.uid + payload["boundary_at"] = s.boundaryAt + checkpoints := s.checkpoints + if checkpoints == nil { + checkpoints = map[string]string{} + } + payload["notice_checkpoints"] = checkpoints + case "poll": + method = http.MethodGet + query := url.Values{"limit": {"100"}, "wait": {"0"}} + for _, id := range ack { + query.Add("ack", id) + } + path += "?" + query.Encode() + case "stop": + method, expectedStatus = http.MethodDelete, http.StatusNoContent + default: + return nil, creator.ErrInvalid + } + status, body, err := gatewayCallWithLimit(ctx, s.target, method, path, payload, 30*time.Second, largeGatewayResponseLimit) + if err != nil { + return nil, fmt.Errorf("event listener %s: %w", action, err) + } + if status != expectedStatus { + return nil, fmt.Errorf("event listener %s: gateway HTTP %d: %s", action, status, body) + } + if action == "stop" { + return nil, nil + } + if action == "start" { + var response struct { + Connected bool `json:"connected"` + Alias string `json:"alias"` + UID string `json:"uid"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("event listener start response: %w", err) + } + if !response.Connected || response.Alias != s.environment.Alias || response.UID != s.uid { + return nil, errors.New("event listener start response identity or connection mismatch") + } + return nil, nil + } + var items []creator.ListenerDelivery + if err := json.Unmarshal(body, &items); err != nil { + return nil, fmt.Errorf("event listener poll response: %w", err) + } + if items == nil { + return nil, errors.New("event listener poll response must be an array") + } + for _, d := range items { + if err := d.Validate(); err != nil { + return nil, fmt.Errorf("event listener invalid %s delivery: %w", d.Kind, err) + } + } + return items, nil +} +func (s gatewayListenerSession) Start(ctx context.Context) ([]creator.ListenerDelivery, error) { + return s.request(ctx, "start", nil) +} +func (s gatewayListenerSession) Poll(ctx context.Context, ack []string) ([]creator.ListenerDelivery, error) { + if s.pollInterval > 0 && !waitListener(ctx, s.pollInterval) { + return nil, ctx.Err() + } + items, err := s.request(ctx, "poll", ack) + if err != nil { + return nil, err + } + if s.coverFiles != nil { + browser := creatorGatewayBrowser{gateway: s.target, environment: s.environment} + fetch := func(ctx context.Context, target string) (string, []byte, error) { + unlock, err := s.resources.LockResources(ctx, []string{s.environment.Alias}, nil) + if err != nil { + return "", nil, err + } + defer unlock() + return browser.GetImage(ctx, target) + } + cacheListenerWorkCovers(ctx, s.coverFiles, fetch, items) + } else { + for _, d := range items { + if d.Notice != nil && d.Notice.WorkCoverURL != "" { + return nil, fmt.Errorf("event cover store is unavailable") + } + } + } + return items, nil +} +func (s gatewayListenerSession) Stop(ctx context.Context) error { + _, err := s.request(ctx, "stop", nil) + return err +} + +func runListenerSession(ctx context.Context, store listenerWriter, accountID, generation string, session listenerSession) (err error) { + items, err := session.Start(ctx) + if err != nil { + return err + } + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + stopErr := session.Stop(cleanup) + if stopErr != nil { + logrus.WithFields(logrus.Fields{"account_id": accountID, "generation": generation}).WithError(stopErr).Error("stop account event listener failed") + } + err = errors.Join(err, stopErr) + }() + healthStatus, healthReason := "starting", "" + for { + if err := ctx.Err(); err != nil { + return err + } + ack := make([]string, 0, len(items)) + if len(items) > 0 { + for _, d := range items { + switch d.Kind { + case "baseline", "checkpoint": + healthStatus, healthReason = "ready", "" + if d.Reason != "" { + healthStatus, healthReason = "gap", d.Reason + } + case "close", "reconnected", "error": + healthStatus, healthReason = "gap", d.Reason + if healthReason == "" { + healthReason = "抖音通知连接中断，正在恢复" + } + } + } + persistenceFailed := false + for { + err := store.RecordListenerDeliveries(ctx, accountID, generation, items) + if err == nil { + break + } + if errors.Is(err, creator.ErrConflict) || errors.Is(err, creator.ErrInvalid) || errors.Is(err, creator.ErrNotFound) { + return err + } + persistenceFailed = true + log := logrus.WithFields(logrus.Fields{"account_id": accountID, "generation": generation, "delivery_count": len(items)}) + log.WithError(err).Error("event persistence failed; retaining unacknowledged deliveries and retrying") + if statusErr := store.UpdateListenerStatus(ctx, accountID, generation, "error", "事件保存失败："+err.Error()); statusErr != nil { + if errors.Is(statusErr, creator.ErrConflict) { + return statusErr + } + log.WithError(statusErr).Error("persist event listener failure status failed") + } + if !waitListener(ctx, 2*time.Second) { + return errors.Join(ctx.Err(), err) + } + } + if persistenceFailed { + if err := store.UpdateListenerStatus(ctx, accountID, generation, healthStatus, healthReason); err != nil { + return err + } + } + logrus.WithFields(logrus.Fields{"account_id": accountID, "generation": generation, "delivery_count": len(items), "last_delivery_id": items[len(items)-1].DeliveryID}).Debug("account event deliveries committed") + for _, d := range items { + ack = append(ack, d.DeliveryID) + } + } + items, err = session.Poll(ctx, ack) + if err != nil { + return err + } + } +} + +func waitListener(ctx context.Context, delay time.Duration) bool { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func listenerSessionForAccount(ctx context.Context, store listenerStore, accounts HubStore, state creator.ListenerState) (gatewayListenerSession, error) { + profile, err := store.GetAccountProfile(ctx, state.AccountID) + if err != nil { + return gatewayListenerSession{}, err + } + environment, err := accounts.GetEnvironmentContextForAccount(ctx, state.AccountID) + if err != nil { + return gatewayListenerSession{}, err + } + if environment.RuntimeID == "" || environment.RuntimeID == missingRuntimeID { + return gatewayListenerSession{}, errors.New("浏览器未运行，请在我的账号启动浏览器") + } + if state.Enabled && profile.LoginStatus != "logged_in" { + return gatewayListenerSession{}, errors.New("账号未登录，请登录并同步账号") + } + target, err := accounts.GetGateway(ctx, environment.Gateway) + if err != nil { + return gatewayListenerSession{}, err + } + coverFiles, _ := store.(eventCoverFiles) + return gatewayListenerSession{target: target, environment: environment, uid: profile.PlatformAccountKey, coverFiles: coverFiles, resources: accounts, + boundaryAt: state.BoundaryAt, checkpoints: state.Checkpoints, pollInterval: 5 * time.Second}, nil +} + +func runAccountEventListener(ctx context.Context, store listenerStore, accounts HubStore, state creator.ListenerState) { + log := logrus.WithFields(logrus.Fields{"account_id": state.AccountID, "generation": state.Generation, "enabled": state.Enabled}) + update := func(status, reason string) { + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if err := store.UpdateListenerStatus(writeCtx, state.AccountID, state.Generation, status, reason); err != nil && !errors.Is(err, creator.ErrConflict) { + log.WithError(err).Error("persist event listener status failed") + } + } + for ctx.Err() == nil { + // Reload the last committed checkpoint after any failed request or restart. + states, readErr := store.ListListenerStates(ctx) + if readErr != nil { + update("error", readErr.Error()) + log.WithError(readErr).Error("reload notification checkpoint failed") + if !waitListener(ctx, 10*time.Second) { + return + } + continue + } + found := false + for _, current := range states { + if current.AccountID == state.AccountID && current.Generation == state.Generation { + state, found = current, true + break + } + } + if !found { + return + } + if state.Enabled { + update("starting", "") + } + var err error + if !state.Enabled { + // A missing runtime has already destroyed its browser-side listener. + environment, lookupErr := accounts.GetEnvironmentContextForAccount(ctx, state.AccountID) + if lookupErr == nil && (environment.RuntimeID == "" || environment.RuntimeID == missingRuntimeID) { + update("stopped", "") + log.Info("account event listener stopped with browser") + return + } + err = lookupErr + } + if err == nil { + var session gatewayListenerSession + session, err = listenerSessionForAccount(ctx, store, accounts, state) + if err == nil { + if state.Enabled { + log.Info("account event listener starting") + err = runListenerSession(ctx, store, state.AccountID, state.Generation, session) + } else { + err = session.Stop(ctx) + if err == nil { + update("stopped", "") + log.Info("account event listener stopped") + return + } + } + } + } + if ctx.Err() != nil { + update("starting", "服务停止或监听设置已变更，等待重新连接") + return + } + if errors.Is(err, creator.ErrConflict) { + return + } + update("error", err.Error()) + log.WithError(err).Error("account event listener failed; retry in 10 seconds") + if !waitListener(ctx, 10*time.Second) { + return + } + } +} + +// One worker per account; a previous generation is fully stopped before the +// next starts. Database generations prevent a late delivery after disable. +func RunAccountEventListeners(ctx context.Context, store listenerStore, accounts HubStore) { + type running struct { + generation string + cancel context.CancelFunc + done chan struct{} + } + active := make(map[string]running) + var workers sync.WaitGroup + defer func() { + for _, worker := range active { + worker.cancel() + } + workers.Wait() + }() + for ctx.Err() == nil { + states, err := store.ListListenerStates(ctx) + if err != nil { + logrus.WithError(err).Error("list account event listeners failed") + } else { + desired := make(map[string]creator.ListenerState, len(states)) + for _, state := range states { + if state.Enabled || state.Status != "stopped" { + desired[state.AccountID] = state + } + } + for id, worker := range active { + select { + case <-worker.done: + worker.cancel() + delete(active, id) + default: + next, exists := desired[id] + if !exists || next.Generation != worker.generation { + worker.cancel() + } + } + } + for id, state := range desired { + if _, exists := active[id]; exists { + continue + } + workerCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + active[id] = running{generation: state.Generation, cancel: cancel, done: done} + workers.Add(1) + go func(state creator.ListenerState) { + defer workers.Done() + defer close(done) + runAccountEventListener(workerCtx, store, accounts, state) + }(state) + } + } + if !waitListener(ctx, 2*time.Second) { + return + } + } +} diff --git a/internal/controlplane/api/event_listener_worker_test.go b/internal/controlplane/api/event_listener_worker_test.go new file mode 100644 index 0000000..0396ac9 --- /dev/null +++ b/internal/controlplane/api/event_listener_worker_test.go @@ -0,0 +1,91 @@ +package api + +import ( + "context" + "errors" + "git.ipao.vip/rogee/creator-hub/internal/creator" + "testing" + "time" +) + +type listenerWriterStub struct { + batches int + fail error +} + +func (s *listenerWriterStub) UpdateListenerStatus(context.Context, string, string, string, string) error { + return nil +} +func (s *listenerWriterStub) RecordListenerDeliveries(context.Context, string, string, []creator.ListenerDelivery) error { + s.batches++ + return s.fail +} + +type listenerSessionStub struct { + starts, polls, stops int + ack []string + startErr, pollErr, stopErr error + initial []creator.ListenerDelivery +} + +func (s *listenerSessionStub) Start(context.Context) ([]creator.ListenerDelivery, error) { + s.starts++ + return s.initial, s.startErr +} +func (s *listenerSessionStub) Poll(_ context.Context, ack []string) ([]creator.ListenerDelivery, error) { + s.polls++ + s.ack = ack + return nil, s.pollErr +} +func (s *listenerSessionStub) Stop(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + s.stops++ + return s.stopErr +} + +func TestListenerSessionAcknowledgesCommittedEvents(t *testing.T) { + failure := errors.New("poll failed") + store := &listenerWriterStub{} + session := &listenerSessionStub{initial: []creator.ListenerDelivery{{DeliveryID: "1", Kind: "open"}}, pollErr: failure} + err := runListenerSession(context.Background(), store, "account", "generation", session) + if !errors.Is(err, failure) || store.batches != 1 || len(session.ack) != 1 || session.ack[0] != "1" || session.stops != 1 { + t.Fatalf("err=%v store=%+v session=%+v", err, store, session) + } +} +func TestListenerSessionDoesNotAcknowledgeFailedWrite(t *testing.T) { + failure := creator.ErrConflict + store := &listenerWriterStub{fail: failure} + session := &listenerSessionStub{initial: []creator.ListenerDelivery{{DeliveryID: "1", Kind: "open"}}} + err := runListenerSession(context.Background(), store, "account", "generation", session) + if !errors.Is(err, failure) || session.polls != 0 || session.stops != 1 { + t.Fatalf("err=%v session=%+v", err, session) + } +} +func TestListenerSessionStartFailure(t *testing.T) { + failure := errors.New("not logged in") + session := &listenerSessionStub{startErr: failure} + if err := runListenerSession(context.Background(), &listenerWriterStub{}, "account", "generation", session); !errors.Is(err, failure) || session.polls != 0 { + t.Fatalf("err=%v session=%+v", err, session) + } +} +func TestListenerSessionCancellationStopsGateway(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + session := &listenerSessionStub{stopErr: errors.New("stop unavailable")} + err := runListenerSession(ctx, &listenerWriterStub{}, "account", "generation", session) + if !errors.Is(err, context.Canceled) || session.stops != 1 || err.Error() == context.Canceled.Error() { + t.Fatalf("err=%v session=%+v", err, session) + } +} +func TestListenerRetryWait(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if waitListener(ctx, time.Hour) { + t.Fatal("ignored cancellation") + } + if !waitListener(context.Background(), time.Millisecond) { + t.Fatal("timer did not fire") + } +} diff --git a/internal/controlplane/api/hub_test.go b/internal/controlplane/api/hub_test.go index daaab5a..c80a607 100644 --- a/internal/controlplane/api/hub_test.go +++ b/internal/controlplane/api/hub_test.go @@ -304,6 +304,12 @@ func (s *memoryStore) DisableNetworkExit(_ context.Context, id string) (hub.Netw return exit, nil } func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) { + if env.MemoryLimitMB == 0 { + env.MemoryLimitMB = 2048 + } + if env.MemoryLimitMB < 512 || env.MemoryLimitMB > 65536 { + return hub.EnvironmentContext{}, false, hub.ErrInvalid + } s.mu.Lock() if existing, ok := s.bindings[env.Alias]; ok { if existing.AccountStatus == "" { @@ -350,19 +356,24 @@ func (s *memoryStore) GetEnvironmentContextForAccount(ctx context.Context, accou s.mu.Unlock() return hub.EnvironmentContext{}, hub.ErrNotFound } -func (s *memoryStore) UpdateAccountFingerprint(_ context.Context, accountID string, fingerprint hub.Fingerprint) (hub.EnvironmentContext, error) { - if fingerprint.ProxyServer != "" || fingerprint.Validate() != nil { +func (s *memoryStore) UpdateEnvironmentSettings(_ context.Context, alias string, fingerprint *hub.Fingerprint, memoryLimitMB *int) (hub.EnvironmentContext, error) { + if (fingerprint != nil && (fingerprint.ProxyServer != "" || fingerprint.Validate() != nil)) || (memoryLimitMB != nil && (*memoryLimitMB < 512 || *memoryLimitMB > 65536)) { return hub.EnvironmentContext{}, hub.ErrInvalid } s.mu.Lock() defer s.mu.Unlock() - for alias, bound := range s.bindings { - if bound.AccountID == accountID { - fingerprint.Seed = bound.Fingerprint.Seed - bound.Fingerprint = fingerprint - s.bindings[alias] = bound - return bound, nil + bound, ok := s.bindings[alias] + if ok { + if fingerprint != nil { + updated := *fingerprint + updated.Seed = bound.Fingerprint.Seed + bound.Fingerprint = updated } + if memoryLimitMB != nil { + bound.MemoryLimitMB = *memoryLimitMB + } + s.bindings[alias] = bound + return bound, nil } return hub.EnvironmentContext{}, hub.ErrNotFound } diff --git a/internal/controlplane/api/private_messages.go b/internal/controlplane/api/private_messages.go new file mode 100644 index 0000000..79646a3 --- /dev/null +++ b/internal/controlplane/api/private_messages.go @@ -0,0 +1,273 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/creator" + "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" +) + +type PrivateSendResult struct { + State, ServerID, Error string + MessageAt *time.Time +} +type privateMessageSender func(context.Context, creator.PrivateMessageSendInput, creator.PrivateMessageReservation) PrivateSendResult + +func registerPrivateMessageRoutes(app *fiber.App, store *creator.Store, send privateMessageSender) { + app.Get("/api/creator/private-messages/conversations", func(c fiber.Ctx) error { + page, size, _, err := creatorPageQuery(c) + if err != nil { + return creatorError(c, err) + } + result, err := store.ListPrivateConversations(c.Context(), c.Query("account_id"), page, size) + if err != nil { + return creatorError(c, err) + } + return c.JSON(result) + }) + app.Get("/api/creator/private-messages/messages", func(c fiber.Ctx) error { + page, size, _, err := creatorPageQuery(c) + if err != nil { + return creatorError(c, err) + } + result, err := store.ListPrivateMessages(c.Context(), c.Query("account_id"), c.Query("peer_uid"), page, size) + if err != nil { + return creatorError(c, err) + } + return c.JSON(result) + }) + app.Get("/api/creator/private-messages/status", func(c fiber.Ctx) error { + result, err := store.ListPrivateSyncStatus(c.Context()) + if err != nil { + return creatorError(c, err) + } + return c.JSON(result) + }) + app.Post("/api/creator/private-messages/send", func(c fiber.Ctx) error { + var input creator.PrivateMessageSendInput + if err := c.Bind().Body(&input); err != nil { + return creatorError(c, creator.ErrInvalid) + } + reservation, err := store.BeginPrivateMessage(c.Context(), input) + if err != nil { + return creatorError(c, err) + } + if !reservation.New { + return c.JSON(reservation.Message) + } + fields := logrus.Fields{"account_id": input.AccountID, "peer_uid": input.PeerUID, "request_id": input.RequestID, "message_id": reservation.Message.ID, "generation": reservation.Generation} + logrus.WithFields(fields).Info("private message send reserved") + // Completion must be saved even when the client closes its connection. + ctx, cancel := context.WithTimeout(context.WithoutCancel(c.Context()), 65*time.Second) + defer cancel() + result := send(ctx, input, reservation) + message, err := store.FinishPrivateMessage(ctx, reservation.Message.ID, result.State, result.ServerID, result.Error, result.MessageAt) + if err != nil { + logrus.WithError(err).WithFields(fields).Error("private message send outcome persistence failed") + return creatorError(c, err) + } + fields["state"] = message.State + fields["server_id"] = message.ServerID + if message.State != "succeeded" { + logrus.WithFields(fields).WithField("reason", message.Error).Error("private message send not confirmed") + } else { + logrus.WithFields(fields).Info("private message platform send confirmed") + } + return c.JSON(message) + }) +} +func decodePrivateSendResult(raw []byte) PrivateSendResult { + var result struct { + Status string `json:"status"` + Code string `json:"code"` + StatusCode json.RawMessage `json:"status_code"` + CheckCode string `json:"check_code"` + CheckMessage string `json:"check_message"` + Success *bool `json:"success"` + Message struct { + ServerID string `json:"server_id"` + } `json:"message"` + } + if err := json.Unmarshal(raw, &result); err != nil { + return PrivateSendResult{State: "unknown", Error: "网关响应无法解析，发送结果未确认：" + err.Error()} + } + reason := result.Code + if len(result.StatusCode) > 0 && string(result.StatusCode) != "null" { + reason += "；status_code=" + string(result.StatusCode) + } + if result.CheckCode != "" { + reason += "；check_code=" + result.CheckCode + } + if result.CheckMessage != "" { + reason += "；" + result.CheckMessage + } + if strings.Trim(string(result.StatusCode), "\"") == "1008" { + return PrivateSendResult{State: "unknown", Error: "聊天客户端网络错误，发送结果未确认：" + reason} + } + if result.Status == "failed" || result.Status == "denied" || result.Status == "unsupported" { + return PrivateSendResult{State: "failed", Error: "发送失败：" + reason} + } + if result.Status != "succeeded" || result.Success == nil || !*result.Success || result.Message.ServerID == "" || result.Message.ServerID == "0" { + return PrivateSendResult{State: "unknown", Error: fmt.Sprintf("发送结果未确认：status=%s %s", result.Status, reason)} + } + if _, err := strconv.ParseUint(result.Message.ServerID, 10, 64); err != nil { + return PrivateSendResult{State: "unknown", Error: "平台返回的消息编号无效"} + } + return PrivateSendResult{State: "succeeded", ServerID: result.Message.ServerID} +} +func privateListenerState(ctx context.Context, store *creator.Store, accountID string) (creator.ListenerState, error) { + states, err := store.ListPrivateListeners(ctx) + if err != nil { + return creator.ListenerState{}, err + } + for _, state := range states { + if state.AccountID == accountID { + return state, nil + } + } + return creator.ListenerState{}, creator.ErrNotFound +} + +func gatewayPrivateMessageSender(accounts HubStore, store *creator.Store) privateMessageSender { + return func(ctx context.Context, input creator.PrivateMessageSendInput, r creator.PrivateMessageReservation) PrivateSendResult { + state, err := privateListenerState(ctx, store, input.AccountID) + if err != nil { + return PrivateSendResult{State: "failed", Error: err.Error()} + } + if !state.Enabled || state.Generation != r.Generation { + return PrivateSendResult{State: "failed", Error: "监听已关闭或状态已变化，未发送"} + } + session, err := listenerSessionForAccount(ctx, store, accounts, state) + if err != nil { + return PrivateSendResult{State: "failed", Error: err.Error()} + } + target, environment := session.target, session.environment + payload := gatewayGenerationPayload(environment) + payload["expected_uid"] = r.UID + payload["action"] = "dm" + payload["target_uid"] = input.PeerUID + payload["text"] = input.Text + payload["confirm"] = true + payload["operation_id"] = r.Message.ID + status, body, err := gatewayCallWithLimit(ctx, target, http.MethodPost, "/v1/browsers/"+url.PathEscape(environment.Alias)+"/douyin/action", payload, 55*time.Second, largeGatewayResponseLimit) + if err != nil { + return PrivateSendResult{State: "unknown", Error: "发送请求已交给网关，但结果未确认：" + err.Error()} + } + if status != http.StatusOK { + state := "unknown" + if status == 400 || status == 409 { + state = "failed" + } + return PrivateSendResult{State: state, Error: fmt.Sprintf("发送网关 HTTP %d：%s", status, body)} + } + return decodePrivateSendResult(body) + } +} + +// Polling uses the IM SDK itself: site notification notices are not reliable +// evidence of individual chat messages. Only enabled listener generations write. +func RunPrivateMessageSync(ctx context.Context, accounts HubStore, store *creator.Store) { + if err := store.RecoverInterruptedPrivateMessages(ctx); err != nil { + logrus.WithError(err).Error("private message interrupted-send recovery failed") + return + } + states := make(map[string]string) + checkpoints := make(map[string]map[string]string) + for { + listeners, err := store.ListPrivateListeners(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + logrus.WithError(err).Error("private message enabled-account query failed") + } + for _, listener := range listeners { + if !listener.Enabled { + continue + } + if ctx.Err() != nil { + return + } + key := listener.AccountID + ":" + listener.Generation + syncCtx, cancel := context.WithTimeout(ctx, 35*time.Second) + next, err := syncPrivateInbox(syncCtx, accounts, store, listener, checkpoints[key]) + cancel() + if err != nil { + if ctx.Err() != nil { + return + } + if errors.Is(err, creator.ErrConflict) { + continue + } + reason := err.Error() + if states[key] != reason { + logrus.WithError(err).WithFields(logrus.Fields{"account_id": listener.AccountID, "generation": listener.Generation}).Error("private message inbox sync failed") + } + states[key] = reason + if statusErr := store.UpdatePrivateSyncStatus(ctx, listener.AccountID, listener.Generation, reason); statusErr != nil && !errors.Is(statusErr, creator.ErrConflict) { + logrus.WithError(statusErr).WithField("account_id", listener.AccountID).Error("private message sync error persistence failed") + } + } else { + if states[key] != "connected" { + logrus.WithFields(logrus.Fields{"account_id": listener.AccountID, "generation": listener.Generation}).Info("private message inbox sync connected") + } + states[key] = "connected" + checkpoints[key] = next + } + } + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + } + } +} +func syncPrivateInbox(ctx context.Context, accounts HubStore, store *creator.Store, listener creator.ListenerState, checkpoints map[string]string) (map[string]string, error) { + profile, err := store.GetAccountProfile(ctx, listener.AccountID) + if err != nil { + return nil, err + } + if profile.LoginStatus != "logged_in" { + return nil, fmt.Errorf("账号未登录") + } + session, err := listenerSessionForAccount(ctx, store, accounts, listener) + if err != nil { + return nil, err + } + target, environment := session.target, session.environment + payload := gatewayGenerationPayload(environment) + payload["expected_uid"] = profile.PlatformAccountKey + if checkpoints != nil { + payload["checkpoints"] = checkpoints + } + status, body, err := gatewayCallWithLimit(ctx, target, http.MethodPost, "/v1/browsers/"+url.PathEscape(environment.Alias)+"/douyin/inbox", payload, 30*time.Second, largeGatewayResponseLimit) + if err != nil { + return nil, err + } + if status != 200 { + return nil, fmt.Errorf("收件同步网关 HTTP %d：%s", status, body) + } + items, err := creator.DecodePrivateInbox(body, profile.PlatformAccountKey) + if err != nil { + return nil, err + } + var response struct { + Checkpoints map[string]string `json:"checkpoints"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, err + } + if response.Checkpoints == nil { + return nil, fmt.Errorf("收件同步未返回进度") + } + if err := store.SavePrivateInbox(ctx, listener.AccountID, listener.Generation, items); err != nil { + return nil, err + } + return response.Checkpoints, nil +} diff --git a/internal/controlplane/api/private_messages_test.go b/internal/controlplane/api/private_messages_test.go new file mode 100644 index 0000000..93b6973 --- /dev/null +++ b/internal/controlplane/api/private_messages_test.go @@ -0,0 +1,130 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/creator" + "github.com/gofiber/fiber/v3" +) + +func TestPrivateMessageRoutes(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("requires CREATORHUB_POSTGRES_TEST_URL") + } + store, accounts, ctx := openCreatorIntegrationStoreForAPITest(t, databaseURL) + owner := account.Account{ID: "private-owner", Platform: creator.PlatformDouyin, Name: "本人", PlatformAccountKey: "123", CredentialReference: account.CredentialReference{ID: "private-credential", Provider: "os_keyring"}, CredentialKey: "creatorhub/private-owner/cookies"} + bridge := &testCredentialBridge{values: make(map[string]string)} + if err := accounts.CreateAccount(ctx, owner, bridge); err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, owner.ID, "123"); err != nil { + t.Fatal(err) + } + if _, err := store.SetListenerEnabled(ctx, owner.ID, true); err != nil { + t.Fatal(err) + } + calls := 0 + send := func(ctx context.Context, i creator.PrivateMessageSendInput, r creator.PrivateMessageReservation) PrivateSendResult { + calls++ + return PrivateSendResult{State: "succeeded", ServerID: "1001"} + } + app := fiber.New() + registerPrivateMessageRoutes(app, store, send) + input := fmt.Sprintf(`{"request_id":"one","account_id":%q,"peer_uid":"456","text":"你好"}`, owner.ID) + for n := 0; n < 2; n++ { + req := httptest.NewRequest("POST", "/api/creator/private-messages/send", strings.NewReader(input)) + req.Header.Set("Content-Type", "application/json") + res, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Fatalf("send status=%d", res.StatusCode) + } + var msg creator.PrivateMessage + if err := json.NewDecoder(res.Body).Decode(&msg); err != nil { + t.Fatal(err) + } + res.Body.Close() + if msg.State != "succeeded" { + t.Fatalf("message=%+v", msg) + } + } + if calls != 1 { + t.Fatalf("send calls=%d", calls) + } + for _, path := range []string{"/api/creator/private-messages/conversations?page=1&page_size=25", "/api/creator/private-messages/messages?account_id=" + owner.ID + "&peer_uid=456", "/api/creator/private-messages/status"} { + res, err := app.Test(httptest.NewRequest("GET", path, nil)) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Fatalf("%s status=%d", path, res.StatusCode) + } + res.Body.Close() + } + res, err := app.Test(httptest.NewRequest("GET", "/api/creator/private-messages/messages?account_id="+owner.ID+"&peer_uid=bad", nil)) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != http.StatusBadRequest { + t.Fatalf("bad uid=%d", res.StatusCode) + } + res.Body.Close() + if _, err := store.SetListenerEnabled(ctx, owner.ID, false); err != nil { + t.Fatal(err) + } + input = strings.Replace(input, `"one"`, `"two"`, 1) + req := httptest.NewRequest("POST", "/api/creator/private-messages/send", strings.NewReader(input)) + req.Header.Set("Content-Type", "application/json") + res, err = app.Test(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 409 || calls != 1 { + t.Fatalf("disabled status=%d calls=%d", res.StatusCode, calls) + } +} +func TestPrivateNetworkFailureIsUnconfirmedNotRejected(t *testing.T) { + result := decodePrivateSendResult([]byte(`{"status":"failed","code":"MESSAGE_REJECTED","status_code":1008}`)) + if result.State != "unknown" || !strings.Contains(result.Error, "1008") { + t.Fatalf("network error=%+v", result) + } +} + +func TestPrivateSendRejectionPreservesReason(t *testing.T) { + result := decodePrivateSendResult([]byte(`{"status":"failed","code":"MESSAGE_REJECTED","status_code":401,"check_code":"restricted","check_message":"暂时无法发送"}`)) + if result.State != "failed" || !strings.Contains(result.Error, "restricted") || !strings.Contains(result.Error, "401") || !strings.Contains(result.Error, "暂时无法发送") { + t.Fatalf("reason=%+v", result) + } +} + +func TestDecodePrivateSendResult(t *testing.T) { + cases := []struct { + raw string + state string + id string + }{ + {`{"status":"succeeded","success":true,"message":{"server_id":"100"}}`, "succeeded", "100"}, + {`{"status":"succeeded","success":false,"message":{"server_id":"100"}}`, "unknown", ""}, + {`{"status":"succeeded","success":true,"message":{}}`, "unknown", ""}, + {`{"status":"failed","code":"IM_SEND_REJECTED"}`, "failed", ""}, + {`{}`, "unknown", ""}, {`bad`, "unknown", ""}, + } + for _, c := range cases { + r := decodePrivateSendResult([]byte(c.raw)) + if r.State != c.state || r.ServerID != c.id { + t.Fatalf("raw=%s result=%+v", c.raw, r) + } + } +} diff --git a/internal/controlplane/api/private_messages_worker_test.go b/internal/controlplane/api/private_messages_worker_test.go new file mode 100644 index 0000000..16107b0 --- /dev/null +++ b/internal/controlplane/api/private_messages_worker_test.go @@ -0,0 +1,169 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/account" + "git.ipao.vip/rogee/creator-hub/internal/creator" + hub "git.ipao.vip/rogee/creator-hub/internal/environment" +) + +func privateWorkerFixture(t *testing.T) (*creator.Store, creator.ListenerState) { + t.Helper() + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("requires CREATORHUB_POSTGRES_TEST_URL") + } + store, accounts, ctx := openCreatorIntegrationStoreForAPITest(t, databaseURL) + owner := account.Account{ID: "dm-owner", Platform: creator.PlatformDouyin, Name: "本人", PlatformAccountKey: "123", CredentialReference: account.CredentialReference{ID: "dm-credential", Provider: "os_keyring"}, CredentialKey: "creatorhub/dm-owner/cookies"} + if err := accounts.CreateAccount(ctx, owner, &testCredentialBridge{values: make(map[string]string)}); err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, owner.ID, "123"); err != nil { + t.Fatal(err) + } + state, err := store.SetListenerEnabled(ctx, owner.ID, true) + if err != nil { + t.Fatal(err) + } + return store, state +} +func TestPrivateGatewaySendAndSync(t *testing.T) { + store, state := privateWorkerFixture(t) + ctx := context.Background() + var mode atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mode.Load() == 1 { + w.WriteHeader(502) + return + } + var payload map[string]any + json.NewDecoder(r.Body).Decode(&payload) + if payload["expected_uid"] != "123" { + t.Errorf("identity=%v", payload) + } + if r.URL.Path == "/v1/browsers/browser/douyin/action" { + w.Write([]byte(`{"status":"succeeded","success":true,"message":{"server_id":"100"}}`)) + return + } + if r.URL.Path != "/v1/browsers/browser/douyin/inbox" { + t.Errorf("path=%s", r.URL.Path) + } + if mode.Load() == 2 { + w.Write([]byte(`{"status":"succeeded","account_uid":"123","messages":[]}`)) + return + } + if mode.Load() == 3 { + w.Write([]byte(`{"status":"failed","code":"IM_SDK_NOT_READY"}`)) + return + } + w.Write([]byte(`{"status":"succeeded","account_uid":"123","messages":[{"peer_uid":"456","server_id":"101","sender_uid":"456","message_type":"7","content":"{\"aweType\":700,\"text\":\"你好\"}","created_at":"1700000000000"}],"checkpoints":{"456":"101"}}`)) + })) + defer server.Close() + accounts := listenerAccountsStub{gateway: hub.Gateway{Endpoint: server.URL, Token: "token"}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser", Gateway: "gateway"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 7}} + input := creator.PrivateMessageSendInput{RequestID: "send-one", AccountID: state.AccountID, PeerUID: "456", Text: "你好"} + reservation, err := store.BeginPrivateMessage(ctx, input) + if err != nil { + t.Fatal(err) + } + result := gatewayPrivateMessageSender(accounts, store)(ctx, input, reservation) + if result.State != "succeeded" || result.ServerID != "100" { + t.Fatalf("send=%+v", result) + } + checkpoints, err := syncPrivateInbox(ctx, accounts, store, state, map[string]string{"456": "99"}) + if err != nil || checkpoints["456"] != "101" { + t.Fatalf("sync=%v %v", checkpoints, err) + } + page, err := store.ListPrivateMessages(ctx, state.AccountID, "456", 1, 25) + if err != nil || page.Total != 2 { + t.Fatalf("page=%+v %v", page, err) + } + for _, m := range []int32{1, 2, 3} { + mode.Store(m) + if _, err := syncPrivateInbox(ctx, accounts, store, state, checkpoints); err == nil { + t.Fatalf("accepted mode %d", m) + } + } + mode.Store(1) + result = gatewayPrivateMessageSender(accounts, store)(ctx, input, reservation) + if result.State != "unknown" { + t.Fatalf("HTTP error=%+v", result) + } + invalid := accounts + invalid.environmentErr = errors.New("missing environment") + result = gatewayPrivateMessageSender(invalid, store)(ctx, input, reservation) + if result.State != "failed" { + t.Fatalf("missing environment=%+v", result) + } + if _, err := syncPrivateInbox(ctx, invalid, store, state, nil); err == nil { + t.Fatal("missing environment accepted") + } + if _, err := store.SetListenerEnabled(ctx, state.AccountID, false); err != nil { + t.Fatal(err) + } + result = gatewayPrivateMessageSender(accounts, store)(ctx, input, reservation) + if result.State != "failed" { + t.Fatalf("disabled=%+v", result) + } + if _, err := privateListenerState(ctx, store, "missing"); !errors.Is(err, creator.ErrNotFound) { + t.Fatalf("missing=%v", err) + } +} +func TestPrivateSyncWorkerRecoveryAndErrors(t *testing.T) { + for _, failure := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "failure"}[failure], func(t *testing.T) { + store, state := privateWorkerFixture(t) + input := creator.PrivateMessageSendInput{RequestID: "interrupted", AccountID: state.AccountID, PeerUID: "456", Text: "你好"} + if _, err := store.BeginPrivateMessage(context.Background(), input); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if failure { + w.WriteHeader(502) + return + } + w.Write([]byte(`{"status":"succeeded","account_uid":"123","messages":[],"checkpoints":{}}`)) + })) + defer server.Close() + accounts := listenerAccountsStub{gateway: hub.Gateway{Endpoint: server.URL}, environment: hub.EnvironmentContext{Env: hub.Env{Alias: "browser", Gateway: "gateway"}, RuntimeID: "runtime", RuntimeNetworkID: "network", BindingVersion: 7}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { defer close(done); RunPrivateMessageSync(ctx, accounts, store) }() + deadline := time.Now().Add(3 * time.Second) + matched := false + for time.Now().Before(deadline) { + statuses, err := store.ListPrivateSyncStatus(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(statuses) == 1 && (failure && statuses[0].Error != "" || !failure && statuses[0].SyncedAt != nil) { + matched = true + break + } + time.Sleep(10 * time.Millisecond) + } + if !matched { + t.Fatal("sync result not persisted") + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("worker did not stop") + } + page, err := store.ListPrivateMessages(context.Background(), state.AccountID, "456", 1, 25) + if err != nil || page.Data[0].State != "unknown" { + t.Fatalf("interrupted=%+v %v", page, err) + } + }) + } +} diff --git a/internal/controlplane/app/app.go b/internal/controlplane/app/app.go index 63a860d..6de768b 100644 --- a/internal/controlplane/app/app.go +++ b/internal/controlplane/app/app.go @@ -97,20 +97,36 @@ func newCommand() *cobra.Command { defer close(creatorScheduleDone) workers.RunCreatorScheduler(creatorScheduleContext, creatorStore, phaseAStore, hubStore) }() + listenerContext, stopListeners := context.WithCancel(command.Context()) + listenerDone := make(chan struct{}) + go func() { + defer close(listenerDone) + api.RunAccountEventListeners(listenerContext, creatorStore, hubStore) + }() gatewayHealthContext, stopGatewayHealth := context.WithCancel(command.Context()) gatewayHealthDone := make(chan struct{}) go func() { defer close(gatewayHealthDone) workers.RunGatewayHealthCheck(gatewayHealthContext, hubStore) }() + privateMessagesContext, stopPrivateMessages := context.WithCancel(command.Context()) + privateMessagesDone := make(chan struct{}) + go func() { + defer close(privateMessagesDone) + api.RunPrivateMessageSync(privateMessagesContext, hubStore, creatorStore) + }() listenErr := newHandler(cfg.webDir, cfg.username, cfg.password, phaseAStore, hubStore, credentials, creatorStore).Listen(cfg.listenAddr, fiber.ListenConfig{ GracefulContext: command.Context(), DisableStartupMessage: true, }) stopCreatorScheduler() + stopListeners() + stopPrivateMessages() stopGatewayHealth() stopHeartbeat() <-creatorScheduleDone + <-listenerDone + <-privateMessagesDone <-gatewayHealthDone <-heartbeatDone return listenErr diff --git a/internal/creator/account_collection_monitor_test.go b/internal/creator/account_collection_monitor_test.go new file mode 100644 index 0000000..576b852 --- /dev/null +++ b/internal/creator/account_collection_monitor_test.go @@ -0,0 +1,110 @@ +package creator + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func TestCreatorPostgresMonitorViewIncludesCollectionCheckpoints(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + accountID := createIntegrationAccount(t, ctx, accounts, fmt.Sprintf("9%d", time.Now().UnixNano())) + otherID := createIntegrationAccount(t, ctx, accounts, fmt.Sprintf("8%d", time.Now().UnixNano())) + for _, id := range []string{accountID, otherID} { + if err := store.EnsureAccountProfile(ctx, id); err != nil { + t.Fatal(err) + } + } + viewFor := func(id string) AccountMonitorView { + t.Helper() + views, err := store.ListAccountMonitorViews(ctx) + if err != nil { + t.Fatal(err) + } + for _, view := range views { + if view.ID == id { + return view + } + } + t.Fatalf("account %s missing from monitor views", id) + return AccountMonitorView{} + } + pending := viewFor(accountID) + if pending.WorksCollection.Status != "pending" || pending.CommentsCollection.Status != "pending" { + t.Fatalf("missing checkpoints must be pending, not healthy: %+v", pending) + } + + now := time.Now().UTC() + works, err := store.beginCheckpoint(ctx, SourceOwned, accountID, "works", now.Add(-time.Hour), now) + if err != nil { + t.Fatal(err) + } + if viewFor(accountID).WorksCollection.Status != "running" { + t.Fatal("list must expose the currently running works checkpoint") + } + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", works, "succeeded", "", nil); err != nil { + t.Fatal(err) + } + completed := viewFor(accountID).WorksCollection + if completed.Status != "succeeded" || completed.LastCompletedAt == nil { + t.Fatalf("list must expose the real last completion: %+v", completed) + } + + comments, err := store.beginCheckpoint(ctx, SourceOwned, accountID, "comments", now.Add(-time.Hour), now) + if err != nil { + t.Fatal(err) + } + if err := store.failCheckpoint(ctx, SourceOwned, accountID, "comments", comments, fmt.Errorf("comments transport failed")); err == nil { + t.Fatal("failed checkpoint must return the collection error") + } + works, err = store.beginCheckpoint(ctx, SourceOwned, accountID, "works", now, now.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", works, "blocked", "browser is awaiting user login", nil); err != nil { + t.Fatal(err) + } + // Competitor checkpoints must never be mistaken for an owned-account result. + if _, err := store.beginCheckpoint(ctx, SourceCompetitor, accountID, "works", now, now.Add(2*time.Hour)); err != nil { + t.Fatal(err) + } + + view := viewFor(accountID) + detail, err := store.GetAccountCollectionStatus(ctx, accountID) + if err != nil { + t.Fatal(err) + } + if view.WorksCollection.Status != detail.Works.Status || view.WorksCollection.LastError != detail.Works.LastError || + view.WorksCollection.LastCompletedAt == nil || !view.WorksCollection.LastCompletedAt.Equal(*detail.Works.LastCompletedAt) { + t.Fatalf("list and detail works checkpoints differ: list=%+v detail=%+v", view.WorksCollection, detail.Works) + } + if !view.WorksCollection.LastCompletedAt.Equal(*completed.LastCompletedAt) { + t.Fatal("blocked attempt must not replace the previous successful completion time") + } + if view.CommentsCollection.Status != "failed" || view.CommentsCollection.LastError != "comments transport failed" || view.CommentsCollection.LastCompletedAt != nil { + t.Fatalf("comment failures must remain independent and not invent completion: %+v", view.CommentsCollection) + } + other := viewFor(otherID) + if other.WorksCollection.Status != "pending" || other.CommentsCollection.Status != "pending" { + t.Fatalf("one account's failures must not affect another: %+v", other) + } + payload, err := json.Marshal(view) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(payload), `"works_collection"`) || !strings.Contains(string(payload), `"comments_collection"`) { + t.Fatalf("monitor API must include both checkpoints: %s", payload) + } +} + +func TestCreatorPostgresMonitorViewDoesNotHideCheckpointReadFailure(t *testing.T) { + store, _, ctx := openCreatorIntegrationStore(t) + if _, err := store.db.ExecContext(ctx, `DROP TABLE creator_collection_checkpoint`); err != nil { + t.Fatal(err) + } + if _, err := store.ListAccountMonitorViews(ctx); err == nil { + t.Fatal("unreadable checkpoints must not be reported as healthy or pending") + } +} diff --git a/internal/creator/accounts.go b/internal/creator/accounts.go index a66c152..9cb5820 100644 --- a/internal/creator/accounts.go +++ b/internal/creator/accounts.go @@ -28,15 +28,17 @@ func accountProfileQuery() string { p.login_username, p.password_configured, p.real_name_status, p.real_name, p.identity_number, p.note, p.business_status, p.big_account, p.reply_requirements, p.login_status, p.login_reason, - p.login_checked_at, p.cooldown_seconds, p.updated_at, p.avatar_url, p.douyin_number + p.login_checked_at, p.cooldown_seconds, p.updated_at, p.avatar_url, p.douyin_number, + COALESCE(l.enabled, false), COALESCE(l.status, 'stopped'), COALESCE(l.reason, ''), l.last_poll_at FROM social_account a JOIN creator_account_profile p ON p.account_id = a.id + LEFT JOIN creator_account_listener l ON l.account_id = a.id WHERE a.account_id = $1` } func scanAccountProfile(scanner interface{ Scan(...any) error }) (AccountProfile, error) { var result AccountProfile - var checkedAt sql.NullTime + var checkedAt, lastPollAt sql.NullTime if err := scanner.Scan( &result.ID, &result.Name, &result.Platform, &result.PlatformAccountKey, &result.RuntimeStatus, @@ -44,10 +46,12 @@ func scanAccountProfile(scanner interface{ Scan(...any) error }) (AccountProfile &result.RealName, &result.IdentityNumber, &result.Note, &result.BusinessStatus, &result.BigAccount, &result.ReplyRequirements, &result.LoginStatus, &result.LoginReason, &checkedAt, &result.CooldownSeconds, &result.UpdatedAt, &result.AvatarURL, &result.DouyinNumber, + &result.ListenerEnabled, &result.ListenerStatus, &result.ListenerReason, &lastPollAt, ); err != nil { return AccountProfile{}, err } result.LoginCheckedAt = nullableTime(checkedAt) + result.ListenerLastPollAt = nullableTime(lastPollAt) return result, nil } @@ -108,14 +112,16 @@ func (s *Store) ListAccountProfiles(ctx context.Context) ([]AccountProfile, erro // AccountMonitorView 自有账号监控列表视图：账号画像 + 作品聚合统计（作品数/最近发布）+ 最新画像指标快照 + 自有作品评论累计。 type AccountMonitorView struct { AccountProfile - WorkCount int64 `json:"work_count"` - LatestPublishedAt *time.Time `json:"latest_published_at,omitempty"` - FollowerCount *int64 `json:"follower_count,omitempty"` - FollowingCount *int64 `json:"following_count,omitempty"` - TotalFavorited *int64 `json:"total_favorited,omitempty"` - AwemeCount *int64 `json:"aweme_count,omitempty"` - FriendCount *int64 `json:"friend_count,omitempty"` - CommentTotal int64 `json:"comment_total"` + WorkCount int64 `json:"work_count"` + LatestPublishedAt *time.Time `json:"latest_published_at,omitempty"` + FollowerCount *int64 `json:"follower_count,omitempty"` + FollowingCount *int64 `json:"following_count,omitempty"` + TotalFavorited *int64 `json:"total_favorited,omitempty"` + PublicWorkCount *int64 `json:"public_work_count,omitempty"` + FriendCount *int64 `json:"friend_count,omitempty"` + CommentTotal int64 `json:"comment_total"` + WorksCollection AccountCheckpointStatus `json:"works_collection"` + CommentsCollection AccountCheckpointStatus `json:"comments_collection"` } // ListAccountMonitorViews 自有账号监控列表：画像 + owned 作品聚合，形态对齐竞品的 ListCompetitorsWithProfile。 @@ -132,15 +138,27 @@ func (s *Store) ListAccountMonitorViews(ctx context.Context) ([]AccountMonitorVi if err != nil { return nil, err } + collectionStates, err := s.accountCollectionStates(ctx) + if err != nil { + return nil, err + } views := make([]AccountMonitorView, 0, len(profiles)) for _, profile := range profiles { - view := AccountMonitorView{AccountProfile: profile} + view := AccountMonitorView{ + AccountProfile: profile, + WorksCollection: AccountCheckpointStatus{Status: "pending"}, + CommentsCollection: AccountCheckpointStatus{Status: "pending"}, + } if stat, exists := stats[profile.ID]; exists { view.WorkCount, view.LatestPublishedAt, view.CommentTotal = stat.WorkCount, stat.LatestPublishedAt, stat.CommentTotal } if metric, exists := latest[profile.ID]; exists { - view.FollowerCount, view.FollowingCount, view.TotalFavorited, view.AwemeCount, view.FriendCount = - metric.FollowerCount, metric.FollowingCount, metric.TotalFavorited, metric.AwemeCount, metric.FriendCount + view.FollowerCount, view.FollowingCount, view.TotalFavorited, view.FriendCount = + metric.FollowerCount, metric.FollowingCount, metric.TotalFavorited, metric.FriendCount + } + if state, exists := collectionStates[profile.ID]; exists { + view.PublicWorkCount = state.PublicWorkCount + view.WorksCollection, view.CommentsCollection = state.Works, state.Comments } views = append(views, view) } @@ -158,14 +176,13 @@ type latestAccountMetricRow struct { FollowerCount *int64 FollowingCount *int64 TotalFavorited *int64 - AwemeCount *int64 FriendCount *int64 } -// latestAccountMetrics 每账号取最新非空画像计数：按采集时间倒序回填，缺失维度保留上一轮的值（详情页同口径）。 +// latestAccountMetrics 回填每账号最新非空画像指标；公开作品总数单独从 works checkpoint 读取。 func (s *Store) latestAccountMetrics(ctx context.Context) (map[string]latestAccountMetricRow, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT account.account_id, metric.follower_count, metric.following_count, metric.total_favorited, metric.aweme_count, metric.friend_count + SELECT account.account_id, metric.follower_count, metric.following_count, metric.total_favorited, metric.friend_count FROM creator_account_metric metric JOIN social_account account ON account.id = metric.account_id ORDER BY account.account_id, metric.collected_at`) @@ -176,8 +193,8 @@ func (s *Store) latestAccountMetrics(ctx context.Context) (map[string]latestAcco latest := make(map[string]latestAccountMetricRow) for rows.Next() { var accountID string - var follower, following, favorited, aweme, friend sql.NullInt64 - if err := rows.Scan(&accountID, &follower, &following, &favorited, &aweme, &friend); err != nil { + var follower, following, favorited, friend sql.NullInt64 + if err := rows.Scan(&accountID, &follower, &following, &favorited, &friend); err != nil { return nil, err } current := latest[accountID] @@ -193,10 +210,6 @@ func (s *Store) latestAccountMetrics(ctx context.Context) (map[string]latestAcco value := favorited.Int64 current.TotalFavorited = &value } - if aweme.Valid { - value := aweme.Int64 - current.AwemeCount = &value - } if friend.Valid { value := friend.Int64 current.FriendCount = &value @@ -206,13 +219,50 @@ func (s *Store) latestAccountMetrics(ctx context.Context) (map[string]latestAcco return latest, rows.Err() } +func (s *Store) accountCollectionStates(ctx context.Context) (map[string]AccountCollectionStatus, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT source_id, collection_kind, status, last_completed_at, COALESCE(last_error, ''), + window_end, public_work_count + FROM creator_collection_checkpoint + WHERE source_type = $1 AND collection_kind IN ('works', 'comments')`, SourceOwned) + if err != nil { + return nil, databaseError(err) + } + defer rows.Close() + states := make(map[string]AccountCollectionStatus) + for rows.Next() { + var sourceID, kind string + var checkpoint AccountCheckpointStatus + var completed, end sql.NullTime + var publicCount sql.NullInt64 + if err := rows.Scan(&sourceID, &kind, &checkpoint.Status, &completed, &checkpoint.LastError, &end, &publicCount); err != nil { + return nil, fmt.Errorf("decode account collection checkpoint: %w", err) + } + checkpoint.LastCompletedAt, checkpoint.WindowEnd = nullableTime(completed), nullableTime(end) + state, exists := states[sourceID] + if !exists { + state.Works.Status, state.Comments.Status = "pending", "pending" + } + if kind == "works" { + state.Works = checkpoint + if publicCount.Valid { + state.PublicWorkCount = &publicCount.Int64 + } + } else { + state.Comments = checkpoint + } + states[sourceID] = state + } + return states, rows.Err() +} + // AccountCollectionStatus 自有账号采集状态（checkpoint 形态，对齐竞品的 sync_status 展示语义）。 func (s *Store) ownedWorkStats(ctx context.Context) (map[string]ownedWorkStat, error) { rows, err := s.db.QueryContext(ctx, ` SELECT w.source_id, COUNT(*) AS work_count, MAX(w.published_at) AS latest_published_at, COALESCE(SUM(w.comments_count), 0) AS comment_total FROM creator_work w - WHERE w.source_type = 'owned' + WHERE w.source_type = 'owned' AND w.is_public = true GROUP BY w.source_id`) if err != nil { return nil, databaseError(err) @@ -234,10 +284,10 @@ func (s *Store) ownedWorkStats(ctx context.Context) (map[string]ownedWorkStat, e // AccountCollectionStatus 自有账号采集状态（checkpoint 形态，对齐竞品的 sync_status 展示语义）。 type AccountCollectionStatus struct { - WorkCount int64 `json:"work_count"` - AwemeCount *int64 `json:"aweme_count"` - Works AccountCheckpointStatus `json:"works"` - Comments AccountCheckpointStatus `json:"comments"` + WorkCount int64 `json:"work_count"` + PublicWorkCount *int64 `json:"public_work_count"` + Works AccountCheckpointStatus `json:"works"` + Comments AccountCheckpointStatus `json:"comments"` } type AccountCheckpointStatus struct { @@ -259,16 +309,14 @@ func (s *Store) GetAccountCollectionStatus(ctx context.Context, accountID string var total sql.NullInt64 err := s.db.QueryRowContext(ctx, ` SELECT - (SELECT COUNT(*) FROM creator_work WHERE source_type = $1 AND source_id = $2), - (SELECT metric.aweme_count FROM creator_account_metric metric - JOIN social_account account ON account.id = metric.account_id - WHERE account.account_id = $2 AND metric.aweme_count IS NOT NULL - ORDER BY metric.collected_at DESC LIMIT 1)`, SourceOwned, accountID).Scan(&status.WorkCount, &total) + (SELECT COUNT(*) FROM creator_work WHERE source_type = $1 AND source_id = $2 AND is_public = true), + (SELECT public_work_count FROM creator_collection_checkpoint + WHERE source_type = $1 AND source_id = $2 AND collection_kind = 'works')`, SourceOwned, accountID).Scan(&status.WorkCount, &total) if err != nil { return AccountCollectionStatus{}, databaseError(err) } if total.Valid { - status.AwemeCount = &total.Int64 + status.PublicWorkCount = &total.Int64 } for _, kind := range []struct { name string @@ -451,6 +499,9 @@ func (s *Store) RecordVerifiedLoginResult(ctx context.Context, accountID, actual if _, err := tx.ExecContext(ctx, `UPDATE creator_account_profile SET login_status = 'logged_in', login_reason = '', login_checked_at = $2, updated_at = $2 WHERE account_id = $1`, id, now); err != nil { return LoginResult{}, databaseError(err) } + if err := releaseLoginBlockedCollection(ctx, tx, accountID); err != nil { + return LoginResult{}, err + } if err := tx.Commit(); err != nil { return LoginResult{}, fmt.Errorf("commit account UID verification: %w", err) } diff --git a/internal/creator/blocked_collection_retry_test.go b/internal/creator/blocked_collection_retry_test.go new file mode 100644 index 0000000..b68472d --- /dev/null +++ b/internal/creator/blocked_collection_retry_test.go @@ -0,0 +1,167 @@ +package creator + +import ( + "context" + "errors" + "reflect" + "sort" + "strings" + "testing" + "time" +) + +func TestBlockedOwnedCollectionRemainsDueWithoutClearingErrors(t *testing.T) { + store, accountStore, ctx := openCreatorIntegrationStore(t) + now := time.Now().UTC().Truncate(time.Second) + var want []string + for _, tc := range []struct { + name, works, comments, login, business, runtime string + due bool + }{ + {"both_blocked", "blocked", "blocked", "logged_in", "normal", "active", true}, + {"works_blocked", "blocked", "succeeded", "logged_in", "normal", "active", true}, + {"comments_blocked", "succeeded", "blocked", "logged_in", "normal", "active", true}, + {"muted_blocked", "blocked", "blocked", "logged_in", "muted", "active", true}, + {"recently_completed", "succeeded", "succeeded", "logged_in", "normal", "active", false}, + {"requires_login", "blocked", "blocked", "needs_login", "normal", "active", false}, + {"banned", "blocked", "blocked", "logged_in", "banned", "active", false}, + {"paused", "blocked", "blocked", "logged_in", "normal", "paused", false}, + } { + accountID := createIntegrationAccount(t, ctx, accountStore, strings.ReplaceAll(tc.name, "_", "-")) + if err := store.EnsureAccountProfile(ctx, accountID); err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE creator_account_profile SET login_status=$2 WHERE account_id=(SELECT id FROM social_account WHERE account_id=$1)`, accountID, tc.login); err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE creator_account_profile SET business_status=$2 WHERE account_id=(SELECT id FROM social_account WHERE account_id=$1)`, accountID, tc.business); err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status=$2 WHERE account_id=$1`, accountID, tc.runtime); err != nil { + t.Fatal(err) + } + for kind, status := range map[string]string{"works": tc.works, "comments": tc.comments} { + reason := "" + if status == "blocked" { + reason = "identity verification unavailable: multiple browser page targets" + } + if _, err := store.db.ExecContext(ctx, `INSERT INTO creator_collection_checkpoint + (source_type,source_id,collection_kind,window_start,window_end,status,last_error,last_completed_at) + VALUES ('owned',$1,$2,$3,$4,$5,$6,$4)`, accountID, kind, now.Add(-24*time.Hour), now, status, reason); err != nil { + t.Fatal(err) + } + } + if tc.due { + want = append(want, accountID) + } + } + got, err := store.ListDueOwnedAccounts(ctx, now, 3600) + if err != nil { + t.Fatal(err) + } + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Fatalf("blocked tasks must be retried only for eligible accounts: got=%v want=%v", got, want) + } + for _, accountID := range want { + status, err := store.GetAccountCollectionStatus(ctx, accountID) + if err != nil { + t.Fatal(err) + } + for _, checkpoint := range []AccountCheckpointStatus{status.Works, status.Comments} { + if checkpoint.Status == "blocked" && checkpoint.LastError == "" { + t.Fatalf("scheduling must not hide the actual error: %+v", checkpoint) + } + } + } +} + +func TestBlockedCheckpointRetryPreservesWindowCursorAndProgress(t *testing.T) { + store, _, ctx := openCreatorIntegrationStore(t) + end := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + start := end.Add(-7 * 24 * time.Hour) + completed := start.Add(-time.Hour) + for _, kind := range []string{"works", "comments"} { + t.Run(kind, func(t *testing.T) { + id := "blocked-progress-" + kind + cursor := "resume-page-2" + if kind == "comments" { + cursor = `{"work_key":"work-1","cursor":"resume-page-2"}` + } + if _, err := store.db.ExecContext(ctx, `INSERT INTO creator_collection_checkpoint + (source_type,source_id,collection_kind,window_start,window_end,status,cursor, + last_error,last_completed_at,public_work_count,public_work_count_in_progress) + VALUES ('owned',$1,$2,$3,$4,'blocked',$5,'runtime unavailable',$6,77,13)`, + id, kind, start, end, cursor, completed); err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < 2; attempt++ { + lease, err := store.beginCheckpoint(ctx, SourceOwned, id, kind, end, end.Add(time.Hour)) + if err != nil || lease == "" { + t.Fatalf("start retry: lease=%q err=%v", lease, err) + } + checkpoint, err := store.checkpoint(ctx, SourceOwned, id, kind) + if err != nil { + t.Fatal(err) + } + var publicCount int64 + var gotCompleted time.Time + if err := store.db.QueryRowContext(ctx, `SELECT public_work_count,last_completed_at FROM creator_collection_checkpoint WHERE source_type='owned' AND source_id=$1 AND collection_kind=$2`, id, kind).Scan(&publicCount, &gotCompleted); err != nil { + t.Fatal(err) + } + if checkpoint.Status != "running" || checkpoint.Cursor != cursor || + !checkpoint.WindowStart.Equal(start) || !checkpoint.WindowEnd.Equal(end) || + publicCount != 77 || checkpoint.PublicWorkCountInProgress != 13 || !gotCompleted.Equal(completed) { + t.Fatalf("blocked retry discarded prior progress: %+v", checkpoint) + } + if err := store.failCheckpoint(ctx, SourceOwned, id, kind, lease, ErrUnavailable); !errors.Is(err, ErrUnavailable) { + t.Fatalf("failed retry must retain its real error: %v", err) + } + } + }) + } +} + +type blockedRetryCollector struct{ cursor string } + +func (c *blockedRetryCollector) VerifyIdentity(context.Context, string) error { return nil } +func (c *blockedRetryCollector) ListWorks(_ context.Context, _, cursor string) (WorkPage, error) { + c.cursor = cursor + return WorkPage{}, nil +} +func (*blockedRetryCollector) ListTopLevelComments(context.Context, string, string) (CommentPage, error) { + return CommentPage{}, nil +} + +func TestBlockedCollectionBecomesSuccessfulOnlyAfterCollection(t *testing.T) { + store, accountStore, ctx := openCreatorIntegrationStore(t) + id := createIntegrationAccount(t, ctx, accountStore, "blocked-retry") + if err := store.EnsureAccountProfile(ctx, id); err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, id, "sec_uid_"+id); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + start, end := now.Add(-24*time.Hour), now.Add(-time.Hour) + if _, err := store.db.ExecContext(ctx, `INSERT INTO creator_collection_checkpoint + (source_type,source_id,collection_kind,window_start,window_end,status,cursor,last_error,public_work_count_in_progress) + VALUES ('owned',$1,'works',$2,$3,'blocked','resume-page-2','browser target unavailable',13)`, id, start, end); err != nil { + t.Fatal(err) + } + collector := &blockedRetryCollector{} + report, err := store.CollectSource(ctx, PlatformDouyin, SourceOwned, id, collector, now) + if err != nil || !report.PaginationComplete || report.PublicWorksSeen != 13 || collector.cursor != "resume-page-2" { + t.Fatalf("collection must resume rather than restart: report=%+v cursor=%q err=%v", report, collector.cursor, err) + } + checkpoint, err := store.checkpoint(ctx, SourceOwned, id, "works") + if err != nil { + t.Fatal(err) + } + status, err := store.GetAccountCollectionStatus(ctx, id) + if err != nil || status.Works.Status != "succeeded" || status.Works.LastError != "" || + status.Works.LastCompletedAt == nil || status.PublicWorkCount == nil || *status.PublicWorkCount != 13 || + !checkpoint.WindowStart.Equal(start) || !checkpoint.WindowEnd.Equal(end) { + t.Fatalf("successful collection must update status and preserve its window: checkpoint=%+v status=%+v err=%v", checkpoint, status, err) + } +} diff --git a/internal/creator/collection.go b/internal/creator/collection.go index 8bcd72c..98836f8 100644 --- a/internal/creator/collection.go +++ b/internal/creator/collection.go @@ -8,12 +8,35 @@ import ( "fmt" "strings" "time" + + "github.com/sirupsen/logrus" ) +const CollectionLoginRequiredReason = "browser is awaiting user login" + +func releaseLoginBlockedCollection(ctx context.Context, tx *sql.Tx, accountID string) error { + result, err := tx.ExecContext(ctx, `UPDATE creator_collection_checkpoint + SET status='idle',last_error='' + WHERE source_type=$1 AND source_id=$2 AND status='blocked' AND last_error=$3`, + SourceOwned, accountID, CollectionLoginRequiredReason) + if err != nil { + return databaseError(err) + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count > 0 { + logrus.WithFields(logrus.Fields{"account_id": accountID, "checkpoint_count": count}).Info("collection login blocks reset inside identity verification transaction") + } + return nil +} + type WorkPage struct { - Items []WorkInput - NextCursor string - HasMore bool + ExcludedWorkKeys []string + Items []WorkInput + NextCursor string + HasMore bool } type CommentPage struct { @@ -34,6 +57,8 @@ type WorkContextCollector interface { } type CollectionReport struct { + PublicWorksSeen int `json:"public_works_seen"` + NonPublicWorksSeen int `json:"non_public_works_seen"` SourceType string `json:"source_type"` SourceID string `json:"source_id"` WorksSeen int `json:"works_seen"` @@ -118,11 +143,12 @@ func publishedAtInCollectionWindow(publishedAt *time.Time, start, end time.Time) } type checkpointState struct { - Cursor string - LeaseToken string - WindowStart time.Time - WindowEnd time.Time - Status string + PublicWorkCountInProgress int64 + Cursor string + LeaseToken string + WindowStart time.Time + WindowEnd time.Time + Status string } func (s *Store) beginCheckpoint(ctx context.Context, sourceType, sourceID, kind string, start, end time.Time) (string, error) { @@ -136,9 +162,10 @@ func (s *Store) beginCheckpoint(ctx context.Context, sourceType, sourceID, kind (source_type, source_id, collection_kind, window_start, window_end, status, lease_token, lease_until, last_started_at, last_error) VALUES ($1, $2, $3, $4, $5, 'running', $6, now() + interval '10 minutes', now(), '') ON CONFLICT (source_type, source_id, collection_kind) DO UPDATE SET - window_start = CASE WHEN creator_collection_checkpoint.status IN ('failed', 'running') THEN creator_collection_checkpoint.window_start ELSE EXCLUDED.window_start END, - window_end = CASE WHEN creator_collection_checkpoint.status IN ('failed', 'running') THEN creator_collection_checkpoint.window_end ELSE EXCLUDED.window_end END, - cursor = CASE WHEN creator_collection_checkpoint.status IN ('failed', 'running') THEN creator_collection_checkpoint.cursor ELSE '' END, + window_start = CASE WHEN creator_collection_checkpoint.status IN ('idle', 'failed', 'running', 'blocked') THEN creator_collection_checkpoint.window_start ELSE EXCLUDED.window_start END, + window_end = CASE WHEN creator_collection_checkpoint.status IN ('idle', 'failed', 'running', 'blocked') THEN creator_collection_checkpoint.window_end ELSE EXCLUDED.window_end END, + cursor = CASE WHEN creator_collection_checkpoint.status IN ('idle', 'failed', 'running', 'blocked') THEN creator_collection_checkpoint.cursor ELSE '' END, + public_work_count_in_progress = CASE WHEN creator_collection_checkpoint.status IN ('idle', 'failed', 'running', 'blocked') THEN creator_collection_checkpoint.public_work_count_in_progress ELSE 0 END, status = 'running', lease_token = EXCLUDED.lease_token, lease_until = EXCLUDED.lease_until, last_started_at = EXCLUDED.last_started_at, last_error = '' WHERE creator_collection_checkpoint.status <> 'running' @@ -156,7 +183,7 @@ func (s *Store) beginCheckpoint(ctx context.Context, sourceType, sourceID, kind func (s *Store) checkpoint(ctx context.Context, sourceType, sourceID, kind string) (checkpointState, error) { var state checkpointState - err := s.db.QueryRowContext(ctx, `SELECT cursor,lease_token,window_start,window_end,status FROM creator_collection_checkpoint WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3`, sourceType, sourceID, kind).Scan(&state.Cursor, &state.LeaseToken, &state.WindowStart, &state.WindowEnd, &state.Status) + err := s.db.QueryRowContext(ctx, `SELECT cursor,lease_token,window_start,window_end,status,public_work_count_in_progress FROM creator_collection_checkpoint WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3`, sourceType, sourceID, kind).Scan(&state.Cursor, &state.LeaseToken, &state.WindowStart, &state.WindowEnd, &state.Status, &state.PublicWorkCountInProgress) if err != nil { return checkpointState{}, rowError(err) } @@ -181,7 +208,7 @@ func (s *Store) NextCollectionWindow(ctx context.Context, sourceType, sourceID s return time.Time{}, time.Time{}, databaseError(err) } end = end.UTC() - if status == "failed" || status == "blocked" || status == "running" { + if status == "idle" || status == "failed" || status == "blocked" || status == "running" { return end.Add(-time.Duration(lookbackDays) * 24 * time.Hour), end, nil } nextEnd := NextFixedRun(end, now.UTC(), interval) @@ -230,11 +257,11 @@ func (s *Store) renewCheckpoint(ctx context.Context, sourceType, sourceID, kind, return nil } -func (s *Store) saveCheckpointCursor(ctx context.Context, sourceType, sourceID, kind, leaseToken, cursor string) error { +func (s *Store) saveCheckpointCursor(ctx context.Context, sourceType, sourceID, kind, leaseToken, cursor string, publicWorkCountDelta *int64) error { if err := s.renewCheckpoint(ctx, sourceType, sourceID, kind, leaseToken); err != nil { return err } - result, err := s.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET cursor=$5 WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3 AND lease_token=$4 AND status='running' AND lease_until > now()`, sourceType, sourceID, kind, leaseToken, cursor) + result, err := s.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET cursor=$5, public_work_count_in_progress=CASE WHEN $6::bigint IS NULL THEN public_work_count_in_progress ELSE public_work_count_in_progress+$6 END WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3 AND lease_token=$4 AND status='running' AND lease_until > now()`, sourceType, sourceID, kind, leaseToken, cursor, publicWorkCountDelta) if err != nil { return databaseError(err) } @@ -246,11 +273,14 @@ func (s *Store) saveCheckpointCursor(ctx context.Context, sourceType, sourceID, return nil } -func (s *Store) finishCheckpoint(ctx context.Context, sourceType, sourceID, kind, leaseToken, status, reason string) error { +func (s *Store) finishCheckpoint(ctx context.Context, sourceType, sourceID, kind, leaseToken, status, reason string, publicWorkCountDelta *int64) error { if status != "succeeded" && status != "failed" && status != "blocked" || leaseToken == "" { return ErrInvalid } - result, err := s.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET status=$5, cursor=CASE WHEN $5='succeeded' THEN '' ELSE cursor END, lease_until=NULL, last_completed_at=CASE WHEN $5='succeeded' THEN now() ELSE last_completed_at END, last_error=$6 WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3 AND lease_token=$4 AND status='running' AND lease_until > now()`, sourceType, sourceID, kind, leaseToken, status, reason) + if publicWorkCountDelta != nil && (sourceType != SourceOwned || kind != "works" || status != "succeeded" || *publicWorkCountDelta < 0) { + return ErrInvalid + } + result, err := s.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET status=$5, cursor=CASE WHEN $5='succeeded' THEN '' ELSE cursor END, public_work_count=CASE WHEN $7::bigint IS NULL THEN public_work_count ELSE public_work_count_in_progress+$7 END, public_work_count_in_progress=CASE WHEN $7::bigint IS NULL THEN public_work_count_in_progress ELSE 0 END, lease_until=NULL, last_completed_at=CASE WHEN $5='succeeded' THEN now() ELSE last_completed_at END, last_error=$6 WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3 AND lease_token=$4 AND status='running' AND lease_until > now()`, sourceType, sourceID, kind, leaseToken, status, reason, publicWorkCountDelta) if err != nil { return databaseError(err) } @@ -267,7 +297,7 @@ func (s *Store) failCheckpoint(ctx context.Context, sourceType, sourceID, kind, if errors.Is(primary, ErrConflict) || errors.Is(primary, ErrUnavailable) { status = "blocked" } - if err := s.finishCheckpoint(ctx, sourceType, sourceID, kind, leaseToken, status, primary.Error()); err != nil { + if err := s.finishCheckpoint(ctx, sourceType, sourceID, kind, leaseToken, status, primary.Error(), nil); err != nil { return errors.Join(primary, err) } return primary @@ -327,6 +357,8 @@ func decodeCommentCheckpoint(value string) (commentCheckpointCursor, error) { return result, nil } +// ListDueOwnedAccounts rechecks blocked collection for runnable, logged-in accounts. +// Selecting a retry does not clear errors or mark collection successful. func (s *Store) ListDueOwnedAccounts(ctx context.Context, now time.Time, intervalSeconds int64) ([]string, error) { if now.IsZero() || intervalSeconds <= 0 || intervalSeconds > maxDurationSeconds { return nil, ErrInvalid @@ -341,11 +373,10 @@ func (s *Store) ListDueOwnedAccounts(ctx context.Context, now time.Time, interva LEFT JOIN creator_collection_checkpoint comments_checkpoint ON comments_checkpoint.source_type='owned' AND comments_checkpoint.source_id=account.account_id AND comments_checkpoint.collection_kind='comments' WHERE account.platform = 'douyin' + AND account.status='active' AND profile.login_status='logged_in' - AND COALESCE(works_checkpoint.status, '') <> 'blocked' - AND COALESCE(comments_checkpoint.status, '') <> 'blocked' AND (works_checkpoint.id IS NULL OR comments_checkpoint.id IS NULL - OR works_checkpoint.status='failed' OR comments_checkpoint.status='failed' + OR works_checkpoint.status IN ('failed','blocked') OR comments_checkpoint.status IN ('failed','blocked') OR works_checkpoint.last_completed_at IS NULL OR comments_checkpoint.last_completed_at IS NULL OR works_checkpoint.last_completed_at <= $1 OR comments_checkpoint.last_completed_at <= $1) @@ -387,17 +418,40 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI return report, err } report.WindowStart, report.WindowEnd = worksCheckpoint.WindowStart, worksCheckpoint.WindowEnd + resetOwnedVisibility := sourceType == SourceOwned && worksCheckpoint.Cursor == "" + report.PublicWorksSeen = int(worksCheckpoint.PublicWorkCountInProgress) seenWorks := map[string]string{} + var pageExcludedWorkKeys []string + var pagePublicWorkCountDelta int64 _, err = collectPagesFromCursor(ctx, worksCheckpoint.Cursor, func(ctx context.Context, cursor string) ([]WorkInput, string, bool, error) { page, err := collector.ListWorks(ctx, sourceID, cursor) if err != nil { return nil, "", false, err } + pageExcludedWorkKeys = page.ExcludedWorkKeys + pagePublicWorkCountDelta = 0 return page.Items, page.NextCursor, page.HasMore, nil }, func(pageItems []WorkInput, nextCursor string, hasMore bool) error { if err := s.renewCheckpoint(ctx, sourceType, sourceID, "works", worksLease); err != nil { return err } + if resetOwnedVisibility { + if err := s.ResetOwnedWorkVisibility(ctx, sourceID); err != nil { + return err + } + resetOwnedVisibility = false + } + report.WorksSeen += len(pageExcludedWorkKeys) + report.NonPublicWorksSeen += len(pageExcludedWorkKeys) + report.PublicWorksSeen += len(pageItems) + pagePublicWorkCountDelta = int64(len(pageItems)) + for _, workKey := range pageExcludedWorkKeys { + if sourceType == SourceOwned { + if err := s.MarkOwnedWorkNonPublic(ctx, sourceID, workKey); err != nil { + return err + } + } + } for _, work := range pageItems { if err := s.renewCheckpoint(ctx, sourceType, sourceID, "works", worksLease); err != nil { return err @@ -425,14 +479,22 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI report.WorksSaved++ } if hasMore { - return s.saveCheckpointCursor(ctx, sourceType, sourceID, "works", worksLease, nextCursor) + var publicWorkCountDelta *int64 + if sourceType == SourceOwned { + publicWorkCountDelta = &pagePublicWorkCountDelta + } + return s.saveCheckpointCursor(ctx, sourceType, sourceID, "works", worksLease, nextCursor, publicWorkCountDelta) } return nil }) if err != nil { return report, s.failCheckpoint(ctx, sourceType, sourceID, "works", worksLease, err) } - if err := s.finishCheckpoint(ctx, sourceType, sourceID, "works", worksLease, "succeeded", ""); err != nil { + var publicWorkCountDelta *int64 + if sourceType == SourceOwned { + publicWorkCountDelta = &pagePublicWorkCountDelta + } + if err := s.finishCheckpoint(ctx, sourceType, sourceID, "works", worksLease, "succeeded", "", publicWorkCountDelta); err != nil { return report, err } commentsLease, err := s.beginCheckpoint(ctx, sourceType, sourceID, "comments", report.WindowStart, report.WindowEnd) @@ -508,7 +570,7 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI if err != nil { return err } - return s.saveCheckpointCursor(ctx, sourceType, sourceID, "comments", commentsLease, checkpoint) + return s.saveCheckpointCursor(ctx, sourceType, sourceID, "comments", commentsLease, checkpoint, nil) } return nil }) @@ -520,12 +582,12 @@ func (s *Store) CollectSource(ctx context.Context, platform, sourceType, sourceI if err != nil { return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err) } - if err := s.saveCheckpointCursor(ctx, sourceType, sourceID, "comments", commentsLease, checkpoint); err != nil { + if err := s.saveCheckpointCursor(ctx, sourceType, sourceID, "comments", commentsLease, checkpoint, nil); err != nil { return report, s.failCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, err) } } } - if err := s.finishCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, "succeeded", ""); err != nil { + if err := s.finishCheckpoint(ctx, sourceType, sourceID, "comments", commentsLease, "succeeded", "", nil); err != nil { return report, err } report.PaginationComplete = true diff --git a/internal/creator/content.go b/internal/creator/content.go index dc31ad2..1626b14 100644 --- a/internal/creator/content.go +++ b/internal/creator/content.go @@ -416,8 +416,8 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) var inserted bool err = tx.QueryRowContext(ctx, ` INSERT INTO creator_work (work_id, platform, work_key, source_type, source_id, author_name, title, body, - published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares, collect_count, play_count) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + published_at, published_at_status, original_url, cover_url, raw_payload, likes, comments_count, shares, collect_count, play_count, is_public) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, TRUE) ON CONFLICT (platform, work_key) DO UPDATE SET author_name = CASE WHEN EXCLUDED.author_name = '' THEN creator_work.author_name ELSE EXCLUDED.author_name END, title = CASE WHEN EXCLUDED.title = '' THEN creator_work.title ELSE EXCLUDED.title END, @@ -432,6 +432,7 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) shares = COALESCE(EXCLUDED.shares, creator_work.shares), collect_count = COALESCE(EXCLUDED.collect_count, creator_work.collect_count), play_count = COALESCE(EXCLUDED.play_count, creator_work.play_count), + is_public = TRUE, updated_at = now() RETURNING work_id, (xmax = 0)`, id, input.Platform, input.WorkKey, input.SourceType, input.SourceID, input.AuthorName, input.Title, input.Body, input.PublishedAt, status, input.OriginalURL, input.CoverURL, @@ -450,6 +451,30 @@ func (s *Store) UpsertWork(ctx context.Context, input WorkInput, now time.Time) return result, inserted, nil } +func (s *Store) ResetOwnedWorkVisibility(ctx context.Context, sourceID string) error { + sourceID = strings.TrimSpace(sourceID) + if sourceID == "" { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, `UPDATE creator_work SET is_public = false WHERE source_type = $1 AND source_id = $2 AND is_public = true`, SourceOwned, sourceID) + if err != nil { + return databaseError(err) + } + return nil +} + +func (s *Store) MarkOwnedWorkNonPublic(ctx context.Context, sourceID, workKey string) error { + sourceID, workKey = strings.TrimSpace(sourceID), strings.TrimSpace(workKey) + if sourceID == "" || workKey == "" { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, `UPDATE creator_work SET is_public = false WHERE source_type = $1 AND source_id = $2 AND work_key = $3`, SourceOwned, sourceID, workKey) + if err != nil { + return databaseError(err) + } + return nil +} + func ptrTime(value time.Time) *time.Time { return &value } func nullableRawPayload(value string) any { @@ -489,7 +514,7 @@ func loadWorkSources(work *Work) { } func (s *Store) GetWork(ctx context.Context, id string) (Work, error) { - result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE work_id = $1`, id)) + result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE work_id = $1 AND is_public = true`, id)) if err != nil { return Work{}, rowError(err) } @@ -502,7 +527,7 @@ func (s *Store) GetWorkByKey(ctx context.Context, platform, workKey string) (Wor if !ValidatePlatform(platform) || workKey == "" { return Work{}, ErrInvalid } - result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE platform = $1 AND work_key = $2`, platform, workKey)) + result, err := scanWork(s.db.QueryRowContext(ctx, workSelect+` WHERE platform = $1 AND work_key = $2 AND is_public = true`, platform, workKey)) if err != nil { return Work{}, rowError(err) } @@ -535,7 +560,7 @@ func buildWorkQuery(filter WorkFilter) (string, []any, error) { return "", nil, ErrInvalid } query, args := workSelect, make([]any, 0, 11) - where := make([]string, 0, 11) + where := []string{"creator_work.is_public = true"} add := func(clause string, value any) { args = append(args, value) where = append(where, fmt.Sprintf(clause, len(args))) diff --git a/internal/creator/covers.go b/internal/creator/covers.go index b218ed7..93f9b09 100644 --- a/internal/creator/covers.go +++ b/internal/creator/covers.go @@ -50,6 +50,10 @@ func (s *Store) SaveWorkCover(ctx context.Context, workID, contentType string, d if err != nil { return err } + return saveCoverFile(base, extension, data) +} + +func saveCoverFile(base, extension string, data []byte) error { if err := os.MkdirAll(filepath.Dir(base), 0755); err != nil { return fmt.Errorf("create cover directory: %w", err) } @@ -91,6 +95,10 @@ func (s *Store) GetWorkCover(ctx context.Context, workID string) (string, error) if err != nil { return "", err } + return getCoverFile(base) +} + +func getCoverFile(base string) (string, error) { var result string for _, extension := range coverExtensions { path := base + extension @@ -105,7 +113,7 @@ func (s *Store) GetWorkCover(ctx context.Context, workID string) (string, error) return "", fmt.Errorf("%w: cover file is not a nonempty regular file", ErrInvalid) } if result != "" { - return "", fmt.Errorf("%w: multiple cover formats exist for work %s", ErrConflict, workID) + return "", fmt.Errorf("%w: multiple cover formats exist for %s", ErrConflict, base) } result = path } @@ -115,6 +123,52 @@ func (s *Store) GetWorkCover(ctx context.Context, workID string) (string, error) return result, nil } +func (s *Store) platformWorkCoverBase(uid, workKey string) (string, error) { + if !coverComponent.MatchString(uid) || !coverComponent.MatchString(workKey) { + return "", ErrInvalid + } + return filepath.Join(s.coverDirectory, uid, workKey), nil +} + +// SavePlatformWorkCover also supports notification works absent from the library. +// It creates only the cover file, never an account or work database record. +func (s *Store) SavePlatformWorkCover(uid, workKey, contentType string, data []byte) error { + mediaType, _, err := mime.ParseMediaType(contentType) + extension, supported := coverExtensions[mediaType] + if err != nil || !supported || len(data) == 0 || len(data) > maxWorkCoverBytes { + return ErrInvalid + } + base, err := s.platformWorkCoverBase(uid, workKey) + if err != nil { + return err + } + return saveCoverFile(base, extension, data) +} + +func (s *Store) GetPlatformWorkCover(uid, workKey string) (string, error) { + base, err := s.platformWorkCoverBase(uid, workKey) + if err != nil { + return "", err + } + return getCoverFile(base) +} + +func (s *Store) GetEventWorkCover(ctx context.Context, eventID string) (string, error) { + if !decimalNoticeID(eventID) { + return "", ErrInvalid + } + var uid, workKey string + err := s.db.QueryRowContext(ctx, `SELECT e.work_author_uid,COALESCE(e.work_id,'') FROM creator_account_event e + JOIN creator_account_listener l ON l.account_id=e.account_id AND l.enabled WHERE e.id::text=$1`, eventID).Scan(&uid, &workKey) + if err != nil { + return "", rowError(err) + } + if uid == "" || workKey == "" { + return "", ErrNotFound + } + return s.GetPlatformWorkCover(uid, workKey) +} + // ListWorksMissingCover backfills both owned and monitored sources, at most 40 per sync. func (s *Store) ListWorksMissingCover(ctx context.Context, sourceType, sourceID string) ([]Work, error) { if (sourceType != SourceOwned && sourceType != SourceCompetitor) || sourceID == "" { diff --git a/internal/creator/environment_login.go b/internal/creator/environment_login.go index 853f259..7e18c44 100644 --- a/internal/creator/environment_login.go +++ b/internal/creator/environment_login.go @@ -65,6 +65,9 @@ func (s *Store) RecordVerifiedEnvironmentLogin(ctx context.Context, alias string if _, err := tx.ExecContext(ctx, `UPDATE creator_account_profile SET login_status='logged_in',login_reason='',login_checked_at=$2,updated_at=$2,sec_uid=$3,avatar_url=$4,douyin_number=$5 WHERE account_id=$1`, rowID.Int64, now, identity.SecUID, identity.AvatarURL, identity.DouyinNumber); err != nil { return LoginResult{}, databaseError(err) } + if err := releaseLoginBlockedCollection(ctx, tx, accountID); err != nil { + return LoginResult{}, err + } if err := tx.Commit(); err != nil { return LoginResult{}, identityWriteError(err, identity.UID) } diff --git a/internal/creator/event_details_test.go b/internal/creator/event_details_test.go new file mode 100644 index 0000000..a76274c --- /dev/null +++ b/internal/creator/event_details_test.go @@ -0,0 +1,141 @@ +package creator + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestEventDetailsValidateWithoutDroppingFailures(t *testing.T) { + d := ListenerDelivery{DeliveryID: "1", Kind: "notice", Notice: &ListenerNotice{EventKey: "2", EventType: "like", WorkAuthorUID: "123", WorkCoverError: "download failed"}} + if err := d.Validate(); err != nil { + t.Fatal(err) + } + d.Notice.WorkAuthorUID = "../123" + if !errors.Is(d.Validate(), ErrInvalid) { + t.Fatal("bad author identity accepted") + } + d.Notice.WorkAuthorUID = "123" + d.Notice.InteractorName = strings.Repeat("x", 32769) + if !errors.Is(d.Validate(), ErrInvalid) { + t.Fatal("oversized profile accepted") + } +} + +func TestEventCoverUsesPlatformIdentityWithoutPlaceholderWork(t *testing.T) { + s := &Store{coverDirectory: t.TempDir()} + if _, err := s.GetPlatformWorkCover("123", "456"); !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + if err := s.SavePlatformWorkCover("123", "456", "image/webp", []byte("image")); err != nil { + t.Fatal(err) + } + path, err := s.GetPlatformWorkCover("123", "456") + if err != nil || path != filepath.Join(s.coverDirectory, "123", "456.webp") { + t.Fatalf("%s %v", path, err) + } + if err := s.SavePlatformWorkCover("../bad", "456", "image/png", []byte("image")); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + if err := s.SavePlatformWorkCover("123", "456", "text/html", []byte("bad")); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + t.Fatal(err) + } + if _, err := s.GetPlatformWorkCover("123", "456"); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } +} + +func TestEventDetailsBackfillPreservesMessageAndTimes(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + store.coverDirectory = t.TempDir() + id := createIntegrationAccount(t, ctx, accounts, fmt.Sprint(time.Now().UnixNano())) + profile, err := store.GetAccountProfile(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err = store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + state, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Microsecond) + d := ListenerDelivery{DeliveryID: "original", Kind: "notice", Baseline: true, Notice: &ListenerNotice{EventKey: "101", EventType: "comment", InteractorUID: "456", WorkID: "987", PlatformEventAt: &now, MessageText: "原评论"}} + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{d}); err != nil { + t.Fatal(err) + } + d.DeliveryID = "details" + d.Notice.InteractorName = "用户昵称" + d.Notice.InteractorSecUID = "MS4wLjABAAAAprofile" + d.Notice.WorkAuthorUID = "123" + d.Notice.WorkCoverURL = "https://p3.douyinpic.com/cover.jpg" + d.Notice.WorkURL = "https://www.douyin.com/video/987" + d.Notice.MessageText = "不应覆盖的重读内容" + later := now.Add(time.Hour) + d.Notice.PlatformEventAt = &later + if err = store.SavePlatformWorkCover("123", "987", "image/jpeg", []byte("image")); err != nil { + t.Fatal(err) + } + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{d}); err != nil { + t.Fatal(err) + } + page, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{AccountID: id}, 1, 25) + if err != nil || page.Total != 1 { + t.Fatalf("%+v %v", page, err) + } + event := page.Data[0] + if event.InteractorName != "用户昵称" || event.InteractorSecUID != "MS4wLjABAAAAprofile" || !event.HasWorkCover { + t.Fatalf("details not filled: %+v", event) + } + if event.MessageText != "原评论" || !event.PlatformEventAt.Equal(now) || !event.Baseline { + t.Fatalf("original changed: %+v", event) + } + if _, err = store.GetEventWorkCover(ctx, event.ID); err != nil { + t.Fatal(err) + } + d.Notice.InteractorName = "" + d.Notice.InteractorSecUID = "" + d.Notice.WorkCoverURL = "" + d.Notice.WorkURL = "" + d.Notice.WorkAuthorUID = "" + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{d}); err != nil { + t.Fatal(err) + } + page, err = store.ListAggregatedEventsPage(ctx, EventAggregationFilter{AccountID: id}, 1, 25) + if err != nil || page.Data[0].InteractorName != "用户昵称" || page.Data[0].WorkURL == "" { + t.Fatalf("partial response erased details: %+v %v", page, err) + } + d.Notice.InteractorUID = "789" + d.Notice.InteractorName = "不同用户" + d.Notice.InteractorSecUID = "other" + d.Notice.WorkID = "111" + d.Notice.WorkAuthorUID = "222" + d.Notice.WorkURL = "https://www.douyin.com/video/111" + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{d}); err != nil { + t.Fatal(err) + } + page, err = store.ListAggregatedEventsPage(ctx, EventAggregationFilter{AccountID: id}, 1, 25) + if err != nil || page.Data[0].InteractorName != "用户昵称" || page.Data[0].InteractorSecUID != "MS4wLjABAAAAprofile" || page.Data[0].WorkAuthorUID != "123" || page.Data[0].WorkURL != "https://www.douyin.com/video/987" { + t.Fatalf("merged notification misattributed details: %+v %v", page, err) + } + if _, err = store.GetEventWorkCover(ctx, "bad"); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + if _, err = store.GetEventWorkCover(ctx, "999999999"); !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + if _, err = store.SetListenerEnabled(ctx, id, false); err != nil { + t.Fatal(err) + } + if _, err = store.GetEventWorkCover(ctx, event.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("disabled account cover remained exposed: %v", err) + } +} diff --git a/internal/creator/event_listener.go b/internal/creator/event_listener.go new file mode 100644 index 0000000..959cf59 --- /dev/null +++ b/internal/creator/event_listener.go @@ -0,0 +1,393 @@ +package creator + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// ListenerDelivery is the gateway's acknowledged notification envelope. +// Control envelopes update listener health; only notices become account events. +type ListenerNotice struct { + EventKey string `json:"event_key"` + EventType string `json:"event_type"` + InteractorUID string `json:"interactor_uid"` + InteractorName string `json:"interactor_name"` + InteractorSecUID string `json:"interactor_sec_uid"` + WorkAuthorUID string `json:"work_author_uid"` + WorkCoverURL string `json:"work_cover_url"` + WorkURL string `json:"work_url"` + WorkCoverError string `json:"work_cover_error"` + CommentID string `json:"comment_id"` + WorkID string `json:"work_id"` + MessageType string `json:"message_type"` + MessageText string `json:"message_text"` + PlatformEventAt *time.Time `json:"platform_event_at"` + GatewayReceivedAt *time.Time `json:"gateway_received_at"` +} + +type ListenerDelivery struct { + DeliveryID string `json:"delivery_id"` + Kind string `json:"kind"` + BoundaryAt *time.Time `json:"boundary_at"` + Reason string `json:"reason"` + Baseline bool `json:"baseline"` + Notice *ListenerNotice `json:"notice"` + Checkpoints map[string]string `json:"checkpoints,omitempty"` + CheckedAt *time.Time `json:"checked_at,omitempty"` +} + +func validEventType(value string) bool { + switch value { + case "like", "comment", "follow", "repost": + return true + } + return false +} +func (d ListenerDelivery) Validate() error { + if strings.TrimSpace(d.DeliveryID) == "" { + return ErrInvalid + } + switch d.Kind { + case "baseline": + if d.BoundaryAt == nil || d.BoundaryAt.IsZero() { + return ErrInvalid + } + case "checkpoint": + if d.BoundaryAt == nil || d.BoundaryAt.IsZero() || d.CheckedAt == nil || d.CheckedAt.IsZero() || d.Checkpoints == nil { + return ErrInvalid + } + for group, id := range d.Checkpoints { + if (group != "700" && group != "960" && group != "961") || !decimalNoticeID(id) { + return ErrInvalid + } + } + case "open", "close", "reconnected", "error": + case "notice": + if d.Notice == nil || strings.TrimSpace(d.Notice.EventKey) == "" || !validEventType(d.Notice.EventType) || len(d.Notice.MessageText) > 400000 { + return ErrInvalid + } + default: + return ErrInvalid + } + if d.Notice != nil { + for _, value := range []string{d.Notice.InteractorName, d.Notice.InteractorSecUID, d.Notice.WorkCoverURL, d.Notice.WorkURL, d.Notice.WorkCoverError} { + if len(value) > 32768 { + return ErrInvalid + } + } + if d.Notice.WorkAuthorUID != "" && !decimalNoticeID(d.Notice.WorkAuthorUID) { + return ErrInvalid + } + } + return nil +} + +func decimalNoticeID(value string) bool { + if len(value) == 0 || len(value) > 30 || value[0] == '0' { + return false + } + for _, c := range value { + if c < '0' || c > '9' { + return false + } + } + return true +} + +const listenerQuery = `SELECT a.account_id, COALESCE(a.platform_account_key, ''), 'douyin', l.enabled, l.generation, l.status, l.boundary_at, l.last_delivery_id, l.reason, l.updated_at, l.notice_checkpoints, l.last_poll_at + FROM creator_account_listener l JOIN social_account a ON a.id = l.account_id` + +func scanListener(scanner interface{ Scan(...any) error }) (ListenerState, error) { + var l ListenerState + var boundary, lastPoll sql.NullTime + var checkpoints []byte + err := scanner.Scan(&l.AccountID, &l.AccountUID, &l.Platform, &l.Enabled, &l.Generation, &l.Status, &boundary, &l.LastDeliveryID, &l.Reason, &l.UpdatedAt, &checkpoints, &lastPoll) + if err != nil { + return l, rowError(err) + } + if err := json.Unmarshal(checkpoints, &l.Checkpoints); err != nil { + return l, fmt.Errorf("invalid notification checkpoints: %w", err) + } + l.BoundaryAt, l.LastPollAt = nullableTime(boundary), nullableTime(lastPoll) + return l, nil +} + +func (s *Store) SetListenerEnabled(ctx context.Context, accountID string, enabled bool) (ListenerState, error) { + if err := s.EnsureAccountProfile(ctx, accountID); err != nil { + return ListenerState{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ListenerState{}, err + } + defer tx.Rollback() + var id int64 + var login, platform, uid string + err = tx.QueryRowContext(ctx, `SELECT a.id, p.login_status, a.platform, COALESCE(a.platform_account_key, '') + FROM social_account a JOIN creator_account_profile p ON p.account_id = a.id WHERE a.account_id = $1 FOR UPDATE OF a, p`, accountID).Scan(&id, &login, &platform, &uid) + if err != nil { + return ListenerState{}, rowError(err) + } + if enabled && (platform != PlatformDouyin || login != "logged_in" || strings.TrimSpace(uid) == "") { + return ListenerState{}, fmt.Errorf("%w: 请先登录并同步账号", ErrConflict) + } + status := "stopping" + if enabled { + status = "starting" + } + _, err = tx.ExecContext(ctx, `INSERT INTO creator_account_listener(account_id, enabled, generation, status) + VALUES($1, $2, $3, $4) + ON CONFLICT(account_id) DO UPDATE SET + enabled = EXCLUDED.enabled, + generation = CASE WHEN creator_account_listener.enabled <> EXCLUDED.enabled THEN EXCLUDED.generation ELSE creator_account_listener.generation END, + status = CASE WHEN creator_account_listener.enabled <> EXCLUDED.enabled THEN EXCLUDED.status ELSE creator_account_listener.status END, + reason = CASE WHEN creator_account_listener.enabled <> EXCLUDED.enabled THEN '' ELSE creator_account_listener.reason END, + boundary_at = CASE WHEN creator_account_listener.enabled <> EXCLUDED.enabled THEN NULL ELSE creator_account_listener.boundary_at END, + notice_checkpoints = CASE WHEN creator_account_listener.enabled <> EXCLUDED.enabled THEN '{}'::jsonb ELSE creator_account_listener.notice_checkpoints END, + last_poll_at = CASE WHEN creator_account_listener.enabled <> EXCLUDED.enabled THEN NULL ELSE creator_account_listener.last_poll_at END, + updated_at = now()`, id, enabled, newID("listener"), status) + if err != nil { + return ListenerState{}, databaseError(err) + } + state, err := scanListener(tx.QueryRowContext(ctx, listenerQuery+` WHERE a.account_id = $1`, accountID)) + if err != nil { + return ListenerState{}, err + } + if err := tx.Commit(); err != nil { + return ListenerState{}, err + } + return state, nil +} + +func (s *Store) ListListenerStates(ctx context.Context) ([]ListenerState, error) { + rows, err := s.db.QueryContext(ctx, listenerQuery+` ORDER BY a.id`) + if err != nil { + return nil, err + } + defer rows.Close() + states := make([]ListenerState, 0) + for rows.Next() { + l, err := scanListener(rows) + if err != nil { + return nil, err + } + states = append(states, l) + } + return states, rows.Err() +} + +func (s *Store) UpdateListenerStatus(ctx context.Context, accountID, generation, status, reason string) error { + switch status { + case "starting", "ready", "gap", "stopping", "stopped", "error": + default: + return ErrInvalid + } + result, err := s.db.ExecContext(ctx, `UPDATE creator_account_listener SET status = $3, reason = $4, updated_at = now() + WHERE account_id = (SELECT id FROM social_account WHERE account_id = $1) AND generation = $2`, accountID, generation, status, reason) + if err != nil { + return databaseError(err) + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return ErrConflict + } + return nil +} + +// The listener row lock serializes reception with turning the account off. +// Acknowledgements are sent only after this transaction commits. +func (s *Store) RecordListenerDeliveries(ctx context.Context, accountID, generation string, deliveries []ListenerDelivery) error { + for _, d := range deliveries { + if err := d.Validate(); err != nil { + return fmt.Errorf("%w: invalid %s delivery", err, d.Kind) + } + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var id int64 + var enabled bool + var current string + err = tx.QueryRowContext(ctx, `SELECT l.account_id, l.enabled, l.generation FROM creator_account_listener l + JOIN social_account a ON a.id = l.account_id WHERE a.account_id = $1 FOR UPDATE OF l`, accountID).Scan(&id, &enabled, ¤t) + if err != nil { + return rowError(err) + } + if !enabled || current != generation { + return ErrConflict + } + for _, d := range deliveries { + switch d.Kind { + case "notice": + e := d.Notice + _, err = tx.ExecContext(ctx, `INSERT INTO creator_account_event(account_id, event_key, event_type, interactor_uid, + comment_id, work_id, message_type, message_text, platform_event_at, gateway_received_at, generation, baseline, + interactor_name, interactor_sec_uid, work_author_uid, work_cover_url, work_url, work_cover_error) + VALUES($1, $2, $3, $4, $5, $6, COALESCE(NULLIF($7, ''), 'text'), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + ON CONFLICT(account_id, event_type, event_key) DO UPDATE SET + interactor_name=CASE WHEN EXCLUDED.interactor_uid=creator_account_event.interactor_uid + THEN COALESCE(NULLIF(EXCLUDED.interactor_name,''),creator_account_event.interactor_name) ELSE creator_account_event.interactor_name END, + interactor_sec_uid=CASE WHEN EXCLUDED.interactor_uid=creator_account_event.interactor_uid + THEN COALESCE(NULLIF(EXCLUDED.interactor_sec_uid,''),creator_account_event.interactor_sec_uid) ELSE creator_account_event.interactor_sec_uid END, + work_author_uid=CASE WHEN EXCLUDED.work_id=creator_account_event.work_id + THEN COALESCE(NULLIF(EXCLUDED.work_author_uid,''),creator_account_event.work_author_uid) ELSE creator_account_event.work_author_uid END, + work_cover_url=CASE WHEN EXCLUDED.work_id=creator_account_event.work_id + THEN COALESCE(NULLIF(EXCLUDED.work_cover_url,''),creator_account_event.work_cover_url) ELSE creator_account_event.work_cover_url END, + work_url=CASE WHEN EXCLUDED.work_id=creator_account_event.work_id + THEN COALESCE(NULLIF(EXCLUDED.work_url,''),creator_account_event.work_url) ELSE creator_account_event.work_url END, + work_cover_error=CASE WHEN EXCLUDED.work_id=creator_account_event.work_id AND EXCLUDED.work_cover_url<>'' + THEN EXCLUDED.work_cover_error ELSE creator_account_event.work_cover_error END`, + id, e.EventKey, e.EventType, e.InteractorUID, e.CommentID, e.WorkID, e.MessageType, e.MessageText, e.PlatformEventAt, e.GatewayReceivedAt, generation, d.Baseline, + e.InteractorName, e.InteractorSecUID, e.WorkAuthorUID, e.WorkCoverURL, e.WorkURL, e.WorkCoverError) + case "checkpoint": + checkpoint, marshalErr := json.Marshal(d.Checkpoints) + if marshalErr != nil { + return marshalErr + } + status := "ready" + if d.Reason != "" { + status = "gap" + } + result, updateErr := tx.ExecContext(ctx, `UPDATE creator_account_listener SET notice_checkpoints = $2::jsonb, + boundary_at = COALESCE(boundary_at, $3), last_poll_at = $4, status = $5, reason = $6 + WHERE account_id = $1 AND (boundary_at IS NULL OR boundary_at = $3)`, id, string(checkpoint), d.BoundaryAt, d.CheckedAt, status, d.Reason) + if updateErr != nil { + return databaseError(updateErr) + } + count, countErr := result.RowsAffected() + if countErr != nil { + return countErr + } + if count != 1 { + return ErrConflict + } + case "baseline": + result, updateErr := tx.ExecContext(ctx, `UPDATE creator_account_listener SET boundary_at = COALESCE(boundary_at, $2), status = 'ready', reason = '' WHERE account_id = $1 AND (boundary_at IS NULL OR boundary_at = $2)`, id, d.BoundaryAt) + if updateErr != nil { + return databaseError(updateErr) + } + count, countErr := result.RowsAffected() + if countErr != nil { + return countErr + } + if count != 1 { + return ErrConflict + } + case "open": + // Only a successfully established boundary proves continuity. An open + // transport alone must not erase a preceding gap. + case "close", "reconnected", "error": + reason := d.Reason + if reason == "" { + reason = "抖音通知连接中断，正在恢复" + } + _, err = tx.ExecContext(ctx, `UPDATE creator_account_listener SET status = 'gap', reason = $2 WHERE account_id = $1`, id, reason) + } + if err != nil { + return databaseError(err) + } + if _, err = tx.ExecContext(ctx, `UPDATE creator_account_listener SET last_delivery_id = $2, updated_at = now() WHERE account_id = $1`, id, d.DeliveryID); err != nil { + return err + } + } + return tx.Commit() +} + +type EventAggregationFilter struct { + AccountID string + EventType string + Hours int +} + +func (f EventAggregationFilter) Validate() error { + if f.EventType != "" && !validEventType(f.EventType) { + return ErrInvalid + } + switch f.Hours { + case 0, 1, 6, 24, 72, 168: + return nil + } + return ErrInvalid +} + +type AggregatedEvent struct { + ListenerNotice + ID string `json:"id"` + ReceivingAccountID string `json:"receiving_account_id"` + AccountName string `json:"account_name"` + AccountUID string `json:"account_uid"` + ReceivedAt time.Time `json:"received_at"` + Baseline bool `json:"baseline"` + Generation string `json:"generation"` + HasWorkCover bool `json:"has_work_cover"` +} + +func (s *Store) ListAggregatedEventsPage(ctx context.Context, filter EventAggregationFilter, page, pageSize int) (Page[AggregatedEvent], error) { + if err := filter.Validate(); err != nil { + return Page[AggregatedEvent]{}, err + } + if _, _, err := pageBounds(page, pageSize); err != nil { + return Page[AggregatedEvent]{}, err + } + result := Page[AggregatedEvent]{Data: make([]AggregatedEvent, 0), Page: page, PageSize: pageSize} + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) + if err != nil { + return result, err + } + defer tx.Rollback() + from := ` FROM creator_account_event e JOIN social_account a ON a.id = e.account_id + JOIN creator_account_listener l ON l.account_id = a.id AND l.enabled = true + WHERE e.event_type IN ('like', 'comment', 'follow', 'repost') + AND ($1 = '' OR a.account_id = $1) AND ($2 = '' OR e.event_type = $2) + AND ($3::int = 0 OR e.received_at >= now() - make_interval(hours => $3::int))` + args := []any{filter.AccountID, filter.EventType, filter.Hours} + if err := tx.QueryRowContext(ctx, `SELECT count(*)`+from, args...).Scan(&result.Total); err != nil { + return result, err + } + rows, err := tx.QueryContext(ctx, `SELECT e.id::text, a.account_id, a.name, COALESCE(a.platform_account_key, ''), e.event_key, e.event_type, + e.interactor_uid, COALESCE(e.comment_id, ''), COALESCE(e.work_id, ''), e.message_type, e.message_text, + e.platform_event_at, e.gateway_received_at, e.received_at, e.baseline, e.generation, + e.interactor_name, e.interactor_sec_uid, e.work_author_uid, e.work_cover_url, e.work_url, e.work_cover_error`+from+` + ORDER BY e.received_at DESC, e.id DESC LIMIT $4 OFFSET $5`, append(args, pageSize, (page-1)*pageSize)...) + if err != nil { + return result, err + } + for rows.Next() { + var e AggregatedEvent + var platformAt, gatewayAt sql.NullTime + if err := rows.Scan(&e.ID, &e.ReceivingAccountID, &e.AccountName, &e.AccountUID, &e.EventKey, &e.EventType, &e.InteractorUID, + &e.CommentID, &e.WorkID, &e.MessageType, &e.MessageText, &platformAt, &gatewayAt, &e.ReceivedAt, &e.Baseline, &e.Generation, + &e.InteractorName, &e.InteractorSecUID, &e.WorkAuthorUID, &e.WorkCoverURL, &e.WorkURL, &e.WorkCoverError); err != nil { + rows.Close() + return result, err + } + e.PlatformEventAt = nullableTime(platformAt) + e.GatewayReceivedAt = nullableTime(gatewayAt) + if e.WorkAuthorUID != "" && e.WorkID != "" { + _, coverErr := s.GetPlatformWorkCover(e.WorkAuthorUID, e.WorkID) + if coverErr != nil && !errors.Is(coverErr, ErrNotFound) { + rows.Close() + return result, coverErr + } + e.HasWorkCover = coverErr == nil + } + result.Data = append(result.Data, e) + } + err = rows.Err() + rows.Close() + if err != nil { + return result, err + } + result.HasNext = page*pageSize < result.Total + return result, tx.Commit() +} diff --git a/internal/creator/event_listener_test.go b/internal/creator/event_listener_test.go new file mode 100644 index 0000000..afb2fa7 --- /dev/null +++ b/internal/creator/event_listener_test.go @@ -0,0 +1,147 @@ +package creator + +import ( + "errors" + "fmt" + "testing" + "time" +) + +func TestEventAggregationFilter(t *testing.T) { + for _, f := range []EventAggregationFilter{{}, {EventType: "comment", Hours: 24, AccountID: "account"}, {EventType: "follow", Hours: 0}} { + if err := f.Validate(); err != nil { + t.Fatalf("valid filter %+v: %v", f, err) + } + } + for _, f := range []EventAggregationFilter{{EventType: "dm"}, {EventType: "unknown"}, {Hours: -1}, {Hours: 2}} { + if !errors.Is(f.Validate(), ErrInvalid) { + t.Fatalf("invalid filter accepted: %+v", f) + } + } +} + +func TestListenerDeliveryValidation(t *testing.T) { + now := time.Now().UTC() + for _, d := range []ListenerDelivery{ + {DeliveryID: "1", Kind: "baseline", BoundaryAt: &now}, + {DeliveryID: "2", Kind: "open"}, + {DeliveryID: "3", Kind: "error", Reason: "connection lost"}, + {DeliveryID: "4", Kind: "notice", Notice: &ListenerNotice{EventKey: "event", EventType: "comment", MessageText: "hello"}}, + } { + if err := d.Validate(); err != nil { + t.Fatalf("valid delivery %+v: %v", d, err) + } + } + for _, d := range []ListenerDelivery{ + {Kind: "baseline", BoundaryAt: &now}, {DeliveryID: "1", Kind: "baseline"}, + {DeliveryID: "1", Kind: "other"}, {DeliveryID: "1", Kind: "notice"}, + {DeliveryID: "1", Kind: "notice", Notice: &ListenerNotice{EventKey: "", EventType: "like"}}, + {DeliveryID: "1", Kind: "notice", Notice: &ListenerNotice{EventKey: "1", EventType: "unknown"}}, + {DeliveryID: "1", Kind: "status"}, + } { + if !errors.Is(d.Validate(), ErrInvalid) { + t.Fatalf("invalid delivery accepted: %+v", d) + } + } +} + +func TestAccountEventListenerLifecycle(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + id := createIntegrationAccount(t, ctx, accounts, fmt.Sprint(time.Now().UnixNano())) + profile, err := store.GetAccountProfile(ctx, id) + if err != nil || profile.ListenerEnabled || profile.ListenerStatus != "stopped" { + t.Fatalf("default off: %+v %v", profile, err) + } + if _, err := store.SetListenerEnabled(ctx, id, true); !errors.Is(err, ErrConflict) { + t.Fatalf("unverified account enabled: %v", err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + state, err := store.SetListenerEnabled(ctx, id, true) + if err != nil || !state.Enabled || state.Status != "starting" { + t.Fatalf("enable: %+v %v", state, err) + } + again, err := store.SetListenerEnabled(ctx, id, true) + if err != nil || again.Generation != state.Generation { + t.Fatalf("idempotent enable: %+v %v", again, err) + } + states, err := store.ListListenerStates(ctx) + if err != nil || len(states) != 1 || states[0].AccountID != id { + t.Fatalf("states=%+v err=%v", states, err) + } + if err := store.UpdateListenerStatus(ctx, id, state.Generation, "unknown", ""); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + if err := store.UpdateListenerStatus(ctx, id, "old", "ready", ""); !errors.Is(err, ErrConflict) { + t.Fatal(err) + } + if err := store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{{Kind: "unknown"}}); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + for _, f := range []EventAggregationFilter{{EventType: "unknown"}, {Hours: 2}} { + if _, err := store.ListAggregatedEventsPage(ctx, f, 1, 25); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + } + if _, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{}, 0, 25); !errors.Is(err, ErrInvalid) { + t.Fatal(err) + } + now := time.Now().UTC() + deliveries := []ListenerDelivery{ + {DeliveryID: "baseline", Kind: "baseline", BoundaryAt: &now}, + {DeliveryID: "ready", Kind: "open"}, + {DeliveryID: "notice", Kind: "notice", Notice: &ListenerNotice{EventKey: "100", EventType: "comment", InteractorUID: "22", MessageText: "测试评论", PlatformEventAt: &now}}, + {DeliveryID: "historical", Kind: "notice", Baseline: true, Notice: &ListenerNotice{EventKey: "101", EventType: "follow"}}, + } + if err := store.RecordListenerDeliveries(ctx, id, state.Generation, deliveries); err != nil { + t.Fatal(err) + } + if err := store.RecordListenerDeliveries(ctx, id, state.Generation, deliveries); err != nil { + t.Fatal(err) + } + page, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{}, 1, 25) + if err != nil || page.Total != 2 || len(page.Data) != 2 { + t.Fatalf("record and deduplicate: %+v %v", page, err) + } + filtered, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{EventType: "comment", Hours: 24, AccountID: id}, 1, 25) + if err != nil || filtered.Total != 1 || filtered.Data[0].MessageText != "测试评论" { + t.Fatalf("filter: %+v %v", filtered, err) + } + persisted, err := store.GetAccountProfile(ctx, id) + if err != nil || !persisted.ListenerEnabled || persisted.ListenerStatus != "ready" { + t.Fatalf("persisted status: %+v %v", persisted, err) + } + if err := store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{{DeliveryID: "close", Kind: "close"}, {DeliveryID: "reopened", Kind: "open"}}); err != nil { + t.Fatal(err) + } + states, err = store.ListListenerStates(ctx) + if err != nil || states[0].Status != "gap" || states[0].Reason == "" { + t.Fatalf("gap must remain visible until baseline: %+v %v", states, err) + } + if err := store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{{DeliveryID: "boundary2", Kind: "baseline", BoundaryAt: &now}}); err != nil { + t.Fatal(err) + } + off, err := store.SetListenerEnabled(ctx, id, false) + if err != nil || off.Enabled || off.Status != "stopping" || off.Generation == state.Generation { + t.Fatalf("disable: %+v %v", off, err) + } + if err := store.RecordListenerDeliveries(ctx, id, state.Generation, deliveries); !errors.Is(err, ErrConflict) { + t.Fatalf("late delivery accepted: %v", err) + } + page, err = store.ListAggregatedEventsPage(ctx, EventAggregationFilter{}, 1, 25) + if err != nil || page.Total != 0 { + t.Fatalf("disabled account visible: %+v %v", page, err) + } + if err := store.UpdateListenerStatus(ctx, id, off.Generation, "stopped", ""); err != nil { + t.Fatal(err) + } + on, err := store.SetListenerEnabled(ctx, id, true) + if err != nil || on.Generation == off.Generation { + t.Fatalf("re-enable: %+v %v", on, err) + } + page, err = store.ListAggregatedEventsPage(ctx, EventAggregationFilter{}, 1, 1) + if err != nil || page.Total != 2 || !page.HasNext { + t.Fatalf("history retained and paginated: %+v %v", page, err) + } +} diff --git a/internal/creator/integration_test.go b/internal/creator/integration_test.go index 623d61e..23ed0d5 100644 --- a/internal/creator/integration_test.go +++ b/internal/creator/integration_test.go @@ -309,6 +309,9 @@ func TestCreatorPostgresContentAndWorkflow(t *testing.T) { if _, err := store.RecordVerifiedLoginResult(ctx, smallID, "sec_uid_"+smallID); err != nil { t.Fatal(err) } + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status='active' WHERE account_id IN ($1,$2)`, bigID, smallID); err != nil { + t.Fatal(err) + } now := time.Now().UTC().Truncate(time.Microsecond) ownedDue, err := store.ListDueOwnedAccounts(ctx, now, 1800) if err != nil || len(ownedDue) != 2 || ownedDue[0] != bigID || ownedDue[1] != smallID { @@ -520,7 +523,7 @@ func TestCreatorPostgresContentAndWorkflow(t *testing.T) { collector := integrationCollector{work: WorkInput{Platform: PlatformDouyin, WorkKey: "creator-it-collected-" + stamp, SourceType: SourceOwned, SourceID: bigID, Title: "Collected", Body: "body", PublishedAt: &published, PublishedAtStatus: "verified", Likes: &likes}, comment: CommentInput{Platform: PlatformDouyin, CommentKey: "creator-it-collected-comment-" + stamp, WorkID: "", Content: "hello", CommentType: "top_level"}} collector.comment.WorkID = "" report, err := store.CollectSource(ctx, PlatformDouyin, SourceOwned, bigID, collector, now) - if err != nil || !report.PaginationComplete || report.WorksSeen != 1 || report.CommentsSaved != 2 { + if err != nil || !report.PaginationComplete || report.WorksSeen != 1 || report.CommentsSaved != 1 { t.Fatalf("collect source workflow failed: report=%+v err=%v", report, err) } } @@ -598,7 +601,7 @@ func TestCreatorPostgresAccountMonitorViewAndCollectionStatus(t *testing.T) { t.Fatalf("monitor view without works: %+v", view) } // 无指标快照：画像指标为空（不伪造成 0）。 - if view.FollowerCount != nil || view.FollowingCount != nil || view.TotalFavorited != nil || view.AwemeCount != nil { + if view.FollowerCount != nil || view.FollowingCount != nil || view.TotalFavorited != nil || view.PublicWorkCount != nil { t.Fatalf("monitor view without metrics must leave profile counters nil: %+v", view) } if view.CommentTotal != 0 { @@ -610,7 +613,7 @@ func TestCreatorPostgresAccountMonitorViewAndCollectionStatus(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", ""); err != nil { + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", "", nil); err != nil { t.Fatal(err) } published := now.Add(-time.Hour) @@ -665,7 +668,7 @@ func TestCreatorPostgresAccountMonitorViewAndCollectionStatus(t *testing.T) { if view == nil || view.FollowerCount == nil || *view.FollowerCount != second { t.Fatalf("monitor view must carry the latest metric snapshot: %+v", view) } - if view.FollowingCount == nil || *view.FollowingCount != following || view.TotalFavorited == nil || *view.TotalFavorited != favorited || view.AwemeCount == nil || *view.AwemeCount != aweme { + if view.FollowingCount == nil || *view.FollowingCount != following || view.TotalFavorited == nil || *view.TotalFavorited != favorited || view.PublicWorkCount != nil { t.Fatalf("monitor view must fall back to the latest non-null counters: %+v", view) } if view.FriendCount == nil || *view.FriendCount != mutual { @@ -696,6 +699,9 @@ func TestCreatorPostgresCollectionAllowsMutedReadAccount(t *testing.T) { store, phaseAStore, ctx := openCreatorIntegrationStore(t) stamp := time.Now().UnixNano() accountID := createIntegrationAccount(t, ctx, phaseAStore, "muted"+fmt.Sprint(stamp)) + if _, err := store.db.ExecContext(ctx, `UPDATE social_account SET status='active' WHERE account_id=$1`, accountID); err != nil { + t.Fatal(err) + } if err := store.EnsureAccountProfile(ctx, accountID); err != nil { t.Fatal(err) } @@ -728,7 +734,7 @@ func TestCreatorPostgresSettingsResetCollectionCheckpoint(t *testing.T) { if err != nil { t.Fatalf("begin checkpoint: %v", err) } - if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", ""); err != nil { + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", "", nil); err != nil { t.Fatalf("finish checkpoint: %v", err) } if _, err := store.db.ExecContext(ctx, `UPDATE creator_collection_checkpoint SET window_start='2020-01-01T00:00:00Z',window_end='2020-01-02T00:00:00Z',cursor='old-cursor' WHERE source_type=$1 AND source_id=$2 AND collection_kind='works'`, SourceOwned, accountID); err != nil { @@ -757,7 +763,7 @@ func TestCreatorPostgresSettingsResetCollectionCheckpoint(t *testing.T) { if _, err := store.UpdateSettings(ctx, SettingsUpdate{LookbackDays: settings.LookbackDays, NewWorkIntervalSeconds: settings.NewWorkIntervalSeconds, MetricInitialIntervalSeconds: settings.MetricInitialIntervalSeconds, MetricMultiplier: settings.MetricMultiplier, MetricMaxIntervalSeconds: settings.MetricMaxIntervalSeconds, MetricAgeSeconds: settings.MetricAgeSeconds, AIProvider: settings.AIProvider, AIModel: settings.AIModel, AIConfigured: settings.AIConfigured}); !errors.Is(err, ErrConflict) { t.Fatalf("settings changed during running checkpoint: err=%v", err) } - if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", ""); err != nil { + if err := store.finishCheckpoint(ctx, SourceOwned, accountID, "works", lease, "succeeded", "", nil); err != nil { t.Fatal(err) } } diff --git a/internal/creator/login_collection_recovery_test.go b/internal/creator/login_collection_recovery_test.go new file mode 100644 index 0000000..3a7921e --- /dev/null +++ b/internal/creator/login_collection_recovery_test.go @@ -0,0 +1,94 @@ +package creator + +import ( + "errors" + "fmt" + "testing" + "time" +) + +func TestVerifiedLoginReleasesOnlyLoginBlockedCollection(t *testing.T) { + for _, environmentLogin := range []bool{false, true} { + t.Run(fmt.Sprintf("environment=%t", environmentLogin), func(t *testing.T) { + s, _, ctx := openCreatorIntegrationStore(t) + var gatewayID int64 + if err := s.db.QueryRowContext(ctx, `INSERT INTO gateway(name,endpoint,token) VALUES('login-recovery','http://localhost:28187','test') RETURNING id`).Scan(&gatewayID); err != nil { + t.Fatal(err) + } + for i, state := range []struct { + status, reason string + release bool + }{ + {"blocked", "browser is awaiting user login", true}, + {"blocked", "gateway unavailable", false}, + {"failed", "cover download failed", false}, + {"running", "", false}, + {"succeeded", "", false}, + } { + accountID, alias, uid := fmt.Sprintf("account-recovery-%d", i), fmt.Sprintf("env-recovery-%d", i), fmt.Sprintf("90000%d", i) + var rowID int64 + if err := s.db.QueryRowContext(ctx, `INSERT INTO social_account(account_id,name,platform,platform_account_key,credential_provider,credential_key,status) VALUES($1,'恢复账号','douyin',$2,'os_keyring',$1,'active') RETURNING id`, accountID, uid).Scan(&rowID); err != nil { + t.Fatal(err) + } + if _, err := s.db.ExecContext(ctx, `INSERT INTO browser_env(alias,name,account_id,gateway_id,fingerprint,profile_id) VALUES($1,$1,$2,$3,jsonb_build_object('seed',$4::bigint),$1)`, alias, rowID, gatewayID, 1001+i); err != nil { + t.Fatal(err) + } + if err := s.EnsureAccountProfile(ctx, accountID); err != nil { + t.Fatal(err) + } + start, end, completed := time.Now().UTC().Add(-48*time.Hour).Truncate(time.Second), time.Now().UTC().Add(-24*time.Hour).Truncate(time.Second), time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + for _, kind := range []string{"works", "comments"} { + if _, err := s.db.ExecContext(ctx, `INSERT INTO creator_collection_checkpoint(source_type,source_id,collection_kind,window_start,window_end,cursor,status,last_error,last_completed_at,public_work_count_in_progress) VALUES($1,$2,$3,$4,$5,'resume-cursor',$6,$7,$8,13)`, SourceOwned, accountID, kind, start, end, state.status, state.reason, completed); err != nil { + t.Fatal(err) + } + } + verify := func(actual string) error { + if environmentLogin { + _, err := s.RecordVerifiedEnvironmentLogin(ctx, alias, PlatformIdentity{UID: actual, Nickname: "恢复账号"}) + return err + } + _, err := s.RecordVerifiedLoginResult(ctx, accountID, actual) + return err + } + if err := verify("wrong-uid"); !errors.Is(err, ErrConflict) { + t.Fatalf("mismatched UID: %v", err) + } + var before string + if err := s.db.QueryRowContext(ctx, `SELECT status FROM creator_collection_checkpoint WHERE source_type=$1 AND source_id=$2 AND collection_kind='works'`, SourceOwned, accountID).Scan(&before); err != nil || before != state.status { + t.Fatalf("failed verification changed collection: %s %v", before, err) + } + if err := verify(uid); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"works", "comments"} { + var status, reason, cursor string + var gotStart, gotEnd, gotCompleted time.Time + var progress int64 + if err := s.db.QueryRowContext(ctx, `SELECT status,last_error,cursor,window_start,window_end,last_completed_at,public_work_count_in_progress FROM creator_collection_checkpoint WHERE source_type=$1 AND source_id=$2 AND collection_kind=$3`, SourceOwned, accountID, kind).Scan(&status, &reason, &cursor, &gotStart, &gotEnd, &gotCompleted, &progress); err != nil { + t.Fatal(err) + } + expectedStatus, expectedReason := state.status, state.reason + if state.release { + expectedStatus, expectedReason = "idle", "" + } + if status != expectedStatus || reason != expectedReason || cursor != "resume-cursor" || !gotStart.Equal(start) || !gotEnd.Equal(end) || !gotCompleted.Equal(completed) || progress != 13 { + t.Fatalf("%s state changed incorrectly: %s %q cursor=%s progress=%d", kind, status, reason, cursor, progress) + } + } + if state.release { + gotStart, gotEnd, err := s.NextCollectionWindow(ctx, SourceOwned, accountID, time.Now().UTC(), time.Hour, 1) + if err != nil || !gotStart.Equal(start) || !gotEnd.Equal(end) { + t.Fatalf("recovery skipped original window: %v %v %v", gotStart, gotEnd, err) + } + if _, err := s.beginCheckpoint(ctx, SourceOwned, accountID, "works", time.Now().UTC().Add(-time.Hour), time.Now().UTC()); err != nil { + t.Fatal(err) + } + checkpoint, err := s.checkpoint(ctx, SourceOwned, accountID, "works") + if err != nil || checkpoint.Cursor != "resume-cursor" || checkpoint.PublicWorkCountInProgress != 13 || !checkpoint.WindowStart.Equal(start) || !checkpoint.WindowEnd.Equal(end) { + t.Fatalf("recovery erased progress: %#v %v", checkpoint, err) + } + } + } + }) + } +} diff --git a/internal/creator/models.go b/internal/creator/models.go index 78c2772..79c7c1b 100644 --- a/internal/creator/models.go +++ b/internal/creator/models.go @@ -52,6 +52,10 @@ type SettingsUpdate struct { } type AccountProfile struct { + ListenerEnabled bool `json:"listener_enabled"` + ListenerStatus string `json:"listener_status"` + ListenerReason string `json:"listener_reason"` + ListenerLastPollAt *time.Time `json:"listener_last_poll_at,omitempty"` ID string `json:"id"` Name string `json:"name"` Platform string `json:"platform"` @@ -400,16 +404,20 @@ type Relation struct { } type ListenerState struct { - AccountID string `json:"account_id"` - Platform string `json:"platform"` - Generation string `json:"generation"` - Status string `json:"status"` - BoundaryAt *time.Time `json:"boundary_at,omitempty"` - LastDeliveryID string `json:"last_delivery_id,omitempty"` - Reason string `json:"reason,omitempty"` - UpdatedAt time.Time `json:"updated_at"` - SessionToken string `json:"-"` - Invalidated bool `json:"-"` + AccountUID string `json:"account_uid"` + Enabled bool `json:"enabled"` + AccountID string `json:"account_id"` + Platform string `json:"platform"` + Generation string `json:"generation"` + Status string `json:"status"` + BoundaryAt *time.Time `json:"boundary_at,omitempty"` + LastDeliveryID string `json:"last_delivery_id,omitempty"` + Checkpoints map[string]string `json:"notice_checkpoints"` + LastPollAt *time.Time `json:"last_poll_at,omitempty"` + Reason string `json:"reason,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + SessionToken string `json:"-"` + Invalidated bool `json:"-"` } type Page[T any] struct { diff --git a/internal/creator/notice_history_test.go b/internal/creator/notice_history_test.go new file mode 100644 index 0000000..3e5a7c2 --- /dev/null +++ b/internal/creator/notice_history_test.go @@ -0,0 +1,73 @@ +package creator + +import ( + "errors" + "fmt" + "testing" + "time" +) + +func TestNotificationHistoryBackfillAcrossEnableGenerations(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + id := createIntegrationAccount(t, ctx, accounts, fmt.Sprint(time.Now().UnixNano())) + profile, err := store.GetAccountProfile(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err = store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + first, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Microsecond) + old := now.Add(-48 * time.Hour) + history := ListenerDelivery{DeliveryID: "old-delivery", Kind: "notice", Baseline: true, Notice: &ListenerNotice{EventKey: "101", EventType: "comment", PlatformEventAt: &old, MessageText: "历史评论"}} + if err = store.RecordListenerDeliveries(ctx, id, first.Generation, []ListenerDelivery{history}); err != nil { + t.Fatal(err) + } + if _, err = store.SetListenerEnabled(ctx, id, false); err != nil { + t.Fatal(err) + } + during := now.Add(-time.Hour) + missing := ListenerDelivery{DeliveryID: "disabled-delivery", Kind: "notice", Baseline: true, Notice: &ListenerNotice{EventKey: "102", EventType: "follow", PlatformEventAt: &during}} + if err = store.RecordListenerDeliveries(ctx, id, first.Generation, []ListenerDelivery{missing}); !errors.Is(err, ErrConflict) { + t.Fatalf("disabled account accepted delivery: %v", err) + } + hidden, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{AccountID: id}, 1, 25) + if err != nil || hidden.Total != 0 { + t.Fatalf("disabled history visible: %+v %v", hidden, err) + } + next, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + checkpoint := ListenerDelivery{DeliveryID: "history-complete", Kind: "checkpoint", BoundaryAt: &now, CheckedAt: &now, Checkpoints: map[string]string{"700": "102"}} + deliveries := []ListenerDelivery{{DeliveryID: "baseline", Kind: "baseline", BoundaryAt: &now}, history, missing, checkpoint} + for i := 0; i < 2; i++ { + if err = store.RecordListenerDeliveries(ctx, id, next.Generation, deliveries); err != nil { + t.Fatal(err) + } + } + result, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{AccountID: id}, 1, 25) + if err != nil || result.Total != 2 { + t.Fatalf("history missing or duplicated: %+v %v", result, err) + } + seen := map[string]bool{} + for _, event := range result.Data { + if !event.Baseline || event.PlatformEventAt == nil { + t.Fatalf("historical time/label lost: %+v", event) + } + seen[event.EventKey] = true + if event.EventKey == "101" && !event.PlatformEventAt.Equal(old) { + t.Fatalf("historical timestamp overwritten: %+v", event) + } + if event.EventKey == "102" && !event.PlatformEventAt.Equal(during) { + t.Fatalf("disabled-period timestamp overwritten: %+v", event) + } + } + if !seen["101"] || !seen["102"] { + t.Fatalf("history incomplete: %+v", seen) + } +} diff --git a/internal/creator/notice_polling_test.go b/internal/creator/notice_polling_test.go new file mode 100644 index 0000000..28e10cb --- /dev/null +++ b/internal/creator/notice_polling_test.go @@ -0,0 +1,95 @@ +package creator + +import ( + "errors" + "fmt" + "testing" + "time" +) + +func TestNotificationCheckpointValidation(t *testing.T) { + now := time.Now().UTC() + valid := ListenerDelivery{DeliveryID: "scan", Kind: "checkpoint", BoundaryAt: &now, CheckedAt: &now, Checkpoints: map[string]string{"960": "9007199254740993"}} + if err := valid.Validate(); err != nil { + t.Fatal(err) + } + for _, d := range []ListenerDelivery{ + {DeliveryID: "scan", Kind: "checkpoint"}, + {DeliveryID: "scan", Kind: "checkpoint", BoundaryAt: &now, CheckedAt: &now, Checkpoints: map[string]string{"invalid": "1"}}, + {DeliveryID: "scan", Kind: "checkpoint", BoundaryAt: &now, CheckedAt: &now, Checkpoints: map[string]string{"960": "bad"}}, + } { + if !errors.Is(d.Validate(), ErrInvalid) { + t.Fatalf("accepted %+v", d) + } + } +} + +func TestNotificationCheckpointTransactionAndRestart(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + id := createIntegrationAccount(t, ctx, accounts, fmt.Sprint(time.Now().UnixNano())) + profile, err := store.GetAccountProfile(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err = store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + state, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Microsecond) + checkpoint := ListenerDelivery{DeliveryID: "scan", Kind: "checkpoint", BoundaryAt: &now, CheckedAt: &now, Checkpoints: map[string]string{"960": "9007199254740993"}} + items := []ListenerDelivery{{DeliveryID: "event", Kind: "notice", Notice: &ListenerNotice{EventKey: "9007199254740993", EventType: "follow", PlatformEventAt: &now}}, checkpoint} + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, items); err != nil { + t.Fatal(err) + } + profile, err = store.GetAccountProfile(ctx, id) + if err != nil || profile.ListenerLastPollAt == nil || !profile.ListenerLastPollAt.Equal(now) { + t.Fatalf("account does not expose last successful scan: %+v %v", profile, err) + } + states, err := store.ListListenerStates(ctx) + if err != nil { + t.Fatal(err) + } + if states[0].Checkpoints["960"] != "9007199254740993" || !states[0].BoundaryAt.Equal(now) || !states[0].LastPollAt.Equal(now) || states[0].Status != "ready" { + t.Fatalf("checkpoint not durable: %+v", states[0]) + } + later := now.Add(time.Minute) + bad := checkpoint + bad.BoundaryAt = &later + bad.Checkpoints = map[string]string{"960": "2"} + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{bad}); !errors.Is(err, ErrConflict) { + t.Fatalf("boundary replaced: %v", err) + } + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, items); err != nil { + t.Fatal(err) + } + page, err := store.ListAggregatedEventsPage(ctx, EventAggregationFilter{}, 1, 25) + if err != nil || page.Total != 1 { + t.Fatalf("duplicate inserted: %+v %v", page, err) + } + gap := checkpoint + gap.Reason = "checkpoint disappeared" + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, []ListenerDelivery{gap}); err != nil { + t.Fatal(err) + } + states, err = store.ListListenerStates(ctx) + if err != nil || states[0].Status != "gap" { + t.Fatalf("missing boundary hidden: %+v %v", states, err) + } + off, err := store.SetListenerEnabled(ctx, id, false) + if err != nil { + t.Fatal(err) + } + if err = store.RecordListenerDeliveries(ctx, id, state.Generation, items); !errors.Is(err, ErrConflict) { + t.Fatal(err) + } + on, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + if on.Generation == off.Generation || on.BoundaryAt != nil || len(on.Checkpoints) != 0 || on.LastPollAt != nil { + t.Fatalf("new enable inherited old boundary: %+v", on) + } +} diff --git a/internal/creator/owned_collection_progress_test.go b/internal/creator/owned_collection_progress_test.go index 899abf7..75d9239 100644 --- a/internal/creator/owned_collection_progress_test.go +++ b/internal/creator/owned_collection_progress_test.go @@ -9,17 +9,25 @@ import ( ) type ownedHistoryCollector struct { - pages map[string]WorkPage - workCursors []string - commentWorks []string + pages map[string]WorkPage + workCursors []string + commentWorks []string + excludedWorkKeys []string + workErr error } func (c *ownedHistoryCollector) ListWorks(_ context.Context, _, cursor string) (WorkPage, error) { c.workCursors = append(c.workCursors, cursor) + if c.workErr != nil { + return WorkPage{}, c.workErr + } page, ok := c.pages[cursor] if !ok { return WorkPage{}, fmt.Errorf("unexpected cursor %q", cursor) } + if cursor == "" { + page.ExcludedWorkKeys = append(page.ExcludedWorkKeys, c.excludedWorkKeys...) + } return page, nil } @@ -28,6 +36,36 @@ func (c *ownedHistoryCollector) ListTopLevelComments(_ context.Context, workKey, return CommentPage{}, nil } +func TestOwnedCollectionFirstPageFailurePreservesExistingVisibility(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + accountID := createIntegrationAccount(t, ctx, accounts, "visibility"+fmt.Sprint(time.Now().UnixNano())) + if err := store.EnsureAccountProfile(ctx, accountID); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + published := now.Add(-time.Hour) + work, _, err := store.UpsertWork(ctx, WorkInput{ + Platform: PlatformDouyin, WorkKey: "owned-before-failed-refresh", SourceType: SourceOwned, + SourceID: accountID, Title: "已采集作品", PublishedAt: &published, PublishedAtStatus: "verified", + }, now) + if err != nil { + t.Fatal(err) + } + + collector := &ownedHistoryCollector{workErr: fmt.Errorf("HTTP 409: browser alias is busy")} + if _, err := store.CollectSource(ctx, PlatformDouyin, SourceOwned, accountID, collector, now); err == nil { + t.Fatal("expected the works-page request to fail") + } + + var isPublic bool + if err := store.db.QueryRowContext(ctx, `SELECT is_public FROM creator_work WHERE work_id = $1`, work.ID).Scan(&isPublic); err != nil { + t.Fatal(err) + } + if !isPublic { + t.Fatal("a failed first page must not hide previously collected public works") + } +} + func TestCreatorPostgresOwnedCollectionIncludesHistoryAndProgress(t *testing.T) { store, accounts, ctx := openCreatorIntegrationStore(t) accountID := createIntegrationAccount(t, ctx, accounts, fmt.Sprint(time.Now().UnixNano())) @@ -53,18 +91,23 @@ func TestCreatorPostgresOwnedCollectionIncludesHistoryAndProgress(t *testing.T) page.Items = append(page.Items, work) pages[cursor] = page } - collector := &ownedHistoryCollector{pages: pages} - total := int64(30) - if err := store.RecordAccountMetric(ctx, AccountMetricInput{AccountID: accountID, CollectedAt: now.Add(-time.Minute), AwemeCount: &total}); err != nil { + privatePublished := recent + privateWork, _, err := store.UpsertWork(ctx, WorkInput{Platform: PlatformDouyin, SourceType: SourceOwned, SourceID: accountID, WorkKey: "owned-history-private", Title: "private", PublishedAt: &privatePublished, PublishedAtStatus: "verified"}, now) + if err != nil { t.Fatal(err) } - // A newer snapshot lacking aweme_count must not erase the known platform total. + collector := &ownedHistoryCollector{pages: pages, excludedWorkKeys: []string{privateWork.WorkKey}} + profileTotal := int64(30) + if err := store.RecordAccountMetric(ctx, AccountMetricInput{AccountID: accountID, CollectedAt: now.Add(-time.Minute), AwemeCount: &profileTotal}); err != nil { + t.Fatal(err) + } + // The profile aweme_count remains separate from the strict public-work total. followers := int64(100) if err := store.RecordAccountMetric(ctx, AccountMetricInput{AccountID: accountID, CollectedAt: now, FollowerCount: &followers}); err != nil { t.Fatal(err) } report, err := store.CollectSource(ctx, PlatformDouyin, SourceOwned, accountID, collector, now) - if err != nil || !report.PaginationComplete || report.WorksSeen != 28 || report.WorksSaved != 28 { + if err != nil || !report.PaginationComplete || report.WorksSeen != 29 || report.WorksSaved != 28 { t.Fatalf("full history collection: report=%+v err=%v", report, err) } if len(collector.workCursors) != 2 || collector.workCursors[1] != "history" { @@ -86,10 +129,11 @@ func TestCreatorPostgresOwnedCollectionIncludesHistoryAndProgress(t *testing.T) if err != nil { t.Fatal(err) } - // Read JSON fields to keep the regression test compilable before implementation. - assertCollectionCounts(t, status, 28, &total) + // Read JSON fields to guard the public collection-count API contract. + publicTotal := int64(28) + assertCollectionCounts(t, status, 28, &publicTotal) views, err := store.ListAccountMonitorViews(ctx) - if err != nil || len(views) != 1 || views[0].WorkCount != 28 || views[0].AwemeCount == nil || *views[0].AwemeCount != total { + if err != nil || len(views) != 1 || views[0].WorkCount != 28 || views[0].PublicWorkCount == nil || *views[0].PublicWorkCount != publicTotal { t.Fatalf("list/detail count consistency: views=%+v err=%v", views, err) } // A full subsequent run must not count duplicate works twice. @@ -100,7 +144,14 @@ func TestCreatorPostgresOwnedCollectionIncludesHistoryAndProgress(t *testing.T) if err != nil { t.Fatal(err) } - assertCollectionCounts(t, status, 28, &total) + assertCollectionCounts(t, status, 28, &publicTotal) + var retainedPrivate, visiblePrivate int + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*), COUNT(*) FILTER (WHERE is_public) FROM creator_work WHERE work_id = $1`, privateWork.ID).Scan(&retainedPrivate, &visiblePrivate); err != nil { + t.Fatal(err) + } + if retainedPrivate != 1 || visiblePrivate != 0 { + t.Fatalf("previously collected non-public work must remain hidden, not deleted: rows=%d visible=%d", retainedPrivate, visiblePrivate) + } } func assertCollectionCounts(t *testing.T, status AccountCollectionStatus, collected int64, total *int64) { @@ -120,7 +171,7 @@ func assertCollectionCounts(t *testing.T, status AccountCollectionStatus, collec if total != nil { want = fmt.Sprint(*total) } - if string(fields["aweme_count"]) != want { + if string(fields["public_work_count"]) != want { t.Fatalf("platform total: status=%s want=%s", data, want) } } @@ -141,5 +192,14 @@ func TestCreatorPostgresCollectionProgressDistinguishesUnknownAndZero(t *testing if err != nil { t.Fatal(err) } + assertCollectionCounts(t, status, 0, nil) + collector := &ownedHistoryCollector{pages: map[string]WorkPage{"": {}}} + if _, err := store.CollectSource(ctx, PlatformDouyin, SourceOwned, accountID, collector, time.Now().UTC()); err != nil { + t.Fatal(err) + } + status, err = store.GetAccountCollectionStatus(ctx, accountID) + if err != nil { + t.Fatal(err) + } assertCollectionCounts(t, status, 0, &zero) } diff --git a/internal/creator/private_messages.go b/internal/creator/private_messages.go new file mode 100644 index 0000000..556c6e5 --- /dev/null +++ b/internal/creator/private_messages.go @@ -0,0 +1,425 @@ +package creator + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type PrivateMessageSendInput struct { + RequestID string `json:"request_id"` + AccountID string `json:"account_id"` + PeerUID string `json:"peer_uid"` + Text string `json:"text"` +} + +func privateUID(v string) bool { + if len(v) == 0 || len(v) > 20 || v == "0" { + return false + } + for _, r := range v { + if r < '0' || r > '9' { + return false + } + } + return true +} +func (i PrivateMessageSendInput) Validate() error { + if strings.TrimSpace(i.RequestID) == "" || len(i.RequestID) > 128 || strings.TrimSpace(i.AccountID) == "" || !privateUID(i.PeerUID) || strings.TrimSpace(i.Text) == "" || utf8.RuneCountInString(i.Text) > 1000 { + return fmt.Errorf("%w: 账号、对方 UID、发送编号和 1–1000 字的消息不能为空", ErrInvalid) + } + return nil +} + +type PrivateMessage struct { + ID string `json:"id"` + AccountID string `json:"account_id"` + PeerUID string `json:"peer_uid"` + PeerName string `json:"peer_name"` + ServerID string `json:"server_id,omitempty"` + RequestID string `json:"request_id,omitempty"` + Direction string `json:"direction"` + MessageType string `json:"message_type"` + Text string `json:"text"` + State string `json:"state"` + Error string `json:"error"` + MessageAt *time.Time `json:"message_at"` + CreatedAt time.Time `json:"created_at"` +} +type PrivateMessageReservation struct { + Message PrivateMessage + New bool + Generation string + UID string +} +type PrivateConversation struct { + AccountID string `json:"account_id"` + AccountName string `json:"account_name"` + PeerUID string `json:"peer_uid"` + PeerName string `json:"peer_name"` + LastText string `json:"last_text"` + LastMessageAt *time.Time `json:"last_message_at"` + MessageCount int `json:"message_count"` +} +type PrivatePage[T any] struct { + Data []T `json:"data"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} +type PrivateSyncStatus struct { + AccountID string `json:"account_id"` + AccountName string `json:"account_name"` + SyncedAt *time.Time `json:"synced_at"` + Error string `json:"error"` +} +type PrivateInboxMessage struct { + PeerUID, PeerName, ServerID, ClientID, Direction, MessageType, Text string + MessageAt *time.Time +} + +// The platform timestamp is optional. Reception time is never substituted for it. +func privateMessageTime(value string) (*time.Time, error) { + if value == "" { + return nil, nil + } + if n, err := strconv.ParseInt(value, 10, 64); err == nil { + if n <= 0 { + return nil, fmt.Errorf("invalid message time") + } + var t time.Time + if n > 100000000000 { + t = time.UnixMilli(n) + } else { + t = time.Unix(n, 0) + } + t = t.UTC() + return &t, nil + } + t, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, fmt.Errorf("invalid message time: %w", err) + } + t = t.UTC() + return &t, nil +} +func DecodePrivateInbox(raw []byte, expectedUID string) ([]PrivateInboxMessage, error) { + var response struct { + Status string `json:"status"` + Code string `json:"code"` + AccountUID string `json:"account_uid"` + Messages *[]struct { + PeerUID string `json:"peer_uid"` + PeerName string `json:"peer_name"` + ServerID string `json:"server_id"` + ClientID string `json:"client_id"` + SenderUID string `json:"sender_uid"` + MessageType string `json:"message_type"` + Content json.RawMessage `json:"content"` + CreatedAt *string `json:"created_at"` + } `json:"messages"` + } + if err := json.Unmarshal(raw, &response); err != nil { + return nil, fmt.Errorf("decode inbox: %w", err) + } + if response.Status != "succeeded" || response.AccountUID != expectedUID || response.Messages == nil { + return nil, fmt.Errorf("invalid inbox response: status=%s code=%s identity=%s", response.Status, response.Code, response.AccountUID) + } + result := make([]PrivateInboxMessage, 0, len(*response.Messages)) + for _, m := range *response.Messages { + if !privateUID(m.PeerUID) || m.PeerUID == expectedUID || !privateUID(m.ServerID) || (m.SenderUID != expectedUID && m.SenderUID != m.PeerUID) { + return nil, fmt.Errorf("invalid inbox message identity or server id") + } + direction := "inbound" + if m.SenderUID == expectedUID { + direction = "outbound" + } + content := m.Content + var str string + if json.Unmarshal(content, &str) == nil { + content = []byte(str) + } + var body struct { + Text string `json:"text"` + AweType int `json:"aweType"` + Type int `json:"type"` + } + if err := json.Unmarshal(content, &body); err != nil { + return nil, fmt.Errorf("message %s content: %w", m.ServerID, err) + } + kind := "unknown" + if m.MessageType == "7" { + switch body.AweType { + case 700: + kind = "text" + case 701: + kind = "image" + case 702: + kind = "audio" + case 703: + kind = "video" + case 704: + kind = "sticker" + } + } + text := "" + if kind == "text" { + text = body.Text + } + rawTime := "" + if m.CreatedAt != nil { + rawTime = *m.CreatedAt + } + at, err := privateMessageTime(rawTime) + if err != nil { + return nil, fmt.Errorf("message %s: %w", m.ServerID, err) + } + result = append(result, PrivateInboxMessage{PeerUID: m.PeerUID, PeerName: m.PeerName, ServerID: m.ServerID, ClientID: m.ClientID, Direction: direction, MessageType: kind, Text: text, MessageAt: at}) + } + return result, nil +} + +const privateMessageQuery = `SELECT m.id,a.account_id,m.peer_uid,m.peer_name,COALESCE(m.server_id,''),COALESCE(m.request_id,''),m.direction,m.message_type,m.text,m.state,m.error,m.message_at,m.created_at FROM creator_private_message m JOIN social_account a ON a.id=m.account_id` + +func scanPrivateMessage(row interface{ Scan(...any) error }) (PrivateMessage, error) { + var m PrivateMessage + var at sql.NullTime + err := row.Scan(&m.ID, &m.AccountID, &m.PeerUID, &m.PeerName, &m.ServerID, &m.RequestID, &m.Direction, &m.MessageType, &m.Text, &m.State, &m.Error, &at, &m.CreatedAt) + m.MessageAt = nullableTime(at) + return m, rowError(err) +} + +func (s *Store) BeginPrivateMessage(ctx context.Context, input PrivateMessageSendInput) (PrivateMessageReservation, error) { + if err := input.Validate(); err != nil { + return PrivateMessageReservation{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return PrivateMessageReservation{}, err + } + defer tx.Rollback() + var id int64 + var generation, uid, login string + var enabled bool + err = tx.QueryRowContext(ctx, `SELECT a.id,l.enabled,l.generation,COALESCE(a.platform_account_key,''),p.login_status FROM social_account a JOIN creator_account_listener l ON l.account_id=a.id JOIN creator_account_profile p ON p.account_id=a.id WHERE a.account_id=$1 AND a.platform='douyin' FOR UPDATE OF l`, input.AccountID).Scan(&id, &enabled, &generation, &uid, &login) + if errors.Is(err, sql.ErrNoRows) || err == nil && (!enabled || login != "logged_in") { + return PrivateMessageReservation{}, fmt.Errorf("%w: 只能给已登录且已开启监听的账号发送私信", ErrConflict) + } + if err != nil { + return PrivateMessageReservation{}, err + } + if uid == input.PeerUID { + return PrivateMessageReservation{}, fmt.Errorf("%w: 不能给自己发送私信", ErrInvalid) + } + var existingID string + err = tx.QueryRowContext(ctx, `INSERT INTO creator_private_message(id,account_id,peer_uid,request_id,direction,message_type,text,state) VALUES($1,$2,$3,$4,'outbound','text',$5,'sending') ON CONFLICT(account_id,request_id) DO NOTHING RETURNING id`, newID("dm"), id, input.PeerUID, input.RequestID, input.Text).Scan(&existingID) + fresh := true + if errors.Is(err, sql.ErrNoRows) { + fresh = false + } else if err != nil { + return PrivateMessageReservation{}, databaseError(err) + } + message, err := scanPrivateMessage(tx.QueryRowContext(ctx, privateMessageQuery+` WHERE a.account_id=$1 AND m.request_id=$2`, input.AccountID, input.RequestID)) + if err != nil { + return PrivateMessageReservation{}, err + } + if message.PeerUID != input.PeerUID || message.Text != input.Text { + return PrivateMessageReservation{}, fmt.Errorf("%w: 同一发送编号的账号、收件人和内容不可更改", ErrConflict) + } + if err := tx.Commit(); err != nil { + return PrivateMessageReservation{}, err + } + return PrivateMessageReservation{Message: message, New: fresh, Generation: generation, UID: uid}, nil +} +func (s *Store) FinishPrivateMessage(ctx context.Context, id, state, serverID, reason string, at *time.Time) (PrivateMessage, error) { + if state != "succeeded" && state != "failed" && state != "unknown" { + return PrivateMessage{}, ErrInvalid + } + if serverID != "" && !privateUID(serverID) { + return PrivateMessage{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return PrivateMessage{}, err + } + defer tx.Rollback() + var accountID int64 + if err := tx.QueryRowContext(ctx, `SELECT account_id FROM creator_private_message WHERE id=$1 FOR UPDATE`, id).Scan(&accountID); err != nil { + return PrivateMessage{}, rowError(err) + } + // A concurrent inbox pull may already contain the server-confirmed message. + // Merge that copy into the reserved row rather than showing it twice. + if serverID != "" { + if _, err := tx.ExecContext(ctx, `DELETE FROM creator_private_message WHERE account_id=$1 AND server_id=$2 AND id<>$3`, accountID, serverID, id); err != nil { + return PrivateMessage{}, err + } + } + result, err := tx.ExecContext(ctx, `UPDATE creator_private_message SET state=$2,server_id=NULLIF($3,''),error=$4,message_at=COALESCE($5,message_at),updated_at=now() WHERE id=$1 AND state='sending'`, id, state, serverID, reason, at) + if err != nil { + return PrivateMessage{}, databaseError(err) + } + n, err := result.RowsAffected() + if err != nil { + return PrivateMessage{}, err + } + if n != 1 { + return PrivateMessage{}, ErrConflict + } + m, err := scanPrivateMessage(tx.QueryRowContext(ctx, privateMessageQuery+` WHERE m.id=$1`, id)) + if err != nil { + return PrivateMessage{}, err + } + if err := tx.Commit(); err != nil { + return PrivateMessage{}, err + } + return m, nil +} +func (s *Store) RecoverInterruptedPrivateMessages(ctx context.Context) error { + _, err := s.db.ExecContext(ctx, `UPDATE creator_private_message SET state='unknown',error=CASE WHEN state='failed' THEN '聊天客户端网络错误（1008），发送结果未确认；请核实对方是否收到，不自动重发' ELSE '服务重启，发送结果未确认；请核实对方是否收到，不自动重发' END,updated_at=now() WHERE state='sending' OR (state='failed' AND error ~ 'status_code=1008($|[;；])')`) + return err +} + +func (s *Store) SavePrivateInbox(ctx context.Context, accountID, generation string, items []PrivateInboxMessage) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var id int64 + var enabled bool + var current string + if err := tx.QueryRowContext(ctx, `SELECT l.account_id,l.enabled,l.generation FROM creator_account_listener l JOIN social_account a ON a.id=l.account_id WHERE a.account_id=$1 FOR UPDATE OF l`, accountID).Scan(&id, &enabled, ¤t); err != nil { + return rowError(err) + } + if !enabled || current != generation { + return ErrConflict + } + for _, m := range items { + if !privateUID(m.PeerUID) || !privateUID(m.ServerID) || (m.Direction != "inbound" && m.Direction != "outbound") { + return ErrInvalid + } + _, err = tx.ExecContext(ctx, `INSERT INTO creator_private_message(id,account_id,peer_uid,peer_name,server_id,direction,message_type,text,state,message_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'succeeded',$9) ON CONFLICT(account_id,server_id) DO UPDATE SET peer_name=COALESCE(NULLIF(EXCLUDED.peer_name,''),creator_private_message.peer_name),message_at=COALESCE(EXCLUDED.message_at,creator_private_message.message_at),updated_at=now()`, newID("dm"), id, m.PeerUID, m.PeerName, m.ServerID, m.Direction, m.MessageType, m.Text, m.MessageAt) + if err != nil { + return databaseError(err) + } + } + if _, err := tx.ExecContext(ctx, `UPDATE creator_account_listener SET dm_synced_at=now(),dm_sync_error='' WHERE account_id=$1`, id); err != nil { + return err + } + return tx.Commit() +} +func (s *Store) UpdatePrivateSyncStatus(ctx context.Context, accountID, generation, reason string) error { + result, err := s.db.ExecContext(ctx, `UPDATE creator_account_listener SET dm_sync_error=$3 WHERE account_id=(SELECT id FROM social_account WHERE account_id=$1) AND generation=$2 AND enabled`, accountID, generation, reason) + if err != nil { + return err + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n != 1 { + return ErrConflict + } + return nil +} + +// Private messaging needs only the listener switch and generation, not the +// unrelated notification collector's delivery/checkpoint state. +func (s *Store) ListPrivateListeners(ctx context.Context) ([]ListenerState, error) { + rows, err := s.db.QueryContext(ctx, `SELECT a.account_id,l.enabled,l.generation FROM creator_account_listener l JOIN social_account a ON a.id=l.account_id ORDER BY a.id DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]ListenerState, 0) + for rows.Next() { + var state ListenerState + if err := rows.Scan(&state.AccountID, &state.Enabled, &state.Generation); err != nil { + return nil, err + } + out = append(out, state) + } + return out, rows.Err() +} + +func (s *Store) ListPrivateSyncStatus(ctx context.Context) ([]PrivateSyncStatus, error) { + rows, err := s.db.QueryContext(ctx, `SELECT a.account_id,a.name,l.dm_synced_at,l.dm_sync_error FROM creator_account_listener l JOIN social_account a ON a.id=l.account_id WHERE l.enabled ORDER BY a.id DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]PrivateSyncStatus, 0) + for rows.Next() { + var s PrivateSyncStatus + var at sql.NullTime + if err := rows.Scan(&s.AccountID, &s.AccountName, &at, &s.Error); err != nil { + return nil, err + } + s.SyncedAt = nullableTime(at) + out = append(out, s) + } + return out, rows.Err() +} +func privatePage[T any](page, size int) PrivatePage[T] { + if page < 1 { + page = 1 + } + if size < 1 { + size = 25 + } + if size > 100 { + size = 100 + } + return PrivatePage[T]{Data: make([]T, 0), Page: page, PageSize: size} +} +func (s *Store) ListPrivateMessages(ctx context.Context, accountID, peerUID string, page, size int) (PrivatePage[PrivateMessage], error) { + out := privatePage[PrivateMessage](page, size) + if accountID == "" || !privateUID(peerUID) { + return out, ErrInvalid + } + const where = ` JOIN creator_account_listener l ON l.account_id=a.id WHERE l.enabled AND a.account_id=$1 AND m.peer_uid=$2` + if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM creator_private_message m JOIN social_account a ON a.id=m.account_id`+where, accountID, peerUID).Scan(&out.Total); err != nil { + return out, err + } + rows, err := s.db.QueryContext(ctx, privateMessageQuery+where+` ORDER BY COALESCE(m.message_at,m.created_at) DESC,m.created_at DESC,m.id DESC LIMIT $3 OFFSET $4`, accountID, peerUID, out.PageSize, (out.Page-1)*out.PageSize) + if err != nil { + return out, err + } + defer rows.Close() + for rows.Next() { + m, err := scanPrivateMessage(rows) + if err != nil { + return out, err + } + out.Data = append(out.Data, m) + } + return out, rows.Err() +} +func (s *Store) ListPrivateConversations(ctx context.Context, accountID string, page, size int) (PrivatePage[PrivateConversation], error) { + out := privatePage[PrivateConversation](page, size) + const base = ` FROM creator_private_message m JOIN social_account a ON a.id=m.account_id JOIN creator_account_listener l ON l.account_id=a.id WHERE l.enabled AND ($1='' OR a.account_id=$1)` + if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM (SELECT m.account_id,m.peer_uid`+base+` GROUP BY m.account_id,m.peer_uid) c`, accountID).Scan(&out.Total); err != nil { + return out, err + } + rows, err := s.db.QueryContext(ctx, `SELECT a.account_id,a.name,m.peer_uid,COALESCE((array_agg(NULLIF(m.peer_name,'') ORDER BY m.created_at DESC) FILTER(WHERE m.peer_name<>''))[1],''),(array_agg(m.text ORDER BY COALESCE(m.message_at,m.created_at) DESC,m.created_at DESC,m.id DESC))[1],(array_agg(m.message_at ORDER BY COALESCE(m.message_at,m.created_at) DESC,m.created_at DESC,m.id DESC))[1],count(*)`+base+` GROUP BY a.id,a.account_id,a.name,m.peer_uid ORDER BY max(COALESCE(m.message_at,m.created_at)) DESC,a.id DESC,m.peer_uid LIMIT $2 OFFSET $3`, accountID, out.PageSize, (out.Page-1)*out.PageSize) + if err != nil { + return out, err + } + defer rows.Close() + for rows.Next() { + var c PrivateConversation + var at sql.NullTime + if err := rows.Scan(&c.AccountID, &c.AccountName, &c.PeerUID, &c.PeerName, &c.LastText, &at, &c.MessageCount); err != nil { + return out, err + } + c.LastMessageAt = nullableTime(at) + out.Data = append(out.Data, c) + } + return out, rows.Err() +} diff --git a/internal/creator/private_messages_test.go b/internal/creator/private_messages_test.go new file mode 100644 index 0000000..b3320b5 --- /dev/null +++ b/internal/creator/private_messages_test.go @@ -0,0 +1,189 @@ +package creator + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +func TestPrivateMessageInputValidation(t *testing.T) { + valid := PrivateMessageSendInput{RequestID: "one", AccountID: "account", PeerUID: "123", Text: "你好"} + if err := valid.Validate(); err != nil { + t.Fatal(err) + } + for _, edit := range []func(*PrivateMessageSendInput){ + func(i *PrivateMessageSendInput) { i.RequestID = "" }, func(i *PrivateMessageSendInput) { i.AccountID = "" }, + func(i *PrivateMessageSendInput) { i.PeerUID = "abc" }, func(i *PrivateMessageSendInput) { i.PeerUID = "0" }, + func(i *PrivateMessageSendInput) { i.Text = " " }, func(i *PrivateMessageSendInput) { i.Text = strings.Repeat("你", 1001) }, + } { + i := valid + edit(&i) + if !errors.Is(i.Validate(), ErrInvalid) { + t.Fatalf("accepted %+v", i) + } + } + valid.Text = strings.Repeat("你", 1000) + if err := valid.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestDecodePrivateInbox(t *testing.T) { + raw := []byte(`{"status":"succeeded","account_uid":"123","messages":[{"peer_uid":"456","peer_name":"好友","server_id":"1001","sender_uid":"456","message_type":"7","content":"{\"aweType\":700,\"type\":0,\"text\":\"你好\"}","created_at":"1700000000123"},{"peer_uid":"456","server_id":"1002","sender_uid":"123","message_type":"7","content":"{\"aweType\":701,\"type\":1,\"resource_url\":\"https://example.invalid\"}","created_at":null}]}`) + items, err := DecodePrivateInbox(raw, "123") + if err != nil { + t.Fatal(err) + } + if len(items) != 2 || items[0].Direction != "inbound" || items[0].Text != "你好" || items[0].MessageAt == nil || items[0].MessageAt.UnixMilli() != 1700000000123 || items[1].Direction != "outbound" || items[1].MessageType != "image" || items[1].MessageAt != nil { + t.Fatalf("items=%+v", items) + } + for _, raw := range []string{`{}`, `{"status":"failed","code":"IM_SDK_NOT_READY"}`, `{"status":"succeeded","account_uid":"999","messages":[]}`, `{"status":"succeeded","account_uid":"123"}`, `{"status":"succeeded","account_uid":"123","messages":[{"peer_uid":"456","server_id":"0","sender_uid":"456","message_type":"7","content":"{}"}]}`, `{"status":"succeeded","account_uid":"123","messages":[{"peer_uid":"456","server_id":"1","sender_uid":"777","message_type":"7","content":"{}"}]}`} { + if _, err := DecodePrivateInbox([]byte(raw), "123"); err == nil { + t.Fatalf("accepted %s", raw) + } + } +} + +func TestPrivateListenerStateDoesNotDependOnNoticeProgress(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + id := createIntegrationAccount(t, ctx, accounts, "independent-listener") + profile, err := store.GetAccountProfile(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + state, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, `ALTER TABLE creator_account_listener DROP COLUMN IF EXISTS notice_checkpoints`); err != nil { + t.Fatal(err) + } + states, err := store.ListPrivateListeners(ctx) + if err != nil || len(states) != 1 || states[0].AccountID != id || !states[0].Enabled || states[0].Generation != state.Generation { + t.Fatalf("states=%+v %v", states, err) + } +} + +func TestPreviouslyMisclassifiedNetworkFailureIsUnconfirmed(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + id := createIntegrationAccount(t, ctx, accounts, "network-reclassification") + profile, err := store.GetAccountProfile(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + if _, err := store.SetListenerEnabled(ctx, id, true); err != nil { + t.Fatal(err) + } + reservation, err := store.BeginPrivateMessage(ctx, PrivateMessageSendInput{RequestID: "old-network", AccountID: id, PeerUID: "456", Text: "你好"}) + if err != nil { + t.Fatal(err) + } + if _, err := store.FinishPrivateMessage(ctx, reservation.Message.ID, "failed", "", "发送失败：MESSAGE_REJECTED；status_code=1008", nil); err != nil { + t.Fatal(err) + } + if err := store.RecoverInterruptedPrivateMessages(ctx); err != nil { + t.Fatal(err) + } + page, err := store.ListPrivateMessages(ctx, id, "456", 1, 25) + if err != nil || page.Data[0].State != "unknown" { + t.Fatalf("network=%+v %v", page, err) + } +} + +func TestPrivateMessagesLifecycle(t *testing.T) { + store, accounts, ctx := openCreatorIntegrationStore(t) + uid := fmt.Sprint(time.Now().UnixNano()) + id := createIntegrationAccount(t, ctx, accounts, uid) + input := PrivateMessageSendInput{RequestID: "request-1", AccountID: id, PeerUID: "456", Text: "你好"} + if _, err := store.BeginPrivateMessage(ctx, input); !errors.Is(err, ErrConflict) { + t.Fatalf("unlistened send=%v", err) + } + profile, err := store.GetAccountProfile(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err := store.RecordVerifiedLoginResult(ctx, id, profile.PlatformAccountKey); err != nil { + t.Fatal(err) + } + listener, err := store.SetListenerEnabled(ctx, id, true) + if err != nil { + t.Fatal(err) + } + first, err := store.BeginPrivateMessage(ctx, input) + if err != nil || !first.New || first.Message.State != "sending" { + t.Fatalf("begin=%+v %v", first, err) + } + duplicate, err := store.BeginPrivateMessage(ctx, input) + if err != nil || duplicate.New || duplicate.Message.ID != first.Message.ID { + t.Fatalf("duplicate=%+v %v", duplicate, err) + } + changed := input + changed.Text = "不同内容" + if _, err := store.BeginPrivateMessage(ctx, changed); !errors.Is(err, ErrConflict) { + t.Fatalf("changed request=%v", err) + } + self := input + self.RequestID = "self" + self.PeerUID = profile.PlatformAccountKey + if _, err := store.BeginPrivateMessage(ctx, self); !errors.Is(err, ErrInvalid) { + t.Fatalf("self=%v", err) + } + at := time.Now().UTC() + batch := []PrivateInboxMessage{{PeerUID: "456", PeerName: "好友", ServerID: "100", Direction: "outbound", MessageType: "text", Text: "你好", MessageAt: &at}, {PeerUID: "456", PeerName: "好友", ServerID: "101", Direction: "inbound", MessageType: "text", Text: "你好呀", MessageAt: &at}} + if err := store.SavePrivateInbox(ctx, id, listener.Generation, batch); err != nil { + t.Fatal(err) + } + if err := store.SavePrivateInbox(ctx, id, listener.Generation, batch); err != nil { + t.Fatal(err) + } + finished, err := store.FinishPrivateMessage(ctx, first.Message.ID, "succeeded", "100", "", &at) + if err != nil || finished.State != "succeeded" { + t.Fatalf("finish=%+v %v", finished, err) + } + page, err := store.ListPrivateMessages(ctx, id, "456", 1, 25) + if err != nil || page.Total != 2 || len(page.Data) != 2 { + t.Fatalf("messages=%+v %v", page, err) + } + conv, err := store.ListPrivateConversations(ctx, id, 1, 25) + if err != nil || conv.Total != 1 || conv.Data[0].MessageCount != 2 { + t.Fatalf("conversations=%+v %v", conv, err) + } + if err := store.UpdatePrivateSyncStatus(ctx, id, listener.Generation, "network failed"); err != nil { + t.Fatal(err) + } + statuses, err := store.ListPrivateSyncStatus(ctx) + if err != nil || len(statuses) != 1 || statuses[0].Error != "network failed" { + t.Fatalf("status=%+v %v", statuses, err) + } + if _, err := store.SetListenerEnabled(ctx, id, false); err != nil { + t.Fatal(err) + } + if err := store.SavePrivateInbox(ctx, id, listener.Generation, batch); !errors.Is(err, ErrConflict) { + t.Fatalf("late sync=%v", err) + } + page, err = store.ListPrivateMessages(ctx, id, "456", 1, 25) + if err != nil || page.Total != 0 || page.Data == nil { + t.Fatalf("disabled=%+v %v", page, err) + } + conv, err = store.ListPrivateConversations(ctx, "", 1, 25) + if err != nil || conv.Total != 0 || conv.Data == nil { + t.Fatalf("disabled conversations=%+v %v", conv, err) + } + if _, err := store.SetListenerEnabled(ctx, id, true); err != nil { + t.Fatal(err) + } + page, err = store.ListPrivateMessages(ctx, id, "456", 1, 25) + if err != nil || page.Total != 2 { + t.Fatalf("reenabled=%+v %v", page, err) + } + _ = context.Background() +} diff --git a/internal/environment/environment.go b/internal/environment/environment.go index 3167a6f..d64ebc7 100644 --- a/internal/environment/environment.go +++ b/internal/environment/environment.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "net" "regexp" "strings" @@ -386,7 +387,10 @@ func (s *Store) DisableNetworkExit(ctx context.Context, id string) (NetworkExit, func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID string) (EnvironmentContext, bool, error) { env.Alias, env.Name = strings.TrimSpace(env.Alias), strings.TrimSpace(env.Name) - if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || !aliasPattern.MatchString(accountID) || + if env.MemoryLimitMB == 0 { + env.MemoryLimitMB = 2048 + } + if !validMemoryLimit(env.MemoryLimitMB) || !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || !aliasPattern.MatchString(accountID) || (exitID != "" && !exitIDPattern.MatchString(exitID)) || !gatewayNamePattern.MatchString(env.Gateway) || env.Fingerprint.ProxyServer != "" { return EnvironmentContext{}, false, ErrInvalid @@ -426,14 +430,14 @@ func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID s } var created string if err := tx.QueryRowContext(ctx, ` - INSERT INTO browser_env (alias, name, gateway_id, fingerprint, account_id, exit_id, version, profile_id) - SELECT $1, $2, gateway.id, jsonb_set($4::jsonb, '{seed}', to_jsonb(nextval('browser_fingerprint_seed_seq'))), account.id, network.id, 1, $1 + INSERT INTO browser_env (alias, name, gateway_id, fingerprint, account_id, exit_id, version, profile_id, memory_limit_mb) + SELECT $1, $2, gateway.id, jsonb_set($4::jsonb, '{seed}', to_jsonb(nextval('browser_fingerprint_seed_seq'))), account.id, network.id, 1, $1, $7 FROM social_account account JOIN gateway ON gateway.name = $3 LEFT JOIN network_exit network ON network.exit_id = NULLIF($6, '') WHERE account.account_id = $5 AND account.status = 'paused' AND ($6 = '' OR network.health_status = 'healthy') - RETURNING alias`, env.Alias, env.Name, env.Gateway, encoded, accountID, exitID).Scan(&created); err != nil { + RETURNING alias`, env.Alias, env.Name, env.Gateway, encoded, accountID, exitID, env.MemoryLimitMB).Scan(&created); err != nil { return EnvironmentContext{}, false, rowError(err) } if _, err := tx.ExecContext(ctx, `UPDATE social_account SET version = version + 1, updated_at = now() WHERE account_id = $1`, accountID); err != nil { @@ -463,7 +467,7 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro var cleanupBindingVersion sql.NullInt64 err = tx.QueryRowContext(ctx, ` SELECT environment.id, environment.alias, environment.name, gateway.name, - environment.fingerprint, environment.created_at, COALESCE(account.account_id, ''), COALESCE(account.status, ''), environment.profile_id, + environment.fingerprint, environment.created_at, environment.memory_limit_mb, COALESCE(account.account_id, ''), COALESCE(account.status, ''), environment.profile_id, environment.version, environment.runtime_cleanup_pending, environment.runtime_cleanup_binding_version, environment.runtime_cleanup_runtime_id, environment.runtime_cleanup_network_id, @@ -479,7 +483,7 @@ func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (Enviro JOIN gateway ON gateway.id = environment.gateway_id LEFT JOIN network_exit network ON network.id = environment.exit_id WHERE environment.alias = $1`, alias). - Scan(&result.ID, &result.Alias, &result.Name, &result.Gateway, &encoded, &result.CreatedAt, + Scan(&result.ID, &result.Alias, &result.Name, &result.Gateway, &encoded, &result.CreatedAt, &result.MemoryLimitMB, &result.AccountID, &result.AccountStatus, &result.ProfileID, &result.BindingVersion, &result.RuntimeCleanupPending, &cleanupBindingVersion, &cleanupRuntimeID, &cleanupNetworkID, @@ -524,40 +528,44 @@ func (s *Store) GetEnvironmentContextForAccount(ctx context.Context, accountID s return s.GetEnvironmentContext(ctx, alias) } -// UpdateAccountFingerprint 更新账号绑定环境的指纹参数并返回最新环境上下文。 -// seed 保持账号派生值不可改，代理由网络出口管理不可存（直连环境强制空代理）。 -func (s *Store) UpdateAccountFingerprint(ctx context.Context, accountID string, fingerprint Fingerprint) (EnvironmentContext, error) { - // 门禁对齐 CreateBoundEnv：存储层拒绝携带代理的指纹（代理由网络出口管理，传错入口直接报错）。 - if !aliasPattern.MatchString(accountID) || fingerprint.ProxyServer != "" || fingerprint.Validate() != nil { +func validMemoryLimit(limit int) bool { return limit >= 512 && limit <= 65536 } + +// UpdateEnvironmentSettings atomically saves settings without changing browser identity or runtime. +func (s *Store) UpdateEnvironmentSettings(ctx context.Context, alias string, fingerprint *Fingerprint, memoryLimitMB *int) (EnvironmentContext, error) { + if !aliasPattern.MatchString(alias) || (fingerprint != nil && (fingerprint.ProxyServer != "" || fingerprint.Validate() != nil)) || + (memoryLimitMB != nil && !validMemoryLimit(*memoryLimitMB)) { return EnvironmentContext{}, ErrInvalid } tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return EnvironmentContext{}, errors.New("begin account fingerprint update") + return EnvironmentContext{}, fmt.Errorf("begin environment settings update: %w", err) } defer tx.Rollback() - var alias string var encoded []byte + var limit int err = tx.QueryRowContext(ctx, ` - SELECT environment.alias, environment.fingerprint - FROM browser_env environment - JOIN social_account account ON account.id = environment.account_id - WHERE account.account_id = $1 - FOR UPDATE OF environment`, accountID).Scan(&alias, &encoded) + SELECT fingerprint, memory_limit_mb FROM browser_env + WHERE alias = $1 FOR UPDATE`, alias).Scan(&encoded, &limit) if err != nil { return EnvironmentContext{}, rowError(err) } - var stored Fingerprint - if err := json.Unmarshal(encoded, &stored); err != nil { - return EnvironmentContext{}, errors.New("decode environment fingerprint") + if fingerprint != nil { + var stored Fingerprint + if err := json.Unmarshal(encoded, &stored); err != nil { + return EnvironmentContext{}, fmt.Errorf("decode environment fingerprint: %w", err) + } + updated := *fingerprint + updated.Seed = stored.Seed + encoded, err = json.Marshal(updated) + if err != nil { + return EnvironmentContext{}, fmt.Errorf("encode environment fingerprint: %w", err) + } } - fingerprint.Seed = stored.Seed - updated, err := json.Marshal(fingerprint) - if err != nil { - return EnvironmentContext{}, errors.New("encode environment fingerprint") + if memoryLimitMB != nil { + limit = *memoryLimitMB } - if _, err := tx.ExecContext(ctx, `UPDATE browser_env SET fingerprint = $2 WHERE alias = $1`, alias, updated); err != nil { - return EnvironmentContext{}, errors.New("update environment fingerprint") + if _, err := tx.ExecContext(ctx, `UPDATE browser_env SET fingerprint = $2, memory_limit_mb = $3 WHERE alias = $1`, alias, encoded, limit); err != nil { + return EnvironmentContext{}, fmt.Errorf("update environment settings: %w", err) } if err := commitHub(tx); err != nil { return EnvironmentContext{}, err diff --git a/internal/environment/fingerprint_update_test.go b/internal/environment/fingerprint_update_test.go index fbfa935..85ee74b 100644 --- a/internal/environment/fingerprint_update_test.go +++ b/internal/environment/fingerprint_update_test.go @@ -37,9 +37,9 @@ func TestUpdateAccountFingerprintPreservesDerivedSeed(t *testing.T) { t.Fatalf("create bound env: created=%v err=%v", created, err) } - updated, err := store.UpdateAccountFingerprint(ctx, "fp-owner", Fingerprint{ + updated, err := store.UpdateEnvironmentSettings(ctx, "fp-owner", &Fingerprint{ Platform: "linux", Timezone: "Asia/Tokyo", Lang: "ja-JP", - }) + }, nil) if err != nil { t.Fatalf("update account fingerprint: %v", err) } @@ -70,16 +70,16 @@ func TestUpdateAccountFingerprintRejectsInvalidInput(t *testing.T) { t.Fatal(err) } - if _, err := store.UpdateAccountFingerprint(ctx, "fp-owner", Fingerprint{Platform: "android"}); !errors.Is(err, ErrInvalid) { + if _, err := store.UpdateEnvironmentSettings(ctx, "fp-owner", &Fingerprint{Platform: "android"}, nil); !errors.Is(err, ErrInvalid) { t.Fatalf("expected ErrInvalid for bad platform, got %v", err) } - if _, err := store.UpdateAccountFingerprint(ctx, "fp-owner", Fingerprint{ProxyServer: "socks5://proxy.example:1080", Timezone: "Asia/Shanghai"}); !errors.Is(err, ErrInvalid) { + if _, err := store.UpdateEnvironmentSettings(ctx, "fp-owner", &Fingerprint{ProxyServer: "socks5://proxy.example:1080", Timezone: "Asia/Shanghai"}, nil); !errors.Is(err, ErrInvalid) { t.Fatalf("stored fingerprint proxy must be rejected, got %v", err) } - if _, err := store.UpdateAccountFingerprint(ctx, "fp-owner", Fingerprint{HardwareConcurrency: -1}); !errors.Is(err, ErrInvalid) { + if _, err := store.UpdateEnvironmentSettings(ctx, "fp-owner", &Fingerprint{HardwareConcurrency: -1}, nil); !errors.Is(err, ErrInvalid) { t.Fatalf("expected ErrInvalid for bad concurrency, got %v", err) } - if _, err := store.UpdateAccountFingerprint(ctx, "missing-owner", Fingerprint{Timezone: "Asia/Shanghai"}); !errors.Is(err, ErrNotFound) { + if _, err := store.UpdateEnvironmentSettings(ctx, "missing-owner", &Fingerprint{Timezone: "Asia/Shanghai"}, nil); !errors.Is(err, ErrNotFound) { t.Fatalf("expected ErrNotFound for unbound account, got %v", err) } // 失败更新不得污染落库值。 @@ -98,9 +98,9 @@ func TestUpdateAccountFingerprintJSONRoundTrip(t *testing.T) { if _, _, err := store.CreateBoundEnv(ctx, Env{Alias: "fp-owner", Name: "fp-owner", Gateway: "gw-fp"}, "fp-owner", ""); err != nil { t.Fatal(err) } - if _, err := store.UpdateAccountFingerprint(ctx, "fp-owner", Fingerprint{ + if _, err := store.UpdateEnvironmentSettings(ctx, "fp-owner", &Fingerprint{ Brand: "Edge", AcceptLang: "zh-CN,en-US", DisableSpoofing: "font,gpu", HardwareConcurrency: 8, - }); err != nil { + }, nil); err != nil { t.Fatal(err) } updated, err := store.GetEnvironmentContextForAccount(ctx, "fp-owner") diff --git a/internal/environment/memory_limit_test.go b/internal/environment/memory_limit_test.go new file mode 100644 index 0000000..e7e0f4b --- /dev/null +++ b/internal/environment/memory_limit_test.go @@ -0,0 +1,86 @@ +package environment + +import ( + "context" + "errors" + "testing" +) + +func TestEnvironmentMemoryLimitPersistence(t *testing.T) { + ctx := context.Background() + s := seedFingerprintAccount(t, ctx, "memory-owner") + for _, limit := range []int{0, 512, 4096, 65536} { + env, err := s.CreateStandaloneEnv(ctx, "gw-fp", Fingerprint{}, limit) + if err != nil { + t.Fatal(err) + } + expected := limit + if expected == 0 { + expected = 2048 + } + got, err := s.GetEnv(ctx, env.Alias) + if err != nil || got.MemoryLimitMB != expected { + t.Fatalf("read limit: %+v %v", got, err) + } + pending, err := s.ListPendingEnvironments(ctx) + if err != nil || pending[0].MemoryLimitMB != expected { + t.Fatalf("pending: %+v %v", pending, err) + } + } + for _, limit := range []int{-1, 511, 65537} { + if _, err := s.CreateStandaloneEnv(ctx, "gw-fp", Fingerprint{}, limit); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid %d: %v", limit, err) + } + if _, _, err := s.CreateBoundEnv(ctx, Env{Alias: "memory-owner", Name: "memory", Gateway: "gw-fp", MemoryLimitMB: limit}, "memory-owner", ""); !errors.Is(err, ErrInvalid) { + t.Fatalf("bound invalid %d: %v", limit, err) + } + } + before, _, err := s.CreateBoundEnv(ctx, Env{Alias: "memory-owner", Name: "memory", Gateway: "gw-fp", MemoryLimitMB: 4096, Fingerprint: Fingerprint{Timezone: "Asia/Shanghai"}}, "memory-owner", "") + if err != nil { + t.Fatal(err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE browser_env SET runtime_id='running-memory',runtime_lease_until=now()+interval '1 hour',fingerprint=jsonb_set(fingerprint,'{hardware_concurrency}','0') WHERE alias=$1`, before.Alias); err != nil { + t.Fatal(err) + } + before, err = s.GetEnvironmentContext(ctx, before.Alias) + if err != nil { + t.Fatal(err) + } + var fingerprintJSON string + if err := s.db.QueryRowContext(ctx, `SELECT fingerprint::text FROM browser_env WHERE alias=$1`, before.Alias).Scan(&fingerprintJSON); err != nil { + t.Fatal(err) + } + limit := 8192 + after, err := s.UpdateEnvironmentSettings(ctx, before.Alias, nil, &limit) + if err != nil { + t.Fatal(err) + } + if after.MemoryLimitMB != limit || after.Fingerprint != before.Fingerprint || after.ProfileID != before.ProfileID || after.RuntimeID != before.RuntimeID || after.BindingVersion != before.BindingVersion || after.Gateway != before.Gateway { + t.Fatalf("memory update changed identity: before=%+v after=%+v", before, after) + } + var updatedJSON string + if err := s.db.QueryRowContext(ctx, `SELECT fingerprint::text FROM browser_env WHERE alias=$1`, before.Alias).Scan(&updatedJSON); err != nil { + t.Fatal(err) + } + if updatedJSON != fingerprintJSON { + t.Fatalf("memory-only edit rewrote fingerprint: before=%s after=%s", fingerprintJSON, updatedJSON) + } + for _, limit := range []int{0, 511, 65537} { + if _, err := s.UpdateEnvironmentSettings(ctx, before.Alias, &Fingerprint{Timezone: "Asia/Tokyo"}, &limit); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid update: %v", err) + } + } + got, err := s.GetEnvironmentContext(ctx, before.Alias) + if err != nil || got.Fingerprint != before.Fingerprint || got.MemoryLimitMB != 8192 { + t.Fatalf("failed update persisted: %+v %v", got, err) + } + listed, err := s.ListEnvs(ctx) + if err != nil { + t.Fatal(err) + } + for _, env := range listed { + if env.MemoryLimitMB < 512 { + t.Fatalf("list missing memory: %+v", env) + } + } +} diff --git a/internal/environment/memory_schema_upgrade_test.go b/internal/environment/memory_schema_upgrade_test.go new file mode 100644 index 0000000..daf11a6 --- /dev/null +++ b/internal/environment/memory_schema_upgrade_test.go @@ -0,0 +1,46 @@ +package environment + +import ( + "context" + "os" + "testing" +) + +func TestEnvironmentMemorySchemaUpgradeAndRepeatedStartup(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL") + } + ctx := context.Background() + url := isolatedDatabaseURL(t, databaseURL) + s := openFullyMigratedHub(t, ctx, url) + if _, err := s.CreateGateway(ctx, "memory-upgrade", "http://127.0.0.1:8090", ""); err != nil { + t.Fatal(err) + } + before, err := s.CreateStandaloneEnv(ctx, "memory-upgrade", Fingerprint{Timezone: "Asia/Shanghai"}, 4096) + if err != nil { + t.Fatal(err) + } + // Reproduce a database deployed before 1055 without touching applied scripts. + if _, err := s.db.ExecContext(ctx, `ALTER TABLE browser_env DROP COLUMN memory_limit_mb; DELETE FROM schema_migration WHERE version=1055`); err != nil { + t.Fatal(err) + } + s.Close() + for range 2 { + s = openFullyMigratedHub(t, ctx, url) + after, err := s.GetEnvironmentContext(ctx, before.Alias) + if err != nil || after.MemoryLimitMB != 2048 || after.ID != before.ID || after.ProfileID != before.ProfileID || after.Fingerprint != before.Fingerprint || after.BindingVersion != before.BindingVersion { + t.Fatalf("upgrade changed identity: %+v %v", after, err) + } + assertDatabaseCount(t, s.db, `SELECT count(*) FROM schema_migration WHERE version=1055`, 1) + for _, limit := range []int{511, 65537} { + if _, err := s.db.ExecContext(ctx, `UPDATE browser_env SET memory_limit_mb=$2 WHERE alias=$1`, before.Alias, limit); err == nil { + t.Fatalf("database accepted invalid %d", limit) + } + } + if _, err := s.db.ExecContext(ctx, `UPDATE browser_env SET memory_limit_mb=NULL WHERE alias=$1`, before.Alias); err == nil { + t.Fatal("database accepted null") + } + s.Close() + } +} diff --git a/internal/environment/migration043_probe_test.go b/internal/environment/migration043_probe_test.go index ae0173d..327b1ab 100644 --- a/internal/environment/migration043_probe_test.go +++ b/internal/environment/migration043_probe_test.go @@ -71,7 +71,8 @@ func TestMigration043ConsolidatedSchemaShape(t *testing.T) { assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'audit_event' AND column_name IN ('confirmation_id','confirmation_version','attempt_id','task_id','runtime_instance_id')`, 0) // 统一登记表包含首次登录 UID 绑定及封面静态文件迁移。 - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration`, 56) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version <= 1049`, 56) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version = 1050`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'social_account' AND column_name = 'platform_account_key' AND is_nullable = 'YES'`, 1) } diff --git a/internal/environment/migrations/1054_notice_polling.sql b/internal/environment/migrations/1054_notice_polling.sql new file mode 100644 index 0000000..b29db55 --- /dev/null +++ b/internal/environment/migrations/1054_notice_polling.sql @@ -0,0 +1,11 @@ +-- Notification list cursors are per account, per enabled generation. +-- A checkpoint and its events are committed in the same transaction. +ALTER TABLE creator_account_listener + ADD COLUMN notice_checkpoints jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN last_poll_at timestamptz, + ADD CONSTRAINT creator_listener_checkpoint_object + CHECK (jsonb_typeof(notice_checkpoints) = 'object'); +-- A boundary established by the removed push adapter cannot seed list polling. +UPDATE creator_account_listener +SET boundary_at = NULL, last_delivery_id = '', status = CASE WHEN enabled THEN 'starting' ELSE 'stopped' END, + reason = '', updated_at = now(); diff --git a/internal/environment/migrations/1055_environment_memory_limit.sql b/internal/environment/migrations/1055_environment_memory_limit.sql new file mode 100644 index 0000000..6be873d --- /dev/null +++ b/internal/environment/migrations/1055_environment_memory_limit.sql @@ -0,0 +1,3 @@ +ALTER TABLE browser_env + ADD COLUMN memory_limit_mb INTEGER NOT NULL DEFAULT 2048 + CHECK (memory_limit_mb BETWEEN 512 AND 65536); diff --git a/internal/environment/migrations/1056_event_details.sql b/internal/environment/migrations/1056_event_details.sql new file mode 100644 index 0000000..c7e3619 --- /dev/null +++ b/internal/environment/migrations/1056_event_details.sql @@ -0,0 +1,7 @@ +ALTER TABLE creator_account_event + ADD COLUMN interactor_name text NOT NULL DEFAULT '', + ADD COLUMN interactor_sec_uid text NOT NULL DEFAULT '', + ADD COLUMN work_author_uid text NOT NULL DEFAULT '', + ADD COLUMN work_cover_url text NOT NULL DEFAULT '', + ADD COLUMN work_url text NOT NULL DEFAULT '', + ADD COLUMN work_cover_error text NOT NULL DEFAULT ''; diff --git a/internal/environment/standalone.go b/internal/environment/standalone.go index bc59dff..04c0f36 100644 --- a/internal/environment/standalone.go +++ b/internal/environment/standalone.go @@ -8,8 +8,11 @@ import ( // CreateStandaloneEnv creates only a browser environment; no social account or // login profile exists until the browser's first verified login. -func (s *Store) CreateStandaloneEnv(ctx context.Context, gateway string, fingerprint Fingerprint) (EnvironmentContext, error) { - if !gatewayNamePattern.MatchString(gateway) || fingerprint.Seed != 0 || fingerprint.ProxyServer != "" || fingerprint.Validate() != nil { +func (s *Store) CreateStandaloneEnv(ctx context.Context, gateway string, fingerprint Fingerprint, memoryLimitMB int) (EnvironmentContext, error) { + if memoryLimitMB == 0 { + memoryLimitMB = 2048 + } + if !validMemoryLimit(memoryLimitMB) || !gatewayNamePattern.MatchString(gateway) || fingerprint.Seed != 0 || fingerprint.ProxyServer != "" || fingerprint.Validate() != nil { return EnvironmentContext{}, ErrInvalid } alias := "env-" + newHubID() @@ -17,7 +20,7 @@ func (s *Store) CreateStandaloneEnv(ctx context.Context, gateway string, fingerp if err != nil { return EnvironmentContext{}, fmt.Errorf("encode browser fingerprint: %w", err) } - if err := s.db.QueryRowContext(ctx, `INSERT INTO browser_env(alias,name,gateway_id,fingerprint,profile_id) SELECT $1,$1,id,jsonb_set($3::jsonb,'{seed}',to_jsonb(nextval('browser_fingerprint_seed_seq'))),$1 FROM gateway WHERE name=$2 RETURNING alias`, alias, gateway, encoded).Scan(&alias); err != nil { + if err := s.db.QueryRowContext(ctx, `INSERT INTO browser_env(alias,name,gateway_id,fingerprint,profile_id,memory_limit_mb) SELECT $1,$1,id,jsonb_set($3::jsonb,'{seed}',to_jsonb(nextval('browser_fingerprint_seed_seq'))),$1,$4 FROM gateway WHERE name=$2 RETURNING alias`, alias, gateway, encoded, memoryLimitMB).Scan(&alias); err != nil { return EnvironmentContext{}, rowError(err) } return s.GetEnvironmentContext(ctx, alias) diff --git a/internal/environment/standalone_order_test.go b/internal/environment/standalone_order_test.go index 7f5eec8..89ff5e3 100644 --- a/internal/environment/standalone_order_test.go +++ b/internal/environment/standalone_order_test.go @@ -18,11 +18,11 @@ func TestPendingEnvironmentDatabaseIDOrder(t *testing.T) { if _, err := store.CreateGateway(ctx, "order-gateway", "http://127.0.0.1:8090", "order-gateway-test-token"); err != nil { t.Fatal(err) } - first, err := store.CreateStandaloneEnv(ctx, "order-gateway", Fingerprint{}) + first, err := store.CreateStandaloneEnv(ctx, "order-gateway", Fingerprint{}, 0) if err != nil { t.Fatal(err) } - second, err := store.CreateStandaloneEnv(ctx, "order-gateway", Fingerprint{}) + second, err := store.CreateStandaloneEnv(ctx, "order-gateway", Fingerprint{}, 0) if err != nil { t.Fatal(err) } diff --git a/internal/environment/store.go b/internal/environment/store.go index fdbf6df..4b30711 100644 --- a/internal/environment/store.go +++ b/internal/environment/store.go @@ -200,6 +200,15 @@ var migration1052 string //go:embed migrations/1053_owned_public_works.sql var migration1053 string +//go:embed migrations/1054_notice_polling.sql +var migration1054 string + +//go:embed migrations/1055_environment_memory_limit.sql +var migration1055 string + +//go:embed migrations/1056_event_details.sql +var migration1056 string + var ( ErrConflict = errors.New("resource conflicts with existing state") ErrInvalid = errors.New("invalid environment input") @@ -241,6 +250,7 @@ type Env struct { Gateway string `json:"gateway"` BrowserVersion string `json:"-"` // legacy database test snapshot; runtime selection is gateway-owned Fingerprint Fingerprint `json:"fingerprint"` + MemoryLimitMB int `json:"memory_limit_mb"` CreatedAt time.Time `json:"created_at"` } @@ -341,7 +351,7 @@ func (s *Store) migrate(ctx context.Context) error { {1029, migration1029}, {1030, migration1030}, {1031, migration1031}, {1032, migration1032}, {1033, migration1033}, {1034, migration1034}, {1035, migration1035}, {1036, migration1036}, {1037, migration1037}, {1038, migration1038}, {1039, migration1039}, {1040, migration1040}, {1041, migration1041}, {1042, migration1042}, {1043, migration1043}, {1044, migration1044}, {1045, migration1045}, {1046, migration1046}, - {43, migration043}, {44, migration044}, {1047, migration1047}, {1048, migration1048}, {1049, migration1049}, {1050, migration1050}, {1051, migration1051}, {1052, migration1052}, {1053, migration1053}} { + {43, migration043}, {44, migration044}, {1047, migration1047}, {1048, migration1048}, {1049, migration1049}, {1050, migration1050}, {1051, migration1051}, {1052, migration1052}, {1053, migration1053}, {1054, migration1054}, {1055, migration1055}, {1056, migration1056}} { var applied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { return errors.New("read environment schema migration state") @@ -465,7 +475,7 @@ func (s *Store) DeleteGateway(ctx context.Context, name string) error { func (s *Store) ListEnvs(ctx context.Context) ([]Env, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT environment.id, environment.alias, environment.name, gateway.name, environment.fingerprint, environment.created_at + SELECT environment.id, environment.alias, environment.name, gateway.name, environment.fingerprint, environment.created_at, environment.memory_limit_mb FROM browser_env environment JOIN gateway ON gateway.id = environment.gateway_id ORDER BY environment.created_at, environment.alias`) @@ -489,7 +499,7 @@ func (s *Store) GetEnv(ctx context.Context, alias string) (Env, error) { return Env{}, ErrInvalid } rows, err := s.db.QueryContext(ctx, ` - SELECT environment.id, environment.alias, environment.name, gateway.name, environment.fingerprint, environment.created_at + SELECT environment.id, environment.alias, environment.name, gateway.name, environment.fingerprint, environment.created_at, environment.memory_limit_mb FROM browser_env environment JOIN gateway ON gateway.id = environment.gateway_id WHERE environment.alias = $1`, alias) @@ -549,7 +559,7 @@ func (s *Store) DeleteAccountEnvironment(ctx context.Context, accountID string) func scanEnv(rows *sql.Rows) (Env, error) { var env Env var encoded []byte - if err := rows.Scan(&env.ID, &env.Alias, &env.Name, &env.Gateway, &encoded, &env.CreatedAt); err != nil { + if err := rows.Scan(&env.ID, &env.Alias, &env.Name, &env.Gateway, &encoded, &env.CreatedAt, &env.MemoryLimitMB); err != nil { return Env{}, errors.New("decode browser env") } if len(encoded) > 0 { diff --git a/internal/platform/douyin/connector.go b/internal/platform/douyin/connector.go index 25122ab..c47456a 100644 --- a/internal/platform/douyin/connector.go +++ b/internal/platform/douyin/connector.go @@ -452,7 +452,10 @@ type worksEnvelope struct { ID string `json:"aweme_id"` Description string `json:"desc"` CreatedAt *int64 `json:"create_time"` - Author *struct { + Status struct { + PrivateStatus *int `json:"private_status"` + } `json:"status"` + Author *struct { Nickname string `json:"nickname"` } `json:"author"` Video *struct { @@ -532,7 +535,7 @@ func parseCreatorWorksPage(body []byte) ([]creatorWorkPageItem, bool, *int64, bo if createdAtInvalid { createdAt = nil } - item := creatorWorkPageItem{ID: candidate.ID, Description: candidate.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares, CollectCount: collects, PlayCount: plays} + item := creatorWorkPageItem{ID: candidate.ID, PrivateStatus: candidate.Status.PrivateStatus, Description: candidate.Description, CreatedAt: createdAt, CreatedAtInvalid: createdAtInvalid, DiggCount: likes, CommentCount: comments, ShareCount: shares, CollectCount: collects, PlayCount: plays} if candidate.Author != nil { item.AuthorName = candidate.Author.Nickname } diff --git a/internal/platform/douyin/creator_collector.go b/internal/platform/douyin/creator_collector.go index ff50503..c85bc39 100644 --- a/internal/platform/douyin/creator_collector.go +++ b/internal/platform/douyin/creator_collector.go @@ -259,21 +259,34 @@ func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor stri if !ok { return creator.WorkPage{}, fmt.Errorf("%w: invalid douyin works response (status=%d body_len=%d head=%.160s)", ErrInvalid, response.Status, len(response.Body), response.Body) } + sourceType, sourceID := c.SourceType, c.SourceID + if sourceType == "" { + sourceType = creator.SourceCompetitor + } + if sourceID == "" { + sourceID = accountKey + } items := make([]creator.WorkInput, 0, len(works)) + excludedWorkKeys := make([]string, 0) for _, work := range works { + if sourceType == creator.SourceOwned { + if work.PrivateStatus == nil { + return creator.WorkPage{}, fmt.Errorf("%w: owned douyin work %s is missing status.private_status", ErrInvalid, work.ID) + } + if *work.PrivateStatus < 0 { + return creator.WorkPage{}, fmt.Errorf("%w: owned douyin work %s has invalid status.private_status", ErrInvalid, work.ID) + } + if *work.PrivateStatus != 0 { + excludedWorkKeys = append(excludedWorkKeys, work.ID) + continue + } + } var published *time.Time if work.CreatedAt != nil && *work.CreatedAt > 0 { value := time.Unix(*work.CreatedAt, 0).UTC() published = &value } likes, comments, shares, collects, plays := work.DiggCount, work.CommentCount, work.ShareCount, work.CollectCount, work.PlayCount - sourceType, sourceID := c.SourceType, c.SourceID - if sourceType == "" { - sourceType = creator.SourceCompetitor - } - if sourceID == "" { - sourceID = accountKey - } status := "pending_verification" if work.CreatedAtInvalid { status = "invalid" @@ -282,7 +295,7 @@ func (c CreatorCollector) ListWorks(ctx context.Context, accountKey, cursor stri } items = append(items, creator.WorkInput{Platform: creator.PlatformDouyin, WorkKey: work.ID, SourceType: sourceType, SourceID: sourceID, AuthorName: work.AuthorName, Body: work.Description, PublishedAt: published, PublishedAtStatus: status, OriginalURL: "https://www.douyin.com/video/" + work.ID, CoverURL: work.CoverURL, Likes: likes, CommentsCount: comments, Shares: shares, CollectCount: collects, PlayCount: plays}) } - page := creator.WorkPage{Items: items, HasMore: hasMore} + page := creator.WorkPage{Items: items, ExcludedWorkKeys: excludedWorkKeys, HasMore: hasMore} if nextCursor != nil { page.NextCursor = strconv.FormatInt(*nextCursor, 10) } @@ -354,6 +367,7 @@ func parseCreatorCommentsPage(body []byte) (creator.CommentPage, error) { } type creatorWorkPageItem struct { + PrivateStatus *int ID string Description string AuthorName string diff --git a/internal/platform/douyin/creator_collector_test.go b/internal/platform/douyin/creator_collector_test.go index e5e31d6..e3582da 100644 --- a/internal/platform/douyin/creator_collector_test.go +++ b/internal/platform/douyin/creator_collector_test.go @@ -72,6 +72,30 @@ func TestCanonicalTargetSecUIDRejectsUniqueIDLookup(t *testing.T) { } } +func TestOwnedCreatorCollectorListWorksOnlyReturnsPublic(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":false,"max_cursor":0,"aweme_list":[{"aweme_id":"101","desc":"public","create_time":1700000000,"status":{"private_status":0}},{"aweme_id":"102","desc":"friends","create_time":1700000001,"status":{"private_status":1}},{"aweme_id":"103","desc":"private","create_time":1700000002,"status":{"private_status":2}}]}`) + browser := &collectorBrowser{response: Response{Status: 200, Body: body}} + page, err := (CreatorCollector{Browser: browser, SourceType: "owned"}).ListWorks(context.Background(), "2328120603967913", "") + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].WorkKey != "101" { + t.Fatalf("owned works page includes non-public works: %+v", page.Items) + } + if len(page.ExcludedWorkKeys) != 2 || page.ExcludedWorkKeys[0] != "102" || page.ExcludedWorkKeys[1] != "103" { + t.Fatalf("excluded work keys = %v, want [102 103]", page.ExcludedWorkKeys) + } +} + +func TestOwnedCreatorCollectorListWorksRequiresVisibilityStatus(t *testing.T) { + body := []byte(`{"status_code":0,"has_more":false,"max_cursor":0,"aweme_list":[{"aweme_id":"101","desc":"unclassified","create_time":1700000000}]}`) + browser := &collectorBrowser{response: Response{Status: 200, Body: body}} + _, err := (CreatorCollector{Browser: browser, SourceType: "owned"}).ListWorks(context.Background(), "2328120603967913", "") + if err == nil { + t.Fatal("owned works without private_status were accepted") + } +} + func TestCreatorCollectorGuardsAndCollection(t *testing.T) { ctx := context.Background() collector := CreatorCollector{} diff --git a/scripts/test_validate_douyin_event_listener.py b/scripts/test_validate_douyin_event_listener.py new file mode 100644 index 0000000..a668aa9 --- /dev/null +++ b/scripts/test_validate_douyin_event_listener.py @@ -0,0 +1,83 @@ +import importlib.util +import unittest +from unittest.mock import Mock +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("validate_douyin_event_listener.py") +spec = importlib.util.spec_from_file_location("listener_probe", MODULE_PATH) +probe = importlib.util.module_from_spec(spec) +assert spec and spec.loader +spec.loader.exec_module(probe) + + +class ListenerProbeTests(unittest.TestCase): + def test_accepts_only_explicit_local_cdp_endpoints(self): + probe.validate_arguments("http://127.0.0.1:19000", "99491952055") + probe.validate_arguments("http://localhost:9222/", "123") + for endpoint in ( + "https://127.0.0.1:19000", + "http://0.0.0.0:19000", + "http://192.168.1.5:19000", + "http://user:pass@localhost:19000", + "http://127.0.0.1:19000/json", + "http://127.0.0.1:19000?token=secret", + ): + with self.subTest(endpoint=endpoint), self.assertRaises(probe.ProbeError): + probe.validate_arguments(endpoint, "123") + + def test_validates_expected_uid_as_decimal_string(self): + for uid in ("", "0", "１２３", "abc", "1" * 21, 123): + with self.subTest(uid=uid), self.assertRaises(probe.ProbeError): + probe.validate_arguments("http://127.0.0.1:19000", uid) + + def test_classifies_readable_api_without_claiming_event_delivery(self): + self.assertEqual(probe.classify_pages([]), "no_matching_logged_in_page") + self.assertEqual(probe.classify_pages([{"matches_expected_uid": True}]), "notification_api_unavailable") + groups = [{"group":group,"http_status":200,"status_code":0,"ids_valid":True,"account_identity_matches":True} for group in (700,960,961)] + pages = [{"matches_expected_uid": True, "notice_list_read_probe": groups}] + self.assertEqual(probe.classify_pages(pages), "notification_api_readable_delivery_unverified") + pages[0]['polling_read_probe'] = {'outcome':'failed'} + self.assertEqual(probe.classify_pages(pages), 'notification_polling_failed') + pages[0]['polling_read_probe'] = {'outcome':'read_and_checkpoint_restore_verified'} + self.assertEqual(probe.classify_pages(pages), 'notification_polling_read_verified_delivery_unverified') + pages[0]['polling_read_probe']['reason'] = 'missing checkpoint' + self.assertEqual(probe.classify_pages(pages), 'notification_polling_gap') + groups[0]['account_identity_matches'] = False + self.assertEqual(probe.classify_pages(pages), "notification_api_unavailable") + self.assertEqual(probe.classify_pages([ + {"matches_expected_uid": True}, {"matches_expected_uid": True} + ]), "multiple_matching_pages") + + def test_readonly_probe_acknowledges_all_chunks_before_restoring_checkpoint(self): + session = Mock() + session.poll.side_effect = [ + [{'delivery_id':str(i), 'kind':'notice'} for i in range(100)], + [{'delivery_id':str(i), 'kind':'notice'} for i in range(100,151)] + + [{'delivery_id':'checkpoint', 'kind':'checkpoint', 'checkpoints':{'700':'1'}, 'ignored_types':{}, 'reason':''}], + ] + summary = probe.drain_readonly_session(session) + self.assertEqual(summary['kinds'], {'notice':151, 'checkpoint':1}) + self.assertEqual(session.ack.call_count, 2) + self.assertEqual(summary['checkpoint']['checkpoints'], {'700':'1'}) + session.poll.side_effect = None + session.poll.return_value = [] + with self.assertRaises(probe.ProbeError): + probe.drain_readonly_session(session) + + def test_probe_does_not_enable_listener_or_send_messages(self): + program = probe._harness_program("123") + self.assertIn("did_not_install_listener", program) + self.assertNotIn("install_expression", program) + self.assertNotIn("/douyin/events", program) + self.assertNotIn("sendMessage", program) + self.assertIn("is_mark_read", program) + self.assertNotIn("method:", program) + self.assertIn('"notice_list_read_probe"', program) + self.assertIn('response.text()', program) + self.assertNotIn('response.json()', program) + compile(program, '', 'exec') + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_douyin_event_listener.py b/scripts/validate_douyin_event_listener.py new file mode 100644 index 0000000..c9bf3a9 --- /dev/null +++ b/scripts/validate_douyin_event_listener.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Read-only notification API probe through browser-harness; not delivery acceptance.""" +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from collections import Counter +from pathlib import Path +from urllib.parse import urlsplit + +LOCAL_HOSTS = {'127.0.0.1', 'localhost', '::1'} +UID_RE = re.compile(r'^[1-9][0-9]{0,19}$') + + +class ProbeError(ValueError): + pass + + +def validate_arguments(cdp_url: str, expected_uid: str) -> None: + try: + parsed = urlsplit(cdp_url) + port = parsed.port + except ValueError as exc: + raise ProbeError('CDP 端口无效') from exc + if (parsed.scheme != 'http' or parsed.hostname not in LOCAL_HOSTS or not port + or parsed.username or parsed.password or parsed.path not in ('', '/') + or parsed.query or parsed.fragment): + raise ProbeError('只允许明确指定本机 http://127.0.0.1:端口 CDP 地址') + if not isinstance(expected_uid, str) or not UID_RE.fullmatch(expected_uid): + raise ProbeError('expected UID 必须为十进制字符串') + + +def classify_pages(pages: list[dict[str, object]]) -> str: + matched = [p for p in pages if p.get('matches_expected_uid')] + if len(matched) > 1: + return 'multiple_matching_pages' + if not matched: + return 'no_matching_logged_in_page' + groups = matched[0].get('notice_list_read_probe', []) + if (len(groups) == 3 and {g.get('group') for g in groups} == {700, 960, 961} + and all(g.get('http_status') == 200 and g.get('status_code') == 0 + and g.get('ids_valid') and g.get('account_identity_matches') for g in groups)): + polling = matched[0].get('polling_read_probe') + if polling: + if polling.get('outcome') != 'read_and_checkpoint_restore_verified': + return 'notification_polling_failed' + if polling.get('reason'): + return 'notification_polling_gap' + return 'notification_polling_read_verified_delivery_unverified' + return 'notification_api_readable_delivery_unverified' + return 'notification_api_unavailable' + + +def drain_readonly_session(session): + """Drain the local queue, including history spanning more than one batch.""" + kinds = Counter() + while True: + batch = session.poll(100, 0) + if not batch: + raise ProbeError('通知读取没有返回确认边界，不能声称历史已读完') + kinds.update(d['kind'] for d in batch) + checkpoint = next((d for d in batch if d['kind'] == 'checkpoint'), None) + session.ack([d['delivery_id'] for d in batch]) + if checkpoint is not None: + return {'kinds':dict(kinds), 'checkpoint':checkpoint} + + +def _harness_program(expected_uid: str) -> str: + # Raw JSON is decoded in Python; JavaScript would round 64-bit notice IDs. + fetch = r'''(async()=>{ + const response=await fetch(URL_VALUE,{credentials:"include",redirect:"error",cache:"no-store",signal:AbortSignal.timeout(10000)}); + return {status:response.status,body:await response.text()}; + })()''' + return f'''import json,re,sys +from collections import Counter +from urllib.parse import urlsplit,urlencode +sys.path.insert(0,{json.dumps(str(Path(__file__).resolve().parent.parent))}) +from browser_gateway.platform.douyin import BrowserResponse,DouyinBrowser +from browser_gateway.platform.notice_polling import NoticePollingSession +from scripts.validate_douyin_event_listener import drain_readonly_session +expected_uid={json.dumps(expected_uid)} +fetch_expression={json.dumps(fetch)} +reports=[] +for target in cdp("Target.getTargets")["targetInfos"]: + url=target.get("url", "") + if target.get("type")!="page" or urlsplit(url).netloc!="www.douyin.com": continue + target_id=target["targetId"] + report={{"page_path":urlsplit(url).path}} + def read(path): + response=js(fetch_expression.replace("URL_VALUE",json.dumps(path)),target_id=target_id) + return response["status"],json.loads(response["body"]) + try: + status,identity=read("/aweme/v1/web/user/profile/self/?device_platform=webapp&aid=6383&channel=channel_pc_web") + uid=str((identity.get("user") or {{}}).get("uid") or "") + report["identity_status"]="read" if status==200 and identity.get("status_code")==0 and uid else "login_required" + report["matches_expected_uid"]=report["identity_status"]=="read" and uid==expected_uid + if report["matches_expected_uid"]: + groups=[] + for group in (700,960,961): + query=urlencode(dict(device_platform="webapp",aid=6383,channel="channel_pc_web",is_new_notice=1,is_mark_read=0,count=1,min_time=0,max_time=0,notice_group=group)) + try: + status,body=read("/aweme/v1/web/notice/?"+query) + items=body.get("notice_list_v2") + valid=isinstance(items,list) + groups.append({{"group":group,"http_status":status,"status_code":body.get("status_code"), + "has_more":body.get("has_more"),"sample_count":len(items) if valid else None, + "notice_types":sorted({{str(n.get("type","unknown")) for n in items}}) if valid else [], + "ids_valid":valid and all(re.fullmatch(r"[1-9][0-9]{{0,29}}",str(n.get("nid_str") or n.get("nid") or "")) for n in items), + "account_identity_matches":valid and all(str(n.get("user_id"))==expected_uid for n in items)}}) + except Exception as exc: groups.append({{"group":group,"error":str(exc)[:240]}}) + report["notice_list_read_probe"]=groups + class ReadOnlyBrowser: + identity=DouyinBrowser.identity + def get(self,alias,url): + raw=js(fetch_expression.replace("URL_VALUE",json.dumps(url)),target_id=target_id) + return BrowserResponse(raw["status"],raw["body"],False) + sessions=[] + try: + browser=ReadOnlyBrowser() + session=NoticePollingSession(browser,"readonly",expected_uid) + sessions.append(session) + first=drain_readonly_session(session) + restored=NoticePollingSession(browser,"readonly",expected_uid,session.boundary_at,session.checkpoints) + sessions.append(restored) + second=drain_readonly_session(restored) + report["polling_read_probe"]={{"outcome":"read_and_checkpoint_restore_verified", + "first_kinds":first["kinds"], "restored_kinds":second["kinds"], + "checkpoint_groups":list(second["checkpoint"]["checkpoints"]), + "ignored_types":first["checkpoint"]["ignored_types"], + "reason":second["checkpoint"]["reason"]}} + except Exception as exc: + report["polling_read_probe"]={{"outcome":"failed","error":str(exc)[:240]}} + finally: + for session in sessions: session.stop() + except Exception as exc: + report.update(identity_status="probe_error",error=str(exc)[:240]) + reports.append(report) +print("CREATORHUB_LISTENER_PROBE="+json.dumps({{"expected_uid":expected_uid,"pages":reports, + "read_only":True,"did_not_install_listener":True}},ensure_ascii=False)) +''' + + +def run_probe(cdp_url: str, expected_uid: str, harness_name: str) -> dict[str, object]: + validate_arguments(cdp_url, expected_uid) + executable = shutil.which('browser-harness') + if not executable: + raise ProbeError('找不到 browser-harness；请按 docs/research/douyin-event-listener.md 安装') + env = os.environ.copy() + env.update(BU_CDP_URL=cdp_url, BU_NAME=harness_name, BH_RECORD='0') + try: + process = subprocess.run([executable], input=_harness_program(expected_uid), text=True, + capture_output=True, check=False, timeout=90, env=env) + except subprocess.TimeoutExpired as exc: + raise ProbeError('browser-harness 超时；未切换页面或启动监听') from exc + if process.returncode: + detail = process.stderr.strip().splitlines() + raise ProbeError(detail[-1][:400] if detail else 'browser-harness 连接失败') + for line in reversed(process.stdout.splitlines()): + if line.startswith('CREATORHUB_LISTENER_PROBE='): + report = json.loads(line.split('=', 1)[1]) + report['outcome'] = classify_pages(report['pages']) + report['checked_at'] = datetime.now(timezone.utc).isoformat() + return report + raise ProbeError('browser-harness 未返回诊断结果') + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cdp-url', required=True, help='该账号浏览器的本机 CDP 地址') + parser.add_argument('--expected-uid', required=True, help='该浏览器绑定的抖音 UID') + parser.add_argument('--harness-name', default='creatorhub-event-listener-readonly') + args = parser.parse_args(argv) + try: + report = run_probe(args.cdp_url, args.expected_uid, args.harness_name) + except (ProbeError, json.JSONDecodeError) as exc: + print(json.dumps({'outcome':'probe_failed','error':str(exc)},ensure_ascii=False),file=sys.stderr) + return 2 + print(json.dumps(report,ensure_ascii=False,indent=2)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/web/package-lock.json b/web/package-lock.json index 36f7e49..fafef02 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -9,6 +9,7 @@ "@ant-design/charts": "^2.6.7", "@ant-design/icons": "^6.3.4", "@ant-design/pro-components": "^3.1.14-7", + "@ant-design/x": "2.9.0", "@umijs/max": "^4.7.19", "antd": "^6.6.5", "react": "^19.3.0", @@ -265,6 +266,36 @@ "react-dom": "^0.14.0 || ^15.0.1 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/@ant-design/x": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@ant-design/x/-/x-2.9.0.tgz", + "integrity": "sha512-m3N2nprrPG/D0n93B/fBFhjpvSaJAb2IDF6w0rX0ycYXSGxtgrJQvqSI33Jhx6gxUHj495zdIzqA1NMFld9iQA==", + "license": "MIT", + "dependencies": { + "@ant-design/colors": "^8.0.0", + "@ant-design/cssinjs": "^2.0.1", + "@ant-design/cssinjs-utils": "^2.0.2", + "@ant-design/fast-color": "^3.0.0", + "@ant-design/icons": "^6.0.0", + "@babel/runtime": "^7.25.6", + "@rc-component/motion": "^1.1.6", + "@rc-component/resize-observer": "^1.0.1", + "@rc-component/util": "^1.4.0", + "clsx": "^2.1.1", + "lodash.throttle": "^4.1.1", + "mermaid": "^11.12.1", + "react-syntax-highlighter": "^16.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ant-design" + }, + "peerDependencies": { + "antd": "^6.1.1", + "react": ">=18.0.0", + "react-dom": ">=18.0.0" + } + }, "node_modules/@antfu/install-pkg": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-0.1.1.tgz", @@ -1284,6 +1315,12 @@ "integrity": "sha512-h0OYmPR3A5Dfbetra/GzxBAzQk8sH7LhRkRUTdagX6nrtlUgJGYCTv4bBK33jsTQw9HDd8PE2x1Ma+iRKEDUsw==", "license": "Apache-2.0" }, + "node_modules/@braintree/sanitize-url": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz", + "integrity": "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==", + "license": "MIT" + }, "node_modules/@chenshuai2144/sketch-color": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@chenshuai2144/sketch-color/-/sketch-color-1.0.9.tgz", @@ -1297,6 +1334,12 @@ "react": ">=16.12.0" } }, + "node_modules/@chevrotain/types": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.1.2.tgz", + "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", + "license": "Apache-2.0" + }, "node_modules/@csstools/postcss-color-function": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@csstools/postcss-color-function/-/postcss-color-function-1.1.1.tgz", @@ -3135,6 +3178,15 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" }, + "node_modules/@mermaid-js/parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.2.1.tgz", + "integrity": "sha512-n12NohV3mrUyUL2o93IgG/ifeW9FTyeJn3zDxkhwa8MJ9Fxg3HQMlA3RiGmD/3UnJvheztkjjQAjA2T4LmUcpw==", + "license": "MIT", + "dependencies": { + "@chevrotain/types": "~11.1.2" + } + }, "node_modules/@module-federation/error-codes": { "version": "0.8.12", "resolved": "https://registry.npmjs.org/@module-federation/error-codes/-/error-codes-0.8.12.tgz", @@ -5551,24 +5603,111 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, "node_modules/@types/d3-array": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", "license": "MIT" }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT" + }, "node_modules/@types/d3-color": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", "license": "MIT" }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT" + }, "node_modules/@types/d3-dispatch": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", "license": "MIT" }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, "node_modules/@types/d3-dsv": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", @@ -5632,6 +5771,12 @@ "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", "license": "MIT" }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT" + }, "node_modules/@types/d3-quadtree": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", @@ -5659,6 +5804,12 @@ "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", "license": "MIT" }, + "node_modules/@types/d3-selection": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.12.tgz", + "integrity": "sha512-Qe/KWYhEiIIxGs7HrAAjMfShxKldx19SJtr5zu53f3afPsdZNz7HHtdTLXo/kqeiWNXVycI24kSnfzBYkTzpgw==", + "license": "MIT" + }, "node_modules/@types/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.2.0.tgz", @@ -5674,12 +5825,37 @@ "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", "license": "MIT" }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT" + }, "node_modules/@types/d3-timer": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", "license": "MIT" }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.9.tgz", + "integrity": "sha512-0sE1406XBYJGiqD3AusTl9ZqC//2mIXix51tbom25gDCA8ri4xnSZg28CaSE8Srl6FClqABUKfsn0qghgdepMA==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -5707,6 +5883,15 @@ "integrity": "sha512-oOMFT8vmCTFncsF1engrs04jatz8/Anwx3De9uxnOK4chgSEgWBvFtpSoJo8u3784JNO+ql5tzRR6phHoRnscQ==", "license": "MIT" }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, "node_modules/@types/history": { "version": "4.7.11", "resolved": "https://registry.npmjs.org/@types/history/-/history-4.7.11.tgz", @@ -5802,6 +5987,12 @@ "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", "license": "MIT" }, + "node_modules/@types/prismjs": { + "version": "1.26.6", + "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.6.tgz", + "integrity": "sha512-vqlvI7qlMvcCBbVe0AKAb4f97//Hy0EBTaiW8AalRnG/xAN5zOiWWyrNqNXeq8+KAuvRewjCVY1+IPxk4RdNYw==", + "license": "MIT" + }, "node_modules/@types/react": { "version": "19.3.0", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", @@ -5875,6 +6066,19 @@ "integrity": "sha512-VgDNokpBoKF+wrdvhAAfS55OMQpL6QRglwTwNC3kIgBrzZxA4WsFj+2eLfEA/uMUDzBcEhYmjSbwQakn/i3ajA==", "license": "MIT" }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, "node_modules/@types/use-sync-external-store": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", @@ -9471,6 +9675,16 @@ "integrity": "sha512-e91nN/1YQDSR8eck99GyY4QIZ5eVcZ+BeaKTW2zei7hoIthyIT1gWVtC1jeThD3hmq3A2wdw9uu26Eu+DB6D0Q==", "license": "MIT" }, + "node_modules/@upsetjs/venn.js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@upsetjs/venn.js/-/venn.js-2.0.0.tgz", + "integrity": "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw==", + "license": "MIT", + "optionalDependencies": { + "d3-selection": "^3.0.0", + "d3-transition": "^3.0.1" + } + }, "node_modules/@utoo/pack": { "version": "1.5.18", "resolved": "https://registry.npmjs.org/@utoo/pack/-/pack-1.5.18.tgz", @@ -11187,6 +11401,36 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -11425,6 +11669,16 @@ "integrity": "sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==", "license": "Apache-2.0" }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/commander": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", @@ -11633,6 +11887,15 @@ "url": "https://opencollective.com/express" } }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "license": "MIT", + "dependencies": { + "layout-base": "^1.0.0" + } + }, "node_modules/cosmiconfig": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz", @@ -11981,6 +12244,54 @@ "integrity": "sha512-qv8s+G47V6Hq+g2kRE5th+ASzzrL7b6l+tap1DHKK25ZQJv3yIFhH96XaQ7NGL+zRW3t/RDbweJf/dJDe5Z5KA==", "license": "MIT" }, + "node_modules/cytoscape": { + "version": "3.34.3", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.3.tgz", + "integrity": "sha512-yfYGhRcGAntq6YBD583j4n0Eg3jIxvWmZtz/5uz9UYkeIStSlMxuUja+ec5j3iBD8nv1rwaOAYMW09tBdkSeaQ==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^2.2.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/cose-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", + "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", + "license": "MIT", + "dependencies": { + "layout-base": "^2.0.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/layout-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", + "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", + "license": "MIT" + }, "node_modules/d": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/d/-/d-1.0.2.tgz", @@ -11994,6 +12305,47 @@ "node": ">=0.12" } }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-array": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", @@ -12006,12 +12358,49 @@ "node": ">=12" } }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/d3-binarytree": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/d3-binarytree/-/d3-binarytree-1.0.2.tgz", "integrity": "sha512-cElUNH+sHu95L04m92pG73t2MEJXKu+GeKUN1TJkFsu93E5W8E9Sc3kHEGJKgenGvj19m6upSn2EunvMgMD2Yw==", "license": "MIT" }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-color": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", @@ -12021,6 +12410,30 @@ "node": ">=12" } }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-dispatch": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", @@ -12030,6 +12443,19 @@ "node": ">=12" } }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-dsv": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", @@ -12214,6 +12640,15 @@ "node": ">=12" } }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/d3-quadtree": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", @@ -12238,6 +12673,46 @@ "integrity": "sha512-PF8GWEL70cHHWpx2jUQXc68r1pyPHIA+St16muk/XRokETzlegj5LriNKg7o4LR0TySug4nHYPJNNRz/W+/Niw==", "license": "BSD-3-Clause" }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC" + }, "node_modules/d3-scale": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", @@ -12267,6 +12742,15 @@ "node": ">=12" } }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", @@ -12312,6 +12796,41 @@ "node": ">=12" } }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/dagre": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/dagre/-/dagre-0.8.5.tgz", @@ -12322,6 +12841,16 @@ "lodash": "^4.17.15" } }, + "node_modules/dagre-d3-es": { + "version": "7.0.14", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.14.tgz", + "integrity": "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg==", + "license": "MIT", + "dependencies": { + "d3": "^7.9.0", + "lodash-es": "^4.17.21" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -12455,6 +12984,19 @@ "node": ">=0.10.0" } }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/decode-uri-component": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", @@ -12666,6 +13208,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delaunator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", + "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -13026,6 +13577,15 @@ "node": ">= 10" } }, + "node_modules/dompurify": { + "version": "3.4.16", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.16.tgz", + "integrity": "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, "node_modules/domutils": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", @@ -13508,6 +14068,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es-toolkit": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.52.0.tgz", + "integrity": "sha512-XTNEJQh1tY1ZJVcf6ayP/2n4ZPyaHlW2FWs7xvw5ddPuhUVjLD3olQVQS7kf58JbAB48iL0uL/jerTrjtV3lDA==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks", + "tests/types", + "tests/browser-compat" + ] + }, "node_modules/es5-ext": { "version": "0.10.64", "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", @@ -14274,6 +14846,15 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fastdom": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fastdom/-/fastdom-1.0.12.tgz", + "integrity": "sha512-LB+xjSTEbjHE1cWsxu+tN2Xqr1kpi+V9aADI7sVM5ZMaXyYGPHULQMzpJMYqOTULK/73pUkWVzzObFRBkPr+hg==", + "license": "MIT", + "dependencies": { + "strictdom": "^1.0.1" + } + }, "node_modules/fastest-levenshtein": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", @@ -14292,6 +14873,19 @@ "reusify": "^1.0.4" } }, + "node_modules/fault": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", + "integrity": "sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==", + "license": "MIT", + "dependencies": { + "format": "^0.2.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -14610,6 +15204,14 @@ "node": ">= 6" } }, + "node_modules/format": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", + "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -15115,6 +15717,12 @@ "lodash": "^4.17.15" } }, + "node_modules/hachure-fill": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", + "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", + "license": "MIT" + }, "node_modules/handle-thing": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", @@ -15246,6 +15854,36 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.1.tgz", + "integrity": "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -15255,6 +15893,21 @@ "he": "bin/he" } }, + "node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/highlightjs-vue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/highlightjs-vue/-/highlightjs-vue-1.0.0.tgz", + "integrity": "sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==", + "license": "CC0-1.0" + }, "node_modules/history": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/history/-/history-5.3.0.tgz", @@ -15629,6 +16282,16 @@ "node": ">=8" } }, + "node_modules/import-meta-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", + "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -15743,6 +16406,30 @@ "node": ">= 0.10" } }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-any-array": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-3.0.0.tgz", @@ -15922,6 +16609,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-docker": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", @@ -16053,6 +16750,16 @@ "node": ">=0.10.0" } }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-inside-container": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", @@ -16769,6 +17476,22 @@ "node": ">=4.0" } }, + "node_modules/katex": { + "version": "0.16.47", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", + "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -16778,6 +17501,11 @@ "json-buffer": "3.0.1" } }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -16808,6 +17536,12 @@ "integrity": "sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==", "license": "MIT" }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", + "license": "MIT" + }, "node_modules/less": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/less/-/less-4.1.3.tgz", @@ -17264,6 +17998,20 @@ "tslib": "^2.0.3" } }, + "node_modules/lowlight": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/lowlight/-/lowlight-1.20.0.tgz", + "integrity": "sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==", + "license": "MIT", + "dependencies": { + "fault": "^1.0.0", + "highlight.js": "~10.7.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -17327,6 +18075,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/marked": { + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", + "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -17455,6 +18215,60 @@ "node": ">= 8" } }, + "node_modules/mermaid": { + "version": "11.17.2", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.17.2.tgz", + "integrity": "sha512-V6K3C8EBdEsPFZXSKMJe6ppQOENxuHARr9GvHX4hh47lAbhMRD9qf4oEK7LoaRQxULMa80/qt5gHO73aCleBBg==", + "license": "MIT", + "dependencies": { + "@braintree/sanitize-url": "^7.1.2", + "@iconify/utils": "^3.0.2", + "@mermaid-js/parser": "^1.2.1", + "@types/d3": "^7.4.3", + "@upsetjs/venn.js": "^2.0.0", + "cytoscape": "^3.34.0", + "cytoscape-cose-bilkent": "^4.1.0", + "cytoscape-fcose": "^2.2.0", + "d3": "^7.9.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.14", + "dayjs": "^1.11.21", + "dompurify": "^3.3.3", + "es-toolkit": "^1.45.1", + "fastdom": "1.0.12", + "katex": "^0.16.47", + "khroma": "^2.1.0", + "marked": "^16.3.0", + "roughjs": "^4.6.6", + "stylis": "^4.3.6", + "ts-dedent": "^2.2.0", + "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" + } + }, + "node_modules/mermaid/node_modules/@antfu/install-pkg": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-2.1.0.tgz", + "integrity": "sha512-sdg9NxU3zR4Mnawfbc/x6GB5Wf17WYud5qOuEuxXjaKpYpMkISSJEjItGebXJ2bQ4DIcly4NYH23mtkGJjvKUw==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.8.0", + "tinyexec": "^1.3.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/mermaid/node_modules/@iconify/utils": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.1.7.tgz", + "integrity": "sha512-JZHlwdID+dy+lTgbYC8NEC4zeugqeYsc6jewvzb4c58kHauJn+X7rNwQjxz5p2qSjqaEeQoLkCIQ9v/H4PK0/w==", + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^2.0.1", + "@iconify/types": "^2.0.0", + "import-meta-resolve": "^4.2.0" + } + }, "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -18456,6 +19270,12 @@ "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", "license": "BlueOak-1.0.0" }, + "node_modules/package-manager-detector": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.9.0.tgz", + "integrity": "sha512-zAMfbta9nPA8AKkA4WdmNAT8R8s6Dk0/xwPsqNnRidMFcLo1YkLuPDTgs2f64XdzS8vbLNnoJ+uUIkgbz7oRHA==", + "license": "MIT" + }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -18500,6 +19320,31 @@ "node": ">= 0.10" } }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -18552,6 +19397,12 @@ "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==", "license": "MIT" }, + "node_modules/path-data-parser": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", + "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", + "license": "MIT" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -18739,6 +19590,22 @@ "integrity": "sha512-3ojrFwjnnw8Q9242TzgXuTD+eKiutbzyslcq1ydfu82Db2y+Ogbmyrkpv0Hgj31qwT3lbS9+QAAO/pIQM35XRw==", "license": "MIT" }, + "node_modules/points-on-curve": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", + "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", + "license": "MIT" + }, + "node_modules/points-on-path": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", + "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", + "license": "MIT", + "dependencies": { + "path-data-parser": "0.1.0", + "points-on-curve": "0.2.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -19707,6 +20574,15 @@ "renderkid": "^3.0.0" } }, + "node_modules/prismjs": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", + "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/probe-image-size": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/probe-image-size/-/probe-image-size-7.4.0.tgz", @@ -19804,6 +20680,16 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" }, + "node_modules/property-information": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", + "integrity": "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/proxy-addr": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", @@ -20869,6 +21755,26 @@ "react": ">=15" } }, + "node_modules/react-syntax-highlighter": { + "version": "16.1.1", + "resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-16.1.1.tgz", + "integrity": "sha512-PjVawBGy80C6YbC5DDZJeUjBmC7skaoEUdvfFQediQHgCL7aKyVHe57SaJGfQsloGDac+gCpTfRdtxzWWKmCXA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.28.4", + "highlight.js": "^10.4.1", + "highlightjs-vue": "^1.0.0", + "lowlight": "^1.17.0", + "prismjs": "^1.30.0", + "refractor": "^5.0.0" + }, + "engines": { + "node": ">= 16.20.2" + }, + "peerDependencies": { + "react": ">= 0.14.0" + } + }, "node_modules/reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", @@ -21114,6 +22020,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/refractor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/refractor/-/refractor-5.0.0.tgz", + "integrity": "sha512-QXOrHQF5jOpjjLfiNk5GFnWhRXvxjUVnlFxkeDmewR5sXkr3iM46Zo+CnRR8B+MDVqkULW4EcLVcRBNOPXHosw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/prismjs": "^1.0.0", + "hastscript": "^9.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/regenerate": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", @@ -21369,6 +22291,12 @@ "node": ">= 0.8" } }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense" + }, "node_modules/rollup": { "version": "4.63.4", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.4.tgz", @@ -21465,6 +22393,18 @@ "node": ">= 12" } }, + "node_modules/roughjs": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", + "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", + "license": "MIT", + "dependencies": { + "hachure-fill": "^0.5.2", + "path-data-parser": "^0.1.0", + "points-on-curve": "^0.2.0", + "points-on-path": "^0.2.1" + } + }, "node_modules/run-applescript": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-5.0.0.tgz", @@ -22188,6 +23128,16 @@ "node": ">=0.10.0" } }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/spdx-correct": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", @@ -22397,6 +23347,12 @@ "node": ">=4" } }, + "node_modules/strictdom": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strictdom/-/strictdom-1.0.1.tgz", + "integrity": "sha512-cEmp9QeXXRmjj/rVp9oyiqcvyocWab/HaoN4+bwFeZ7QzykJD6L3yD4v12K1x0tHpqRqVpJevN3gW7kyM39Bqg==", + "license": "MIT" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -23235,6 +24191,15 @@ "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", "license": "MIT" }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -23354,6 +24319,15 @@ "node": ">=8" } }, + "node_modules/ts-dedent": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", + "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", + "license": "MIT", + "engines": { + "node": ">=6.10" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -23790,6 +24764,19 @@ "base64-arraybuffer": "^1.0.2" } }, + "node_modules/uuid": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/v8-compile-cache": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.4.0.tgz", diff --git a/web/package.json b/web/package.json index 93b2fc8..8a929df 100644 --- a/web/package.json +++ b/web/package.json @@ -10,6 +10,7 @@ "@ant-design/charts": "^2.6.7", "@ant-design/icons": "^6.3.4", "@ant-design/pro-components": "^3.1.14-7", + "@ant-design/x": "2.9.0", "@umijs/max": "^4.7.19", "antd": "^6.6.5", "react": "^19.3.0", diff --git a/web/src/components/accounts/AccountCreateModal.tsx b/web/src/components/accounts/AccountCreateModal.tsx index 29987bf..2aef6d0 100644 --- a/web/src/components/accounts/AccountCreateModal.tsx +++ b/web/src/components/accounts/AccountCreateModal.tsx @@ -1,11 +1,12 @@ import { useEffect, useState } from 'react'; -import { Alert, Button, Collapse, Form, Modal, Select, Typography } from 'antd'; +import { Alert, Button, Collapse, Form, InputNumber, Modal, Select, Typography } from 'antd'; import { creatorCreate, creatorGet } from '@/services/api'; import { conflictMessage } from '@/utils/helpers'; import FingerprintFields, { fingerprintPayload, type FingerprintValues } from '@/components/accounts/FingerprintFields'; interface Values { gateway: string; + memory_limit_mb: number; fingerprint?: FingerprintValues; } @@ -42,6 +43,7 @@ export default function AccountCreateModal({ onClose, onSuccess }: { onClose: () try { const environment = await creatorCreate('/creator/environments', { gateway: values.gateway, + memory_limit_mb: values.memory_limit_mb, fingerprint: fingerprintPayload(values.fingerprint), }); if (!environment?.alias) throw new Error('创建环境未返回环境标识'); @@ -72,6 +74,11 @@ export default function AccountCreateModal({ onClose, onSuccess }: { onClose: () updateFilters({ accountId })} + options={accounts.map((account) => ({ value: account.id, label: account.name || '待登录' }))} /> + updateFilters({ hours })} + options={[{ value: 0, label: '全部时间' }, { value: 1, label: '最近 1 小时' }, { value: 6, label: '最近 6 小时' }, + { value: 24, label: '最近 1 天' }, { value: 72, label: '最近 3 天' }, { value: 168, label: '最近 7 天' }]} /> + + 仅展示已开启监听账号的事件；停止监听的账号不在此列表展示，历史记录仍保留。时间范围按接收时间计算，仅记录事件，不执行自动互动。私信在独立收件箱展示。 + {error ? : null} + rowKey="id" columns={columns} dataSource={events} loading={pending} sticky={sticky} + scroll={{ x: 'max-content' }} + locale={{ emptyText: '尚未收到事件。请在“我的账号”开启监听，并保持账号浏览器运行。' }} + pagination={{ ...tablePagination(), current: pagination.page, pageSize: pagination.pageSize, total, + onChange: (page, pageSize) => setPagination({ page: pageSize !== pagination.pageSize ? 1 : page, pageSize }) }} /> + + ); +} diff --git a/web/src/pages/creator/messages/index.tsx b/web/src/pages/creator/messages/index.tsx new file mode 100644 index 0000000..9e21b4f --- /dev/null +++ b/web/src/pages/creator/messages/index.tsx @@ -0,0 +1,401 @@ +import { Bubble, Conversations, Sender } from '@ant-design/x'; +import { Alert, App, Button, Card, Empty, Flex, Form, Input, Modal, Select, Spin, Splitter, Tooltip, Typography } from 'antd'; +import { CheckCircleOutlined, CloseCircleOutlined, LoadingOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons'; +import { useEffect, useRef, useState } from 'react'; +import { creatorCreate, creatorGet } from '@/services/api'; +import { conflictMessage, dateTime } from '@/utils/helpers'; +import { + clearSentDraft, conversationKey, conversationQuery, mergeConversations, mergeMessages, + messagesQuery, messageStateLabel, messageTypeLabel, updateDraft, +} from '@/utils/private-messages'; +import type { ChatDrafts, Conversation, PrivateMessage } from '@/utils/private-messages'; + +type Account = { id: string; name: string; platform_account_key: string; listener_enabled: boolean }; +type SyncStatus = { account_id: string; account_name: string; error: string }; +type Page = { data: T[]; total: number }; +type SendValues = { account_id: string; peer_uid: string; text: string }; +type SendAttempt = { request_id: string; text: string; state: string; error: string; pending: boolean; version: number; messageId?: string }; +const PAGE_SIZE = 25; +const MESSAGE_PAGE_SIZE = 50; +const HISTORY_HINT = '仅展示聊天客户端已加载会话的最近 50 条消息，不代表完整历史；已发送不代表对方已收到或已读。'; +const accountName = (name?: string) => name?.trim() || '未命名账号'; +const STATE_ICONS: Record = { + succeeded: CheckCircleOutlined, sending: LoadingOutlined, + failed: CloseCircleOutlined, unknown: QuestionCircleOutlined, +}; + +async function readPage(path: string): Promise> { + const result = await creatorGet(path) as Page; + if (!Array.isArray(result?.data) || !Number.isFinite(result.total) || result.total < 0) { + throw new Error('私信列表返回格式错误'); + } + return result; +} + +export default function PrivateMessagesPage() { + const { message } = App.useApp(); + const [accounts, setAccounts] = useState([]); + const accountsRef = useRef([]); + const [statuses, setStatuses] = useState([]); + const [accountId, setAccountId] = useState(); + const [contacts, setContacts] = useState([]); + const [contactTotal, setContactTotal] = useState(0); + const [contactError, setContactError] = useState(''); + const [contactLoading, setContactLoading] = useState(false); + const [moreContacts, setMoreContacts] = useState(false); + const contactScope = useRef(0); + const contactPages = useRef(1); + const contactsBusy = useRef(false); + const newConversation = useRef(undefined); + const [activeKey, setActiveKey] = useState(); + const activeKeyRef = useRef(undefined); + activeKeyRef.current = activeKey; + const active = contacts.find((contact) => conversationKey(contact) === activeKey); + const [messages, setMessages] = useState([]); + const [messageTotal, setMessageTotal] = useState(0); + const [messageError, setMessageError] = useState(''); + const [messageLoading, setMessageLoading] = useState(false); + const [moreMessages, setMoreMessages] = useState(false); + const messageScope = useRef(0); + const messagePages = useRef(1); + const messagesBusy = useRef(false); + const messageRevision = useRef(0); + const [drafts, setDrafts] = useState({}); + const draftsRef = useRef({}); + const [attempts, setAttempts] = useState>({}); + const attemptsRef = useRef>({}); + const inFlight = useRef(new Set()); + const mounted = useRef(true); + const [sendOpen, setSendOpen] = useState(false); + const [sendForm] = Form.useForm(); + const [newChatError, setNewChatError] = useState(''); + + function changeDraft(key: string, text: string) { + draftsRef.current = updateDraft(draftsRef.current, key, text); + setDrafts(draftsRef.current); + } + + function saveAttempt(key: string, attempt: SendAttempt) { + const previous = attemptsRef.current[key] || []; + attemptsRef.current = { + ...attemptsRef.current, + [key]: [...previous.filter((item) => item.request_id !== attempt.request_id), attempt], + }; + setAttempts(attemptsRef.current); + } + + function clearDraft(key: string, version: number) { + draftsRef.current = clearSentDraft(draftsRef.current, key, version); + setDrafts(draftsRef.current); + } + + function reconcileAttempts(key: string, rows: PrivateMessage[]) { + for (const attempt of attemptsRef.current[key] || []) { + if (attempt.pending || attempt.state === 'succeeded' || attempt.state === 'failed') continue; + const row = rows.find((item) => item.id === attempt.messageId); + if (!row) continue; + saveAttempt(key, { ...attempt, state: row.state, error: row.error }); + if (row.state === 'succeeded') clearDraft(key, attempt.version); + } + } + + useEffect(() => { + mounted.current = true; + return () => { mounted.current = false; }; + }, []); + + useEffect(() => { + const scope = ++contactScope.current; + let stopped = false; + let timer: ReturnType; + contactPages.current = 1; + contactsBusy.current = false; + setMoreContacts(false); + const created = newConversation.current; + newConversation.current = undefined; + setContacts(created ? [created] : []); + setContactTotal(0); + setContactError(''); + setContactLoading(true); + setActiveKey(created ? conversationKey(created) : undefined); + async function refresh() { + try { + const [accountRows, statusRows] = await Promise.all([ + creatorGet('/creator/accounts') as Promise, + creatorGet('/creator/private-messages/status') as Promise, + ]); + if (!Array.isArray(accountRows) || !Array.isArray(statusRows)) throw new Error('私信账号状态返回格式错误'); + if (stopped) return; + accountsRef.current = accountRows; + setAccounts(accountRows); + setStatuses(statusRows); + const pages = await Promise.all(Array.from({ length: contactPages.current }, (_, index) => + readPage(`/creator/private-messages/conversations?${conversationQuery({ accountId }, index + 1, PAGE_SIZE)}`))); + if (stopped) return; + setContacts((previous) => mergeConversations(previous, pages.flatMap((page) => page.data))); + setContactTotal(pages[0].total); + setContactError(''); + } catch (cause) { + console.error('private-messages contacts read failed', { account_id: accountId, pages: contactPages.current, scope }); + if (!stopped) setContactError(conflictMessage(cause, '私信会话加载失败')); + } finally { + if (!stopped) { + setContactLoading(false); + timer = setTimeout(() => { void refresh(); }, 5000); + } + } + } + void refresh(); + return () => { stopped = true; clearTimeout(timer); }; + }, [accountId]); + + useEffect(() => { + if (!activeKey && contacts.length) setActiveKey(conversationKey(contacts[0])); + }, [activeKey, contacts]); + + useEffect(() => { + const scope = ++messageScope.current; + let stopped = false; + let timer: ReturnType; + messagePages.current = 1; + messagesBusy.current = false; + setMoreMessages(false); + setMessages([]); + setMessageTotal(0); + setMessageError(''); + if (!active) { setMessageLoading(false); return; } + const selected = active; + const key = conversationKey(selected); + setMessageLoading(true); + async function refresh() { + const revision = messageRevision.current; + try { + const pages = await Promise.all(Array.from({ length: messagePages.current }, (_, index) => + readPage(`/creator/private-messages/messages?${messagesQuery(selected.account_id, selected.peer_uid, index + 1, MESSAGE_PAGE_SIZE)}`))); + if (stopped || activeKeyRef.current !== key || revision !== messageRevision.current) return; + const rows = pages.flatMap((page) => page.data); + setMessages((previous) => mergeMessages(previous, rows)); + setMessageTotal(pages[0].total); + reconcileAttempts(key, rows); + setMessageError(''); + } catch (cause) { + console.error('private-messages messages read failed', { account_id: selected.account_id, peer_uid: selected.peer_uid, pages: messagePages.current, scope }); + if (!stopped && activeKeyRef.current === key) setMessageError(conflictMessage(cause, '私信消息加载失败')); + } finally { + if (!stopped) { + setMessageLoading(false); + timer = setTimeout(() => { void refresh(); }, 5000); + } + } + } + void refresh(); + return () => { stopped = true; clearTimeout(timer); }; + }, [activeKey]); + + async function loadMoreContacts() { + if (contactsBusy.current) return; + const scope = contactScope.current; + const page = contactPages.current + 1; + contactsBusy.current = true; + setMoreContacts(true); + try { + const result = await readPage(`/creator/private-messages/conversations?${conversationQuery({ accountId }, page, PAGE_SIZE)}`); + if (scope !== contactScope.current || !mounted.current) return; + setContacts((previous) => mergeConversations(result.data, previous)); + setContactTotal(result.total); + contactPages.current = page; + setContactError(''); + } catch (cause) { + console.error('private-messages more contacts failed', { account_id: accountId, page, scope }); + if (scope === contactScope.current && mounted.current) setContactError(conflictMessage(cause, '更多联系人加载失败')); + } finally { + if (scope === contactScope.current && mounted.current) { contactsBusy.current = false; setMoreContacts(false); } + } + } + + async function loadEarlierMessages() { + if (!active || messagesBusy.current) return; + const selected = active; + const key = conversationKey(selected); + const scope = messageScope.current; + const page = messagePages.current + 1; + messagesBusy.current = true; + setMoreMessages(true); + try { + const result = await readPage(`/creator/private-messages/messages?${messagesQuery(selected.account_id, selected.peer_uid, page, MESSAGE_PAGE_SIZE)}`); + if (scope !== messageScope.current || activeKeyRef.current !== key || !mounted.current) return; + // Older pages add missing rows; they must not replace fresher polling or send results. + setMessages((previous) => mergeMessages(result.data, previous)); + setMessageTotal(result.total); + messagePages.current = page; + reconcileAttempts(key, result.data); + setMessageError(''); + } catch (cause) { + console.error('private-messages earlier messages failed', { account_id: selected.account_id, peer_uid: selected.peer_uid, page, scope }); + if (scope === messageScope.current && mounted.current) setMessageError(conflictMessage(cause, '更早消息加载失败')); + } finally { + if (scope === messageScope.current && mounted.current) { messagesBusy.current = false; setMoreMessages(false); } + } + } + + async function sendText(selected: Conversation, value: string) { + const key = conversationKey(selected); + const text = value.trim(); + if (!text || inFlight.current.has(key)) return; + if (text.length > 2000) { message.error('文字内容不能超过 2000 字'); return; } + if (!accountsRef.current.some((account) => account.id === selected.account_id && account.listener_enabled)) { + message.error('该账号未开启监听，无法发送私信'); + return; + } + // An unresolved submission keeps its original request ID; editing a draft never resends it. + if ((attemptsRef.current[key] || []).some((attempt) => attempt.text === text && attempt.state !== 'succeeded' && attempt.state !== 'failed')) return; + const request_id = crypto.randomUUID(); + const snapshot: SendAttempt = { request_id, text, version: draftsRef.current[key]?.version ?? 0, state: 'sending', error: '', pending: true }; + inFlight.current.add(key); + saveAttempt(key, snapshot); + try { + const result = await creatorCreate('/creator/private-messages/send', { + request_id, account_id: selected.account_id, peer_uid: selected.peer_uid, text, + }) as PrivateMessage; + if (!mounted.current) return; + if (!result?.id || !result.state) throw new Error('发送接口缺少消息或发送状态，结果未确认'); + saveAttempt(key, { ...snapshot, state: result.state, error: result.error, messageId: result.id, pending: false }); + if (activeKeyRef.current === key) { + messageRevision.current++; + setMessages((previous) => mergeMessages(previous, [result])); + } + if (result.state === 'succeeded') clearDraft(key, snapshot.version); + if (result.state === 'failed' || result.state === 'unknown') { + console.error('private-messages send result', { account_id: selected.account_id, peer_uid: selected.peer_uid, request_id, state: result.state }); + } + } catch (cause) { + const status = (cause as { status?: number; response?: { status?: number } })?.status + ?? (cause as { response?: { status?: number } })?.response?.status; + console.error('private-messages send failed', { account_id: selected.account_id, peer_uid: selected.peer_uid, request_id, status }); + if (mounted.current) saveAttempt(key, { ...snapshot, pending: false, state: status && status >= 400 && status < 500 ? 'failed' : 'unknown', error: conflictMessage(cause, '发送结果未确认，请核对聊天记录，不会自动重发') }); + } finally { + inFlight.current.delete(key); + } + } + + async function createConversation(values: SendValues) { + const selectedAccount = accountsRef.current.find((account) => account.id === values.account_id && account.listener_enabled); + if (!selectedAccount) { setNewChatError('该账号未开启监听，无法发送私信'); return; } + const conversation: Conversation = contacts.find((contact) => contact.account_id === selectedAccount.id && contact.peer_uid === values.peer_uid.trim()) || { + account_id: selectedAccount.id, account_name: selectedAccount.name, peer_uid: values.peer_uid.trim(), + peer_name: '', last_text: '', last_message_at: '', message_count: 0, + }; + const key = conversationKey(conversation); + setContacts((previous) => mergeConversations(previous, [conversation])); + changeDraft(key, values.text); + setActiveKey(key); + if (accountId && accountId !== selectedAccount.id) { + newConversation.current = conversation; + setAccountId(selectedAccount.id); + } + setSendOpen(false); + sendForm.resetFields(); + await sendText(conversation, values.text); + } + + const enabledAccounts = accounts.filter((account) => account.listener_enabled); + const accountOptions = enabledAccounts.map((account) => ({ value: account.id, label: accountName(account.name) })); + const selectedAccount = accounts.find((account) => account.id === active?.account_id); + const canSend = !!selectedAccount?.listener_enabled; + const activeAttempts = activeKey ? attempts[activeKey] || [] : []; + const visibleContacts = contacts.filter((contact) => (!accountId || contact.account_id === accountId) + && enabledAccounts.some((account) => account.id === contact.account_id)); + + return ( + + + + {contactTotal} 个私信会话 + + + {HISTORY_HINT} 仅手动发送文字，不自动回复或群发。 + {contactError && } + {statuses.filter((status) => status.error).map((status) => )} + + + + + + + + + } + + ); +} diff --git a/web/src/utils/account-collection-status.ts b/web/src/utils/account-collection-status.ts new file mode 100644 index 0000000..a1ce8a9 --- /dev/null +++ b/web/src/utils/account-collection-status.ts @@ -0,0 +1,41 @@ +export type CollectionCheckpoint = { + status?: string; + last_completed_at?: string | null; + last_error?: string | null; +}; + +const labels: Record = { + idle: { text: '待采集', color: 'default' }, + pending: { text: '待采集', color: 'default' }, + succeeded: { text: '采集完成', color: 'success' }, + running: { text: '采集运行中', color: 'processing' }, + failed: { text: '采集失败', color: 'error' }, + blocked: { text: '采集受阻', color: 'warning' }, +}; + +export function collectionStatusPresentation(checkpoint?: CollectionCheckpoint | null) { + const reason = checkpoint?.last_error || ''; + if (reason.includes('browser is awaiting user login')) { + return { + text: '需要登录', + color: 'warning', + reason: '浏览器需要登录，请登录并同步账号后重试。', + }; + } + const state = checkpoint?.status ? labels[checkpoint.status] : undefined; + return { ...(state || { text: '状态未知', color: 'default' }), reason }; +} + +const accountStatusPriority = ['需要登录', '采集失败', '采集受阻', '状态未知', '采集运行中', '待采集', '采集完成']; + +export function accountCollectionStatusPresentation( + works?: CollectionCheckpoint | null, + comments?: CollectionCheckpoint | null, +) { + const states = [works, comments].map(collectionStatusPresentation); + const primary = states.reduce((current, candidate) => + accountStatusPriority.indexOf(current.text) <= accountStatusPriority.indexOf(candidate.text) ? current : candidate, + ); + const reasons = [...new Set(states.map(state => state.reason).filter(Boolean))]; + return { ...primary, reason: reasons.join('；') }; +} diff --git a/web/src/utils/account-events.ts b/web/src/utils/account-events.ts new file mode 100644 index 0000000..d9b6b55 --- /dev/null +++ b/web/src/utils/account-events.ts @@ -0,0 +1,45 @@ +export const eventTypeLabels: Record = { + like: '点赞', comment: '评论', follow: '关注', repost: '转发', +}; + +export function eventTypeLabel(type: string): string { + return eventTypeLabels[type] ?? type; +} + +export function listenerStatusLabel(status: string): string { + const labels: Record = { + stopped: '已关闭', starting: '启动中', ready: '接收正常', gap: '可能有遗漏', stopping: '关闭中', error: '异常', + }; + return labels[status] ?? status; +} + +export function messageTypeLabel(type: string): string { + const labels: Record = { + image: '图片消息', voice: '语音消息', video: '视频消息', sticker: '表情消息', non_text: '非文本消息', text: '—', + }; + return labels[type] ?? type; +} + +export function eventUserProfileURL(secUID?: string): string | undefined { + return secUID && /^[A-Za-z0-9_-]{1,256}$/.test(secUID) + ? `https://www.douyin.com/user/${encodeURIComponent(secUID)}` : undefined; +} + +export function eventWorkURL(workID: string, provided?: string): string | undefined { + if (!/^[1-9][0-9]{0,29}$/.test(workID)) return undefined; + if (provided) return new RegExp(`^https://www\\.douyin\\.com/(video|note)/${workID}$`).test(provided) ? provided : undefined; + return `https://www.douyin.com/video/${workID}`; +} + +export interface EventFilters { + accountId?: string; + eventType?: string; + hours: number; +} + +export function eventQuery(filters: EventFilters, page: number, pageSize: number): string { + const query = new URLSearchParams({ page: String(page), page_size: String(pageSize), hours: String(filters.hours) }); + if (filters.accountId) query.set('account_id', filters.accountId); + if (filters.eventType) query.set('event_type', filters.eventType); + return query.toString(); +} diff --git a/web/src/utils/helpers.ts b/web/src/utils/helpers.ts index bbbd3c9..49e27c9 100644 --- a/web/src/utils/helpers.ts +++ b/web/src/utils/helpers.ts @@ -2,7 +2,9 @@ export function dateTime(value?: string | number | null): string { if (!value) return '—'; const date = new Date(value); - return Number.isNaN(date.getTime()) ? String(value) : date.toLocaleString(); + if (Number.isNaN(date.getTime())) return String(value); + const twoDigits = (part: number) => String(part).padStart(2, '0'); + return `${String(date.getFullYear()).slice(-2)}/${twoDigits(date.getMonth() + 1)}/${twoDigits(date.getDate())} ${twoDigits(date.getHours())}:${twoDigits(date.getMinutes())}:${twoDigits(date.getSeconds())}`; } // 优先展示后端具体 reason；409/503 冲突类错误即使无 hint 也直接透出后端 @@ -58,7 +60,7 @@ export function extractShareURL(value?: string): string { return candidates?.find((candidate) => platformForShareURL(candidate)) || candidates?.[0] || ''; } -// 账号可恢复性判定（对齐归档版 accountReadiness）：绑定、出口健康、账号暂停三层。 +// 分别判定采集调度、环境启动和账号恢复：未启动阻止调度，但不能阻止启动。 export const readinessReasonText: Record = { account_paused: '账号已暂停', binding_missing: '未绑定运行环境', @@ -75,28 +77,30 @@ export interface AccountReadiness { label: string; reason: string; canResume: boolean; + canStart: boolean; ready: boolean; } export function accountReadiness(account: any, binding: any, bindingError: any = null): AccountReadiness { - if (bindingError) return { label: '环境状态未知', reason: 'environment_unavailable', canResume: false, ready: false }; - if (!binding) return { label: '未绑定运行环境', reason: 'binding_missing', canResume: false, ready: false }; + if (bindingError) return { label: '环境状态未知', reason: 'environment_unavailable', canResume: false, canStart: false, ready: false }; + if (!binding) return { label: '未绑定运行环境', reason: 'binding_missing', canResume: false, canStart: false, ready: false }; let blocked = ''; if (binding.runtime_cleanup_pending) { blocked = 'runtime_stop_pending'; } else if (binding.network_exit_id && binding.network_exit_health !== 'healthy') { blocked = 'network_exit_unhealthy'; } - if (blocked) return { label: readinessReasonText[blocked], reason: blocked, canResume: false, ready: false }; + if (blocked) return { label: readinessReasonText[blocked], reason: blocked, canResume: false, canStart: false, ready: false }; if (account.runtime_status === 'paused') { const stopped = !binding.runtime_id; - return { label: stopped ? '资源就绪，可恢复' : '等待运行环境停止', reason: stopped ? '' : 'runtime_stop_pending', canResume: stopped, ready: false }; + return { label: stopped ? '资源就绪，可恢复' : '等待运行环境停止', reason: stopped ? '' : 'runtime_stop_pending', canResume: stopped, canStart: false, ready: false }; } const schedule = binding.schedule_block_reason || ''; return { label: schedule ? readinessReasonText[schedule] || `不可调度：${schedule}` : '可调度', reason: schedule, canResume: false, + canStart: !schedule || schedule === 'runtime_missing', ready: !schedule, }; } diff --git a/web/src/utils/metadata.tsx b/web/src/utils/metadata.tsx index c6e693a..0b86d63 100644 --- a/web/src/utils/metadata.tsx +++ b/web/src/utils/metadata.tsx @@ -5,6 +5,8 @@ import { EyeOutlined, LineChartOutlined, CommentOutlined, + BellOutlined, + MessageOutlined, SettingOutlined, ProfileOutlined, ClusterOutlined, @@ -23,6 +25,8 @@ const rules: Array<{ test: (path: string) => boolean; title: string; subtitle: s { test: (p) => p.startsWith('/accounts/'), title: '账号详情', subtitle: '查看账号状态、登录身份与运行环境绑定。' }, { test: (p) => p === '/creator/competitors', title: '作品分析', subtitle: '分别查看竞品与自有账号作品，按账号、发布时间和表现筛选排序。' }, { test: (p) => p === '/creator/comments', title: '评论聚合', subtitle: '分别查看我的作品与竞品作品的已采集评论，按所属账号和评论发布时间筛选。' }, + { test: (p) => p === '/creator/events', title: '事件聚合', subtitle: '查看已开启账号的点赞、评论、关注与转发通知；私信在独立收件箱展示。' }, + { test: (p) => p === '/creator/messages', title: '私信管理', subtitle: '查看并回复已监听账号的文字私信会话。' }, { test: (p) => p === '/creator/settings', title: '采集设置', subtitle: '统一配置采集窗口、指标采集和已批准的服务。' }, { test: (p) => p.startsWith('/creator/competitors/'), title: '竞品账号详情', subtitle: '查看竞品画像与作品列表，点击作品可追溯指标统计。' }, { test: (p) => p === '/network-exits', title: '网络出口', subtitle: '维护固定出口、健康观测与账号绑定。' }, @@ -42,6 +46,8 @@ export const menu: MenuDataItem[] = [ { path: '/accounts/monitoring', name: '监控账号', icon: }, { path: '/creator/competitors', name: '作品分析', icon: }, { path: '/creator/comments', name: '评论聚合', icon: }, + { path: '/creator/events', name: '事件聚合', icon: }, + { path: '/creator/messages', name: '私信管理', icon: }, { path: '/creator/settings', name: '采集设置', icon: }, ], }, diff --git a/web/src/utils/private-messages.ts b/web/src/utils/private-messages.ts new file mode 100644 index 0000000..3800296 --- /dev/null +++ b/web/src/utils/private-messages.ts @@ -0,0 +1,126 @@ +export interface Conversation { + account_id: string; + account_name: string; + peer_uid: string; + peer_name: string; + last_text: string; + last_message_at: string; + message_count: number; +} + +export interface PrivateMessage { + id: string; + account_id: string; + peer_uid: string; + direction: string; + message_type: string; + text: string; + state: string; + error: string; + message_at: string; + created_at: string; +} + +export type ChatDrafts = Record; + +export function conversationKey(conversation: Pick): string { + return `${conversation.account_id}:${conversation.peer_uid}`; +} + +function timestamp(value: string): number { + const time = Date.parse(value); + return Number.isFinite(time) ? time : 0; +} + +export function mergeConversations(previous: Conversation[], incoming: Conversation[]): Conversation[] { + const merged = new Map(previous.map((conversation) => [conversationKey(conversation), conversation])); + const seen = new Set(); + for (const conversation of incoming) { + const key = conversationKey(conversation); + if (!seen.has(key)) merged.set(key, conversation); + seen.add(key); + } + return [...merged.values()].sort((a, b) => timestamp(b.last_message_at) - timestamp(a.last_message_at) + || conversationKey(a).localeCompare(conversationKey(b))); +} + +export function mergeMessages(previous: PrivateMessage[], incoming: PrivateMessage[]): PrivateMessage[] { + const merged = new Map(previous.map((message) => [message.id, message])); + const seen = new Set(); + for (const message of incoming) { + if (!seen.has(message.id)) merged.set(message.id, message); + seen.add(message.id); + } + return [...merged.values()].sort((a, b) => timestamp(a.message_at || a.created_at) - timestamp(b.message_at || b.created_at) + || a.id.localeCompare(b.id, undefined, { numeric: true })); +} + +export function updateDraft(drafts: ChatDrafts, key: string, text: string): ChatDrafts { + return { ...drafts, [key]: { text, version: (drafts[key]?.version ?? 0) + 1 } }; +} + +export function clearSentDraft(drafts: ChatDrafts, key: string, version: number): ChatDrafts { + if (!drafts[key] || drafts[key].version !== version) return drafts; + return updateDraft(drafts, key, ''); +} + +export interface MessageFilters { + accountId?: string; +} + +export function conversationQuery(filters: MessageFilters, page: number, pageSize: number): string { + const query = new URLSearchParams({ page: String(page), page_size: String(pageSize) }); + if (filters.accountId) query.set('account_id', filters.accountId); + return query.toString(); +} + +export function messagesQuery(accountId: string, peerUid: string, page: number, pageSize: number): string { + const query = new URLSearchParams({ + account_id: accountId, + peer_uid: peerUid, + page: String(page), + page_size: String(pageSize), + }); + return query.toString(); +} + +export const messageDirectionLabels: Record = { + inbound: '收到', + outbound: '发出', +}; + +export function messageDirectionLabel(direction: string): string { + return messageDirectionLabels[direction] ?? direction; +} + +export const messageStateLabels: Record = { + succeeded: '已发送', + sending: '发送中', + failed: '发送失败', + unknown: '结果未确认', +}; + +export const messageStateTones: Record = { + succeeded: 'success', + sending: 'processing', + failed: 'error', + unknown: 'warning', +}; + +export function messageStateLabel(state: string): string { + return messageStateLabels[state] ?? state; +} + +export function messageStateTone(state: string): string { + return messageStateTones[state] ?? 'default'; +} + +export const messageTypeLabels: Record = { + text: '文字', + image: '图片', + unknown: '未知类型', +}; + +export function messageTypeLabel(type: string): string { + return messageTypeLabels[type] ?? type; +} diff --git a/web/tests/account-actions.test.cjs b/web/tests/account-actions.test.cjs new file mode 100644 index 0000000..79b78d7 --- /dev/null +++ b/web/tests/account-actions.test.cjs @@ -0,0 +1,136 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const { Module } = require('node:module'); +const test = require('node:test'); +const ts = require('typescript'); + +function compile(file, imports = {}) { + const loaded = new Module(file, module); + loaded.require = id => imports[id] || require(id); + loaded._compile(ts.transpileModule(readFileSync(file, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, file); + return loaded.exports; +} +const collection = compile(resolve(__dirname, '../src/utils/account-collection-status.ts')); +const running = { alias: 'environment', state: 'running', status: 'ready', gateway_reachable: true }; +const stopped = { ...running, state: 'runtime_missing', status: 'stopped' }; +const account = { id: 'account', login_status: 'logged_in', runtime_status: 'active', environment: { alias: 'environment', runtime_id: 'old-record' } }; + +function setup({ current = account, snapshots = [running], result = { status: 'logged_in', account_id: 'account' }, actionError } = {}) { + const calls = []; + const queue = [...snapshots]; + const api = { + creatorGet: async path => { + calls.push(['get', path]); + if (path === '/phase-a/accounts/account') return current; + if (path === '/browsers/environment') return queue.length > 1 ? queue.shift() : queue[0]; + throw new Error('Unexpected request: ' + path); + }, + creatorAction: async (path, body) => { + calls.push(['post', path, body]); + if (actionError) throw actionError; + if (path === '/creator/accounts/account/verify') return result; + if (path === '/browsers/environment/start' || path === '/phase-a/accounts/account/start') return {}; + throw new Error('Unexpected action: ' + path); + }, + }; + return { calls, ...compile(resolve(__dirname, '../src/components/accounts/account-actions.ts'), { + '@/services/api': api, '@/utils/account-collection-status': collection, + }) }; +} + +for (const [row, runtime, expected] of [ + [{ pending_environment: true }, undefined, ['login', '登录并同步账号']], + [{ ...account, runtime_status: 'paused' }, running, ['resume', '恢复运行']], + [account, undefined, ['check', '检查状态']], + [account, { ...running, gateway_reachable: false }, ['check', '检查状态']], + [account, { ...running, state: 'recorded' }, ['check', '检查状态']], + [account, { ...running, cleanup_pending: true }, ['check', '检查状态']], + [account, stopped, ['start', '启动环境']], + [account, running, ['sync', '同步资料']], + [{ ...account, login_status: 'manual_login' }, running, ['login', '重新登录']], + [{ ...account, works_collection: { status: 'blocked', last_error: 'browser is awaiting user login' } }, running, ['login', '重新登录']], +]) { + test(`primary action ${expected.join('/')} uses actual browser state`, () => { + const s = setup(); + assert.deepEqual(s.accountPrimaryAction(row, runtime), { key: expected[0], label: expected[1] }); + }); +} + +test('latest successful verification overrides an older login-required collection result', () => { + const s = setup(); + assert.equal(s.accountPrimaryAction({ ...account, works_collection: { status: 'blocked', last_error: 'browser is awaiting user login' } }, running, 'logged_in').key, 'sync'); +}); + +test('normal sync checks live state and verifies identity without restarting or requesting QR', async () => { + const s = setup(); + assert.equal((await s.performAccountPrimaryAction('account', 'sync')).status, 'logged_in'); + assert.deepEqual(s.calls.map(call => call.slice(0, 2)), [ + ['get', '/phase-a/accounts/account'], ['get', '/browsers/environment'], ['post', '/creator/accounts/account/verify'], + ]); +}); + +test('login-required result is explicit; other failures must not be disguised as QR login', async () => { + const s = setup({ result: { status: 'manual_login', account_id: 'account', reason: 'awaiting_login' } }); + assert.equal((await s.performAccountPrimaryAction('account', 'login')).status, 'manual_login'); + assert.ok(s.calls.every(call => !call[1].includes('/qr'))); + await assert.rejects(setup({ actionError: new Error('identity UID mismatch') }).performAccountPrimaryAction('account', 'sync'), /UID mismatch/); + await assert.rejects(setup({ result: { status: 'unknown' } }).performAccountPrimaryAction('account', 'sync'), /未识别/); + await assert.rejects(setup({ result: { status: 'manual_login', reason: 'platform_restricted', account_id: 'account' } }).performAccountPrimaryAction('account', 'login'), /未识别/); +}); + +test('starting is not reported successful until the gateway reports a running browser', async () => { + const s = setup({ snapshots: [stopped, stopped] }); + await assert.rejects(s.performAccountPrimaryAction('account', 'start'), /启动尚未完成/); + assert.ok(!s.calls.some(call => call[1].endsWith('/verify'))); +}); + +test('starting a stopped browser does not restore or overwrite the account', async () => { + const s = setup({ snapshots: [stopped, running] }); + await s.performAccountPrimaryAction('account', 'start'); + assert.deepEqual(s.calls.filter(call => call[0] === 'post').map(call => call[1]), ['/browsers/environment/start', '/creator/accounts/account/verify']); +}); + +test('starting an already running browser only verifies it', async () => { + const s = setup(); + await s.performAccountPrimaryAction('account', 'start'); + assert.ok(!s.calls.some(call => call[1] === '/browsers/environment/start')); +}); + +test('restoring a paused account is explicit and uses the dedicated account start route', async () => { + const s = setup({ current: { ...account, runtime_status: 'paused' } }); + await s.performAccountPrimaryAction('account', 'resume'); + assert.equal(s.calls[1][1], '/phase-a/accounts/account/start'); + assert.ok(!s.calls.some(call => call[1] === '/browsers/environment/start')); +}); + +test('stale login or sync buttons never implicitly restore a paused account', async () => { + for (const intent of ['sync', 'login', 'start']) { + const s = setup({ current: { ...account, runtime_status: 'paused' } }); + await assert.rejects(s.performAccountPrimaryAction('account', intent), /先恢复运行/); + assert.equal(s.calls.length, 1); + } +}); + +test('a stopped browser is checked without mutation and stale sync does not start it', async () => { + const s = setup({ snapshots: [stopped] }); + assert.equal((await s.performAccountPrimaryAction('account', 'check')).status, 'stopped'); + assert.ok(s.calls.every(call => call[0] === 'get')); + await assert.rejects(setup({ snapshots: [stopped] }).performAccountPrimaryAction('account', 'sync'), /启动环境/); +}); + +test('gateway failure, wrong alias and unexpected browser state stop the action', async () => { + for (const runtime of [{ ...running, gateway_reachable: false }, { ...running, alias: 'other' }, { ...running, state: 'recorded' }, { ...running, cleanup_pending: true }]) { + const s = setup({ snapshots: [runtime] }); + await assert.rejects(s.performAccountPrimaryAction('account', 'start')); + assert.ok(s.calls.every(call => call[0] === 'get')); + } +}); + +test('wrong account responses, missing binding and changed verification identity fail visibly', async () => { + await assert.rejects(setup({ current: { ...account, id: 'other' } }).performAccountPrimaryAction('account', 'sync'), /账号/); + await assert.rejects(setup({ current: { ...account, environment: null } }).performAccountPrimaryAction('account', 'sync'), /环境/); + await assert.rejects(setup({ result: { status: 'logged_in', account_id: 'other' } }).performAccountPrimaryAction('account', 'sync'), /账号/); +}); diff --git a/web/tests/account-business-status.test.cjs b/web/tests/account-business-status.test.cjs index a2186ea..7a25ae3 100644 --- a/web/tests/account-business-status.test.cjs +++ b/web/tests/account-business-status.test.cjs @@ -61,8 +61,8 @@ test('业务状态列紧邻现有状态列，运行状态判定保持不变', () const titles = columns.map((column) => property(column, 'title')?.text); const index = titles.indexOf('业务状态'); assert.ok(index > 0, '必须有独立的业务状态列'); - assert.equal(titles[index - 1], '状态'); - assert.equal(titles[index + 1], '操作'); + assert.equal(titles[index - 1], '调度状态'); + assert.equal(titles[index + 1], '采集状态'); const readiness = property(columns[index - 1], 'render').getText(source); assert.match(readiness, /accountReadiness\(account, account\.environment \?\? null\)/); }); diff --git a/web/tests/account-collection-status.test.cjs b/web/tests/account-collection-status.test.cjs new file mode 100644 index 0000000..d10d60e --- /dev/null +++ b/web/tests/account-collection-status.test.cjs @@ -0,0 +1,93 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const { Module } = require('node:module'); +const ts = require('typescript'); + +const file = path.join(__dirname, '../src/utils/account-collection-status.ts'); +const loaded = new Module(file, module); +loaded.filename = file; +loaded.paths = module.paths; +loaded._compile(ts.transpileModule(fs.readFileSync(file, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, +}).outputText, file); +const { collectionStatusPresentation, accountCollectionStatusPresentation } = loaded.exports; + +for (const [status, text, color] of [ + ['idle', '待采集', 'default'], + ['pending', '待采集', 'default'], + ['running', '采集运行中', 'processing'], + ['succeeded', '采集完成', 'success'], + ['failed', '采集失败', 'error'], + ['blocked', '采集受阻', 'warning'], +]) { + test(`collection status ${status} has an explicit label`, () => { + assert.deepEqual(collectionStatusPresentation({ status }), { text, color, reason: '' }); + }); +} + +test('login errors are visible in Chinese and never render healthy', () => { + for (const status of ['blocked', 'failed', 'succeeded']) { + const result = collectionStatusPresentation({ status, last_error: 'browser is awaiting user login' }); + assert.equal(result.text, '需要登录'); + assert.equal(result.color, 'warning'); + assert.equal(result.reason, '浏览器需要登录，请登录并同步账号后重试。'); + } +}); + +test('unknown errors retain their original information', () => { + const reason = 'Douyin response HTTP 502: unexpected identity payload'; + assert.equal(collectionStatusPresentation({ status: 'blocked', last_error: reason }).reason, reason); +}); + +test('missing and unrecognized states are unknown, never completed or pending', () => { + for (const checkpoint of [undefined, null, {}, { status: 'unrecognized' }]) { + assert.equal(collectionStatusPresentation(checkpoint).text, '状态未知'); + assert.equal(collectionStatusPresentation(checkpoint).color, 'default'); + } +}); + +for (const [left, right, text] of [ + ['succeeded', 'succeeded', '采集完成'], + ['succeeded', 'pending', '待采集'], + ['idle', 'idle', '待采集'], + ['pending', 'running', '采集运行中'], + ['running', 'blocked', '采集受阻'], + ['blocked', 'failed', '采集失败'], + ['failed', 'running', '采集失败'], + ['succeeded', 'unknown', '状态未知'], + ['running', 'unknown', '状态未知'], +]) { + test(`combined collection state prioritizes ${text} for ${left}/${right}, independent of source order`, () => { + assert.equal(accountCollectionStatusPresentation({ status: left }, { status: right }).text, text); + assert.equal(accountCollectionStatusPresentation({ status: right }, { status: left }).text, text); + }); +} + +test('combined login-required state takes priority and preserves all distinct errors', () => { + const result = accountCollectionStatusPresentation( + { status: 'blocked', last_error: 'browser is awaiting user login' }, + { status: 'failed', last_error: 'network HTTP 502' }, + ); + assert.deepEqual(result, { + text: '需要登录', color: 'warning', + reason: '浏览器需要登录，请登录并同步账号后重试。；network HTTP 502', + }); +}); + +test('combined errors are deduplicated without adding source labels or times', () => { + const result = accountCollectionStatusPresentation( + { status: 'failed', last_error: 'network HTTP 502', last_completed_at: '2026-10-06T16:47:46Z' }, + { status: 'blocked', last_error: 'network HTTP 502', last_completed_at: '2026-10-06T15:00:00Z' }, + ); + assert.equal(result.text, '采集失败'); + assert.equal(result.reason, 'network HTTP 502'); + assert.doesNotMatch(JSON.stringify(result), /作品|评论|2026-10-06/); +}); + +test('missing collection data must not become a combined healthy state', () => { + assert.equal(accountCollectionStatusPresentation().text, '状态未知'); + assert.equal(accountCollectionStatusPresentation({ status: 'succeeded' }).text, '状态未知'); + assert.equal(accountCollectionStatusPresentation(undefined, { status: 'succeeded' }).text, '状态未知'); +}); diff --git a/web/tests/account-create-modal.test.cjs b/web/tests/account-create-modal.test.cjs index 4049b38..f9ead50 100644 --- a/web/tests/account-create-modal.test.cjs +++ b/web/tests/account-create-modal.test.cjs @@ -13,7 +13,7 @@ function setup({ gateways = [{ name: 'gw-a' }, { name: 'gw-b' }], loadError, res const marker = name => Object.assign(() => null, { displayName: name }); const Form = Object.assign(marker('Form'), { Item: marker('Form.Item'), useForm: () => [{ submit() { calls.push('submit'); } }] }); const antd = { Form }; - for (const name of ['Modal', 'Alert', 'Button', 'Collapse', 'Select']) antd[name] = marker(name); + for (const name of ['Modal', 'Alert', 'Button', 'Collapse', 'Select', 'InputNumber']) antd[name] = marker(name); antd.Typography = { Paragraph: marker('Paragraph'), Link: marker('Link') }; const fingerprint = new Module(resolve(__dirname, '../src/components/accounts/FingerprintFields.tsx'), module); fingerprint.require = id => id === 'antd' ? antd : requireWeb(id); @@ -48,16 +48,16 @@ function setup({ gateways = [{ name: 'gw-a' }, { name: 'gw-b' }], loadError, res }; let tree = render(); const cleanup = () => effects.forEach(effect => effect.cleanup()); - const find = (name, node = tree) => { + const find = (name, node = tree, predicate = () => true) => { if (!node || typeof node !== 'object') return; - if (node.type?.displayName === name) return node; - for (const child of [node.props?.children].flat(Infinity)) { if (!child) continue; const found = find(name, child); if (found) return found; } + if (node.type?.displayName === name && predicate(node)) return node; + for (const child of [node.props?.children].flat(Infinity)) { if (!child) continue; const found = find(name, child, predicate); if (found) return found; } }; return { calls, successes, closes, cleanup, find, render() { tree = render(); return tree; } }; } const flush = async () => { for (let i = 0; i < 8; i++) await Promise.resolve(); }; -test('网关必选、指纹默认折叠；创建只提交网关和可选指纹', async () => { +test('网关必选、指纹默认折叠；创建提交网关、内存上限和可选指纹', async () => { const s = setup(); await flush(); s.render(); assert.deepEqual(s.calls[0], { load: '/gateways' }); assert.deepEqual(s.find('Select').props.options.map(x => x.value), ['gw-a', 'gw-b']); @@ -66,15 +66,15 @@ test('网关必选、指纹默认折叠；创建只提交网关和可选指纹', assert.equal(s.find('Collapse').props.defaultActiveKey, undefined); s.find('Modal').props.onOk(); assert.equal(s.calls.at(-1), 'submit'); - await s.find('Form').props.onFinish({ gateway: 'gw-b', fingerprint: { timezone: ' Asia/Shanghai ', disable_spoofing: ['font', 'audio'] } }); - assert.deepEqual(s.calls.at(-1), { path: '/creator/environments', data: { gateway: 'gw-b', fingerprint: { timezone: 'Asia/Shanghai', disable_spoofing: 'font,audio' } } }); + await s.find('Form').props.onFinish({ gateway: 'gw-b', memory_limit_mb: 4096, fingerprint: { timezone: ' Asia/Shanghai ', disable_spoofing: ['font', 'audio'] } }); + assert.deepEqual(s.calls.at(-1), { path: '/creator/environments', data: { gateway: 'gw-b', memory_limit_mb: 4096, fingerprint: { timezone: 'Asia/Shanghai', disable_spoofing: 'font,audio' } } }); assert.equal(s.successes.length, 1); assert.equal(s.closes.length, 0); }); test('不填写指纹时使用默认配置；提交期间不能关闭', async () => { const s = setup(); await flush(); s.render(); - const save = s.find('Form').props.onFinish({ gateway: 'gw-a' }); + const save = s.find('Form').props.onFinish({ gateway: 'gw-a', memory_limit_mb: 2048 }); const busy = s.render(); assert.equal(busy.props.confirmLoading, true); assert.equal(busy.props.closable, false); @@ -85,7 +85,26 @@ test('不填写指纹时使用默认配置；提交期间不能关闭', async () busy.props.onCancel(); assert.equal(s.closes.length, 0); await save; - assert.deepEqual(s.calls.at(-1).data, { gateway: 'gw-a', fingerprint: {} }); + assert.deepEqual(s.calls.at(-1).data, { gateway: 'gw-a', memory_limit_mb: 2048, fingerprint: {} }); +}); + +test('内存设置在指纹区域外，默认2048 MB，校验整数范围并以256步进', async () => { + const s = setup(); await flush(); s.render(); + const field = s.find('Form.Item', undefined, node => node.props.name === 'memory_limit_mb'); + assert.ok(field, '必须提供独立内存字段'); + assert.equal(field.props.initialValue, 2048); + assert.equal(field.props.label, '浏览器内存上限（MB）'); + assert.match(field.props.extra, /默认2048 MB（2GB）.*下次启动生效/); + const input = s.find('InputNumber'); + assert.equal(input.props.min, 512); + assert.equal(input.props.max, 65536); + assert.equal(input.props.step, 256); + const Schema = requireWeb('@rc-component/async-validator').default; + const schema = new Schema({ memory_limit_mb: field.props.rules }); + for (const value of [512, 2048, 4096, 65536]) await schema.validate({ memory_limit_mb: value }); + for (const value of [undefined, null, 511, 65537, 2048.5, '2048']) { + await assert.rejects(schema.validate({ memory_limit_mb: value })); + } }); test('创建失败显示原因，不关闭弹窗也不触发列表刷新', async () => { diff --git a/web/tests/account-detail-resources.test.cjs b/web/tests/account-detail-resources.test.cjs index 22778d6..f0061f6 100644 --- a/web/tests/account-detail-resources.test.cjs +++ b/web/tests/account-detail-resources.test.cjs @@ -8,6 +8,11 @@ const ts = require('typescript'); const root = path.join(__dirname, '..'); const element = (type, props = {}) => ({ type, props }); const flush = () => new Promise((resolve) => setImmediate(resolve)); +const collectionFile = path.join(root, 'src/utils/account-collection-status.ts'); +const collectionModule = new Module(collectionFile); +collectionModule._compile(ts.transpileModule(fs.readFileSync(collectionFile, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }, +}).outputText, collectionFile); // Execute the real page/component with isolated hooks and API calls, without a browser. function harness(relativePath, api = {}, componentName = 'default') { @@ -99,6 +104,7 @@ function harness(relativePath, api = {}, componentName = 'default') { jsonOptions: (method, body) => ({ method, body: JSON.stringify(body) }), request: async (url, options) => { calls.requests.push({ url, options }); if (api.requestError) throw api.requestError; return api.requestResult; }, }, + '@/utils/account-collection-status': collectionModule.exports, '@/utils/helpers': { accountReadiness: () => ({ ready: true, canResume: true, label: '就绪' }), conflictMessage: (error) => error.message, @@ -225,13 +231,13 @@ test('owned account metrics are one horizontal row with each number above its la const page = harness(detailPath, { detail, metrics: [{ follower_count: 12500, following_count: 0, total_favorited: null, friend_count: 8, aweme_count: 100 }] }); const tree = await page.mount(); const dataCard = card(tree, '账号数据'); - const row = byType(dataCard, 'Flex').find((node) => byType(node, 'Statistic').length === 5 && !node.props.vertical); - assert.ok(row, 'all five metrics must share a horizontal Flex'); + const row = byType(dataCard, 'Flex').find((node) => byType(node, 'Statistic').length === 4 && !node.props.vertical); + assert.ok(row, 'all four profile metrics must share a horizontal Flex'); assert.notEqual(row.props.wrap, 'wrap'); const cells = byType(row, 'Flex').filter((node) => node.props.vertical); - assert.equal(cells.length, 5); - const labels = ['粉丝', '关注', '获赞', '互关', '作品总数']; - const values = ['1.3w', '0', '—', '8', '100']; + assert.equal(cells.length, 4); + const labels = ['粉丝', '关注', '获赞', '互关']; + const values = ['1.3w', '0', '—', '8']; cells.forEach((cell, index) => { const [number, label] = cell.props.children; assert.equal(number.type, 'Statistic'); @@ -241,16 +247,51 @@ test('owned account metrics are one horizontal row with each number above its la assert.equal(byType(dataCard, 'Descriptions.Item').length, 0); }); -test('作品总数保留最近非空值，不会被下一次缺失字段的快照清空', async () => { - const page = harness(detailPath, { detail, metrics: [{ aweme_count: 100 }, { follower_count: 20, aweme_count: null }] }); +test('公开作品总数来自采集检查点，不把资料接口 aweme_count 当作公开总数', async () => { + const page = harness(detailPath, { + detail, + metrics: [{ follower_count: 20, aweme_count: 100 }], + collectionStatus: { work_count: 28, public_work_count: 100, works: { status: 'succeeded' } }, + }); const tree = await page.mount(); - assert.equal(byType(card(tree, '账号数据'), 'Statistic')[4].props.value, '100'); + const stats = byType(card(tree, '账号数据'), 'Statistic'); + assert.equal(stats.length, 4); + assert.equal(stats.some(({ props }) => props.value === '100'), false); }); -test('详情作品总数使用完整采集历史，而非仅依赖最近指标记录', async () => { - const page = harness(detailPath, { detail, metrics: [{ follower_count: 20 }], collectionStatus: { work_count: 28, aweme_count: 100 } }); +test('详情与列表一致展示需要登录、独立评论结果和真实上次完成时间', async () => { + const page = harness(detailPath, { detail, collectionStatus: { + works: { status: 'blocked', last_error: 'browser is awaiting user login', last_completed_at: '2026-10-06T16:47:46Z' }, + comments: { status: 'failed', last_error: 'comments HTTP 502' }, + } }); const tree = await page.mount(); - assert.equal(byType(card(tree, '账号数据'), 'Statistic')[4].props.value, '100'); + const status = card(tree, '账号状态'); + const fields = byType(status, 'Descriptions.Item'); + const works = fields.find(({ props }) => props.label === '作品采集'); + const comments = fields.find(({ props }) => props.label === '评论采集'); + assert.match(text(works), /需要登录/); + assert.match(text(works), /浏览器需要登录/); + assert.match(text(works), /上次完成 2026-10-06T16:47:46Z/); + assert.doesNotMatch(text(works), /browser is awaiting user login/); + assert.match(text(comments), /采集失败/); + assert.match(text(comments), /comments HTTP 502/); + assert.doesNotMatch(text(comments), /上次完成/); + page.unmount(); +}); + +test('详情采集状态请求失败明确提示，不冒充待采集或正常', async () => { + const page = harness(detailPath, { detail, collectionStatusError: new Error('checkpoint query unavailable') }); + const tree = await page.mount(); + const failure = byType(tree, 'Alert').find(({ props }) => props.title === '采集状态读取失败'); + assert.ok(failure); + assert.equal(failure.props.description, 'checkpoint query unavailable'); + const fields = byType(card(tree, '账号状态'), 'Descriptions.Item').filter(({ props }) => ['作品采集', '评论采集'].includes(props.label)); + assert.equal(fields.length, 2); + fields.forEach((field) => { + assert.match(text(field), /状态未知/); + assert.doesNotMatch(text(field), /待采集|采集完成/); + }); + page.unmount(); }); for (const [name, props] of [ @@ -262,7 +303,7 @@ for (const [name, props] of [ const works = Array.from({ length: 6 }, (_, i) => ({ id: `work-${i}`, title: `作品${i}` })); const page = harness('src/components/creator/WorksGallery.tsx', { workPages: { 1: { data: works, total: 30, hasNext: true } }, - collectionStatus: { work_count: 30, aweme_count: 30, works: { status: 'succeeded' } }, + collectionStatus: { work_count: 30, public_work_count: 30, works: { status: 'succeeded' } }, }); const tree = await page.mount(props); const columns = byType(card(tree, '作品列表'), 'Col'); @@ -311,7 +352,7 @@ test('作品封面加载占位与成功图片均为3:4，图片裁切显示并 test('缺失封面也保持3:4，不使用远程图片代替', async () => { const cover = harness('src/components/creator/WorksGallery.tsx', { - services: { fetchCreatorBinary: async () => { throw new Error('封面不存在'); } }, + services: { fetchCreatorBinary: async () => { throw Object.assign(new Error('封面不存在'), {status:404}); } }, }, 'WorkCover'); const tree = await cover.mount({ workID: 'missing-work' }); assert.equal(tree.type, 'Flex'); @@ -323,7 +364,7 @@ test('缺失封面也保持3:4，不使用远程图片代替', async () => { test('自有账号作品展示独立采集进度，历史作品可翻页且筛选不改变进度分母', async () => { const api = { - collectionStatus: { work_count: 28, aweme_count: 100, works: { status: 'succeeded' } }, + collectionStatus: { work_count: 28, public_work_count: 100, works: { status: 'succeeded' } }, workPages: { 1: { data: [{ id: 'new-work' }], total: 28, hasNext: true }, 2: { data: [{ id: 'historical-work' }], total: 28, hasNext: false }, @@ -331,7 +372,7 @@ test('自有账号作品展示独立采集进度，历史作品可翻页且筛 }; const page = harness('src/components/creator/WorksGallery.tsx', api); let tree = await page.mount({ sourceId: 'account/1', sourceType: 'owned' }); - assert.match(text(tree), /已采集 28 \/ 总计 100/); + assert.match(text(tree), /已采集公开作品 28 \/ 公开作品总数 100/); assert.match(text(tree), /尚未采集完整/); assert.equal(page.calls.collectionReads[0], '/creator/accounts/account%2F1/collection-status'); assert.equal(byType(tree, 'Pagination')[0].props.total, 28); @@ -350,13 +391,13 @@ test('自有账号作品展示独立采集进度，历史作品可翻页且筛 tree = page.render(); assert.equal(page.calls.lists.at(-1).page, 1); assert.equal(page.calls.lists.at(-1).filters.min_likes_growth, 10); - assert.match(text(tree), /已采集 28 \/ 总计 100/); + assert.match(text(tree), /已采集公开作品 28 \/ 公开作品总数 100/); assert.match(text(tree), /符合筛选 0 个作品/); }); test('作品进度区分未知、零作品与数量不一致，不把采集成功当作完整', async () => { - for (const [count, total, expected] of [[7, null, /总计 未知/], [0, 0, /已采集 0 \/ 总计 0/], [7, 3, /数量与抖音总数不一致/]]) { - const page = harness('src/components/creator/WorksGallery.tsx', { collectionStatus: { work_count: count, aweme_count: total, works: { status: 'succeeded' } } }); + for (const [count, total, expected] of [[7, null, /公开作品总数 未知/], [0, 0, /已采集公开作品 0 \/ 公开作品总数 0/], [7, 3, /已采集公开作品数高于抖音公开作品总数/]]) { + const page = harness('src/components/creator/WorksGallery.tsx', { collectionStatus: { work_count: count, public_work_count: total, works: { status: 'succeeded' } } }); const tree = await page.mount({ sourceId: 'account-1', sourceType: 'owned' }); assert.match(text(tree), expected); if (total === null) assert.match(text(tree), /暂无法判断是否完整/); @@ -372,13 +413,13 @@ test('读取作品进度失败明确提示，已采集作品仍然展示，刷 assert.equal(byType(tree, 'Alert')[0].props.title, '进度读取失败'); assert.match(text(tree), /采集进度暂不可用/); api.collectionStatusError = null; - api.collectionStatus = { work_count: 1, aweme_count: 1, works: { status: 'succeeded' } }; + api.collectionStatus = { work_count: 1, public_work_count: 1, works: { status: 'succeeded' } }; findAll(tree, (node) => node.type === 'Button' && node.props['aria-label'] === '刷新作品')[0].props.onClick(); page.render(); await flush(); tree = page.render(); assert.equal(byType(tree, 'Alert').length, 0); - assert.match(text(tree), /已采集 1 \/ 总计 1/); + assert.match(text(tree), /已采集公开作品 1 \/ 公开作品总数 1/); assert.equal(page.calls.collectionReads.length, 2); }); @@ -391,7 +432,7 @@ test('监控账号仍展示已采集列表，不请求自有账号采集进度', test('作品列表读取失败明确提示并可重试，不影响独立采集进度', async (t) => { t.mock.method(console, 'error', () => {}); - const api = { workListError: new Error('作品读取失败'), collectionStatus: { work_count: 28, aweme_count: 100, works: { status: 'running' } } }; + const api = { workListError: new Error('作品读取失败'), collectionStatus: { work_count: 28, public_work_count: 100, works: { status: 'running' } } }; const page = harness('src/components/creator/WorksGallery.tsx', api); let tree = await page.mount({ sourceId: 'account-1', sourceType: 'owned' }); assert.equal(byType(tree, 'Alert')[0].props.title, '作品读取失败'); @@ -479,7 +520,7 @@ test('detail retains missing data, no-environment state, trends, and account act const page = harness(detailPath, { detail: { ...detail, environment: null, runtime_status: 'paused' } }); let tree = await page.mount(); assert.match(text(card(tree, '账号状态')), /尚未绑定运行环境/); - assert.deepEqual(byType(card(tree, '账号数据'), 'Statistic').map((node) => node.props.value), ['—', '—', '—', '—', '—']); + assert.deepEqual(byType(card(tree, '账号数据'), 'Statistic').map((node) => node.props.value), ['—', '—', '—', '—']); await byType(card(tree, '账号状态'), 'Button')[0].props.onClick(); assert.equal(page.calls.requests[0].url, '/phase-a/accounts/account-1/start'); await byType(page.calls.actions, 'Button')[1].props.onClick(); diff --git a/web/tests/account-login-modal.test.cjs b/web/tests/account-login-modal.test.cjs new file mode 100644 index 0000000..8ff1c91 --- /dev/null +++ b/web/tests/account-login-modal.test.cjs @@ -0,0 +1,94 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const Module = require('node:module'); +const React = require('react'); +const ts = require('typescript'); + +const file = resolve(__dirname, '../src/components/accounts/AccountLoginModal.tsx'); +const compiled = ts.transpileModule(readFileSync(file, 'utf8'), { + fileName: file, + compilerOptions: { module: ts.ModuleKind.CommonJS, jsx: ts.JsxEmit.ReactJSX, target: ts.ScriptTarget.ES2020, inlineSourceMap: true, inlineSources: true }, +}).outputText; + +function harness(state = { phase: 'loading', logs: [] }) { + const effects = [], loginCalls = []; + const antd = Object.fromEntries(['Alert', 'Button', 'Flex', 'Image', 'Modal', 'Spin'].map((name) => [name, () => null])); + antd.Typography = { Text: () => null, Title: () => null }; + const request = () => assert.fail('The modal must delegate requests to the shared login flow'); + let stopped = false; + const loaded = new Module(file, module); + loaded.filename = file; + loaded.require = (id) => { + if (id === 'react') return { + useState: (initial) => [initial?.phase ? state : initial, () => {}], + useRef: (initial) => ({ current: initial }), + useEffect: (callback) => effects.push(callback), + }; + if (id === 'antd') return antd; + if (id === '@ant-design/icons') return { ReloadOutlined: () => null }; + if (id === '@/requestErrorConfig') return { request }; + if (id === '@/utils/helpers') return { dateTime: value => value }; + if (id === './account-login') return { startAccountLogin: (...args) => { loginCalls.push(args); return () => { stopped = true; }; } }; + return require(id); + }; + loaded._compile(compiled, file); + return { antd, request, effects, loginCalls, render: loaded.exports.default, stopped: () => stopped }; +} + +function find(node, type) { + if (Array.isArray(node)) return node.map((child) => find(child, type)).find(Boolean); + if (!React.isValidElement(node)) return undefined; + return node.type === type ? node : find(node.props.children, type); +} + +const defaultDescription = '使用抖音扫码，并在手机上确认。状态自动检查，二维码到期自动刷新。关闭弹窗将停止检查。'; + +for (const resource of [undefined, 'accounts', 'environments']) { + test(`login modal keeps the existing sequential flow for ${resource || 'default accounts'}`, () => { + const h = harness(); + let successes = 0, closes = 0; + const tree = h.render({ accountID: 'account/one', accountName: '所选账号', resource, onClose: () => { closes++; }, onSuccess: () => { successes++; } }); + assert.equal(tree.props.title, '登录二维码 · 所选账号'); + assert.equal(find(tree, h.antd.Alert).props.description, defaultDescription); + const stop = h.effects[0](); + assert.equal(h.loginCalls.length, 1); + assert.equal(h.loginCalls[0][0], 'account/one'); + assert.equal(h.loginCalls[0][1], h.request); + assert.equal(h.loginCalls[0][4], resource || 'accounts'); + assert.equal(successes, 0); + h.loginCalls[0][3]({ status: 'logged_in' }); + assert.equal(successes, 1); + tree.props.onCancel(); + assert.equal(closes, 1); + stop(); + assert.equal(h.stopped(), true); + }); + test(`${resource || 'default accounts'} login keeps the existing PNG Image display`, () => { + const h = harness({ phase: 'waiting', logs: [], qr: { + content_type: 'image/png', image_base64: 'cG5n', qr_detected: true, + expires_at: new Date(Date.now() + 120000).toISOString(), + } }); + const tree = h.render({ accountID: 'one', accountName: '账号', resource, onClose() {}, onSuccess() {} }); + assert.deepEqual(find(tree, h.antd.Image).props, { + src: 'data:image/png;base64,cG5n', alt: '抖音登录二维码', width: 240, preview: false, + }); + assert.equal(find(tree, h.antd.Alert).props.description, defaultDescription); + }); +} + +test('account login preserves the actual error instead of hiding it in scan instructions', () => { + const h = harness({ phase: 'error', error: '已绑定 UID 与浏览器账号不一致', logs: [] }); + const tree = h.render({ accountID: 'one', accountName: '账号', onClose() {}, onSuccess() {} }); + const alert = find(tree, h.antd.Alert); + assert.equal(alert.props.type, 'error'); + assert.equal(alert.props.title, '登录流程已停止'); + assert.equal(alert.props.description, '已绑定 UID 与浏览器账号不一致'); +}); + +test('account login success copy remains unchanged', () => { + const h = harness({ phase: 'success', logs: [] }); + const tree = h.render({ accountID: 'one', accountName: '账号', onClose() {}, onSuccess() {} }); + assert.equal(find(tree, h.antd.Alert).props.description, '账号已登录，检查已停止。'); +}); diff --git a/web/tests/account-login.test.cjs b/web/tests/account-login.test.cjs index 43d08f9..8902d49 100644 --- a/web/tests/account-login.test.cjs +++ b/web/tests/account-login.test.cjs @@ -176,3 +176,12 @@ test('列表与编辑页共享自动登录弹窗，不再提供手动核验', () assert.doesNotMatch(text, /扫码后核验|刷新登录二维码|二维码有效期|登录二维码（剩余/); } }); + +test('环境扫码保留原有核验、二维码与账号绑定结果', async () => { + const paths = [], states = [], successes = []; + const responses = [pending, qr(), { status: 'logged_in', account_id: 'bound-account' }]; + const stop = startAccountLogin('env/one', async (path) => { paths.push(path); return responses.shift(); }, state => states.push(state), result => successes.push(result), 'environments'); + await flush(); stop(); + assert.deepEqual(paths, ['/creator/environments/env%2Fone/verify', '/creator/environments/env%2Fone/login-qr', '/creator/environments/env%2Fone/verify']); + assert.equal(states.at(-1).phase, 'success'); assert.equal(successes[0].account_id, 'bound-account'); +}); diff --git a/web/tests/account-management-list.test.cjs b/web/tests/account-management-list.test.cjs index d98fcde..c2433b7 100644 --- a/web/tests/account-management-list.test.cjs +++ b/web/tests/account-management-list.test.cjs @@ -54,29 +54,57 @@ test('我的账号列不显示 UID，头像与监控账号一致且详情入口 assert.match(code, /history\.push\(`\/accounts\/\$\{account\.id\}`\)/); }); -test('作品总数来自抖音，已采集数量只表示进度', () => { +test('运行环境只展示状态、网关和代理，不显示名称或别名', () => { + const render = accountRenderer('AccountManagementList.tsx', '运行环境'); + const requireWeb = require('node:module').createRequire(`${__dirname}/../package.json`); + const { renderToStaticMarkup } = requireWeb('react-dom/server'); + for (const [environment, expected] of [ + [{ runtime_id: 'runtime-1', gateway: 'gateway-1', network_exit_id: 'proxy-1', network_exit_health: 'healthy' }, ['运行中', '网关 gateway-1', '出口 proxy-1 · 健康']], + [{ runtime_cleanup_pending: true, runtime_id: 'runtime-1', gateway: 'gateway-2', network_exit_id: 'proxy-2', network_exit_health: 'unhealthy' }, ['停止待确认', '网关 gateway-2', '出口 proxy-2 · unhealthy']], + [{}, ['未启动', '网关 —', '机器直连']], + [{ network_exit_id: 'proxy-3' }, ['未启动', '网关 —', '出口 proxy-3 · 状态未知']], + ]) { + for (const pending_environment of [undefined, 'env-de0f681e61ba10330090bfe8']) { + const html = renderToStaticMarkup(render(undefined, { pending_environment, environment: { + ...environment, name: '不应展示的环境名称', alias: 'env-de0f681e61ba10330090bfe8', + } })); + assert.ok(!html.includes('不应展示的环境名称')); + assert.ok(!html.includes('env-de0f681e61ba10330090bfe8')); + for (const text of expected) assert.ok(html.includes(text), `应保留 ${text}`); + } + } + assert.match(renderToStaticMarkup(render(undefined, {})), /未绑定运行环境/); +}); + +test('作品数据列只用普通字体展示 N/M', () => { const render = accountRenderer('AccountManagementList.tsx', '作品数据'); const requireWeb = require('node:module').createRequire(`${__dirname}/../package.json`); - const html = requireWeb('react-dom/server').renderToStaticMarkup(render(undefined, { aweme_count: 100, work_count: 7 })); - assert.match(html, /100 个作品/); - assert.match(html, /已采集 7 \/ 总计 100/); - assert.doesNotMatch(html, />7 个作品/); - const unknown = requireWeb('react-dom/server').renderToStaticMarkup(render(undefined, { work_count: 7 })); - assert.match(unknown, /作品总数未知/); - assert.match(unknown, /已采集 7 \/ 总计 未知/); - const zero = requireWeb('react-dom/server').renderToStaticMarkup(render(undefined, { aweme_count: 0, work_count: 0 })); - assert.match(zero, /0 个作品/); - assert.match(zero, /已采集 0 \/ 总计 0/); + const { renderToStaticMarkup } = requireWeb('react-dom/server'); + const cell = render(undefined, { public_work_count: 100, work_count: 7, latest_published_at: '2026-01-01' }); + const textContent = element => [element.props.children].flat(Infinity).join(''); + assert.equal(textContent(cell), '7/100'); + assert.equal(cell.props.type, undefined, '使用正常字体颜色'); + assert.equal(cell.props.style, undefined, '不使用小号字体'); + assert.doesNotMatch(renderToStaticMarkup(cell), /公开作品|已采集|最近发布/); + const unknown = render(undefined, { work_count: 7 }); + assert.equal(textContent(unknown), '7/未知'); + const zero = render(undefined, { public_work_count: 0, work_count: 0 }); + assert.equal(textContent(zero), '0/0'); +}); + +test('最近发布使用独立列', () => { + const render = accountRenderer('AccountManagementList.tsx', '最近发布'); + assert.equal(render(undefined, { latest_published_at: '2026-01-01' }), '2026-01-01'); }); test('未登录环境没有真实作品总数，不把占位零当作抖音数据', () => { const render = accountRenderer('AccountManagementList.tsx', '作品数据'); const requireWeb = require('node:module').createRequire(`${__dirname}/../package.json`); const html = requireWeb('react-dom/server').renderToStaticMarkup(render(undefined, { - pending_environment: true, aweme_count: 0, work_count: 0, + pending_environment: true, public_work_count: 0, work_count: 0, })); assert.match(html, /登录后显示作品数据/); - assert.doesNotMatch(html, /0 个作品|总计 0/); + assert.doesNotMatch(html, /0 个公开作品|公开作品总数 0/); }); test('列表展示粉丝、关注、获赞、互关、评论独立数值列', () => { @@ -94,25 +122,23 @@ test('列表展示粉丝、关注、获赞、互关、评论独立数值列', () } }); -test('操作列包含「启动」：先启动环境再弹出登录画面', () => { - assert.match(code, /onClick=\{\(\) => startAccount\(account\)\}/); - assert.match(code, /\/phase-a\/accounts\/\$\{encodeURIComponent\(row\.id\)\}\/start/); - assert.match(code, /setLoginTarget\(row\)/); - assert.match(code, / { + assert.match(code, /accountPrimaryAction\(account, browser, verifiedLogin\[account\.id\]\)/); + assert.match(code, /performPrimary\(account, primary\.key\)/); + assert.match(code, / { assert.match(code, /loginTarget \? \(/); assert.match(code, /onClose=\{\(\) => setLoginTarget\(null\)\}/); - assert.match(code, /onSuccess=\{\(\) => \{ void load\(\); \}\}/); + assert.match(code, /\[loginTarget\.id\]: 'logged_in'/); + assert.match(code, /void load\(\)/); assert.doesNotMatch(code, /刷新登录画面|核验浏览器身份|requestLoginQR|verifyLogin/); }); -test('启动失败走 conflictMessage 透传 409 就绪原因', () => { - assert.ok( - code.includes( - "messageApi.error(conflictMessage(startError, '启动未完成：账号或环境不满足就绪条件，请按不可运行原因处理后重试。'));", - ), - ); +test('操作失败走 conflictMessage 透传原因并保留排查信息', () => { + assert.ok(code.includes("messageApi.error(conflictMessage(cause, '账号操作未完成。'))")); + assert.ok(code.includes("console.error('账号主操作失败'")); }); diff --git a/web/tests/account-management-unified-list.test.cjs b/web/tests/account-management-unified-list.test.cjs index 5310a76..ee51177 100644 --- a/web/tests/account-management-unified-list.test.cjs +++ b/web/tests/account-management-unified-list.test.cjs @@ -6,38 +6,67 @@ const { test } = require('node:test'); const ts = require('typescript'); const requireWeb = createRequire(resolve(__dirname, '../package.json')); const file = resolve(__dirname, '../src/components/accounts/AccountManagementList.tsx'); +const eventsFile = resolve(__dirname, '../src/utils/account-events.ts'); +const collectionFile = resolve(__dirname, '../src/utils/account-collection-status.ts'); +const collectionModule = new Module(collectionFile, module); +collectionModule._compile(ts.transpileModule(readFileSync(collectionFile, 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, collectionFile); +const eventsModule = new Module(eventsFile, module); +eventsModule._compile(ts.transpileModule(readFileSync(eventsFile, 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS } }).outputText, eventsFile); -function setup({ accounts = [{ id: 'account-1', name: '真实昵称', business_status: 'normal', tags: ['主账号'] }], environments = [{ alias: 'env-1', gateway: 'gw-a' }], loadError, actionError } = {}) { - const states = [], calls = [], messages = [], paths = []; +function setup({ accounts = [{ id: 'account-1', name: '真实昵称', business_status: 'normal', tags: ['主账号'], login_status: 'logged_in', environment: { alias: 'env-account' } }], environments = [{ alias: 'env-1', gateway: 'gw-a' }], browsers = [{ alias: 'env-account', state: 'running', status: 'ready', gateway_reachable: true }], currentAccount, verification, liveBrowserError, browserError, loadError, actionError } = {}) { + const states = [], calls = [], messages = [], paths = [], confirmations = []; let cursor = 0, started = false; const marker = displayName => Object.assign(() => null, { displayName }); const form = { setFieldsValue(values) { form.values = values; }, async validateFields() { return form.values; } }; const antd = {}; - for (const name of ['Alert', 'Avatar', 'Button', 'Card', 'Flex', 'Modal', 'Popconfirm', 'Select', 'Space', 'Table', 'Tag']) antd[name] = marker(name); + for (const name of ['Alert', 'Avatar', 'Button', 'Card', 'Dropdown', 'Flex', 'Modal', 'Popconfirm', 'Select', 'Space', 'Switch', 'Tooltip', 'Table', 'Tag']) antd[name] = marker(name); antd.Typography = { Text: marker('Text'), Link: marker('Link') }; antd.Form = Object.assign(marker('Form'), { Item: marker('Form.Item'), useForm: () => [form] }); - antd.App = { useApp: () => ({ message: { success: text => messages.push(text), error: text => messages.push(text) } }) }; + antd.App = { useApp: () => ({ message: { success: text => messages.push(text), error: text => messages.push(text) }, modal: { confirm: options => confirmations.push(options) } }) }; const loaded = new Module(file, module); - const options = { accounts, environments, loadError, actionError }; + const actionFile = resolve(__dirname, '../src/components/accounts/account-actions.ts'); + const actionLoaded = new Module(actionFile, module); + const options = { accounts, environments, browsers, currentAccount, verification, liveBrowserError, browserError, loadError, actionError }; loaded.require = id => { if (id === 'react') return { useState(value) { const index = cursor++; if (!(index in states)) states[index] = value; return [states[index], next => { states[index] = typeof next === 'function' ? next(states[index]) : next; }]; }, + useRef(value) { const index = cursor++; if (!(index in states)) states[index] = { current: value }; return states[index]; }, useCallback: callback => callback, useEffect(effect) { if (!started) { started = true; effect(); } }, }; if (id === 'antd') return antd; if (id === '@umijs/max') return { history: { push: path => paths.push(path) } }; + if (id === './account-actions') return actionLoaded.exports; if (id === '@/services/api') return { - async creatorGet(path) { calls.push({ path }); if (options.loadError) throw options.loadError; return path === '/creator/environments' ? options.environments : options.accounts; }, - async creatorAction(path) { calls.push({ path }); if (options.actionError) throw options.actionError; }, - async creatorUpdate(path, data) { calls.push({ path, data }); if (options.actionError) throw options.actionError; }, + async creatorGet(path) { + calls.push({ path }); + if (options.loadError) throw options.loadError; + if (path === '/creator/environments') return options.environments; + if (path === '/creator/accounts/monitor-views') return options.accounts; + if (path === '/browsers') { if (options.browserError) throw options.browserError; return options.browsers; } + if (path.startsWith('/phase-a/accounts/')) return options.currentAccount || options.accounts.find(account => account.id === decodeURIComponent(path.split('/').pop())); + if (path.startsWith('/browsers/')) { if (options.liveBrowserError) throw options.liveBrowserError; return options.browsers.find(browser => browser.alias === decodeURIComponent(path.split('/').pop())); } + throw new Error('Unexpected request: ' + path); + }, + async creatorAction(path) { + calls.push({ path }); + if (options.actionError) throw options.actionError; + if (path.endsWith('/verify')) return options.verification || { status: 'logged_in', account_id: path.split('/')[3] }; + if (path.endsWith('/pause')) options.accounts.forEach(account => { if (path.includes(`/${account.id}/`)) account.runtime_status = 'paused'; }); + if (path.startsWith('/browsers/') && path.endsWith('/stop')) options.browsers.forEach(browser => { if (path.includes(`/${browser.alias}/`)) Object.assign(browser, { state: 'runtime_missing', status: 'stopped' }); }); + }, + async creatorUpdate(path, data) { calls.push({ path, data }); if (options.actionError) throw options.actionError; if (path.endsWith('/listener')) return { enabled: data.enabled, status: data.enabled ? 'starting' : 'stopping', reason: '' }; }, async remove(resource, id) { calls.push({ resource, id }); if (options.actionError) throw options.actionError; }, }; - if (id === '@/utils/helpers') return { accountReadiness: () => ({ ready: true, label: '可运行' }), dateTime: value => value, conflictMessage: (error, fallback) => error.message || fallback }; + if (id === '@/utils/account-events') return eventsModule.exports; + if (id === '@/utils/account-collection-status') return collectionModule.exports; + if (id === '@/utils/helpers') return { accountReadiness: () => ({ ready: true, canResume: true, label: '可运行' }), dateTime: value => value, conflictMessage: (error, fallback) => error.message || fallback }; if (id === '@/utils/table') return { fixedLeft: column => ({ ...column, fixed: 'left' }), fixedRight: column => ({ ...column, fixed: 'right' }), tablePagination: () => ({ pageSize: 20, hideOnSinglePage: true }), useTableSticky: () => ({ offsetHeader: 56 }) }; if (id.startsWith('@/components/accounts/')) return { default: marker(id.split('/').at(-1)) }; return requireWeb(id); }; + actionLoaded.require = loaded.require; + actionLoaded._compile(ts.transpileModule(readFileSync(actionFile, 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, actionFile); loaded._compile(ts.transpileModule(readFileSync(file, 'utf8'), { fileName: file, compilerOptions: { module: ts.ModuleKind.CommonJS, jsx: ts.JsxEmit.ReactJSX, inlineSourceMap: true, inlineSources: true } }).outputText, file); let tree; const render = () => { cursor = 0; tree = loaded.exports.default(); return tree; }; @@ -49,7 +78,7 @@ function setup({ accounts = [{ id: 'account-1', name: '真实昵称', business_s const column = title => find('Table').props.columns.find(column => column.title === title); const cell = (title, row) => { const col = column(title); return col.render(row[col.dataIndex], row); }; render(); - return { render, find, findAll, cell, calls, messages, paths, form, options }; + return { render, find, findAll, cell, calls, messages, paths, form, options, confirmations }; } const flush = async () => { for (let i = 0; i < 10; i++) await Promise.resolve(); }; const text = node => node == null ? '' : typeof node === 'object' ? [node.props?.children].flat(Infinity).map(text).join('') : String(node); @@ -79,7 +108,7 @@ for (const rowCount of [1, 20, 100]) { assert.equal(columns[0].width, 240); assert.equal(columns.at(-1).title, '操作'); assert.equal(columns.at(-1).fixed, 'right'); - assert.equal(columns.at(-1).width, 300); + assert.equal(columns.at(-1).width, 200); assert.ok(columns.slice(1, -1).every(column => !column.fixed)); assert.equal(scroll.x, 'max-content', '宽屏也应按内容宽度滚动，而不是挤压两侧列'); assert.equal(Object.hasOwn(scroll, 'y'), false, '不应启用表格内部纵向滚动'); @@ -117,8 +146,8 @@ test('待登录行缺失资料显示默认值，不伪造账号、业务状态 assert.equal(text(s.cell('作品数据', row)), '登录后显示作品数据'); assert.equal(text(s.cell('标签', row)), '未设置'); assert.equal(text(s.cell('业务状态', row)), '—'); - assert.equal(text(s.cell('状态', row)), '待登录'); - assert.match(text(s.cell('运行环境', row)), /env-1未启动网关 gw-a机器直连/); + assert.equal(text(s.cell('调度状态', row)), '待登录'); + assert.equal(text(s.cell('运行环境', row)), '未启动网关 gw-a机器直连'); const actions = s.cell('操作', row); assert.equal(s.findAll('Button', actions).length, 1); assert.equal(text(actions), '登录并同步账号'); @@ -145,10 +174,12 @@ test('登录使用真实环境标识，成功后同一列表刷新为真实账 assert.deepEqual(s.find('Table').props.dataSource.map(row => row.id), ['account-new']); }); -test('待登录环境真实名称、网关、运行与出口信息仍被保留', async () => { +test('待登录环境保留名称数据但只展示状态、网关与出口信息', async () => { const s = await ready({ accounts: [], environments: [{ alias: 'env-2', name: '环境二', gateway: 'gw-b', runtime_id: 'runtime-2', runtime_cleanup_pending: true, network_exit: { id: 'exit-2', health_status: 'healthy' } }] }); const row = s.find('Table').props.dataSource[0]; assert.ok(row); - assert.match(text(s.cell('运行环境', row)), /环境二停止待确认网关 gw-b出口 exit-2 · 健康/); + assert.equal(row.environment.name, '环境二', '隐藏名称不改变环境数据'); + assert.equal(row.environment.alias, 'env-2', '别名仍用于登录操作'); + assert.equal(text(s.cell('运行环境', row)), '停止待确认网关 gw-b出口 exit-2 · 健康'); }); test('空列表不生成占位行，环境加载失败不冒充成功并支持重试', async () => { @@ -164,48 +195,53 @@ test('空列表不生成占位行，环境加载失败不冒充成功并支持 assert.equal(s.find('Table').props.dataSource.length, 2); }); -test('已登录账号仍可访问详情、编辑、启动、删除和保存标签', async () => { +test('正常账号仅显示同步资料和更多，不启动浏览器、不弹二维码；编辑与删除保留', async () => { const s = await ready({ environments: [] }); let row = s.find('Table').props.dataSource[0]; s.find('Link', s.cell('账号', row)).props.onClick(); assert.equal(s.paths.at(-1), '/accounts/account-1'); const actions = s.cell('操作', row); - s.findAll('Button', actions).find(button => text(button) === '编辑账号').props.onClick(); + assert.equal(s.findAll('Button', actions).length, 2); + const button = s.findAll('Button', actions).find(button => text(button) === '同步资料'); + await button.props.onClick(); s.render(); + assert.ok(s.calls.some(call => call.path === '/creator/accounts/account-1/verify')); + assert.ok(!s.calls.some(call => call.path.endsWith('/start'))); + assert.equal(s.find('AccountLoginModal'), undefined); + const menu = s.find('Dropdown', actions).props.menu; + await menu.onClick({ key: 'edit' }); assert.equal(s.paths.at(-1), '/accounts/account-1/edit'); - await s.findAll('Button', actions).find(button => text(button) === '启动').props.onClick(); s.render(); - assert.ok(s.calls.some(call => call.path === '/phase-a/accounts/account-1/start')); - let login = s.find('AccountLoginModal'); assert.equal(login.props.accountID, 'account-1'); - login.props.onSuccess(); await flush(); s.render(); - login = s.find('AccountLoginModal'); login.props.onClose(); s.render(); - s.findAll('Button', actions).find(button => text(button) === '编辑标签').props.onClick(); s.render(); + await menu.onClick({ key: 'tags' }); s.render(); assert.deepEqual(s.form.values.tags, ['主账号']); s.form.values = { tags: ['新标签'] }; await s.find('Modal').props.onOk(); s.render(); row = s.find('Table').props.dataSource[0]; assert.deepEqual(row.tags, ['新标签']); - assert.ok(s.calls.some(call => call.path === '/creator/accounts/account-1/tags')); - await s.find('Popconfirm', actions).props.onConfirm(); + await menu.onClick({ key: 'delete' }); + assert.equal(s.calls.some(call => call.resource === 'accounts'), false, '删除前必须确认'); + await s.confirmations.at(-1).onOk(); assert.ok(s.calls.some(call => call.resource === 'accounts' && call.id === 'account-1')); }); test('已登录账号缺失业务状态有明确占位，真实指标和标签保持原样', async () => { - const s = await ready({ environments: [], accounts: [{ id: 'account-2', name: '', follower_count: 20000, aweme_count: 100, work_count: 7, latest_published_at: '2026-01-01', environment: { alias: 'env-2', runtime_id: 'runtime-2', network_exit_id: 'exit-2', network_exit_health: 'unchecked' } }] }); + const s = await ready({ environments: [], accounts: [{ id: 'account-2', name: '', follower_count: 20000, public_work_count: 100, work_count: 7, latest_published_at: '2026-01-01', environment: { alias: 'env-2', runtime_id: 'runtime-2', network_exit_id: 'exit-2', network_exit_health: 'unchecked' } }] }); const row = s.find('Table').props.dataSource[0]; assert.equal(s.cell('粉丝', row), '2.0w'); assert.equal(s.cell('关注', row), '—'); assert.equal(text(s.cell('业务状态', row)), '—'); - assert.match(text(s.cell('作品数据', row)), /100 个作品已采集 7 \/ 总计 100最近发布 2026-01-01/); - assert.match(text(s.cell('运行环境', row)), /env-2运行中网关 —出口 exit-2 · unchecked/); - assert.equal(text(s.cell('状态', row)), '可运行'); + assert.equal(text(s.cell('作品数据', row)), '7/100'); + assert.equal(text(s.cell('最近发布', row)), '2026-01-01'); + assert.equal(text(s.cell('运行环境', row)), '运行中网关 —出口 exit-2 · unchecked'); + assert.equal(text(s.cell('调度状态', row)), '可运行'); }); test('账号操作失败显示实际错误，标签取消及新增关闭刷新保持可用', async () => { const s = await ready({ actionError: new Error('操作冲突') }); const row = s.find('Table').props.dataSource[0]; const actions = s.cell('操作', row); - const button = label => s.findAll('Button', actions).find(button => text(button) === label); - await button('启动').props.onClick(); s.render(); assert.equal(s.find('AccountLoginModal'), undefined); - await s.find('Popconfirm', actions).props.onConfirm(); - button('编辑标签').props.onClick(); s.render(); await s.find('Modal').props.onOk(); s.render(); + await s.findAll('Button', actions).find(button => text(button) === '同步资料').props.onClick(); s.render(); assert.equal(s.find('AccountLoginModal'), undefined); + const menu = s.find('Dropdown', actions).props.menu; + await menu.onClick({ key: 'delete' }); + await assert.rejects(s.confirmations.at(-1).onOk(), /操作冲突/); + await menu.onClick({ key: 'tags' }); s.render(); await s.find('Modal').props.onOk(); s.render(); assert.equal(s.messages.filter(message => message === '操作冲突').length, 3); assert.equal(s.find('Modal').props.open, true); s.find('Modal').props.onCancel(); s.render(); assert.equal(s.find('Modal').props.open, false); @@ -216,3 +252,150 @@ test('账号操作失败显示实际错误，标签取消及新增关闭刷新 assert.equal(s.find('AccountCreateModal'), undefined); assert.equal(s.find('Table').props.dataSource.length, 2); }); + +test('监听默认关闭，登录后开关保存真实状态，未登录环境不提供开关', async () => { + const s = await ready({ accounts: [{ id: 'account', name: '账号', login_status: 'logged_in', listener_enabled: false, listener_status: 'stopped' }] }); + let row = s.find('Table').props.dataSource.find(row => !row.pending_environment); + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.checked, false); + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.disabled, false); + const pending = s.find('Table').props.dataSource.find(row => row.pending_environment); + assert.equal(s.find('Switch', s.cell('监听状态', pending)), undefined); + assert.match(text(s.cell('监听状态', pending)), /登录后可开启/); + s.find('Switch', s.cell('监听状态', row)).props.onChange(true); await flush(); s.render(); + row = s.find('Table').props.dataSource.find(row => row.id === 'account'); + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.checked, true); + assert.match(text(s.cell('监听状态', row)), /启动中/); + assert.deepEqual(s.calls.find(call => call.path.endsWith('/listener')).data, { enabled: true }); + s.find('Switch', s.cell('监听状态', row)).props.onChange(false); await flush(); s.render(); + row = s.find('Table').props.dataSource.find(row => row.id === 'account'); + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.checked, false); + assert.match(text(s.cell('监听状态', row)), /关闭中/); +}); + +test('未登录账号不能开启，但仍能关闭已开启监听，保存失败不假装成功', async () => { + const s = await ready({ environments: [], actionError: new Error('监听保存失败'), accounts: [{ id: 'account', name: '账号', login_status: 'not_logged_in', listener_enabled: false, listener_status: 'error', listener_reason: '登录已失效' }] }); + let row = s.find('Table').props.dataSource[0]; + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.disabled, true); + assert.equal(s.find('Tooltip', s.cell('监听状态', row)).props.title, '登录已失效；尚未完成通知检查'); + row.listener_enabled = true; + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.disabled, false); + s.find('Switch', s.cell('监听状态', row)).props.onChange(false); await flush(); s.render(); + row = s.find('Table').props.dataSource[0]; + assert.equal(s.find('Switch', s.cell('监听状态', row)).props.checked, true); + assert.ok(s.messages.includes('监听保存失败')); +}); + +test('采集状态只展示一个 Tag，异常优先，时间和分类不展示，原因仅悬停可见', async () => { + const s = await ready({ environments: [], accounts: [{ + id: 'account', business_status: 'normal', + works_collection: { status: 'blocked', last_error: 'browser is awaiting user login', last_completed_at: '2026-10-06T16:47:46Z' }, + comments_collection: { status: 'succeeded', last_completed_at: '2026-10-06T15:00:00Z' }, + }] }); + const row = s.find('Table').props.dataSource[0]; + const cell = s.cell('采集状态', row); + assert.equal(s.findAll('Tag', cell).length, 1); + assert.equal(text(cell), '需要登录'); + assert.equal(s.find('Tooltip', cell).props.title, '浏览器需要登录，请登录并同步账号后重试。'); + assert.doesNotMatch(JSON.stringify(cell), /作品|评论|上次完成|2026-10-06/); + assert.equal(text(s.cell('业务状态', row)), '正常'); + assert.equal(text(s.cell('调度状态', row)), '可运行'); +}); + +test('未登录环境只显示待登录 Tag，不显示采集成功或完成时间', async () => { + const s = await ready({ accounts: [] }); + const row = s.find('Table').props.dataSource[0]; + const cell = s.cell('采集状态', row); + assert.equal(cell.type.displayName, 'Tag'); + assert.equal(text(cell), '待登录'); + assert.equal(s.find('Tooltip', cell), undefined); +}); + +test('错误只在悬停提示中展示，任一失败不被其他成功或缺失状态掩盖', async () => { + const s = await ready({ environments: [], accounts: [{ id: 'account', works_collection: { status: 'failed', last_error: 'network HTTP 502' } }] }); + const row = s.find('Table').props.dataSource[0]; + const cell = s.cell('采集状态', row); + assert.equal(text(cell), '采集失败'); + assert.equal(s.find('Tooltip', cell).props.title, 'network HTTP 502'); +}); + +test('成功仅显示一个 Tag，没有异常提示；缺失状态不显示完成', async () => { + const s = await ready({ environments: [], accounts: [ + { id: 'success', works_collection: { status: 'succeeded' }, comments_collection: { status: 'succeeded' } }, + { id: 'unknown', works_collection: { status: 'succeeded' } }, + ] }); + const rows = s.find('Table').props.dataSource; + const success = s.cell('采集状态', rows.find(row => row.id === 'success')); + const unknown = s.cell('采集状态', rows.find(row => row.id === 'unknown')); + assert.equal(success.type.displayName, 'Tag'); + assert.equal(text(success), '采集完成'); + assert.equal(s.find('Tooltip', success), undefined); + assert.equal(unknown.type.displayName, 'Tag'); + assert.equal(text(unknown), '状态未知'); +}); + +test('详情保留作品与评论的详细状态，读取失败不冒充待采集', () => { + const source = readFileSync(resolve(__dirname, '../src/pages/accounts/$id/index.tsx'), 'utf8'); + assert.match(source, /collectionStatusPresentation/); + assert.match(source, /作品采集/); + assert.match(source, /评论采集/); + assert.doesNotMatch(source, /const collectionStatusMeta/); + assert.match(source, /采集状态读取失败/); +}); + +test('真实未登录才弹二维码，正常登录资料不被旧采集错误误导', async () => { + const s = await ready({ environments: [], verification: { status: 'manual_login', reason: 'awaiting_login', account_id: 'account-1' } }); + let row = s.find('Table').props.dataSource[0]; + await s.findAll('Button', s.cell('操作', row)).find(button => text(button) === '同步资料').props.onClick(); s.render(); + assert.equal(s.find('AccountLoginModal').props.accountID, 'account-1'); + row = s.find('Table').props.dataSource[0]; + assert.match(text(s.cell('操作', row)), /重新登录/); + s.find('AccountLoginModal').props.onClose(); s.render(); + s.options.verification = undefined; + await s.findAll('Button', s.cell('操作', row)).find(button => text(button) === '重新登录').props.onClick(); s.render(); + assert.equal(s.find('AccountLoginModal'), undefined); + assert.match(text(s.cell('操作', s.find('Table').props.dataSource[0])), /同步资料/); +}); + +test('检查登录状态发现需要登录时不自行弹二维码，确认登录成功会更新主操作', async () => { + const s = await ready({ environments: [], accounts: [{ id: 'account-1', name: '账号', environment: { alias: 'env-account' } }], verification: { status: 'manual_login', reason: 'awaiting_login', account_id: 'account-1' } }); + let row = s.find('Table').props.dataSource[0]; + await s.findAll('Button', s.cell('操作', row)).find(button => text(button) === '检查状态').props.onClick(); s.render(); + assert.equal(s.find('AccountLoginModal'), undefined); + assert.ok(s.messages.includes('状态已更新，账号需要重新登录。')); + row = s.find('Table').props.dataSource[0]; + await s.findAll('Button', s.cell('操作', row)).find(button => text(button) === '重新登录').props.onClick(); s.render(); + const dialog = s.find('AccountLoginModal'); + assert.ok(dialog); + dialog.props.onSuccess(); await flush(); s.render(); + assert.match(text(s.cell('操作', s.find('Table').props.dataSource[0])), /同步资料/); +}); + +test('暂停账号和停止浏览器分别确认并使用不同路径，停止不暂停账号', async () => { + const s = await ready({ environments: [] }); + let row = s.find('Table').props.dataSource[0]; + let menu = s.find('Dropdown', s.cell('操作', row)).props.menu; + await menu.onClick({ key: 'stop' }); + assert.equal(s.calls.some(call => call.path.endsWith('/stop')), false); + assert.match(s.confirmations.at(-1).content, /不暂停账号/); + await s.confirmations.at(-1).onOk(); s.render(); + assert.ok(s.calls.some(call => call.path === '/browsers/env-account/stop')); + assert.ok(!s.calls.some(call => call.path.endsWith('/pause'))); + assert.match(text(s.cell('操作', s.find('Table').props.dataSource[0])), /启动环境/); + row = s.find('Table').props.dataSource[0]; + menu = s.find('Dropdown', s.cell('操作', row)).props.menu; + await menu.onClick({ key: 'pause' }); + await s.confirmations.at(-1).onOk(); s.render(); + assert.ok(s.calls.some(call => call.path === '/phase-a/accounts/account-1/pause')); + assert.match(text(s.cell('操作', s.find('Table').props.dataSource[0])), /恢复运行/); +}); + +test('实时状态读取失败不使用旧运行记录，显示检查状态及实际错误', async () => { + const s = await ready({ environments: [], browserError: new Error('网关状态读取失败'), liveBrowserError: new Error('网关仍不可达') }); + const row = s.find('Table').props.dataSource[0]; + assert.match(text(s.cell('操作', row)), /检查状态/); + assert.ok(s.findAll('Alert').some(alert => alert.props.title === '浏览器状态读取失败')); + await s.findAll('Button', s.cell('操作', row)).find(button => text(button) === '检查状态').props.onClick(); s.render(); + assert.ok(s.messages.includes('网关仍不可达')); + assert.equal(s.find('AccountLoginModal'), undefined); + assert.ok(!s.calls.some(call => call.path.endsWith('/start'))); +}); diff --git a/web/tests/account-memory-edit.test.cjs b/web/tests/account-memory-edit.test.cjs new file mode 100644 index 0000000..24b0d25 --- /dev/null +++ b/web/tests/account-memory-edit.test.cjs @@ -0,0 +1,104 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { test } = require('node:test'); +const { Module, createRequire } = require('node:module'); +const { resolve } = require('node:path'); +const ts = require('typescript'); +const requireWeb = createRequire(resolve(__dirname, '../package.json')); +const file = resolve(__dirname, '../src/pages/accounts/$id/edit.tsx'); + +function setup({ environment = { alias: 'env-a', memory_limit_mb: 8192, runtime_id: 'running', fingerprint: { seed: 42 }, profile_id: 'existing', gateway: 'gw-a' }, saveError } = {}) { + const states = [], forms = [], effects = [], calls = [], messages = []; + let cursor = 0, formCursor = 0; + const marker = name => Object.assign(() => null, { displayName: name }); + const Form = Object.assign(marker('Form'), { + Item: marker('Form.Item'), + useForm() { + const index = formCursor++; + forms[index] ||= { values: {}, setFieldsValue(values) { Object.assign(this.values, values); }, getFieldValue(name) { return this.values[name]; }, async validateFields() { return this.values; } }; + return [forms[index]]; + }, + }); + const antd = { Form, App: { useApp: () => ({ message: { success: text => messages.push(text), error: text => messages.push(text) } }) } }; + for (const name of ['Alert', 'Button', 'Card', 'Col', 'Flex', 'Row', 'Select', 'Space', 'InputNumber']) antd[name] = marker(name); + antd.Input = Object.assign(marker('Input'), { Password: marker('Password'), TextArea: marker('TextArea') }); + antd.Descriptions = Object.assign(marker('Descriptions'), { Item: marker('Descriptions.Item') }); + antd.Typography = { Title: marker('Title'), Text: marker('Text'), Paragraph: marker('Paragraph') }; + const loaded = new Module(file, module); + loaded.require = id => { + if (id === 'react') return { + useState(value) { const index = cursor++; if (!(index in states)) states[index] = value; return [states[index], next => { states[index] = typeof next === 'function' ? next(states[index]) : next; }]; }, + useCallback: callback => callback, + useEffect(effect) { if (!effects.length) effects.push(effect); }, + }; + if (id === 'antd') return antd; + if (id === '@umijs/max') return { useParams: () => ({ id: 'account-a' }), history: { push() {} } }; + if (id === '@/components/PageActions') return { usePageActions() {}, usePageInfo() {} }; + if (id === '@/services/api') return { + async getList() { return { data: [{ id: 'account-a', platform: 'douyin' }] }; }, + async getOne() { return { environment }; }, + async creatorUpdate(path, data) { calls.push({ path, data }); if (saveError) throw saveError; return { ...environment, ...data }; }, + }; + if (id === '@/utils/helpers') return { conflictMessage: (error, fallback) => error.message || fallback, dateTime: String }; + if (id === '@/components/accounts/FingerprintFields') return { default: marker('FingerprintFields'), fingerprintFormValues: fingerprint => ({ fingerprint }), fingerprintPayload: value => value }; + if (id.startsWith('@/components/')) return { default: marker(id) }; + return requireWeb(id); + }; + loaded._compile(ts.transpileModule(readFileSync(file, 'utf8'), { fileName: file, compilerOptions: { module: ts.ModuleKind.CommonJS, jsx: ts.JsxEmit.ReactJSX, inlineSourceMap: true, inlineSources: true } }).outputText, file); + let tree; + function render() { cursor = 0; formCursor = 0; tree = loaded.exports.default(); return tree; } + function all(name, node = tree) { + if (!node || typeof node !== 'object') return []; + return [...(node.type?.displayName === name ? [node] : []), ...[node.props?.children].flat(Infinity).flatMap(child => child == null ? [] : all(name, child))]; + } + render(); + return { forms, calls, messages, render, all, async load() { effects[0](); for (let i = 0; i < 8; i++) await Promise.resolve(); render(); } }; +} + +test('编辑回填真实内存上限，运行中独立保存只发内存字段，不触发stop/start或指纹保存', async () => { + const s = setup(); await s.load(); + const field = s.all('Form.Item').find(node => node.props.name === 'memory_limit_mb'); + assert.ok(field, '编辑页必须提供独立内存设置'); + const memoryForm = s.forms.find(form => Object.hasOwn(form.values, 'memory_limit_mb')); + assert.equal(memoryForm.values.memory_limit_mb, 8192); + assert.equal(field.props.initialValue, undefined); + assert.equal(s.all('InputNumber')[0].props.disabled, undefined); + const form = s.all('Form').find(node => node.props.form === memoryForm); + assert.equal(form.props.disabled, false, '运行中也允许保存内存设置'); + memoryForm.values.memory_limit_mb = 4096; + const saving = form.props.onFinish(memoryForm.values); + s.render(); + assert.equal(s.all('Form').find(node => node.props.form === memoryForm).props.disabled, true); + await saving; + assert.deepEqual(s.calls, [{ path: '/creator/accounts/account-a/fingerprint', data: { memory_limit_mb: 4096 } }]); + assert.match(s.messages.at(-1), /下次启动.*生效/); +}); + +test('编辑缺失内存字段不伪造2048；原生规则拒绝空值、越界与小数', async () => { + const s = setup({ environment: { alias: 'env-a' } }); await s.load(); + const field = s.all('Form.Item').find(node => node.props.name === 'memory_limit_mb'); + assert.ok(field); + const memoryForm = s.forms.find(form => Object.hasOwn(form.values, 'memory_limit_mb')); + assert.equal(memoryForm.values.memory_limit_mb, undefined); + const Schema = requireWeb('@rc-component/async-validator').default; + const schema = new Schema({ memory_limit_mb: field.props.rules }); + for (const value of [512, 65536]) await schema.validate({ memory_limit_mb: value }); + for (const value of [undefined, null, 511, 65537, 512.5]) await assert.rejects(schema.validate({ memory_limit_mb: value })); +}); + +test('未绑定环境不允许保存内存上限，也不创建环境', async () => { + const s = setup({ environment: null }); await s.load(); + const memoryForm = s.forms.find(form => Object.hasOwn(form.values, 'memory_limit_mb')); + const form = s.all('Form').find(node => node.props.form === memoryForm); + assert.equal(form.props.disabled, true); + await form.props.onFinish({ memory_limit_mb: 4096 }); + assert.deepEqual(s.calls, []); +}); + +test('内存保存失败明确显示原因，不提示成功', async () => { + const s = setup({ saveError: new Error('环境不可用') }); await s.load(); + const memoryForm = s.forms.find(form => Object.hasOwn(form.values, 'memory_limit_mb')); + assert.ok(memoryForm); + await s.all('Form').find(node => node.props.form === memoryForm).props.onFinish({ memory_limit_mb: 4096 }); + assert.deepEqual(s.messages, ['环境不可用']); +}); diff --git a/web/tests/account-start-readiness.test.cjs b/web/tests/account-start-readiness.test.cjs new file mode 100644 index 0000000..7415d5a --- /dev/null +++ b/web/tests/account-start-readiness.test.cjs @@ -0,0 +1,133 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { createRequire, Module } = require('node:module'); +const { resolve } = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); +const ts = require('typescript'); + +const requireWeb = createRequire(resolve(__dirname, '../package.json')); +function compile(relativePath, imports = {}) { + const path = resolve(__dirname, relativePath); + const loaded = new Module(path, module); + loaded.require = id => imports[id] || requireWeb(id); + loaded._compile(ts.transpileModule(readFileSync(path, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, path); + return loaded.exports; +} +const helpers = compile('../src/utils/helpers.ts'); +const actions = compile('../src/components/accounts/account-actions.ts', { + '@/services/api': {}, + '@/utils/account-collection-status': compile('../src/utils/account-collection-status.ts'), +}); +const account = { + id: '1004291301', runtime_status: 'active', login_status: 'logged_in', + environment: { alias: 'environment', schedule_block_reason: 'runtime_missing' }, +}; +const stopped = { alias: 'environment', state: 'missing', status: '未记录运行实例', gateway_reachable: true }; + +const source = ts.createSourceFile('AccountManagementList.tsx', + readFileSync(resolve(__dirname, '../src/components/accounts/AccountManagementList.tsx'), 'utf8'), + ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +let renderer; +function visit(node) { + if (ts.isObjectLiteralExpression(node) && node.properties.some(property => + ts.isPropertyAssignment(property) && property.name.getText(source) === 'title' && property.initializer.text === '操作')) { + renderer = node.properties.find(property => ts.isPropertyAssignment(property) && property.name.getText(source) === 'render').initializer.getText(source); + } + ts.forEachChild(node, visit); +} +visit(source); +assert.ok(renderer, '必须存在账号操作列'); + +function renderActions(row = account, browser = stopped, actionBusy = '') { + const calls = []; + const code = ts.transpileModule(`exports.render = ${renderer};`, { + compilerOptions: { jsx: ts.JsxEmit.ReactJSX, module: ts.ModuleKind.CommonJS }, + }).outputText; + const context = { + exports: {}, require: requireWeb, ...requireWeb('antd'), + DownOutlined: requireWeb('@ant-design/icons').DownOutlined, + ...helpers, ...actions, browsers: browser ? [browser] : [], verifiedLogin: {}, actionBusy, + performPrimary: (row, intent) => calls.push([row.id, intent]), moreAction() {}, setEnvironmentTarget() {}, + }; + vm.runInNewContext(code, context); + const cell = context.exports.render(undefined, row); + const [tooltip, dropdown] = cell.props.children; + return { button: tooltip.props.children, tooltip, menu: dropdown.props.menu, calls }; +} + +test('未启动只阻止采集调度，不阻止启动环境', () => { + const readiness = helpers.accountReadiness(account, account.environment); + assert.equal(readiness.ready, false); + assert.equal(readiness.reason, 'runtime_missing'); + assert.equal(readiness.label, '运行环境未启动'); + assert.equal(readiness.canStart, true); +}); + +test('可调度账号仍可启动，未知阻塞原因不能绕过', () => { + assert.equal(helpers.accountReadiness(account, {}).canStart, true); + assert.equal(helpers.accountReadiness(account, { schedule_block_reason: 'unknown_block' }).canStart, false); +}); + +for (const [name, row, binding, error] of [ + ['未绑定', account, null], + ['状态读取失败', account, account.environment, new Error('unavailable')], + ['停止待确认', account, { ...account.environment, runtime_cleanup_pending: true }], + ['固定出口异常', account, { ...account.environment, network_exit_id: 'proxy', network_exit_health: 'unhealthy' }], + ['账号暂停', { ...account, runtime_status: 'paused' }, account.environment], + ['暂停账号仍在停止', { ...account, runtime_status: 'paused' }, { ...account.environment, runtime_id: 'runtime' }], +]) { + test(`${name}不能通过启动环境绕过限制`, () => { + assert.equal(helpers.accountReadiness(row, binding, error).canStart, false); + }); +} + +test('1004291301 未启动时主按钮和更多菜单均允许启动', () => { + const { button, tooltip, menu, calls } = renderActions(); + assert.equal(button.props.children, '启动环境'); + assert.equal(button.props.disabled, false); + assert.equal(tooltip.props.title, undefined); + assert.equal(menu.items.find(item => item.key === 'start').disabled, false); + button.props.onClick(); + assert.deepEqual(calls, [['1004291301', 'start']]); +}); + +test('需要重新登录的未启动账号可点击重新登录', () => { + const { button, calls } = renderActions({ ...account, login_status: 'manual_login' }); + assert.equal(button.props.children, '重新登录'); + assert.equal(button.props.disabled, false); + button.props.onClick(); + assert.deepEqual(calls, [['1004291301', 'login']]); +}); + +for (const environment of [ + { ...account.environment, runtime_cleanup_pending: true }, + { ...account.environment, network_exit_id: 'proxy', network_exit_health: 'unhealthy' }, + { ...account.environment, schedule_block_reason: 'unknown_block' }, +]) { + test(`列表启动入口仍遵守限制 ${JSON.stringify(environment)}`, () => { + const { button, tooltip, menu } = renderActions({ ...account, environment }); + assert.equal(button.props.disabled, true); + assert.ok(tooltip.props.title); + assert.equal(menu.items.find(item => item.key === 'start').disabled, true); + }); +} + +test('暂停账号保留恢复运行入口，不能从更多菜单直接启动', () => { + const paused = { ...account, runtime_status: 'paused' }; + const { button, menu } = renderActions(paused); + assert.equal(button.props.children, '恢复运行'); + assert.equal(button.props.disabled, false); + assert.equal(menu.items.find(item => item.key === 'start').disabled, true); +}); + +test('操作执行中仍禁用主按钮，运行中不重复启动', () => { + assert.equal(renderActions(account, stopped, account.id).button.props.disabled, true); + const running = { ...stopped, state: 'running', status: 'ready' }; + const { button, menu } = renderActions({ ...account, environment: { alias: 'environment' } }, running); + assert.equal(button.props.children, '同步资料'); + assert.equal(button.props.disabled, false); + assert.equal(menu.items.find(item => item.key === 'start').disabled, true); +}); diff --git a/web/tests/comment-aggregation.test.cjs b/web/tests/comment-aggregation.test.cjs index 0902552..cb2a2d1 100644 --- a/web/tests/comment-aggregation.test.cjs +++ b/web/tests/comment-aggregation.test.cjs @@ -51,6 +51,7 @@ Module._load = function (request, parent, isMain) { if (request === "@/utils/helpers") return { conflictMessage: (error, fallback) => error?.message || fallback, + dateTime: (value) => value, }; } return originalLoad.call(this, request, parent, isMain); diff --git a/web/tests/comment-work-cover.test.cjs b/web/tests/comment-work-cover.test.cjs index 72080fd..c357726 100644 --- a/web/tests/comment-work-cover.test.cjs +++ b/web/tests/comment-work-cover.test.cjs @@ -29,6 +29,7 @@ function harness(fetch) { const revoked = []; let cursor = 0; let effect; + let effectIndex; const react = { createElement: (type, props, ...children) => ({ type, @@ -49,6 +50,7 @@ function harness(fetch) { const index = cursor++; if (!effects[index]) { effect = fn; + effectIndex = index; effects[index] = { deps }; } }; @@ -86,13 +88,13 @@ function harness(fetch) { cursor = 0; const tree = Component(props); if (effect) { - effects[2].cleanup = effect(); + effects[effectIndex].cleanup = effect(); effect = null; } return tree; }, cleanup() { - effects[2]?.cleanup?.(); + effects[effectIndex]?.cleanup?.(); }, }; } @@ -118,7 +120,7 @@ test("comment thumbnails load only the cached local cover and release its image test("missing cached cover is explicit and never replaced by a remote image", async () => { const h = harness(async () => { - throw new Error("作品封面尚未缓存"); + throw Object.assign(new Error("作品封面尚未缓存"), {status:404}); }); const props = { workID: "missing-cover", title: "作品封面" }; h.render(props); @@ -130,3 +132,22 @@ test("missing cached cover is explicit and never replaced by a remote image", as h.cleanup(); assert.deepEqual(h.revoked, []); }); + + +test("event thumbnail uses its event endpoint at a restrained width", async () => { + const h=harness(async()=>({image:true}));const props={workID:'123',coverPath:'/creator/events/7/cover',width:48}; + h.render(props);await tick();const tree=h.render(props); + assert.equal(tree.props.style.width,48);assert.deepEqual(h.calls,['/creator/events/7/cover']);h.cleanup(); +}); +test("server errors and invalid image data are not disguised as uncached covers", async () => { + const h=harness(async()=>{throw Object.assign(new Error('read failed'),{status:500})}); + const props={workID:'123'};h.render(props);await tick(); + assert.equal(h.render(props).props.children[0].props.children[0],'封面读取失败'); + const loaded=harness(async()=>({image:true}));loaded.render(props);await tick();loaded.render(props).props.onError(); + assert.equal(loaded.render(props).props.children[0].props.children[0],'封面读取失败');loaded.cleanup(); +}); +test("a cover resolved after unmount cannot allocate an image URL", async () => { + let resolve;const h=harness(()=>new Promise(r=>resolve=r));const props={workID:'123'}; + h.render(props);h.cleanup();resolve({image:true});await tick(); + assert.equal(h.render(props).type,'Flex');assert.deepEqual(h.revoked,[]); +}); diff --git a/web/tests/date-time-format.test.cjs b/web/tests/date-time-format.test.cjs new file mode 100644 index 0000000..da50726 --- /dev/null +++ b/web/tests/date-time-format.test.cjs @@ -0,0 +1,27 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve, dirname } = require('node:path'); +const Module = require('node:module'); +const ts = require('../node_modules/typescript'); + +const helperPath = resolve(__dirname, '../src/utils/helpers.ts'); +const compiled = ts.transpileModule(readFileSync(helperPath, 'utf8'), { + fileName: helperPath, + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, +}).outputText; +const helperModule = new Module(helperPath, module); +helperModule.filename = helperPath; +helperModule.paths = Module._nodeModulePaths(dirname(helperPath)); +helperModule._compile(compiled, helperPath); +const { dateTime } = helperModule.exports; + +test('dateTime uses YY/MM/DD HH:mm:ss with a 24-hour clock', () => { + const late = new Date(2026, 9, 6, 23, 4, 5); + const midnight = new Date(2026, 9, 6, 0, 4, 5); + + assert.equal(dateTime(late.toISOString()), '26/10/06 23:04:05'); + assert.equal(dateTime(midnight.toISOString()), '26/10/06 00:04:05'); + assert.equal(dateTime(null), '—'); + assert.equal(dateTime('not-a-date'), 'not-a-date'); +}); diff --git a/web/tests/ellipsis-tooltip.test.cjs b/web/tests/ellipsis-tooltip.test.cjs index 7c6ec8e..2be16d7 100644 --- a/web/tests/ellipsis-tooltip.test.cjs +++ b/web/tests/ellipsis-tooltip.test.cjs @@ -7,7 +7,6 @@ const vm = require('node:vm'); const sourceRoot = path.join(__dirname, '../src'); const cases = [ - 'components/accounts/AccountManagementList.tsx', 'pages/accounts/$id/index.tsx', 'pages/accounts/$id/edit.tsx', 'pages/accounts/monitoring/$id/index.tsx', @@ -16,6 +15,8 @@ const cases = [ 'pages/network-exits/index.tsx', 'pages/network-exits/$id/index.tsx', 'pages/accounts/import.tsx', + 'pages/creator/events/index.tsx', + 'pages/creator/messages/index.tsx', ]; function attribute(opening, name) { @@ -97,18 +98,6 @@ function renderField(file, index, bindings) { }); } -for (const [label, env, expected] of [ - ['long name', { name: '完整运行环境名称'.repeat(30), alias: '别名' }, '完整运行环境名称'.repeat(30)], - ['alias', { name: '', alias: '完整环境别名'.repeat(30) }, '完整环境别名'.repeat(30)], - ['empty environment name', {}, '—'], -]) { - test(`account environment hover preserves ${label}`, () => { - const field = renderField('components/accounts/AccountManagementList.tsx', -1, { env }); - assert.equal(field.props.children, expected); - assert.equal(field.props.ellipsis.tooltip, true); - }); -} - for (const visible of [false, true]) { test(`gateway token hover respects visible=${visible}`, () => { const token = 'complete-gateway-token'.repeat(20); diff --git a/web/tests/event-aggregation-page.test.cjs b/web/tests/event-aggregation-page.test.cjs new file mode 100644 index 0000000..78e7d83 --- /dev/null +++ b/web/tests/event-aggregation-page.test.cjs @@ -0,0 +1,103 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const Module = require('node:module'); +const ts = require('typescript'); +const React = require('react'); + +function transpile(file) { return ts.transpileModule(readFileSync(file,'utf8'), {compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2020}}).outputText; } +function harness(options={}) { + const states=[],effects=[],queued=[],calls=[]; + let cursor=0; + const antd={}; + for(const name of ['Alert','Avatar','Button','Card','Flex','Select','Table','Tag','Tooltip']){antd[name]=()=>null;antd[name].displayName=name;} + antd.Typography={Text:()=>null,Paragraph:()=>null,Link:()=>null}; + const utilityFile=resolve(__dirname,'../src/utils/account-events.ts'); + const utility=new Module(utilityFile,module);utility._compile(transpile(utilityFile),utilityFile); + const file=resolve(__dirname,'../src/pages/creator/events/index.tsx'); + const loaded=new Module(file,module);loaded.filename=file; + loaded.require=id=>{ + if(id==='react')return{useState(initial){const index=cursor++;if(!(index in states))states[index]=initial;return[states[index],value=>{states[index]=typeof value==='function'?value(states[index]):value;}]}, + useEffect(callback,deps){const index=cursor++;const prior=effects[index];if(!prior||deps.some((v,i)=>!Object.is(v,prior.deps[i]))){if(prior?.cleanup)prior.cleanup();effects[index]={deps};queued.push(()=>{effects[index].cleanup=callback();});}}}; + if(id==='react/jsx-runtime')return require(id); + if(id==='antd')return antd; + if(id==='@ant-design/icons')return{ReloadOutlined:()=>null,VideoCameraOutlined:()=>null}; + if(id==='@/components/creator/WorksGallery')return{WorkCover:Object.assign(()=>null,{displayName:'WorkCover'})}; + if(id==='@/utils/account-events')return utility.exports; + if(id==='@/utils/table')return{useTableSticky:()=>false,tablePagination:()=>({})}; + if(id==='@/utils/helpers')return{dateTime:value=>`time:${value}`,conflictMessage:(cause,fallback)=>cause.message||fallback}; + if(id==='@/services/api')return{creatorGet:async path=>{calls.push(path);if(options.get)return options.get(path);if(path==='/creator/accounts')return options.accounts||[];return options.page||{data:[],total:0};}}; + return require(id); + }; + loaded._compile(transpile(file),file); + return{antd,calls,render(){cursor=0;return loaded.exports.default();},async effects(){while(queued.length)queued.shift()();for(let i=0;i<8;i++)await Promise.resolve();},dispose(){for(const effect of effects)effect?.cleanup?.();}}; +} +function all(node,predicate){const result=[];function visit(value){if(Array.isArray(value))return value.forEach(visit);if(!React.isValidElement(value))return;if(predicate(value))result.push(value);visit(value.props.children);}visit(node);return result;} +function text(node){if(node==null)return'';if(typeof node==='string'||typeof node==='number')return String(node);if(Array.isArray(node))return node.map(text).join('');return React.isValidElement(node)?text(node.props.children):'';} +function select(h,tree,label){return all(tree,node=>node.type===h.antd.Select&&node.props['aria-label']===label)[0];} +function table(h,tree){return all(tree,node=>node.type===h.antd.Table)[0];} + +test('only enabled accounts are offered, event filters are server-wide and changes reset page',async t=>{ + const h=harness({accounts:[{id:'on',name:'开启账号',platform_account_key:'1',listener_enabled:true},{id:'off',name:'关闭账号',listener_enabled:false}],page:{data:[{id:'event'}],total:40}});t.after(()=>h.dispose()); + h.render();await h.effects();let tree=h.render(); + assert.deepEqual(select(h,tree,'所属账号').props.options.map(o=>o.value),['on']); + assert.equal(select(h,tree,'所属账号').props.options[0].label,'开启账号'); + assert.equal(table(h,tree).props.pagination.total,40); + table(h,tree).props.pagination.onChange(2,25);h.render();await h.effects();tree=h.render(); + assert.equal(table(h,tree).props.pagination.current,2); + select(h,tree,'所属账号').props.onChange('on');h.render();await h.effects();tree=h.render(); + assert.equal(table(h,tree).props.pagination.current,1); + select(h,tree,'事件类型').props.onChange('dm');h.render();await h.effects();tree=h.render(); + select(h,tree,'接收时间范围').props.onChange(24);h.render();await h.effects();tree=h.render(); + const query=new URLSearchParams(h.calls.filter(p=>p.startsWith('/creator/events')).at(-1).split('?')[1]); + assert.equal(query.get('event_type'),'dm');assert.equal(query.get('account_id'),'on');assert.equal(query.get('hours'),'24');assert.equal(query.get('page'),'1'); + table(h,tree).props.pagination.onChange(3,50);h.render();await h.effects();tree=h.render(); + assert.equal(table(h,tree).props.pagination.current,1);assert.equal(table(h,tree).props.pagination.pageSize,50); + all(tree,node=>node.type===h.antd.Button)[0].props.onClick();h.render();await h.effects(); + assert.equal(h.calls.filter(p=>p.startsWith('/creator/events')).length,7); +}); + +test('event contents distinguish missing data, non-text messages and baseline history',async t=>{ + const h=harness();t.after(()=>h.dispose());h.render();await h.effects();const cols=table(h,h.render()).props.columns; + const column=title=>cols.find(c=>c.title===title); + assert.equal(text(column('事件类型').render('comment')),'评论'); + assert.equal(text(column('内容').render(null,{message_text:'测试内容',message_type:'text'})),'测试内容'); + assert.equal(text(column('内容').render(null,{message_text:'',message_type:'image'})),'图片消息'); + assert.equal(text(column('内容').render(null,{message_text:'',message_type:'text'})),'—'); + assert.equal(column('发生时间').render(null),'—');assert.equal(column('发生时间').render('date'),'time:date'); + assert.equal(column('对应作品').render(''),'—'); + const link=column('对应作品').render('123',{id:'event',work_id:'123',work_url:'https://www.douyin.com/video/123',has_work_cover:true}); + assert.equal(link.props.href,'https://www.douyin.com/video/123'); + const cover=all(link,n=>n.type.displayName==='WorkCover')[0]; + assert.equal(cover.props.width,48);assert.equal(cover.props.coverPath,'/creator/events/event/cover'); + const missing=column('对应作品').render('123',{id:'event',work_id:'123',work_url:'',has_work_cover:false,work_cover_error:'下载失败'}); + assert.equal(missing.props.href,'https://www.douyin.com/video/123'); + assert.equal(all(missing,n=>n.type===h.antd.Avatar)[0].props.size,48); + assert.equal(all(missing,n=>n.type===h.antd.Tooltip)[0].props.title,'下载失败'); + assert.equal(text(column('来源').render(true)),'历史记录');assert.equal(text(column('来源').render(false)),'开启后通知'); + const user=column('互动用户').render(null,{interactor_uid:'22',interactor_name:'用户',interactor_sec_uid:'MS4wLjABAAAAprofile'}); + const profile=all(user,n=>n.type===h.antd.Typography.Link)[0]; + assert.equal(profile.props.href,'https://www.douyin.com/user/MS4wLjABAAAAprofile'); + assert.equal(profile.props.target,'_blank');assert.match(text(user),/用户/);assert.match(text(user),/UID：22/); + const unknown=column('互动用户').render(null,{interactor_uid:'22',interactor_name:'',interactor_sec_uid:''}); + assert.equal(all(unknown,n=>n.type===h.antd.Typography.Link).length,0);assert.match(text(unknown),/昵称未获取/); + assert.equal(text(column('所属账号').render(null,{account_name:'账号',account_uid:'1'})),'账号'); + assert.deepEqual(cols.slice(0,2).map(c=>c.title),['发生时间','接收时间']); +}); + +test('read failures are visible and retries do not fake an empty success',async t=>{ + let failed=true;const h=harness({get:async path=>{if(failed)throw new Error('服务不可用');return path==='/creator/accounts'?[]:{data:[{id:'1'}],total:1};}});t.after(()=>h.dispose()); + h.render();await h.effects();let tree=h.render(); + const error=all(tree,n=>n.type===h.antd.Alert)[0];assert.equal(error.props.title,'事件读取失败');assert.equal(error.props.description,'服务不可用'); + failed=false;all(tree,n=>n.type===h.antd.Button)[0].props.onClick();h.render();await h.effects();tree=h.render(); + assert.equal(all(tree,n=>n.type===h.antd.Alert).length,0);assert.equal(table(h,tree).props.dataSource.length,1); +}); + +test('a late response for previous filters cannot overwrite current events',async t=>{ + let resolveOld; + const h=harness({get:path=>{if(path==='/creator/accounts')return Promise.resolve([]);if(path.includes('event_type=comment'))return Promise.resolve({data:[{id:'current'}],total:1});return new Promise(resolve=>{resolveOld=resolve;});}});t.after(()=>h.dispose()); + let tree=h.render();await h.effects();select(h,tree,'事件类型').props.onChange('comment');h.render();await h.effects(); + resolveOld({data:[{id:'old'}],total:99});await h.effects();tree=h.render(); + assert.equal(table(h,tree).props.dataSource[0].id,'current');assert.equal(table(h,tree).props.pagination.total,1); +}); diff --git a/web/tests/event-aggregation.test.cjs b/web/tests/event-aggregation.test.cjs new file mode 100644 index 0000000..3089ccd --- /dev/null +++ b/web/tests/event-aggregation.test.cjs @@ -0,0 +1,75 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const ts = require('typescript'); + +function loadUtility() { + const filename = path.resolve(__dirname, '../src/utils/account-events.ts'); + const source = fs.readFileSync(filename, 'utf8'); + const compiled = ts.transpileModule(source, {compilerOptions:{module:ts.ModuleKind.CommonJS, target:ts.ScriptTarget.ES2020}}).outputText; + const module = {exports:{}}; + new Function('module','exports',compiled)(module,module.exports); + return module.exports; +} + +test('event links require real profile identifiers and matching canonical work URLs', () => { + const {eventUserProfileURL,eventWorkURL}=loadUtility(); + assert.equal(eventUserProfileURL('MS4wLjABAAAAprofile'),'https://www.douyin.com/user/MS4wLjABAAAAprofile'); + for(const id of [undefined,'','../123','not a profile']) assert.equal(eventUserProfileURL(id),undefined); + assert.equal(eventWorkURL('123'),'https://www.douyin.com/video/123'); + assert.equal(eventWorkURL('123','https://www.douyin.com/note/123'),'https://www.douyin.com/note/123'); + for(const url of ['https://www.douyin.com/video/456','https://wwwXdouyinYcom/video/123','https://example.com/video/123']) assert.equal(eventWorkURL('123',url),undefined); + for(const id of ['', '0', '../123']) assert.equal(eventWorkURL(id),undefined); +}); + +test('listener status distinguishes stopped, startup, ready, interruption and error', () => { + const {listenerStatusLabel} = loadUtility(); + assert.equal(listenerStatusLabel('stopped'), '已关闭'); + assert.equal(listenerStatusLabel('starting'), '启动中'); + assert.equal(listenerStatusLabel('ready'), '接收正常'); + assert.equal(listenerStatusLabel('gap'), '可能有遗漏'); + assert.equal(listenerStatusLabel('stopping'), '关闭中'); + assert.equal(listenerStatusLabel('error'), '异常'); + assert.equal(listenerStatusLabel('unrecognized'), 'unrecognized'); +}); + +test('aggregation query includes pagination and filters without inventing defaults', () => { + const {eventQuery} = loadUtility(); + const query = new URLSearchParams(eventQuery({accountId:'account 1', eventType:'comment', hours:0},2,50)); + assert.equal(query.get('account_id'), 'account 1'); + assert.equal(query.get('event_type'), 'comment'); + assert.equal(query.get('hours'), '0'); + assert.equal(query.get('page'), '2'); + assert.equal(query.get('page_size'), '50'); + const empty = new URLSearchParams(eventQuery({hours:24},1,25)); + assert.equal(empty.has('account_id'),false); assert.equal(empty.has('event_type'),false); +}); + +test('all supported event types and non-text messages have readable labels', () => { + const {eventTypeLabel,messageTypeLabel} = loadUtility(); + for (const kind of ['like','comment','follow','repost']) assert.notEqual(eventTypeLabel(kind),kind); + for (const kind of ['image','voice','video','sticker','non_text']) assert.notEqual(messageTypeLabel(kind),kind); + assert.equal(messageTypeLabel('text'),'—'); +}); + +test('owned account list has an opt-in listener column and blocks pending environments', () => { + const source = fs.readFileSync(path.resolve(__dirname,'../src/components/accounts/AccountManagementList.tsx'),'utf8'); + assert.match(source,/title: '监听状态'/); + assert.match(source,/checked=\{Boolean\(row.listener_enabled\)\}/); + assert.match(source,/row.login_status !== 'logged_in' && !row.listener_enabled/); + assert.match(source,/if \(row.pending_environment\) return/); + assert.match(source,/\/listener/); +}); + +test('event aggregation is listed in the left menu and keeps source scope explicit', () => { + const metadata = fs.readFileSync(path.resolve(__dirname,'../src/utils/metadata.tsx'),'utf8'); + assert.match(metadata,/\/creator\/events/); assert.match(metadata,/事件聚合/); + const source = fs.readFileSync(path.resolve(__dirname,'../src/pages/creator/events/index.tsx'),'utf8'); + assert.match(source,/listener_enabled/); + assert.match(source,/setPagination\(\{ page: 1/); + assert.match(source,/停止监听的账号不在此列表展示/); + assert.doesNotMatch(source,/label: '私信'/); + assert.match(source,/私信在独立收件箱展示/); + assert.match(source,/开启后通知/); +}); diff --git a/web/tests/private-messages-page.test.cjs b/web/tests/private-messages-page.test.cjs new file mode 100644 index 0000000..c00b185 --- /dev/null +++ b/web/tests/private-messages-page.test.cjs @@ -0,0 +1,546 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const Module = require('node:module'); +const ts = require('typescript'); +const React = require('react'); + +function transpile(file) { + return ts.transpileModule(readFileSync(file, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, jsx: ts.JsxEmit.ReactJSX, target: ts.ScriptTarget.ES2020 }, + }).outputText; +} +const accounts = [ + { id: 'a', name: '账号甲', platform_account_key: '11', listener_enabled: true }, + { id: 'b', name: '', platform_account_key: '22', listener_enabled: true }, + { id: 'off', name: '未监听', platform_account_key: '33', listener_enabled: false }, +]; +const contacts = [ + { account_id: 'a', account_name: '账号甲', peer_uid: '100', peer_name: '联系人', last_text: '甲的最近消息', last_message_at: '2026-06-01T12:00:00Z', message_count: 2 }, + { account_id: 'b', account_name: '', peer_uid: '100', peer_name: '', last_text: '乙的最近消息', last_message_at: '2026-06-01T11:00:00Z', message_count: 1 }, +]; +function record(id, overrides = {}) { + return { id: String(id), account_id: 'a', peer_uid: '100', direction: 'inbound', message_type: 'text', text: `消息${id}`, state: 'succeeded', error: '', message_at: `2026-06-01T12:00:0${id}Z`, created_at: '', ...overrides }; +} +function deferred() { + let resolve, reject; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + return { promise, resolve, reject }; +} +function harness(options = {}) { + const states = [], effects = [], queued = [], calls = [], sendCalls = [], logs = [], notices = []; + const timers = new Map(); + let cursor = 0, uuid = 0, timerID = 0; + const antd = {}, x = {}; + for (const name of ['Alert', 'Button', 'Card', 'Flex', 'Modal', 'Select', 'Table', 'Tag', 'Empty', 'Spin', 'Splitter', 'Tooltip']) { + antd[name] = () => null; + } + antd.Splitter.Panel = () => null; + antd.Typography = { Text: () => null, Paragraph: () => null }; + antd.App = { useApp: () => ({ message: { success(value) { notices.push(value); }, error(value) { notices.push(value); } } }) }; + antd.Input = () => null; + antd.Input.TextArea = () => null; + antd.Form = () => null; + antd.Form.useForm = () => [{ resetFields() {}, setFieldsValue() {} }]; + antd.Form.Item = () => null; + x.Conversations = () => null; + x.Bubble = () => null; + x.Bubble.List = () => null; + x.Sender = () => null; + const utilFile = resolve(__dirname, '../src/utils/private-messages.ts'); + const util = new Module(utilFile, module); + util._compile(transpile(utilFile), utilFile); + const file = resolve(__dirname, '../src/pages/creator/messages/index.tsx'); + const loaded = new Module(file, module); + loaded.filename = file; + loaded.require = (id) => { + if (id === 'react') return { + useRef(initial) { + const index = cursor++; + if (!(index in states)) states[index] = { current: initial }; + return states[index]; + }, + useState(initial) { + const index = cursor++; + if (!(index in states)) states[index] = typeof initial === 'function' ? initial() : initial; + return [states[index], (value) => { states[index] = typeof value === 'function' ? value(states[index]) : value; }]; + }, + useEffect(callback, deps) { + const index = cursor++; + const prior = effects[index]; + if (!prior || deps.some((v, i) => !Object.is(v, prior.deps[i]))) { + prior?.cleanup?.(); + effects[index] = { deps }; + queued.push(() => { effects[index].cleanup = callback(); }); + } + }, + }; + if (id === 'react/jsx-runtime') return require(id); + if (id === 'antd') return antd; + if (id === '@ant-design/x') return x; + if (id === '@ant-design/icons') return Object.fromEntries(['PlusOutlined', 'CheckCircleOutlined', 'LoadingOutlined', 'CloseCircleOutlined', 'QuestionCircleOutlined'].map((name) => [name, name])); + if (id === '@/utils/private-messages') return util.exports; + if (id === '@/utils/helpers') return { dateTime: (value) => value ? `time:${value}` : '—', conflictMessage: (cause, fallback) => cause?.body?.reason || cause?.message || fallback }; + if (id === '@/services/api') return { + creatorGet: async (path) => { + calls.push(path); + if (options.get) { + const result = options.get(path); + if (result !== undefined) return result; + } + if (path === '/creator/accounts') return options.accounts || accounts; + if (path === '/creator/private-messages/status') return options.statuses || []; + const query = new URLSearchParams(path.split('?')[1]); + if (path.startsWith('/creator/private-messages/conversations?')) { + const rows = (options.contacts || contacts).filter((c) => !query.get('account_id') || c.account_id === query.get('account_id')); + return { data: rows, total: rows.length }; + } + if (path.startsWith('/creator/private-messages/messages?')) return options.messages || { data: [], total: 0 }; + throw new Error(`Unexpected API: ${path}`); + }, + creatorCreate: async (path, payload) => { + sendCalls.push({ path, payload }); + if (options.send) return options.send(path, payload); + return record('sent', { ...payload, direction: 'outbound', state: 'succeeded' }); + }, + }; + return require(id); + }; + // Timers and UUIDs are isolated per harness; polling never opens a browser. + function globals(callback) { + const prior = [globalThis.__dmTimers, globalThis.__dmCrypto, globalThis.__dmConsole]; + globalThis.__dmTimers = { set(fn) { const id = ++timerID; timers.set(id, fn); return id; }, clear(id) { timers.delete(id); } }; + globalThis.__dmCrypto = { randomUUID: () => `request-${++uuid}` }; + globalThis.__dmConsole = { error: (...args) => logs.push(args) }; + try { return callback(); } finally { + [globalThis.__dmTimers, globalThis.__dmCrypto, globalThis.__dmConsole] = prior; + } + } + // Recompile with the per-instance globals installed so asynchronous callbacks retain them. + globals(() => loaded._compile(`const setTimeout = globalThis.__dmTimers.set; const clearTimeout = globalThis.__dmTimers.clear; const crypto = globalThis.__dmCrypto; const console = globalThis.__dmConsole;\n${transpile(file)}`, file)); + const h = { + antd, x, calls, sendCalls, logs, notices, + render() { cursor = 0; return loaded.exports.default(); }, + async flush() { + let tree; + for (let i = 0; i < 6; i++) { + tree = h.render(); + while (queued.length) queued.shift()(); + for (let j = 0; j < 16; j++) await Promise.resolve(); + } + return h.render(); + }, + async poll() { + const pending = [...timers.values()]; + timers.clear(); + pending.forEach((fn) => fn()); + return h.flush(); + }, + dispose() { for (const effect of effects) effect?.cleanup?.(); }, + }; + return h; +} +function all(node, predicate) { + const result = []; + function visit(value) { + if (Array.isArray(value)) return value.forEach(visit); + if (!React.isValidElement(value)) return; + if (predicate(value)) result.push(value); + visit(value.props.children); + } + visit(node); + return result; +} +function text(node) { + if (node == null) return ''; + if (typeof node === 'string' || typeof node === 'number') return String(node); + if (Array.isArray(node)) return node.map(text).join(''); + return React.isValidElement(node) ? text(node.props.children) : ''; +} +const find = (tree, type) => all(tree, (node) => node.type === type)[0]; +const button = (h, tree, label) => all(tree, (node) => node.type === h.antd.Button).find((node) => text(node.props.children) === label); +const alerts = (h, tree) => all(tree, (node) => node.type === h.antd.Alert).map((node) => `${text(node.props.title)} ${text(node.props.description)}`).join('\n'); +const sender = (h, tree) => find(tree, h.x.Sender); +const list = (h, tree) => find(tree, h.x.Bubble.List); +const conversations = (h, tree) => find(tree, h.x.Conversations); +async function choose(h, key) { + conversations(h, h.render()).props.onActiveChange(key); + return h.flush(); +} +async function draft(h, value) { + sender(h, h.render()).props.onChange(value); + return h.flush(); +} + +test('native X chat replaces both tables and reply/login modals; account filter defaults to all', async (t) => { + const h = harness(); t.after(h.dispose); + const tree = await h.flush(); + assert.equal(all(tree, (n) => n.type === h.antd.Table).length, 0); + assert.ok(find(tree, h.antd.Splitter)); + assert.ok(conversations(h, tree)); assert.ok(list(h, tree)); assert.ok(sender(h, tree)); + assert.equal(all(tree, (n) => n.type === h.antd.Modal).length, 0); + assert.equal(button(h, tree, '登录私信'), undefined); + const filter = all(tree, (n) => n.type === h.antd.Select && n.props['aria-label'] === '所属账号')[0]; + assert.equal(filter.props.value, undefined); + assert.deepEqual(filter.props.options.map((o) => o.value), ['a', 'b']); + assert.match(text(tree), /最近 50 条/); + assert.equal(alerts(h, tree), ''); + assert.match(text(tree), /已发送不代表对方已收到/); + assert.equal(sender(h, tree).props.submitType, 'enter'); + const source = readFileSync(resolve(__dirname, '../src/pages/creator/messages/index.tsx'), 'utf8'); + assert.doesNotMatch(source, /AccountLoginModal|LoginOutlined|qr_value|login-qr|useXAgent|useXChat|useXStream|allowSpeech|onPasteFile|from ['"]@\/utils\/table/); + for (const filename of ['AccountLoginModal.tsx', 'account-login.ts']) { + assert.doesNotMatch(readFileSync(resolve(__dirname, '../src/components/accounts', filename), 'utf8'), /qr_value|private-messages\/accounts/); + } +}); + +test('same peer on different accounts has distinct stable keys, source names and last-message previews', async (t) => { + const h = harness(); t.after(h.dispose); + let tree = await h.flush(); + const items = conversations(h, tree).props.items; + assert.deepEqual(items.map((c) => c.key), ['a:100', 'b:100']); + assert.equal(items[0].label.type, h.antd.Flex); + assert.equal(items[0].label.props.vertical, true); + assert.deepEqual(React.Children.toArray(items[0].label.props.children).map(text), ['联系人', '账号甲', '甲的最近消息']); + assert.match(text(items[1].label), /100.*未命名账号.*乙的最近消息/); + tree = await choose(h, 'b:100'); + assert.match(text(tree), /发送账号：未命名账号/); + const query = new URLSearchParams(h.calls.filter((p) => p.startsWith('/creator/private-messages/messages?')).at(-1).split('?')[1]); + assert.equal(query.get('account_id'), 'b'); assert.equal(query.get('peer_uid'), '100'); +}); + +test('messages are chronological, received left/sent right, with real times, errors and non-text placeholders', async (t) => { + const h = harness({ messages: { data: [ + record(4, { message_type: 'image', text: 'not rendered as image' }), + record(3, { direction: 'outbound', state: 'unknown', error: 'SDK 1008 完整错误' }), + record(2, { direction: 'outbound', state: 'failed', error: '发送失败完整错误' }), + record(1, { direction: 'outbound', state: 'sending' }), + record(0), + ], total: 5 } }); t.after(h.dispose); + const tree = await h.flush(); const bubbles = list(h, tree).props; + assert.deepEqual(bubbles.items.map((m) => m.key), ['0', '1', '2', '3', '4']); + assert.equal(bubbles.role.inbound.placement, 'start'); assert.equal(bubbles.role.outbound.placement, 'end'); + assert.equal(bubbles.items[0].role, 'inbound'); assert.equal(bubbles.items[2].role, 'outbound'); + for (const [index, icon, title] of [[1, 'LoadingOutlined', '发送中'], [2, 'CloseCircleOutlined', '发送失败：发送失败完整错误'], [3, 'QuestionCircleOutlined', '结果未确认：SDK 1008 完整错误']]) { + const tooltip = all(bubbles.items[index].footer, (n) => n.type === h.antd.Tooltip).find((n) => n.props.title === title); + assert.equal(tooltip.props.title, title); + assert.ok(find(tooltip, icon)); + assert.doesNotMatch(text(bubbles.items[index].footer), /发送中|发送失败|结果未确认|完整错误/); + } + assert.match(text(bubbles.items[0].footer), /time:2026-06-01T12:00:00Z/); + assert.match(text(bubbles.items[4].content), /图片.*暂不支持/); + assert.equal(all(bubbles.items[4].content, (n) => n.type === 'img').length, 0); + assert.doesNotMatch(bubbles.items.map((m) => text(m.footer)).join(''), /已读|已送达/); +}); + +test('load more contacts merges and deduplicates; polling retains already loaded contacts', async (t) => { + let polled = false; + const h = harness({ get(path) { + if (!path.startsWith('/creator/private-messages/conversations?')) return; + const page = Number(new URLSearchParams(path.split('?')[1]).get('page')); + return { data: page === 1 ? [{ ...contacts[0], last_text: polled ? '更新后的摘要' : contacts[0].last_text }] : [contacts[0], contacts[1]], total: 3 }; + } }); t.after(h.dispose); + let tree = await h.flush(); + await button(h, tree, '加载更多联系人').props.onClick(); tree = await h.flush(); + assert.deepEqual(conversations(h, tree).props.items.map((c) => c.key), ['a:100', 'b:100']); + polled = true; tree = await h.poll(); + assert.equal(conversations(h, tree).props.items.length, 2); + assert.match(text(conversations(h, tree).props.items[0].label), /更新后的摘要/); + assert.ok(h.calls.some((p) => p.startsWith('/creator/private-messages/conversations?') && new URLSearchParams(p.split('?')[1]).get('page') === '2')); +}); + +test('earlier messages are prepended, deduplicated and retained during polling', async (t) => { + let polling = false; + const h = harness({ get(path) { + if (!path.startsWith('/creator/private-messages/messages?')) return; + const page = Number(new URLSearchParams(path.split('?')[1]).get('page')); + return { data: page === 1 ? [record(3, { text: polling ? '最新更新' : '消息3' }), record(2)] : [record(2), record(1)], total: 4 }; + } }); t.after(h.dispose); + let tree = await h.flush(); + await button(h, tree, '加载更早消息').props.onClick(); tree = await h.flush(); + assert.deepEqual(list(h, tree).props.items.map((m) => m.key), ['1', '2', '3']); + polling = true; tree = await h.poll(); + assert.deepEqual(list(h, tree).props.items.map((m) => m.key), ['1', '2', '3']); + assert.equal(list(h, tree).props.items.at(-1).content, '最新更新'); +}); + +test('account filtering resets contact pages and late responses cannot restore the old source', async (t) => { + const late = deferred(); + const other = harness({ get(path) { + if (!path.startsWith('/creator/private-messages/conversations?')) return; + const q = new URLSearchParams(path.split('?')[1]); + if (q.get('account_id') === 'b') return { data: [contacts[1]], total: 1 }; + if (q.get('page') === '2') return late.promise; + return { data: [contacts[0]], total: 30 }; + } }); t.after(other.dispose); + let tree = await other.flush(); + const loading = button(other, tree, '加载更多联系人').props.onClick(); + const filter = all(tree, (n) => n.type === other.antd.Select && n.props['aria-label'] === '所属账号')[0]; + filter.props.onChange('b'); tree = await other.flush(); + late.resolve({ data: [contacts[0]], total: 30 }); await loading; tree = await other.flush(); + assert.deepEqual(conversations(other, tree).props.items.map((c) => c.key), ['b:100']); + const q = new URLSearchParams(other.calls.filter((p) => p.startsWith('/creator/private-messages/conversations?')).at(-1).split('?')[1]); + assert.equal(q.get('account_id'), 'b'); assert.equal(q.get('page'), '1'); +}); + +test('dependency help is absent while real account sync errors stay visible', async (t) => { + const h = harness({ statuses: [{ account_id: 'a', account_name: '肉宝儿', error: '实际读取错误' }] }); t.after(h.dispose); + const tree = await h.flush(); + assert.doesNotMatch(alerts(h, tree), /私信同步依赖|同步失败时请查看/); + assert.match(alerts(h, tree), /肉宝儿：私信同步异常.*实际读取错误/); +}); + +test('successful bubbles use icons and record timestamps have no visible label', async (t) => { + const h = harness({ messages: { data: [record('x', { direction: 'outbound', message_at: '', created_at: '2026-06-01T12:00:00Z' })], total: 1 } }); t.after(h.dispose); + const footer = list(h, await h.flush()).props.items[0].footer; + assert.match(text(footer), /^time:2026-06-01T12:00:00Z$/); + const tooltips = all(footer, (n) => n.type === h.antd.Tooltip); + assert.ok(tooltips.some((n) => n.props.title === '记录时间')); + const status = tooltips.find((n) => n.props.title === '已发送'); + assert.ok(status); assert.ok(find(status, 'CheckCircleOutlined')); +}); + +test('late message responses do not leak into the newly selected conversation', async (t) => { + const late = deferred(); + const h = harness({ get(path) { + if (!path.startsWith('/creator/private-messages/messages?')) return; + const account = new URLSearchParams(path.split('?')[1]).get('account_id'); + return account === 'a' ? late.promise : { data: [record('b', { account_id: 'b', text: '乙的消息' })], total: 1 }; + } }); t.after(h.dispose); + await h.flush(); let tree = await choose(h, 'b:100'); + late.resolve({ data: [record('a', { text: '甲的旧响应' })], total: 1 }); tree = await h.flush(); + assert.deepEqual(list(h, tree).props.items.map((m) => m.content), ['乙的消息']); + assert.match(text(tree), /发送账号：未命名账号/); +}); + +test('drafts are independent even for the same peer and typing never sends', async (t) => { + const h = harness(); t.after(h.dispose); + await h.flush(); await draft(h, '甲的草稿'); + let tree = await choose(h, 'b:100'); assert.equal(sender(h, tree).props.value, ''); + await draft(h, '乙的草稿'); tree = await choose(h, 'a:100'); + assert.equal(sender(h, tree).props.value, '甲的草稿'); + tree = await choose(h, 'b:100'); assert.equal(sender(h, tree).props.value, '乙的草稿'); + await h.poll(); assert.equal(h.sendCalls.length, 0); +}); + +test('send snapshots the source/peer; switching chats never sends to the new selection or clears its draft', async (t) => { + const late = deferred(); const h = harness({ send: () => late.promise }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, '甲的发送内容'); + const sending = sender(h, tree).props.onSubmit('甲的发送内容'); + tree = await choose(h, 'b:100'); await draft(h, '乙的草稿'); + assert.equal(h.sendCalls.length, 1); + assert.deepEqual(h.sendCalls[0], { path: '/creator/private-messages/send', payload: { account_id: 'a', peer_uid: '100', text: '甲的发送内容', request_id: 'request-1' } }); + late.resolve(record('sent', { direction: 'outbound', state: 'succeeded' })); await sending; tree = await h.flush(); + assert.equal(sender(h, tree).props.value, '乙的草稿'); + assert.equal(list(h, tree).props.items.some((m) => m.key === 'sent'), false); + tree = await choose(h, 'a:100'); assert.equal(sender(h, tree).props.value, ''); +}); + +test('successful send cannot erase input edited while the request was pending', async (t) => { + const late = deferred(); const h = harness({ send: () => late.promise }); t.after(h.dispose); + await h.flush(); const tree = await draft(h, '第一条'); + const sending = sender(h, tree).props.onSubmit('第一条'); + await draft(h, '第二条草稿'); + late.resolve(record('sent', { direction: 'outbound' })); await sending; + assert.equal(sender(h, await h.flush()).props.value, '第二条草稿'); +}); + +for (const state of ['failed', 'unknown', 'sending']) { + test(`${state} retains draft and complete feedback; polls and input changes never resend`, async (t) => { + const h = harness({ send: () => record('attempt', { direction: 'outbound', state, error: '完整发送错误' }) }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, '不能丢的正文'); + await sender(h, tree).props.onSubmit('不能丢的正文'); tree = await h.flush(); + assert.equal(sender(h, tree).props.value, '不能丢的正文'); + assert.match(alerts(h, tree), /完整发送错误/); + assert.match(alerts(h, tree), /request-1/); + if (state !== 'failed') { + await sender(h, tree).props.onSubmit('不能丢的正文'); + assert.equal(h.sendCalls.length, 1, 'uncertain or in-flight attempts must not be resent'); + } + await draft(h, '新的草稿'); await h.poll(); + assert.equal(h.sendCalls.length, 1); + assert.doesNotMatch(JSON.stringify(h.logs), /不能丢的正文|新的草稿/); + }); +} + +test('failed messages can be explicitly retried with a fresh request ID, never automatically', async (t) => { + let attempt = 0; + const h = harness({ send: () => record('sent', { direction: 'outbound', state: ++attempt === 1 ? 'failed' : 'succeeded', error: attempt === 1 ? '失败' : '' }) }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, '手动重发'); + await sender(h, tree).props.onSubmit('手动重发'); tree = await h.flush(); + await h.poll(); assert.equal(h.sendCalls.length, 1); + await sender(h, tree).props.onSubmit('手动重发'); tree = await h.flush(); + assert.equal(h.sendCalls.length, 2); + assert.notEqual(h.sendCalls[0].payload.request_id, h.sendCalls[1].payload.request_id); + assert.equal(sender(h, tree).props.value, ''); +}); + +test('network errors remain visible and preserve uncertain request IDs without exposing request bodies in logs', async (t) => { + const cause = Object.assign(new Error('网络中断，结果未确认'), { body: { text: '私密正文' } }); + const h = harness({ send: () => { throw cause; } }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, '私密正文'); + await sender(h, tree).props.onSubmit('私密正文'); tree = await h.flush(); + assert.match(alerts(h, tree), /网络中断，结果未确认/); + assert.match(alerts(h, tree), /request-1/); + assert.equal(sender(h, tree).props.value, '私密正文'); + await sender(h, tree).props.onSubmit('私密正文'); await h.poll(); + assert.equal(h.sendCalls.length, 1); + assert.match(JSON.stringify(h.logs), /account_id.*a.*peer_uid.*100.*request_id.*request-1/); + assert.doesNotMatch(JSON.stringify(h.logs), /私密正文/); +}); + +test('listener closure disables sending while preserving the selected chat, draft and server errors', async (t) => { + let enabled = true; + const h = harness({ get(path) { + if (path === '/creator/accounts') return accounts.map((a) => a.id === 'a' ? { ...a, listener_enabled: enabled } : a); + } }); t.after(h.dispose); + await h.flush(); await draft(h, '关闭前草稿'); enabled = false; + let tree = await h.poll(); + assert.equal(sender(h, tree).props.disabled, true); + assert.match(alerts(h, tree), /监听.*关闭|未开启监听/); + await sender(h, tree).props.onSubmit('关闭前草稿'); tree = await h.flush(); + assert.equal(h.sendCalls.length, 0); assert.equal(sender(h, tree).props.value, '关闭前草稿'); +}); + +test('server conflict reasons are not hidden by generic send feedback', async (t) => { + const h = harness({ send() { throw { status: 409, body: { reason: '监听已关闭：generation=9' } }; } }); t.after(h.dispose); + await h.flush(); const tree = await draft(h, '草稿'); + await sender(h, tree).props.onSubmit('草稿'); + assert.match(alerts(h, await h.flush()), /监听已关闭：generation=9/); +}); + +test('new text conversation uses the sole modal and locks its explicit source instead of the filter', async (t) => { + const h = harness(); t.after(h.dispose); + let tree = await h.flush(); button(h, tree, '新建私信').props.onClick(); tree = await h.flush(); + assert.equal(all(tree, (n) => n.type === h.antd.Modal).length, 1); + const modal = find(tree, h.antd.Modal); assert.equal(modal.props.title, '新建私信'); + const form = find(modal, h.antd.Form); + const uid = all(form, (n) => n.type === h.antd.Form.Item && n.props.name === 'peer_uid')[0]; + assert.equal(uid.props.rules[1].pattern.test('invalid'), false); + await form.props.onFinish({ account_id: 'b', peer_uid: '999', text: '首次消息' }); tree = await h.flush(); + assert.deepEqual(h.sendCalls[0].payload, { request_id: 'request-1', account_id: 'b', peer_uid: '999', text: '首次消息' }); + assert.match(text(tree), /发送账号：未命名账号/); + assert.equal(all(tree, (n) => n.type === h.antd.Modal).length, 0); + assert.equal(conversations(h, tree).props.activeKey, 'b:999'); +}); + +test('sync errors and read failures remain visible without replacing existing chat data', async (t) => { + let fail = false; + const h = harness({ statuses: [{ account_id: 'a', account_name: '账号甲', error: '聊天客户端尚未就绪' }], messages: { data: [record(1)], total: 1 }, get(path) { + if (fail && path.startsWith('/creator/private-messages/messages?')) throw new Error('消息读取真实错误'); + if (fail && path.startsWith('/creator/private-messages/conversations?')) throw new Error('联系人读取真实错误'); + } }); t.after(h.dispose); + let tree = await h.flush(); assert.match(alerts(h, tree), /聊天客户端尚未就绪/); + fail = true; tree = await h.poll(); + assert.match(alerts(h, tree), /消息读取真实错误/); assert.match(alerts(h, tree), /联系人读取真实错误/); + assert.equal(list(h, tree).props.items.length, 1); + assert.equal(conversations(h, tree).props.items.length, 2); +}); + +test('a new conversation on another source changes the filter and preserves failure feedback', async (t) => { + const h = harness({ send: (_path, payload) => record('attempt', { account_id: payload.account_id, peer_uid: payload.peer_uid, direction: 'outbound', state: 'failed', error: '首次发送真实失败' }) }); t.after(h.dispose); + let tree = await h.flush(); + all(tree, (n) => n.type === h.antd.Select && n.props['aria-label'] === '所属账号')[0].props.onChange('a'); + tree = await h.flush(); button(h, tree, '新建私信').props.onClick(); tree = await h.flush(); + await find(tree, h.antd.Form).props.onFinish({ account_id: 'b', peer_uid: '999', text: '乙的新草稿' }); tree = await h.flush(); + const filter = all(tree, (n) => n.type === h.antd.Select && n.props['aria-label'] === '所属账号')[0]; + assert.equal(filter.props.value, 'b'); + assert.equal(conversations(h, tree).props.activeKey, 'b:999'); + assert.ok(conversations(h, tree).props.items.some((c) => c.key === 'b:999')); + assert.equal(conversations(h, tree).props.items.some((c) => c.key.startsWith('a:')), false); + assert.match(alerts(h, tree), /首次发送真实失败/); + assert.equal(sender(h, tree).props.value, '乙的新草稿'); +}); + +test('an older contact page cannot overwrite a newer polling summary', async (t) => { + const late = deferred(); let latest = false; + const h = harness({ get(path) { + if (!path.startsWith('/creator/private-messages/conversations?')) return; + const page = new URLSearchParams(path.split('?')[1]).get('page'); + return page === '2' ? late.promise : { data: [{ ...contacts[0], last_text: latest ? '刚更新的摘要' : '旧摘要' }], total: 30 }; + } }); t.after(h.dispose); + let tree = await h.flush(); const loading = button(h, tree, '加载更多联系人').props.onClick(); + latest = true; tree = await h.poll(); + late.resolve({ data: [{ ...contacts[0], last_text: '旧分页摘要' }, contacts[1]], total: 30 }); await loading; tree = await h.flush(); + assert.match(text(conversations(h, tree).props.items[0].label), /刚更新的摘要/); +}); + +test('late earlier-message pages cannot appear in another chat', async (t) => { + const late = deferred(); + const h = harness({ get(path) { + if (!path.startsWith('/creator/private-messages/messages?')) return; + const q = new URLSearchParams(path.split('?')[1]); + if (q.get('account_id') === 'b') return { data: [record('b', { account_id: 'b', text: '乙的内容' })], total: 1 }; + if (q.get('page') === '2') return late.promise; + return { data: [record(3)], total: 60 }; + } }); t.after(h.dispose); + let tree = await h.flush(); const loading = button(h, tree, '加载更早消息').props.onClick(); + await choose(h, 'b:100'); late.resolve({ data: [record(1)], total: 60 }); await loading; tree = await h.flush(); + assert.deepEqual(list(h, tree).props.items.map((m) => m.content), ['乙的内容']); +}); + +for (const editWhileWaiting of [false, true]) { + test(`polling resolves unknown requests without resend; newer draft preserved=${editWhileWaiting}`, async (t) => { + let sent = false; + const h = harness({ send: () => { sent = true; return record('attempt', { direction: 'outbound', state: 'unknown', error: 'SDK 1008' }); }, get(path) { + if (sent && path.startsWith('/creator/private-messages/messages?')) return { data: [record('attempt', { direction: 'outbound', state: 'succeeded' })], total: 1 }; + } }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, '原草稿'); await sender(h, tree).props.onSubmit('原草稿'); tree = await h.flush(); + assert.match(alerts(h, tree), /request-1/); + if (editWhileWaiting) await draft(h, '修改后的草稿'); + tree = await h.poll(); + assert.equal(h.sendCalls.length, 1); + assert.equal(sender(h, tree).props.value, editWhileWaiting ? '修改后的草稿' : ''); + assert.doesNotMatch(alerts(h, tree), /request-1/); + }); +} + +test('pagination errors and malformed data are visible without discarding existing rows', async (t) => { + const h = harness({ get(path) { + if (path.startsWith('/creator/private-messages/conversations?')) { + return new URLSearchParams(path.split('?')[1]).get('page') === '2' ? Promise.reject(new Error('联系人第二页错误')) : { data: [contacts[0]], total: 30 }; + } + if (path.startsWith('/creator/private-messages/messages?')) { + return new URLSearchParams(path.split('?')[1]).get('page') === '2' ? { data: null, total: 60 } : { data: [record(3)], total: 60 }; + } + } }); t.after(h.dispose); + let tree = await h.flush(); + await button(h, tree, '加载更多联系人').props.onClick(); await button(h, tree, '加载更早消息').props.onClick(); tree = await h.flush(); + assert.match(alerts(h, tree), /联系人第二页错误/); assert.match(alerts(h, tree), /返回格式错误/); + assert.equal(conversations(h, tree).props.items.length, 1); assert.equal(list(h, tree).props.items.length, 1); +}); + +test('pending requests use the native loading SendButton, not an AI stop/cancel action', async (t) => { + const late = deferred(); const h = harness({ send: () => late.promise }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, '发送中的内容'); + const sending = sender(h, tree).props.onSubmit('发送中的内容'); tree = await h.flush(); + const props = sender(h, tree).props; + const SendButton = () => null; + assert.equal(typeof props.suffix, 'function'); + const action = props.suffix(null, { components: { SendButton } }); + assert.equal(action.type, SendButton); assert.equal(action.props.loading, true); + assert.equal(props.onCancel, undefined); + assert.equal(props.disabled, false, 'input stays editable while a send is pending'); + late.resolve(record('sent', { direction: 'outbound' })); await sending; await h.flush(); +}); + +test('double submission, whitespace and oversized input cannot create unintended sends', async (t) => { + const late = deferred(); const h = harness({ send: () => late.promise }); t.after(h.dispose); + await h.flush(); let tree = await draft(h, ' '); await sender(h, tree).props.onSubmit(' '); + tree = await draft(h, '字'.repeat(2001)); await sender(h, tree).props.onSubmit('字'.repeat(2001)); + assert.equal(h.sendCalls.length, 0); assert.ok(h.notices.some((n) => n.includes('2000'))); + tree = await draft(h, '手动消息'); const send = sender(h, tree).props.onSubmit('手动消息'); + await sender(h, tree).props.onSubmit('手动消息'); assert.equal(h.sendCalls.length, 1); + late.resolve(record('sent', { direction: 'outbound' })); await send; await h.flush(); +}); + +test('missing timestamps stay unknown and no-enabled-account empty state cannot send', async (t) => { + const h = harness({ accounts: [], contacts: [] }); t.after(h.dispose); + const tree = await h.flush(); + assert.ok(find(tree, h.antd.Empty)); assert.equal(button(h, tree, '新建私信').props.disabled, true); + assert.equal(sender(h, tree), undefined); assert.equal(h.sendCalls.length, 0); + const other = harness({ messages: { data: [record('x', { message_at: '', created_at: '' })], total: 1 } }); t.after(other.dispose); + const footer = list(other, await other.flush()).props.items[0].footer; + assert.match(text(footer), /时间未知/); +}); diff --git a/web/tests/private-messages.test.cjs b/web/tests/private-messages.test.cjs new file mode 100644 index 0000000..40e4344 --- /dev/null +++ b/web/tests/private-messages.test.cjs @@ -0,0 +1,87 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const Module = require('node:module'); +const ts = require('typescript'); +const file = resolve(__dirname, '../src/utils/private-messages.ts'); +const loaded = new Module(file, module); +loaded._compile(ts.transpileModule(readFileSync(file, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, +}).outputText, file); +const util = loaded.exports; + +const contact = (account_id, peer_uid, time = '') => ({ account_id, peer_uid, account_name: '', peer_name: '', last_text: '', last_message_at: time, message_count: 0 }); +const message = (id, time = '', overrides = {}) => ({ id, account_id: 'a', peer_uid: '100', direction: 'inbound', message_type: 'text', text: '', state: 'succeeded', error: '', message_at: time, created_at: '', ...overrides }); + +test('conversation and message API queries preserve pagination, source and encoded peers', () => { + assert.equal(util.conversationQuery({}, 2, 25), 'page=2&page_size=25'); + const query = new URLSearchParams(util.conversationQuery({ accountId: 'a/b' }, 1, 25)); + assert.equal(query.get('account_id'), 'a/b'); + const messages = new URLSearchParams(util.messagesQuery('a/b', '100+20', 3, 20)); + assert.equal(messages.get('account_id'), 'a/b'); assert.equal(messages.get('peer_uid'), '100+20'); + assert.equal(messages.get('page'), '3'); assert.equal(messages.get('page_size'), '20'); +}); + +test('conversation identity includes account and peer, never peer alone', () => { + assert.equal(util.conversationKey(contact('a', '100')), 'a:100'); + assert.notEqual(util.conversationKey(contact('a', '100')), util.conversationKey(contact('b', '100'))); + assert.notEqual(util.conversationKey(contact('a', '100')), util.conversationKey(contact('a', '200'))); +}); + +test('contact pages merge by compound key, update summaries, sort newest first and retain older rows', () => { + const old = contact('a', '100', '2026-06-01T10:00:00Z'); + const updated = { ...old, last_text: '最新摘要', last_message_at: '2026-06-01T12:00:00Z' }; + const previous = [old, contact('b', '100', '2026-06-01T11:00:00Z')]; + const result = util.mergeConversations(previous, [updated, updated]); + assert.deepEqual(result.map(util.conversationKey), ['a:100', 'b:100']); + assert.equal(result[0].last_text, '最新摘要'); assert.equal(previous[0].last_text, ''); + assert.equal(util.mergeConversations(result, []).length, 2); +}); + +test('missing and invalid contact times stay behind known times, equal-time ordering is deterministic', () => { + const rows = [contact('b', '100'), contact('c', '100', 'invalid'), contact('a', '100', '2026-06-01T12:00:00Z')]; + assert.deepEqual(util.mergeConversations([], rows).map(util.conversationKey), ['a:100', 'b:100', 'c:100']); +}); + +test('older message pages merge without duplicates, ascending times and updated send results', () => { + const latest = [message('3', '2026-06-01T12:00:03Z'), message('2', '2026-06-01T12:00:02Z', { state: 'sending' })]; + const updated = message('2', '2026-06-01T12:00:02Z', { state: 'succeeded' }); + const result = util.mergeMessages(latest, [updated, message('1', '2026-06-01T12:00:01Z'), updated]); + assert.deepEqual(result.map((m) => m.id), ['1', '2', '3']); assert.equal(result[1].state, 'succeeded'); + assert.equal(latest[1].state, 'sending'); assert.equal(util.mergeMessages(result, []).length, 3); +}); + +test('message ordering handles real storage times and exact large numeric IDs without number rounding', () => { + const rows = [message('9007199254740993', '', { created_at: '2026-06-01T12:00:00Z' }), message('9007199254740992', '', { created_at: '2026-06-01T12:00:00Z' }), message('2'), message('10')]; + assert.deepEqual(util.mergeMessages([], rows).map((m) => m.id), ['2', '10', '9007199254740992', '9007199254740993']); +}); + +test('draft updates isolate sessions and success only clears the exact submitted revision', () => { + let drafts = util.updateDraft({}, 'a:100', '甲'); + drafts = util.updateDraft(drafts, 'b:100', '乙'); + const original = drafts; + const cleared = util.clearSentDraft(drafts, 'a:100', drafts['a:100'].version); + assert.equal(cleared['a:100'].text, ''); assert.equal(cleared['b:100'].text, '乙'); + assert.equal(original['a:100'].text, '甲'); + const newer = util.updateDraft(drafts, 'a:100', '新输入'); + assert.strictEqual(util.clearSentDraft(newer, 'a:100', drafts['a:100'].version), newer); + assert.strictEqual(util.clearSentDraft(drafts, 'missing', 1), drafts); +}); + +test('retyping the same value counts as new input and cannot be erased by an old success', () => { + const original = util.updateDraft({}, 'a:100', '相同文本'); + const changed = util.updateDraft(util.updateDraft(original, 'a:100', '其他输入'), 'a:100', '相同文本'); + assert.strictEqual(util.clearSentDraft(changed, 'a:100', original['a:100'].version), changed); +}); + +test('direction, state and type labels preserve unknown server values without faking delivery or reads', () => { + assert.equal(util.messageDirectionLabel('inbound'), '收到'); assert.equal(util.messageDirectionLabel('outbound'), '发出'); + assert.equal(util.messageDirectionLabel('other'), 'other'); + for (const [state, label, tone] of [['succeeded', '已发送', 'success'], ['sending', '发送中', 'processing'], ['failed', '发送失败', 'error'], ['unknown', '结果未确认', 'warning']]) { + assert.equal(util.messageStateLabel(state), label); assert.equal(util.messageStateTone(state), tone); + } + assert.equal(util.messageStateLabel('other'), 'other'); assert.equal(util.messageStateTone('other'), 'default'); + assert.equal(util.messageTypeLabel('text'), '文字'); assert.equal(util.messageTypeLabel('image'), '图片'); + assert.equal(util.messageTypeLabel('unknown'), '未知类型'); assert.equal(util.messageTypeLabel('video'), 'video'); +});