diff --git a/browser_gateway/browser/cdp.py b/browser_gateway/browser/cdp.py index ed2d0e8..59d73d4 100644 --- a/browser_gateway/browser/cdp.py +++ b/browser_gateway/browser/cdp.py @@ -15,6 +15,7 @@ import websocket LOG = logging.getLogger("creatorhub.browser.cdp") CONTROL_TIMEOUT = 15.0 +RESOLVE_TIMEOUT = 90.0 class BrowserError(RuntimeError): @@ -30,6 +31,13 @@ class CDPConnection: self._next_id = 0 self._pending: deque[dict] = deque() + def notify(self, method: str, params: dict | None = None) -> None: + with self._lock: + self._next_id += 1 + self.socket.send( + json.dumps({"id": self._next_id, "method": method, "params": params or {}}) + ) + def command(self, method: str, params: dict | None = None) -> dict: with self._lock: self._next_id += 1 diff --git a/browser_gateway/platform/douyin.py b/browser_gateway/platform/douyin.py index ccf3d08..a511a11 100644 --- a/browser_gateway/platform/douyin.py +++ b/browser_gateway/platform/douyin.py @@ -21,7 +21,12 @@ from urllib.parse import urlsplit import websocket -from ..browser.cdp import CONTROL_TIMEOUT, BrowserError, CDPConnection +from ..browser.cdp import ( + CONTROL_TIMEOUT, + RESOLVE_TIMEOUT, + BrowserError, + CDPConnection, +) from ..browser.response import ( BrowserLoginQRResponse, BrowserMediaResponse, @@ -245,25 +250,28 @@ class DouyinBrowser: with self.connection(alias) as cdp: if cdp.evaluate("location.origin") not in {"null", self.origin}: raise DouyinError("restricted browser origin changed") - navigation = cdp.evaluate( - "window.location.assign(" + json.dumps(target) + "); true" - ) - if not isinstance(navigation, bool) or not navigation: - raise DouyinError("Douyin share URL navigation failed") - deadline = time.monotonic() + CONTROL_TIMEOUT + try: + cdp.command("Page.enable") + cdp.notify("Page.navigate", {"url": target}) + except (OSError, websocket.WebSocketException) as exc: + raise DouyinError("Douyin share URL navigation failed") from exc + deadline = time.monotonic() + RESOLVE_TIMEOUT final_url = "" while time.monotonic() < deadline: try: - current_url = cdp.evaluate("location.href") + event = cdp.wait_event( + "Page.frameNavigated", + lambda params: isinstance(params.get("frame"), dict) + and not params["frame"].get("parentId"), + timeout=max(0.01, deadline - time.monotonic()), + ) except BrowserError: - if is_douyin_content_url(target): - final_url = target - break - raise + break + frame = event.get("params", {}).get("frame", {}) + current_url = frame.get("url") if isinstance(frame, dict) else None if isinstance(current_url, str) and is_douyin_content_url(current_url): final_url = current_url break - time.sleep(0.1) if not final_url: raise DouyinError("Douyin share URL did not resolve to a supported page") return final_url diff --git a/browser_gateway/platform/xiaohongshu.py b/browser_gateway/platform/xiaohongshu.py index b8abd63..ece0f75 100644 --- a/browser_gateway/platform/xiaohongshu.py +++ b/browser_gateway/platform/xiaohongshu.py @@ -6,7 +6,9 @@ import json import time from urllib.parse import parse_qs, urlsplit -from ..browser.cdp import CONTROL_TIMEOUT, BrowserError +import websocket + +from ..browser.cdp import RESOLVE_TIMEOUT, BrowserError from ..browser.response import ( BrowserResponse, detect_challenge, @@ -81,25 +83,28 @@ class XiaohongshuBrowser(DouyinBrowser): with self.connection(alias) as cdp: if cdp.evaluate("location.origin") not in {"null", self.origin}: raise DouyinError("restricted browser origin changed") - navigation = cdp.evaluate( - "window.location.assign(" + json.dumps(target) + "); true" - ) - if not isinstance(navigation, bool) or not navigation: - raise DouyinError("Xiaohongshu share URL navigation failed") - deadline = time.monotonic() + CONTROL_TIMEOUT + try: + cdp.command("Page.enable") + cdp.notify("Page.navigate", {"url": target}) + except (OSError, websocket.WebSocketException) as exc: + raise DouyinError("Xiaohongshu share URL navigation failed") from exc + deadline = time.monotonic() + RESOLVE_TIMEOUT final_url = "" while time.monotonic() < deadline: try: - current_url = cdp.evaluate("location.href") + event = cdp.wait_event( + "Page.frameNavigated", + lambda params: isinstance(params.get("frame"), dict) + and not params["frame"].get("parentId"), + timeout=max(0.01, deadline - time.monotonic()), + ) except BrowserError: - if _is_xiaohongshu_page_url(target): - final_url = target - break - raise + break + frame = event.get("params", {}).get("frame", {}) + current_url = frame.get("url") if isinstance(frame, dict) else None if isinstance(current_url, str) and _is_xiaohongshu_page_url(current_url): final_url = current_url break - time.sleep(0.1) if not final_url: raise DouyinError( "Xiaohongshu share URL did not resolve to a supported page" diff --git a/browser_gateway/runtime.py b/browser_gateway/runtime.py index 43d7b0d..cf0f198 100644 --- a/browser_gateway/runtime.py +++ b/browser_gateway/runtime.py @@ -34,7 +34,7 @@ PROFILE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$") RUNTIME_CLEANUP_SENTINEL = "runtime-not-found" DEFAULT_CLEANUP_TIMEOUT = 30.0 -DEFAULT_READY_TIMEOUT = 15.0 +DEFAULT_READY_TIMEOUT = 60.0 DEFAULT_DISPLAY_START = 100 DEFAULT_DISPLAY_END = 199 DEFAULT_CDP_PORT_START = 19000 diff --git a/browser_gateway/server/http.py b/browser_gateway/server/http.py index 4f5b00a..5fcde51 100644 --- a/browser_gateway/server/http.py +++ b/browser_gateway/server/http.py @@ -35,12 +35,6 @@ from ..platform.douyin import ( ) from ..platform.xiaohongshu import XiaohongshuBrowser, is_xiaohongshu_share_url from ..proxy import ProxyExit -from ..runtime import ( - BrowserRuntimeError, - NativeRuntimeManager, - has_control, - validate_runtime_input, -) from ..runtime import ( NETWORK_ID_RE as _NETWORK_ID_RE, ) @@ -50,6 +44,12 @@ from ..runtime import ( from ..runtime import ( RUNTIME_ID_RE as _RUNTIME_ID_RE, ) +from ..runtime import ( + BrowserRuntimeError, + NativeRuntimeManager, + has_control, + validate_runtime_input, +) from ..runtime import ( parse_proxy_exit as _parse_proxy_exit, ) @@ -832,12 +832,18 @@ class GatewayHandler(BaseHTTPRequestHandler): return value def _respond(self, status: int, body: bytes) -> None: - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - if body: - self.wfile.write(body) + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + except BrokenPipeError: + LOG.info( + "gateway client disconnected before response", + extra={"status": status}, + ) def json_bytes(value: object) -> bytes: @@ -1275,7 +1281,7 @@ def load_config(env: Mapping[str, str] | None = None) -> dict: "node_name": node_name, "token": token, "cleanup_timeout": _positive_float(env, "RUNTIME_CLEANUP_TIMEOUT", 30.0, 300.0), - "ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 15.0, 300.0), + "ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 60.0, 300.0), "min_free_bytes": _integer(env, "RUNTIME_MIN_FREE_BYTES", 20 * 1024**3), "log_max_bytes": _integer(env, "RUNTIME_LOG_MAX_BYTES", 1 * 1024**3), "profile_cache_max_bytes": _integer(env, "PROFILE_CACHE_MAX_BYTES", 20 * 1024**3), diff --git a/browser_gateway/test_gateway.py b/browser_gateway/test_gateway.py index 9c979d1..abdd5c8 100644 --- a/browser_gateway/test_gateway.py +++ b/browser_gateway/test_gateway.py @@ -155,6 +155,7 @@ class GatewayValidationTests(unittest.TestCase): ) self.assertEqual(config["listen"], ("", 8081)) self.assertEqual(config["browser_path"], "/bin/true") + self.assertEqual(config["ready_timeout"], 60.0) self.assertIsNone(config["external_display"]) external = load_config( { @@ -756,7 +757,7 @@ class BrowserTests(unittest.TestCase): cdp.evaluate.side_effect = ["null", True, "https://www.douyin.com/video/123"] cdp.command.return_value = {"frameId": "frame-1"} cdp.wait_event.return_value = { - "frame": {"url": "https://www.douyin.com/video/123"} + "params": {"frame": {"url": "https://www.douyin.com/video/123"}} } browser = DouyinBrowser() self._with_connection(browser, cast(BrowserCDP, cdp)) @@ -765,12 +766,11 @@ class BrowserTests(unittest.TestCase): browser.resolve("safe", "https://www.douyin.com/video/123"), "https://www.douyin.com/video/123", ) - self.assertEqual(cdp.evaluate.call_count, 3) + self.assertEqual(cdp.evaluate.call_count, 1) self.assertEqual(cdp.evaluate.call_args_list[0].args, ("location.origin",)) - self.assertIn( - "window.location.assign(\"https://www.douyin.com/video/123\")", - cdp.evaluate.call_args_list[1].args[0], - ) + self.assertEqual(cdp.command.call_args_list[0].args, ("Page.enable",)) + self.assertEqual(cdp.notify.call_args_list[0].args, ("Page.navigate", {"url": "https://www.douyin.com/video/123"})) + def test_browser_fetch_uses_manually_logged_session(self) -> None: cdp = BrowserCDP( [ diff --git a/browser_gateway/test_xiaohongshu.py b/browser_gateway/test_xiaohongshu.py index 88ed782..ad61a71 100644 --- a/browser_gateway/test_xiaohongshu.py +++ b/browser_gateway/test_xiaohongshu.py @@ -98,7 +98,7 @@ class XiaohongshuBrowserTests(unittest.TestCase): ] cdp.command.return_value = {"frameId": "frame-1"} cdp.wait_event.return_value = { - "frame": {"url": "https://www.xiaohongshu.com/explore/n-1"} + "params": {"frame": {"url": "https://www.xiaohongshu.com/explore/n-1"}} } @contextmanager diff --git a/deploy/browser-gateway.env.example b/deploy/browser-gateway.env.example index b99d545..471efc1 100644 --- a/deploy/browser-gateway.env.example +++ b/deploy/browser-gateway.env.example @@ -6,7 +6,7 @@ BROWSER_PROFILE_ROOT=~/.local/share/creatorhub/browser-profiles BROWSER_PATH=~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome NODE_NAME= RUNTIME_CLEANUP_TIMEOUT=30 -RUNTIME_READY_TIMEOUT=15 +RUNTIME_READY_TIMEOUT=60 RUNTIME_MIN_FREE_BYTES=21474836480 RUNTIME_LOG_MAX_BYTES=1073741824 PROFILE_CACHE_MAX_BYTES=21474836480 diff --git a/internal/controlplane/api/creator.go b/internal/controlplane/api/creator.go index c0b4c58..1dc92db 100644 --- a/internal/controlplane/api/creator.go +++ b/internal/controlplane/api/creator.go @@ -1441,7 +1441,7 @@ func (browser creatorGatewayBrowser) Get(ctx context.Context, target string) (do func (browser creatorGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) { payload := gatewayGenerationPayload(browser.environment) payload["url"] = target - status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/resolve", payload, 30*time.Second) + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/resolve", payload, gatewayBrowserOperationTimeout) if err != nil { return "", err } @@ -1535,6 +1535,7 @@ type competitorSharePreview struct { AccountID string `json:"account_id"` Platform string `json:"platform"` PlatformAccountKey string `json:"platform_account_key"` + UniqueID string `json:"unique_id,omitempty"` Nickname string `json:"nickname"` AvatarURL string `json:"avatar_url,omitempty"` HomepageURL string `json:"homepage_url"` @@ -1545,6 +1546,7 @@ func (preview competitorSharePreview) input() creator.CompetitorInput { return creator.CompetitorInput{ Platform: preview.Platform, PlatformAccountKey: preview.PlatformAccountKey, + UniqueID: preview.UniqueID, Nickname: preview.Nickname, AvatarURL: preview.AvatarURL, HomepageURL: preview.HomepageURL, @@ -1720,13 +1722,21 @@ func previewDouyinCompetitorShare(ctx context.Context, store *creator.Store, pha if err != nil { return competitorSharePreview{}, err } - target, err := (douyin.CreatorCollector{Browser: browser}).ResolveWork(ctx, workKey) - if err != nil { - return competitorSharePreview{}, fmt.Errorf("%w: target identity verification failed: %v", creator.ErrConflict, err) + var target douyin.TargetProfile + page, pageErr := browser.Get(ctx, canonicalURL) + if pageErr == nil && page.Status >= http.StatusOK && page.Status < http.StatusMultipleChoices { + target, _ = douyin.ParseShareTargetHTML(page.Body, workKey) + } + if target.SecUID == "" { + target, err = (douyin.CreatorCollector{Browser: browser}).ResolveWork(ctx, workKey) + if err != nil { + return competitorSharePreview{}, fmt.Errorf("%w: target identity verification failed: %v", creator.ErrConflict, err) + } } return competitorSharePreview{ Platform: creator.PlatformDouyin, PlatformAccountKey: target.SecUID, + UniqueID: target.UniqueID, Nickname: target.Nickname, AvatarURL: target.AvatarURL, HomepageURL: "https://www.douyin.com/user/" + target.SecUID, @@ -1826,6 +1836,7 @@ func previewDouyinCompetitor(ctx context.Context, store *creator.Store, phaseASt "account_id": accountID, "platform": creator.PlatformDouyin, "platform_account_key": target.SecUID, + "unique_id": target.UniqueID, "nickname": target.Nickname, "avatar_url": target.AvatarURL, "homepage_url": "https://www.douyin.com/user/" + target.SecUID, diff --git a/internal/controlplane/api/environments.go b/internal/controlplane/api/environments.go index 26bd9d0..5b26860 100644 --- a/internal/controlplane/api/environments.go +++ b/internal/controlplane/api/environments.go @@ -66,10 +66,11 @@ type runtimeCleanupStore interface { } const ( - gatewayLongTimeout = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取 - gatewayReconcileDelay = 100 * time.Millisecond - gatewayReconcileAttempts = 10 - missingRuntimeID = "runtime-not-found" + gatewayLongTimeout = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取 + gatewayBrowserOperationTimeout = 90 * time.Second + gatewayReconcileDelay = 100 * time.Millisecond + gatewayReconcileAttempts = 10 + missingRuntimeID = "runtime-not-found" ) var ( diff --git a/internal/creator/content.go b/internal/creator/content.go index a400a86..82a0c31 100644 --- a/internal/creator/content.go +++ b/internal/creator/content.go @@ -39,6 +39,7 @@ func validateCreatorTags(tags []string) error { func (s *Store) CreateCompetitor(ctx context.Context, input CompetitorInput) (Competitor, error) { input.Platform = strings.TrimSpace(input.Platform) input.PlatformAccountKey = strings.TrimSpace(input.PlatformAccountKey) + input.UniqueID = strings.TrimSpace(input.UniqueID) input.Nickname = strings.TrimSpace(input.Nickname) input.AvatarURL = strings.TrimSpace(input.AvatarURL) input.HomepageURL = strings.TrimSpace(input.HomepageURL) @@ -46,16 +47,16 @@ func (s *Store) CreateCompetitor(ctx context.Context, input CompetitorInput) (Co input.Tags = []string{} } if !ValidatePlatform(input.Platform) || input.PlatformAccountKey == "" || input.HomepageURL == "" || - utf8.RuneCountInString(input.PlatformAccountKey) > 255 || utf8.RuneCountInString(input.Nickname) > 255 || + utf8.RuneCountInString(input.PlatformAccountKey) > 255 || utf8.RuneCountInString(input.UniqueID) > 255 || utf8.RuneCountInString(input.Nickname) > 255 || utf8.RuneCountInString(input.AvatarURL) > 1000 || validateHomepage(input.HomepageURL) != nil || validateCreatorTags(input.Tags) != nil { return Competitor{}, ErrInvalid } id := newID("competitor") if _, err := s.db.ExecContext(ctx, ` - INSERT INTO creator_competitor (id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags, next_sync_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, now())`, - id, input.Platform, input.PlatformAccountKey, input.Nickname, input.AvatarURL, input.HomepageURL, input.Tags); err != nil { + INSERT INTO creator_competitor (id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags, next_sync_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())`, + id, input.Platform, input.PlatformAccountKey, input.UniqueID, input.Nickname, input.AvatarURL, input.HomepageURL, input.Tags); err != nil { return Competitor{}, databaseError(err) } return s.GetCompetitor(ctx, id) @@ -65,7 +66,7 @@ func scanCompetitor(scanner interface{ Scan(...any) error }) (Competitor, error) var result Competitor var tags pgtype.FlatArray[string] var leaseUntil, lastSync, nextSync sql.NullTime - if err := scanner.Scan(&result.ID, &result.Platform, &result.PlatformAccountKey, &result.Nickname, + if err := scanner.Scan(&result.ID, &result.Platform, &result.PlatformAccountKey, &result.UniqueID, &result.Nickname, &result.AvatarURL, &result.HomepageURL, pgtype.NewMap().SQLScanner(&tags), &result.Enabled, &result.SyncStatus, &result.SyncCursor, &result.SyncError, &leaseUntil, &lastSync, &nextSync, &result.CreatedAt, &result.UpdatedAt); err != nil { return Competitor{}, err @@ -79,14 +80,14 @@ func scanCompetitor(scanner interface{ Scan(...any) error }) (Competitor, error) func (s *Store) GetCompetitor(ctx context.Context, id string) (Competitor, error) { result, err := scanCompetitor(s.db.QueryRowContext(ctx, ` - SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags, + SELECT id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags, enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at FROM creator_competitor WHERE id = $1`, id)) return result, rowError(err) } func (s *Store) ListCompetitors(ctx context.Context, platform string) ([]Competitor, error) { - query := `SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags, + query := `SELECT id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags, enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at FROM creator_competitor` args := []any{} @@ -209,7 +210,7 @@ func (s *Store) ListDueCompetitors(ctx context.Context, now time.Time) ([]Compet if now.IsZero() { return nil, ErrInvalid } - rows, err := s.db.QueryContext(ctx, `SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags, + rows, err := s.db.QueryContext(ctx, `SELECT id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags, enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at FROM creator_competitor WHERE enabled AND next_sync_at IS NOT NULL AND next_sync_at <= $1 diff --git a/internal/creator/integration_test.go b/internal/creator/integration_test.go index fa6f79f..83a26b0 100644 --- a/internal/creator/integration_test.go +++ b/internal/creator/integration_test.go @@ -192,11 +192,11 @@ func TestCreatorPostgresContentAndWorkflow(t *testing.T) { if err != nil || !inserted { t.Fatalf("insert work: work=%+v inserted=%v err=%v", work, inserted, err) } - competitor, err := store.CreateCompetitor(ctx, CompetitorInput{Platform: PlatformDouyin, PlatformAccountKey: "sec_uid_competitor_" + stamp, Nickname: "Competitor", HomepageURL: "https://www.douyin.com/user/sec_uid_competitor_" + stamp, Tags: []string{"重点监测"}}) + competitor, err := store.CreateCompetitor(ctx, CompetitorInput{Platform: PlatformDouyin, PlatformAccountKey: "sec_uid_competitor_" + stamp, UniqueID: "competitor_" + stamp, Nickname: "Competitor", HomepageURL: "https://www.douyin.com/user/sec_uid_competitor_" + stamp, Tags: []string{"重点监测"}}) if err != nil { t.Fatal(err) } - if len(competitor.Tags) != 1 || competitor.Tags[0] != "重点监测" { + if competitor.UniqueID != "competitor_"+stamp || len(competitor.Tags) != 1 || competitor.Tags[0] != "重点监测" { t.Fatalf("create competitor tags: %+v", competitor.Tags) } updatedCompetitor, err := store.UpdateCompetitorTags(ctx, competitor.ID, []string{"已分类"}) diff --git a/internal/creator/migrations/036_competitor_unique_id.sql b/internal/creator/migrations/036_competitor_unique_id.sql new file mode 100644 index 0000000..b4c7b7b --- /dev/null +++ b/internal/creator/migrations/036_competitor_unique_id.sql @@ -0,0 +1,7 @@ +ALTER TABLE creator_competitor +ADD COLUMN IF NOT EXISTS unique_id text NOT NULL DEFAULT ''; + +ALTER TABLE creator_competitor +DROP CONSTRAINT IF EXISTS creator_competitor_unique_id_check, +ADD CONSTRAINT creator_competitor_unique_id_check +CHECK (char_length(unique_id) <= 255); diff --git a/internal/creator/models.go b/internal/creator/models.go index 0c914f1..09b5bbb 100644 --- a/internal/creator/models.go +++ b/internal/creator/models.go @@ -107,6 +107,7 @@ type Competitor struct { ID string `json:"id"` Platform string `json:"platform"` PlatformAccountKey string `json:"platform_account_key"` + UniqueID string `json:"unique_id,omitempty"` Nickname string `json:"nickname"` AvatarURL string `json:"avatar_url"` HomepageURL string `json:"homepage_url"` @@ -125,6 +126,7 @@ type Competitor struct { type CompetitorInput struct { Platform string `json:"platform"` PlatformAccountKey string `json:"platform_account_key"` + UniqueID string `json:"unique_id"` Nickname string `json:"nickname"` AvatarURL string `json:"avatar_url"` HomepageURL string `json:"homepage_url"` diff --git a/internal/creator/store.go b/internal/creator/store.go index 16d2b1d..8b9689c 100644 --- a/internal/creator/store.go +++ b/internal/creator/store.go @@ -77,6 +77,9 @@ var migration034 string //go:embed migrations/035_account_deletion.sql var migration035 string +//go:embed migrations/036_competitor_unique_id.sql +var migration036 string + type SecretReference struct { ID string Provider string @@ -173,6 +176,7 @@ func (s *Store) migrate(ctx context.Context) error { {version: 33, sql: migration033}, {version: 34, sql: migration034}, {version: 35, sql: migration035}, + {version: 36, sql: migration036}, } for _, migration := range migrations { var applied bool diff --git a/internal/platform/douyin/share.go b/internal/platform/douyin/share.go new file mode 100644 index 0000000..7d74524 --- /dev/null +++ b/internal/platform/douyin/share.go @@ -0,0 +1,161 @@ +package douyin + +import ( + "bytes" + "encoding/json" +) + +const routerDataMarker = "window._ROUTER_DATA" + +// ParseShareTargetHTML extracts only the author identity needed to add a +// competitor. It intentionally ignores the work's playback and engagement data. +func ParseShareTargetHTML(body []byte, expectedWorkKey string) (TargetProfile, bool) { + if len(body) == 0 || len(body) > 2<<20 || !workKeyPattern.MatchString(expectedWorkKey) { + return TargetProfile{}, false + } + root, ok := extractRouterData(body) + if !ok { + return TargetProfile{}, false + } + loaderData, ok := root["loaderData"].(map[string]any) + if !ok { + return TargetProfile{}, false + } + page, ok := loaderData["video_(id)/page"].(map[string]any) + if !ok { + return TargetProfile{}, false + } + videoInfo, ok := page["videoInfoRes"].(map[string]any) + if !ok { + return TargetProfile{}, false + } + items, ok := videoInfo["item_list"].([]any) + if !ok || len(items) == 0 { + return TargetProfile{}, false + } + item, ok := items[0].(map[string]any) + if !ok { + return TargetProfile{}, false + } + if workKey, ok := stringValue(item["aweme_id"]); ok && workKey != expectedWorkKey { + return TargetProfile{}, false + } + author, ok := item["author"].(map[string]any) + if !ok { + return TargetProfile{}, false + } + return parseShareAuthor(author) +} + +func extractRouterData(body []byte) (map[string]any, bool) { + marker := []byte(routerDataMarker) + for offset := 0; ; { + index := bytes.Index(body[offset:], marker) + if index < 0 { + return nil, false + } + index += offset + len(marker) + equals := bytes.IndexByte(body[index:], '=') + if equals < 0 { + return nil, false + } + start := index + equals + 1 + for start < len(body) && (body[start] == ' ' || body[start] == '\n' || body[start] == '\r' || body[start] == '\t') { + start++ + } + if start >= len(body) || body[start] != '{' { + offset = index + continue + } + end, ok := jsonObjectEnd(body, start) + if !ok { + return nil, false + } + var root map[string]any + if json.Unmarshal(body[start:end], &root) == nil { + return root, true + } + offset = end + } +} + +func jsonObjectEnd(body []byte, start int) (int, bool) { + depth := 0 + inString := false + escaped := false + for index := start; index < len(body); index++ { + char := body[index] + if inString { + if escaped { + escaped = false + } else if char == '\\' { + escaped = true + } else if char == '"' { + inString = false + } + continue + } + switch char { + case '"': + inString = true + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return index + 1, true + } + } + } + return 0, false +} + +func parseShareAuthor(author map[string]any) (TargetProfile, bool) { + secUID, ok := stringValue(author["sec_uid"]) + if !ok || !keyPattern.MatchString(secUID) { + return TargetProfile{}, false + } + uid := stringOrEmpty(author["uid"]) + uniqueID := stringOrEmpty(author["unique_id"]) + if uid != "" && !keyPattern.MatchString(uid) || uniqueID != "" && !keyPattern.MatchString(uniqueID) { + return TargetProfile{}, false + } + return TargetProfile{ + UID: uid, + SecUID: secUID, + UniqueID: uniqueID, + Nickname: stringOrEmpty(author["nickname"]), + AvatarURL: avatarURL(author), + }, true +} + +func stringValue(value any) (string, bool) { + result, ok := value.(string) + return result, ok && result != "" +} + +func stringOrEmpty(value any) string { + result, _ := value.(string) + return result +} + +func avatarURL(author map[string]any) string { + for _, key := range []string{"avatar_thumb", "avatar_larger", "avatar_medium", "avatar"} { + value := author[key] + if direct, ok := value.(string); ok && direct != "" { + return direct + } + container, ok := value.(map[string]any) + if !ok { + continue + } + urls, ok := container["url_list"].([]any) + if !ok || len(urls) == 0 { + continue + } + if result, ok := urls[0].(string); ok { + return result + } + } + return "" +} diff --git a/internal/platform/douyin/share_test.go b/internal/platform/douyin/share_test.go new file mode 100644 index 0000000..dabe0b5 --- /dev/null +++ b/internal/platform/douyin/share_test.go @@ -0,0 +1,33 @@ +package douyin + +import "testing" + +func TestParseShareTargetHTML(t *testing.T) { + body := []byte(``) + profile, ok := ParseShareTargetHTML(body, "7667952980074745151") + if !ok { + t.Fatal("expected share author to parse") + } + if profile.SecUID != "MS4wLjABAAAAexample" || profile.UniqueID != "muzi_code" || + profile.Nickname != "木子不写代码" || profile.AvatarURL != "https://example.com/avatar.jpg" { + t.Fatalf("unexpected share profile: %+v", profile) + } +} + +func TestParseShareTargetHTMLRejectsWrongWork(t *testing.T) { + body := []byte(``) + if _, ok := ParseShareTargetHTML(body, "2"); ok { + t.Fatal("accepted a mismatched work id") + } +} diff --git a/web/src/styles/globals.css b/web/src/styles/globals.css index 64ff066..2df4602 100644 --- a/web/src/styles/globals.css +++ b/web/src/styles/globals.css @@ -1,5 +1,6 @@ @import 'tailwindcss'; @import 'tw-animate-css'; +@import 'remixicon/fonts/remixicon.css'; @custom-variant dark (&:is(.dark *)); diff --git a/web/vite.config.ts b/web/vite.config.ts index 3ae8f67..62c6683 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -46,6 +46,7 @@ function creatorHubAssetFetchDest(): Plugin { if (request.headers['sec-fetch-dest']) return next() const pathname = request.url?.split(/[?#]/, 1)[0] ?? '' if (/\.css$/.test(pathname)) request.headers['sec-fetch-dest'] = 'style' + else if (/\.(?:woff2?|eot|ttf|otf)$/.test(pathname)) request.headers['sec-fetch-dest'] = 'font' else if (/\.(?:[cm]?[jt]sx?|mjs)$/.test(pathname)) request.headers['sec-fetch-dest'] = 'script' next() })