fix gateway share resolution and competitor identity
This commit is contained in:
@@ -15,6 +15,7 @@ import websocket
|
||||
LOG = logging.getLogger("creatorhub.browser.cdp")
|
||||
|
||||
CONTROL_TIMEOUT = 15.0
|
||||
RESOLVE_TIMEOUT = 90.0
|
||||
|
||||
|
||||
class BrowserError(RuntimeError):
|
||||
@@ -30,6 +31,13 @@ class CDPConnection:
|
||||
self._next_id = 0
|
||||
self._pending: deque[dict] = deque()
|
||||
|
||||
def notify(self, method: str, params: dict | None = None) -> None:
|
||||
with self._lock:
|
||||
self._next_id += 1
|
||||
self.socket.send(
|
||||
json.dumps({"id": self._next_id, "method": method, "params": params or {}})
|
||||
)
|
||||
|
||||
def command(self, method: str, params: dict | None = None) -> dict:
|
||||
with self._lock:
|
||||
self._next_id += 1
|
||||
|
||||
@@ -21,7 +21,12 @@ from urllib.parse import urlsplit
|
||||
|
||||
import websocket
|
||||
|
||||
from ..browser.cdp import CONTROL_TIMEOUT, BrowserError, CDPConnection
|
||||
from ..browser.cdp import (
|
||||
CONTROL_TIMEOUT,
|
||||
RESOLVE_TIMEOUT,
|
||||
BrowserError,
|
||||
CDPConnection,
|
||||
)
|
||||
from ..browser.response import (
|
||||
BrowserLoginQRResponse,
|
||||
BrowserMediaResponse,
|
||||
@@ -245,25 +250,28 @@ class DouyinBrowser:
|
||||
with self.connection(alias) as cdp:
|
||||
if cdp.evaluate("location.origin") not in {"null", self.origin}:
|
||||
raise DouyinError("restricted browser origin changed")
|
||||
navigation = cdp.evaluate(
|
||||
"window.location.assign(" + json.dumps(target) + "); true"
|
||||
)
|
||||
if not isinstance(navigation, bool) or not navigation:
|
||||
raise DouyinError("Douyin share URL navigation failed")
|
||||
deadline = time.monotonic() + CONTROL_TIMEOUT
|
||||
try:
|
||||
cdp.command("Page.enable")
|
||||
cdp.notify("Page.navigate", {"url": target})
|
||||
except (OSError, websocket.WebSocketException) as exc:
|
||||
raise DouyinError("Douyin share URL navigation failed") from exc
|
||||
deadline = time.monotonic() + RESOLVE_TIMEOUT
|
||||
final_url = ""
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current_url = cdp.evaluate("location.href")
|
||||
event = cdp.wait_event(
|
||||
"Page.frameNavigated",
|
||||
lambda params: isinstance(params.get("frame"), dict)
|
||||
and not params["frame"].get("parentId"),
|
||||
timeout=max(0.01, deadline - time.monotonic()),
|
||||
)
|
||||
except BrowserError:
|
||||
if is_douyin_content_url(target):
|
||||
final_url = target
|
||||
break
|
||||
raise
|
||||
break
|
||||
frame = event.get("params", {}).get("frame", {})
|
||||
current_url = frame.get("url") if isinstance(frame, dict) else None
|
||||
if isinstance(current_url, str) and is_douyin_content_url(current_url):
|
||||
final_url = current_url
|
||||
break
|
||||
time.sleep(0.1)
|
||||
if not final_url:
|
||||
raise DouyinError("Douyin share URL did not resolve to a supported page")
|
||||
return final_url
|
||||
|
||||
@@ -6,7 +6,9 @@ import json
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from ..browser.cdp import CONTROL_TIMEOUT, BrowserError
|
||||
import websocket
|
||||
|
||||
from ..browser.cdp import RESOLVE_TIMEOUT, BrowserError
|
||||
from ..browser.response import (
|
||||
BrowserResponse,
|
||||
detect_challenge,
|
||||
@@ -81,25 +83,28 @@ class XiaohongshuBrowser(DouyinBrowser):
|
||||
with self.connection(alias) as cdp:
|
||||
if cdp.evaluate("location.origin") not in {"null", self.origin}:
|
||||
raise DouyinError("restricted browser origin changed")
|
||||
navigation = cdp.evaluate(
|
||||
"window.location.assign(" + json.dumps(target) + "); true"
|
||||
)
|
||||
if not isinstance(navigation, bool) or not navigation:
|
||||
raise DouyinError("Xiaohongshu share URL navigation failed")
|
||||
deadline = time.monotonic() + CONTROL_TIMEOUT
|
||||
try:
|
||||
cdp.command("Page.enable")
|
||||
cdp.notify("Page.navigate", {"url": target})
|
||||
except (OSError, websocket.WebSocketException) as exc:
|
||||
raise DouyinError("Xiaohongshu share URL navigation failed") from exc
|
||||
deadline = time.monotonic() + RESOLVE_TIMEOUT
|
||||
final_url = ""
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current_url = cdp.evaluate("location.href")
|
||||
event = cdp.wait_event(
|
||||
"Page.frameNavigated",
|
||||
lambda params: isinstance(params.get("frame"), dict)
|
||||
and not params["frame"].get("parentId"),
|
||||
timeout=max(0.01, deadline - time.monotonic()),
|
||||
)
|
||||
except BrowserError:
|
||||
if _is_xiaohongshu_page_url(target):
|
||||
final_url = target
|
||||
break
|
||||
raise
|
||||
break
|
||||
frame = event.get("params", {}).get("frame", {})
|
||||
current_url = frame.get("url") if isinstance(frame, dict) else None
|
||||
if isinstance(current_url, str) and _is_xiaohongshu_page_url(current_url):
|
||||
final_url = current_url
|
||||
break
|
||||
time.sleep(0.1)
|
||||
if not final_url:
|
||||
raise DouyinError(
|
||||
"Xiaohongshu share URL did not resolve to a supported page"
|
||||
|
||||
@@ -34,7 +34,7 @@ PROFILE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$")
|
||||
RUNTIME_CLEANUP_SENTINEL = "runtime-not-found"
|
||||
|
||||
DEFAULT_CLEANUP_TIMEOUT = 30.0
|
||||
DEFAULT_READY_TIMEOUT = 15.0
|
||||
DEFAULT_READY_TIMEOUT = 60.0
|
||||
DEFAULT_DISPLAY_START = 100
|
||||
DEFAULT_DISPLAY_END = 199
|
||||
DEFAULT_CDP_PORT_START = 19000
|
||||
|
||||
@@ -35,12 +35,6 @@ from ..platform.douyin import (
|
||||
)
|
||||
from ..platform.xiaohongshu import XiaohongshuBrowser, is_xiaohongshu_share_url
|
||||
from ..proxy import ProxyExit
|
||||
from ..runtime import (
|
||||
BrowserRuntimeError,
|
||||
NativeRuntimeManager,
|
||||
has_control,
|
||||
validate_runtime_input,
|
||||
)
|
||||
from ..runtime import (
|
||||
NETWORK_ID_RE as _NETWORK_ID_RE,
|
||||
)
|
||||
@@ -50,6 +44,12 @@ from ..runtime import (
|
||||
from ..runtime import (
|
||||
RUNTIME_ID_RE as _RUNTIME_ID_RE,
|
||||
)
|
||||
from ..runtime import (
|
||||
BrowserRuntimeError,
|
||||
NativeRuntimeManager,
|
||||
has_control,
|
||||
validate_runtime_input,
|
||||
)
|
||||
from ..runtime import (
|
||||
parse_proxy_exit as _parse_proxy_exit,
|
||||
)
|
||||
@@ -832,12 +832,18 @@ class GatewayHandler(BaseHTTPRequestHandler):
|
||||
return value
|
||||
|
||||
def _respond(self, status: int, body: bytes) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
try:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
except BrokenPipeError:
|
||||
LOG.info(
|
||||
"gateway client disconnected before response",
|
||||
extra={"status": status},
|
||||
)
|
||||
|
||||
|
||||
def json_bytes(value: object) -> bytes:
|
||||
@@ -1275,7 +1281,7 @@ def load_config(env: Mapping[str, str] | None = None) -> dict:
|
||||
"node_name": node_name,
|
||||
"token": token,
|
||||
"cleanup_timeout": _positive_float(env, "RUNTIME_CLEANUP_TIMEOUT", 30.0, 300.0),
|
||||
"ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 15.0, 300.0),
|
||||
"ready_timeout": _positive_float(env, "RUNTIME_READY_TIMEOUT", 60.0, 300.0),
|
||||
"min_free_bytes": _integer(env, "RUNTIME_MIN_FREE_BYTES", 20 * 1024**3),
|
||||
"log_max_bytes": _integer(env, "RUNTIME_LOG_MAX_BYTES", 1 * 1024**3),
|
||||
"profile_cache_max_bytes": _integer(env, "PROFILE_CACHE_MAX_BYTES", 20 * 1024**3),
|
||||
|
||||
@@ -155,6 +155,7 @@ class GatewayValidationTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(config["listen"], ("", 8081))
|
||||
self.assertEqual(config["browser_path"], "/bin/true")
|
||||
self.assertEqual(config["ready_timeout"], 60.0)
|
||||
self.assertIsNone(config["external_display"])
|
||||
external = load_config(
|
||||
{
|
||||
@@ -756,7 +757,7 @@ class BrowserTests(unittest.TestCase):
|
||||
cdp.evaluate.side_effect = ["null", True, "https://www.douyin.com/video/123"]
|
||||
cdp.command.return_value = {"frameId": "frame-1"}
|
||||
cdp.wait_event.return_value = {
|
||||
"frame": {"url": "https://www.douyin.com/video/123"}
|
||||
"params": {"frame": {"url": "https://www.douyin.com/video/123"}}
|
||||
}
|
||||
browser = DouyinBrowser()
|
||||
self._with_connection(browser, cast(BrowserCDP, cdp))
|
||||
@@ -765,12 +766,11 @@ class BrowserTests(unittest.TestCase):
|
||||
browser.resolve("safe", "https://www.douyin.com/video/123"),
|
||||
"https://www.douyin.com/video/123",
|
||||
)
|
||||
self.assertEqual(cdp.evaluate.call_count, 3)
|
||||
self.assertEqual(cdp.evaluate.call_count, 1)
|
||||
self.assertEqual(cdp.evaluate.call_args_list[0].args, ("location.origin",))
|
||||
self.assertIn(
|
||||
"window.location.assign(\"https://www.douyin.com/video/123\")",
|
||||
cdp.evaluate.call_args_list[1].args[0],
|
||||
)
|
||||
self.assertEqual(cdp.command.call_args_list[0].args, ("Page.enable",))
|
||||
self.assertEqual(cdp.notify.call_args_list[0].args, ("Page.navigate", {"url": "https://www.douyin.com/video/123"}))
|
||||
|
||||
def test_browser_fetch_uses_manually_logged_session(self) -> None:
|
||||
cdp = BrowserCDP(
|
||||
[
|
||||
|
||||
@@ -98,7 +98,7 @@ class XiaohongshuBrowserTests(unittest.TestCase):
|
||||
]
|
||||
cdp.command.return_value = {"frameId": "frame-1"}
|
||||
cdp.wait_event.return_value = {
|
||||
"frame": {"url": "https://www.xiaohongshu.com/explore/n-1"}
|
||||
"params": {"frame": {"url": "https://www.xiaohongshu.com/explore/n-1"}}
|
||||
}
|
||||
|
||||
@contextmanager
|
||||
|
||||
@@ -6,7 +6,7 @@ BROWSER_PROFILE_ROOT=~/.local/share/creatorhub/browser-profiles
|
||||
BROWSER_PATH=~/.local/share/creatorhub/browsers/fingerprint-chromium/148.0.7778.215/chrome
|
||||
NODE_NAME=
|
||||
RUNTIME_CLEANUP_TIMEOUT=30
|
||||
RUNTIME_READY_TIMEOUT=15
|
||||
RUNTIME_READY_TIMEOUT=60
|
||||
RUNTIME_MIN_FREE_BYTES=21474836480
|
||||
RUNTIME_LOG_MAX_BYTES=1073741824
|
||||
PROFILE_CACHE_MAX_BYTES=21474836480
|
||||
|
||||
@@ -1441,7 +1441,7 @@ func (browser creatorGatewayBrowser) Get(ctx context.Context, target string) (do
|
||||
func (browser creatorGatewayBrowser) Resolve(ctx context.Context, target string) (string, error) {
|
||||
payload := gatewayGenerationPayload(browser.environment)
|
||||
payload["url"] = target
|
||||
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/resolve", payload, 30*time.Second)
|
||||
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/resolve", payload, gatewayBrowserOperationTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -1535,6 +1535,7 @@ type competitorSharePreview struct {
|
||||
AccountID string `json:"account_id"`
|
||||
Platform string `json:"platform"`
|
||||
PlatformAccountKey string `json:"platform_account_key"`
|
||||
UniqueID string `json:"unique_id,omitempty"`
|
||||
Nickname string `json:"nickname"`
|
||||
AvatarURL string `json:"avatar_url,omitempty"`
|
||||
HomepageURL string `json:"homepage_url"`
|
||||
@@ -1545,6 +1546,7 @@ func (preview competitorSharePreview) input() creator.CompetitorInput {
|
||||
return creator.CompetitorInput{
|
||||
Platform: preview.Platform,
|
||||
PlatformAccountKey: preview.PlatformAccountKey,
|
||||
UniqueID: preview.UniqueID,
|
||||
Nickname: preview.Nickname,
|
||||
AvatarURL: preview.AvatarURL,
|
||||
HomepageURL: preview.HomepageURL,
|
||||
@@ -1720,13 +1722,21 @@ func previewDouyinCompetitorShare(ctx context.Context, store *creator.Store, pha
|
||||
if err != nil {
|
||||
return competitorSharePreview{}, err
|
||||
}
|
||||
target, err := (douyin.CreatorCollector{Browser: browser}).ResolveWork(ctx, workKey)
|
||||
if err != nil {
|
||||
return competitorSharePreview{}, fmt.Errorf("%w: target identity verification failed: %v", creator.ErrConflict, err)
|
||||
var target douyin.TargetProfile
|
||||
page, pageErr := browser.Get(ctx, canonicalURL)
|
||||
if pageErr == nil && page.Status >= http.StatusOK && page.Status < http.StatusMultipleChoices {
|
||||
target, _ = douyin.ParseShareTargetHTML(page.Body, workKey)
|
||||
}
|
||||
if target.SecUID == "" {
|
||||
target, err = (douyin.CreatorCollector{Browser: browser}).ResolveWork(ctx, workKey)
|
||||
if err != nil {
|
||||
return competitorSharePreview{}, fmt.Errorf("%w: target identity verification failed: %v", creator.ErrConflict, err)
|
||||
}
|
||||
}
|
||||
return competitorSharePreview{
|
||||
Platform: creator.PlatformDouyin,
|
||||
PlatformAccountKey: target.SecUID,
|
||||
UniqueID: target.UniqueID,
|
||||
Nickname: target.Nickname,
|
||||
AvatarURL: target.AvatarURL,
|
||||
HomepageURL: "https://www.douyin.com/user/" + target.SecUID,
|
||||
@@ -1826,6 +1836,7 @@ func previewDouyinCompetitor(ctx context.Context, store *creator.Store, phaseASt
|
||||
"account_id": accountID,
|
||||
"platform": creator.PlatformDouyin,
|
||||
"platform_account_key": target.SecUID,
|
||||
"unique_id": target.UniqueID,
|
||||
"nickname": target.Nickname,
|
||||
"avatar_url": target.AvatarURL,
|
||||
"homepage_url": "https://www.douyin.com/user/" + target.SecUID,
|
||||
|
||||
@@ -66,10 +66,11 @@ type runtimeCleanupStore interface {
|
||||
}
|
||||
|
||||
const (
|
||||
gatewayLongTimeout = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取
|
||||
gatewayReconcileDelay = 100 * time.Millisecond
|
||||
gatewayReconcileAttempts = 10
|
||||
missingRuntimeID = "runtime-not-found"
|
||||
gatewayLongTimeout = 11 * time.Minute // 覆盖网关侧最长 10 分钟的镜像拉取
|
||||
gatewayBrowserOperationTimeout = 90 * time.Second
|
||||
gatewayReconcileDelay = 100 * time.Millisecond
|
||||
gatewayReconcileAttempts = 10
|
||||
missingRuntimeID = "runtime-not-found"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -39,6 +39,7 @@ func validateCreatorTags(tags []string) error {
|
||||
func (s *Store) CreateCompetitor(ctx context.Context, input CompetitorInput) (Competitor, error) {
|
||||
input.Platform = strings.TrimSpace(input.Platform)
|
||||
input.PlatformAccountKey = strings.TrimSpace(input.PlatformAccountKey)
|
||||
input.UniqueID = strings.TrimSpace(input.UniqueID)
|
||||
input.Nickname = strings.TrimSpace(input.Nickname)
|
||||
input.AvatarURL = strings.TrimSpace(input.AvatarURL)
|
||||
input.HomepageURL = strings.TrimSpace(input.HomepageURL)
|
||||
@@ -46,16 +47,16 @@ func (s *Store) CreateCompetitor(ctx context.Context, input CompetitorInput) (Co
|
||||
input.Tags = []string{}
|
||||
}
|
||||
if !ValidatePlatform(input.Platform) || input.PlatformAccountKey == "" || input.HomepageURL == "" ||
|
||||
utf8.RuneCountInString(input.PlatformAccountKey) > 255 || utf8.RuneCountInString(input.Nickname) > 255 ||
|
||||
utf8.RuneCountInString(input.PlatformAccountKey) > 255 || utf8.RuneCountInString(input.UniqueID) > 255 || utf8.RuneCountInString(input.Nickname) > 255 ||
|
||||
utf8.RuneCountInString(input.AvatarURL) > 1000 || validateHomepage(input.HomepageURL) != nil ||
|
||||
validateCreatorTags(input.Tags) != nil {
|
||||
return Competitor{}, ErrInvalid
|
||||
}
|
||||
id := newID("competitor")
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO creator_competitor (id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags, next_sync_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, now())`,
|
||||
id, input.Platform, input.PlatformAccountKey, input.Nickname, input.AvatarURL, input.HomepageURL, input.Tags); err != nil {
|
||||
INSERT INTO creator_competitor (id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags, next_sync_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())`,
|
||||
id, input.Platform, input.PlatformAccountKey, input.UniqueID, input.Nickname, input.AvatarURL, input.HomepageURL, input.Tags); err != nil {
|
||||
return Competitor{}, databaseError(err)
|
||||
}
|
||||
return s.GetCompetitor(ctx, id)
|
||||
@@ -65,7 +66,7 @@ func scanCompetitor(scanner interface{ Scan(...any) error }) (Competitor, error)
|
||||
var result Competitor
|
||||
var tags pgtype.FlatArray[string]
|
||||
var leaseUntil, lastSync, nextSync sql.NullTime
|
||||
if err := scanner.Scan(&result.ID, &result.Platform, &result.PlatformAccountKey, &result.Nickname,
|
||||
if err := scanner.Scan(&result.ID, &result.Platform, &result.PlatformAccountKey, &result.UniqueID, &result.Nickname,
|
||||
&result.AvatarURL, &result.HomepageURL, pgtype.NewMap().SQLScanner(&tags), &result.Enabled, &result.SyncStatus, &result.SyncCursor,
|
||||
&result.SyncError, &leaseUntil, &lastSync, &nextSync, &result.CreatedAt, &result.UpdatedAt); err != nil {
|
||||
return Competitor{}, err
|
||||
@@ -79,14 +80,14 @@ func scanCompetitor(scanner interface{ Scan(...any) error }) (Competitor, error)
|
||||
|
||||
func (s *Store) GetCompetitor(ctx context.Context, id string) (Competitor, error) {
|
||||
result, err := scanCompetitor(s.db.QueryRowContext(ctx, `
|
||||
SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags,
|
||||
SELECT id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags,
|
||||
enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at
|
||||
FROM creator_competitor WHERE id = $1`, id))
|
||||
return result, rowError(err)
|
||||
}
|
||||
|
||||
func (s *Store) ListCompetitors(ctx context.Context, platform string) ([]Competitor, error) {
|
||||
query := `SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags,
|
||||
query := `SELECT id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags,
|
||||
enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at
|
||||
FROM creator_competitor`
|
||||
args := []any{}
|
||||
@@ -209,7 +210,7 @@ func (s *Store) ListDueCompetitors(ctx context.Context, now time.Time) ([]Compet
|
||||
if now.IsZero() {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id, platform, platform_account_key, nickname, avatar_url, homepage_url, tags,
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id, platform, platform_account_key, unique_id, nickname, avatar_url, homepage_url, tags,
|
||||
enabled, sync_status, sync_cursor, sync_error, sync_lease_until, last_sync_at, next_sync_at, created_at, updated_at
|
||||
FROM creator_competitor
|
||||
WHERE enabled AND next_sync_at IS NOT NULL AND next_sync_at <= $1
|
||||
|
||||
@@ -192,11 +192,11 @@ func TestCreatorPostgresContentAndWorkflow(t *testing.T) {
|
||||
if err != nil || !inserted {
|
||||
t.Fatalf("insert work: work=%+v inserted=%v err=%v", work, inserted, err)
|
||||
}
|
||||
competitor, err := store.CreateCompetitor(ctx, CompetitorInput{Platform: PlatformDouyin, PlatformAccountKey: "sec_uid_competitor_" + stamp, Nickname: "Competitor", HomepageURL: "https://www.douyin.com/user/sec_uid_competitor_" + stamp, Tags: []string{"重点监测"}})
|
||||
competitor, err := store.CreateCompetitor(ctx, CompetitorInput{Platform: PlatformDouyin, PlatformAccountKey: "sec_uid_competitor_" + stamp, UniqueID: "competitor_" + stamp, Nickname: "Competitor", HomepageURL: "https://www.douyin.com/user/sec_uid_competitor_" + stamp, Tags: []string{"重点监测"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(competitor.Tags) != 1 || competitor.Tags[0] != "重点监测" {
|
||||
if competitor.UniqueID != "competitor_"+stamp || len(competitor.Tags) != 1 || competitor.Tags[0] != "重点监测" {
|
||||
t.Fatalf("create competitor tags: %+v", competitor.Tags)
|
||||
}
|
||||
updatedCompetitor, err := store.UpdateCompetitorTags(ctx, competitor.ID, []string{"已分类"})
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
ALTER TABLE creator_competitor
|
||||
ADD COLUMN IF NOT EXISTS unique_id text NOT NULL DEFAULT '';
|
||||
|
||||
ALTER TABLE creator_competitor
|
||||
DROP CONSTRAINT IF EXISTS creator_competitor_unique_id_check,
|
||||
ADD CONSTRAINT creator_competitor_unique_id_check
|
||||
CHECK (char_length(unique_id) <= 255);
|
||||
@@ -107,6 +107,7 @@ type Competitor struct {
|
||||
ID string `json:"id"`
|
||||
Platform string `json:"platform"`
|
||||
PlatformAccountKey string `json:"platform_account_key"`
|
||||
UniqueID string `json:"unique_id,omitempty"`
|
||||
Nickname string `json:"nickname"`
|
||||
AvatarURL string `json:"avatar_url"`
|
||||
HomepageURL string `json:"homepage_url"`
|
||||
@@ -125,6 +126,7 @@ type Competitor struct {
|
||||
type CompetitorInput struct {
|
||||
Platform string `json:"platform"`
|
||||
PlatformAccountKey string `json:"platform_account_key"`
|
||||
UniqueID string `json:"unique_id"`
|
||||
Nickname string `json:"nickname"`
|
||||
AvatarURL string `json:"avatar_url"`
|
||||
HomepageURL string `json:"homepage_url"`
|
||||
|
||||
@@ -77,6 +77,9 @@ var migration034 string
|
||||
//go:embed migrations/035_account_deletion.sql
|
||||
var migration035 string
|
||||
|
||||
//go:embed migrations/036_competitor_unique_id.sql
|
||||
var migration036 string
|
||||
|
||||
type SecretReference struct {
|
||||
ID string
|
||||
Provider string
|
||||
@@ -173,6 +176,7 @@ func (s *Store) migrate(ctx context.Context) error {
|
||||
{version: 33, sql: migration033},
|
||||
{version: 34, sql: migration034},
|
||||
{version: 35, sql: migration035},
|
||||
{version: 36, sql: migration036},
|
||||
}
|
||||
for _, migration := range migrations {
|
||||
var applied bool
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package douyin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
const routerDataMarker = "window._ROUTER_DATA"
|
||||
|
||||
// ParseShareTargetHTML extracts only the author identity needed to add a
|
||||
// competitor. It intentionally ignores the work's playback and engagement data.
|
||||
func ParseShareTargetHTML(body []byte, expectedWorkKey string) (TargetProfile, bool) {
|
||||
if len(body) == 0 || len(body) > 2<<20 || !workKeyPattern.MatchString(expectedWorkKey) {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
root, ok := extractRouterData(body)
|
||||
if !ok {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
loaderData, ok := root["loaderData"].(map[string]any)
|
||||
if !ok {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
page, ok := loaderData["video_(id)/page"].(map[string]any)
|
||||
if !ok {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
videoInfo, ok := page["videoInfoRes"].(map[string]any)
|
||||
if !ok {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
items, ok := videoInfo["item_list"].([]any)
|
||||
if !ok || len(items) == 0 {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
item, ok := items[0].(map[string]any)
|
||||
if !ok {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
if workKey, ok := stringValue(item["aweme_id"]); ok && workKey != expectedWorkKey {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
author, ok := item["author"].(map[string]any)
|
||||
if !ok {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
return parseShareAuthor(author)
|
||||
}
|
||||
|
||||
func extractRouterData(body []byte) (map[string]any, bool) {
|
||||
marker := []byte(routerDataMarker)
|
||||
for offset := 0; ; {
|
||||
index := bytes.Index(body[offset:], marker)
|
||||
if index < 0 {
|
||||
return nil, false
|
||||
}
|
||||
index += offset + len(marker)
|
||||
equals := bytes.IndexByte(body[index:], '=')
|
||||
if equals < 0 {
|
||||
return nil, false
|
||||
}
|
||||
start := index + equals + 1
|
||||
for start < len(body) && (body[start] == ' ' || body[start] == '\n' || body[start] == '\r' || body[start] == '\t') {
|
||||
start++
|
||||
}
|
||||
if start >= len(body) || body[start] != '{' {
|
||||
offset = index
|
||||
continue
|
||||
}
|
||||
end, ok := jsonObjectEnd(body, start)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
var root map[string]any
|
||||
if json.Unmarshal(body[start:end], &root) == nil {
|
||||
return root, true
|
||||
}
|
||||
offset = end
|
||||
}
|
||||
}
|
||||
|
||||
func jsonObjectEnd(body []byte, start int) (int, bool) {
|
||||
depth := 0
|
||||
inString := false
|
||||
escaped := false
|
||||
for index := start; index < len(body); index++ {
|
||||
char := body[index]
|
||||
if inString {
|
||||
if escaped {
|
||||
escaped = false
|
||||
} else if char == '\\' {
|
||||
escaped = true
|
||||
} else if char == '"' {
|
||||
inString = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch char {
|
||||
case '"':
|
||||
inString = true
|
||||
case '{':
|
||||
depth++
|
||||
case '}':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return index + 1, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func parseShareAuthor(author map[string]any) (TargetProfile, bool) {
|
||||
secUID, ok := stringValue(author["sec_uid"])
|
||||
if !ok || !keyPattern.MatchString(secUID) {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
uid := stringOrEmpty(author["uid"])
|
||||
uniqueID := stringOrEmpty(author["unique_id"])
|
||||
if uid != "" && !keyPattern.MatchString(uid) || uniqueID != "" && !keyPattern.MatchString(uniqueID) {
|
||||
return TargetProfile{}, false
|
||||
}
|
||||
return TargetProfile{
|
||||
UID: uid,
|
||||
SecUID: secUID,
|
||||
UniqueID: uniqueID,
|
||||
Nickname: stringOrEmpty(author["nickname"]),
|
||||
AvatarURL: avatarURL(author),
|
||||
}, true
|
||||
}
|
||||
|
||||
func stringValue(value any) (string, bool) {
|
||||
result, ok := value.(string)
|
||||
return result, ok && result != ""
|
||||
}
|
||||
|
||||
func stringOrEmpty(value any) string {
|
||||
result, _ := value.(string)
|
||||
return result
|
||||
}
|
||||
|
||||
func avatarURL(author map[string]any) string {
|
||||
for _, key := range []string{"avatar_thumb", "avatar_larger", "avatar_medium", "avatar"} {
|
||||
value := author[key]
|
||||
if direct, ok := value.(string); ok && direct != "" {
|
||||
return direct
|
||||
}
|
||||
container, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
urls, ok := container["url_list"].([]any)
|
||||
if !ok || len(urls) == 0 {
|
||||
continue
|
||||
}
|
||||
if result, ok := urls[0].(string); ok {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package douyin
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseShareTargetHTML(t *testing.T) {
|
||||
body := []byte(`<script>window._ROUTER_DATA = {
|
||||
"loaderData": {"video_(id)/page": {"videoInfoRes": {"item_list": [{
|
||||
"aweme_id": "7667952980074745151",
|
||||
"author": {
|
||||
"uid": "123456789",
|
||||
"sec_uid": "MS4wLjABAAAAexample",
|
||||
"unique_id": "muzi_code",
|
||||
"nickname": "木子不写代码",
|
||||
"avatar_thumb": {"url_list": ["https://example.com/avatar.jpg"]}
|
||||
}
|
||||
}]}}}
|
||||
};</script>`)
|
||||
profile, ok := ParseShareTargetHTML(body, "7667952980074745151")
|
||||
if !ok {
|
||||
t.Fatal("expected share author to parse")
|
||||
}
|
||||
if profile.SecUID != "MS4wLjABAAAAexample" || profile.UniqueID != "muzi_code" ||
|
||||
profile.Nickname != "木子不写代码" || profile.AvatarURL != "https://example.com/avatar.jpg" {
|
||||
t.Fatalf("unexpected share profile: %+v", profile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShareTargetHTMLRejectsWrongWork(t *testing.T) {
|
||||
body := []byte(`<script>window._ROUTER_DATA = {"loaderData":{"video_(id)/page":{"videoInfoRes":{"item_list":[{"aweme_id":"1","author":{"sec_uid":"MS4wLjABAAAAexample"}}]}}}};</script>`)
|
||||
if _, ok := ParseShareTargetHTML(body, "2"); ok {
|
||||
t.Fatal("accepted a mismatched work id")
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
@import 'tailwindcss';
|
||||
@import 'tw-animate-css';
|
||||
@import 'remixicon/fonts/remixicon.css';
|
||||
|
||||
@custom-variant dark (&:is(.dark *));
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ function creatorHubAssetFetchDest(): Plugin {
|
||||
if (request.headers['sec-fetch-dest']) return next()
|
||||
const pathname = request.url?.split(/[?#]/, 1)[0] ?? ''
|
||||
if (/\.css$/.test(pathname)) request.headers['sec-fetch-dest'] = 'style'
|
||||
else if (/\.(?:woff2?|eot|ttf|otf)$/.test(pathname)) request.headers['sec-fetch-dest'] = 'font'
|
||||
else if (/\.(?:[cm]?[jt]sx?|mjs)$/.test(pathname)) request.headers['sec-fetch-dest'] = 'script'
|
||||
next()
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user