rogee/creator-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: continue CreatorHub plan01 implementation

Browse Source
This commit is contained in:
rogee
2026-09-12 20:17:29 +08:00
parent 1fbf126b21
commit 860ecaf7ea
52 changed files with 9441 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/docker-gateway/douyin.go
+74 -13
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"io"
"net"
"net/http"
"net/url"
"regexp"
@@ -15,6 +16,7 @@ import (
"git.ipao.vip/rogee/creator-hub/internal/douyin"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"golang.org/x/net/websocket"
)
@@ -23,6 +25,7 @@ const (
douyinOriginURL = "https://www.douyin.com/"
douyinIdentityPath = "/aweme/v1/web/user/profile/self/"
douyinWorksPath = "/aweme/v1/web/aweme/post/"
douyinCommentsPath = "/aweme/v1/web/comment/list/"
douyinResponseLimit = 1 << 20
browserControlTimeout = 15 * time.Second
)
@@ -71,7 +74,11 @@ func (api gateway) setDouyinCookies(c fiber.Ctx) error {
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
return writeError(c, statusFor(err), err)
}
if api.browser == nil || api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies) != nil {
if api.browser == nil {
return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
}
if err := api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies); err != nil {
logrus.WithError(err).WithField("browser_id", c.Params("id")).Error("restricted browser cookie operation failed")
return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
}
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
@@ -99,6 +106,7 @@ func (api gateway) getDouyin(c fiber.Ctx) error {
}
response, err := api.browser.Get(c.Context(), c.Params("id"), input.URL)
if err != nil {
logrus.WithError(err).WithField("browser_id", c.Params("id")).Error("restricted browser fetch operation failed")
return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
}
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
@@ -174,10 +182,22 @@ func validDouyinURL(raw string) bool {
query := parsed.Query()
switch parsed.Path {
case douyinIdentityPath:
return parsed.RawQuery == ""
return len(query) == 2 && len(query["aid"]) == 1 && query.Get("aid") == "6383" &&
len(query["device_platform"]) == 1 && query.Get("device_platform") == "webapp"
case douyinWorksPath:
return len(query) == 3 && len(query["sec_user_id"]) == 1 && douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) &&
len(query["count"]) == 1 && query.Get("count") == "20" && len(query["max_cursor"]) == 1 && query.Get("max_cursor") == "0"
if len(query) != 3 || len(query["sec_user_id"]) != 1 || !douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) ||
len(query["count"]) != 1 || query.Get("count") != "20" || len(query["max_cursor"]) != 1 {
return false
}
cursor, err := strconv.ParseInt(query.Get("max_cursor"), 10, 64)
return err == nil && cursor >= 0
case douyinCommentsPath:
if len(query) != 3 || len(query["aweme_id"]) != 1 || !douyinAccountKeyPattern.MatchString(query.Get("aweme_id")) ||
len(query["count"]) != 1 || query.Get("count") != "20" || len(query["cursor"]) != 1 {
return false
}
cursor, err := strconv.ParseInt(query.Get("cursor"), 10, 64)
return err == nil && cursor >= 0
default:
return false
}
@@ -227,11 +247,15 @@ func (browser cdpBrowser) SetCookies(ctx context.Context, alias string, cookies
LoaderID string `json:"loaderId"`
ErrorText string `json:"errorText"`
}
if err := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events); err != nil ||
navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" {
navigationErr := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events)
logrus.WithFields(logrus.Fields{"frame_id": navigation.FrameID, "loader_id": navigation.LoaderID,
"error_text": navigation.ErrorText, "event_count": len(events), "command_error": navigationErr != nil}).Debug("restricted browser navigation response")
if navigationErr != nil || navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" {
logrus.WithFields(logrus.Fields{"frame_id": navigation.FrameID, "loader_id": navigation.LoaderID,
"error_text": navigation.ErrorText, "event_count": len(events)}).Error("restricted browser navigation response invalid")
return errors.New("restricted browser navigation failed")
}
if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, navigation.LoaderID, events); err != nil {
if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, events); err != nil {
return err
}
return cdpCommand(connection, &commandID, "Network.setCookies", map[string]any{"cookies": cdpCookies}, nil, nil)
@@ -286,6 +310,18 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket
if browser.endpoint != nil {
base = browser.endpoint(alias)
}
baseURL, err := url.Parse(base)
if err != nil || baseURL.Scheme != "http" || baseURL.Hostname() == "" || baseURL.Port() == "" {
return nil, errors.New("restricted browser unavailable")
}
if net.ParseIP(baseURL.Hostname()) == nil {
addresses, lookupErr := net.LookupIP(baseURL.Hostname())
if lookupErr != nil || len(addresses) == 0 {
return nil, errors.New("restricted browser unavailable")
}
baseURL.Host = net.JoinHostPort(addresses[0].String(), baseURL.Port())
}
base = strings.TrimRight(baseURL.String(), "/")
request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/json/list", nil)
if err != nil {
return nil, errors.New("restricted browser unavailable")
@@ -300,6 +336,7 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
response, err := client.Do(request)
if err != nil {
logrus.WithError(err).WithField("browser_id", alias).Error("restricted browser discovery failed")
return nil, errors.New("restricted browser unavailable")
}
defer response.Body.Close()
@@ -322,7 +359,6 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return nil, errors.New("restricted browser unavailable")
}
baseURL, _ := url.Parse(base)
pageTarget := ""
for _, target := range targets {
if target.Type != "page" {
@@ -352,6 +388,7 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket
}
connection, err := config.DialContext(ctx)
if err != nil {
logrus.WithError(err).WithField("browser_id", alias).Error("restricted browser websocket failed")
return nil, errors.New("restricted browser unavailable")
}
deadline := time.Now().Add(browserControlTimeout)
@@ -359,6 +396,7 @@ func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket
deadline = contextDeadline
}
_ = connection.SetDeadline(deadline)
logrus.WithFields(logrus.Fields{"browser_id": alias, "cdp_endpoint": pageTarget}).Debug("restricted browser websocket connected")
return connection, nil
}
@@ -373,11 +411,13 @@ type cdpMessage struct {
func cdpCommand(connection *websocket.Conn, commandID *int, method string, parameters any, output any, events *[]cdpMessage) error {
*commandID = *commandID + 1
if err := websocket.JSON.Send(connection, map[string]any{"id": *commandID, "method": method, "params": parameters}); err != nil {
logrus.WithError(err).WithField("cdp_method", method).Error("restricted browser command send failed")
return errors.New("restricted browser command failed")
}
for range 128 {
var reply cdpMessage
if err := websocket.JSON.Receive(connection, &reply); err != nil {
logrus.WithError(err).WithField("cdp_method", method).Error("restricted browser command receive failed")
return errors.New("restricted browser command failed")
}
if reply.ID != *commandID {
@@ -387,6 +427,7 @@ func cdpCommand(connection *websocket.Conn, commandID *int, method string, param
continue
}
if len(reply.Error) != 0 || len(reply.Result) == 0 || bytes.Equal(bytes.TrimSpace(reply.Result), []byte("null")) {
logrus.WithFields(logrus.Fields{"cdp_method": method, "cdp_error": string(reply.Error), "has_result": len(reply.Result) != 0}).Error("restricted browser command returned failure")
return errors.New("restricted browser command failed")
}
if output != nil && json.Unmarshal(reply.Result, output) != nil {
@@ -397,17 +438,32 @@ func cdpCommand(connection *websocket.Conn, commandID *int, method string, param
return errors.New("restricted browser command failed")
}
func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID, loaderID string, events []cdpMessage) error {
func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID string, events []cdpMessage) error {
deadline := time.Now().Add(10 * time.Second)
if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
deadline = contextDeadline
}
_ = connection.SetReadDeadline(deadline)
for attempts := 0; attempts < 256; attempts++ {
bufferedLifecycle := map[string]bool{}
for _, event := range events {
if event.Method != "Page.lifecycleEvent" {
continue
}
var lifecycle struct {
FrameID string `json:"frameId"`
LoaderID string `json:"loaderId"`
Name string `json:"name"`
}
if json.Unmarshal(event.Params, &lifecycle) == nil && lifecycle.FrameID == frameID && (lifecycle.Name == "DOMContentLoaded" || lifecycle.Name == "load") {
bufferedLifecycle[lifecycle.LoaderID+":"+lifecycle.Name] = true
}
}
for {
var event cdpMessage
if len(events) != 0 {
event, events = events[0], events[1:]
} else if websocket.JSON.Receive(connection, &event) != nil {
} else if err := websocket.JSON.Receive(connection, &event); err != nil {
logrus.WithError(err).Error("restricted browser lifecycle receive failed")
return errors.New("restricted browser navigation failed")
}
if event.Method != "Page.lifecycleEvent" {
@@ -418,7 +474,11 @@ func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandI
LoaderID string `json:"loaderId"`
Name string `json:"name"`
}
if json.Unmarshal(event.Params, &lifecycle) != nil || lifecycle.FrameID != frameID || lifecycle.LoaderID != loaderID || lifecycle.Name != "load" {
if err := json.Unmarshal(event.Params, &lifecycle); err != nil {
logrus.WithError(err).Debug("restricted browser lifecycle decode failed")
continue
}
if lifecycle.FrameID != frameID || lifecycle.Name != "DOMContentLoaded" && lifecycle.Name != "load" || bufferedLifecycle[lifecycle.LoaderID+":"+lifecycle.Name] {
continue
}
var evaluated struct {
@@ -429,11 +489,12 @@ func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandI
if err := cdpCommand(connection, commandID, "Runtime.evaluate", map[string]any{
"expression": "location.origin", "returnByValue": true,
}, &evaluated, nil); err != nil || evaluated.Result.Value != douyinOrigin {
logrus.WithFields(logrus.Fields{"frame_id": lifecycle.FrameID, "loader_id": lifecycle.LoaderID,
"origin": evaluated.Result.Value}).Error("restricted browser navigation origin failed")
return errors.New("restricted browser navigation failed")
}
return nil
}
return errors.New("restricted browser navigation failed")
}
func detectDouyinChallenge(status int, body string) douyin.Challenge {
cmd/docker-gateway/douyin_test.go
+9 -7
View File
@@ -16,6 +16,8 @@ import (
"golang.org/x/net/websocket"
)
const douyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/?aid=6383&device_platform=webapp"
type fakeRestrictedBrowser struct {
cookies []douyin.Cookie
urls []string
@@ -90,7 +92,7 @@ func TestGatewayRestrictedDouyinContract(t *testing.T) {
}
response.Body.Close()
identityURL := "https://www.douyin.com" + douyinIdentityPath
identityURL := douyinIdentityURL
getBody := `{` + generation + `,"url":"` + identityURL + `"}`
response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody)))
body, _ := io.ReadAll(response.Body)
@@ -222,7 +224,7 @@ func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) {
}})
if !onlyOldLoader {
afterReply = append(afterReply, map[string]any{"method": "Page.lifecycleEvent", "params": map[string]any{
"frameId": "frame-new", "loaderId": "loader-new", "name": "load",
"frameId": "frame-new", "loaderId": "loader-final", "name": "load",
}})
}
case "Runtime.evaluate":
@@ -269,7 +271,7 @@ func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) {
}}); err != nil {
t.Fatal(err)
}
result, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath)
result, err := browser.Get(context.Background(), "account-a", douyinIdentityURL)
if err != nil || result.Status != 412 || result.Challenge != douyin.ChallengeCaptcha {
t.Fatalf("unexpected CDP response: %#v err=%v", result, err)
}
@@ -297,13 +299,13 @@ func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) {
}
pageOrigin, fetchMode = douyinOrigin, "redirect"
mu.Unlock()
if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil {
if _, err := browser.Get(context.Background(), "account-a", douyinIdentityURL); err == nil {
t.Fatal("accepted a redirected fetch")
}
mu.Lock()
fetchMode = "too_large"
mu.Unlock()
if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil {
if _, err := browser.Get(context.Background(), "account-a", douyinIdentityURL); err == nil {
t.Fatal("accepted a response at the 1 MiB limit")
}
mu.Lock()
@@ -395,9 +397,9 @@ func TestCDPDiscoveryRequiresOneJSONValue(t *testing.T) {
func TestDouyinURLContract(t *testing.T) {
for target, want := range map[string]bool{
"https://www.douyin.com" + douyinIdentityPath: true,
douyinIdentityURL: true,
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0": true,
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=1": false,
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=1": true,
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0&method=publish": false,
"https://www.douyin.com/aweme/v1/web/commit/item/": false,
} {
cmd/docker-gateway/main.go
+5
View File
@@ -919,6 +919,11 @@ func (api gateway) requireGeneration(id string, input generationRequest) (string
}
version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64)
if input.RuntimeID == "" || runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkIDLabel] != input.NetworkID {
logrus.WithFields(logrus.Fields{
"browser_id": id, "input_runtime_id": input.RuntimeID, "actual_runtime_id": runtimeID,
"input_binding_version": input.BindingVersion, "actual_binding_version": version,
"input_network_id": input.NetworkID, "actual_network_id": labels[networkIDLabel],
}).Warn("browser generation mismatch")
return "", false, errGenerationConflict
}
return runtimeID, true, nil