package main import ( "bytes" "context" "encoding/json" "errors" "io" "net/http" "net/url" "regexp" "strconv" "strings" "time" "git.ipao.vip/rogee/creator-hub/internal/douyin" "github.com/gofiber/fiber/v3" "golang.org/x/net/websocket" ) const ( douyinOrigin = "https://www.douyin.com" douyinOriginURL = "https://www.douyin.com/" douyinIdentityPath = "/aweme/v1/web/user/profile/self/" douyinWorksPath = "/aweme/v1/web/aweme/post/" douyinResponseLimit = 1 << 20 browserControlTimeout = 15 * time.Second ) var douyinAccountKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$`) type restrictedBrowserResponse struct { Status int Body string Challenge douyin.Challenge } type restrictedBrowser interface { SetCookies(context.Context, string, []douyin.Cookie) error Get(context.Context, string, string) (restrictedBrowserResponse, error) } type douyinGenerationRequest struct { BindingVersion int64 `json:"binding_version"` RuntimeID string `json:"runtime_id"` NetworkID string `json:"network_id"` NetworkExitID string `json:"network_exit_id"` } type douyinCookieRequest struct { douyinGenerationRequest Cookies []douyin.Cookie `json:"cookies"` } type douyinGetRequest struct { douyinGenerationRequest URL string `json:"url"` } func (api gateway) setDouyinCookies(c fiber.Ctx) error { var input douyinCookieRequest if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) || !validDouyinCookies(input.Cookies) { return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request")) } _, release, err := api.locks.acquire(c.Params("id")) if err != nil { return writeError(c, statusFor(err), err) } defer release() if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { return writeError(c, statusFor(err), err) } if api.browser == nil || api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies) != nil { return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) } if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { return writeError(c, statusFor(err), err) } return c.SendStatus(http.StatusNoContent) } func (api gateway) getDouyin(c fiber.Ctx) error { var input douyinGetRequest if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) || !validDouyinURL(input.URL) { return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request")) } _, release, err := api.locks.acquire(c.Params("id")) if err != nil { return writeError(c, statusFor(err), err) } defer release() if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { return writeError(c, statusFor(err), err) } if api.browser == nil { return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) } response, err := api.browser.Get(c.Context(), c.Params("id"), input.URL) if err != nil { return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) } if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { return writeError(c, statusFor(err), err) } return writeJSON(c, http.StatusOK, map[string]any{ "status": response.Status, "body": response.Body, "challenge": response.Challenge, }) } func decodeRestrictedBrowserRequest(body []byte, target any) error { decoder := json.NewDecoder(bytes.NewReader(body)) decoder.DisallowUnknownFields() if err := decoder.Decode(target); err != nil { return err } var trailing json.RawMessage if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return errors.New("request must contain one JSON value") } return nil } func validDouyinGeneration(input douyinGenerationRequest) bool { return input.BindingVersion > 0 && exitIDPattern.MatchString(input.RuntimeID) && exitIDPattern.MatchString(input.NetworkID) && (input.NetworkExitID == "" || exitIDPattern.MatchString(input.NetworkExitID)) } func (api gateway) requireDouyinGeneration(alias string, input douyinGenerationRequest) error { runtimeID, labels, networks, err := api.managedContainerState(alias) if err != nil { return err } version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) if runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkIDLabel] != input.NetworkID || labels[networkExitLabel] != input.NetworkExitID { return errGenerationConflict } generation, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, input.BindingVersion, input.RuntimeID, api.self, input.NetworkID, false) if err != nil { return err } if !exists || !generation.RuntimeAttached || generation.SelfMember == "" || len(generation.GatewayMembers) == 0 || len(networks) != 1 || networks[generation.Name] != generation.ID { return errGenerationConflict } return nil } func validDouyinCookies(cookies []douyin.Cookie) bool { if len(cookies) == 0 || len(cookies) > 64 { return false } for _, cookie := range cookies { domain := strings.ToLower(strings.TrimSpace(cookie.Domain)) if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 || domain != cookie.Domain || cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") || (domain != "douyin.com" && !strings.HasSuffix(domain, ".douyin.com")) || !strings.HasPrefix(cookie.Path, "/") || strings.ContainsAny(cookie.Path, ";\r\n\x00") || (cookie.SameSite != "" && cookie.SameSite != "Lax" && cookie.SameSite != "Strict" && cookie.SameSite != "None") { return false } } return true } func validDouyinURL(raw string) bool { parsed, err := url.Parse(raw) if err != nil || parsed.Scheme != "https" || parsed.Host != "www.douyin.com" || parsed.User != nil || parsed.Fragment != "" { return false } query := parsed.Query() switch parsed.Path { case douyinIdentityPath: return parsed.RawQuery == "" case douyinWorksPath: return len(query) == 3 && len(query["sec_user_id"]) == 1 && douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) && len(query["count"]) == 1 && query.Get("count") == "20" && len(query["max_cursor"]) == 1 && query.Get("max_cursor") == "0" default: return false } } type cdpBrowser struct { endpoint func(string) string client *http.Client } func (browser cdpBrowser) SetCookies(ctx context.Context, alias string, cookies []douyin.Cookie) error { connection, err := browser.connect(ctx, alias) if err != nil { return err } defer connection.Close() commandID := 0 cdpCookies := make([]map[string]any, 0, len(cookies)) for _, cookie := range cookies { value := map[string]any{ "name": cookie.Name, "value": cookie.Value, "url": douyinOriginURL, "path": cookie.Path, "secure": cookie.Secure, "httpOnly": cookie.HTTPOnly, } if cookie.SameSite != "" { value["sameSite"] = cookie.SameSite } if cookie.Expires != 0 { value["expires"] = cookie.Expires } cdpCookies = append(cdpCookies, value) } if err := cdpCommand(connection, &commandID, "Network.enable", map[string]any{}, nil, nil); err != nil { return err } if err := cdpCommand(connection, &commandID, "Network.clearBrowserCookies", map[string]any{}, nil, nil); err != nil { return err } if err := cdpCommand(connection, &commandID, "Page.enable", map[string]any{}, nil, nil); err != nil { return err } if err := cdpCommand(connection, &commandID, "Page.setLifecycleEventsEnabled", map[string]bool{"enabled": true}, nil, nil); err != nil { return err } var events []cdpMessage var navigation struct { FrameID string `json:"frameId"` LoaderID string `json:"loaderId"` ErrorText string `json:"errorText"` } if err := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events); err != nil || navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" { return errors.New("restricted browser navigation failed") } if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, navigation.LoaderID, events); err != nil { return err } return cdpCommand(connection, &commandID, "Network.setCookies", map[string]any{"cookies": cdpCookies}, nil, nil) } func (browser cdpBrowser) Get(ctx context.Context, alias, target string) (restrictedBrowserResponse, error) { connection, err := browser.connect(ctx, alias) if err != nil { return restrictedBrowserResponse{}, err } defer connection.Close() commandID := 0 var currentOrigin struct { Result struct { Value string `json:"value"` } `json:"result"` } if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{ "expression": "location.origin", "returnByValue": true, }, ¤tOrigin, nil); err != nil || currentOrigin.Result.Value != douyinOrigin { return restrictedBrowserResponse{}, errors.New("restricted browser origin changed") } encodedURL, _ := json.Marshal(target) expression := `(async()=>{const r=await fetch(` + string(encodedURL) + `,{credentials:"include",redirect:"error"});` + `if(!r.body)return {status:r.status,body:"",too_large:false};const q=r.body.getReader(),d=new TextDecoder();let n=0,b="";` + `for(;;){const x=await q.read();if(x.done)break;if(n+x.value.byteLength>=` + strconv.Itoa(douyinResponseLimit) + `){await q.cancel();return {too_large:true};}n+=x.value.byteLength;b+=d.decode(x.value,{stream:true});}` + `b+=d.decode();return {status:r.status,body:b,too_large:false};})()` var evaluated struct { Result struct { Value struct { Status int `json:"status"` Body string `json:"body"` TooLarge bool `json:"too_large"` } `json:"value"` } `json:"result"` ExceptionDetails json.RawMessage `json:"exceptionDetails"` } if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{ "expression": expression, "awaitPromise": true, "returnByValue": true, }, &evaluated, nil); err != nil || len(evaluated.ExceptionDetails) != 0 || evaluated.Result.Value.TooLarge || evaluated.Result.Value.Status < 200 || evaluated.Result.Value.Status > 599 || (evaluated.Result.Value.Status >= 300 && evaluated.Result.Value.Status < 400) { return restrictedBrowserResponse{}, errors.New("restricted browser fetch failed") } body := evaluated.Result.Value.Body return restrictedBrowserResponse{Status: evaluated.Result.Value.Status, Body: body, Challenge: detectDouyinChallenge(evaluated.Result.Value.Status, body)}, nil } func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket.Conn, error) { base := "http://" + namePrefix + alias + ":9222" if browser.endpoint != nil { base = browser.endpoint(alias) } request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/json/list", nil) if err != nil { return nil, errors.New("restricted browser unavailable") } client := http.Client{Timeout: browserControlTimeout} if browser.client != nil { client = *browser.client } if client.Timeout <= 0 { client.Timeout = browserControlTimeout } client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } response, err := client.Do(request) if err != nil { return nil, errors.New("restricted browser unavailable") } defer response.Body.Close() var targets []struct { Type string `json:"type"` WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` } if response.StatusCode != http.StatusOK { return nil, errors.New("restricted browser unavailable") } body, err := io.ReadAll(io.LimitReader(response.Body, (64<<10)+1)) if err != nil || len(body) > 64<<10 { return nil, errors.New("restricted browser unavailable") } decoder := json.NewDecoder(bytes.NewReader(body)) if decoder.Decode(&targets) != nil || len(targets) > 32 { return nil, errors.New("restricted browser unavailable") } var trailing json.RawMessage if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { return nil, errors.New("restricted browser unavailable") } baseURL, _ := url.Parse(base) pageTarget := "" for _, target := range targets { if target.Type != "page" { continue } if pageTarget != "" { return nil, errors.New("restricted browser unavailable") } websocketURL, parseErr := url.Parse(target.WebSocketDebuggerURL) if parseErr != nil || websocketURL.Scheme != "ws" || !strings.HasPrefix(websocketURL.Path, "/devtools/page/") { return nil, errors.New("restricted browser unavailable") } if websocketURL.Hostname() == "localhost" || websocketURL.Hostname() == "127.0.0.1" || websocketURL.Hostname() == "::1" { websocketURL.Host = baseURL.Host } if websocketURL.Host != baseURL.Host { return nil, errors.New("restricted browser unavailable") } pageTarget = websocketURL.String() } if pageTarget == "" { return nil, errors.New("restricted browser unavailable") } config, err := websocket.NewConfig(pageTarget, "devtools://devtools") if err != nil { return nil, errors.New("restricted browser unavailable") } connection, err := config.DialContext(ctx) if err != nil { return nil, errors.New("restricted browser unavailable") } deadline := time.Now().Add(browserControlTimeout) if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { deadline = contextDeadline } _ = connection.SetDeadline(deadline) return connection, nil } type cdpMessage struct { ID int `json:"id"` Method string `json:"method"` Params json.RawMessage `json:"params"` Result json.RawMessage `json:"result"` Error json.RawMessage `json:"error"` } func cdpCommand(connection *websocket.Conn, commandID *int, method string, parameters any, output any, events *[]cdpMessage) error { *commandID = *commandID + 1 if err := websocket.JSON.Send(connection, map[string]any{"id": *commandID, "method": method, "params": parameters}); err != nil { return errors.New("restricted browser command failed") } for range 128 { var reply cdpMessage if err := websocket.JSON.Receive(connection, &reply); err != nil { return errors.New("restricted browser command failed") } if reply.ID != *commandID { if events != nil && reply.ID == 0 && reply.Method != "" { *events = append(*events, reply) } continue } if len(reply.Error) != 0 || len(reply.Result) == 0 || bytes.Equal(bytes.TrimSpace(reply.Result), []byte("null")) { return errors.New("restricted browser command failed") } if output != nil && json.Unmarshal(reply.Result, output) != nil { return errors.New("restricted browser command failed") } return nil } return errors.New("restricted browser command failed") } func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID, loaderID string, events []cdpMessage) error { deadline := time.Now().Add(10 * time.Second) if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { deadline = contextDeadline } _ = connection.SetReadDeadline(deadline) for attempts := 0; attempts < 256; attempts++ { var event cdpMessage if len(events) != 0 { event, events = events[0], events[1:] } else if websocket.JSON.Receive(connection, &event) != nil { return errors.New("restricted browser navigation failed") } if event.Method != "Page.lifecycleEvent" { continue } var lifecycle struct { FrameID string `json:"frameId"` LoaderID string `json:"loaderId"` Name string `json:"name"` } if json.Unmarshal(event.Params, &lifecycle) != nil || lifecycle.FrameID != frameID || lifecycle.LoaderID != loaderID || lifecycle.Name != "load" { continue } var evaluated struct { Result struct { Value string `json:"value"` } `json:"result"` } if err := cdpCommand(connection, commandID, "Runtime.evaluate", map[string]any{ "expression": "location.origin", "returnByValue": true, }, &evaluated, nil); err != nil || evaluated.Result.Value != douyinOrigin { return errors.New("restricted browser navigation failed") } return nil } return errors.New("restricted browser navigation failed") } func detectDouyinChallenge(status int, body string) douyin.Challenge { lower := strings.ToLower(body) if status == http.StatusPreconditionFailed || strings.Contains(lower, "captcha") || strings.Contains(lower, "verify_center_decision_conf") { return douyin.ChallengeCaptcha } if strings.Contains(lower, "device_challenge") || strings.Contains(lower, "device verification") { return douyin.ChallengeDevice } return douyin.ChallengeNone }