creator-hub/cmd/docker-gateway/douyin.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"git.ipao.vip/rogee/creator-hub/internal/douyin"
	"github.com/gofiber/fiber/v3"
	"golang.org/x/net/websocket"
)

const (
	douyinOrigin          = "https://www.douyin.com"
	douyinOriginURL       = "https://www.douyin.com/"
	douyinIdentityPath    = "/aweme/v1/web/user/profile/self/"
	douyinWorksPath       = "/aweme/v1/web/aweme/post/"
	douyinResponseLimit   = 1 << 20
	browserControlTimeout = 15 * time.Second
)

var douyinAccountKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$`)

type restrictedBrowserResponse struct {
	Status    int
	Body      string
	Challenge douyin.Challenge
}

type restrictedBrowser interface {
	SetCookies(context.Context, string, []douyin.Cookie) error
	Get(context.Context, string, string) (restrictedBrowserResponse, error)
}

type douyinGenerationRequest struct {
	BindingVersion int64  `json:"binding_version"`
	RuntimeID      string `json:"runtime_id"`
	NetworkID      string `json:"network_id"`
	NetworkExitID  string `json:"network_exit_id"`
}

type douyinCookieRequest struct {
	douyinGenerationRequest
	Cookies []douyin.Cookie `json:"cookies"`
}

type douyinGetRequest struct {
	douyinGenerationRequest
	URL string `json:"url"`
}

func (api gateway) setDouyinCookies(c fiber.Ctx) error {
	var input douyinCookieRequest
	if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) ||
		!validDouyinCookies(input.Cookies) {
		return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request"))
	}
	_, release, err := api.locks.acquire(c.Params("id"))
	if err != nil {
		return writeError(c, statusFor(err), err)
	}
	defer release()
	if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
		return writeError(c, statusFor(err), err)
	}
	if api.browser == nil || api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies) != nil {
		return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
	}
	if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
		return writeError(c, statusFor(err), err)
	}
	return c.SendStatus(http.StatusNoContent)
}

func (api gateway) getDouyin(c fiber.Ctx) error {
	var input douyinGetRequest
	if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) ||
		!validDouyinURL(input.URL) {
		return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request"))
	}
	_, release, err := api.locks.acquire(c.Params("id"))
	if err != nil {
		return writeError(c, statusFor(err), err)
	}
	defer release()
	if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
		return writeError(c, statusFor(err), err)
	}
	if api.browser == nil {
		return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
	}
	response, err := api.browser.Get(c.Context(), c.Params("id"), input.URL)
	if err != nil {
		return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
	}
	if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
		return writeError(c, statusFor(err), err)
	}
	return writeJSON(c, http.StatusOK, map[string]any{
		"status": response.Status, "body": response.Body, "challenge": response.Challenge,
	})
}

func decodeRestrictedBrowserRequest(body []byte, target any) error {
	decoder := json.NewDecoder(bytes.NewReader(body))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(target); err != nil {
		return err
	}
	var trailing json.RawMessage
	if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
		return errors.New("request must contain one JSON value")
	}
	return nil
}

func validDouyinGeneration(input douyinGenerationRequest) bool {
	return input.BindingVersion > 0 && exitIDPattern.MatchString(input.RuntimeID) &&
		exitIDPattern.MatchString(input.NetworkID) && exitIDPattern.MatchString(input.NetworkExitID)
}

func (api gateway) requireDouyinGeneration(alias string, input douyinGenerationRequest) error {
	runtimeID, labels, networks, err := api.managedContainerState(alias)
	if err != nil {
		return err
	}
	version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64)
	if runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkIDLabel] != input.NetworkID ||
		labels[networkExitLabel] != input.NetworkExitID {
		return errGenerationConflict
	}
	generation, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, input.BindingVersion,
		input.RuntimeID, api.self, input.NetworkID, false)
	if err != nil {
		return err
	}
	if !exists || !generation.RuntimeAttached || generation.SelfMember == "" || len(generation.GatewayMembers) == 0 ||
		len(networks) != 1 || networks[generation.Name] != generation.ID {
		return errGenerationConflict
	}
	return nil
}

func validDouyinCookies(cookies []douyin.Cookie) bool {
	if len(cookies) == 0 || len(cookies) > 64 {
		return false
	}
	for _, cookie := range cookies {
		domain := strings.ToLower(strings.TrimSpace(cookie.Domain))
		if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 ||
			domain != cookie.Domain || cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") ||
			(domain != "douyin.com" && !strings.HasSuffix(domain, ".douyin.com")) || !strings.HasPrefix(cookie.Path, "/") ||
			strings.ContainsAny(cookie.Path, ";\r\n\x00") || (cookie.SameSite != "" && cookie.SameSite != "Lax" &&
			cookie.SameSite != "Strict" && cookie.SameSite != "None") {
			return false
		}
	}
	return true
}

func validDouyinURL(raw string) bool {
	parsed, err := url.Parse(raw)
	if err != nil || parsed.Scheme != "https" || parsed.Host != "www.douyin.com" || parsed.User != nil || parsed.Fragment != "" {
		return false
	}
	query := parsed.Query()
	switch parsed.Path {
	case douyinIdentityPath:
		return parsed.RawQuery == ""
	case douyinWorksPath:
		return len(query) == 3 && len(query["sec_user_id"]) == 1 && douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) &&
			len(query["count"]) == 1 && query.Get("count") == "20" && len(query["max_cursor"]) == 1 && query.Get("max_cursor") == "0"
	default:
		return false
	}
}

type cdpBrowser struct {
	endpoint func(string) string
	client   *http.Client
}

func (browser cdpBrowser) SetCookies(ctx context.Context, alias string, cookies []douyin.Cookie) error {
	connection, err := browser.connect(ctx, alias)
	if err != nil {
		return err
	}
	defer connection.Close()
	commandID := 0
	cdpCookies := make([]map[string]any, 0, len(cookies))
	for _, cookie := range cookies {
		value := map[string]any{
			"name": cookie.Name, "value": cookie.Value, "url": douyinOriginURL, "path": cookie.Path,
			"secure": cookie.Secure, "httpOnly": cookie.HTTPOnly,
		}
		if cookie.SameSite != "" {
			value["sameSite"] = cookie.SameSite
		}
		if cookie.Expires != 0 {
			value["expires"] = cookie.Expires
		}
		cdpCookies = append(cdpCookies, value)
	}
	if err := cdpCommand(connection, &commandID, "Network.enable", map[string]any{}, nil, nil); err != nil {
		return err
	}
	if err := cdpCommand(connection, &commandID, "Network.clearBrowserCookies", map[string]any{}, nil, nil); err != nil {
		return err
	}
	if err := cdpCommand(connection, &commandID, "Page.enable", map[string]any{}, nil, nil); err != nil {
		return err
	}
	if err := cdpCommand(connection, &commandID, "Page.setLifecycleEventsEnabled", map[string]bool{"enabled": true}, nil, nil); err != nil {
		return err
	}
	var events []cdpMessage
	var navigation struct {
		FrameID   string `json:"frameId"`
		LoaderID  string `json:"loaderId"`
		ErrorText string `json:"errorText"`
	}
	if err := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events); err != nil ||
		navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" {
		return errors.New("restricted browser navigation failed")
	}
	if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, navigation.LoaderID, events); err != nil {
		return err
	}
	return cdpCommand(connection, &commandID, "Network.setCookies", map[string]any{"cookies": cdpCookies}, nil, nil)
}

func (browser cdpBrowser) Get(ctx context.Context, alias, target string) (restrictedBrowserResponse, error) {
	connection, err := browser.connect(ctx, alias)
	if err != nil {
		return restrictedBrowserResponse{}, err
	}
	defer connection.Close()
	commandID := 0
	var currentOrigin struct {
		Result struct {
			Value string `json:"value"`
		} `json:"result"`
	}
	if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{
		"expression": "location.origin", "returnByValue": true,
	}, &currentOrigin, nil); err != nil || currentOrigin.Result.Value != douyinOrigin {
		return restrictedBrowserResponse{}, errors.New("restricted browser origin changed")
	}
	encodedURL, _ := json.Marshal(target)
	expression := `(async()=>{const r=await fetch(` + string(encodedURL) + `,{credentials:"include",redirect:"error"});` +
		`if(!r.body)return {status:r.status,body:"",too_large:false};const q=r.body.getReader(),d=new TextDecoder();let n=0,b="";` +
		`for(;;){const x=await q.read();if(x.done)break;if(n+x.value.byteLength>=` + strconv.Itoa(douyinResponseLimit) +
		`){await q.cancel();return {too_large:true};}n+=x.value.byteLength;b+=d.decode(x.value,{stream:true});}` +
		`b+=d.decode();return {status:r.status,body:b,too_large:false};})()`
	var evaluated struct {
		Result struct {
			Value struct {
				Status   int    `json:"status"`
				Body     string `json:"body"`
				TooLarge bool   `json:"too_large"`
			} `json:"value"`
		} `json:"result"`
		ExceptionDetails json.RawMessage `json:"exceptionDetails"`
	}
	if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{
		"expression": expression, "awaitPromise": true, "returnByValue": true,
	}, &evaluated, nil); err != nil || len(evaluated.ExceptionDetails) != 0 || evaluated.Result.Value.TooLarge ||
		evaluated.Result.Value.Status < 200 || evaluated.Result.Value.Status > 599 ||
		(evaluated.Result.Value.Status >= 300 && evaluated.Result.Value.Status < 400) {
		return restrictedBrowserResponse{}, errors.New("restricted browser fetch failed")
	}
	body := evaluated.Result.Value.Body
	return restrictedBrowserResponse{Status: evaluated.Result.Value.Status, Body: body, Challenge: detectDouyinChallenge(evaluated.Result.Value.Status, body)}, nil
}

func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket.Conn, error) {
	base := "http://" + namePrefix + alias + ":9222"
	if browser.endpoint != nil {
		base = browser.endpoint(alias)
	}
	request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/json/list", nil)
	if err != nil {
		return nil, errors.New("restricted browser unavailable")
	}
	client := http.Client{Timeout: browserControlTimeout}
	if browser.client != nil {
		client = *browser.client
	}
	if client.Timeout <= 0 {
		client.Timeout = browserControlTimeout
	}
	client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
	response, err := client.Do(request)
	if err != nil {
		return nil, errors.New("restricted browser unavailable")
	}
	defer response.Body.Close()
	var targets []struct {
		Type                 string `json:"type"`
		WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
	}
	if response.StatusCode != http.StatusOK {
		return nil, errors.New("restricted browser unavailable")
	}
	body, err := io.ReadAll(io.LimitReader(response.Body, (64<<10)+1))
	if err != nil || len(body) > 64<<10 {
		return nil, errors.New("restricted browser unavailable")
	}
	decoder := json.NewDecoder(bytes.NewReader(body))
	if decoder.Decode(&targets) != nil || len(targets) > 32 {
		return nil, errors.New("restricted browser unavailable")
	}
	var trailing json.RawMessage
	if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
		return nil, errors.New("restricted browser unavailable")
	}
	baseURL, _ := url.Parse(base)
	pageTarget := ""
	for _, target := range targets {
		if target.Type != "page" {
			continue
		}
		if pageTarget != "" {
			return nil, errors.New("restricted browser unavailable")
		}
		websocketURL, parseErr := url.Parse(target.WebSocketDebuggerURL)
		if parseErr != nil || websocketURL.Scheme != "ws" || !strings.HasPrefix(websocketURL.Path, "/devtools/page/") {
			return nil, errors.New("restricted browser unavailable")
		}
		if websocketURL.Hostname() == "localhost" || websocketURL.Hostname() == "127.0.0.1" || websocketURL.Hostname() == "::1" {
			websocketURL.Host = baseURL.Host
		}
		if websocketURL.Host != baseURL.Host {
			return nil, errors.New("restricted browser unavailable")
		}
		pageTarget = websocketURL.String()
	}
	if pageTarget == "" {
		return nil, errors.New("restricted browser unavailable")
	}
	config, err := websocket.NewConfig(pageTarget, "devtools://devtools")
	if err != nil {
		return nil, errors.New("restricted browser unavailable")
	}
	connection, err := config.DialContext(ctx)
	if err != nil {
		return nil, errors.New("restricted browser unavailable")
	}
	deadline := time.Now().Add(browserControlTimeout)
	if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
		deadline = contextDeadline
	}
	_ = connection.SetDeadline(deadline)
	return connection, nil
}

type cdpMessage struct {
	ID     int             `json:"id"`
	Method string          `json:"method"`
	Params json.RawMessage `json:"params"`
	Result json.RawMessage `json:"result"`
	Error  json.RawMessage `json:"error"`
}

func cdpCommand(connection *websocket.Conn, commandID *int, method string, parameters any, output any, events *[]cdpMessage) error {
	*commandID = *commandID + 1
	if err := websocket.JSON.Send(connection, map[string]any{"id": *commandID, "method": method, "params": parameters}); err != nil {
		return errors.New("restricted browser command failed")
	}
	for range 128 {
		var reply cdpMessage
		if err := websocket.JSON.Receive(connection, &reply); err != nil {
			return errors.New("restricted browser command failed")
		}
		if reply.ID != *commandID {
			if events != nil && reply.ID == 0 && reply.Method != "" {
				*events = append(*events, reply)
			}
			continue
		}
		if len(reply.Error) != 0 || len(reply.Result) == 0 || bytes.Equal(bytes.TrimSpace(reply.Result), []byte("null")) {
			return errors.New("restricted browser command failed")
		}
		if output != nil && json.Unmarshal(reply.Result, output) != nil {
			return errors.New("restricted browser command failed")
		}
		return nil
	}
	return errors.New("restricted browser command failed")
}

func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID, loaderID string, events []cdpMessage) error {
	deadline := time.Now().Add(10 * time.Second)
	if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
		deadline = contextDeadline
	}
	_ = connection.SetReadDeadline(deadline)
	for attempts := 0; attempts < 256; attempts++ {
		var event cdpMessage
		if len(events) != 0 {
			event, events = events[0], events[1:]
		} else if websocket.JSON.Receive(connection, &event) != nil {
			return errors.New("restricted browser navigation failed")
		}
		if event.Method != "Page.lifecycleEvent" {
			continue
		}
		var lifecycle struct {
			FrameID  string `json:"frameId"`
			LoaderID string `json:"loaderId"`
			Name     string `json:"name"`
		}
		if json.Unmarshal(event.Params, &lifecycle) != nil || lifecycle.FrameID != frameID || lifecycle.LoaderID != loaderID || lifecycle.Name != "load" {
			continue
		}
		var evaluated struct {
			Result struct {
				Value string `json:"value"`
			} `json:"result"`
		}
		if err := cdpCommand(connection, commandID, "Runtime.evaluate", map[string]any{
			"expression": "location.origin", "returnByValue": true,
		}, &evaluated, nil); err != nil || evaluated.Result.Value != douyinOrigin {
			return errors.New("restricted browser navigation failed")
		}
		return nil
	}
	return errors.New("restricted browser navigation failed")
}

func detectDouyinChallenge(status int, body string) douyin.Challenge {
	lower := strings.ToLower(body)
	if status == http.StatusPreconditionFailed || strings.Contains(lower, "captcha") || strings.Contains(lower, "verify_center_decision_conf") {
		return douyin.ChallengeCaptcha
	}
	if strings.Contains(lower, "device_challenge") || strings.Contains(lower, "device verification") {
		return douyin.ChallengeDevice
	}
	return douyin.ChallengeNone
}