diff --git a/cmd/control-plane/douyin.go b/cmd/control-plane/douyin.go new file mode 100644 index 0000000..31468ad --- /dev/null +++ b/cmd/control-plane/douyin.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +type douyinGatewayBrowser struct { + gateway hub.Gateway + environment hub.EnvironmentContext +} + +type douyinGatewayRequest struct { + BindingVersion int64 `json:"binding_version"` + RuntimeID string `json:"runtime_id"` + NetworkID string `json:"network_id"` + NetworkExitID string `json:"network_exit_id"` + Cookies []douyin.Cookie `json:"cookies,omitempty"` + URL string `json:"url,omitempty"` +} + +func (browser douyinGatewayBrowser) SetCookies(ctx context.Context, cookies []douyin.Cookie) error { + request, err := browser.request() + if err != nil { + return err + } + request.Cookies = cookies + status, _, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/cookies", request, 30*time.Second) + if err != nil || status != http.StatusNoContent { + return errors.New("restricted browser operation failed") + } + return nil +} + +func (browser douyinGatewayBrowser) Get(ctx context.Context, target string) (douyin.Response, error) { + request, err := browser.request() + if err != nil { + return douyin.Response{}, err + } + request.URL = target + status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost, + "/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/get", request, 30*time.Second) + if err != nil || status != http.StatusOK { + return douyin.Response{}, errors.New("restricted browser operation failed") + } + var response struct { + Status int `json:"status"` + Body string `json:"body"` + Challenge douyin.Challenge `json:"challenge"` + } + if json.Unmarshal(body, &response) != nil || response.Status < 100 || response.Status > 599 || + (response.Challenge != douyin.ChallengeNone && response.Challenge != douyin.ChallengeCaptcha && + response.Challenge != douyin.ChallengeDevice) { + return douyin.Response{}, errors.New("restricted browser returned an invalid response") + } + return douyin.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil +} + +func (browser douyinGatewayBrowser) request() (douyinGatewayRequest, error) { + environment := browser.environment + if browser.gateway.Endpoint == "" || browser.gateway.Token == "" || !accountRunnable(environment) || + !gatewayGenerationIDPattern.MatchString(environment.AccountID) || !gatewayGenerationIDPattern.MatchString(environment.Alias) || + !gatewayGenerationIDPattern.MatchString(environment.BindingID) || + !gatewayGenerationIDPattern.MatchString(environment.Exit.ID) || environment.Exit.HealthStatus != "healthy" || environment.RuntimeCleanupPending || + !gatewayGenerationIDPattern.MatchString(environment.RuntimeInstanceID) || !gatewayGenerationIDPattern.MatchString(environment.RuntimeID) || + !gatewayGenerationIDPattern.MatchString(environment.RuntimeNetworkID) || + environment.BindingVersion < 1 { + return douyinGatewayRequest{}, errors.New("restricted browser is not ready") + } + return douyinGatewayRequest{BindingVersion: environment.BindingVersion, RuntimeID: environment.RuntimeID, + NetworkID: environment.RuntimeNetworkID, NetworkExitID: environment.Exit.ID}, nil +} diff --git a/cmd/control-plane/douyin_test.go b/cmd/control-plane/douyin_test.go new file mode 100644 index 0000000..73cb85c --- /dev/null +++ b/cmd/control-plane/douyin_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +const testDouyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/" + +func TestDouyinGatewayBrowserFencesAccountGeneration(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + requests++ + if request.Header.Get("Authorization") != "Bearer gateway-token-1" { + t.Fatalf("missing gateway authorization") + } + var body map[string]any + if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) || + body["runtime_id"] != "runtime-a" || body["network_id"] != "network-a" || body["network_exit_id"] != "exit-a" { + t.Fatalf("generation fence missing: %#v", body) + } + switch request.URL.Path { + case "/v1/browsers/account-a/douyin/cookies": + cookies, ok := body["cookies"].([]any) + if !ok || len(cookies) != 1 { + t.Fatalf("cookies missing: %#v", body) + } + response.WriteHeader(http.StatusNoContent) + case "/v1/browsers/account-a/douyin/get": + if body["url"] != testDouyinIdentityURL { + t.Fatalf("unexpected URL: %#v", body) + } + _ = json.NewEncoder(response).Encode(map[string]any{"status": 412, "body": `{"captcha":true}`, "challenge": "captcha"}) + default: + response.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + browser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()} + if err := browser.SetCookies(context.Background(), []douyin.Cookie{{Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/"}}); err != nil { + t.Fatal(err) + } + result, err := browser.Get(context.Background(), testDouyinIdentityURL) + if err != nil || result.Status != 412 || result.Challenge != douyin.ChallengeCaptcha || requests != 2 { + t.Fatalf("unexpected result: %#v requests=%d err=%v", result, requests, err) + } +} + +func TestDouyinGatewayBrowserFailsClosedWithoutReadyBinding(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests++ })) + defer server.Close() + environment := readyDouyinEnvironment() + environment.Exit.HealthStatus = "unhealthy" + browser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: environment} + if _, err := browser.Get(context.Background(), testDouyinIdentityURL); err == nil || requests != 0 { + t.Fatalf("unready binding reached gateway: requests=%d err=%v", requests, err) + } +} + +func TestDouyinGatewayBrowserDoesNotEchoCredentialOnFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + response.WriteHeader(http.StatusBadGateway) + _, _ = response.Write([]byte(`{"error":"private-session"}`)) + })) + defer server.Close() + browser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()} + err := browser.SetCookies(context.Background(), []douyin.Cookie{{Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/"}}) + if err == nil || strings.Contains(err.Error(), "private-session") { + t.Fatalf("gateway failure leaked credential: %v", err) + } +} + +func readyDouyinEnvironment() hub.EnvironmentContext { + return hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Gateway: "gateway-a"}, AccountID: "account-a", + AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "binding-a", BindingVersion: 2, + Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "healthy"}, RuntimeInstanceID: "instance-a", + RuntimeID: "runtime-a", RuntimeNetworkID: "network-a"} +} diff --git a/cmd/docker-gateway/douyin.go b/cmd/docker-gateway/douyin.go new file mode 100644 index 0000000..6124421 --- /dev/null +++ b/cmd/docker-gateway/douyin.go @@ -0,0 +1,448 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "github.com/gofiber/fiber/v3" + "golang.org/x/net/websocket" +) + +const ( + douyinOrigin = "https://www.douyin.com" + douyinOriginURL = "https://www.douyin.com/" + douyinIdentityPath = "/aweme/v1/web/user/profile/self/" + douyinWorksPath = "/aweme/v1/web/aweme/post/" + douyinResponseLimit = 1 << 20 + browserControlTimeout = 15 * time.Second +) + +var douyinAccountKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$`) + +type restrictedBrowserResponse struct { + Status int + Body string + Challenge douyin.Challenge +} + +type restrictedBrowser interface { + SetCookies(context.Context, string, []douyin.Cookie) error + Get(context.Context, string, string) (restrictedBrowserResponse, error) +} + +type douyinGenerationRequest struct { + BindingVersion int64 `json:"binding_version"` + RuntimeID string `json:"runtime_id"` + NetworkID string `json:"network_id"` + NetworkExitID string `json:"network_exit_id"` +} + +type douyinCookieRequest struct { + douyinGenerationRequest + Cookies []douyin.Cookie `json:"cookies"` +} + +type douyinGetRequest struct { + douyinGenerationRequest + URL string `json:"url"` +} + +func (api gateway) setDouyinCookies(c fiber.Ctx) error { + var input douyinCookieRequest + if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) || + !validDouyinCookies(input.Cookies) { + return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request")) + } + _, release, err := api.locks.acquire(c.Params("id")) + if err != nil { + return writeError(c, statusFor(err), err) + } + defer release() + if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { + return writeError(c, statusFor(err), err) + } + if api.browser == nil || api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies) != nil { + return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) + } + if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { + return writeError(c, statusFor(err), err) + } + return c.SendStatus(http.StatusNoContent) +} + +func (api gateway) getDouyin(c fiber.Ctx) error { + var input douyinGetRequest + if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) || + !validDouyinURL(input.URL) { + return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request")) + } + _, release, err := api.locks.acquire(c.Params("id")) + if err != nil { + return writeError(c, statusFor(err), err) + } + defer release() + if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { + return writeError(c, statusFor(err), err) + } + if api.browser == nil { + return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) + } + response, err := api.browser.Get(c.Context(), c.Params("id"), input.URL) + if err != nil { + return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed")) + } + if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil { + return writeError(c, statusFor(err), err) + } + return writeJSON(c, http.StatusOK, map[string]any{ + "status": response.Status, "body": response.Body, "challenge": response.Challenge, + }) +} + +func decodeRestrictedBrowserRequest(body []byte, target any) error { + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return errors.New("request must contain one JSON value") + } + return nil +} + +func validDouyinGeneration(input douyinGenerationRequest) bool { + return input.BindingVersion > 0 && exitIDPattern.MatchString(input.RuntimeID) && + exitIDPattern.MatchString(input.NetworkID) && exitIDPattern.MatchString(input.NetworkExitID) +} + +func (api gateway) requireDouyinGeneration(alias string, input douyinGenerationRequest) error { + runtimeID, labels, networks, err := api.managedContainerState(alias) + if err != nil { + return err + } + version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) + if runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkIDLabel] != input.NetworkID || + labels[networkExitLabel] != input.NetworkExitID { + return errGenerationConflict + } + generation, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, input.BindingVersion, + input.RuntimeID, api.self, input.NetworkID, false) + if err != nil { + return err + } + if !exists || !generation.RuntimeAttached || generation.SelfMember == "" || len(generation.GatewayMembers) == 0 || + len(networks) != 1 || networks[generation.Name] != generation.ID { + return errGenerationConflict + } + return nil +} + +func validDouyinCookies(cookies []douyin.Cookie) bool { + if len(cookies) == 0 || len(cookies) > 64 { + return false + } + for _, cookie := range cookies { + domain := strings.ToLower(strings.TrimSpace(cookie.Domain)) + if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 || + domain != cookie.Domain || cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") || + (domain != "douyin.com" && !strings.HasSuffix(domain, ".douyin.com")) || !strings.HasPrefix(cookie.Path, "/") || + strings.ContainsAny(cookie.Path, ";\r\n\x00") || (cookie.SameSite != "" && cookie.SameSite != "Lax" && + cookie.SameSite != "Strict" && cookie.SameSite != "None") { + return false + } + } + return true +} + +func validDouyinURL(raw string) bool { + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "https" || parsed.Host != "www.douyin.com" || parsed.User != nil || parsed.Fragment != "" { + return false + } + query := parsed.Query() + switch parsed.Path { + case douyinIdentityPath: + return parsed.RawQuery == "" + case douyinWorksPath: + return len(query) == 3 && len(query["sec_user_id"]) == 1 && douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) && + len(query["count"]) == 1 && query.Get("count") == "20" && len(query["max_cursor"]) == 1 && query.Get("max_cursor") == "0" + default: + return false + } +} + +type cdpBrowser struct { + endpoint func(string) string + client *http.Client +} + +func (browser cdpBrowser) SetCookies(ctx context.Context, alias string, cookies []douyin.Cookie) error { + connection, err := browser.connect(ctx, alias) + if err != nil { + return err + } + defer connection.Close() + commandID := 0 + cdpCookies := make([]map[string]any, 0, len(cookies)) + for _, cookie := range cookies { + value := map[string]any{ + "name": cookie.Name, "value": cookie.Value, "url": douyinOriginURL, "path": cookie.Path, + "secure": cookie.Secure, "httpOnly": cookie.HTTPOnly, + } + if cookie.SameSite != "" { + value["sameSite"] = cookie.SameSite + } + if cookie.Expires != 0 { + value["expires"] = cookie.Expires + } + cdpCookies = append(cdpCookies, value) + } + if err := cdpCommand(connection, &commandID, "Network.enable", map[string]any{}, nil, nil); err != nil { + return err + } + if err := cdpCommand(connection, &commandID, "Network.clearBrowserCookies", map[string]any{}, nil, nil); err != nil { + return err + } + if err := cdpCommand(connection, &commandID, "Page.enable", map[string]any{}, nil, nil); err != nil { + return err + } + if err := cdpCommand(connection, &commandID, "Page.setLifecycleEventsEnabled", map[string]bool{"enabled": true}, nil, nil); err != nil { + return err + } + var events []cdpMessage + var navigation struct { + FrameID string `json:"frameId"` + LoaderID string `json:"loaderId"` + ErrorText string `json:"errorText"` + } + if err := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events); err != nil || + navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" { + return errors.New("restricted browser navigation failed") + } + if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, navigation.LoaderID, events); err != nil { + return err + } + return cdpCommand(connection, &commandID, "Network.setCookies", map[string]any{"cookies": cdpCookies}, nil, nil) +} + +func (browser cdpBrowser) Get(ctx context.Context, alias, target string) (restrictedBrowserResponse, error) { + connection, err := browser.connect(ctx, alias) + if err != nil { + return restrictedBrowserResponse{}, err + } + defer connection.Close() + commandID := 0 + var currentOrigin struct { + Result struct { + Value string `json:"value"` + } `json:"result"` + } + if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{ + "expression": "location.origin", "returnByValue": true, + }, ¤tOrigin, nil); err != nil || currentOrigin.Result.Value != douyinOrigin { + return restrictedBrowserResponse{}, errors.New("restricted browser origin changed") + } + encodedURL, _ := json.Marshal(target) + expression := `(async()=>{const r=await fetch(` + string(encodedURL) + `,{credentials:"include",redirect:"error"});` + + `if(!r.body)return {status:r.status,body:"",too_large:false};const q=r.body.getReader(),d=new TextDecoder();let n=0,b="";` + + `for(;;){const x=await q.read();if(x.done)break;if(n+x.value.byteLength>=` + strconv.Itoa(douyinResponseLimit) + + `){await q.cancel();return {too_large:true};}n+=x.value.byteLength;b+=d.decode(x.value,{stream:true});}` + + `b+=d.decode();return {status:r.status,body:b,too_large:false};})()` + var evaluated struct { + Result struct { + Value struct { + Status int `json:"status"` + Body string `json:"body"` + TooLarge bool `json:"too_large"` + } `json:"value"` + } `json:"result"` + ExceptionDetails json.RawMessage `json:"exceptionDetails"` + } + if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{ + "expression": expression, "awaitPromise": true, "returnByValue": true, + }, &evaluated, nil); err != nil || len(evaluated.ExceptionDetails) != 0 || evaluated.Result.Value.TooLarge || + evaluated.Result.Value.Status < 200 || evaluated.Result.Value.Status > 599 || + (evaluated.Result.Value.Status >= 300 && evaluated.Result.Value.Status < 400) { + return restrictedBrowserResponse{}, errors.New("restricted browser fetch failed") + } + body := evaluated.Result.Value.Body + return restrictedBrowserResponse{Status: evaluated.Result.Value.Status, Body: body, Challenge: detectDouyinChallenge(evaluated.Result.Value.Status, body)}, nil +} + +func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket.Conn, error) { + base := "http://" + namePrefix + alias + ":9222" + if browser.endpoint != nil { + base = browser.endpoint(alias) + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/json/list", nil) + if err != nil { + return nil, errors.New("restricted browser unavailable") + } + client := http.Client{Timeout: browserControlTimeout} + if browser.client != nil { + client = *browser.client + } + if client.Timeout <= 0 { + client.Timeout = browserControlTimeout + } + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + response, err := client.Do(request) + if err != nil { + return nil, errors.New("restricted browser unavailable") + } + defer response.Body.Close() + var targets []struct { + Type string `json:"type"` + WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` + } + if response.StatusCode != http.StatusOK { + return nil, errors.New("restricted browser unavailable") + } + body, err := io.ReadAll(io.LimitReader(response.Body, (64<<10)+1)) + if err != nil || len(body) > 64<<10 { + return nil, errors.New("restricted browser unavailable") + } + decoder := json.NewDecoder(bytes.NewReader(body)) + if decoder.Decode(&targets) != nil || len(targets) > 32 { + return nil, errors.New("restricted browser unavailable") + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return nil, errors.New("restricted browser unavailable") + } + baseURL, _ := url.Parse(base) + pageTarget := "" + for _, target := range targets { + if target.Type != "page" { + continue + } + if pageTarget != "" { + return nil, errors.New("restricted browser unavailable") + } + websocketURL, parseErr := url.Parse(target.WebSocketDebuggerURL) + if parseErr != nil || websocketURL.Scheme != "ws" || !strings.HasPrefix(websocketURL.Path, "/devtools/page/") { + return nil, errors.New("restricted browser unavailable") + } + if websocketURL.Hostname() == "localhost" || websocketURL.Hostname() == "127.0.0.1" || websocketURL.Hostname() == "::1" { + websocketURL.Host = baseURL.Host + } + if websocketURL.Host != baseURL.Host { + return nil, errors.New("restricted browser unavailable") + } + pageTarget = websocketURL.String() + } + if pageTarget == "" { + return nil, errors.New("restricted browser unavailable") + } + config, err := websocket.NewConfig(pageTarget, "devtools://devtools") + if err != nil { + return nil, errors.New("restricted browser unavailable") + } + connection, err := config.DialContext(ctx) + if err != nil { + return nil, errors.New("restricted browser unavailable") + } + deadline := time.Now().Add(browserControlTimeout) + if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { + deadline = contextDeadline + } + _ = connection.SetDeadline(deadline) + return connection, nil +} + +type cdpMessage struct { + ID int `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + Result json.RawMessage `json:"result"` + Error json.RawMessage `json:"error"` +} + +func cdpCommand(connection *websocket.Conn, commandID *int, method string, parameters any, output any, events *[]cdpMessage) error { + *commandID = *commandID + 1 + if err := websocket.JSON.Send(connection, map[string]any{"id": *commandID, "method": method, "params": parameters}); err != nil { + return errors.New("restricted browser command failed") + } + for range 128 { + var reply cdpMessage + if err := websocket.JSON.Receive(connection, &reply); err != nil { + return errors.New("restricted browser command failed") + } + if reply.ID != *commandID { + if events != nil && reply.ID == 0 && reply.Method != "" { + *events = append(*events, reply) + } + continue + } + if len(reply.Error) != 0 || len(reply.Result) == 0 || bytes.Equal(bytes.TrimSpace(reply.Result), []byte("null")) { + return errors.New("restricted browser command failed") + } + if output != nil && json.Unmarshal(reply.Result, output) != nil { + return errors.New("restricted browser command failed") + } + return nil + } + return errors.New("restricted browser command failed") +} + +func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID, loaderID string, events []cdpMessage) error { + deadline := time.Now().Add(10 * time.Second) + if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { + deadline = contextDeadline + } + _ = connection.SetReadDeadline(deadline) + for attempts := 0; attempts < 256; attempts++ { + var event cdpMessage + if len(events) != 0 { + event, events = events[0], events[1:] + } else if websocket.JSON.Receive(connection, &event) != nil { + return errors.New("restricted browser navigation failed") + } + if event.Method != "Page.lifecycleEvent" { + continue + } + var lifecycle struct { + FrameID string `json:"frameId"` + LoaderID string `json:"loaderId"` + Name string `json:"name"` + } + if json.Unmarshal(event.Params, &lifecycle) != nil || lifecycle.FrameID != frameID || lifecycle.LoaderID != loaderID || lifecycle.Name != "load" { + continue + } + var evaluated struct { + Result struct { + Value string `json:"value"` + } `json:"result"` + } + if err := cdpCommand(connection, commandID, "Runtime.evaluate", map[string]any{ + "expression": "location.origin", "returnByValue": true, + }, &evaluated, nil); err != nil || evaluated.Result.Value != douyinOrigin { + return errors.New("restricted browser navigation failed") + } + return nil + } + return errors.New("restricted browser navigation failed") +} + +func detectDouyinChallenge(status int, body string) douyin.Challenge { + lower := strings.ToLower(body) + if status == http.StatusPreconditionFailed || strings.Contains(lower, "captcha") || strings.Contains(lower, "verify_center_decision_conf") { + return douyin.ChallengeCaptcha + } + if strings.Contains(lower, "device_challenge") || strings.Contains(lower, "device verification") { + return douyin.ChallengeDevice + } + return douyin.ChallengeNone +} diff --git a/cmd/docker-gateway/douyin_cdp_integration_test.go b/cmd/docker-gateway/douyin_cdp_integration_test.go new file mode 100644 index 0000000..83f3c3f --- /dev/null +++ b/cmd/docker-gateway/douyin_cdp_integration_test.go @@ -0,0 +1,205 @@ +package main + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/douyin" +) + +func TestCDPBrowserRealNavigationIsolation(t *testing.T) { + if os.Getenv("CREATORHUB_REAL_CDP_TEST") != "1" { + t.Skip("set CREATORHUB_REAL_CDP_TEST=1 to run against local Chrome") + } + chrome, err := exec.LookPath("google-chrome") + if err != nil { + t.Skip("google-chrome is unavailable") + } + + var mu sync.Mutex + phase := "redirect" + release := make(chan struct{}) + mainRequests := make(chan string, 4) + loginRequests := make(chan string, 1) + subdomainRequests := make(chan string, 1) + server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + host := request.Host + if name, _, splitErr := net.SplitHostPort(host); splitErr == nil { + host = name + } + switch host { + case "login.douyin.com": + loginRequests <- request.Header.Get("Cookie") + _, _ = response.Write([]byte("redirected")) + case "api.douyin.com": + subdomainRequests <- request.Header.Get("Cookie") + _, _ = response.Write([]byte("ok")) + case "www.douyin.com": + if request.URL.Path == "/" { + mainRequests <- request.Header.Get("Cookie") + } + mu.Lock() + currentPhase, currentRelease := phase, release + mu.Unlock() + if currentPhase == "redirect" { + http.Redirect(response, request, "https://login.douyin.com/landing", http.StatusFound) + return + } + if currentPhase == "blocked" { + <-currentRelease + } + response.Header().Set("Content-Type", "text/html") + _, _ = response.Write([]byte(``)) + default: + response.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + allowedHosts := map[string]bool{"www.douyin.com": true, "login.douyin.com": true, "api.douyin.com": true} + proxy := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + host := request.Host + if name, _, splitErr := net.SplitHostPort(host); splitErr == nil { + host = name + } + if request.Method != http.MethodConnect || !allowedHosts[host] { + response.WriteHeader(http.StatusForbidden) + return + } + upstream, dialErr := net.Dial("tcp", server.Listener.Addr().String()) + if dialErr != nil { + response.WriteHeader(http.StatusBadGateway) + return + } + client, _, hijackErr := response.(http.Hijacker).Hijack() + if hijackErr != nil { + upstream.Close() + return + } + _, _ = client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")) + go func() { + _, _ = io.Copy(upstream, client) + _ = upstream.Close() + }() + _, _ = io.Copy(client, upstream) + _ = client.Close() + })) + t.Cleanup(proxy.Close) + + profile := t.TempDir() + command := exec.Command(chrome, "--headless=new", "--no-sandbox", "--disable-gpu", "--disable-background-networking", "--disable-quic", + "--disable-dev-shm-usage", "--no-first-run", "--no-default-browser-check", "--password-store=basic", "--use-mock-keychain", + "--ignore-certificate-errors", "--proxy-server="+proxy.URL, + "--remote-debugging-address=127.0.0.1", "--remote-debugging-port=0", "--remote-allow-origins=*", + "--user-data-dir="+profile, "about:blank") + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := command.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + _ = command.Wait() + }) + + var debugPort string + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(25 * time.Millisecond) { + content, readErr := os.ReadFile(filepath.Join(profile, "DevToolsActivePort")) + if readErr == nil { + debugPort = strings.SplitN(string(content), "\n", 2)[0] + break + } + } + if debugPort == "" { + t.Fatal("Chrome did not expose a DevTools port") + } + time.Sleep(250 * time.Millisecond) + browser := cdpBrowser{endpoint: func(string) string { return "http://127.0.0.1:" + debugPort }} + cookie := []douyin.Cookie{{Name: "sessionid", Value: "fake-secret", Domain: ".douyin.com", Path: "/", Secure: true}} + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + redirectErr := browser.SetCookies(ctx, "account-a", cookie) + if redirectErr == nil { + cancel() + t.Fatal("accepted a redirected navigation") + } + cancel() + select { + case got := <-loginRequests: + if strings.Contains(got, "fake-secret") { + t.Fatal("cookie leaked during redirected navigation") + } + case <-time.After(3 * time.Second): + t.Fatalf("redirect target was not reached: %v", redirectErr) + } + + mu.Lock() + phase = "success" + mu.Unlock() + ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + if err := browser.SetCookies(ctx, "account-a", cookie); err != nil { + cancel() + t.Fatal(err) + } + cancel() + select { + case <-mainRequests: + case <-time.After(3 * time.Second): + t.Fatal("successful navigation was not observed") + } + + mu.Lock() + phase = "blocked" + release = make(chan struct{}) + currentRelease := release + mu.Unlock() + defer func() { + select { + case <-currentRelease: + default: + close(currentRelease) + } + }() + done := make(chan error, 1) + ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + go func() { done <- browser.SetCookies(ctx, "account-a", cookie) }() + select { + case got := <-mainRequests: + if strings.Contains(got, "fake-secret") { + cancel() + t.Fatal("cookie leaked during navigation") + } + case <-time.After(3 * time.Second): + cancel() + t.Fatal("blocked navigation was not observed") + } + select { + case err := <-done: + cancel() + t.Fatalf("navigation completed before its loader: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(currentRelease) + if err := <-done; err != nil { + cancel() + t.Fatal(err) + } + cancel() + select { + case got := <-subdomainRequests: + if strings.Contains(got, "fake-secret") { + t.Fatal("host-only cookie leaked to a subdomain") + } + case <-time.After(3 * time.Second): + t.Fatal("subdomain probe did not run") + } +} diff --git a/cmd/docker-gateway/douyin_test.go b/cmd/docker-gateway/douyin_test.go new file mode 100644 index 0000000..cd9d420 --- /dev/null +++ b/cmd/docker-gateway/douyin_test.go @@ -0,0 +1,408 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/douyin" + "golang.org/x/net/websocket" +) + +type fakeRestrictedBrowser struct { + cookies []douyin.Cookie + urls []string + response restrictedBrowserResponse + after func() +} + +func (browser *fakeRestrictedBrowser) SetCookies(_ context.Context, _ string, cookies []douyin.Cookie) error { + browser.cookies = cookies + if browser.after != nil { + browser.after() + } + return nil +} + +func (browser *fakeRestrictedBrowser) Get(_ context.Context, _ string, target string) (restrictedBrowserResponse, error) { + browser.urls = append(browser.urls, target) + if browser.after != nil { + browser.after() + } + return browser.response, nil +} + +func TestGatewayRestrictedDouyinContract(t *testing.T) { + labels := map[string]string{ + managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "2", networkIDLabel: "network-a", networkExitLabel: "exit-a", + } + self, _ := os.Hostname() + var stateMu sync.Mutex + containerNetworks := map[string]string{"creatorhub_browser-account-a": "network-a"} + runtimeAttached := true + server := httptest.NewServer(withAliasReservations(self, func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/containers/" + namePrefix + "account-a/json": + stateMu.Lock() + labelCopy, networkCopy := map[string]string{}, map[string]any{} + for key, value := range labels { + labelCopy[key] = value + } + for name, id := range containerNetworks { + networkCopy[name] = map[string]string{"NetworkID": id} + } + stateMu.Unlock() + _ = json.NewEncoder(response).Encode(map[string]any{"Id": "runtime-a", "Config": map[string]any{"Labels": labelCopy}, + "NetworkSettings": map[string]any{"Networks": networkCopy}}) + case "/networks/network-a": + stateMu.Lock() + members := map[string]any{self: map[string]string{"Name": self, "IPv4Address": "127.0.0.1/8"}} + if runtimeAttached { + members["runtime-a"] = map[string]string{"Name": namePrefix + "account-a", "IPv4Address": "127.0.0.2/8"} + } + stateMu.Unlock() + _ = json.NewEncoder(response).Encode(map[string]any{ + "Id": "network-a", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "2"}, + "Containers": members, + }) + default: + response.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + browser := &fakeRestrictedBrowser{response: restrictedBrowserResponse{Status: 200, Body: `{"status_code":0}`, Challenge: douyin.ChallengeNone}} + app := newGatewayWithBrowser(docker, "creatorhub_browser", testToken, self, browser) + generation := `"binding_version":2,"runtime_id":"runtime-a","network_id":"network-a","network_exit_id":"exit-a"` + + cookieBody := `{` + generation + `,"cookies":[{"name":"sessionid","value":"private-session","domain":".douyin.com","path":"/"}]}` + response, err := app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/cookies", strings.NewReader(cookieBody))) + if err != nil || response.StatusCode != http.StatusNoContent || len(browser.cookies) != 1 || browser.cookies[0].Value != "private-session" { + t.Fatalf("set cookies failed: status=%d cookies=%#v err=%v", response.StatusCode, browser.cookies, err) + } + response.Body.Close() + + identityURL := "https://www.douyin.com" + douyinIdentityPath + getBody := `{` + generation + `,"url":"` + identityURL + `"}` + response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody))) + body, _ := io.ReadAll(response.Body) + response.Body.Close() + if err != nil || response.StatusCode != http.StatusOK || len(browser.urls) != 1 || browser.urls[0] != identityURL || + !strings.Contains(string(body), `\"status_code\":0`) || strings.Contains(string(body), "private-session") { + t.Fatalf("get failed or leaked cookies: status=%d urls=%#v body=%s err=%v", response.StatusCode, browser.urls, body, err) + } + + for name, request := range map[string]*http.Request{ + "unauthenticated": httptest.NewRequest(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody)), + "invalid account": authed(http.MethodPost, "/v1/browsers/AccountA/douyin/get", strings.NewReader(getBody)), + "generic URL": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(`{`+generation+`,"url":"https://example.com/"}`)), + "generic CDP": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(`{`+generation+`,"url":"`+identityURL+`","method":"Runtime.evaluate"}`)), + "trailing null": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody+`null`)), + "stale generation": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(`{"binding_version":1,"runtime_id":"runtime-a","network_id":"network-a","network_exit_id":"exit-a","url":"`+identityURL+`"}`)), + } { + t.Run(name, func(t *testing.T) { + before := len(browser.urls) + response, err := app.Test(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + want := http.StatusBadRequest + if name == "unauthenticated" { + want = http.StatusUnauthorized + } else if name == "stale generation" { + want = http.StatusConflict + } + if response.StatusCode != want || len(browser.urls) != before { + t.Fatalf("status=%d want=%d calls=%d want=%d", response.StatusCode, want, len(browser.urls), before) + } + }) + } + + stateMu.Lock() + runtimeAttached = false + stateMu.Unlock() + before := len(browser.urls) + response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody))) + if err != nil || response.StatusCode != http.StatusConflict || len(browser.urls) != before { + t.Fatalf("wrong network membership reached browser: status=%d calls=%d want=%d err=%v", response.StatusCode, len(browser.urls), before, err) + } + response.Body.Close() + stateMu.Lock() + runtimeAttached = true + stateMu.Unlock() + + browser.after = func() { + stateMu.Lock() + containerNetworks["other-tenant"] = "network-b" + stateMu.Unlock() + } + response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/cookies", strings.NewReader(cookieBody))) + if err != nil || response.StatusCode != http.StatusConflict { + t.Fatalf("post-operation cross-network attachment was accepted: status=%d err=%v", response.StatusCode, err) + } + response.Body.Close() + stateMu.Lock() + delete(containerNetworks, "other-tenant") + stateMu.Unlock() + + browser.after = func() { + stateMu.Lock() + labels[networkIDLabel] = "network-replaced" + stateMu.Unlock() + } + response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/cookies", strings.NewReader(cookieBody))) + if err != nil || response.StatusCode != http.StatusConflict { + t.Fatalf("post-operation generation replacement was accepted: status=%d err=%v", response.StatusCode, err) + } + response.Body.Close() +} + +func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) { + var server *httptest.Server + var mu sync.Mutex + methods := []string{} + secretSeen := false + cookieNames := []string{} + cookiesHostOnly := true + setCookieCalls := 0 + pageOrigin := douyinOrigin + onlyOldLoader := false + fetchMode := "ok" + fetchExpression := "" + mux := http.NewServeMux() + mux.HandleFunc("/json/list", func(response http.ResponseWriter, request *http.Request) { + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/devtools/page/one" + _ = json.NewEncoder(response).Encode([]map[string]string{{"type": "page", "webSocketDebuggerUrl": wsURL}}) + }) + mux.Handle("/devtools/page/one", websocket.Server{ + Handshake: func(*websocket.Config, *http.Request) error { return nil }, + Handler: func(connection *websocket.Conn) { + for { + var command struct { + ID int `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + } + if websocket.JSON.Receive(connection, &command) != nil { + return + } + mu.Lock() + methods = append(methods, command.Method) + secretSeen = secretSeen || strings.Contains(string(command.Params), "private-session") + result := any(map[string]any{}) + var beforeReply, afterReply []map[string]any + switch command.Method { + case "Network.clearBrowserCookies": + cookieNames = nil + case "Network.setCookies": + var params struct { + Cookies []map[string]any `json:"cookies"` + } + _ = json.Unmarshal(command.Params, ¶ms) + setCookieCalls++ + cookieNames = cookieNames[:0] + for _, cookie := range params.Cookies { + cookieNames = append(cookieNames, cookie["name"].(string)) + _, hasDomain := cookie["domain"] + cookiesHostOnly = cookiesHostOnly && !hasDomain && cookie["url"] == douyinOriginURL + } + case "Page.navigate": + result = map[string]any{"frameId": "frame-new", "loaderId": "loader-new"} + beforeReply = append(beforeReply, map[string]any{"method": "Page.lifecycleEvent", "params": map[string]any{ + "frameId": "frame-old", "loaderId": "loader-old", "name": "load", + }}) + if !onlyOldLoader { + afterReply = append(afterReply, map[string]any{"method": "Page.lifecycleEvent", "params": map[string]any{ + "frameId": "frame-new", "loaderId": "loader-new", "name": "load", + }}) + } + case "Runtime.evaluate": + var params struct { + Expression string `json:"expression"` + } + _ = json.Unmarshal(command.Params, ¶ms) + if params.Expression == "location.origin" { + result = map[string]any{"result": map[string]any{"value": pageOrigin}} + } else { + fetchExpression = params.Expression + value := map[string]any{"status": 412, "body": `{"captcha":true}`, "too_large": false} + if fetchMode == "redirect" { + value = map[string]any{"status": 302, "body": "", "too_large": false} + } else if fetchMode == "too_large" { + value = map[string]any{"too_large": true} + } + result = map[string]any{"result": map[string]any{"value": map[string]any{ + "status": value["status"], "body": value["body"], "too_large": value["too_large"], + }}} + } + } + mu.Unlock() + for _, event := range beforeReply { + _ = websocket.JSON.Send(connection, event) + } + _ = websocket.JSON.Send(connection, map[string]any{"id": command.ID, "result": result}) + for _, event := range afterReply { + _ = websocket.JSON.Send(connection, event) + } + } + }, + }) + server = httptest.NewServer(mux) + defer server.Close() + browser := cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()} + if err := browser.SetCookies(context.Background(), "account-a", []douyin.Cookie{{ + Name: "old_auth", Value: "old-session", Domain: ".douyin.com", Path: "/", + }}); err != nil { + t.Fatal(err) + } + if err := browser.SetCookies(context.Background(), "account-a", []douyin.Cookie{{ + Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/", + }}); err != nil { + t.Fatal(err) + } + result, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath) + if err != nil || result.Status != 412 || result.Challenge != douyin.ChallengeCaptcha { + t.Fatalf("unexpected CDP response: %#v err=%v", result, err) + } + mu.Lock() + if !secretSeen || !cookiesHostOnly || strings.Join(cookieNames, ",") != "sessionid" || + strings.Join(methods, ",") != "Network.enable,Network.clearBrowserCookies,Page.enable,Page.setLifecycleEventsEnabled,Page.navigate,Runtime.evaluate,Network.setCookies,"+ + "Network.enable,Network.clearBrowserCookies,Page.enable,Page.setLifecycleEventsEnabled,Page.navigate,Runtime.evaluate,Network.setCookies,Runtime.evaluate,Runtime.evaluate" || + !strings.Contains(fetchExpression, `redirect:"error"`) || !strings.Contains(fetchExpression, "getReader()") || + !strings.Contains(fetchExpression, "q.cancel()") || !strings.Contains(fetchExpression, ">=1048576") || strings.Contains(fetchExpression, "r.text()") { + mu.Unlock() + t.Fatalf("unexpected CDP contract: methods=%#v cookies=%#v secret_seen=%v expression=%s", methods, cookieNames, secretSeen, fetchExpression) + } + pageOrigin = "https://login.douyin.com" + setCookiesBeforeRedirect := setCookieCalls + mu.Unlock() + if err := browser.SetCookies(context.Background(), "account-a", []douyin.Cookie{{ + Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/", + }}); err == nil { + t.Fatal("accepted navigation redirected to a Douyin subdomain") + } + mu.Lock() + if setCookieCalls != setCookiesBeforeRedirect { + mu.Unlock() + t.Fatal("set cookies before rejecting redirected navigation") + } + pageOrigin, fetchMode = douyinOrigin, "redirect" + mu.Unlock() + if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil { + t.Fatal("accepted a redirected fetch") + } + mu.Lock() + fetchMode = "too_large" + mu.Unlock() + if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil { + t.Fatal("accepted a response at the 1 MiB limit") + } + mu.Lock() + onlyOldLoader = true + setCookiesBeforeOldLoader := setCookieCalls + mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := browser.SetCookies(ctx, "account-a", []douyin.Cookie{{ + Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/", + }}); err == nil { + t.Fatal("accepted an old page load event for the new navigation") + } + mu.Lock() + defer mu.Unlock() + if setCookieCalls != setCookiesBeforeOldLoader { + t.Fatal("set cookies before the new loader completed") + } +} + +func TestCDPDiscoveryDoesNotFollowRedirects(t *testing.T) { + redirected := 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/json/list" { + http.Redirect(response, request, "/redirected", http.StatusFound) + return + } + redirected++ + response.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + browser := cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()} + if _, err := browser.connect(context.Background(), "account-a"); err == nil || redirected != 0 { + t.Fatalf("discovery redirect was followed: redirected=%d err=%v", redirected, err) + } +} + +func TestCDPDiscoveryRequiresOnePage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + _ = json.NewEncoder(response).Encode([]map[string]string{ + {"type": "page", "webSocketDebuggerUrl": "ws://localhost/devtools/page/one"}, + {"type": "page", "webSocketDebuggerUrl": "ws://localhost/devtools/page/two"}, + }) + })) + defer server.Close() + browser := cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()} + if _, err := browser.connect(context.Background(), "account-a"); err == nil { + t.Fatal("accepted a profile with multiple page targets") + } +} + +func TestCDPDiscoveryRequiresOneJSONValue(t *testing.T) { + for name, suffix := range map[string]string{ + "null": "null", "other value": `{}`, "garbage": "garbage", "oversized": strings.Repeat(" ", 64<<10), "whitespace": " \n\t", + } { + t.Run(name, func(t *testing.T) { + websocketAttempts := 0 + var server *httptest.Server + mux := http.NewServeMux() + mux.HandleFunc("/json/list", func(response http.ResponseWriter, request *http.Request) { + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/devtools/page/one" + _, _ = response.Write([]byte(`[{"type":"page","webSocketDebuggerUrl":"` + wsURL + `"}]` + suffix)) + }) + mux.Handle("/devtools/page/one", websocket.Server{ + Handshake: func(*websocket.Config, *http.Request) error { + websocketAttempts++ + return nil + }, + Handler: func(connection *websocket.Conn) {}, + }) + server = httptest.NewServer(mux) + defer server.Close() + + connection, err := (cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()}).connect(context.Background(), "account-a") + if connection != nil { + connection.Close() + } + valid := name == "whitespace" + wantAttempts := 0 + if valid { + wantAttempts = 1 + } + if (err == nil) != valid || websocketAttempts != wantAttempts { + t.Fatalf("err=%v websocket attempts=%d", err, websocketAttempts) + } + }) + } +} + +func TestDouyinURLContract(t *testing.T) { + for target, want := range map[string]bool{ + "https://www.douyin.com" + douyinIdentityPath: true, + "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0": true, + "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=1": false, + "https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0&method=publish": false, + "https://www.douyin.com/aweme/v1/web/commit/item/": false, + } { + if got := validDouyinURL(target); got != want { + t.Fatalf("validDouyinURL(%q)=%v want=%v", target, got, want) + } + } +} diff --git a/cmd/docker-gateway/main.go b/cmd/docker-gateway/main.go index 2e1da61..a5f0177 100644 --- a/cmd/docker-gateway/main.go +++ b/cmd/docker-gateway/main.go @@ -113,6 +113,7 @@ type gateway struct { network string self string token string + browser restrictedBrowser proxies *memoryProxyRegistry locks *dockerAliasReservations } @@ -276,8 +277,12 @@ func newGateway(client dockerClient, network, token string) *fiber.App { } func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber.App { + return newGatewayWithBrowser(client, network, token, self, cdpBrowser{}) +} + +func newGatewayWithBrowser(client dockerClient, network, token, self string, browser restrictedBrowser) *fiber.App { api := gateway{docker: client, network: network, self: self, token: token, proxies: newMemoryProxyRegistry(), - locks: &dockerAliasReservations{docker: client, self: self}} + locks: &dockerAliasReservations{docker: client, self: self}, browser: browser} app := fiber.New(fiber.Config{ AppName: "CreatorHub Docker gateway", BodyLimit: 1 << 20, @@ -300,6 +305,8 @@ func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber app.Get("/v1/browsers", api.list) app.Post("/v1/browsers", api.create) app.Post("/v1/browsers/:id/proxy", api.restoreProxy) + app.Post("/v1/browsers/:id/douyin/cookies", api.setDouyinCookies) + app.Post("/v1/browsers/:id/douyin/get", api.getDouyin) app.Post("/v1/browsers/:id/:action", api.changeState) app.Delete("/v1/browsers/:id", api.remove) return app @@ -901,33 +908,47 @@ func (api gateway) requireGeneration(id string, input generationRequest) (string } func (api gateway) managedContainer(id string) (string, map[string]string, error) { + runtimeID, labels, _, err := api.managedContainerState(id) + return runtimeID, labels, err +} + +func (api gateway) managedContainerState(id string) (string, map[string]string, map[string]string, error) { if !runtimeIDPattern.MatchString(id) { - return "", nil, errInvalidRuntimeID + return "", nil, nil, errInvalidRuntimeID } result, err := api.docker.request(http.MethodGet, "/containers/"+url.PathEscape(namePrefix+id)+"/json", nil) if err != nil { - return "", nil, err + return "", nil, nil, err } defer result.Body.Close() if result.StatusCode == http.StatusNotFound { - return "", nil, os.ErrNotExist + return "", nil, nil, os.ErrNotExist } if result.StatusCode != http.StatusOK { - return "", nil, fmt.Errorf("Docker inspect returned %s", result.Status) + return "", nil, nil, fmt.Errorf("Docker inspect returned %s", result.Status) } var inspected struct { ID string `json:"Id"` Config struct { Labels map[string]string `json:"Labels"` } `json:"Config"` + NetworkSettings struct { + Networks map[string]struct { + NetworkID string `json:"NetworkID"` + } `json:"Networks"` + } `json:"NetworkSettings"` } if err := json.NewDecoder(result.Body).Decode(&inspected); err != nil { - return "", nil, fmt.Errorf("decode Docker inspect: %w", err) + return "", nil, nil, fmt.Errorf("decode Docker inspect: %w", err) } if inspected.Config.Labels[managedLabel] != "true" || inspected.Config.Labels[idLabel] != id { - return "", nil, errUnmanagedContainer + return "", nil, nil, errUnmanagedContainer } - return inspected.ID, inspected.Config.Labels, nil + networks := make(map[string]string, len(inspected.NetworkSettings.Networks)) + for name, network := range inspected.NetworkSettings.Networks { + networks[name] = network.NetworkID + } + return inspected.ID, inspected.Config.Labels, networks, nil } // pullIfMissing 在镜像不在本地时从远端仓库拉取；镜像缺失属于可恢复错误，调用方可直接重试。 diff --git a/go.mod b/go.mod index f008678..9f962a4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/sirupsen/logrus v1.10.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 + golang.org/x/net v0.57.0 ) require ( @@ -37,7 +38,6 @@ require ( github.com/valyala/fasthttp v1.73.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.54.0 // indirect - golang.org/x/net v0.57.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect diff --git a/internal/douyin/connector.go b/internal/douyin/connector.go new file mode 100644 index 0000000..98eb2ac --- /dev/null +++ b/internal/douyin/connector.go @@ -0,0 +1,304 @@ +package douyin + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + "unicode/utf8" +) + +const ( + StateSucceeded = "succeeded" + StatePolicyHold = "policy_hold" + StateNeedsConfirmation = "needs_confirmation" + + ReasonAuthInvalid = "douyin_auth_invalid" + ReasonForbidden = "douyin_forbidden" + ReasonRateLimited = "douyin_rate_limited" + ReasonChallenge = "douyin_challenge" + ReasonUnknown = "douyin_result_unknown" + ReasonIdentityMatch = "douyin_identity_mismatch" + ReasonSucceeded = "douyin_sync_succeeded" + + identityEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/self/" + worksEndpoint = "https://www.douyin.com/aweme/v1/web/aweme/post/" +) + +var ( + keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`) + credentialKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._/-]{0,126}$`) +) + +var ErrInvalid = errors.New("invalid douyin connector input") + +type Challenge string + +const ( + ChallengeNone Challenge = "" + ChallengeCaptcha Challenge = "captcha" + ChallengeDevice Challenge = "device" +) + +type Cookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Secure bool `json:"secure,omitempty"` + HTTPOnly bool `json:"http_only,omitempty"` + SameSite string `json:"same_site,omitempty"` + Expires float64 `json:"expires,omitempty"` +} + +type Response struct { + Status int + Body []byte + Challenge Challenge +} + +// Browser is the deliberately narrow contract the restricted browser control +// plane must implement. It does not permit arbitrary CDP commands. +type Browser interface { + SetCookies(context.Context, []Cookie) error + Get(context.Context, string) (Response, error) +} + +type SecretReference struct { + Provider string + Key string +} + +type SecretResolver interface { + Resolve(context.Context, SecretReference) ([]byte, error) +} + +type Work struct { + ID string `json:"id"` + Description string `json:"description"` + CreatedAt int64 `json:"created_at"` + DiggCount int64 `json:"digg_count"` + CommentCount int64 `json:"comment_count"` + ShareCount int64 `json:"share_count"` + PlayCount int64 `json:"play_count"` +} + +type Evidence struct { + Phase string `json:"phase"` + HTTPStatus int `json:"http_status,omitempty"` + IdentityVerified bool `json:"identity_verified"` + WorksSeen int `json:"works_seen,omitempty"` + HasMore bool `json:"has_more,omitempty"` +} + +type Result struct { + State string `json:"state"` + ReasonCode string `json:"reason_code"` + Evidence Evidence `json:"evidence"` +} + +// Store owns both persistence/audit and fail-closed account/runtime handling. +// Hold must pause the account and stop its existing bound runtime without retry. +type Store interface { + Complete(context.Context, string, []Work, Evidence) error + Hold(context.Context, string, string, string, Evidence) error +} + +type Connector struct { + Browser Browser + Secrets SecretResolver + Store Store +} + +type Request struct { + AccountID string + PlatformAccountKey string + Credential SecretReference +} + +func (connector Connector) Sync(ctx context.Context, request Request) (Result, error) { + if connector.Browser == nil || connector.Secrets == nil || connector.Store == nil || !keyPattern.MatchString(request.AccountID) || + !keyPattern.MatchString(request.PlatformAccountKey) || + (request.Credential.Provider != "os_keyring" && request.Credential.Provider != "secret_manager") || + !credentialKeyPattern.MatchString(request.Credential.Key) { + return Result{}, ErrInvalid + } + credential, err := connector.Secrets.Resolve(ctx, request.Credential) + if err != nil { + return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"}) + } + cookies, err := parseCredential(credential) + if err != nil { + return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"}) + } + if err := connector.Browser.SetCookies(ctx, cookies); err != nil { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "login"}) + } + + identityResponse, err := connector.Browser.Get(ctx, identityEndpoint) + if err != nil { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "identity"}) + } + if state, reason := classify(identityResponse); state != "" { + return connector.stop(ctx, request.AccountID, state, reason, Evidence{Phase: "identity", HTTPStatus: identityResponse.Status}) + } + identity, ok := parseIdentity(identityResponse.Body) + if !ok { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, + Evidence{Phase: "identity", HTTPStatus: identityResponse.Status}) + } + if request.PlatformAccountKey != identity.User.UID && request.PlatformAccountKey != identity.User.SecUID && + request.PlatformAccountKey != identity.User.UniqueID { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonIdentityMatch, + Evidence{Phase: "identity", HTTPStatus: identityResponse.Status}) + } + + query := url.Values{"sec_user_id": {identity.User.SecUID}, "count": {"20"}, "max_cursor": {"0"}} + worksResponse, err := connector.Browser.Get(ctx, worksEndpoint+"?"+query.Encode()) + if err != nil { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, + Evidence{Phase: "works", IdentityVerified: true}) + } + if state, reason := classify(worksResponse); state != "" { + return connector.stop(ctx, request.AccountID, state, reason, + Evidence{Phase: "works", HTTPStatus: worksResponse.Status, IdentityVerified: true}) + } + works, hasMore, ok := parseWorks(worksResponse.Body) + evidence := Evidence{Phase: "works", HTTPStatus: worksResponse.Status, IdentityVerified: true, WorksSeen: len(works), HasMore: hasMore} + if !ok { + return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + } + if err := connector.Store.Complete(ctx, request.AccountID, works, evidence); err != nil { + result, holdErr := connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence) + return result, errors.Join(err, holdErr) + } + return Result{State: StateSucceeded, ReasonCode: ReasonSucceeded, Evidence: evidence}, nil +} + +func (connector Connector) stop(ctx context.Context, accountID, state, reason string, evidence Evidence) (Result, error) { + result := Result{State: state, ReasonCode: reason, Evidence: evidence} + holdContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + return result, connector.Store.Hold(holdContext, accountID, state, reason, evidence) +} + +func classify(response Response) (string, string) { + switch response.Challenge { + case ChallengeCaptcha, ChallengeDevice: + return StateNeedsConfirmation, ReasonChallenge + case ChallengeNone: + default: + return StateNeedsConfirmation, ReasonUnknown + } + switch response.Status { + case http.StatusUnauthorized: + return StatePolicyHold, ReasonAuthInvalid + case http.StatusForbidden: + return StatePolicyHold, ReasonForbidden + case http.StatusTooManyRequests: + return StatePolicyHold, ReasonRateLimited + case http.StatusOK: + return "", "" + default: + return StateNeedsConfirmation, ReasonUnknown + } +} + +func parseCredential(raw []byte) ([]Cookie, error) { + if len(raw) == 0 || len(raw) > 64<<10 { + return nil, ErrInvalid + } + var bundle struct { + Cookies []Cookie `json:"cookies"` + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&bundle); err != nil || len(bundle.Cookies) == 0 || len(bundle.Cookies) > 64 { + return nil, ErrInvalid + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return nil, ErrInvalid + } + for index := range bundle.Cookies { + cookie := &bundle.Cookies[index] + cookie.Domain = strings.ToLower(strings.TrimSpace(cookie.Domain)) + if cookie.Path == "" { + cookie.Path = "/" + } + if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 || + cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") || + (cookie.Domain != "douyin.com" && !strings.HasSuffix(cookie.Domain, ".douyin.com")) || + !strings.HasPrefix(cookie.Path, "/") || strings.ContainsAny(cookie.Path, ";\r\n\x00") || + (cookie.SameSite != "" && cookie.SameSite != "Lax" && + cookie.SameSite != "Strict" && cookie.SameSite != "None") { + return nil, ErrInvalid + } + } + return bundle.Cookies, nil +} + +type identityEnvelope struct { + StatusCode *int `json:"status_code"` + User *struct { + UID string `json:"uid"` + SecUID string `json:"sec_uid"` + UniqueID string `json:"unique_id"` + } `json:"user"` +} + +func parseIdentity(body []byte) (identityEnvelope, bool) { + var identity identityEnvelope + if len(body) > 1<<20 || json.Unmarshal(body, &identity) != nil || identity.StatusCode == nil || *identity.StatusCode != 0 || identity.User == nil || + !keyPattern.MatchString(identity.User.UID) || !keyPattern.MatchString(identity.User.SecUID) || + (identity.User.UniqueID != "" && !keyPattern.MatchString(identity.User.UniqueID)) { + return identityEnvelope{}, false + } + return identity, true +} + +type worksEnvelope struct { + StatusCode *int `json:"status_code"` + HasMore *bool `json:"has_more"` + Works []struct { + ID string `json:"aweme_id"` + Description string `json:"desc"` + CreatedAt *int64 `json:"create_time"` + Statistics *struct { + DiggCount *int64 `json:"digg_count"` + CommentCount *int64 `json:"comment_count"` + ShareCount *int64 `json:"share_count"` + PlayCount *int64 `json:"play_count"` + } `json:"statistics"` + } `json:"aweme_list"` +} + +func parseWorks(body []byte) ([]Work, bool, bool) { + var envelope worksEnvelope + if len(body) > 4<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 || + envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 { + return nil, false, false + } + works := make([]Work, 0, len(envelope.Works)) + seen := make(map[string]bool, len(envelope.Works)) + for _, candidate := range envelope.Works { + if !keyPattern.MatchString(candidate.ID) || seen[candidate.ID] || candidate.CreatedAt == nil || *candidate.CreatedAt <= 0 || + candidate.Statistics == nil || candidate.Statistics.DiggCount == nil || candidate.Statistics.CommentCount == nil || + candidate.Statistics.ShareCount == nil || candidate.Statistics.PlayCount == nil || + utf8.RuneCountInString(candidate.Description) > 4096 || *candidate.Statistics.DiggCount < 0 || + *candidate.Statistics.CommentCount < 0 || *candidate.Statistics.ShareCount < 0 || *candidate.Statistics.PlayCount < 0 { + return nil, false, false + } + seen[candidate.ID] = true + works = append(works, Work{ID: candidate.ID, Description: candidate.Description, CreatedAt: *candidate.CreatedAt, + DiggCount: *candidate.Statistics.DiggCount, CommentCount: *candidate.Statistics.CommentCount, + ShareCount: *candidate.Statistics.ShareCount, PlayCount: *candidate.Statistics.PlayCount}) + } + return works, *envelope.HasMore, true +} diff --git a/internal/douyin/connector_test.go b/internal/douyin/connector_test.go new file mode 100644 index 0000000..76f48e1 --- /dev/null +++ b/internal/douyin/connector_test.go @@ -0,0 +1,241 @@ +package douyin + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +const credential = `{"cookies":[{"name":"sessionid","value":"private-session","domain":".douyin.com","path":"/","secure":true,"http_only":true,"same_site":"Lax"}]}` + +var secretReference = SecretReference{Provider: "os_keyring", Key: "creatorhub/account-a"} + +type fakeSecrets struct { + value []byte + err error +} + +func (secrets fakeSecrets) Resolve(_ context.Context, _ SecretReference) ([]byte, error) { + return secrets.value, secrets.err +} + +type fakeBrowser struct { + responses []Response + err error + cookies []Cookie + urls []string +} + +func (browser *fakeBrowser) SetCookies(_ context.Context, cookies []Cookie) error { + browser.cookies = cookies + return browser.err +} + +func (browser *fakeBrowser) Get(_ context.Context, target string) (Response, error) { + browser.urls = append(browser.urls, target) + if browser.err != nil { + return Response{}, browser.err + } + response := browser.responses[0] + browser.responses = browser.responses[1:] + return response, nil +} + +type fakeStore struct { + works []Work + holds []Result + completeCalls int + holdContextErr error + completeErr error + holdErr error +} + +func (store *fakeStore) Complete(_ context.Context, _ string, works []Work, _ Evidence) error { + store.completeCalls++ + store.works = works + return store.completeErr +} + +func (store *fakeStore) Hold(ctx context.Context, _ string, state, reason string, evidence Evidence) error { + store.holdContextErr = ctx.Err() + store.holds = append(store.holds, Result{State: state, ReasonCode: reason, Evidence: evidence}) + return store.holdErr +} + +func identityBody(uid, secUID, uniqueID string) []byte { + body, _ := json.Marshal(map[string]any{"status_code": 0, "user": map[string]string{ + "uid": uid, "sec_uid": secUID, "unique_id": uniqueID, + }}) + return body +} + +func TestSyncLogsInVerifiesIdentityAndReadsOwnWorks(t *testing.T) { + browser := &fakeBrowser{responses: []Response{ + {Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, + {Status: 200, Body: []byte(`{"status_code":0,"has_more":true,"aweme_list":[{"aweme_id":"work-1","desc":"hello","create_time":123,"statistics":{"digg_count":4,"comment_count":3,"share_count":2,"play_count":1}}]}`)}, + }} + store := &fakeStore{} + result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference, + }) + if err != nil || result.State != StateSucceeded || !result.Evidence.IdentityVerified || result.Evidence.WorksSeen != 1 || !result.Evidence.HasMore { + t.Fatalf("unexpected result: %#v err=%v", result, err) + } + if len(browser.cookies) != 1 || browser.cookies[0].Value != "private-session" || len(browser.urls) != 2 || + browser.urls[0] != identityEndpoint || !strings.Contains(browser.urls[1], "sec_user_id=sec-a") { + t.Fatalf("connector did not use the bound browser session: cookies=%#v urls=%#v", browser.cookies, browser.urls) + } + if len(store.works) != 1 || store.works[0].ID != "work-1" || store.works[0].PlayCount != 1 || len(store.holds) != 0 { + t.Fatalf("unexpected persisted works or hold: works=%#v holds=%#v", store.works, store.holds) + } + encoded, _ := json.Marshal(result) + if strings.Contains(string(encoded), "private-session") { + t.Fatalf("audit result leaked credential: %s", encoded) + } +} + +func TestSyncMapsRiskSignalsAndNeverRetries(t *testing.T) { + tests := []struct { + name string + response Response + state string + reason string + }{ + {name: "authentication invalid", response: Response{Status: 401}, state: StatePolicyHold, reason: ReasonAuthInvalid}, + {name: "forbidden", response: Response{Status: 403}, state: StatePolicyHold, reason: ReasonForbidden}, + {name: "rate limited", response: Response{Status: 429}, state: StatePolicyHold, reason: ReasonRateLimited}, + {name: "captcha", response: Response{Status: 200, Challenge: ChallengeCaptcha}, state: StateNeedsConfirmation, reason: ReasonChallenge}, + {name: "device challenge", response: Response{Status: 200, Challenge: ChallengeDevice}, state: StateNeedsConfirmation, reason: ReasonChallenge}, + {name: "unknown status", response: Response{Status: 502}, state: StateNeedsConfirmation, reason: ReasonUnknown}, + {name: "unknown challenge", response: Response{Status: 200, Challenge: "future"}, state: StateNeedsConfirmation, reason: ReasonUnknown}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + browser := &fakeBrowser{responses: []Response{test.response}} + store := &fakeStore{} + result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference, + }) + if err != nil || result.State != test.state || result.ReasonCode != test.reason || len(store.holds) != 1 { + t.Fatalf("unexpected stop: result=%#v holds=%#v err=%v", result, store.holds, err) + } + if len(browser.urls) != 1 { + t.Fatalf("risk response was retried: %#v", browser.urls) + } + }) + } +} + +func TestSyncFailsClosedOnIdentityAndUnknownResults(t *testing.T) { + tests := []struct { + name string + browser *fakeBrowser + key string + reason string + phase string + }{ + {name: "identity mismatch", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}}}, key: "another", reason: ReasonIdentityMatch, phase: "identity"}, + {name: "malformed identity", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: []byte(`{"status_code":0}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "identity"}, + {name: "identity status missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: []byte(`{"user":{"uid":"uid-a","sec_uid":"sec-a","unique_id":"handle-a"}}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "identity"}, + {name: "browser failure", browser: &fakeBrowser{err: errors.New("transport details must stay internal")}, key: "sec-a", reason: ReasonUnknown, phase: "login"}, + {name: "malformed works", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"aweme_list":[{"aweme_id":""}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "works list missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "works has_more missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"aweme_list":[]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "work create_time missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "work create_time null", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":null,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "work statistics missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "work statistics null", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1,"statistics":null}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "work statistic missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + {name: "work statistic null", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":null}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := &fakeStore{} + result, err := (Connector{Browser: test.browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: test.key, Credential: secretReference, + }) + if err != nil || result.State != StateNeedsConfirmation || result.ReasonCode != test.reason || result.Evidence.Phase != test.phase || len(store.holds) != 1 || store.completeCalls != 0 { + t.Fatalf("unexpected fail-closed result: %#v holds=%#v err=%v", result, store.holds, err) + } + }) + } +} + +func TestSyncRejectsInvalidCredentialWithoutLeakingIt(t *testing.T) { + browser := &fakeBrowser{} + store := &fakeStore{} + result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(`{"cookies":[{"name":"sessionid","value":"secret","domain":"evil.example"}]}`)}, Store: store}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference, + }) + if err != nil || result.State != StatePolicyHold || result.ReasonCode != ReasonAuthInvalid || len(browser.urls) != 0 || len(browser.cookies) != 0 || len(store.holds) != 1 { + t.Fatalf("unexpected invalid credential result: %#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err) + } + encoded, _ := json.Marshal(result) + if strings.Contains(string(encoded), "secret") || strings.Contains(string(encoded), "evil") { + t.Fatalf("stop evidence leaked credential: %s", encoded) + } +} + +func TestSyncStopsWhenSecretReferenceCannotResolve(t *testing.T) { + browser := &fakeBrowser{} + store := &fakeStore{} + result, err := (Connector{Browser: browser, Secrets: fakeSecrets{err: errors.New("secret unavailable")}, Store: store}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference, + }) + if err != nil || result.State != StatePolicyHold || result.ReasonCode != ReasonAuthInvalid || len(browser.urls) != 0 || len(store.holds) != 1 { + t.Fatalf("unavailable secret did not fail closed: result=%#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err) + } +} + +func TestSyncFailsClosedWhenPersistenceIsUnknown(t *testing.T) { + browser := &fakeBrowser{responses: []Response{ + {Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, + {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[]}`)}, + }} + store := &fakeStore{completeErr: errors.New("database result unknown")} + result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference, + }) + if err == nil || result.State != StateNeedsConfirmation || result.ReasonCode != ReasonUnknown || len(store.holds) != 1 { + t.Fatalf("persistence uncertainty did not stop: result=%#v holds=%#v err=%v", result, store.holds, err) + } +} + +func TestCredentialAndWorkValidation(t *testing.T) { + invalidCredentials := []string{ + ``, `{}`, `{"cookies":[]}`, `{"cookies":[{"name":"a","value":"b","domain":".douyin.com","extra":true}]}`, + `{"cookies":[{"name":"a;bad","value":"b","domain":".douyin.com"}]}`, + credential + `true`, credential + `[]`, credential + `null`, credential + `garbage`, + } + for _, input := range invalidCredentials { + if _, err := parseCredential([]byte(input)); !errors.Is(err, ErrInvalid) { + t.Fatalf("accepted invalid credential bundle: %q", input) + } + } + if _, _, ok := parseWorks([]byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"same","desc":"a","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}},{"aweme_id":"same","desc":"b","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}}]}`)); ok { + t.Fatal("accepted duplicate work ids") + } +} + +func TestSyncHoldsWithCancelledRequestContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + store := &fakeStore{} + result, err := (Connector{Browser: &fakeBrowser{}, Secrets: fakeSecrets{err: context.Canceled}, Store: store}).Sync(ctx, Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference, + }) + if err != nil || result.State != StatePolicyHold || len(store.holds) != 1 || store.holdContextErr != nil { + t.Fatalf("cancelled request did not durably hold: result=%#v holds=%#v context_err=%v err=%v", result, store.holds, store.holdContextErr, err) + } +} + +func TestSyncRejectsNonSecretCredentialReference(t *testing.T) { + _, err := (Connector{Browser: &fakeBrowser{}, Secrets: fakeSecrets{}, Store: &fakeStore{}}).Sync(context.Background(), Request{ + AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: SecretReference{Provider: "plain_text", Key: "raw-secret"}, + }) + if !errors.Is(err, ErrInvalid) { + t.Fatalf("accepted non-secret credential reference: %v", err) + } +}