HH-868: add fail-closed Douyin read-only connector core (#31)
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/douyin"
|
||||
"git.ipao.vip/rogee/creator-hub/internal/hub"
|
||||
)
|
||||
|
||||
type douyinGatewayBrowser struct {
|
||||
gateway hub.Gateway
|
||||
environment hub.EnvironmentContext
|
||||
}
|
||||
|
||||
type douyinGatewayRequest struct {
|
||||
BindingVersion int64 `json:"binding_version"`
|
||||
RuntimeID string `json:"runtime_id"`
|
||||
NetworkID string `json:"network_id"`
|
||||
NetworkExitID string `json:"network_exit_id"`
|
||||
Cookies []douyin.Cookie `json:"cookies,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
func (browser douyinGatewayBrowser) SetCookies(ctx context.Context, cookies []douyin.Cookie) error {
|
||||
request, err := browser.request()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Cookies = cookies
|
||||
status, _, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
|
||||
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/cookies", request, 30*time.Second)
|
||||
if err != nil || status != http.StatusNoContent {
|
||||
return errors.New("restricted browser operation failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (browser douyinGatewayBrowser) Get(ctx context.Context, target string) (douyin.Response, error) {
|
||||
request, err := browser.request()
|
||||
if err != nil {
|
||||
return douyin.Response{}, err
|
||||
}
|
||||
request.URL = target
|
||||
status, body, err := gatewayCall(ctx, browser.gateway, http.MethodPost,
|
||||
"/v1/browsers/"+url.PathEscape(browser.environment.Alias)+"/douyin/get", request, 30*time.Second)
|
||||
if err != nil || status != http.StatusOK {
|
||||
return douyin.Response{}, errors.New("restricted browser operation failed")
|
||||
}
|
||||
var response struct {
|
||||
Status int `json:"status"`
|
||||
Body string `json:"body"`
|
||||
Challenge douyin.Challenge `json:"challenge"`
|
||||
}
|
||||
if json.Unmarshal(body, &response) != nil || response.Status < 100 || response.Status > 599 ||
|
||||
(response.Challenge != douyin.ChallengeNone && response.Challenge != douyin.ChallengeCaptcha &&
|
||||
response.Challenge != douyin.ChallengeDevice) {
|
||||
return douyin.Response{}, errors.New("restricted browser returned an invalid response")
|
||||
}
|
||||
return douyin.Response{Status: response.Status, Body: []byte(response.Body), Challenge: response.Challenge}, nil
|
||||
}
|
||||
|
||||
func (browser douyinGatewayBrowser) request() (douyinGatewayRequest, error) {
|
||||
environment := browser.environment
|
||||
if browser.gateway.Endpoint == "" || browser.gateway.Token == "" || !accountRunnable(environment) ||
|
||||
!gatewayGenerationIDPattern.MatchString(environment.AccountID) || !gatewayGenerationIDPattern.MatchString(environment.Alias) ||
|
||||
!gatewayGenerationIDPattern.MatchString(environment.BindingID) ||
|
||||
!gatewayGenerationIDPattern.MatchString(environment.Exit.ID) || environment.Exit.HealthStatus != "healthy" || environment.RuntimeCleanupPending ||
|
||||
!gatewayGenerationIDPattern.MatchString(environment.RuntimeInstanceID) || !gatewayGenerationIDPattern.MatchString(environment.RuntimeID) ||
|
||||
!gatewayGenerationIDPattern.MatchString(environment.RuntimeNetworkID) ||
|
||||
environment.BindingVersion < 1 {
|
||||
return douyinGatewayRequest{}, errors.New("restricted browser is not ready")
|
||||
}
|
||||
return douyinGatewayRequest{BindingVersion: environment.BindingVersion, RuntimeID: environment.RuntimeID,
|
||||
NetworkID: environment.RuntimeNetworkID, NetworkExitID: environment.Exit.ID}, nil
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/douyin"
|
||||
"git.ipao.vip/rogee/creator-hub/internal/hub"
|
||||
)
|
||||
|
||||
const testDouyinIdentityURL = "https://www.douyin.com/aweme/v1/web/user/profile/self/"
|
||||
|
||||
func TestDouyinGatewayBrowserFencesAccountGeneration(t *testing.T) {
|
||||
requests := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
requests++
|
||||
if request.Header.Get("Authorization") != "Bearer gateway-token-1" {
|
||||
t.Fatalf("missing gateway authorization")
|
||||
}
|
||||
var body map[string]any
|
||||
if json.NewDecoder(request.Body).Decode(&body) != nil || body["binding_version"] != float64(2) ||
|
||||
body["runtime_id"] != "runtime-a" || body["network_id"] != "network-a" || body["network_exit_id"] != "exit-a" {
|
||||
t.Fatalf("generation fence missing: %#v", body)
|
||||
}
|
||||
switch request.URL.Path {
|
||||
case "/v1/browsers/account-a/douyin/cookies":
|
||||
cookies, ok := body["cookies"].([]any)
|
||||
if !ok || len(cookies) != 1 {
|
||||
t.Fatalf("cookies missing: %#v", body)
|
||||
}
|
||||
response.WriteHeader(http.StatusNoContent)
|
||||
case "/v1/browsers/account-a/douyin/get":
|
||||
if body["url"] != testDouyinIdentityURL {
|
||||
t.Fatalf("unexpected URL: %#v", body)
|
||||
}
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{"status": 412, "body": `{"captcha":true}`, "challenge": "captcha"})
|
||||
default:
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
browser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()}
|
||||
if err := browser.SetCookies(context.Background(), []douyin.Cookie{{Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := browser.Get(context.Background(), testDouyinIdentityURL)
|
||||
if err != nil || result.Status != 412 || result.Challenge != douyin.ChallengeCaptcha || requests != 2 {
|
||||
t.Fatalf("unexpected result: %#v requests=%d err=%v", result, requests, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDouyinGatewayBrowserFailsClosedWithoutReadyBinding(t *testing.T) {
|
||||
requests := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests++ }))
|
||||
defer server.Close()
|
||||
environment := readyDouyinEnvironment()
|
||||
environment.Exit.HealthStatus = "unhealthy"
|
||||
browser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: environment}
|
||||
if _, err := browser.Get(context.Background(), testDouyinIdentityURL); err == nil || requests != 0 {
|
||||
t.Fatalf("unready binding reached gateway: requests=%d err=%v", requests, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDouyinGatewayBrowserDoesNotEchoCredentialOnFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
response.WriteHeader(http.StatusBadGateway)
|
||||
_, _ = response.Write([]byte(`{"error":"private-session"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
browser := douyinGatewayBrowser{gateway: hub.Gateway{Endpoint: server.URL, Token: "gateway-token-1"}, environment: readyDouyinEnvironment()}
|
||||
err := browser.SetCookies(context.Background(), []douyin.Cookie{{Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/"}})
|
||||
if err == nil || strings.Contains(err.Error(), "private-session") {
|
||||
t.Fatalf("gateway failure leaked credential: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readyDouyinEnvironment() hub.EnvironmentContext {
|
||||
return hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Gateway: "gateway-a"}, AccountID: "account-a",
|
||||
AccountStatus: "active", AuthorizationStatus: "authorized", BindingID: "binding-a", BindingVersion: 2,
|
||||
Exit: hub.NetworkExit{ID: "exit-a", HealthStatus: "healthy"}, RuntimeInstanceID: "instance-a",
|
||||
RuntimeID: "runtime-a", RuntimeNetworkID: "network-a"}
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/douyin"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"golang.org/x/net/websocket"
|
||||
)
|
||||
|
||||
const (
|
||||
douyinOrigin = "https://www.douyin.com"
|
||||
douyinOriginURL = "https://www.douyin.com/"
|
||||
douyinIdentityPath = "/aweme/v1/web/user/profile/self/"
|
||||
douyinWorksPath = "/aweme/v1/web/aweme/post/"
|
||||
douyinResponseLimit = 1 << 20
|
||||
browserControlTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
var douyinAccountKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$`)
|
||||
|
||||
type restrictedBrowserResponse struct {
|
||||
Status int
|
||||
Body string
|
||||
Challenge douyin.Challenge
|
||||
}
|
||||
|
||||
type restrictedBrowser interface {
|
||||
SetCookies(context.Context, string, []douyin.Cookie) error
|
||||
Get(context.Context, string, string) (restrictedBrowserResponse, error)
|
||||
}
|
||||
|
||||
type douyinGenerationRequest struct {
|
||||
BindingVersion int64 `json:"binding_version"`
|
||||
RuntimeID string `json:"runtime_id"`
|
||||
NetworkID string `json:"network_id"`
|
||||
NetworkExitID string `json:"network_exit_id"`
|
||||
}
|
||||
|
||||
type douyinCookieRequest struct {
|
||||
douyinGenerationRequest
|
||||
Cookies []douyin.Cookie `json:"cookies"`
|
||||
}
|
||||
|
||||
type douyinGetRequest struct {
|
||||
douyinGenerationRequest
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
func (api gateway) setDouyinCookies(c fiber.Ctx) error {
|
||||
var input douyinCookieRequest
|
||||
if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) ||
|
||||
!validDouyinCookies(input.Cookies) {
|
||||
return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request"))
|
||||
}
|
||||
_, release, err := api.locks.acquire(c.Params("id"))
|
||||
if err != nil {
|
||||
return writeError(c, statusFor(err), err)
|
||||
}
|
||||
defer release()
|
||||
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
|
||||
return writeError(c, statusFor(err), err)
|
||||
}
|
||||
if api.browser == nil || api.browser.SetCookies(c.Context(), c.Params("id"), input.Cookies) != nil {
|
||||
return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
|
||||
}
|
||||
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
|
||||
return writeError(c, statusFor(err), err)
|
||||
}
|
||||
return c.SendStatus(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (api gateway) getDouyin(c fiber.Ctx) error {
|
||||
var input douyinGetRequest
|
||||
if !runtimeIDPattern.MatchString(c.Params("id")) || decodeRestrictedBrowserRequest(c.Body(), &input) != nil || !validDouyinGeneration(input.douyinGenerationRequest) ||
|
||||
!validDouyinURL(input.URL) {
|
||||
return writeError(c, http.StatusBadRequest, errors.New("invalid restricted browser request"))
|
||||
}
|
||||
_, release, err := api.locks.acquire(c.Params("id"))
|
||||
if err != nil {
|
||||
return writeError(c, statusFor(err), err)
|
||||
}
|
||||
defer release()
|
||||
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
|
||||
return writeError(c, statusFor(err), err)
|
||||
}
|
||||
if api.browser == nil {
|
||||
return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
|
||||
}
|
||||
response, err := api.browser.Get(c.Context(), c.Params("id"), input.URL)
|
||||
if err != nil {
|
||||
return writeError(c, http.StatusBadGateway, errors.New("restricted browser operation failed"))
|
||||
}
|
||||
if err := api.requireDouyinGeneration(c.Params("id"), input.douyinGenerationRequest); err != nil {
|
||||
return writeError(c, statusFor(err), err)
|
||||
}
|
||||
return writeJSON(c, http.StatusOK, map[string]any{
|
||||
"status": response.Status, "body": response.Body, "challenge": response.Challenge,
|
||||
})
|
||||
}
|
||||
|
||||
func decodeRestrictedBrowserRequest(body []byte, target any) error {
|
||||
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var trailing json.RawMessage
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return errors.New("request must contain one JSON value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validDouyinGeneration(input douyinGenerationRequest) bool {
|
||||
return input.BindingVersion > 0 && exitIDPattern.MatchString(input.RuntimeID) &&
|
||||
exitIDPattern.MatchString(input.NetworkID) && exitIDPattern.MatchString(input.NetworkExitID)
|
||||
}
|
||||
|
||||
func (api gateway) requireDouyinGeneration(alias string, input douyinGenerationRequest) error {
|
||||
runtimeID, labels, networks, err := api.managedContainerState(alias)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64)
|
||||
if runtimeID != input.RuntimeID || version != input.BindingVersion || labels[networkIDLabel] != input.NetworkID ||
|
||||
labels[networkExitLabel] != input.NetworkExitID {
|
||||
return errGenerationConflict
|
||||
}
|
||||
generation, _, exists, err := api.docker.inspectTenantNetwork(api.network, alias, input.BindingVersion,
|
||||
input.RuntimeID, api.self, input.NetworkID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists || !generation.RuntimeAttached || generation.SelfMember == "" || len(generation.GatewayMembers) == 0 ||
|
||||
len(networks) != 1 || networks[generation.Name] != generation.ID {
|
||||
return errGenerationConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validDouyinCookies(cookies []douyin.Cookie) bool {
|
||||
if len(cookies) == 0 || len(cookies) > 64 {
|
||||
return false
|
||||
}
|
||||
for _, cookie := range cookies {
|
||||
domain := strings.ToLower(strings.TrimSpace(cookie.Domain))
|
||||
if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 ||
|
||||
domain != cookie.Domain || cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") ||
|
||||
(domain != "douyin.com" && !strings.HasSuffix(domain, ".douyin.com")) || !strings.HasPrefix(cookie.Path, "/") ||
|
||||
strings.ContainsAny(cookie.Path, ";\r\n\x00") || (cookie.SameSite != "" && cookie.SameSite != "Lax" &&
|
||||
cookie.SameSite != "Strict" && cookie.SameSite != "None") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validDouyinURL(raw string) bool {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host != "www.douyin.com" || parsed.User != nil || parsed.Fragment != "" {
|
||||
return false
|
||||
}
|
||||
query := parsed.Query()
|
||||
switch parsed.Path {
|
||||
case douyinIdentityPath:
|
||||
return parsed.RawQuery == ""
|
||||
case douyinWorksPath:
|
||||
return len(query) == 3 && len(query["sec_user_id"]) == 1 && douyinAccountKeyPattern.MatchString(query.Get("sec_user_id")) &&
|
||||
len(query["count"]) == 1 && query.Get("count") == "20" && len(query["max_cursor"]) == 1 && query.Get("max_cursor") == "0"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type cdpBrowser struct {
|
||||
endpoint func(string) string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (browser cdpBrowser) SetCookies(ctx context.Context, alias string, cookies []douyin.Cookie) error {
|
||||
connection, err := browser.connect(ctx, alias)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer connection.Close()
|
||||
commandID := 0
|
||||
cdpCookies := make([]map[string]any, 0, len(cookies))
|
||||
for _, cookie := range cookies {
|
||||
value := map[string]any{
|
||||
"name": cookie.Name, "value": cookie.Value, "url": douyinOriginURL, "path": cookie.Path,
|
||||
"secure": cookie.Secure, "httpOnly": cookie.HTTPOnly,
|
||||
}
|
||||
if cookie.SameSite != "" {
|
||||
value["sameSite"] = cookie.SameSite
|
||||
}
|
||||
if cookie.Expires != 0 {
|
||||
value["expires"] = cookie.Expires
|
||||
}
|
||||
cdpCookies = append(cdpCookies, value)
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Network.enable", map[string]any{}, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Network.clearBrowserCookies", map[string]any{}, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Page.enable", map[string]any{}, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Page.setLifecycleEventsEnabled", map[string]bool{"enabled": true}, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
var events []cdpMessage
|
||||
var navigation struct {
|
||||
FrameID string `json:"frameId"`
|
||||
LoaderID string `json:"loaderId"`
|
||||
ErrorText string `json:"errorText"`
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Page.navigate", map[string]string{"url": douyinOriginURL}, &navigation, &events); err != nil ||
|
||||
navigation.ErrorText != "" || navigation.FrameID == "" || navigation.LoaderID == "" {
|
||||
return errors.New("restricted browser navigation failed")
|
||||
}
|
||||
if err := waitForDouyinPage(ctx, connection, &commandID, navigation.FrameID, navigation.LoaderID, events); err != nil {
|
||||
return err
|
||||
}
|
||||
return cdpCommand(connection, &commandID, "Network.setCookies", map[string]any{"cookies": cdpCookies}, nil, nil)
|
||||
}
|
||||
|
||||
func (browser cdpBrowser) Get(ctx context.Context, alias, target string) (restrictedBrowserResponse, error) {
|
||||
connection, err := browser.connect(ctx, alias)
|
||||
if err != nil {
|
||||
return restrictedBrowserResponse{}, err
|
||||
}
|
||||
defer connection.Close()
|
||||
commandID := 0
|
||||
var currentOrigin struct {
|
||||
Result struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{
|
||||
"expression": "location.origin", "returnByValue": true,
|
||||
}, ¤tOrigin, nil); err != nil || currentOrigin.Result.Value != douyinOrigin {
|
||||
return restrictedBrowserResponse{}, errors.New("restricted browser origin changed")
|
||||
}
|
||||
encodedURL, _ := json.Marshal(target)
|
||||
expression := `(async()=>{const r=await fetch(` + string(encodedURL) + `,{credentials:"include",redirect:"error"});` +
|
||||
`if(!r.body)return {status:r.status,body:"",too_large:false};const q=r.body.getReader(),d=new TextDecoder();let n=0,b="";` +
|
||||
`for(;;){const x=await q.read();if(x.done)break;if(n+x.value.byteLength>=` + strconv.Itoa(douyinResponseLimit) +
|
||||
`){await q.cancel();return {too_large:true};}n+=x.value.byteLength;b+=d.decode(x.value,{stream:true});}` +
|
||||
`b+=d.decode();return {status:r.status,body:b,too_large:false};})()`
|
||||
var evaluated struct {
|
||||
Result struct {
|
||||
Value struct {
|
||||
Status int `json:"status"`
|
||||
Body string `json:"body"`
|
||||
TooLarge bool `json:"too_large"`
|
||||
} `json:"value"`
|
||||
} `json:"result"`
|
||||
ExceptionDetails json.RawMessage `json:"exceptionDetails"`
|
||||
}
|
||||
if err := cdpCommand(connection, &commandID, "Runtime.evaluate", map[string]any{
|
||||
"expression": expression, "awaitPromise": true, "returnByValue": true,
|
||||
}, &evaluated, nil); err != nil || len(evaluated.ExceptionDetails) != 0 || evaluated.Result.Value.TooLarge ||
|
||||
evaluated.Result.Value.Status < 200 || evaluated.Result.Value.Status > 599 ||
|
||||
(evaluated.Result.Value.Status >= 300 && evaluated.Result.Value.Status < 400) {
|
||||
return restrictedBrowserResponse{}, errors.New("restricted browser fetch failed")
|
||||
}
|
||||
body := evaluated.Result.Value.Body
|
||||
return restrictedBrowserResponse{Status: evaluated.Result.Value.Status, Body: body, Challenge: detectDouyinChallenge(evaluated.Result.Value.Status, body)}, nil
|
||||
}
|
||||
|
||||
func (browser cdpBrowser) connect(ctx context.Context, alias string) (*websocket.Conn, error) {
|
||||
base := "http://" + namePrefix + alias + ":9222"
|
||||
if browser.endpoint != nil {
|
||||
base = browser.endpoint(alias)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/json/list", nil)
|
||||
if err != nil {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
client := http.Client{Timeout: browserControlTimeout}
|
||||
if browser.client != nil {
|
||||
client = *browser.client
|
||||
}
|
||||
if client.Timeout <= 0 {
|
||||
client.Timeout = browserControlTimeout
|
||||
}
|
||||
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
var targets []struct {
|
||||
Type string `json:"type"`
|
||||
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, (64<<10)+1))
|
||||
if err != nil || len(body) > 64<<10 {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||
if decoder.Decode(&targets) != nil || len(targets) > 32 {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
var trailing json.RawMessage
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
baseURL, _ := url.Parse(base)
|
||||
pageTarget := ""
|
||||
for _, target := range targets {
|
||||
if target.Type != "page" {
|
||||
continue
|
||||
}
|
||||
if pageTarget != "" {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
websocketURL, parseErr := url.Parse(target.WebSocketDebuggerURL)
|
||||
if parseErr != nil || websocketURL.Scheme != "ws" || !strings.HasPrefix(websocketURL.Path, "/devtools/page/") {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
if websocketURL.Hostname() == "localhost" || websocketURL.Hostname() == "127.0.0.1" || websocketURL.Hostname() == "::1" {
|
||||
websocketURL.Host = baseURL.Host
|
||||
}
|
||||
if websocketURL.Host != baseURL.Host {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
pageTarget = websocketURL.String()
|
||||
}
|
||||
if pageTarget == "" {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
config, err := websocket.NewConfig(pageTarget, "devtools://devtools")
|
||||
if err != nil {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
connection, err := config.DialContext(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.New("restricted browser unavailable")
|
||||
}
|
||||
deadline := time.Now().Add(browserControlTimeout)
|
||||
if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
|
||||
deadline = contextDeadline
|
||||
}
|
||||
_ = connection.SetDeadline(deadline)
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
type cdpMessage struct {
|
||||
ID int `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error json.RawMessage `json:"error"`
|
||||
}
|
||||
|
||||
func cdpCommand(connection *websocket.Conn, commandID *int, method string, parameters any, output any, events *[]cdpMessage) error {
|
||||
*commandID = *commandID + 1
|
||||
if err := websocket.JSON.Send(connection, map[string]any{"id": *commandID, "method": method, "params": parameters}); err != nil {
|
||||
return errors.New("restricted browser command failed")
|
||||
}
|
||||
for range 128 {
|
||||
var reply cdpMessage
|
||||
if err := websocket.JSON.Receive(connection, &reply); err != nil {
|
||||
return errors.New("restricted browser command failed")
|
||||
}
|
||||
if reply.ID != *commandID {
|
||||
if events != nil && reply.ID == 0 && reply.Method != "" {
|
||||
*events = append(*events, reply)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(reply.Error) != 0 || len(reply.Result) == 0 || bytes.Equal(bytes.TrimSpace(reply.Result), []byte("null")) {
|
||||
return errors.New("restricted browser command failed")
|
||||
}
|
||||
if output != nil && json.Unmarshal(reply.Result, output) != nil {
|
||||
return errors.New("restricted browser command failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("restricted browser command failed")
|
||||
}
|
||||
|
||||
func waitForDouyinPage(ctx context.Context, connection *websocket.Conn, commandID *int, frameID, loaderID string, events []cdpMessage) error {
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
|
||||
deadline = contextDeadline
|
||||
}
|
||||
_ = connection.SetReadDeadline(deadline)
|
||||
for attempts := 0; attempts < 256; attempts++ {
|
||||
var event cdpMessage
|
||||
if len(events) != 0 {
|
||||
event, events = events[0], events[1:]
|
||||
} else if websocket.JSON.Receive(connection, &event) != nil {
|
||||
return errors.New("restricted browser navigation failed")
|
||||
}
|
||||
if event.Method != "Page.lifecycleEvent" {
|
||||
continue
|
||||
}
|
||||
var lifecycle struct {
|
||||
FrameID string `json:"frameId"`
|
||||
LoaderID string `json:"loaderId"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if json.Unmarshal(event.Params, &lifecycle) != nil || lifecycle.FrameID != frameID || lifecycle.LoaderID != loaderID || lifecycle.Name != "load" {
|
||||
continue
|
||||
}
|
||||
var evaluated struct {
|
||||
Result struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := cdpCommand(connection, commandID, "Runtime.evaluate", map[string]any{
|
||||
"expression": "location.origin", "returnByValue": true,
|
||||
}, &evaluated, nil); err != nil || evaluated.Result.Value != douyinOrigin {
|
||||
return errors.New("restricted browser navigation failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("restricted browser navigation failed")
|
||||
}
|
||||
|
||||
func detectDouyinChallenge(status int, body string) douyin.Challenge {
|
||||
lower := strings.ToLower(body)
|
||||
if status == http.StatusPreconditionFailed || strings.Contains(lower, "captcha") || strings.Contains(lower, "verify_center_decision_conf") {
|
||||
return douyin.ChallengeCaptcha
|
||||
}
|
||||
if strings.Contains(lower, "device_challenge") || strings.Contains(lower, "device verification") {
|
||||
return douyin.ChallengeDevice
|
||||
}
|
||||
return douyin.ChallengeNone
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/douyin"
|
||||
)
|
||||
|
||||
func TestCDPBrowserRealNavigationIsolation(t *testing.T) {
|
||||
if os.Getenv("CREATORHUB_REAL_CDP_TEST") != "1" {
|
||||
t.Skip("set CREATORHUB_REAL_CDP_TEST=1 to run against local Chrome")
|
||||
}
|
||||
chrome, err := exec.LookPath("google-chrome")
|
||||
if err != nil {
|
||||
t.Skip("google-chrome is unavailable")
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
phase := "redirect"
|
||||
release := make(chan struct{})
|
||||
mainRequests := make(chan string, 4)
|
||||
loginRequests := make(chan string, 1)
|
||||
subdomainRequests := make(chan string, 1)
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
host := request.Host
|
||||
if name, _, splitErr := net.SplitHostPort(host); splitErr == nil {
|
||||
host = name
|
||||
}
|
||||
switch host {
|
||||
case "login.douyin.com":
|
||||
loginRequests <- request.Header.Get("Cookie")
|
||||
_, _ = response.Write([]byte("redirected"))
|
||||
case "api.douyin.com":
|
||||
subdomainRequests <- request.Header.Get("Cookie")
|
||||
_, _ = response.Write([]byte("ok"))
|
||||
case "www.douyin.com":
|
||||
if request.URL.Path == "/" {
|
||||
mainRequests <- request.Header.Get("Cookie")
|
||||
}
|
||||
mu.Lock()
|
||||
currentPhase, currentRelease := phase, release
|
||||
mu.Unlock()
|
||||
if currentPhase == "redirect" {
|
||||
http.Redirect(response, request, "https://login.douyin.com/landing", http.StatusFound)
|
||||
return
|
||||
}
|
||||
if currentPhase == "blocked" {
|
||||
<-currentRelease
|
||||
}
|
||||
response.Header().Set("Content-Type", "text/html")
|
||||
_, _ = response.Write([]byte(`<script>addEventListener("load",()=>setTimeout(()=>fetch("https://api.douyin.com/probe"),300))</script>`))
|
||||
default:
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
allowedHosts := map[string]bool{"www.douyin.com": true, "login.douyin.com": true, "api.douyin.com": true}
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
host := request.Host
|
||||
if name, _, splitErr := net.SplitHostPort(host); splitErr == nil {
|
||||
host = name
|
||||
}
|
||||
if request.Method != http.MethodConnect || !allowedHosts[host] {
|
||||
response.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
upstream, dialErr := net.Dial("tcp", server.Listener.Addr().String())
|
||||
if dialErr != nil {
|
||||
response.WriteHeader(http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
client, _, hijackErr := response.(http.Hijacker).Hijack()
|
||||
if hijackErr != nil {
|
||||
upstream.Close()
|
||||
return
|
||||
}
|
||||
_, _ = client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n"))
|
||||
go func() {
|
||||
_, _ = io.Copy(upstream, client)
|
||||
_ = upstream.Close()
|
||||
}()
|
||||
_, _ = io.Copy(client, upstream)
|
||||
_ = client.Close()
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
|
||||
profile := t.TempDir()
|
||||
command := exec.Command(chrome, "--headless=new", "--no-sandbox", "--disable-gpu", "--disable-background-networking", "--disable-quic",
|
||||
"--disable-dev-shm-usage", "--no-first-run", "--no-default-browser-check", "--password-store=basic", "--use-mock-keychain",
|
||||
"--ignore-certificate-errors", "--proxy-server="+proxy.URL,
|
||||
"--remote-debugging-address=127.0.0.1", "--remote-debugging-port=0", "--remote-allow-origins=*",
|
||||
"--user-data-dir="+profile, "about:blank")
|
||||
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
if err := command.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL)
|
||||
_ = command.Wait()
|
||||
})
|
||||
|
||||
var debugPort string
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(25 * time.Millisecond) {
|
||||
content, readErr := os.ReadFile(filepath.Join(profile, "DevToolsActivePort"))
|
||||
if readErr == nil {
|
||||
debugPort = strings.SplitN(string(content), "\n", 2)[0]
|
||||
break
|
||||
}
|
||||
}
|
||||
if debugPort == "" {
|
||||
t.Fatal("Chrome did not expose a DevTools port")
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
browser := cdpBrowser{endpoint: func(string) string { return "http://127.0.0.1:" + debugPort }}
|
||||
cookie := []douyin.Cookie{{Name: "sessionid", Value: "fake-secret", Domain: ".douyin.com", Path: "/", Secure: true}}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
redirectErr := browser.SetCookies(ctx, "account-a", cookie)
|
||||
if redirectErr == nil {
|
||||
cancel()
|
||||
t.Fatal("accepted a redirected navigation")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case got := <-loginRequests:
|
||||
if strings.Contains(got, "fake-secret") {
|
||||
t.Fatal("cookie leaked during redirected navigation")
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("redirect target was not reached: %v", redirectErr)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
phase = "success"
|
||||
mu.Unlock()
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
|
||||
if err := browser.SetCookies(ctx, "account-a", cookie); err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-mainRequests:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("successful navigation was not observed")
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
phase = "blocked"
|
||||
release = make(chan struct{})
|
||||
currentRelease := release
|
||||
mu.Unlock()
|
||||
defer func() {
|
||||
select {
|
||||
case <-currentRelease:
|
||||
default:
|
||||
close(currentRelease)
|
||||
}
|
||||
}()
|
||||
done := make(chan error, 1)
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
|
||||
go func() { done <- browser.SetCookies(ctx, "account-a", cookie) }()
|
||||
select {
|
||||
case got := <-mainRequests:
|
||||
if strings.Contains(got, "fake-secret") {
|
||||
cancel()
|
||||
t.Fatal("cookie leaked during navigation")
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
cancel()
|
||||
t.Fatal("blocked navigation was not observed")
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
cancel()
|
||||
t.Fatalf("navigation completed before its loader: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
close(currentRelease)
|
||||
if err := <-done; err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case got := <-subdomainRequests:
|
||||
if strings.Contains(got, "fake-secret") {
|
||||
t.Fatal("host-only cookie leaked to a subdomain")
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("subdomain probe did not run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.ipao.vip/rogee/creator-hub/internal/douyin"
|
||||
"golang.org/x/net/websocket"
|
||||
)
|
||||
|
||||
type fakeRestrictedBrowser struct {
|
||||
cookies []douyin.Cookie
|
||||
urls []string
|
||||
response restrictedBrowserResponse
|
||||
after func()
|
||||
}
|
||||
|
||||
func (browser *fakeRestrictedBrowser) SetCookies(_ context.Context, _ string, cookies []douyin.Cookie) error {
|
||||
browser.cookies = cookies
|
||||
if browser.after != nil {
|
||||
browser.after()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (browser *fakeRestrictedBrowser) Get(_ context.Context, _ string, target string) (restrictedBrowserResponse, error) {
|
||||
browser.urls = append(browser.urls, target)
|
||||
if browser.after != nil {
|
||||
browser.after()
|
||||
}
|
||||
return browser.response, nil
|
||||
}
|
||||
|
||||
func TestGatewayRestrictedDouyinContract(t *testing.T) {
|
||||
labels := map[string]string{
|
||||
managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "2", networkIDLabel: "network-a", networkExitLabel: "exit-a",
|
||||
}
|
||||
self, _ := os.Hostname()
|
||||
var stateMu sync.Mutex
|
||||
containerNetworks := map[string]string{"creatorhub_browser-account-a": "network-a"}
|
||||
runtimeAttached := true
|
||||
server := httptest.NewServer(withAliasReservations(self, func(response http.ResponseWriter, request *http.Request) {
|
||||
switch request.URL.Path {
|
||||
case "/containers/" + namePrefix + "account-a/json":
|
||||
stateMu.Lock()
|
||||
labelCopy, networkCopy := map[string]string{}, map[string]any{}
|
||||
for key, value := range labels {
|
||||
labelCopy[key] = value
|
||||
}
|
||||
for name, id := range containerNetworks {
|
||||
networkCopy[name] = map[string]string{"NetworkID": id}
|
||||
}
|
||||
stateMu.Unlock()
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{"Id": "runtime-a", "Config": map[string]any{"Labels": labelCopy},
|
||||
"NetworkSettings": map[string]any{"Networks": networkCopy}})
|
||||
case "/networks/network-a":
|
||||
stateMu.Lock()
|
||||
members := map[string]any{self: map[string]string{"Name": self, "IPv4Address": "127.0.0.1/8"}}
|
||||
if runtimeAttached {
|
||||
members["runtime-a"] = map[string]string{"Name": namePrefix + "account-a", "IPv4Address": "127.0.0.2/8"}
|
||||
}
|
||||
stateMu.Unlock()
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{
|
||||
"Id": "network-a", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false,
|
||||
"Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "2"},
|
||||
"Containers": members,
|
||||
})
|
||||
default:
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}
|
||||
browser := &fakeRestrictedBrowser{response: restrictedBrowserResponse{Status: 200, Body: `{"status_code":0}`, Challenge: douyin.ChallengeNone}}
|
||||
app := newGatewayWithBrowser(docker, "creatorhub_browser", testToken, self, browser)
|
||||
generation := `"binding_version":2,"runtime_id":"runtime-a","network_id":"network-a","network_exit_id":"exit-a"`
|
||||
|
||||
cookieBody := `{` + generation + `,"cookies":[{"name":"sessionid","value":"private-session","domain":".douyin.com","path":"/"}]}`
|
||||
response, err := app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/cookies", strings.NewReader(cookieBody)))
|
||||
if err != nil || response.StatusCode != http.StatusNoContent || len(browser.cookies) != 1 || browser.cookies[0].Value != "private-session" {
|
||||
t.Fatalf("set cookies failed: status=%d cookies=%#v err=%v", response.StatusCode, browser.cookies, err)
|
||||
}
|
||||
response.Body.Close()
|
||||
|
||||
identityURL := "https://www.douyin.com" + douyinIdentityPath
|
||||
getBody := `{` + generation + `,"url":"` + identityURL + `"}`
|
||||
response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody)))
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if err != nil || response.StatusCode != http.StatusOK || len(browser.urls) != 1 || browser.urls[0] != identityURL ||
|
||||
!strings.Contains(string(body), `\"status_code\":0`) || strings.Contains(string(body), "private-session") {
|
||||
t.Fatalf("get failed or leaked cookies: status=%d urls=%#v body=%s err=%v", response.StatusCode, browser.urls, body, err)
|
||||
}
|
||||
|
||||
for name, request := range map[string]*http.Request{
|
||||
"unauthenticated": httptest.NewRequest(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody)),
|
||||
"invalid account": authed(http.MethodPost, "/v1/browsers/AccountA/douyin/get", strings.NewReader(getBody)),
|
||||
"generic URL": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(`{`+generation+`,"url":"https://example.com/"}`)),
|
||||
"generic CDP": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(`{`+generation+`,"url":"`+identityURL+`","method":"Runtime.evaluate"}`)),
|
||||
"trailing null": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody+`null`)),
|
||||
"stale generation": authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(`{"binding_version":1,"runtime_id":"runtime-a","network_id":"network-a","network_exit_id":"exit-a","url":"`+identityURL+`"}`)),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
before := len(browser.urls)
|
||||
response, err := app.Test(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response.Body.Close()
|
||||
want := http.StatusBadRequest
|
||||
if name == "unauthenticated" {
|
||||
want = http.StatusUnauthorized
|
||||
} else if name == "stale generation" {
|
||||
want = http.StatusConflict
|
||||
}
|
||||
if response.StatusCode != want || len(browser.urls) != before {
|
||||
t.Fatalf("status=%d want=%d calls=%d want=%d", response.StatusCode, want, len(browser.urls), before)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
stateMu.Lock()
|
||||
runtimeAttached = false
|
||||
stateMu.Unlock()
|
||||
before := len(browser.urls)
|
||||
response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/get", strings.NewReader(getBody)))
|
||||
if err != nil || response.StatusCode != http.StatusConflict || len(browser.urls) != before {
|
||||
t.Fatalf("wrong network membership reached browser: status=%d calls=%d want=%d err=%v", response.StatusCode, len(browser.urls), before, err)
|
||||
}
|
||||
response.Body.Close()
|
||||
stateMu.Lock()
|
||||
runtimeAttached = true
|
||||
stateMu.Unlock()
|
||||
|
||||
browser.after = func() {
|
||||
stateMu.Lock()
|
||||
containerNetworks["other-tenant"] = "network-b"
|
||||
stateMu.Unlock()
|
||||
}
|
||||
response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/cookies", strings.NewReader(cookieBody)))
|
||||
if err != nil || response.StatusCode != http.StatusConflict {
|
||||
t.Fatalf("post-operation cross-network attachment was accepted: status=%d err=%v", response.StatusCode, err)
|
||||
}
|
||||
response.Body.Close()
|
||||
stateMu.Lock()
|
||||
delete(containerNetworks, "other-tenant")
|
||||
stateMu.Unlock()
|
||||
|
||||
browser.after = func() {
|
||||
stateMu.Lock()
|
||||
labels[networkIDLabel] = "network-replaced"
|
||||
stateMu.Unlock()
|
||||
}
|
||||
response, err = app.Test(authed(http.MethodPost, "/v1/browsers/account-a/douyin/cookies", strings.NewReader(cookieBody)))
|
||||
if err != nil || response.StatusCode != http.StatusConflict {
|
||||
t.Fatalf("post-operation generation replacement was accepted: status=%d err=%v", response.StatusCode, err)
|
||||
}
|
||||
response.Body.Close()
|
||||
}
|
||||
|
||||
func TestCDPBrowserUsesOnlyNarrowCommands(t *testing.T) {
|
||||
var server *httptest.Server
|
||||
var mu sync.Mutex
|
||||
methods := []string{}
|
||||
secretSeen := false
|
||||
cookieNames := []string{}
|
||||
cookiesHostOnly := true
|
||||
setCookieCalls := 0
|
||||
pageOrigin := douyinOrigin
|
||||
onlyOldLoader := false
|
||||
fetchMode := "ok"
|
||||
fetchExpression := ""
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/json/list", func(response http.ResponseWriter, request *http.Request) {
|
||||
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/devtools/page/one"
|
||||
_ = json.NewEncoder(response).Encode([]map[string]string{{"type": "page", "webSocketDebuggerUrl": wsURL}})
|
||||
})
|
||||
mux.Handle("/devtools/page/one", websocket.Server{
|
||||
Handshake: func(*websocket.Config, *http.Request) error { return nil },
|
||||
Handler: func(connection *websocket.Conn) {
|
||||
for {
|
||||
var command struct {
|
||||
ID int `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if websocket.JSON.Receive(connection, &command) != nil {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
methods = append(methods, command.Method)
|
||||
secretSeen = secretSeen || strings.Contains(string(command.Params), "private-session")
|
||||
result := any(map[string]any{})
|
||||
var beforeReply, afterReply []map[string]any
|
||||
switch command.Method {
|
||||
case "Network.clearBrowserCookies":
|
||||
cookieNames = nil
|
||||
case "Network.setCookies":
|
||||
var params struct {
|
||||
Cookies []map[string]any `json:"cookies"`
|
||||
}
|
||||
_ = json.Unmarshal(command.Params, ¶ms)
|
||||
setCookieCalls++
|
||||
cookieNames = cookieNames[:0]
|
||||
for _, cookie := range params.Cookies {
|
||||
cookieNames = append(cookieNames, cookie["name"].(string))
|
||||
_, hasDomain := cookie["domain"]
|
||||
cookiesHostOnly = cookiesHostOnly && !hasDomain && cookie["url"] == douyinOriginURL
|
||||
}
|
||||
case "Page.navigate":
|
||||
result = map[string]any{"frameId": "frame-new", "loaderId": "loader-new"}
|
||||
beforeReply = append(beforeReply, map[string]any{"method": "Page.lifecycleEvent", "params": map[string]any{
|
||||
"frameId": "frame-old", "loaderId": "loader-old", "name": "load",
|
||||
}})
|
||||
if !onlyOldLoader {
|
||||
afterReply = append(afterReply, map[string]any{"method": "Page.lifecycleEvent", "params": map[string]any{
|
||||
"frameId": "frame-new", "loaderId": "loader-new", "name": "load",
|
||||
}})
|
||||
}
|
||||
case "Runtime.evaluate":
|
||||
var params struct {
|
||||
Expression string `json:"expression"`
|
||||
}
|
||||
_ = json.Unmarshal(command.Params, ¶ms)
|
||||
if params.Expression == "location.origin" {
|
||||
result = map[string]any{"result": map[string]any{"value": pageOrigin}}
|
||||
} else {
|
||||
fetchExpression = params.Expression
|
||||
value := map[string]any{"status": 412, "body": `{"captcha":true}`, "too_large": false}
|
||||
if fetchMode == "redirect" {
|
||||
value = map[string]any{"status": 302, "body": "", "too_large": false}
|
||||
} else if fetchMode == "too_large" {
|
||||
value = map[string]any{"too_large": true}
|
||||
}
|
||||
result = map[string]any{"result": map[string]any{"value": map[string]any{
|
||||
"status": value["status"], "body": value["body"], "too_large": value["too_large"],
|
||||
}}}
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
for _, event := range beforeReply {
|
||||
_ = websocket.JSON.Send(connection, event)
|
||||
}
|
||||
_ = websocket.JSON.Send(connection, map[string]any{"id": command.ID, "result": result})
|
||||
for _, event := range afterReply {
|
||||
_ = websocket.JSON.Send(connection, event)
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
server = httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
browser := cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()}
|
||||
if err := browser.SetCookies(context.Background(), "account-a", []douyin.Cookie{{
|
||||
Name: "old_auth", Value: "old-session", Domain: ".douyin.com", Path: "/",
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := browser.SetCookies(context.Background(), "account-a", []douyin.Cookie{{
|
||||
Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/",
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath)
|
||||
if err != nil || result.Status != 412 || result.Challenge != douyin.ChallengeCaptcha {
|
||||
t.Fatalf("unexpected CDP response: %#v err=%v", result, err)
|
||||
}
|
||||
mu.Lock()
|
||||
if !secretSeen || !cookiesHostOnly || strings.Join(cookieNames, ",") != "sessionid" ||
|
||||
strings.Join(methods, ",") != "Network.enable,Network.clearBrowserCookies,Page.enable,Page.setLifecycleEventsEnabled,Page.navigate,Runtime.evaluate,Network.setCookies,"+
|
||||
"Network.enable,Network.clearBrowserCookies,Page.enable,Page.setLifecycleEventsEnabled,Page.navigate,Runtime.evaluate,Network.setCookies,Runtime.evaluate,Runtime.evaluate" ||
|
||||
!strings.Contains(fetchExpression, `redirect:"error"`) || !strings.Contains(fetchExpression, "getReader()") ||
|
||||
!strings.Contains(fetchExpression, "q.cancel()") || !strings.Contains(fetchExpression, ">=1048576") || strings.Contains(fetchExpression, "r.text()") {
|
||||
mu.Unlock()
|
||||
t.Fatalf("unexpected CDP contract: methods=%#v cookies=%#v secret_seen=%v expression=%s", methods, cookieNames, secretSeen, fetchExpression)
|
||||
}
|
||||
pageOrigin = "https://login.douyin.com"
|
||||
setCookiesBeforeRedirect := setCookieCalls
|
||||
mu.Unlock()
|
||||
if err := browser.SetCookies(context.Background(), "account-a", []douyin.Cookie{{
|
||||
Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/",
|
||||
}}); err == nil {
|
||||
t.Fatal("accepted navigation redirected to a Douyin subdomain")
|
||||
}
|
||||
mu.Lock()
|
||||
if setCookieCalls != setCookiesBeforeRedirect {
|
||||
mu.Unlock()
|
||||
t.Fatal("set cookies before rejecting redirected navigation")
|
||||
}
|
||||
pageOrigin, fetchMode = douyinOrigin, "redirect"
|
||||
mu.Unlock()
|
||||
if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil {
|
||||
t.Fatal("accepted a redirected fetch")
|
||||
}
|
||||
mu.Lock()
|
||||
fetchMode = "too_large"
|
||||
mu.Unlock()
|
||||
if _, err := browser.Get(context.Background(), "account-a", "https://www.douyin.com"+douyinIdentityPath); err == nil {
|
||||
t.Fatal("accepted a response at the 1 MiB limit")
|
||||
}
|
||||
mu.Lock()
|
||||
onlyOldLoader = true
|
||||
setCookiesBeforeOldLoader := setCookieCalls
|
||||
mu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := browser.SetCookies(ctx, "account-a", []douyin.Cookie{{
|
||||
Name: "sessionid", Value: "private-session", Domain: ".douyin.com", Path: "/",
|
||||
}}); err == nil {
|
||||
t.Fatal("accepted an old page load event for the new navigation")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if setCookieCalls != setCookiesBeforeOldLoader {
|
||||
t.Fatal("set cookies before the new loader completed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDPDiscoveryDoesNotFollowRedirects(t *testing.T) {
|
||||
redirected := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path == "/json/list" {
|
||||
http.Redirect(response, request, "/redirected", http.StatusFound)
|
||||
return
|
||||
}
|
||||
redirected++
|
||||
response.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
browser := cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()}
|
||||
if _, err := browser.connect(context.Background(), "account-a"); err == nil || redirected != 0 {
|
||||
t.Fatalf("discovery redirect was followed: redirected=%d err=%v", redirected, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDPDiscoveryRequiresOnePage(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
_ = json.NewEncoder(response).Encode([]map[string]string{
|
||||
{"type": "page", "webSocketDebuggerUrl": "ws://localhost/devtools/page/one"},
|
||||
{"type": "page", "webSocketDebuggerUrl": "ws://localhost/devtools/page/two"},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
browser := cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()}
|
||||
if _, err := browser.connect(context.Background(), "account-a"); err == nil {
|
||||
t.Fatal("accepted a profile with multiple page targets")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDPDiscoveryRequiresOneJSONValue(t *testing.T) {
|
||||
for name, suffix := range map[string]string{
|
||||
"null": "null", "other value": `{}`, "garbage": "garbage", "oversized": strings.Repeat(" ", 64<<10), "whitespace": " \n\t",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
websocketAttempts := 0
|
||||
var server *httptest.Server
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/json/list", func(response http.ResponseWriter, request *http.Request) {
|
||||
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/devtools/page/one"
|
||||
_, _ = response.Write([]byte(`[{"type":"page","webSocketDebuggerUrl":"` + wsURL + `"}]` + suffix))
|
||||
})
|
||||
mux.Handle("/devtools/page/one", websocket.Server{
|
||||
Handshake: func(*websocket.Config, *http.Request) error {
|
||||
websocketAttempts++
|
||||
return nil
|
||||
},
|
||||
Handler: func(connection *websocket.Conn) {},
|
||||
})
|
||||
server = httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
connection, err := (cdpBrowser{endpoint: func(string) string { return server.URL }, client: server.Client()}).connect(context.Background(), "account-a")
|
||||
if connection != nil {
|
||||
connection.Close()
|
||||
}
|
||||
valid := name == "whitespace"
|
||||
wantAttempts := 0
|
||||
if valid {
|
||||
wantAttempts = 1
|
||||
}
|
||||
if (err == nil) != valid || websocketAttempts != wantAttempts {
|
||||
t.Fatalf("err=%v websocket attempts=%d", err, websocketAttempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDouyinURLContract(t *testing.T) {
|
||||
for target, want := range map[string]bool{
|
||||
"https://www.douyin.com" + douyinIdentityPath: true,
|
||||
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0": true,
|
||||
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=1": false,
|
||||
"https://www.douyin.com" + douyinWorksPath + "?sec_user_id=sec-a&count=20&max_cursor=0&method=publish": false,
|
||||
"https://www.douyin.com/aweme/v1/web/commit/item/": false,
|
||||
} {
|
||||
if got := validDouyinURL(target); got != want {
|
||||
t.Fatalf("validDouyinURL(%q)=%v want=%v", target, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -113,6 +113,7 @@ type gateway struct {
|
||||
network string
|
||||
self string
|
||||
token string
|
||||
browser restrictedBrowser
|
||||
proxies *memoryProxyRegistry
|
||||
locks *dockerAliasReservations
|
||||
}
|
||||
@@ -276,8 +277,12 @@ func newGateway(client dockerClient, network, token string) *fiber.App {
|
||||
}
|
||||
|
||||
func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber.App {
|
||||
return newGatewayWithBrowser(client, network, token, self, cdpBrowser{})
|
||||
}
|
||||
|
||||
func newGatewayWithBrowser(client dockerClient, network, token, self string, browser restrictedBrowser) *fiber.App {
|
||||
api := gateway{docker: client, network: network, self: self, token: token, proxies: newMemoryProxyRegistry(),
|
||||
locks: &dockerAliasReservations{docker: client, self: self}}
|
||||
locks: &dockerAliasReservations{docker: client, self: self}, browser: browser}
|
||||
app := fiber.New(fiber.Config{
|
||||
AppName: "CreatorHub Docker gateway",
|
||||
BodyLimit: 1 << 20,
|
||||
@@ -300,6 +305,8 @@ func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber
|
||||
app.Get("/v1/browsers", api.list)
|
||||
app.Post("/v1/browsers", api.create)
|
||||
app.Post("/v1/browsers/:id/proxy", api.restoreProxy)
|
||||
app.Post("/v1/browsers/:id/douyin/cookies", api.setDouyinCookies)
|
||||
app.Post("/v1/browsers/:id/douyin/get", api.getDouyin)
|
||||
app.Post("/v1/browsers/:id/:action", api.changeState)
|
||||
app.Delete("/v1/browsers/:id", api.remove)
|
||||
return app
|
||||
@@ -901,33 +908,47 @@ func (api gateway) requireGeneration(id string, input generationRequest) (string
|
||||
}
|
||||
|
||||
func (api gateway) managedContainer(id string) (string, map[string]string, error) {
|
||||
runtimeID, labels, _, err := api.managedContainerState(id)
|
||||
return runtimeID, labels, err
|
||||
}
|
||||
|
||||
func (api gateway) managedContainerState(id string) (string, map[string]string, map[string]string, error) {
|
||||
if !runtimeIDPattern.MatchString(id) {
|
||||
return "", nil, errInvalidRuntimeID
|
||||
return "", nil, nil, errInvalidRuntimeID
|
||||
}
|
||||
result, err := api.docker.request(http.MethodGet, "/containers/"+url.PathEscape(namePrefix+id)+"/json", nil)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
return "", nil, nil, err
|
||||
}
|
||||
defer result.Body.Close()
|
||||
if result.StatusCode == http.StatusNotFound {
|
||||
return "", nil, os.ErrNotExist
|
||||
return "", nil, nil, os.ErrNotExist
|
||||
}
|
||||
if result.StatusCode != http.StatusOK {
|
||||
return "", nil, fmt.Errorf("Docker inspect returned %s", result.Status)
|
||||
return "", nil, nil, fmt.Errorf("Docker inspect returned %s", result.Status)
|
||||
}
|
||||
var inspected struct {
|
||||
ID string `json:"Id"`
|
||||
Config struct {
|
||||
Labels map[string]string `json:"Labels"`
|
||||
} `json:"Config"`
|
||||
NetworkSettings struct {
|
||||
Networks map[string]struct {
|
||||
NetworkID string `json:"NetworkID"`
|
||||
} `json:"Networks"`
|
||||
} `json:"NetworkSettings"`
|
||||
}
|
||||
if err := json.NewDecoder(result.Body).Decode(&inspected); err != nil {
|
||||
return "", nil, fmt.Errorf("decode Docker inspect: %w", err)
|
||||
return "", nil, nil, fmt.Errorf("decode Docker inspect: %w", err)
|
||||
}
|
||||
if inspected.Config.Labels[managedLabel] != "true" || inspected.Config.Labels[idLabel] != id {
|
||||
return "", nil, errUnmanagedContainer
|
||||
return "", nil, nil, errUnmanagedContainer
|
||||
}
|
||||
return inspected.ID, inspected.Config.Labels, nil
|
||||
networks := make(map[string]string, len(inspected.NetworkSettings.Networks))
|
||||
for name, network := range inspected.NetworkSettings.Networks {
|
||||
networks[name] = network.NetworkID
|
||||
}
|
||||
return inspected.ID, inspected.Config.Labels, networks, nil
|
||||
}
|
||||
|
||||
// pullIfMissing 在镜像不在本地时从远端仓库拉取；镜像缺失属于可恢复错误，调用方可直接重试。
|
||||
|
||||
@@ -8,6 +8,7 @@ require (
|
||||
github.com/sirupsen/logrus v1.10.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
golang.org/x/net v0.57.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -37,7 +38,6 @@ require (
|
||||
github.com/valyala/fasthttp v1.73.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
package douyin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
StateSucceeded = "succeeded"
|
||||
StatePolicyHold = "policy_hold"
|
||||
StateNeedsConfirmation = "needs_confirmation"
|
||||
|
||||
ReasonAuthInvalid = "douyin_auth_invalid"
|
||||
ReasonForbidden = "douyin_forbidden"
|
||||
ReasonRateLimited = "douyin_rate_limited"
|
||||
ReasonChallenge = "douyin_challenge"
|
||||
ReasonUnknown = "douyin_result_unknown"
|
||||
ReasonIdentityMatch = "douyin_identity_mismatch"
|
||||
ReasonSucceeded = "douyin_sync_succeeded"
|
||||
|
||||
identityEndpoint = "https://www.douyin.com/aweme/v1/web/user/profile/self/"
|
||||
worksEndpoint = "https://www.douyin.com/aweme/v1/web/aweme/post/"
|
||||
)
|
||||
|
||||
var (
|
||||
keyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$`)
|
||||
credentialKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._/-]{0,126}$`)
|
||||
)
|
||||
|
||||
var ErrInvalid = errors.New("invalid douyin connector input")
|
||||
|
||||
type Challenge string
|
||||
|
||||
const (
|
||||
ChallengeNone Challenge = ""
|
||||
ChallengeCaptcha Challenge = "captcha"
|
||||
ChallengeDevice Challenge = "device"
|
||||
)
|
||||
|
||||
type Cookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
Secure bool `json:"secure,omitempty"`
|
||||
HTTPOnly bool `json:"http_only,omitempty"`
|
||||
SameSite string `json:"same_site,omitempty"`
|
||||
Expires float64 `json:"expires,omitempty"`
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Status int
|
||||
Body []byte
|
||||
Challenge Challenge
|
||||
}
|
||||
|
||||
// Browser is the deliberately narrow contract the restricted browser control
|
||||
// plane must implement. It does not permit arbitrary CDP commands.
|
||||
type Browser interface {
|
||||
SetCookies(context.Context, []Cookie) error
|
||||
Get(context.Context, string) (Response, error)
|
||||
}
|
||||
|
||||
type SecretReference struct {
|
||||
Provider string
|
||||
Key string
|
||||
}
|
||||
|
||||
type SecretResolver interface {
|
||||
Resolve(context.Context, SecretReference) ([]byte, error)
|
||||
}
|
||||
|
||||
type Work struct {
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
DiggCount int64 `json:"digg_count"`
|
||||
CommentCount int64 `json:"comment_count"`
|
||||
ShareCount int64 `json:"share_count"`
|
||||
PlayCount int64 `json:"play_count"`
|
||||
}
|
||||
|
||||
type Evidence struct {
|
||||
Phase string `json:"phase"`
|
||||
HTTPStatus int `json:"http_status,omitempty"`
|
||||
IdentityVerified bool `json:"identity_verified"`
|
||||
WorksSeen int `json:"works_seen,omitempty"`
|
||||
HasMore bool `json:"has_more,omitempty"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
State string `json:"state"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Evidence Evidence `json:"evidence"`
|
||||
}
|
||||
|
||||
// Store owns both persistence/audit and fail-closed account/runtime handling.
|
||||
// Hold must pause the account and stop its existing bound runtime without retry.
|
||||
type Store interface {
|
||||
Complete(context.Context, string, []Work, Evidence) error
|
||||
Hold(context.Context, string, string, string, Evidence) error
|
||||
}
|
||||
|
||||
type Connector struct {
|
||||
Browser Browser
|
||||
Secrets SecretResolver
|
||||
Store Store
|
||||
}
|
||||
|
||||
type Request struct {
|
||||
AccountID string
|
||||
PlatformAccountKey string
|
||||
Credential SecretReference
|
||||
}
|
||||
|
||||
func (connector Connector) Sync(ctx context.Context, request Request) (Result, error) {
|
||||
if connector.Browser == nil || connector.Secrets == nil || connector.Store == nil || !keyPattern.MatchString(request.AccountID) ||
|
||||
!keyPattern.MatchString(request.PlatformAccountKey) ||
|
||||
(request.Credential.Provider != "os_keyring" && request.Credential.Provider != "secret_manager") ||
|
||||
!credentialKeyPattern.MatchString(request.Credential.Key) {
|
||||
return Result{}, ErrInvalid
|
||||
}
|
||||
credential, err := connector.Secrets.Resolve(ctx, request.Credential)
|
||||
if err != nil {
|
||||
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
|
||||
}
|
||||
cookies, err := parseCredential(credential)
|
||||
if err != nil {
|
||||
return connector.stop(ctx, request.AccountID, StatePolicyHold, ReasonAuthInvalid, Evidence{Phase: "login"})
|
||||
}
|
||||
if err := connector.Browser.SetCookies(ctx, cookies); err != nil {
|
||||
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "login"})
|
||||
}
|
||||
|
||||
identityResponse, err := connector.Browser.Get(ctx, identityEndpoint)
|
||||
if err != nil {
|
||||
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, Evidence{Phase: "identity"})
|
||||
}
|
||||
if state, reason := classify(identityResponse); state != "" {
|
||||
return connector.stop(ctx, request.AccountID, state, reason, Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
|
||||
}
|
||||
identity, ok := parseIdentity(identityResponse.Body)
|
||||
if !ok {
|
||||
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown,
|
||||
Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
|
||||
}
|
||||
if request.PlatformAccountKey != identity.User.UID && request.PlatformAccountKey != identity.User.SecUID &&
|
||||
request.PlatformAccountKey != identity.User.UniqueID {
|
||||
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonIdentityMatch,
|
||||
Evidence{Phase: "identity", HTTPStatus: identityResponse.Status})
|
||||
}
|
||||
|
||||
query := url.Values{"sec_user_id": {identity.User.SecUID}, "count": {"20"}, "max_cursor": {"0"}}
|
||||
worksResponse, err := connector.Browser.Get(ctx, worksEndpoint+"?"+query.Encode())
|
||||
if err != nil {
|
||||
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown,
|
||||
Evidence{Phase: "works", IdentityVerified: true})
|
||||
}
|
||||
if state, reason := classify(worksResponse); state != "" {
|
||||
return connector.stop(ctx, request.AccountID, state, reason,
|
||||
Evidence{Phase: "works", HTTPStatus: worksResponse.Status, IdentityVerified: true})
|
||||
}
|
||||
works, hasMore, ok := parseWorks(worksResponse.Body)
|
||||
evidence := Evidence{Phase: "works", HTTPStatus: worksResponse.Status, IdentityVerified: true, WorksSeen: len(works), HasMore: hasMore}
|
||||
if !ok {
|
||||
return connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
||||
}
|
||||
if err := connector.Store.Complete(ctx, request.AccountID, works, evidence); err != nil {
|
||||
result, holdErr := connector.stop(ctx, request.AccountID, StateNeedsConfirmation, ReasonUnknown, evidence)
|
||||
return result, errors.Join(err, holdErr)
|
||||
}
|
||||
return Result{State: StateSucceeded, ReasonCode: ReasonSucceeded, Evidence: evidence}, nil
|
||||
}
|
||||
|
||||
func (connector Connector) stop(ctx context.Context, accountID, state, reason string, evidence Evidence) (Result, error) {
|
||||
result := Result{State: state, ReasonCode: reason, Evidence: evidence}
|
||||
holdContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
return result, connector.Store.Hold(holdContext, accountID, state, reason, evidence)
|
||||
}
|
||||
|
||||
func classify(response Response) (string, string) {
|
||||
switch response.Challenge {
|
||||
case ChallengeCaptcha, ChallengeDevice:
|
||||
return StateNeedsConfirmation, ReasonChallenge
|
||||
case ChallengeNone:
|
||||
default:
|
||||
return StateNeedsConfirmation, ReasonUnknown
|
||||
}
|
||||
switch response.Status {
|
||||
case http.StatusUnauthorized:
|
||||
return StatePolicyHold, ReasonAuthInvalid
|
||||
case http.StatusForbidden:
|
||||
return StatePolicyHold, ReasonForbidden
|
||||
case http.StatusTooManyRequests:
|
||||
return StatePolicyHold, ReasonRateLimited
|
||||
case http.StatusOK:
|
||||
return "", ""
|
||||
default:
|
||||
return StateNeedsConfirmation, ReasonUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func parseCredential(raw []byte) ([]Cookie, error) {
|
||||
if len(raw) == 0 || len(raw) > 64<<10 {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
var bundle struct {
|
||||
Cookies []Cookie `json:"cookies"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&bundle); err != nil || len(bundle.Cookies) == 0 || len(bundle.Cookies) > 64 {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
var trailing json.RawMessage
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
for index := range bundle.Cookies {
|
||||
cookie := &bundle.Cookies[index]
|
||||
cookie.Domain = strings.ToLower(strings.TrimSpace(cookie.Domain))
|
||||
if cookie.Path == "" {
|
||||
cookie.Path = "/"
|
||||
}
|
||||
if cookie.Name == "" || len(cookie.Name) > 256 || len(cookie.Value) > 4096 || len(cookie.Domain) > 256 || len(cookie.Path) > 256 ||
|
||||
cookie.Expires < 0 || strings.ContainsAny(cookie.Name, ";\r\n\x00") || strings.ContainsAny(cookie.Value, ";\r\n\x00") ||
|
||||
(cookie.Domain != "douyin.com" && !strings.HasSuffix(cookie.Domain, ".douyin.com")) ||
|
||||
!strings.HasPrefix(cookie.Path, "/") || strings.ContainsAny(cookie.Path, ";\r\n\x00") ||
|
||||
(cookie.SameSite != "" && cookie.SameSite != "Lax" &&
|
||||
cookie.SameSite != "Strict" && cookie.SameSite != "None") {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
}
|
||||
return bundle.Cookies, nil
|
||||
}
|
||||
|
||||
type identityEnvelope struct {
|
||||
StatusCode *int `json:"status_code"`
|
||||
User *struct {
|
||||
UID string `json:"uid"`
|
||||
SecUID string `json:"sec_uid"`
|
||||
UniqueID string `json:"unique_id"`
|
||||
} `json:"user"`
|
||||
}
|
||||
|
||||
func parseIdentity(body []byte) (identityEnvelope, bool) {
|
||||
var identity identityEnvelope
|
||||
if len(body) > 1<<20 || json.Unmarshal(body, &identity) != nil || identity.StatusCode == nil || *identity.StatusCode != 0 || identity.User == nil ||
|
||||
!keyPattern.MatchString(identity.User.UID) || !keyPattern.MatchString(identity.User.SecUID) ||
|
||||
(identity.User.UniqueID != "" && !keyPattern.MatchString(identity.User.UniqueID)) {
|
||||
return identityEnvelope{}, false
|
||||
}
|
||||
return identity, true
|
||||
}
|
||||
|
||||
type worksEnvelope struct {
|
||||
StatusCode *int `json:"status_code"`
|
||||
HasMore *bool `json:"has_more"`
|
||||
Works []struct {
|
||||
ID string `json:"aweme_id"`
|
||||
Description string `json:"desc"`
|
||||
CreatedAt *int64 `json:"create_time"`
|
||||
Statistics *struct {
|
||||
DiggCount *int64 `json:"digg_count"`
|
||||
CommentCount *int64 `json:"comment_count"`
|
||||
ShareCount *int64 `json:"share_count"`
|
||||
PlayCount *int64 `json:"play_count"`
|
||||
} `json:"statistics"`
|
||||
} `json:"aweme_list"`
|
||||
}
|
||||
|
||||
func parseWorks(body []byte) ([]Work, bool, bool) {
|
||||
var envelope worksEnvelope
|
||||
if len(body) > 4<<20 || json.Unmarshal(body, &envelope) != nil || envelope.StatusCode == nil || *envelope.StatusCode != 0 ||
|
||||
envelope.HasMore == nil || envelope.Works == nil || len(envelope.Works) > 20 {
|
||||
return nil, false, false
|
||||
}
|
||||
works := make([]Work, 0, len(envelope.Works))
|
||||
seen := make(map[string]bool, len(envelope.Works))
|
||||
for _, candidate := range envelope.Works {
|
||||
if !keyPattern.MatchString(candidate.ID) || seen[candidate.ID] || candidate.CreatedAt == nil || *candidate.CreatedAt <= 0 ||
|
||||
candidate.Statistics == nil || candidate.Statistics.DiggCount == nil || candidate.Statistics.CommentCount == nil ||
|
||||
candidate.Statistics.ShareCount == nil || candidate.Statistics.PlayCount == nil ||
|
||||
utf8.RuneCountInString(candidate.Description) > 4096 || *candidate.Statistics.DiggCount < 0 ||
|
||||
*candidate.Statistics.CommentCount < 0 || *candidate.Statistics.ShareCount < 0 || *candidate.Statistics.PlayCount < 0 {
|
||||
return nil, false, false
|
||||
}
|
||||
seen[candidate.ID] = true
|
||||
works = append(works, Work{ID: candidate.ID, Description: candidate.Description, CreatedAt: *candidate.CreatedAt,
|
||||
DiggCount: *candidate.Statistics.DiggCount, CommentCount: *candidate.Statistics.CommentCount,
|
||||
ShareCount: *candidate.Statistics.ShareCount, PlayCount: *candidate.Statistics.PlayCount})
|
||||
}
|
||||
return works, *envelope.HasMore, true
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package douyin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const credential = `{"cookies":[{"name":"sessionid","value":"private-session","domain":".douyin.com","path":"/","secure":true,"http_only":true,"same_site":"Lax"}]}`
|
||||
|
||||
var secretReference = SecretReference{Provider: "os_keyring", Key: "creatorhub/account-a"}
|
||||
|
||||
type fakeSecrets struct {
|
||||
value []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (secrets fakeSecrets) Resolve(_ context.Context, _ SecretReference) ([]byte, error) {
|
||||
return secrets.value, secrets.err
|
||||
}
|
||||
|
||||
type fakeBrowser struct {
|
||||
responses []Response
|
||||
err error
|
||||
cookies []Cookie
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (browser *fakeBrowser) SetCookies(_ context.Context, cookies []Cookie) error {
|
||||
browser.cookies = cookies
|
||||
return browser.err
|
||||
}
|
||||
|
||||
func (browser *fakeBrowser) Get(_ context.Context, target string) (Response, error) {
|
||||
browser.urls = append(browser.urls, target)
|
||||
if browser.err != nil {
|
||||
return Response{}, browser.err
|
||||
}
|
||||
response := browser.responses[0]
|
||||
browser.responses = browser.responses[1:]
|
||||
return response, nil
|
||||
}
|
||||
|
||||
type fakeStore struct {
|
||||
works []Work
|
||||
holds []Result
|
||||
completeCalls int
|
||||
holdContextErr error
|
||||
completeErr error
|
||||
holdErr error
|
||||
}
|
||||
|
||||
func (store *fakeStore) Complete(_ context.Context, _ string, works []Work, _ Evidence) error {
|
||||
store.completeCalls++
|
||||
store.works = works
|
||||
return store.completeErr
|
||||
}
|
||||
|
||||
func (store *fakeStore) Hold(ctx context.Context, _ string, state, reason string, evidence Evidence) error {
|
||||
store.holdContextErr = ctx.Err()
|
||||
store.holds = append(store.holds, Result{State: state, ReasonCode: reason, Evidence: evidence})
|
||||
return store.holdErr
|
||||
}
|
||||
|
||||
func identityBody(uid, secUID, uniqueID string) []byte {
|
||||
body, _ := json.Marshal(map[string]any{"status_code": 0, "user": map[string]string{
|
||||
"uid": uid, "sec_uid": secUID, "unique_id": uniqueID,
|
||||
}})
|
||||
return body
|
||||
}
|
||||
|
||||
func TestSyncLogsInVerifiesIdentityAndReadsOwnWorks(t *testing.T) {
|
||||
browser := &fakeBrowser{responses: []Response{
|
||||
{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")},
|
||||
{Status: 200, Body: []byte(`{"status_code":0,"has_more":true,"aweme_list":[{"aweme_id":"work-1","desc":"hello","create_time":123,"statistics":{"digg_count":4,"comment_count":3,"share_count":2,"play_count":1}}]}`)},
|
||||
}}
|
||||
store := &fakeStore{}
|
||||
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
|
||||
})
|
||||
if err != nil || result.State != StateSucceeded || !result.Evidence.IdentityVerified || result.Evidence.WorksSeen != 1 || !result.Evidence.HasMore {
|
||||
t.Fatalf("unexpected result: %#v err=%v", result, err)
|
||||
}
|
||||
if len(browser.cookies) != 1 || browser.cookies[0].Value != "private-session" || len(browser.urls) != 2 ||
|
||||
browser.urls[0] != identityEndpoint || !strings.Contains(browser.urls[1], "sec_user_id=sec-a") {
|
||||
t.Fatalf("connector did not use the bound browser session: cookies=%#v urls=%#v", browser.cookies, browser.urls)
|
||||
}
|
||||
if len(store.works) != 1 || store.works[0].ID != "work-1" || store.works[0].PlayCount != 1 || len(store.holds) != 0 {
|
||||
t.Fatalf("unexpected persisted works or hold: works=%#v holds=%#v", store.works, store.holds)
|
||||
}
|
||||
encoded, _ := json.Marshal(result)
|
||||
if strings.Contains(string(encoded), "private-session") {
|
||||
t.Fatalf("audit result leaked credential: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncMapsRiskSignalsAndNeverRetries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response Response
|
||||
state string
|
||||
reason string
|
||||
}{
|
||||
{name: "authentication invalid", response: Response{Status: 401}, state: StatePolicyHold, reason: ReasonAuthInvalid},
|
||||
{name: "forbidden", response: Response{Status: 403}, state: StatePolicyHold, reason: ReasonForbidden},
|
||||
{name: "rate limited", response: Response{Status: 429}, state: StatePolicyHold, reason: ReasonRateLimited},
|
||||
{name: "captcha", response: Response{Status: 200, Challenge: ChallengeCaptcha}, state: StateNeedsConfirmation, reason: ReasonChallenge},
|
||||
{name: "device challenge", response: Response{Status: 200, Challenge: ChallengeDevice}, state: StateNeedsConfirmation, reason: ReasonChallenge},
|
||||
{name: "unknown status", response: Response{Status: 502}, state: StateNeedsConfirmation, reason: ReasonUnknown},
|
||||
{name: "unknown challenge", response: Response{Status: 200, Challenge: "future"}, state: StateNeedsConfirmation, reason: ReasonUnknown},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
browser := &fakeBrowser{responses: []Response{test.response}}
|
||||
store := &fakeStore{}
|
||||
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
|
||||
})
|
||||
if err != nil || result.State != test.state || result.ReasonCode != test.reason || len(store.holds) != 1 {
|
||||
t.Fatalf("unexpected stop: result=%#v holds=%#v err=%v", result, store.holds, err)
|
||||
}
|
||||
if len(browser.urls) != 1 {
|
||||
t.Fatalf("risk response was retried: %#v", browser.urls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncFailsClosedOnIdentityAndUnknownResults(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
browser *fakeBrowser
|
||||
key string
|
||||
reason string
|
||||
phase string
|
||||
}{
|
||||
{name: "identity mismatch", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}}}, key: "another", reason: ReasonIdentityMatch, phase: "identity"},
|
||||
{name: "malformed identity", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: []byte(`{"status_code":0}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "identity"},
|
||||
{name: "identity status missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: []byte(`{"user":{"uid":"uid-a","sec_uid":"sec-a","unique_id":"handle-a"}}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "identity"},
|
||||
{name: "browser failure", browser: &fakeBrowser{err: errors.New("transport details must stay internal")}, key: "sec-a", reason: ReasonUnknown, phase: "login"},
|
||||
{name: "malformed works", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"aweme_list":[{"aweme_id":""}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "works list missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "works has_more missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"aweme_list":[]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "work create_time missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "work create_time null", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":null,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "work statistics missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "work statistics null", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1,"statistics":null}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "work statistic missing", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
{name: "work statistic null", browser: &fakeBrowser{responses: []Response{{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")}, {Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"work-1","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":null}}]}`)}}}, key: "sec-a", reason: ReasonUnknown, phase: "works"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
result, err := (Connector{Browser: test.browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: test.key, Credential: secretReference,
|
||||
})
|
||||
if err != nil || result.State != StateNeedsConfirmation || result.ReasonCode != test.reason || result.Evidence.Phase != test.phase || len(store.holds) != 1 || store.completeCalls != 0 {
|
||||
t.Fatalf("unexpected fail-closed result: %#v holds=%#v err=%v", result, store.holds, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncRejectsInvalidCredentialWithoutLeakingIt(t *testing.T) {
|
||||
browser := &fakeBrowser{}
|
||||
store := &fakeStore{}
|
||||
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(`{"cookies":[{"name":"sessionid","value":"secret","domain":"evil.example"}]}`)}, Store: store}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
|
||||
})
|
||||
if err != nil || result.State != StatePolicyHold || result.ReasonCode != ReasonAuthInvalid || len(browser.urls) != 0 || len(browser.cookies) != 0 || len(store.holds) != 1 {
|
||||
t.Fatalf("unexpected invalid credential result: %#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err)
|
||||
}
|
||||
encoded, _ := json.Marshal(result)
|
||||
if strings.Contains(string(encoded), "secret") || strings.Contains(string(encoded), "evil") {
|
||||
t.Fatalf("stop evidence leaked credential: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncStopsWhenSecretReferenceCannotResolve(t *testing.T) {
|
||||
browser := &fakeBrowser{}
|
||||
store := &fakeStore{}
|
||||
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{err: errors.New("secret unavailable")}, Store: store}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
|
||||
})
|
||||
if err != nil || result.State != StatePolicyHold || result.ReasonCode != ReasonAuthInvalid || len(browser.urls) != 0 || len(store.holds) != 1 {
|
||||
t.Fatalf("unavailable secret did not fail closed: result=%#v browser=%#v holds=%#v err=%v", result, browser, store.holds, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncFailsClosedWhenPersistenceIsUnknown(t *testing.T) {
|
||||
browser := &fakeBrowser{responses: []Response{
|
||||
{Status: 200, Body: identityBody("uid-a", "sec-a", "handle-a")},
|
||||
{Status: 200, Body: []byte(`{"status_code":0,"has_more":false,"aweme_list":[]}`)},
|
||||
}}
|
||||
store := &fakeStore{completeErr: errors.New("database result unknown")}
|
||||
result, err := (Connector{Browser: browser, Secrets: fakeSecrets{value: []byte(credential)}, Store: store}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
|
||||
})
|
||||
if err == nil || result.State != StateNeedsConfirmation || result.ReasonCode != ReasonUnknown || len(store.holds) != 1 {
|
||||
t.Fatalf("persistence uncertainty did not stop: result=%#v holds=%#v err=%v", result, store.holds, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCredentialAndWorkValidation(t *testing.T) {
|
||||
invalidCredentials := []string{
|
||||
``, `{}`, `{"cookies":[]}`, `{"cookies":[{"name":"a","value":"b","domain":".douyin.com","extra":true}]}`,
|
||||
`{"cookies":[{"name":"a;bad","value":"b","domain":".douyin.com"}]}`,
|
||||
credential + `true`, credential + `[]`, credential + `null`, credential + `garbage`,
|
||||
}
|
||||
for _, input := range invalidCredentials {
|
||||
if _, err := parseCredential([]byte(input)); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("accepted invalid credential bundle: %q", input)
|
||||
}
|
||||
}
|
||||
if _, _, ok := parseWorks([]byte(`{"status_code":0,"has_more":false,"aweme_list":[{"aweme_id":"same","desc":"a","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}},{"aweme_id":"same","desc":"b","create_time":1,"statistics":{"digg_count":0,"comment_count":0,"share_count":0,"play_count":0}}]}`)); ok {
|
||||
t.Fatal("accepted duplicate work ids")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncHoldsWithCancelledRequestContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
store := &fakeStore{}
|
||||
result, err := (Connector{Browser: &fakeBrowser{}, Secrets: fakeSecrets{err: context.Canceled}, Store: store}).Sync(ctx, Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: secretReference,
|
||||
})
|
||||
if err != nil || result.State != StatePolicyHold || len(store.holds) != 1 || store.holdContextErr != nil {
|
||||
t.Fatalf("cancelled request did not durably hold: result=%#v holds=%#v context_err=%v err=%v", result, store.holds, store.holdContextErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncRejectsNonSecretCredentialReference(t *testing.T) {
|
||||
_, err := (Connector{Browser: &fakeBrowser{}, Secrets: fakeSecrets{}, Store: &fakeStore{}}).Sync(context.Background(), Request{
|
||||
AccountID: "account-a", PlatformAccountKey: "sec-a", Credential: SecretReference{Provider: "plain_text", Key: "raw-secret"},
|
||||
})
|
||||
if !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("accepted non-secret credential reference: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user