package main import ( "bytes" "encoding/json" "fmt" "io" "net" "net/http" "net/http/httptest" "os" "strconv" "strings" "sync" "testing" "time" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" ) const testToken = "unit-test-gateway-token" func authed(method, target string, body io.Reader) *http.Request { request := httptest.NewRequest(method, target, body) request.Header.Set("Authorization", "Bearer "+testToken) return request } func testDocker(handler http.HandlerFunc) (dockerClient, *httptest.Server) { self, _ := os.Hostname() var networkMu sync.Mutex networkMembers := map[string]map[string]bool{} networkDeleted := map[string]bool{} server := httptest.NewServer(withAliasReservations(self, func(response http.ResponseWriter, request *http.Request) { if strings.HasPrefix(request.URL.Path, "/networks/network-") { alias := strings.TrimPrefix(request.URL.Path, "/networks/network-") alias = strings.TrimSuffix(strings.TrimSuffix(alias, "/disconnect"), "/connect") networkMu.Lock() if request.Method == http.MethodGet { if networkDeleted[alias] { networkMu.Unlock() response.WriteHeader(http.StatusNotFound) return } members := map[string]any{} for id := range networkMembers[alias] { members[id] = map[string]string{"Name": id, "IPv4Address": "127.0.0.1/8"} } networkMu.Unlock() _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-" + alias, "Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias, bindingVersionLabel: "1"}, "Containers": members, }) return } if strings.HasSuffix(request.URL.Path, "/disconnect") { var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) delete(networkMembers[alias], body.Container) } else if strings.HasSuffix(request.URL.Path, "/connect") { var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) if networkMembers[alias] == nil { networkMembers[alias] = map[string]bool{} } networkMembers[alias][body.Container] = true } else if request.Method == http.MethodDelete { networkDeleted[alias] = true } networkMu.Unlock() if request.Method == http.MethodDelete { response.WriteHeader(http.StatusNoContent) } else { response.WriteHeader(http.StatusOK) } return } if strings.HasPrefix(request.URL.Path, "/networks/creatorhub_browser-") { if request.Method != http.MethodGet { if request.Method == http.MethodDelete { response.WriteHeader(http.StatusNoContent) } else { response.WriteHeader(http.StatusOK) } return } alias := strings.TrimPrefix(request.URL.Path, "/networks/creatorhub_browser-") networkMu.Lock() if networkDeleted[alias] { networkMu.Unlock() response.WriteHeader(http.StatusNotFound) return } if networkMembers[alias] == nil { networkMembers[alias] = map[string]bool{self: true} } members := map[string]any{} for id := range networkMembers[alias] { members[id] = map[string]string{"Name": id, "IPv4Address": "127.0.0.1/8"} } networkMu.Unlock() _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-" + alias, "Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias, bindingVersionLabel: "1"}, "Containers": members, }) return } handler(response, request) })) return dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, server } func withAliasReservations(self string, next http.HandlerFunc) http.HandlerFunc { var mu sync.Mutex type reservation struct { id, generation string } locks := map[string]reservation{} return func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet && request.URL.Path == "/containers/"+self+"/json" { _, _ = response.Write([]byte(`{"Id":"` + self + `","Image":"gateway-image-id","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) return } name := request.URL.Query().Get("name") if request.Method == http.MethodPost && request.URL.Path == "/containers/create" && strings.HasPrefix(name, reservationPrefix) { mu.Lock() defer mu.Unlock() if locks[name].id != "" { response.WriteHeader(http.StatusConflict) return } var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) id := "reservation-" + strings.TrimPrefix(name, reservationPrefix) locks[name] = reservation{id: id, generation: payload.Labels[reservationGenLabel]} response.WriteHeader(http.StatusCreated) _ = json.NewEncoder(response).Encode(map[string]string{"Id": id}) return } if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+reservationPrefix) && strings.HasSuffix(request.URL.Path, "/json") { name := strings.TrimSuffix(strings.TrimPrefix(request.URL.Path, "/containers/"), "/json") mu.Lock() current, found := locks[name] mu.Unlock() if !found { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{"Id": current.id, "Config": map[string]any{"Labels": map[string]string{ reservationLabel: "true", idLabel: strings.TrimPrefix(name, reservationPrefix), reservationGenLabel: current.generation, }}}) return } if request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/reservation-") { id := strings.TrimPrefix(request.URL.Path, "/containers/") mu.Lock() for name, current := range locks { if current.id == id { delete(locks, name) break } } mu.Unlock() response.WriteHeader(http.StatusNoContent) return } next(response, request) } } func decodeJSONBody(t *testing.T, response *http.Response) map[string]any { t.Helper() var body map[string]any if err := json.NewDecoder(response.Body).Decode(&body); err != nil { t.Fatalf("decode JSON body: %v", err) } return body } const testCreateBody = `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` + `"cmd":["--fingerprint=1000","--lang=zh-CN","about:blank"],"volume":"creatorhub-profile-account-a",` + `"binding_version":1,"network_exit_id":"exit-1",` + `"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` const testGenerationBody = `{"binding_version":1,"runtime_id":"container-id"}` func TestAliasReservationNameDoesNotOverlapRuntimeNamespace(t *testing.T) { if strings.HasPrefix(reservationPrefix, namePrefix) || strings.HasPrefix(namePrefix, reservationPrefix) { t.Fatalf("reservation and runtime prefixes overlap: reservation=%q runtime=%q", reservationPrefix, namePrefix) } generation := "" server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": if request.URL.Query().Get("name") == namePrefix+"lock-account-a" { response.WriteHeader(http.StatusConflict) return } var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} _, release, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a") if err != nil { t.Fatalf("runtime alias lock-account-a occupied account-a reservation: %v", err) } release() } func TestAliasReservationRecoversInvalidCreateAndReleaseResponses(t *testing.T) { t.Run("create disconnect after apply", func(t *testing.T) { generation := "" removed := false server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] connection, _, _ := response.(http.Hijacker).Hijack() _ = connection.Close() case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": removed = true response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || !removed { t.Fatalf("disconnected reservation create was not reconciled by immutable generation: removed=%v err=%v", removed, err) } }) t.Run("invalid create body", func(t *testing.T) { reservationExists, removed := false, false generation := "" server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] reservationExists = true response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": if !reservationExists { response.WriteHeader(http.StatusNotFound) return } _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": reservationExists, removed = false, true response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || !removed { t.Fatalf("invalid reservation response was not reconciled: removed=%v err=%v", removed, err) } }) t.Run("nonempty foreign create id", func(t *testing.T) { generation := "" deletes := 0 server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"foreign-id"}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) case request.Method == http.MethodDelete: deletes++ response.WriteHeader(http.StatusNotFound) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || deletes != 0 { t.Fatalf("foreign 201 id was accepted or deleted: deletes=%d err=%v", deletes, err) } }) t.Run("201 inspect disconnect", func(t *testing.T) { deletes := 0 server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": connection, _, _ := response.(http.Hijacker).Hijack() _ = connection.Close() case request.Method == http.MethodDelete: deletes++ response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} if _, _, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a"); err == nil || deletes != 0 { t.Fatalf("unverified 201 reservation was deleted: deletes=%d err=%v", deletes, err) } }) t.Run("delete 404 requires generation absence", func(t *testing.T) { generation := "" deletes := 0 server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": deletes++ response.WriteHeader(http.StatusNotFound) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} locks := &dockerAliasReservations{docker: docker, self: "gateway-self"} id, _, err := locks.acquire("account-a") if err != nil { t.Fatal(err) } if err := locks.remove("account-a", generation, id); err == nil || deletes != 1 { t.Fatalf("404 was accepted while the reservation generation remained: deletes=%d err=%v", deletes, err) } }) t.Run("release retry", func(t *testing.T) { deletes := 0 generation := "" server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": deletes++ if deletes == 1 { response.WriteHeader(http.StatusInternalServerError) return } response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} _, release, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a") if err != nil { t.Fatal(err) } release() if deletes != 2 { t.Fatalf("reservation release did not retry the immutable id: deletes=%d", deletes) } }) t.Run("release does not delete successor", func(t *testing.T) { generation := "" deletes := 0 server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Image":"gateway-image","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) generation = payload.Labels[reservationGenLabel] response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"reservation-id"}`)) case request.Method == http.MethodDelete && request.URL.Path == "/containers/reservation-id": deletes++ response.WriteHeader(http.StatusInternalServerError) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+reservationPrefix+"account-a/json": if deletes == 0 { _, _ = response.Write([]byte(`{"Id":"reservation-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"` + generation + `"}}}`)) return } _, _ = response.Write([]byte(`{"Id":"successor-id","Config":{"Labels":{"` + reservationLabel + `":"true","` + idLabel + `":"account-a","` + reservationGenLabel + `":"successor"}}}`)) default: t.Fatalf("unexpected Docker request %s %s generation=%s", request.Method, request.URL.String(), generation) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} _, release, err := (&dockerAliasReservations{docker: docker, self: "gateway-self"}).acquire("account-a") if err != nil { t.Fatal(err) } release() if deletes != 1 { t.Fatalf("reservation release deleted a successor generation: deletes=%d", deletes) } }) } func TestGatewayCreatesNetworkDisabledStoppedRecoveryContainer(t *testing.T) { created := false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): _, _ = response.Write([]byte(`{}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): var payload map[string]any _ = json.NewDecoder(request.Body).Decode(&payload) host := payload["HostConfig"].(map[string]any) labels := payload["Labels"].(map[string]any) encoded, _ := json.Marshal(payload["Cmd"]) if host["NetworkMode"] != "none" || labels[networkExitLabel] != "" || strings.Contains(string(encoded), "proxy") { t.Fatalf("unsafe stopped recovery payload: %#v", payload) } created = true response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"stopped-container"}`)) default: t.Fatalf("stopped recovery unexpectedly called Docker %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) body := `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` + `"cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` + `"binding_version":1,"network_exit_id":"","network_exit":{},"stopped":true}` response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) if response.Code != http.StatusCreated || !created { t.Fatalf("stopped recovery create failed: status=%d body=%s", response.Code, response.Body.String()) } } func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { var created map[string]any docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) _, _ = response.Write([]byte(`{}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): if got := request.URL.Query().Get("name"); got != namePrefix+"account-a" { t.Fatalf("unexpected container name %q", got) } if err := json.NewDecoder(request.Body).Decode(&created); err != nil { t.Fatal(err) } response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/container-id/start"): response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) if response.Code != http.StatusCreated { t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String()) } if created["Image"] != "registry.example/browser:1.2.3" { t.Fatalf("gateway must run the platform-specified image: %#v", created["Image"]) } if created["User"] != browserUser || created["Entrypoint"].([]any)[0] != browserEntrypoint { t.Fatalf("runtime identity is not fixed: user=%#v entrypoint=%#v", created["User"], created["Entrypoint"]) } cmd := created["Cmd"].([]any) if len(cmd) != 5 || cmd[0] != "--fingerprint=1000" || !strings.HasPrefix(cmd[2].(string), "--proxy-server=http://docker-gateway:") || cmd[3] != "--disable-non-proxied-udp" || cmd[4] != "about:blank" { t.Fatalf("cmd must be passed through verbatim: %#v", created["Cmd"]) } host := created["HostConfig"].(map[string]any) if host["NetworkMode"] != "network-account-a" || host["ReadonlyRootfs"] != true { t.Fatalf("missing container isolation: %#v", host) } tmpfs := host["Tmpfs"].(map[string]any) if tmpfs["/tmp/.X11-unix"] == nil || tmpfs["/home/ubuntu"] == nil { t.Fatalf("missing writable runtime paths: %#v", tmpfs) } mount := host["Mounts"].([]any)[0].(map[string]any) if mount["Source"] != "creatorhub-profile-account-a" || mount["Target"] != "/data" { t.Fatalf("profile volume must come from the request: %#v", mount) } labels := created["Labels"].(map[string]any) if labels[managedLabel] != "true" || labels[idLabel] != "account-a" || labels[nameLabel] != "账号甲" { t.Fatalf("missing ownership labels: %#v", labels) } } func TestGatewayCreatesDirectBrowserWithoutProxyArguments(t *testing.T) { var created map[string]any docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): _, _ = response.Write([]byte(`{}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): if err := json.NewDecoder(request.Body).Decode(&created); err != nil { t.Fatal(err) } response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/container-id/start"): response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) body := `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` + `"cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` + `"binding_version":1,"network_exit_id":"","network_exit":{}}` response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) if response.Code != http.StatusCreated { t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String()) } cmd := created["Cmd"].([]any) encoded, _ := json.Marshal(cmd) if len(cmd) != 2 || strings.Contains(string(encoded), "proxy") { t.Fatalf("direct runtime received proxy arguments: %#v", cmd) } host := created["HostConfig"].(map[string]any) labels := created["Labels"].(map[string]any) if host["NetworkMode"] != "network-account-a" || labels[networkExitLabel] != "" || labels[proxyPortLabel] != "0" { t.Fatalf("direct runtime metadata is invalid: host=%#v labels=%#v", host, labels) } } func TestGatewayCreateUsesCapturedNetworkIDAcrossNameReplacement(t *testing.T) { networkID := "" members := map[string]string{} usedNetworkID, touchedReplacement := "", false server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): reference := strings.TrimPrefix(request.URL.Path, "/networks/") if reference == "network-n2" { touchedReplacement = true } if networkID == "" || (reference != "creatorhub_browser-account-a" && reference != networkID) { response.WriteHeader(http.StatusNotFound) return } containers := map[string]any{} for id, name := range members { containers[id] = map[string]string{"Name": name, "IPv4Address": "127.0.0.1/8"} } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": networkID, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": containers, }) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": networkID = "network-n1" response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) case request.Method == http.MethodPost && request.URL.Path == "/networks/network-n1/connect": members["gateway-self"] = "gateway-self" response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { HostConfig map[string]any `json:"HostConfig"` } _ = json.NewDecoder(request.Body).Decode(&payload) usedNetworkID, _ = payload.HostConfig["NetworkMode"].(string) networkID, members = "network-n2", map[string]string{"replacement": "replacement"} response.WriteHeader(http.StatusNotFound) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) if response.Code != http.StatusBadGateway || usedNetworkID != "network-n1" || touchedReplacement || networkID != "network-n2" || members["replacement"] == "" { t.Fatalf("stale create crossed network generation: status=%d mode=%q touchedN2=%v network=%q members=%v body=%s", response.Code, usedNetworkID, touchedReplacement, networkID, members, response.Body.String()) } } func TestGatewayDockerInspectContainsNoProxyCredentials(t *testing.T) { var created map[string]any docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): if err := json.NewDecoder(request.Body).Decode(&created); err != nil { t.Fatal(err) } response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/container-id/start"): response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) body := strings.Replace(testCreateBody, `"protocol":"socks5","host":"proxy.example","port":1080`, `"protocol":"socks5","host":"proxy.example","port":1080,"username":"operator","password":"ephemeral"`, 1) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) if response.Code != http.StatusCreated { t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String()) } inspect, _ := json.Marshal(created) for _, secret := range []string{"operator", "ephemeral", "operator:ephemeral@", "proxy.example"} { if bytes.Contains(inspect, []byte(secret)) { t.Fatalf("Docker inspect leaked proxy credential %q: %s", secret, inspect) } } if !bytes.Contains(inspect, []byte("--proxy-server=http://docker-gateway:")) { t.Fatalf("Docker inspect is missing the secret-free proxy configuration: %s", inspect) } } func TestGatewayPullsMissingImageOnCreate(t *testing.T) { tests := []struct { name string ref string fromImage string tag string }{{ name: "tagged ref splits repository and tag", ref: "registry.example/browser:2.0.0", fromImage: "registry.example/browser", tag: "2.0.0", }, { name: "digest ref is pulled as a whole", ref: "registry.example/browser@sha256:b9f23b8e3ac640174db0dfa49e9095fe7eb06f5db55a4e7550d979b35ff3a1b7", fromImage: "registry.example/browser@sha256:b9f23b8e3ac640174db0dfa49e9095fe7eb06f5db55a4e7550d979b35ff3a1b7", tag: "", }} for _, test := range tests { t.Run(test.name, func(t *testing.T) { pulled := false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/images/create"): pulled = true if request.URL.Query().Get("fromImage") != test.fromImage || request.URL.Query().Get("tag") != test.tag { t.Fatalf("unexpected pull query %s", request.URL.RawQuery) } _, _ = response.Write([]byte(`{"status":"Download complete"}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"): response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) body := `{"alias":"account-a","name":"账号甲","image":"` + test.ref + `","cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` + `"binding_version":1,"network_exit_id":"exit-1",` + `"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) if response.Code != http.StatusCreated || !pulled { t.Fatalf("expected pull-then-create, status=%d pulled=%v body=%s", response.Code, pulled, response.Body.String()) } }) } } func TestGatewayRejectsCreateWithoutValidToken(t *testing.T) { docker, server := testDocker(func(http.ResponseWriter, *http.Request) { t.Fatal("no Docker request is expected for an unauthorized call") }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) for name, header := range map[string]string{ "missing": "", "malformed": testToken, "wrong": "Bearer not-the-token", } { request := httptest.NewRequest(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody)) if header != "" { request.Header.Set("Authorization", header) } response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, request) if response.Code != http.StatusUnauthorized { t.Fatalf("%s token: expected 401, got %d: %s", name, response.Code, response.Body.String()) } } response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/healthz", nil)) if response.Code != http.StatusNoContent { t.Fatalf("healthz must stay unauthenticated, got %d", response.Code) } } func TestGatewayRejectsInvalidCreateRequest(t *testing.T) { tests := map[string]string{ "unknown field": `{"alias":"account-a","seed":1}`, "invalid alias": `{"alias":"AccountA","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1"],"volume":"creatorhub-profile-account-a"}`, "invalid image": `{"alias":"account-a","name":"甲","image":"","cmd":["--fingerprint=1"],"volume":"creatorhub-profile-account-a"}`, "empty cmd": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":[],"volume":"creatorhub-profile-account-a"}`, "invalid volume": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1"],"volume":"bad volume!"}`, "proxy override": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1","--proxy-server=http://direct:8080","about:blank"],"volume":"creatorhub-profile-account-a","network_exit":{"protocol":"socks5","host":"proxy","port":1080}}`, } for name, body := range tests { t.Run(name, func(t *testing.T) { handler := newGateway(dockerClient{}, "creatorhub_browser", testToken) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) if response.Code != http.StatusBadRequest { t.Fatalf("expected 400, got %d: %s", response.Code, response.Body.String()) } }) } } func TestGatewayRejectsOversizedCreateRequest(t *testing.T) { handler := newGateway(dockerClient{}, "creatorhub_browser", testToken) request := authed(http.MethodPost, "/v1/browsers", strings.NewReader(strings.Repeat("x", (1<<20)+1))) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, request) if response.Code != http.StatusRequestEntityTooLarge { t.Fatalf("expected 413 for oversized body, status=%d body=%s", response.Code, response.Body.String()) } handler.Post("/request-limit", func(fiber.Ctx) error { return fiber.ErrRequestEntityTooLarge }) jsonResponse, err := handler.Test(httptest.NewRequest(http.MethodPost, "/request-limit", nil)) if err != nil { t.Fatal(err) } defer jsonResponse.Body.Close() var body map[string]string decodeErr := json.NewDecoder(jsonResponse.Body).Decode(&body) contentType := jsonResponse.Header.Get("Content-Type") if jsonResponse.StatusCode != http.StatusRequestEntityTooLarge || decodeErr != nil || body["error"] == "" || !strings.HasPrefix(contentType, "application/json") { t.Fatalf("expected JSON 413 envelope, status=%d body=%v decode=%v content-type=%q", jsonResponse.StatusCode, body, decodeErr, contentType) } } func TestGatewayReconcilesContainerWhenCreateResponseHasNoID(t *testing.T) { created, started := false, false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) _, _ = response.Write([]byte(`{}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): if !created { response.WriteHeader(http.StatusNotFound) return } _, _ = response.Write([]byte(`{"Id":"actual-container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-account-a"}}}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): created = true response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":""}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/actual-container-id/start": started = true response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) if response.Code != http.StatusCreated || !started { t.Fatalf("expected invalid create response reconciliation, status=%d started=%v body=%s", response.Code, started, response.Body.String()) } } func TestGatewayRemovesFailedContainerAndPreservesProfile(t *testing.T) { removed := false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) _, _ = response.Write([]byte(`{}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"failed-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/failed-id/start"): http.Error(response, "start failed", http.StatusInternalServerError) case request.Method == http.MethodDelete && strings.Contains(request.URL.Path, "/containers/failed-id"): removed = request.URL.Query().Get("force") == "1" && request.URL.Query().Get("v") == "0" response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) if response.Code != http.StatusBadGateway || !removed { t.Fatalf("expected failed container cleanup with preserved volume, status=%d removed=%v body=%s", response.Code, removed, response.Body.String()) } } func TestGatewayDoesNotEchoProxyCredentialsFromDockerErrors(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/") { response.WriteHeader(http.StatusOK) return } if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"+namePrefix) { response.WriteHeader(http.StatusNotFound) return } if request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create") { response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(`invalid cmd --proxy-server=http://operator:ephemeral@proxy.example:8080`)) return } t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path) }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) body := strings.Replace(testCreateBody, `"protocol":"socks5","host":"proxy.example","port":1080`, `"protocol":"http","host":"proxy.example","port":8080,"username":"operator","password":"ephemeral"`, 1) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) if response.Code != http.StatusBadGateway || strings.Contains(response.Body.String(), "operator") || strings.Contains(response.Body.String(), "ephemeral") || strings.Contains(response.Body.String(), "proxy.example") { t.Fatalf("gateway leaked proxy material: status=%d body=%s", response.Code, response.Body.String()) } } func TestGatewayListsBrowsers(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { if request.Method != http.MethodGet || request.URL.Path != "/containers/json" { t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } _, _ = response.Write([]byte(`[{"Id":"container-id","State":"running","Status":"Up","Labels":{` + `"` + idLabel + `":"account-a","` + nameLabel + `":"账号甲"}}]`)) }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodGet, "/v1/browsers", nil)) var browsers []browser if response.Code != http.StatusOK || json.NewDecoder(response.Body).Decode(&browsers) != nil || len(browsers) != 1 || browsers[0].Alias != "account-a" || browsers[0].Name != "账号甲" || browsers[0].Endpoint != "http://creatorhub-browser-account-a:9222" { t.Fatalf("unexpected list response status=%d body=%s", response.Code, response.Body.String()) } } func TestGatewayRestartRestoresExistingProxyListener(t *testing.T) { reserved, err := net.Listen("tcp4", "127.0.0.1:0") if err != nil { t.Fatal(err) } port := reserved.Addr().(*net.TCPAddr).Port _ = reserved.Close() labels := map[string]string{ managedLabel: "true", idLabel: "account-a", nameLabel: "账号甲", bindingVersionLabel: "1", networkExitLabel: "exit-1", proxyPortLabel: strconv.Itoa(port), networkIDLabel: "network-account-a", } docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasSuffix(request.URL.Path, "/containers/creatorhub-browser-account-a/json"): _ = json.NewEncoder(response).Encode(map[string]any{"Id": "container-id", "Config": map[string]any{"Labels": labels}}) case request.Method == http.MethodGet && request.URL.Path == "/containers/json": _ = json.NewEncoder(response).Encode([]map[string]any{{"Id": "container-id", "State": "running", "Status": "Up", "Labels": labels}}) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) recovery := `{"binding_version":1,"runtime_id":"container-id","network_id":"network-account-a","network_exit_id":"exit-1","network_exit":{"protocol":"http","host":"127.0.0.1","port":1}}` response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(recovery))) if response.Code != http.StatusNoContent { t.Fatalf("proxy recovery failed: %d %s", response.Code, response.Body.String()) } response = httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodGet, "/v1/browsers", nil)) var browsers []browser if response.Code != http.StatusOK || json.NewDecoder(response.Body).Decode(&browsers) != nil || len(browsers) != 1 || !browsers[0].ProxyReady { t.Fatalf("restarted gateway did not report restored proxy: %d %s", response.Code, response.Body.String()) } } func TestGatewayRestoreFinalFenceRemovesStaleProxy(t *testing.T) { reserved, err := net.Listen("tcp4", "127.0.0.1:0") if err != nil { t.Fatal(err) } port := reserved.Addr().(*net.TCPAddr).Port _ = reserved.Close() registry := newMemoryProxyRegistry() networkID := "network-n1" replaced := false containerReads := 0 exit := gatewayProxyExit{Protocol: "socks5", Host: "proxy.example", Port: 1080} proxyURL, cleanup, err := registry.configure("account-a", 1, "127.0.0.1", port, exit, "network-n1") if err != nil || !registry.bind("account-a", 1, proxyURL, "container-c1", "network-n1") { t.Fatalf("seed existing proxy generation: %v", err) } defer cleanup() labels := map[string]string{ managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "1", networkExitLabel: "exit-1", proxyPortLabel: strconv.Itoa(port), networkIDLabel: "network-n1", } server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": containerReads++ registry.mu.Lock() proxy := registry.proxies["account-a"] bound := proxy != nil && proxy.runtimeID == "container-c1" registry.mu.Unlock() if bound && containerReads == 4 && !replaced { networkID, replaced = "network-n2", true } _ = json.NewEncoder(response).Encode(map[string]any{"Id": "container-c1", "Config": map[string]any{"Labels": labels}}) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): reference := strings.TrimPrefix(request.URL.Path, "/networks/") if reference != networkID && reference != "creatorhub_browser-account-a" { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": networkID, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": map[string]any{ "container-c1": map[string]string{"Name": namePrefix + "account-a", "IPv4Address": "127.0.0.3/8"}, "gateway-self": map[string]string{"Name": "gateway-self", "IPv4Address": "127.0.0.1/8"}, }, }) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-self", token: testToken, proxies: registry, locks: &dockerAliasReservations{docker: docker, self: "gateway-self"}} app := fiber.New() app.Use("/v1", api.authorize) app.Post("/v1/browsers/:id/proxy", api.restoreProxy) body := `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` response := httptest.NewRecorder() adaptor.FiberApp(app).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(body))) registry.mu.Lock() proxy := registry.proxies["account-a"] registry.mu.Unlock() if response.Code != http.StatusConflict || !replaced || networkID != "network-n2" || proxy != nil { t.Fatalf("restore final fence accepted a replaced network: status=%d replaced=%v network=%q proxy=%v body=%s", response.Code, replaced, networkID, proxy, response.Body.String()) } } func TestGatewayProxyFenceRejectsGatewayAddressChange(t *testing.T) { labels := map[string]string{managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "1", networkExitLabel: "exit-1", networkIDLabel: "network-n1"} server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch request.URL.Path { case "/containers/" + namePrefix + "account-a/json": _ = json.NewEncoder(response).Encode(map[string]any{"Id": "container-c1", "Config": map[string]any{"Labels": labels}}) case "/containers/gateway-self/json": _, _ = response.Write([]byte(`{"Id":"gateway-self","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case "/networks/network-n1": _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": map[string]any{ "container-c1": map[string]string{"Name": namePrefix + "account-a", "IPv4Address": "127.0.0.2/8"}, "gateway-self": map[string]string{"Name": "gateway-self", "IPv4Address": "127.0.0.4/8"}, }, }) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-self"} input := proxyRestoreRequest{BindingVersion: 1, RuntimeID: "container-c1", NetworkID: "network-n1", NetworkExitID: "exit-1"} expected := tenantNetworkGeneration{ID: "network-n1", Name: "creatorhub_browser-account-a", RuntimeAttached: true, GatewayMembers: []string{"gateway-self"}, SelfMember: "gateway-self"} if err := api.requireProxyNetworkGeneration("account-a", input, expected, "127.0.0.3"); err != errGenerationConflict { t.Fatalf("gateway address change crossed proxy fence: %v", err) } } func TestGatewayLifecycleUsesInspectedImmutableContainerID(t *testing.T) { tests := []struct { method string path string dockerPath string }{ {http.MethodPost, "/v1/browsers/account-a/start", "/containers/container-id/start"}, {http.MethodPost, "/v1/browsers/account-a/stop", "/containers/container-id/stop"}, {http.MethodDelete, "/v1/browsers/account-a", "/containers/container-id"}, } for _, test := range tests { t.Run(test.method+" "+test.path, func(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodGet { networkLabel := "" if test.path == "/v1/browsers/account-a/start" { networkLabel = `,"` + networkIDLabel + `":"network-id"` } _, _ = response.Write([]byte(`{"Id":"container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1"` + networkLabel + `}}}`)) return } if request.URL.Path != test.dockerPath { t.Fatalf("unexpected Docker path %s", request.URL.String()) } response.WriteHeader(http.StatusNoContent) }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() var body io.Reader if test.path == "/v1/browsers/account-a/start" { body = strings.NewReader(`{"binding_version":1,"runtime_id":"container-id","network_id":"network-id"}`) } else { body = strings.NewReader(testGenerationBody) } adaptor.FiberApp(handler).ServeHTTP(response, authed(test.method, test.path, body)) if response.Code != http.StatusNoContent { t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String()) } }) } } func TestGatewayStartRejectsEmptyNetworkGeneration(t *testing.T) { mutations := 0 docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { if request.Method != http.MethodGet { mutations++ } _, _ = response.Write([]byte(`{"Id":"container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1"}}}`)) }) defer server.Close() response := httptest.NewRecorder() adaptor.FiberApp(newGateway(docker, "creatorhub_browser", testToken)).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/start", strings.NewReader(`{"binding_version":1,"runtime_id":"container-id"}`))) if response.Code != http.StatusBadRequest || mutations != 0 { t.Fatalf("empty network generation reached Docker: status=%d mutations=%d body=%s", response.Code, mutations, response.Body.String()) } } func TestGatewayDeleteUsesImmutableNetworkIDAcrossCleanupRetry(t *testing.T) { containerExists, cleanupFails, containerDeletes := true, true, 0 networkExists := true networkMembers := map[string]any{ "container-id": map[string]string{"Name": namePrefix + "account-a"}, "gateway-self": map[string]string{"Name": "gateway-self"}, } server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"): if !containerExists { response.WriteHeader(http.StatusNotFound) return } _, _ = response.Write([]byte(`{"Id":"container-id","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-id"}}}`)) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): containerExists = false containerDeletes++ response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): if !networkExists { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-id", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": networkMembers, }) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): if request.URL.Path != "/networks/network-id/disconnect" { t.Fatalf("network cleanup did not use immutable id: %s", request.URL.Path) } if cleanupFails { response.WriteHeader(http.StatusInternalServerError) return } var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) delete(networkMembers, body.Container) response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): if request.URL.Path != "/networks/network-id" { t.Fatalf("network delete did not use immutable id: %s", request.URL.Path) } networkExists = false response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() deleteBody := `{"binding_version":1,"runtime_id":"container-id","network_id":"network-id"}` adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(deleteBody))) if response.Code != http.StatusAccepted || !containerExists || containerDeletes != 0 { t.Fatalf("expected alias reservation with pending cleanup, status=%d exists=%v deletes=%d body=%s", response.Code, containerExists, containerDeletes, response.Body.String()) } cleanupFails = false response = httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(deleteBody))) if response.Code != http.StatusNoContent || containerExists || containerDeletes != 1 { t.Fatalf("idempotent cleanup retry failed: status=%d deletes=%d body=%s", response.Code, containerDeletes, response.Body.String()) } } func TestGatewayDeleteWithoutContainerOrNetworkGenerationFailsClosed(t *testing.T) { networkRequests := 0 server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": response.WriteHeader(http.StatusNotFound) case strings.HasPrefix(request.URL.Path, "/networks/"): networkRequests++ response.WriteHeader(http.StatusInternalServerError) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() body := `{"binding_version":1,"runtime_id":"runtime-not-found"}` adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(body))) if response.Code != http.StatusConflict || networkRequests != 0 { t.Fatalf("legacy cleanup discovered a replacement network: status=%d networkRequests=%d body=%s", response.Code, networkRequests, response.Body.String()) } } func TestGatewayRejectsStaleProxyRestoreAfterReplacementGeneration(t *testing.T) { type dockerState struct { sync.Mutex containerID string containerLabels map[string]string networkID string networkMembers map[string]string containerReads int cleanupMutations []string } state := &dockerState{ containerID: "container-c1", containerLabels: map[string]string{managedLabel: "true", idLabel: "account-a", bindingVersionLabel: "1", networkExitLabel: "exit-1", proxyPortLabel: "12345", networkIDLabel: "network-n1"}, networkID: "network-n1", networkMembers: map[string]string{"container-c1": namePrefix + "account-a", "gateway-self": "gateway-self"}, } r1Captured := make(chan struct{}) resumeR1 := make(chan struct{}) var releaseOnce sync.Once release := func() { releaseOnce.Do(func() { close(resumeR1) }) } defer release() dockerServer := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": state.Lock() containerID := state.containerID labels := state.containerLabels state.containerReads++ first := state.containerReads == 1 state.Unlock() if first { close(r1Captured) <-resumeR1 } if containerID == "" { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{"Id": containerID, "Config": map[string]any{"Labels": labels}}) case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1" || request.URL.Path == "/networks/network-n2"): state.Lock() networkID := state.networkID members := make(map[string]map[string]string, len(state.networkMembers)) for id, name := range state.networkMembers { members[id] = map[string]string{"Name": name, "IPv4Address": "127.0.0.1/8"} } state.Unlock() if networkID == "" { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": networkID, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": members, }) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) requestedNetwork := strings.TrimSuffix(strings.TrimPrefix(request.URL.Path, "/networks/"), "/disconnect") state.Lock() if requestedNetwork != state.networkID { state.Unlock() response.WriteHeader(http.StatusNotFound) return } state.cleanupMutations = append(state.cleanupMutations, "disconnect:"+requestedNetwork+":"+body.Container) delete(state.networkMembers, body.Container) state.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): requestedNetwork := strings.TrimPrefix(request.URL.Path, "/networks/") state.Lock() if requestedNetwork != state.networkID { state.Unlock() response.WriteHeader(http.StatusNotFound) return } state.cleanupMutations = append(state.cleanupMutations, "delete-network:"+requestedNetwork) state.networkID = "" state.networkMembers = nil state.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): requestedContainer := strings.TrimPrefix(request.URL.Path, "/containers/") state.Lock() if requestedContainer != state.containerID { state.Unlock() response.WriteHeader(http.StatusNotFound) return } state.cleanupMutations = append(state.cleanupMutations, "delete-container:"+requestedContainer) state.containerID = "" state.containerLabels = nil state.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": state.Lock() state.networkID = "network-n2" state.networkMembers = map[string]string{} state.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"network-n2"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): state.Lock() state.networkMembers["gateway-self"] = "gateway-self" state.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) state.Lock() state.containerID = "container-c2" state.containerLabels = payload.Labels state.networkMembers["container-c2"] = namePrefix + "account-a" state.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-c2"}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/container-c2/start": response.WriteHeader(http.StatusNoContent) default: t.Errorf("unexpected Docker request %s %s", request.Method, request.URL.String()) response.WriteHeader(http.StatusInternalServerError) } })) defer dockerServer.Close() registry := newMemoryProxyRegistry() proxyURL, _, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{Protocol: "socks5", Host: "proxy.example", Port: 1080}) if err != nil || !registry.bind("account-a", 1, proxyURL, "container-c1", "network-n1") { t.Fatalf("seed C1 proxy: %v", err) } defer func() { registry.remove("account-a", 1, "container-c2") registry.remove("account-a", 1, "container-c1") }() api := gateway{ docker: dockerClient{baseURL: dockerServer.URL, client: dockerServer.Client(), slow: dockerServer.Client()}, network: "creatorhub_browser", self: "gateway-self", token: testToken, proxies: registry, } api.locks = &dockerAliasReservations{docker: api.docker, self: api.self} app := fiber.New() app.Use("/v1", api.authorize) app.Post("/v1/browsers", api.create) app.Post("/v1/browsers/:id/proxy", api.restoreProxy) app.Delete("/v1/browsers/:id", api.remove) gatewayServer := httptest.NewServer(adaptor.FiberApp(app)) defer gatewayServer.Close() defer release() type result struct { status int body string } call := func(method, path, body string) result { request, _ := http.NewRequest(method, gatewayServer.URL+path, strings.NewReader(body)) request.Header.Set("Authorization", "Bearer "+testToken) response, err := gatewayServer.Client().Do(request) if err != nil { return result{body: err.Error()} } defer response.Body.Close() responseBody, _ := io.ReadAll(response.Body) return result{status: response.StatusCode, body: string(responseBody)} } r1Result := make(chan result, 1) go func() { r1Result <- call(http.MethodPost, "/v1/browsers/account-a/proxy", `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}`) }() select { case <-r1Captured: case <-time.After(5 * time.Second): t.Fatal("R1 did not capture C1/N1") } r2Delete := call(http.MethodDelete, "/v1/browsers/account-a", `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1"}`) if r2Delete.status != http.StatusNoContent { t.Fatalf("R2 cleanup failed: status=%d body=%s", r2Delete.status, r2Delete.body) } r2Create := call(http.MethodPost, "/v1/browsers", testCreateBody) if r2Create.status != http.StatusCreated { t.Fatalf("R2 replacement create failed: status=%d body=%s", r2Create.status, r2Create.body) } release() select { case stale := <-r1Result: if stale.status != http.StatusConflict { t.Fatalf("R1 stale restore was not fenced: status=%d body=%s", stale.status, stale.body) } case <-time.After(5 * time.Second): t.Fatal("R1 did not finish") } state.Lock() defer state.Unlock() proxy := registry.proxies["account-a"] wantCleanup := []string{ "disconnect:network-n1:container-c1", "disconnect:network-n1:gateway-self", "delete-network:network-n1", "delete-container:container-c1", } if state.containerID != "container-c2" || state.networkID != "network-n2" || state.networkMembers["container-c2"] == "" || proxy == nil || proxy.runtimeID != "container-c2" || strings.Join(state.cleanupMutations, ",") != strings.Join(wantCleanup, ",") { t.Fatalf("stale R1 affected replacement generation: container=%q network=%q members=%v proxy=%v mutations=%v", state.containerID, state.networkID, state.networkMembers, proxy, state.cleanupMutations) } } func TestGatewayRejectsStaleCreateBeforeNetworkOrProxyMutation(t *testing.T) { type dockerState struct { sync.Mutex containerID string containerLabels map[string]string containerReads int networkExists bool networkCreates int networkConnects int gatewayConnected bool containerCreates int } state := &dockerState{} r1Inspected := make(chan struct{}) resumeR1 := make(chan struct{}) var releaseOnce sync.Once release := func() { releaseOnce.Do(func() { close(resumeR1) }) } defer release() dockerServer := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": state.Lock() containerID, labels := state.containerID, state.containerLabels state.containerReads++ first := state.containerReads == 1 state.Unlock() if first { close(r1Inspected) <-resumeR1 } if containerID == "" { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{"Id": containerID, "Config": map[string]any{"Labels": labels}}) case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1"): state.Lock() exists, connected := state.networkExists, state.gatewayConnected state.Unlock() if !exists { response.WriteHeader(http.StatusNotFound) return } members := map[string]any{} if connected { members["gateway-self"] = map[string]string{"Name": "gateway-self", "IPv4Address": "127.0.0.1/8"} } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": members, }) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": state.Lock() state.networkExists = true state.networkCreates++ state.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): state.Lock() state.networkConnects++ state.gatewayConnected = true state.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) state.Lock() state.containerID = "container-c1" state.containerLabels = payload.Labels state.containerCreates++ state.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-c1"}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/container-c1/start": response.WriteHeader(http.StatusNoContent) default: t.Errorf("unexpected Docker request %s %s", request.Method, request.URL.String()) response.WriteHeader(http.StatusInternalServerError) } })) defer dockerServer.Close() docker := dockerClient{baseURL: dockerServer.URL, client: dockerServer.Client(), slow: dockerServer.Client()} registry := newMemoryProxyRegistry() defer registry.remove("account-a", 1, "container-c1") api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-self", token: testToken, proxies: registry, locks: &dockerAliasReservations{docker: docker, self: "gateway-self"}} app := fiber.New() app.Use("/v1", api.authorize) app.Post("/v1/browsers", api.create) gatewayServer := httptest.NewServer(adaptor.FiberApp(app)) defer gatewayServer.Close() call := func() (int, string) { request, _ := http.NewRequest(http.MethodPost, gatewayServer.URL+"/v1/browsers", strings.NewReader(testCreateBody)) request.Header.Set("Authorization", "Bearer "+testToken) response, err := gatewayServer.Client().Do(request) if err != nil { return 0, err.Error() } defer response.Body.Close() body, _ := io.ReadAll(response.Body) return response.StatusCode, string(body) } r1Result := make(chan struct { status int body string }, 1) go func() { status, body := call() r1Result <- struct { status int body string }{status, body} }() select { case <-r1Inspected: case <-time.After(5 * time.Second): t.Fatal("R1 did not inspect the empty alias") } status, body := call() if status != http.StatusCreated { release() t.Fatalf("R2 create failed: status=%d body=%s", status, body) } release() select { case stale := <-r1Result: if stale.status != http.StatusConflict { t.Fatalf("R1 stale create was not fenced: status=%d body=%s", stale.status, stale.body) } case <-time.After(5 * time.Second): t.Fatal("R1 did not finish") } state.Lock() defer state.Unlock() proxy := registry.proxies["account-a"] if state.containerID != "container-c1" || state.networkCreates != 1 || state.networkConnects != 1 || state.containerCreates != 1 || proxy == nil || proxy.runtimeID != "container-c1" { t.Fatalf("stale create left side effects: container=%q networkCreates=%d connects=%d containerCreates=%d proxy=%v", state.containerID, state.networkCreates, state.networkConnects, state.containerCreates, proxy) } } func TestGatewayTwoReplicaCreateRestoreRemoveProxyContract(t *testing.T) { type member struct{ name, ip string } type dockerState struct { sync.Mutex containerID string containerLabels map[string]string networkID string networkMembers map[string]member sequence int } state := &dockerState{} dockerServer := httptest.NewServer(withAliasReservations("gateway-g1", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-g2/json": _, _ = response.Write([]byte(`{"Id":"gateway-g2","Image":"gateway-image-id","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": state.Lock() id, labels := state.containerID, state.containerLabels state.Unlock() if id == "" { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{"Id": id, "Config": map[string]any{"Labels": labels}}) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): state.Lock() id := state.networkID members := map[string]any{} for memberID, current := range state.networkMembers { members[memberID] = map[string]string{"Name": current.name, "IPv4Address": current.ip} } state.Unlock() if id == "" { response.WriteHeader(http.StatusNotFound) return } reference := strings.TrimPrefix(request.URL.Path, "/networks/") if reference != "creatorhub_browser-account-a" && reference != id { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": id, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": members, }) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": state.Lock() state.sequence++ state.networkID = fmt.Sprintf("network-n%d", state.sequence) state.networkMembers = map[string]member{} id := state.networkID state.Unlock() response.WriteHeader(http.StatusCreated) _ = json.NewEncoder(response).Encode(map[string]string{"Id": id}) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) state.Lock() ip := "127.0.0.3/8" if body.Container == "gateway-g1" { ip = "127.0.0.1/8" } else if body.Container == "gateway-g2" { ip = "127.0.0.2/8" } state.networkMembers[body.Container] = member{name: body.Container, ip: ip} state.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": var payload struct { Labels map[string]string `json:"Labels"` } _ = json.NewDecoder(request.Body).Decode(&payload) state.Lock() id := fmt.Sprintf("container-c%d", state.sequence) state.containerID, state.containerLabels = id, payload.Labels state.networkMembers[id] = member{name: namePrefix + "account-a", ip: "127.0.0.3/8"} state.Unlock() response.WriteHeader(http.StatusCreated) _ = json.NewEncoder(response).Encode(map[string]string{"Id": id}) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"): response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) state.Lock() delete(state.networkMembers, body.Container) state.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): state.Lock() if len(state.networkMembers) != 0 { state.Unlock() response.WriteHeader(http.StatusConflict) return } state.networkID = "" state.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): id := strings.TrimPrefix(request.URL.Path, "/containers/") state.Lock() if id != state.containerID { state.Unlock() response.WriteHeader(http.StatusNotFound) return } delete(state.networkMembers, id) state.containerID, state.containerLabels = "", nil state.Unlock() response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer dockerServer.Close() docker := dockerClient{baseURL: dockerServer.URL, client: dockerServer.Client(), slow: dockerServer.Client()} newReplica := func(self string) (*fiber.App, *memoryProxyRegistry) { registry := newMemoryProxyRegistry() api := gateway{docker: docker, network: "creatorhub_browser", self: self, token: testToken, proxies: registry, locks: &dockerAliasReservations{docker: docker, self: self}} app := fiber.New() app.Use("/v1", api.authorize) app.Post("/v1/browsers", api.create) app.Post("/v1/browsers/:id/proxy", api.restoreProxy) app.Delete("/v1/browsers/:id", api.remove) return app, registry } g1, proxiesG1 := newReplica("gateway-g1") g2, proxiesG2 := newReplica("gateway-g2") call := func(app *fiber.App, method, path, body string) *httptest.ResponseRecorder { response := httptest.NewRecorder() adaptor.FiberApp(app).ServeHTTP(response, authed(method, path, strings.NewReader(body))) return response } created := call(g1, http.MethodPost, "/v1/browsers", testCreateBody) if created.Code != http.StatusCreated { t.Fatalf("G1 create failed: %d %s", created.Code, created.Body.String()) } state.Lock() c1, n1, port := state.containerID, state.networkID, state.containerLabels[proxyPortLabel] state.Unlock() restoreBody := `{"binding_version":1,"runtime_id":"` + c1 + `","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` if restored := call(g2, http.MethodPost, "/v1/browsers/account-a/proxy", restoreBody); restored.Code != http.StatusNoContent { t.Fatalf("G2 restore failed: %d %s", restored.Code, restored.Body.String()) } if proxiesG1.proxies["account-a"].runtimeID != c1 || proxiesG2.proxies["account-a"].runtimeID != c1 || port == "" { t.Fatal("both replicas did not bind the same runtime generation") } removeBody := `{"binding_version":1,"runtime_id":"` + c1 + `","network_id":"` + n1 + `"}` if removed := call(g2, http.MethodDelete, "/v1/browsers/account-a", removeBody); removed.Code != http.StatusNoContent { t.Fatalf("G2 remove failed: %d %s", removed.Code, removed.Body.String()) } if replacement := call(g2, http.MethodPost, "/v1/browsers", testCreateBody); replacement.Code != http.StatusCreated { t.Fatalf("G2 replacement create failed: %d %s", replacement.Code, replacement.Body.String()) } state.Lock() c2 := state.containerID _, g1Attached := state.networkMembers["gateway-g1"] _, g2Attached := state.networkMembers["gateway-g2"] _, c2Attached := state.networkMembers[c2] state.Unlock() if c2 == c1 || g1Attached || !g2Attached || !c2Attached || proxiesG1.proxies["account-a"].runtimeID != c1 || proxiesG2.proxies["account-a"].runtimeID != c2 { t.Fatalf("cross-process proxy release contract failed: c1=%q c2=%q g1=%v g2=%v c2Attached=%v", c1, c2, g1Attached, g2Attached, c2Attached) } state.Lock() n2 := state.networkID port = state.containerLabels[proxyPortLabel] state.Unlock() restoreBody = `{"binding_version":1,"runtime_id":"` + c2 + `","network_id":"` + n2 + `","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` if restored := call(g1, http.MethodPost, "/v1/browsers/account-a/proxy", restoreBody); restored.Code != http.StatusNoContent { t.Fatalf("G1 successor restore failed: %d %s", restored.Code, restored.Body.String()) } state.Lock() _, g1Attached = state.networkMembers["gateway-g1"] state.Unlock() if !g1Attached || proxiesG1.proxies["account-a"].runtimeID != c2 || proxiesG2.proxies["account-a"].runtimeID != c2 { t.Fatalf("stale replica did not replace its proxy generation: c1=%q c2=%q g1=%v g1Proxy=%v g2Proxy=%v", c1, c2, g1Attached, proxiesG1.proxies["account-a"], proxiesG2.proxies["account-a"]) } proxiesG1.remove("account-a", 1, c1) proxiesG2.remove("account-a", 1, c2) } func TestGatewayRemoveFencesNetworkReplacementAndMemberChanges(t *testing.T) { for _, test := range []struct { name string replaceOnDisconnect bool addMemberOnDisconnect bool wantNetworkID string }{ {name: "N1 replaced by N2 after inspect", replaceOnDisconnect: true, wantNetworkID: "network-n2"}, {name: "trusted member joins before delete", addMemberOnDisconnect: true, wantNetworkID: "network-n1"}, } { t.Run(test.name, func(t *testing.T) { var mu sync.Mutex networkID := "network-n1" members := map[string]string{"container-c1": namePrefix + "account-a", "gateway-g1": "gateway-g1"} containerDeleted, networkDeletes := false, 0 server := httptest.NewServer(withAliasReservations("gateway-g1", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-n1"}}}`)) case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-g2/json": _, _ = response.Write([]byte(`{"Id":"gateway-g2","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1" || request.URL.Path == "/networks/network-n2"): mu.Lock() id := networkID current := map[string]any{} for memberID, name := range members { current[memberID] = map[string]string{"Name": name} } mu.Unlock() _ = json.NewEncoder(response).Encode(map[string]any{ "Id": id, "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": current, }) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) mu.Lock() if test.replaceOnDisconnect { networkID = "network-n2" mu.Unlock() response.WriteHeader(http.StatusNotFound) return } delete(members, body.Container) if test.addMemberOnDisconnect && body.Container == "gateway-g1" { members["gateway-g2"] = "gateway-g2" } mu.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): mu.Lock() networkDeletes++ mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodDelete && request.URL.Path == "/containers/container-c1": mu.Lock() containerDeleted = true mu.Unlock() response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} registry := newMemoryProxyRegistry() proxyURL, _, err := registry.configure("account-a", 1, "127.0.0.1", 0, gatewayProxyExit{Protocol: "socks5", Host: "proxy.example", Port: 1080}) if err != nil || !registry.bind("account-a", 1, proxyURL, "container-c1") { t.Fatal("seed proxy generation") } defer registry.remove("account-a", 1, "container-c1") api := gateway{docker: docker, network: "creatorhub_browser", self: "gateway-g1", token: testToken, proxies: registry, locks: &dockerAliasReservations{docker: docker, self: "gateway-g1"}} app := fiber.New() app.Use("/v1", api.authorize) app.Delete("/v1/browsers/:id", api.remove) response := httptest.NewRecorder() adaptor.FiberApp(app).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(`{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1"}`))) mu.Lock() defer mu.Unlock() if response.Code != http.StatusConflict || networkID != test.wantNetworkID || containerDeleted || networkDeletes != 0 || registry.proxies["account-a"] == nil { t.Fatalf("remove crossed network fence: status=%d network=%q containerDeleted=%v networkDeletes=%d proxy=%v body=%s", response.Code, networkID, containerDeleted, networkDeletes, registry.proxies["account-a"], response.Body.String()) } }) } } func TestGatewayCreateFailureRemovesCreatedNetworkGeneration(t *testing.T) { for _, failure := range []string{"network-create-id", "configure", "container-create", "start"} { t.Run(failure, func(t *testing.T) { var mu sync.Mutex networkExists, containerExists := false, false members := map[string]string{} networkDeletes := 0 server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": mu.Lock() exists := containerExists mu.Unlock() if !exists { response.WriteHeader(http.StatusNotFound) return } _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkIDLabel + `":"network-n1"}}}`)) case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1"): mu.Lock() exists := networkExists current := map[string]any{} for id, name := range members { ip := "127.0.0.1/8" if failure == "configure" && id == "gateway-self" { ip = "192.0.2.1/24" } current[id] = map[string]string{"Name": name, "IPv4Address": ip} } mu.Unlock() if !exists { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": current, }) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": mu.Lock() networkExists = true mu.Unlock() response.WriteHeader(http.StatusCreated) if failure == "network-create-id" { _, _ = response.Write([]byte(`{"Id":""}`)) } else { _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) } case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect") && !strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) mu.Lock() members[body.Container] = body.Container mu.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && request.URL.Path == "/containers/create": if failure == "container-create" { response.WriteHeader(http.StatusInternalServerError) return } mu.Lock() containerExists = true members["container-c1"] = namePrefix + "account-a" mu.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"container-c1"}`)) case request.Method == http.MethodPost && request.URL.Path == "/containers/container-c1/start": if failure == "start" { response.WriteHeader(http.StatusInternalServerError) return } response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodDelete && request.URL.Path == "/containers/container-c1": mu.Lock() containerExists = false delete(members, "container-c1") mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) mu.Lock() delete(members, body.Container) mu.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && request.URL.Path == "/networks/network-n1": mu.Lock() networkExists = false networkDeletes++ mu.Unlock() response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} handler := newGatewayWithSelf(docker, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) mu.Lock() defer mu.Unlock() wantNetwork, wantDeletes := false, 1 if failure == "network-create-id" { wantNetwork, wantDeletes = true, 0 } if response.Code != http.StatusBadGateway || networkExists != wantNetwork || networkDeletes != wantDeletes || containerExists { t.Fatalf("%s failure left managed resources: status=%d network=%v deletes=%d container=%v body=%s", failure, response.Code, networkExists, networkDeletes, containerExists, response.Body.String()) } }) } } func TestGatewayNetworkCreateDisconnectDoesNotDiscoverReplacementByName(t *testing.T) { networkReads, networkDeletes := 0, 0 networkCreated := false server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): networkReads++ if networkCreated { _, _ = response.Write([]byte(`{"Id":"network-n2","Name":"creatorhub_browser-account-a","Driver":"bridge","Labels":{"` + managedLabel + `":"true","` + networkRoleLabel + `":"` + browserNetworkRole + `","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1"},"Containers":{}}`)) return } response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": networkCreated = true connection, _, _ := response.(http.Hijacker).Hijack() _ = connection.Close() case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): networkDeletes++ response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(testCreateBody))) if response.Code != http.StatusBadGateway || !networkCreated || networkReads != 1 || networkDeletes != 0 { t.Fatalf("unknown N1 was discovered or cleaned as N2: status=%d created=%v reads=%d deletes=%d body=%s", response.Code, networkCreated, networkReads, networkDeletes, response.Body.String()) } } func TestGatewayCompensatesConnectThatAppliedBeforeError(t *testing.T) { for _, restore := range []bool{false, true} { name := "create self" failedMember := "gateway-self" if restore { name, failedMember = "restore runtime", "container-c1" } t.Run(name, func(t *testing.T) { networkExists := false members := map[string]string{} disconnected := []string{} networkDeletes := 0 server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": if !restore { response.WriteHeader(http.StatusNotFound) return } _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkExitLabel + `":"exit-1","` + networkIDLabel + `":"network-n1","` + proxyPortLabel + `":"12345"}}}`)) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): if !networkExists { response.WriteHeader(http.StatusNotFound) return } containers := map[string]any{} for id, memberName := range members { containers[id] = map[string]string{"Name": memberName, "IPv4Address": "127.0.0.1/8"} } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": containers, }) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": networkExists = true response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect") && !strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) members[body.Container] = body.Container if body.Container == failedMember { response.WriteHeader(http.StatusInternalServerError) return } response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) delete(members, body.Container) disconnected = append(disconnected, body.Container) response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && request.URL.Path == "/networks/network-n1": networkExists = false networkDeletes++ response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, "creatorhub_browser", testToken, "gateway-self") method, path, body := http.MethodPost, "/v1/browsers", testCreateBody if restore { path = "/v1/browsers/account-a/proxy" body = `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` } response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(method, path, strings.NewReader(body))) wantDeletes, wantDisconnects := 1, 1 if restore { wantDeletes, wantDisconnects = 0, 0 } if response.Code != http.StatusBadGateway || networkExists || networkDeletes != wantDeletes || len(disconnected) != wantDisconnects || (wantDisconnects == 1 && disconnected[0] != failedMember) { t.Fatalf("applied connect was not compensated: status=%d network=%v deletes=%d disconnected=%v body=%s", response.Code, networkExists, networkDeletes, disconnected, response.Body.String()) } }) } } func TestGatewayRestoreFailureRemovesCreatedNetworkGeneration(t *testing.T) { var mu sync.Mutex networkExists := false members := map[string]string{} networkDeletes, containerDeletes := 0, 0 server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && request.URL.Path == "/containers/"+namePrefix+"account-a/json": _, _ = response.Write([]byte(`{"Id":"container-c1","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"1","` + networkExitLabel + `":"exit-1","` + networkIDLabel + `":"network-n1","` + proxyPortLabel + `":"12345"}}}`)) case request.Method == http.MethodGet && (request.URL.Path == "/networks/creatorhub_browser-account-a" || request.URL.Path == "/networks/network-n1"): mu.Lock() exists := networkExists current := map[string]any{} for id, name := range members { ip := "127.0.0.3/8" if id == "gateway-self" { ip = "192.0.2.1/24" } current[id] = map[string]string{"Name": name, "IPv4Address": ip} } mu.Unlock() if !exists { response.WriteHeader(http.StatusNotFound) return } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-n1", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": current, }) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": mu.Lock() networkExists = true mu.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"network-n1"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) mu.Lock() members[body.Container] = body.Container mu.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): var body struct { Container string `json:"Container"` } _ = json.NewDecoder(request.Body).Decode(&body) mu.Lock() delete(members, body.Container) mu.Unlock() response.WriteHeader(http.StatusOK) case request.Method == http.MethodDelete && request.URL.Path == "/networks/network-n1": mu.Lock() networkExists = false networkDeletes++ mu.Unlock() response.WriteHeader(http.StatusNoContent) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): mu.Lock() containerDeletes++ mu.Unlock() response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} handler := newGatewayWithSelf(docker, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() body := `{"binding_version":1,"runtime_id":"container-c1","network_id":"network-n1","network_exit_id":"exit-1","network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(body))) mu.Lock() defer mu.Unlock() if response.Code != http.StatusBadGateway || networkExists || networkDeletes != 0 || containerDeletes != 0 { t.Fatalf("restore failure left managed network: status=%d network=%v networkDeletes=%d containerDeletes=%d body=%s", response.Code, networkExists, networkDeletes, containerDeletes, response.Body.String()) } } func TestGatewayMapsDockerServiceFailureToBadGateway(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, _ *http.Request) { http.Error(response, "daemon unavailable", http.StatusInternalServerError) }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(testGenerationBody))) if response.Code != http.StatusBadGateway { t.Fatalf("expected 502 for Docker failure, got %d: %s", response.Code, response.Body.String()) } } func TestGatewayRefusesUnmanagedContainer(t *testing.T) { deleted := false docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { switch request.Method { case http.MethodGet: _, _ = response.Write([]byte(`{"Config":{"Labels":{}}}`)) case http.MethodDelete: deleted = true response.WriteHeader(http.StatusNoContent) } }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) request := authed(http.MethodDelete, "/v1/browsers/foreign", strings.NewReader(testGenerationBody)) response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, request) if response.Code != http.StatusForbidden || deleted { t.Fatalf("expected unmanaged container to be rejected, status=%d deleted=%v", response.Code, deleted) } } func TestGatewayRejectsStaleGenerationBeforeDockerMutation(t *testing.T) { for _, request := range []struct { method, path string }{ {http.MethodPost, "/v1/browsers/account-a/stop"}, {http.MethodPost, "/v1/browsers/account-a/start"}, {http.MethodDelete, "/v1/browsers/account-a"}, } { t.Run(request.method, func(t *testing.T) { mutations := 0 docker, server := testDocker(func(response http.ResponseWriter, dockerRequest *http.Request) { if dockerRequest.Method != http.MethodGet { mutations++ } _, _ = response.Write([]byte(`{"Id":"new-container","Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a","` + bindingVersionLabel + `":"2"}}}`)) }) defer server.Close() handler := newGateway(docker, "creatorhub_browser", testToken) response := httptest.NewRecorder() body := testGenerationBody if strings.HasSuffix(request.path, "/start") { body = `{"binding_version":1,"runtime_id":"container-id","network_id":"network-id"}` } adaptor.FiberApp(handler).ServeHTTP(response, authed(request.method, request.path, strings.NewReader(body))) if response.Code != http.StatusConflict || mutations != 0 { t.Fatalf("stale generation reached Docker mutation: status=%d mutations=%d body=%s", response.Code, mutations, response.Body.String()) } }) } } func TestGatewayRejectsStaleDeleteDuringNewNetworkCreation(t *testing.T) { mutations := 0 server := httptest.NewServer(withAliasReservations("gateway-self", func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"): response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-new", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "2"}, }) default: mutations++ response.WriteHeader(http.StatusNoContent) } })) defer server.Close() handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, "creatorhub_browser", testToken, "gateway-self") response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", strings.NewReader(testGenerationBody))) if response.Code != http.StatusConflict || mutations != 0 { t.Fatalf("stale delete crossed the new network generation: status=%d mutations=%d body=%s", response.Code, mutations, response.Body.String()) } } func TestEnsureTenantNetworkConnectsGatewayOnlyToRuntimeNetwork(t *testing.T) { created, connected := false, false server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { switch { case request.Method == http.MethodGet && !created: response.WriteHeader(http.StatusNotFound) case request.Method == http.MethodPost && request.URL.Path == "/networks/create": var body map[string]any _ = json.NewDecoder(request.Body).Decode(&body) labels := body["Labels"].(map[string]any) if body["Name"] != "creatorhub_browser-account-a" || labels[idLabel] != "account-a" { t.Fatalf("unexpected isolated network create: %#v", body) } created = true response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"Id":"network-id"}`)) case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): connected = true response.WriteHeader(http.StatusOK) case request.Method == http.MethodGet && request.URL.Path == "/containers/gateway-id/json": _, _ = response.Write([]byte(`{"Id":"gateway-id","Config":{"Labels":{"` + gatewayMemberLabel + `":"true"}}}`)) case request.Method == http.MethodGet: members := map[string]any{} if connected { members["gateway-id"] = map[string]string{"Name": "gateway-id", "IPv4Address": "127.0.0.3/8"} } _ = json.NewEncoder(response).Encode(map[string]any{ "Id": "network-id", "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a", bindingVersionLabel: "1"}, "Containers": members, }) default: t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path) } })) defer server.Close() docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} generation, bindHost, err := docker.ensureTenantNetwork("creatorhub_browser", "account-a", "gateway-id", 1, "", "", false) if err != nil || !created || !connected || generation.Name != "creatorhub_browser-account-a" || bindHost != "127.0.0.3" { t.Fatalf("isolated network was not created and connected: generation=%#v host=%q created=%v connected=%v err=%v", generation, bindHost, created, connected, err) } } func TestLoadConfigRequiresGatewayToken(t *testing.T) { t.Setenv("GATEWAY_TOKEN", "short") if _, err := loadConfig(); err == nil { t.Fatal("expected a short gateway token to be rejected") } } func TestLoadConfigRejectsControlNetwork(t *testing.T) { t.Setenv("GATEWAY_TOKEN", testToken) t.Setenv("BROWSER_NETWORK", controlNetworkName) if _, err := loadConfig(); err == nil { t.Fatal("expected control network configuration to be rejected") } }