HH-803: add stable network exit orchestration (#19)

This commit is contained in:
2026-08-29 07:59:55 +08:00
parent 884505bb55
commit f1ccfa0096
25 changed files with 6082 additions and 368 deletions
+301 -85
View File
@@ -27,29 +27,33 @@ import (
)
const (
browserUser = "1000:1000"
browserEntrypoint = "/usr/local/bin/docker-entrypoint.sh"
managedLabel = "io.creatorhub.managed"
idLabel = "io.creatorhub.runtime-id"
nameLabel = "io.creatorhub.display-name"
networkRoleLabel = "io.creatorhub.network-role"
browserNetworkRole = "browser"
controlNetworkName = "creatorhub_control"
namePrefix = "creatorhub-browser-"
pullTimeout = 10 * time.Minute
browserUser = "1000:1000"
browserEntrypoint = "/usr/local/bin/docker-entrypoint.sh"
managedLabel = "io.creatorhub.managed"
idLabel = "io.creatorhub.runtime-id"
nameLabel = "io.creatorhub.display-name"
bindingVersionLabel = "io.creatorhub.binding-version"
networkExitLabel = "io.creatorhub.network-exit-id"
proxyPortLabel = "io.creatorhub.proxy-port"
networkRoleLabel = "io.creatorhub.network-role"
browserNetworkRole = "browser"
controlNetworkName = "creatorhub_control"
namePrefix = "creatorhub-browser-"
pullTimeout = 10 * time.Minute
)
var (
runtimeIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
runtimeIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
networkNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`)
imageRefPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$`)
volumePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`)
imageRefPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$`)
volumePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`)
exitIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`)
)
var (
errInvalidRuntimeID = errors.New("invalid runtime id")
errUnmanagedContainer = errors.New("refusing to operate on a container not owned by CreatorHub")
errUnauthorized = errors.New("gateway token rejected")
errInvalidRuntimeID = errors.New("invalid runtime id")
errUnmanagedContainer = errors.New("refusing to operate on a container not owned by CreatorHub")
errUnauthorized = errors.New("gateway token rejected")
)
type serviceConfig struct {
@@ -69,25 +73,42 @@ type dockerClient struct {
type gateway struct {
docker dockerClient
network string
self string
token string
proxies *memoryProxyRegistry
}
// createRequest 全量字段由平台下发;网关不做业务决策,只做输入合法性校验。
type createRequest struct {
Alias string `json:"alias"`
Name string `json:"name"`
Image string `json:"image"`
Cmd []string `json:"cmd"`
Volume string `json:"volume"`
Alias string `json:"alias"`
Name string `json:"name"`
Image string `json:"image"`
Cmd []string `json:"cmd"`
Volume string `json:"volume"`
BindingVersion int64 `json:"binding_version"`
NetworkExitID string `json:"network_exit_id"`
NetworkExit gatewayProxyExit `json:"network_exit"`
Stopped bool `json:"stopped,omitempty"`
}
type gatewayProxyExit struct {
Protocol string `json:"protocol"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password"`
}
type browser struct {
ID string `json:"id"`
Alias string `json:"alias"`
Name string `json:"name"`
State string `json:"state"`
Status string `json:"status"`
Endpoint string `json:"endpoint"`
ID string `json:"id"`
Alias string `json:"alias"`
Name string `json:"name"`
State string `json:"state"`
Status string `json:"status"`
Endpoint string `json:"endpoint"`
BindingVersion int64 `json:"binding_version"`
NetworkExitID string `json:"network_exit_id"`
ProxyReady bool `json:"proxy_ready"`
}
func main() {
@@ -184,9 +205,6 @@ func run(command *cobra.Command, cfg serviceConfig) error {
client: &http.Client{Transport: transport, Timeout: 30 * time.Second},
slow: &http.Client{Transport: transport},
}
if err := docker.ensureBrowserNetwork(cfg.network); err != nil {
return err
}
logrus.WithFields(logrus.Fields{
"service": "docker-gateway",
"listen_addr": cfg.listenAddr,
@@ -199,7 +217,12 @@ func run(command *cobra.Command, cfg serviceConfig) error {
}
func newGateway(client dockerClient, network, token string) *fiber.App {
api := gateway{docker: client, network: network, token: token}
self, _ := os.Hostname()
return newGatewayWithSelf(client, network, token, self)
}
func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber.App {
api := gateway{docker: client, network: network, self: self, token: token, proxies: newMemoryProxyRegistry()}
app := fiber.New(fiber.Config{
AppName: "CreatorHub Docker gateway",
BodyLimit: 1 << 20,
@@ -221,6 +244,7 @@ func newGateway(client dockerClient, network, token string) *fiber.App {
app.Use("/v1", api.authorize)
app.Get("/v1/browsers", api.list)
app.Post("/v1/browsers", api.create)
app.Post("/v1/browsers/:id/proxy", api.restoreProxy)
app.Post("/v1/browsers/:id/:action", api.changeState)
app.Delete("/v1/browsers/:id", api.remove)
return app
@@ -265,13 +289,18 @@ func (api gateway) list(c fiber.Ctx) error {
if name == "" {
name = alias
}
bindingVersion, _ := strconv.ParseInt(container.Labels[bindingVersionLabel], 10, 64)
proxyPort, _ := strconv.Atoi(container.Labels[proxyPortLabel])
browsers = append(browsers, browser{
ID: container.ID,
Alias: alias,
Name: name,
State: container.State,
Status: container.Status,
Endpoint: "http://" + namePrefix + alias + ":9222",
ID: container.ID,
Alias: alias,
Name: name,
State: container.State,
Status: container.Status,
Endpoint: "http://" + namePrefix + alias + ":9222",
BindingVersion: bindingVersion,
NetworkExitID: container.Labels[networkExitLabel],
ProxyReady: api.proxies.ready(alias, proxyPort),
})
}
return writeJSON(c, http.StatusOK, browsers)
@@ -282,7 +311,7 @@ func (api gateway) create(c fiber.Ctx) error {
decoder := json.NewDecoder(bytes.NewReader(c.Body()))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&input); err != nil {
return writeError(c, http.StatusBadRequest, errors.New("body must contain only alias, name, image, cmd and volume"))
return writeError(c, http.StatusBadRequest, errors.New("body must contain only alias, name, image, cmd, volume, binding_version, network_exit_id, network_exit and stopped"))
}
if err := validateCreate(input); err != nil {
return writeError(c, http.StatusBadRequest, err)
@@ -290,22 +319,48 @@ func (api gateway) create(c fiber.Ctx) error {
if err := api.docker.pullIfMissing(c.Context(), input.Image); err != nil {
return writeError(c, http.StatusBadGateway, err)
}
network, proxyServer, undoProxy := "none", "", func() {}
if !input.Stopped {
var err error
var bindHost string
network, bindHost, err = api.docker.ensureTenantNetwork(api.network, input.Alias, api.self)
if err != nil {
return writeError(c, http.StatusBadGateway, errors.New("configure isolated browser network"))
}
proxyServer, undoProxy, err = api.proxies.configure(input.Alias, bindHost, 0, input.NetworkExit)
if err != nil {
return writeError(c, http.StatusBadGateway, errors.New("configure in-memory proxy"))
}
}
keepProxy := false
defer func() {
if !keepProxy {
undoProxy()
}
}()
pidsLimit := int64(512)
cmd := append([]string{}, input.Cmd...)
if !input.Stopped {
cmd = append(cmd[:len(cmd)-1], "--proxy-server="+proxyServer, "--disable-non-proxied-udp", cmd[len(cmd)-1])
}
payload := map[string]any{
"Image": input.Image,
"User": browserUser,
"Entrypoint": []string{browserEntrypoint},
"Cmd": input.Cmd,
"Cmd": cmd,
"Env": []string{"REMOTE_DEBUGGING_PORT=9222"},
"Labels": map[string]string{
managedLabel: "true",
idLabel: input.Alias,
nameLabel: input.Name,
managedLabel: "true",
idLabel: input.Alias,
nameLabel: input.Name,
bindingVersionLabel: strconv.FormatInt(input.BindingVersion, 10),
networkExitLabel: input.NetworkExitID,
proxyPortLabel: strconv.Itoa(proxyPort(proxyServer)),
},
"ExposedPorts": map[string]any{"9222/tcp": map[string]any{}},
"HostConfig": map[string]any{
"NetworkMode": api.network,
"NetworkMode": network,
"ReadonlyRootfs": true,
"CapDrop": []string{"ALL"},
"SecurityOpt": []string{"no-new-privileges"},
@@ -331,7 +386,11 @@ func (api gateway) create(c fiber.Ctx) error {
}
defer result.Body.Close()
if result.StatusCode != http.StatusCreated {
return forwardDockerError(c, result)
status := http.StatusBadGateway
if result.StatusCode == http.StatusConflict {
status = http.StatusConflict
}
return writeError(c, status, errors.New("Docker container creation failed"))
}
var created struct {
ID string `json:"Id"`
@@ -343,13 +402,16 @@ func (api gateway) create(c fiber.Ctx) error {
}
return writeError(c, http.StatusBadGateway, errors.New("Docker returned an invalid container id; container was removed while preserving its Profile volume"))
}
if err := api.docker.expect(http.MethodPost, "/containers/"+url.PathEscape(created.ID)+"/start", nil, http.StatusNoContent, http.StatusNotModified); err != nil {
cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(created.ID)+"?force=1&v=0", nil, http.StatusNoContent)
if cleanupErr != nil {
return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and cleanup failed: %w; cleanup: %v", err, cleanupErr))
if !input.Stopped {
if err := api.docker.expect(http.MethodPost, "/containers/"+url.PathEscape(created.ID)+"/start", nil, http.StatusNoContent, http.StatusNotModified); err != nil {
cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(created.ID)+"?force=1&v=0", nil, http.StatusNoContent)
if cleanupErr != nil {
return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and cleanup failed: %w; cleanup: %v", err, cleanupErr))
}
return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and was removed while preserving its Profile volume: %w", err))
}
return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and was removed while preserving its Profile volume: %w", err))
}
keepProxy = !input.Stopped
return writeJSON(c, http.StatusCreated, map[string]string{"id": created.ID, "alias": input.Alias})
}
@@ -366,7 +428,11 @@ func validateCreate(input createRequest) error {
if !volumePattern.MatchString(input.Volume) {
return errors.New("volume must be a valid volume name")
}
if len(input.Cmd) == 0 || len(input.Cmd) > 64 {
if input.BindingVersion < 1 || (!input.Stopped && !exitIDPattern.MatchString(input.NetworkExitID)) ||
(input.Stopped && (input.NetworkExitID != "" || input.NetworkExit != (gatewayProxyExit{}))) {
return errors.New("binding_version and network_exit_id must identify the current binding")
}
if len(input.Cmd) == 0 || len(input.Cmd) > 64 || input.Cmd[len(input.Cmd)-1] != "about:blank" {
return errors.New("cmd must contain 1..64 arguments")
}
total := 0
@@ -374,14 +440,34 @@ func validateCreate(input createRequest) error {
if arg == "" || hasControlRunes(arg) {
return errors.New("cmd arguments must be non-empty visible strings")
}
if strings.HasPrefix(arg, "--proxy-server") || arg == "--disable-non-proxied-udp" {
return errors.New("proxy arguments are platform-controlled")
}
total += len(arg)
}
if total > 4096 {
return errors.New("cmd arguments exceed 4096 characters")
}
if input.Stopped {
return nil
}
proxy := input.NetworkExit
if (proxy.Protocol != "http" && proxy.Protocol != "https" && proxy.Protocol != "socks4" && proxy.Protocol != "socks5") ||
proxy.Host == "" || len(proxy.Host) > 253 || strings.ContainsAny(proxy.Host, "@/[]?# \t\r\n") ||
proxy.Port < 1 || proxy.Port > 65535 || (proxy.Username == "" && proxy.Password != "") ||
len(proxy.Username) > 255 || len(proxy.Password) > 255 ||
hasControlRunes(proxy.Username) || hasControlRunes(proxy.Password) {
return errors.New("network_exit must contain a valid proxy endpoint and optional credentials")
}
return nil
}
func proxyPort(proxyServer string) int {
parsed, _ := url.Parse(proxyServer)
port, _ := strconv.Atoi(parsed.Port())
return port
}
func hasControlRunes(value string) bool {
for _, r := range value {
if r < 0x20 || r == 0x7f {
@@ -415,31 +501,77 @@ func (api gateway) changeState(c fiber.Ctx) error {
func (api gateway) remove(c fiber.Ctx) error {
id := c.Params("id")
if err := api.requireManaged(id); err != nil {
err := api.requireManaged(id)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return writeError(c, statusFor(err), err)
}
path := "/containers/" + url.PathEscape(namePrefix+id) + "?force=1&v=0"
if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent); err != nil {
return writeError(c, http.StatusBadGateway, err)
if err == nil {
path := "/containers/" + url.PathEscape(namePrefix+id) + "?force=1&v=0"
if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent, http.StatusNotFound); err != nil {
return writeError(c, http.StatusBadGateway, err)
}
}
api.proxies.remove(id)
if err := api.docker.removeTenantNetwork(api.network, id, api.self); err != nil {
return c.Status(http.StatusAccepted).JSON(map[string]string{
"status": "container_removed_network_cleanup_pending",
})
}
c.Status(http.StatusNoContent)
return nil
}
func (api gateway) restoreProxy(c fiber.Ctx) error {
input := struct {
BindingVersion int64 `json:"binding_version"`
NetworkExitID string `json:"network_exit_id"`
NetworkExit gatewayProxyExit `json:"network_exit"`
}{}
decoder := json.NewDecoder(bytes.NewReader(c.Body()))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&input); err != nil || input.BindingVersion < 1 || !exitIDPattern.MatchString(input.NetworkExitID) ||
validateCreate(createRequest{Alias: c.Params("id"), Name: "x", Image: "x", Cmd: []string{"about:blank"}, Volume: "x",
BindingVersion: input.BindingVersion, NetworkExitID: input.NetworkExitID, NetworkExit: input.NetworkExit}) != nil {
return writeError(c, http.StatusBadRequest, errors.New("invalid proxy recovery request"))
}
labels, err := api.managedLabels(c.Params("id"))
if err != nil {
return writeError(c, statusFor(err), err)
}
version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64)
port, _ := strconv.Atoi(labels[proxyPortLabel])
if version != input.BindingVersion || labels[networkExitLabel] != input.NetworkExitID || port < 1 {
return writeError(c, http.StatusConflict, errors.New("container binding does not match recovery request"))
}
_, bindHost, err := api.docker.ensureTenantNetwork(api.network, c.Params("id"), api.self)
if err != nil {
return writeError(c, http.StatusBadGateway, errors.New("restore isolated browser network"))
}
if _, _, err := api.proxies.configure(c.Params("id"), bindHost, port, input.NetworkExit); err != nil {
return writeError(c, http.StatusBadGateway, errors.New("restore in-memory proxy"))
}
return c.SendStatus(http.StatusNoContent)
}
func (api gateway) requireManaged(id string) error {
_, err := api.managedLabels(id)
return err
}
func (api gateway) managedLabels(id string) (map[string]string, error) {
if !runtimeIDPattern.MatchString(id) {
return errInvalidRuntimeID
return nil, errInvalidRuntimeID
}
result, err := api.docker.request(http.MethodGet, "/containers/"+url.PathEscape(namePrefix+id)+"/json", nil)
if err != nil {
return err
return nil, err
}
defer result.Body.Close()
if result.StatusCode == http.StatusNotFound {
return os.ErrNotExist
return nil, os.ErrNotExist
}
if result.StatusCode != http.StatusOK {
return fmt.Errorf("Docker inspect returned %s", result.Status)
return nil, fmt.Errorf("Docker inspect returned %s", result.Status)
}
var inspected struct {
Config struct {
@@ -447,12 +579,12 @@ func (api gateway) requireManaged(id string) error {
} `json:"Config"`
}
if err := json.NewDecoder(result.Body).Decode(&inspected); err != nil {
return fmt.Errorf("decode Docker inspect: %w", err)
return nil, fmt.Errorf("decode Docker inspect: %w", err)
}
if inspected.Config.Labels[managedLabel] != "true" || inspected.Config.Labels[idLabel] != id {
return errUnmanagedContainer
return nil, errUnmanagedContainer
}
return nil
return inspected.Config.Labels, nil
}
// pullIfMissing 在镜像不在本地时从远端仓库拉取;镜像缺失属于可恢复错误,调用方可直接重试。
@@ -541,16 +673,34 @@ func (docker dockerClient) expect(method, path string, payload any, allowed ...i
return fmt.Errorf("Docker returned %s: %s", response.Status, strings.TrimSpace(string(message)))
}
func (docker dockerClient) ensureBrowserNetwork(name string) error {
if err := validateBrowserNetwork(name); err != nil {
return err
func tenantNetworkName(base, alias string) (string, error) {
name := base + "-" + alias
if !networkNamePattern.MatchString(name) {
return "", errors.New("isolated browser network name is invalid")
}
response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil)
if err != nil {
return fmt.Errorf("inspect browser network: %w", err)
return name, nil
}
func (docker dockerClient) ensureTenantNetwork(base, alias, self string) (string, string, error) {
name, err := tenantNetworkName(base, alias)
if err != nil || self == "" {
return "", "", errors.New("isolated browser network identity is invalid")
}
defer response.Body.Close()
if response.StatusCode == http.StatusOK {
inspect := func() (map[string]struct {
Name string `json:"Name"`
IPv4Address string `json:"IPv4Address"`
}, error) {
response, requestErr := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil)
if requestErr != nil {
return nil, requestErr
}
defer response.Body.Close()
if response.StatusCode == http.StatusNotFound {
return nil, os.ErrNotExist
}
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("inspect isolated browser network returned %s", response.Status)
}
var network struct {
Name string `json:"Name"`
Driver string `json:"Driver"`
@@ -558,33 +708,99 @@ func (docker dockerClient) ensureBrowserNetwork(name string) error {
Attachable bool `json:"Attachable"`
Ingress bool `json:"Ingress"`
Labels map[string]string `json:"Labels"`
Containers map[string]struct {
Name string `json:"Name"`
IPv4Address string `json:"IPv4Address"`
} `json:"Containers"`
}
if err := json.NewDecoder(response.Body).Decode(&network); err != nil {
return fmt.Errorf("decode browser network: %w", err)
return nil, err
}
if network.Name != name || network.Driver != "bridge" || network.Internal || network.Attachable || network.Ingress ||
network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole {
return fmt.Errorf("browser network %q is not a CreatorHub bridge", name)
network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias {
return nil, errors.New("isolated browser network is not owned by this runtime")
}
return network.Containers, nil
}
containers, err := inspect()
if errors.Is(err, os.ErrNotExist) {
if err := docker.expect(http.MethodPost, "/networks/create", map[string]any{
"Name": name, "CheckDuplicate": true, "Driver": "bridge",
"Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias},
}, http.StatusCreated); err != nil {
return "", "", err
}
containers = map[string]struct {
Name string `json:"Name"`
IPv4Address string `json:"IPv4Address"`
}{}
} else if err != nil {
return "", "", err
}
findIP := func() string {
for id, container := range containers {
if id == self || strings.HasPrefix(id, self) || strings.HasPrefix(self, id) || container.Name == self {
host, _, _ := net.ParseCIDR(container.IPv4Address)
if host != nil {
return host.String()
}
}
}
return ""
}
if bindHost := findIP(); bindHost != "" {
return name, bindHost, nil
}
if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(name)+"/connect", map[string]any{
"Container": self, "EndpointConfig": map[string]any{"Aliases": []string{browserProxyHost}},
}, http.StatusOK); err != nil {
return "", "", err
}
containers, err = inspect()
if err != nil {
return "", "", err
}
if bindHost := findIP(); bindHost != "" {
return name, bindHost, nil
}
return "", "", errors.New("Docker did not assign the gateway an isolated network address")
}
func (docker dockerClient) removeTenantNetwork(base, alias, self string) error {
name, err := tenantNetworkName(base, alias)
if err != nil {
return err
}
response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode == http.StatusNotFound {
return nil
}
if response.StatusCode != http.StatusNotFound {
return fmt.Errorf("inspect browser network returned %s", response.Status)
var network struct {
Name string `json:"Name"`
Labels map[string]string `json:"Labels"`
}
return docker.expect(http.MethodPost, "/networks/create", map[string]any{
"Name": name,
"CheckDuplicate": true,
"Driver": "bridge",
"Labels": map[string]string{
managedLabel: "true",
networkRoleLabel: browserNetworkRole,
},
}, http.StatusCreated)
if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&network) != nil || network.Name != name ||
network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias {
return errors.New("refusing to remove an unowned browser network")
}
if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(name)+"/disconnect", map[string]any{
"Container": self, "Force": true,
}, http.StatusOK, http.StatusNotFound); err != nil {
return fmt.Errorf("disconnect isolated browser network: %w", err)
}
if err := docker.expect(http.MethodDelete, "/networks/"+url.PathEscape(name), nil, http.StatusNoContent, http.StatusNotFound); err != nil {
return fmt.Errorf("remove isolated browser network: %w", err)
}
return nil
}
func validateBrowserNetwork(name string) error {
if !networkNamePattern.MatchString(name) {
return errors.New("BROWSER_NETWORK is invalid")
if !networkNamePattern.MatchString(name) || len(name) > 31 {
return errors.New("BROWSER_NETWORK must be a valid network prefix of at most 31 characters")
}
if name == controlNetworkName {
return errors.New("BROWSER_NETWORK must not reuse the control network")
+251 -92
View File
@@ -1,10 +1,14 @@
package main
import (
"bytes"
"encoding/json"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"testing"
@@ -21,7 +25,27 @@ func authed(method, target string, body io.Reader) *http.Request {
}
func testDocker(handler http.HandlerFunc) (dockerClient, *httptest.Server) {
server := httptest.NewServer(handler)
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if strings.HasPrefix(request.URL.Path, "/networks/creatorhub_browser-") {
if request.Method != http.MethodGet {
if request.Method == http.MethodDelete {
response.WriteHeader(http.StatusNoContent)
} else {
response.WriteHeader(http.StatusOK)
}
return
}
alias := strings.TrimPrefix(request.URL.Path, "/networks/creatorhub_browser-")
self, _ := os.Hostname()
_ = json.NewEncoder(response).Encode(map[string]any{
"Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false,
"Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias},
"Containers": map[string]any{self: map[string]string{"Name": self, "IPv4Address": "127.0.0.1/8"}},
})
return
}
handler(response, request)
}))
return dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, server
}
@@ -35,7 +59,43 @@ func decodeJSONBody(t *testing.T, response *http.Response) map[string]any {
}
const testCreateBody = `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` +
`"cmd":["--fingerprint=1000","--lang=zh-CN","about:blank"],"volume":"creatorhub-profile-account-a"}`
`"cmd":["--fingerprint=1000","--lang=zh-CN","about:blank"],"volume":"creatorhub-profile-account-a",` +
`"binding_version":1,"network_exit_id":"exit-1",` +
`"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}`
func TestGatewayCreatesNetworkDisabledStoppedRecoveryContainer(t *testing.T) {
created := false
docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"):
_, _ = response.Write([]byte(`{}`))
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"):
var payload map[string]any
_ = json.NewDecoder(request.Body).Decode(&payload)
host := payload["HostConfig"].(map[string]any)
labels := payload["Labels"].(map[string]any)
encoded, _ := json.Marshal(payload["Cmd"])
if host["NetworkMode"] != "none" || labels[networkExitLabel] != "" || strings.Contains(string(encoded), "proxy") {
t.Fatalf("unsafe stopped recovery payload: %#v", payload)
}
created = true
response.WriteHeader(http.StatusCreated)
_, _ = response.Write([]byte(`{"Id":"stopped-container"}`))
default:
t.Fatalf("stopped recovery unexpectedly called Docker %s %s", request.Method, request.URL.String())
}
})
defer server.Close()
handler := newGateway(docker, "creatorhub_browser", testToken)
body := `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` +
`"cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` +
`"binding_version":1,"network_exit_id":"","network_exit":{},"stopped":true}`
response := httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body)))
if response.Code != http.StatusCreated || !created {
t.Fatalf("stopped recovery create failed: status=%d body=%s", response.Code, response.Body.String())
}
}
func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) {
var created map[string]any
@@ -75,11 +135,12 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) {
t.Fatalf("runtime identity is not fixed: user=%#v entrypoint=%#v", created["User"], created["Entrypoint"])
}
cmd := created["Cmd"].([]any)
if len(cmd) != 3 || cmd[0] != "--fingerprint=1000" || cmd[2] != "about:blank" {
if len(cmd) != 5 || cmd[0] != "--fingerprint=1000" || !strings.HasPrefix(cmd[2].(string), "--proxy-server=http://docker-gateway:") ||
cmd[3] != "--disable-non-proxied-udp" || cmd[4] != "about:blank" {
t.Fatalf("cmd must be passed through verbatim: %#v", created["Cmd"])
}
host := created["HostConfig"].(map[string]any)
if host["NetworkMode"] != "creatorhub_browser" || host["ReadonlyRootfs"] != true {
if host["NetworkMode"] != "creatorhub_browser-account-a" || host["ReadonlyRootfs"] != true {
t.Fatalf("missing container isolation: %#v", host)
}
tmpfs := host["Tmpfs"].(map[string]any)
@@ -96,12 +157,51 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) {
}
}
func TestGatewayDockerInspectContainsNoProxyCredentials(t *testing.T) {
var created map[string]any
docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"):
response.WriteHeader(http.StatusOK)
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"):
if err := json.NewDecoder(request.Body).Decode(&created); err != nil {
t.Fatal(err)
}
response.WriteHeader(http.StatusCreated)
_, _ = response.Write([]byte(`{"Id":"container-id"}`))
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/container-id/start"):
response.WriteHeader(http.StatusNoContent)
default:
t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String())
}
})
defer server.Close()
handler := newGateway(docker, "creatorhub_browser", testToken)
body := strings.Replace(testCreateBody, `"protocol":"socks5","host":"proxy.example","port":1080`,
`"protocol":"socks5","host":"proxy.example","port":1080,"username":"operator","password":"ephemeral"`, 1)
response := httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body)))
if response.Code != http.StatusCreated {
t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String())
}
inspect, _ := json.Marshal(created)
for _, secret := range []string{"operator", "ephemeral", "operator:ephemeral@", "proxy.example"} {
if bytes.Contains(inspect, []byte(secret)) {
t.Fatalf("Docker inspect leaked proxy credential %q: %s", secret, inspect)
}
}
if !bytes.Contains(inspect, []byte("--proxy-server=http://docker-gateway:")) {
t.Fatalf("Docker inspect is missing the secret-free proxy configuration: %s", inspect)
}
}
func TestGatewayPullsMissingImageOnCreate(t *testing.T) {
tests := []struct {
name string
ref string
fromImage string
tag string
name string
ref string
fromImage string
tag string
}{{
name: "tagged ref splits repository and tag",
ref: "registry.example/browser:2.0.0",
@@ -139,7 +239,9 @@ func TestGatewayPullsMissingImageOnCreate(t *testing.T) {
handler := newGateway(docker, "creatorhub_browser", testToken)
body := `{"alias":"account-a","name":"账号甲","image":"` + test.ref +
`","cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a"}`
`","cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` +
`"binding_version":1,"network_exit_id":"exit-1",` +
`"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}`
response := httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body)))
@@ -187,6 +289,7 @@ func TestGatewayRejectsInvalidCreateRequest(t *testing.T) {
"invalid image": `{"alias":"account-a","name":"甲","image":"","cmd":["--fingerprint=1"],"volume":"creatorhub-profile-account-a"}`,
"empty cmd": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":[],"volume":"creatorhub-profile-account-a"}`,
"invalid volume": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1"],"volume":"bad volume!"}`,
"proxy override": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1","--proxy-server=http://direct:8080","about:blank"],"volume":"creatorhub-profile-account-a","network_exit":{"protocol":"socks5","host":"proxy","port":1080}}`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
@@ -281,6 +384,31 @@ func TestGatewayRemovesFailedContainerAndPreservesProfile(t *testing.T) {
}
}
func TestGatewayDoesNotEchoProxyCredentialsFromDockerErrors(t *testing.T) {
docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/") {
response.WriteHeader(http.StatusOK)
return
}
if request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create") {
response.WriteHeader(http.StatusInternalServerError)
_, _ = response.Write([]byte(`invalid cmd --proxy-server=http://operator:ephemeral@proxy.example:8080`))
return
}
t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path)
})
defer server.Close()
handler := newGateway(docker, "creatorhub_browser", testToken)
body := strings.Replace(testCreateBody, `"protocol":"socks5","host":"proxy.example","port":1080`,
`"protocol":"http","host":"proxy.example","port":8080,"username":"operator","password":"ephemeral"`, 1)
response := httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body)))
if response.Code != http.StatusBadGateway || strings.Contains(response.Body.String(), "operator") ||
strings.Contains(response.Body.String(), "ephemeral") || strings.Contains(response.Body.String(), "proxy.example") {
t.Fatalf("gateway leaked proxy material: status=%d body=%s", response.Code, response.Body.String())
}
}
func TestGatewayListsBrowsers(t *testing.T) {
docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodGet || request.URL.Path != "/containers/json" {
@@ -303,6 +431,43 @@ func TestGatewayListsBrowsers(t *testing.T) {
}
}
func TestGatewayRestartRestoresExistingProxyListener(t *testing.T) {
reserved, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
port := reserved.Addr().(*net.TCPAddr).Port
_ = reserved.Close()
labels := map[string]string{
managedLabel: "true", idLabel: "account-a", nameLabel: "账号甲",
bindingVersionLabel: "3", networkExitLabel: "exit-1", proxyPortLabel: strconv.Itoa(port),
}
docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.Method == http.MethodGet && strings.HasSuffix(request.URL.Path, "/containers/creatorhub-browser-account-a/json"):
_ = json.NewEncoder(response).Encode(map[string]any{"Config": map[string]any{"Labels": labels}})
case request.Method == http.MethodGet && request.URL.Path == "/containers/json":
_ = json.NewEncoder(response).Encode([]map[string]any{{"Id": "container-id", "State": "running", "Status": "Up", "Labels": labels}})
default:
t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String())
}
})
defer server.Close()
handler := newGateway(docker, "creatorhub_browser", testToken)
recovery := `{"binding_version":3,"network_exit_id":"exit-1","network_exit":{"protocol":"http","host":"127.0.0.1","port":1}}`
response := httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(recovery)))
if response.Code != http.StatusNoContent {
t.Fatalf("proxy recovery failed: %d %s", response.Code, response.Body.String())
}
response = httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodGet, "/v1/browsers", nil))
var browsers []browser
if response.Code != http.StatusOK || json.NewDecoder(response.Body).Decode(&browsers) != nil || len(browsers) != 1 || !browsers[0].ProxyReady {
t.Fatalf("restarted gateway did not report restored proxy: %d %s", response.Code, response.Body.String())
}
}
func TestGatewayLifecycle(t *testing.T) {
tests := []struct {
method string
@@ -337,6 +502,55 @@ func TestGatewayLifecycle(t *testing.T) {
}
}
func TestGatewayDeleteDistinguishesContainerRemovalFromNetworkCleanup(t *testing.T) {
containerExists, cleanupFails, containerDeletes := true, true, 0
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"):
if !containerExists {
response.WriteHeader(http.StatusNotFound)
return
}
_, _ = response.Write([]byte(`{"Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a"}}}`))
case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"):
containerExists = false
containerDeletes++
response.WriteHeader(http.StatusNoContent)
case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"):
_ = json.NewEncoder(response).Encode(map[string]any{
"Name": "creatorhub_browser-account-a",
"Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a"},
})
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"):
if cleanupFails {
response.WriteHeader(http.StatusInternalServerError)
return
}
response.WriteHeader(http.StatusOK)
case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"):
response.WriteHeader(http.StatusNoContent)
default:
t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String())
}
}))
defer server.Close()
handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()},
"creatorhub_browser", testToken, "gateway-self")
response := httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil))
if response.Code != http.StatusAccepted || containerExists || containerDeletes != 1 {
t.Fatalf("expected definite container removal with pending cleanup, status=%d exists=%v deletes=%d body=%s",
response.Code, containerExists, containerDeletes, response.Body.String())
}
cleanupFails = false
response = httptest.NewRecorder()
adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil))
if response.Code != http.StatusNoContent || containerDeletes != 1 {
t.Fatalf("idempotent cleanup retry failed: status=%d deletes=%d body=%s", response.Code, containerDeletes, response.Body.String())
}
}
func TestGatewayMapsDockerServiceFailureToBadGateway(t *testing.T) {
docker, server := testDocker(func(response http.ResponseWriter, _ *http.Request) {
http.Error(response, "daemon unavailable", http.StatusInternalServerError)
@@ -374,94 +588,39 @@ func TestGatewayRefusesUnmanagedContainer(t *testing.T) {
}
}
func TestEnsureBrowserNetwork(t *testing.T) {
var created struct {
Name string `json:"Name"`
Driver string `json:"Driver"`
Labels map[string]string `json:"Labels"`
}
docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) {
switch request.Method {
case http.MethodGet:
func TestEnsureTenantNetworkConnectsGatewayOnlyToRuntimeNetwork(t *testing.T) {
created, connected := false, false
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.Method == http.MethodGet && !created:
response.WriteHeader(http.StatusNotFound)
case http.MethodPost:
if err := json.NewDecoder(request.Body).Decode(&created); err != nil {
t.Fatal(err)
case request.Method == http.MethodPost && request.URL.Path == "/networks/create":
var body map[string]any
_ = json.NewDecoder(request.Body).Decode(&body)
labels := body["Labels"].(map[string]any)
if body["Name"] != "creatorhub_browser-account-a" || labels[idLabel] != "account-a" {
t.Fatalf("unexpected isolated network create: %#v", body)
}
created = true
response.WriteHeader(http.StatusCreated)
}
})
defer server.Close()
if err := docker.ensureBrowserNetwork("creatorhub_browser"); err != nil {
t.Fatal(err)
}
if created.Name != "creatorhub_browser" || created.Driver != "bridge" ||
created.Labels[managedLabel] != "true" || created.Labels[networkRoleLabel] != browserNetworkRole {
t.Fatalf("network is not a CreatorHub bridge: %#v", created)
}
}
func TestEnsureBrowserNetworkRejectsUnsafeExistingNetwork(t *testing.T) {
valid := map[string]any{
"Name": "creatorhub_browser",
"Driver": "bridge",
"Internal": false,
"Attachable": false,
"Ingress": false,
"Labels": map[string]string{
managedLabel: "true",
networkRoleLabel: browserNetworkRole,
},
}
validDocker, validServer := testDocker(func(response http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(response).Encode(valid)
})
defer validServer.Close()
if err := validDocker.ensureBrowserNetwork("creatorhub_browser"); err != nil {
t.Fatalf("expected owned bridge network to be accepted: %v", err)
}
tests := map[string]func(map[string]any){
"wrong name": func(network map[string]any) { network["Name"] = "other" },
"wrong driver": func(network map[string]any) { network["Driver"] = "overlay" },
"internal": func(network map[string]any) { network["Internal"] = true },
"attachable": func(network map[string]any) { network["Attachable"] = true },
"ingress": func(network map[string]any) { network["Ingress"] = true },
"missing ownership": func(network map[string]any) {
network["Labels"] = map[string]string{networkRoleLabel: browserNetworkRole}
},
"wrong role": func(network map[string]any) {
network["Labels"] = map[string]string{managedLabel: "true", networkRoleLabel: "control"}
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
network := make(map[string]any, len(valid))
for key, value := range valid {
network[key] = value
}
mutate(network)
docker, server := testDocker(func(response http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(response).Encode(network)
case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"):
connected = true
response.WriteHeader(http.StatusOK)
case request.Method == http.MethodGet:
_ = json.NewEncoder(response).Encode(map[string]any{
"Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false,
"Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a"},
"Containers": map[string]any{"gateway-id": map[string]string{"Name": "gateway-id", "IPv4Address": "127.0.0.3/8"}},
})
defer server.Close()
if err := docker.ensureBrowserNetwork("creatorhub_browser"); err == nil {
t.Fatal("expected unsafe existing network to be rejected")
}
})
}
}
func TestEnsureBrowserNetworkRejectsControlNetwork(t *testing.T) {
requested := false
docker, server := testDocker(func(http.ResponseWriter, *http.Request) { requested = true })
default:
t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path)
}
}))
defer server.Close()
if err := docker.ensureBrowserNetwork(controlNetworkName); err == nil || requested {
t.Fatalf("expected control network to be rejected before Docker request, requested=%v err=%v", requested, err)
docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}
name, bindHost, err := docker.ensureTenantNetwork("creatorhub_browser", "account-a", "gateway-id")
if err != nil || !created || !connected || name != "creatorhub_browser-account-a" || bindHost != "127.0.0.3" {
t.Fatalf("isolated network was not created and connected: name=%q host=%q created=%v connected=%v err=%v", name, bindHost, created, connected, err)
}
}
+362
View File
@@ -0,0 +1,362 @@
package main
import (
"bufio"
"context"
"crypto/tls"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
"sync"
"time"
)
const browserProxyHost = "docker-gateway"
type memoryProxyRegistry struct {
mu sync.Mutex
proxies map[string]*memoryProxy
}
type memoryProxy struct {
mu sync.RWMutex
exit gatewayProxyExit
bindHost string
listener net.Listener
server *http.Server
url string
}
func newMemoryProxyRegistry() *memoryProxyRegistry {
return &memoryProxyRegistry{proxies: map[string]*memoryProxy{}}
}
func (registry *memoryProxyRegistry) configure(alias, bindHost string, port int, exit gatewayProxyExit) (string, func(), error) {
registry.mu.Lock()
defer registry.mu.Unlock()
if proxy := registry.proxies[alias]; proxy != nil {
if proxy.bindHost == bindHost && (port == 0 || proxy.listener.Addr().(*net.TCPAddr).Port == port) && proxy.exit == exit {
return proxy.url, func() {}, nil
}
delete(registry.proxies, alias)
_ = proxy.server.Close()
}
listener, err := net.Listen("tcp4", net.JoinHostPort(bindHost, strconv.Itoa(port)))
if err != nil {
return "", nil, err
}
actualPort := listener.Addr().(*net.TCPAddr).Port
proxy := &memoryProxy{exit: exit, bindHost: bindHost, listener: listener, url: "http://" + net.JoinHostPort(browserProxyHost, strconv.Itoa(actualPort))}
proxy.server = &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second}
registry.proxies[alias] = proxy
go func() { _ = proxy.server.Serve(listener) }()
undo := func() {
registry.mu.Lock()
defer registry.mu.Unlock()
if registry.proxies[alias] == proxy {
delete(registry.proxies, alias)
_ = proxy.server.Close()
}
}
return proxy.url, undo, nil
}
func (registry *memoryProxyRegistry) ready(alias string, port int) bool {
registry.mu.Lock()
defer registry.mu.Unlock()
proxy := registry.proxies[alias]
return proxy != nil && proxy.listener.Addr().(*net.TCPAddr).Port == port
}
func (registry *memoryProxyRegistry) remove(alias string) {
registry.mu.Lock()
proxy := registry.proxies[alias]
delete(registry.proxies, alias)
registry.mu.Unlock()
if proxy != nil {
_ = proxy.server.Close()
}
}
func (proxy *memoryProxy) ServeHTTP(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodConnect {
proxy.tunnel(response, request)
return
}
proxy.mu.RLock()
exit := proxy.exit
proxy.mu.RUnlock()
transport := &http.Transport{DisableKeepAlives: true}
if exit.Protocol == "http" || exit.Protocol == "https" {
upstream := &url.URL{Scheme: exit.Protocol, Host: net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port))}
if exit.Username != "" {
upstream.User = url.UserPassword(exit.Username, exit.Password)
}
transport.Proxy = http.ProxyURL(upstream)
} else {
transport.DialContext = proxy.dialContext
}
defer transport.CloseIdleConnections()
outbound := request.Clone(request.Context())
outbound.RequestURI = ""
outbound.Header.Del("Proxy-Authorization")
result, err := transport.RoundTrip(outbound)
if err != nil {
http.Error(response, "proxy connection failed", http.StatusBadGateway)
return
}
defer result.Body.Close()
for key, values := range result.Header {
for _, value := range values {
response.Header().Add(key, value)
}
}
response.WriteHeader(result.StatusCode)
_, _ = io.Copy(response, result.Body)
}
func (proxy *memoryProxy) tunnel(response http.ResponseWriter, request *http.Request) {
upstream, err := proxy.dialContext(request.Context(), "tcp", request.Host)
if err != nil {
http.Error(response, "proxy connection failed", http.StatusBadGateway)
return
}
client, buffered, err := http.NewResponseController(response).Hijack()
if err != nil {
_ = upstream.Close()
http.Error(response, "proxy tunnel unavailable", http.StatusInternalServerError)
return
}
if _, err := buffered.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil || buffered.Flush() != nil {
_ = client.Close()
_ = upstream.Close()
return
}
done := make(chan struct{}, 2)
go func() { _, _ = io.Copy(upstream, client); done <- struct{}{} }()
go func() { _, _ = io.Copy(client, upstream); done <- struct{}{} }()
<-done
_ = client.Close()
_ = upstream.Close()
}
func (proxy *memoryProxy) dialContext(ctx context.Context, _, target string) (net.Conn, error) {
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
proxy.mu.RLock()
exit := proxy.exit
proxy.mu.RUnlock()
switch exit.Protocol {
case "http", "https":
return dialHTTPProxy(ctx, exit, target)
case "socks4":
return dialSOCKS4Proxy(ctx, exit, target)
case "socks5":
return dialSOCKS5Proxy(ctx, exit, target)
default:
return nil, errors.New("unsupported proxy protocol")
}
}
func dialHTTPProxy(ctx context.Context, exit gatewayProxyExit, target string) (net.Conn, error) {
address := net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port))
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address)
if err != nil {
return nil, err
}
if exit.Protocol == "https" {
tlsConnection := tls.Client(connection, &tls.Config{ServerName: exit.Host, MinVersion: tls.VersionTLS12})
if err := tlsConnection.HandshakeContext(ctx); err != nil {
_ = connection.Close()
return nil, err
}
connection = tlsConnection
}
request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)}
if exit.Username != "" {
request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(exit.Username+":"+exit.Password)))
}
if deadline, ok := ctx.Deadline(); ok {
_ = connection.SetDeadline(deadline)
}
if err := request.Write(connection); err != nil {
_ = connection.Close()
return nil, err
}
result, err := http.ReadResponse(bufio.NewReader(connection), request)
if err != nil {
_ = connection.Close()
return nil, err
}
if result.StatusCode != http.StatusOK {
_ = result.Body.Close()
_ = connection.Close()
return nil, fmt.Errorf("upstream proxy returned %s", result.Status)
}
_ = connection.SetDeadline(time.Time{})
return connection, nil
}
func dialSOCKS4Proxy(ctx context.Context, exit gatewayProxyExit, target string) (net.Conn, error) {
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port)))
if err != nil {
return nil, err
}
host, portText, err := net.SplitHostPort(target)
if err != nil {
_ = connection.Close()
return nil, err
}
port, err := strconv.Atoi(portText)
if err != nil || port < 1 || port > 65535 {
_ = connection.Close()
return nil, errors.New("invalid SOCKS4 target")
}
payload := []byte{4, 1, byte(port >> 8), byte(port), 0, 0, 0, 1}
if ip := net.ParseIP(host).To4(); ip != nil {
copy(payload[4:8], ip)
}
payload = append(payload, exit.Username...)
payload = append(payload, 0)
if net.ParseIP(host).To4() == nil {
payload = append(payload, host...)
payload = append(payload, 0)
}
if err := exchangeSOCKS(ctx, connection, payload, 8); err != nil {
return nil, err
}
return connection, nil
}
func dialSOCKS5Proxy(ctx context.Context, exit gatewayProxyExit, target string) (net.Conn, error) {
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port)))
if err != nil {
return nil, err
}
methods := []byte{5, 1, 0}
if exit.Username != "" {
methods = []byte{5, 1, 2}
}
if deadline, ok := ctx.Deadline(); ok {
_ = connection.SetDeadline(deadline)
}
if _, err := connection.Write(methods); err != nil {
_ = connection.Close()
return nil, err
}
selection := make([]byte, 2)
if _, err := io.ReadFull(connection, selection); err != nil || selection[0] != 5 || selection[1] == 0xff {
_ = connection.Close()
return nil, errors.New("SOCKS5 authentication method rejected")
}
if selection[1] == 2 {
if len(exit.Username) > 255 || len(exit.Password) > 255 {
_ = connection.Close()
return nil, errors.New("SOCKS5 credentials too long")
}
auth := append([]byte{1, byte(len(exit.Username))}, exit.Username...)
auth = append(auth, byte(len(exit.Password)))
auth = append(auth, exit.Password...)
if _, err := connection.Write(auth); err != nil {
_ = connection.Close()
return nil, err
}
result := make([]byte, 2)
if _, err := io.ReadFull(connection, result); err != nil || result[1] != 0 {
_ = connection.Close()
return nil, errors.New("SOCKS5 authentication rejected")
}
} else if exit.Username != "" {
_ = connection.Close()
return nil, errors.New("SOCKS5 proxy skipped required authentication")
}
host, portText, err := net.SplitHostPort(target)
if err != nil {
_ = connection.Close()
return nil, err
}
port, err := strconv.Atoi(portText)
if err != nil || port < 1 || port > 65535 {
_ = connection.Close()
return nil, errors.New("invalid SOCKS5 target")
}
request := []byte{5, 1, 0}
if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
request = append(request, 1)
request = append(request, ip.To4()...)
} else if ip != nil {
request = append(request, 4)
request = append(request, ip.To16()...)
} else {
if len(host) > 255 {
_ = connection.Close()
return nil, errors.New("SOCKS5 target too long")
}
request = append(request, 3, byte(len(host)))
request = append(request, host...)
}
portBytes := make([]byte, 2)
binary.BigEndian.PutUint16(portBytes, uint16(port))
request = append(request, portBytes...)
if _, err := connection.Write(request); err != nil {
_ = connection.Close()
return nil, err
}
header := make([]byte, 4)
if _, err := io.ReadFull(connection, header); err != nil || header[0] != 5 || header[1] != 0 {
_ = connection.Close()
return nil, errors.New("SOCKS5 proxy rejected connection")
}
addressLength := 0
switch header[3] {
case 1:
addressLength = 4
case 4:
addressLength = 16
case 3:
var length [1]byte
if _, err := io.ReadFull(connection, length[:]); err != nil {
_ = connection.Close()
return nil, err
}
addressLength = int(length[0])
default:
_ = connection.Close()
return nil, errors.New("invalid SOCKS5 response")
}
if _, err := io.CopyN(io.Discard, connection, int64(addressLength+2)); err != nil {
_ = connection.Close()
return nil, err
}
_ = connection.SetDeadline(time.Time{})
return connection, nil
}
func exchangeSOCKS(ctx context.Context, connection net.Conn, request []byte, responseBytes int) error {
if deadline, ok := ctx.Deadline(); ok {
_ = connection.SetDeadline(deadline)
}
if _, err := connection.Write(request); err != nil {
_ = connection.Close()
return err
}
if responseBytes > 0 {
response := make([]byte, responseBytes)
if _, err := io.ReadFull(connection, response); err != nil {
_ = connection.Close()
return err
}
if responseBytes == 8 && response[1] != 90 {
_ = connection.Close()
return errors.New("SOCKS4 proxy rejected connection")
}
}
_ = connection.SetDeadline(time.Time{})
return nil
}
+154
View File
@@ -0,0 +1,154 @@
package main
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
)
func TestMemoryProxyUsesSOCKS5Credentials(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
done := make(chan error, 1)
go func() {
connection, err := listener.Accept()
if err != nil {
done <- err
return
}
defer connection.Close()
greeting := make([]byte, 3)
if _, err := io.ReadFull(connection, greeting); err != nil {
done <- err
return
}
_, _ = connection.Write([]byte{5, 2})
authHeader := make([]byte, 2)
_, _ = io.ReadFull(connection, authHeader)
username := make([]byte, int(authHeader[1]))
_, _ = io.ReadFull(connection, username)
var passwordLength [1]byte
_, _ = io.ReadFull(connection, passwordLength[:])
password := make([]byte, int(passwordLength[0]))
_, _ = io.ReadFull(connection, password)
if string(username) != "operator" || string(password) != "ephemeral" {
done <- io.ErrUnexpectedEOF
return
}
_, _ = connection.Write([]byte{1, 0})
requestHeader := make([]byte, 5)
_, _ = io.ReadFull(connection, requestHeader)
host := make([]byte, int(requestHeader[4]))
_, _ = io.ReadFull(connection, host)
port := make([]byte, 2)
_, _ = io.ReadFull(connection, port)
if string(host) != "example.com" || binary.BigEndian.Uint16(port) != 443 {
done <- io.ErrUnexpectedEOF
return
}
if _, err = connection.Write([]byte{5, 0, 0, 1, 127, 0, 0, 1, 0, 0}); err != nil {
done <- err
return
}
var tunneled [1]byte
_, err = io.ReadFull(connection, tunneled[:])
if err == nil && tunneled[0] != 'x' {
err = io.ErrUnexpectedEOF
}
done <- err
}()
host, portText, _ := net.SplitHostPort(listener.Addr().String())
port, _ := net.LookupPort("tcp", portText)
registry := newMemoryProxyRegistry()
proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{
Protocol: "socks5", Host: host, Port: port, Username: "operator", Password: "ephemeral",
})
if err != nil {
t.Fatal(err)
}
defer cleanup()
parsed, _ := url.Parse(proxyURL)
connection, err := net.Dial("tcp", strings.Replace(parsed.Host, browserProxyHost, "127.0.0.1", 1))
if err != nil {
t.Fatal(err)
}
if _, err := fmt.Fprint(connection, "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n"); err != nil {
t.Fatal(err)
}
response, err := http.ReadResponse(bufio.NewReader(connection), &http.Request{Method: http.MethodConnect})
if err != nil || response.StatusCode != http.StatusOK {
t.Fatalf("memory proxy CONNECT failed: response=%v err=%v", response, err)
}
if _, err := connection.Write([]byte{'x'}); err != nil {
t.Fatal(err)
}
_ = connection.Close()
if err := <-done; err != nil {
t.Fatal(err)
}
}
func TestMemoryProxyUsesAbsoluteFormForHTTPUpstream(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodConnect {
http.Error(response, "CONNECT forbidden", http.StatusMethodNotAllowed)
return
}
if !request.URL.IsAbs() || request.URL.String() != "http://example.com/plain" {
t.Fatalf("expected absolute-form request, got %q", request.URL.String())
}
if request.Header.Get("Proxy-Authorization") == "" {
t.Fatal("upstream proxy credentials were not applied")
}
_, _ = response.Write([]byte("forwarded"))
}))
defer upstream.Close()
address, _ := url.Parse(upstream.URL)
port, _ := strconv.Atoi(address.Port())
registry := newMemoryProxyRegistry()
proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{
Protocol: "http", Host: address.Hostname(), Port: port, Username: "operator", Password: "ephemeral",
})
if err != nil {
t.Fatal(err)
}
defer cleanup()
proxyAddress := strings.Replace(strings.TrimPrefix(proxyURL, "http://"), browserProxyHost, "127.0.0.1", 1)
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(&url.URL{Scheme: "http", Host: proxyAddress})}}
response, err := client.Get("http://example.com/plain")
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
body, _ := io.ReadAll(response.Body)
if response.StatusCode != http.StatusOK || string(body) != "forwarded" {
t.Fatalf("plain HTTP was not forwarded: status=%d body=%s", response.StatusCode, body)
}
}
func TestMemoryProxyRejectsCrossAliasAddress(t *testing.T) {
registry := newMemoryProxyRegistry()
proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1})
if err != nil {
t.Fatal(err)
}
defer cleanup()
parsed, _ := url.Parse(proxyURL)
if connection, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.2", parsed.Port()), 100*time.Millisecond); err == nil {
_ = connection.Close()
t.Fatal("another tenant address could reach account-a proxy")
}
}