HH-738: add constrained browser container control (#4)

This commit is contained in:
2026-08-27 18:07:34 +08:00
parent 8aa194b264
commit 4b054d6f04
16 changed files with 1925 additions and 1 deletions
+63
View File
@@ -0,0 +1,63 @@
package main
import (
"log"
"net/http"
"net/http/httputil"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
func main() {
gateway, err := url.Parse(env("DOCKER_GATEWAY_URL", "http://127.0.0.1:8081"))
if err != nil {
log.Fatal(err)
}
proxy := httputil.NewSingleHostReverseProxy(gateway)
originalDirector := proxy.Director
proxy.Director = func(request *http.Request) {
originalDirector(request)
request.URL.Path = "/v1/browsers" + strings.TrimPrefix(request.URL.Path, "/api/browsers")
request.Host = gateway.Host
}
mux := http.NewServeMux()
mux.Handle("/api/browsers", proxy)
mux.Handle("/api/browsers/", proxy)
mux.HandleFunc("GET /healthz", func(response http.ResponseWriter, _ *http.Request) {
response.WriteHeader(http.StatusNoContent)
})
mux.Handle("/", spaHandler(env("WEB_DIR", "web/dist")))
server := &http.Server{
Addr: env("LISTEN_ADDR", ":8080"),
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
IdleTimeout: 60 * time.Second,
}
log.Printf("CreatorHub control plane listening on %s", server.Addr)
log.Fatal(server.ListenAndServe())
}
func spaHandler(directory string) http.Handler {
files := http.FileServer(http.Dir(directory))
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
name := filepath.Join(directory, filepath.Clean(strings.TrimPrefix(request.URL.Path, "/")))
if info, err := os.Stat(name); err == nil && !info.IsDir() {
files.ServeHTTP(response, request)
return
}
http.ServeFile(response, request, filepath.Join(directory, "index.html"))
})
}
func env(name, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}
+400
View File
@@ -0,0 +1,400 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
)
const (
managedLabel = "io.creatorhub.managed"
idLabel = "io.creatorhub.runtime-id"
networkRoleLabel = "io.creatorhub.network-role"
browserNetworkRole = "browser"
controlNetworkName = "creatorhub_control"
bridgeICCOption = "com.docker.network.bridge.enable_icc"
namePrefix = "creatorhub-browser-"
)
var runtimeIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
var networkNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`)
type config struct {
image string
network string
}
type dockerClient struct {
baseURL string
client *http.Client
}
type gateway struct {
docker dockerClient
config config
}
type createRequest struct {
Name string `json:"name"`
Seed int64 `json:"seed"`
}
type browser struct {
ID string `json:"id"`
Name string `json:"name"`
State string `json:"state"`
Status string `json:"status"`
Profile string `json:"profile"`
Endpoint string `json:"endpoint"`
}
func main() {
image := os.Getenv("BROWSER_IMAGE")
if image == "" || strings.HasSuffix(image, ":latest") || !strings.Contains(image, ":") {
log.Fatal("BROWSER_IMAGE must be a pinned image tag")
}
transport := &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", env("DOCKER_SOCKET", "/var/run/docker.sock"))
},
}
docker := dockerClient{
baseURL: "http://docker/v1.43",
client: &http.Client{Transport: transport, Timeout: 30 * time.Second},
}
network := env("BROWSER_NETWORK", "creatorhub_browser")
if err := docker.ensureInternalNetwork(network); err != nil {
log.Fatal(err)
}
handler := newGateway(docker, config{image: image, network: network})
server := &http.Server{
Addr: env("LISTEN_ADDR", ":8081"),
Handler: handler,
ReadHeaderTimeout: 5 * time.Second,
IdleTimeout: 60 * time.Second,
}
log.Printf("restricted Docker gateway listening on %s", server.Addr)
log.Fatal(server.ListenAndServe())
}
func newGateway(client dockerClient, cfg config) http.Handler {
api := gateway{docker: client, config: cfg}
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(response http.ResponseWriter, _ *http.Request) {
response.WriteHeader(http.StatusNoContent)
})
mux.HandleFunc("GET /v1/browsers", api.list)
mux.HandleFunc("POST /v1/browsers", api.create)
mux.HandleFunc("POST /v1/browsers/{id}/{action}", api.changeState)
mux.HandleFunc("DELETE /v1/browsers/{id}", api.remove)
return mux
}
func (api gateway) list(response http.ResponseWriter, _ *http.Request) {
filters, _ := json.Marshal(map[string][]string{"label": {managedLabel + "=true"}})
result, err := api.docker.request(http.MethodGet, "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), nil)
if err != nil {
writeError(response, http.StatusBadGateway, err)
return
}
defer result.Body.Close()
if result.StatusCode != http.StatusOK {
forwardDockerError(response, result)
return
}
var containers []struct {
ID string `json:"Id"`
State string `json:"State"`
Status string `json:"Status"`
Labels map[string]string `json:"Labels"`
}
if err := json.NewDecoder(result.Body).Decode(&containers); err != nil {
writeError(response, http.StatusBadGateway, fmt.Errorf("decode Docker response: %w", err))
return
}
browsers := make([]browser, 0, len(containers))
for _, container := range containers {
id := container.Labels[idLabel]
if !runtimeIDPattern.MatchString(id) {
continue
}
browsers = append(browsers, browser{
ID: container.ID,
Name: id,
State: container.State,
Status: container.Status,
Profile: "creatorhub-profile-" + id,
Endpoint: "http://" + namePrefix + id + ":9222",
})
}
writeJSON(response, http.StatusOK, browsers)
}
func (api gateway) create(response http.ResponseWriter, request *http.Request) {
var input createRequest
decoder := json.NewDecoder(http.MaxBytesReader(response, request.Body, 1<<20))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&input); err != nil {
writeError(response, http.StatusBadRequest, errors.New("body must contain only name and numeric seed"))
return
}
if !runtimeIDPattern.MatchString(input.Name) || input.Seed < 1 || input.Seed > 2147483647 {
writeError(response, http.StatusBadRequest, errors.New("name must match [a-z0-9][a-z0-9-]{0,31} and seed must be 1..2147483647"))
return
}
pidsLimit := int64(512)
payload := map[string]any{
"Image": api.config.image,
"Cmd": []string{"--fingerprint=" + strconv.FormatInt(input.Seed, 10), "about:blank"},
"Env": []string{"REMOTE_DEBUGGING_PORT=9222"},
"Labels": map[string]string{
managedLabel: "true",
idLabel: input.Name,
},
"ExposedPorts": map[string]any{"9222/tcp": map[string]any{}},
"HostConfig": map[string]any{
"NetworkMode": api.config.network,
"ReadonlyRootfs": true,
"CapDrop": []string{"ALL"},
"SecurityOpt": []string{"no-new-privileges"},
"PidsLimit": &pidsLimit,
"Memory": int64(1 << 30),
"NanoCpus": int64(2_000_000_000),
"Tmpfs": map[string]string{
"/tmp": "rw,nosuid,nodev,noexec,mode=1777,size=256m",
"/tmp/.X11-unix": "rw,nosuid,nodev,noexec,mode=1777,size=1m",
"/dev/shm": "rw,nosuid,nodev,noexec,size=256m",
"/home/ubuntu": "rw,nosuid,nodev,noexec,uid=1000,gid=1000,mode=700,size=64m",
},
"Mounts": []map[string]any{{
"Type": "volume",
"Source": "creatorhub-profile-" + input.Name,
"Target": "/data",
}},
},
}
result, err := api.docker.request(http.MethodPost, "/containers/create?name="+url.QueryEscape(namePrefix+input.Name), payload)
if err != nil {
writeError(response, http.StatusBadGateway, err)
return
}
defer result.Body.Close()
if result.StatusCode != http.StatusCreated {
forwardDockerError(response, result)
return
}
var created struct {
ID string `json:"Id"`
}
if err := json.NewDecoder(result.Body).Decode(&created); err != nil || created.ID == "" {
writeError(response, http.StatusBadGateway, errors.New("Docker returned an invalid container id"))
return
}
if err := api.docker.expect(http.MethodPost, "/containers/"+url.PathEscape(created.ID)+"/start", nil, http.StatusNoContent, http.StatusNotModified); err != nil {
writeError(response, http.StatusBadGateway, fmt.Errorf("container created but did not start: %w", err))
return
}
writeJSON(response, http.StatusCreated, map[string]string{"id": created.ID, "name": input.Name})
}
func (api gateway) changeState(response http.ResponseWriter, request *http.Request) {
id := request.PathValue("id")
action := request.PathValue("action")
if err := api.requireManaged(id); err != nil {
writeError(response, statusFor(err), err)
return
}
var path string
switch action {
case "start":
path = "/containers/" + url.PathEscape(namePrefix+id) + "/start"
case "stop":
path = "/containers/" + url.PathEscape(namePrefix+id) + "/stop?t=10"
default:
writeError(response, http.StatusNotFound, errors.New("unknown action"))
return
}
if err := api.docker.expect(http.MethodPost, path, nil, http.StatusNoContent, http.StatusNotModified); err != nil {
writeError(response, http.StatusBadGateway, err)
return
}
response.WriteHeader(http.StatusNoContent)
}
func (api gateway) remove(response http.ResponseWriter, request *http.Request) {
id := request.PathValue("id")
if err := api.requireManaged(id); err != nil {
writeError(response, statusFor(err), err)
return
}
path := "/containers/" + url.PathEscape(namePrefix+id) + "?force=1&v=0"
if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent); err != nil {
writeError(response, http.StatusBadGateway, err)
return
}
response.WriteHeader(http.StatusNoContent)
}
func (api gateway) requireManaged(id string) error {
if !runtimeIDPattern.MatchString(id) {
return errors.New("invalid runtime id")
}
result, err := api.docker.request(http.MethodGet, "/containers/"+url.PathEscape(namePrefix+id)+"/json", nil)
if err != nil {
return err
}
defer result.Body.Close()
if result.StatusCode == http.StatusNotFound {
return os.ErrNotExist
}
if result.StatusCode != http.StatusOK {
return fmt.Errorf("Docker inspect returned %s", result.Status)
}
var inspected struct {
Config struct {
Labels map[string]string `json:"Labels"`
} `json:"Config"`
}
if err := json.NewDecoder(result.Body).Decode(&inspected); err != nil {
return fmt.Errorf("decode Docker inspect: %w", err)
}
if inspected.Config.Labels[managedLabel] != "true" || inspected.Config.Labels[idLabel] != id {
return errors.New("refusing to operate on a container not owned by CreatorHub")
}
return nil
}
func (docker dockerClient) request(method, path string, payload any) (*http.Response, error) {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return nil, err
}
body = bytes.NewReader(encoded)
}
request, err := http.NewRequest(method, docker.baseURL+path, body)
if err != nil {
return nil, err
}
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
return docker.client.Do(request)
}
func (docker dockerClient) expect(method, path string, payload any, allowed ...int) error {
response, err := docker.request(method, path, payload)
if err != nil {
return err
}
defer response.Body.Close()
for _, status := range allowed {
if response.StatusCode == status {
return nil
}
}
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return fmt.Errorf("Docker returned %s: %s", response.Status, strings.TrimSpace(string(message)))
}
func (docker dockerClient) ensureInternalNetwork(name string) error {
if !networkNamePattern.MatchString(name) {
return errors.New("BROWSER_NETWORK is invalid")
}
if name == controlNetworkName {
return errors.New("BROWSER_NETWORK must not reuse the control network")
}
response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil)
if err != nil {
return fmt.Errorf("inspect browser network: %w", err)
}
defer response.Body.Close()
if response.StatusCode == http.StatusOK {
var network struct {
Name string `json:"Name"`
Driver string `json:"Driver"`
Internal bool `json:"Internal"`
Attachable bool `json:"Attachable"`
Ingress bool `json:"Ingress"`
Options map[string]string `json:"Options"`
Labels map[string]string `json:"Labels"`
}
if err := json.NewDecoder(response.Body).Decode(&network); err != nil {
return fmt.Errorf("decode browser network: %w", err)
}
if network.Name != name || network.Driver != "bridge" || !network.Internal || network.Attachable || network.Ingress ||
network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole ||
len(network.Options) != 1 || network.Options[bridgeICCOption] != "false" {
return fmt.Errorf("browser network %q is not an isolated CreatorHub bridge", name)
}
return nil
}
if response.StatusCode != http.StatusNotFound {
return fmt.Errorf("inspect browser network returned %s", response.Status)
}
return docker.expect(http.MethodPost, "/networks/create", map[string]any{
"Name": name,
"CheckDuplicate": true,
"Driver": "bridge",
"Internal": true,
"Options": map[string]string{bridgeICCOption: "false"},
"Labels": map[string]string{
managedLabel: "true",
networkRoleLabel: browserNetworkRole,
},
}, http.StatusCreated)
}
func statusFor(err error) int {
if errors.Is(err, os.ErrNotExist) {
return http.StatusNotFound
}
if strings.Contains(err.Error(), "refusing") {
return http.StatusForbidden
}
return http.StatusBadRequest
}
func forwardDockerError(response http.ResponseWriter, result *http.Response) {
message, _ := io.ReadAll(io.LimitReader(result.Body, 4096))
status := http.StatusBadGateway
if result.StatusCode == http.StatusConflict {
status = http.StatusConflict
}
writeError(response, status, fmt.Errorf("Docker returned %s: %s", result.Status, strings.TrimSpace(string(message))))
}
func writeError(response http.ResponseWriter, status int, err error) {
writeJSON(response, status, map[string]string{"error": err.Error()})
}
func writeJSON(response http.ResponseWriter, status int, value any) {
response.Header().Set("Content-Type", "application/json")
response.WriteHeader(status)
_ = json.NewEncoder(response).Encode(value)
}
func env(name, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}
+185
View File
@@ -0,0 +1,185 @@
package main
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestGatewayCreatesConstrainedBrowser(t *testing.T) {
var created map[string]any
docker := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.Method == http.MethodPost && request.URL.Path == "/v1.43/containers/create":
if got := request.URL.Query().Get("name"); got != namePrefix+"account-a" {
t.Fatalf("unexpected container name %q", got)
}
if err := json.NewDecoder(request.Body).Decode(&created); err != nil {
t.Fatal(err)
}
response.WriteHeader(http.StatusCreated)
_, _ = response.Write([]byte(`{"Id":"container-id"}`))
case request.Method == http.MethodPost && request.URL.Path == "/v1.43/containers/container-id/start":
response.WriteHeader(http.StatusNoContent)
default:
t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String())
}
}))
defer docker.Close()
handler := newGateway(dockerClient{baseURL: docker.URL + "/v1.43", client: docker.Client()}, config{
image: "registry.example/browser:1.2.3",
network: "creatorhub_browser",
})
request := httptest.NewRequest(http.MethodPost, "/v1/browsers", strings.NewReader(`{"name":"account-a","seed":1000}`))
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusCreated {
t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String())
}
if created["Image"] != "registry.example/browser:1.2.3" {
t.Fatalf("gateway accepted an unexpected image: %#v", created["Image"])
}
host := created["HostConfig"].(map[string]any)
if host["NetworkMode"] != "creatorhub_browser" || host["ReadonlyRootfs"] != true {
t.Fatalf("missing container isolation: %#v", host)
}
tmpfs := host["Tmpfs"].(map[string]any)
if tmpfs["/tmp/.X11-unix"] == nil || tmpfs["/home/ubuntu"] == nil {
t.Fatalf("missing writable runtime paths: %#v", tmpfs)
}
labels := created["Labels"].(map[string]any)
if labels[managedLabel] != "true" || labels[idLabel] != "account-a" {
t.Fatalf("missing ownership labels: %#v", labels)
}
}
func TestGatewayRefusesUnmanagedContainer(t *testing.T) {
deleted := false
docker := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
switch request.Method {
case http.MethodGet:
_, _ = response.Write([]byte(`{"Config":{"Labels":{}}}`))
case http.MethodDelete:
deleted = true
response.WriteHeader(http.StatusNoContent)
}
}))
defer docker.Close()
handler := newGateway(dockerClient{baseURL: docker.URL, client: docker.Client()}, config{})
request := httptest.NewRequest(http.MethodDelete, "/v1/browsers/foreign", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusForbidden || deleted {
t.Fatalf("expected unmanaged container to be rejected, status=%d deleted=%v", response.Code, deleted)
}
}
func TestEnsureInternalNetwork(t *testing.T) {
var created struct {
Name string `json:"Name"`
Driver string `json:"Driver"`
Internal bool `json:"Internal"`
Options map[string]string `json:"Options"`
Labels map[string]string `json:"Labels"`
}
docker := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
switch request.Method {
case http.MethodGet:
response.WriteHeader(http.StatusNotFound)
case http.MethodPost:
if err := json.NewDecoder(request.Body).Decode(&created); err != nil {
t.Fatal(err)
}
response.WriteHeader(http.StatusCreated)
}
}))
defer docker.Close()
client := dockerClient{baseURL: docker.URL, client: docker.Client()}
if err := client.ensureInternalNetwork("creatorhub_browser"); err != nil {
t.Fatal(err)
}
if created.Name != "creatorhub_browser" || created.Driver != "bridge" || !created.Internal ||
created.Options[bridgeICCOption] != "false" || len(created.Options) != 1 ||
created.Labels[managedLabel] != "true" || created.Labels[networkRoleLabel] != browserNetworkRole {
t.Fatalf("network is not isolated: %#v", created)
}
}
func TestEnsureInternalNetworkRejectsUnsafeExistingNetwork(t *testing.T) {
valid := map[string]any{
"Name": "creatorhub_browser",
"Driver": "bridge",
"Internal": true,
"Attachable": false,
"Ingress": false,
"Options": map[string]string{bridgeICCOption: "false"},
"Labels": map[string]string{
managedLabel: "true",
networkRoleLabel: browserNetworkRole,
},
}
validDocker := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(response).Encode(valid)
}))
defer validDocker.Close()
client := dockerClient{baseURL: validDocker.URL, client: validDocker.Client()}
if err := client.ensureInternalNetwork("creatorhub_browser"); err != nil {
t.Fatalf("expected owned isolated network to be accepted: %v", err)
}
tests := map[string]func(map[string]any){
"wrong name": func(network map[string]any) { network["Name"] = "other" },
"wrong driver": func(network map[string]any) { network["Driver"] = "overlay" },
"external": func(network map[string]any) { network["Internal"] = false },
"attachable": func(network map[string]any) { network["Attachable"] = true },
"ingress": func(network map[string]any) { network["Ingress"] = true },
"missing ownership": func(network map[string]any) {
network["Labels"] = map[string]string{networkRoleLabel: browserNetworkRole}
},
"wrong role": func(network map[string]any) {
network["Labels"] = map[string]string{managedLabel: "true", networkRoleLabel: "control"}
},
"missing ICC": func(network map[string]any) { network["Options"] = map[string]string{} },
"enabled ICC": func(network map[string]any) { network["Options"] = map[string]string{bridgeICCOption: "true"} },
"extra option": func(network map[string]any) {
network["Options"] = map[string]string{bridgeICCOption: "false", "unexpected": "value"}
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
network := make(map[string]any, len(valid))
for key, value := range valid {
network[key] = value
}
mutate(network)
docker := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(response).Encode(network)
}))
defer docker.Close()
client := dockerClient{baseURL: docker.URL, client: docker.Client()}
if err := client.ensureInternalNetwork("creatorhub_browser"); err == nil {
t.Fatal("expected unsafe existing network to be rejected")
}
})
}
}
func TestEnsureInternalNetworkRejectsControlNetwork(t *testing.T) {
requested := false
docker := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requested = true }))
defer docker.Close()
client := dockerClient{baseURL: docker.URL, client: docker.Client()}
if err := client.ensureInternalNetwork(controlNetworkName); err == nil || requested {
t.Fatalf("expected control network to be rejected before Docker request, requested=%v err=%v", requested, err)
}
}