diff --git a/cmd/control-plane/hub.go b/cmd/control-plane/hub.go index e34b8ff..e8660df 100644 --- a/cmd/control-plane/hub.go +++ b/cmd/control-plane/hub.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "strings" "sync" "time" @@ -30,7 +31,20 @@ type hubStore interface { ListEnvs(ctx context.Context) ([]hub.Env, error) GetEnv(ctx context.Context, alias string) (hub.Env, error) UpgradeEnv(ctx context.Context, alias, version string) error - DeleteEnv(ctx context.Context, alias string) error + CreateNetworkExit(ctx context.Context, exit hub.NetworkExit, credentialReferenceID string) (hub.NetworkExit, error) + ListNetworkExits(ctx context.Context) ([]hub.NetworkExit, error) + GetNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error) + GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error) + RecordNetworkExitCheck(ctx context.Context, id string, observation hub.ExitObservation, failureReason string) (hub.NetworkExit, string, error) + DisableNetworkExit(ctx context.Context, id string) (hub.NetworkExit, error) + CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) + GetEnvironmentContext(ctx context.Context, alias string) (hub.EnvironmentContext, error) + ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error + RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (hub.EnvironmentContext, error) + ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (hub.EnvironmentContext, error) + ReleaseRuntime(ctx context.Context, alias string) error + SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error + AppendEnvironmentAction(ctx context.Context, eventType string, action hub.EnvironmentAction) error } const ( @@ -64,18 +78,32 @@ func gatewayCall(ctx context.Context, target hub.Gateway, method, path string, b return 0, nil, requestErr } defer response.Body.Close() - responseBody, _ = io.ReadAll(io.LimitReader(response.Body, 1<<20)) - return response.StatusCode, responseBody, nil + responseBody, err = io.ReadAll(io.LimitReader(response.Body, 1<<20)) + return response.StatusCode, responseBody, err } -func gatewayCreatePayload(env hub.Env, imageRef string) map[string]any { - cmd := append(env.Fingerprint.Args(), "about:blank") +func gatewayCreatePayload(environment hub.EnvironmentContext, imageRef string, networkExit gatewayNetworkExit) map[string]any { + fingerprint := environment.Fingerprint + fingerprint.ProxyServer = "" + fingerprint.DisableNonProxiedUDP = false + cmd := append(fingerprint.Args(), "about:blank") return map[string]any{ - "alias": env.Alias, - "name": env.Name, - "image": imageRef, - "cmd": cmd, - "volume": "creatorhub-profile-" + env.Alias, + "alias": environment.Alias, + "name": environment.Name, + "image": imageRef, + "cmd": cmd, + "volume": "creatorhub-profile-" + environment.Alias, + "binding_version": environment.BindingVersion, + "network_exit_id": environment.Exit.ID, + "network_exit": networkExit, + } +} + +func gatewayProxyPayload(environment hub.EnvironmentContext, networkExit gatewayNetworkExit) map[string]any { + return map[string]any{ + "binding_version": environment.BindingVersion, + "network_exit_id": environment.Exit.ID, + "network_exit": networkExit, } } @@ -98,7 +126,7 @@ func gatewayUnreachable(err error) error { return gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf("gateway unreachable: %v", err)} } -func reconcileGatewayBrowser(ctx context.Context, target hub.Gateway, alias string) (bool, error) { +func reconcileGatewayContainer(ctx context.Context, target hub.Gateway, alias string) (containerStatus, bool, error) { var lastErr error for attempt := 0; attempt < gatewayReconcileAttempts; attempt++ { if attempt > 0 { @@ -106,7 +134,7 @@ func reconcileGatewayBrowser(ctx context.Context, target hub.Gateway, alias stri select { case <-ctx.Done(): timer.Stop() - return false, ctx.Err() + return containerStatus{}, false, ctx.Err() case <-timer.C: } } @@ -119,19 +147,38 @@ func reconcileGatewayBrowser(ctx context.Context, target hub.Gateway, alias stri lastErr = gatewayRejected(status, body) continue } - lastErr = nil - var browsers []containerStatus - if err := json.Unmarshal(body, &browsers); err != nil { - lastErr = fmt.Errorf("decode gateway browser list: %w", err) + browsers, parseErr := parseGatewayBrowserList(body) + if parseErr != nil { + lastErr = parseErr continue } + lastErr = nil for _, browser := range browsers { if browser.Alias == alias { - return true, nil + return browser, true, nil } } } - return false, lastErr + return containerStatus{}, false, lastErr +} + +func parseGatewayBrowserList(body []byte) ([]containerStatus, error) { + var entries []json.RawMessage + if err := json.Unmarshal(body, &entries); err != nil || entries == nil { + return nil, errors.New("gateway returned an invalid browser list") + } + browsers := make([]containerStatus, 0, len(entries)) + aliases := make(map[string]bool, len(entries)) + for _, entry := range entries { + var browser *containerStatus + if err := json.Unmarshal(entry, &browser); err != nil || browser == nil || browser.ID == "" || browser.Alias == "" || browser.State == "" || + browser.BindingVersion < 0 || aliases[browser.Alias] { + return nil, errors.New("gateway returned an invalid browser list") + } + aliases[browser.Alias] = true + browsers = append(browsers, *browser) + } + return browsers, nil } func errorFromBody(body []byte, status int) string { @@ -145,36 +192,56 @@ func errorFromBody(body []byte, status int) string { } type containerStatus struct { - ID string `json:"id"` - Alias string `json:"alias"` - Name string `json:"name"` - State string `json:"state"` - Status string `json:"status"` - Endpoint string `json:"endpoint"` + ID string `json:"id"` + Alias string `json:"alias"` + Name string `json:"name"` + State string `json:"state"` + Status string `json:"status"` + Endpoint string `json:"endpoint"` + BindingVersion int64 `json:"binding_version"` + NetworkExitID string `json:"network_exit_id"` + ProxyReady bool `json:"proxy_ready"` } type envView struct { hub.Env - State string `json:"state"` - Status string `json:"status"` - ContainerID string `json:"container_id"` - Endpoint string `json:"endpoint"` + State string `json:"state"` + Status string `json:"status"` + ContainerID string `json:"container_id"` + Endpoint string `json:"endpoint"` + NetworkExitID string `json:"network_exit_id"` + BindingVersion int64 `json:"binding_version"` + RecoveryRequired bool `json:"recovery_required"` + CleanupPending bool `json:"cleanup_pending"` +} + +func containerMatchesBinding(container containerStatus, environment hub.EnvironmentContext) bool { + return container.BindingVersion == environment.BindingVersion && container.NetworkExitID == environment.Exit.ID } func registerHub(app *fiber.App, store hubStore) { - var operations sync.Mutex + registerHubWithNetwork(app, store, defaultNetworkExitProbe(), resolveExitCredential) +} + +var runtimeOperations sync.Mutex + +func registerHubWithNetwork(app *fiber.App, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) { // ponytail: one control-plane instance is serialized globally; use keyed/distributed locks if replicas or throughput require it. serialized := func(handler fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { - operations.Lock() - defer operations.Unlock() + runtimeOperations.Lock() + defer runtimeOperations.Unlock() return handler(c) } } - app.Get("/api/browsers", listBrowsers(store)) - app.Post("/api/browsers", serialized(createBrowser(store))) - app.Post("/api/browsers/:alias/:action", serialized(browserAction(store))) + app.Get("/api/browsers", serialized(listBrowsers(store, probe, resolve))) + app.Post("/api/browsers", serialized(createBrowser(store, probe, resolve))) + app.Post("/api/browsers/:alias/:action", serialized(browserAction(store, probe, resolve))) app.Delete("/api/browsers/:alias", serialized(deleteBrowser(store))) + app.Get("/api/network-exits", listNetworkExits(store)) + app.Post("/api/network-exits", serialized(createNetworkExit(store))) + app.Post("/api/network-exits/:id/check", serialized(checkNetworkExit(store, probe))) + app.Post("/api/network-exits/:id/disable", serialized(disableNetworkExit(store, probe, resolve))) app.Get("/api/browser-images", func(c fiber.Ctx) error { images, err := store.ListImages(c.Context(), false) @@ -259,39 +326,116 @@ func registerHub(app *fiber.App, store hubStore) { }) } -func listBrowsers(store hubStore) fiber.Handler { +func listNetworkExits(store hubStore) fiber.Handler { + return func(c fiber.Ctx) error { + exits, err := store.ListNetworkExits(c.Context()) + if err != nil { + return hubError(c, err) + } + return c.JSON(exits) + } +} + +func createNetworkExit(store hubStore) fiber.Handler { + return func(c fiber.Ctx) error { + var input struct { + Protocol string `json:"protocol"` + Host string `json:"host"` + Port int `json:"port"` + CredentialReference struct { + ID string `json:"id"` + } `json:"credential_reference"` + ExpectedPublicIP string `json:"expected_public_ip"` + ExpectedRegion string `json:"expected_region"` + } + if err := decodeHubJSON(c, &input); err != nil { + return hubError(c, err) + } + exit, err := store.CreateNetworkExit(c.Context(), hub.NetworkExit{ + Protocol: input.Protocol, Host: input.Host, Port: input.Port, + ExpectedPublicIP: input.ExpectedPublicIP, ExpectedRegion: input.ExpectedRegion, + }, input.CredentialReference.ID) + if err != nil { + return hubError(c, err) + } + return c.Status(fiber.StatusCreated).JSON(exit) + } +} + +func checkNetworkExit(store hubStore, probe networkExitProbe) fiber.Handler { + return func(c fiber.Ctx) error { + access, err := store.GetNetworkExitAccess(c.Context(), c.Params("id")) + if err != nil { + return hubError(c, err) + } + if access.HealthStatus == "disabled" { + return hubError(c, hub.ErrConflict) + } + observation, failureReason := probe.Check(c.Context(), access) + exit, reason, err := store.RecordNetworkExitCheck(c.Context(), access.ID, observation, failureReason) + if err != nil { + return hubError(c, err) + } + return c.JSON(struct { + hub.NetworkExit + ReasonCode string `json:"reason_code"` + }{exit, reason}) + } +} + +func disableNetworkExit(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { + return func(c fiber.Ctx) error { + exit, err := store.DisableNetworkExit(c.Context(), c.Params("id")) + if err != nil { + return hubError(c, err) + } + if err := reconcileRuntimeLeasesUnlocked(c.Context(), store, probe, resolve); err != nil { + return hubError(c, err) + } + return c.JSON(exit) + } +} + +func verifyNetworkExit(ctx context.Context, store hubStore, probe networkExitProbe, id string) (hub.NetworkExitAccess, string, error) { + access, err := store.GetNetworkExitAccess(ctx, id) + if err != nil { + return hub.NetworkExitAccess{}, "exit_unavailable", err + } + if access.HealthStatus == "disabled" { + return hub.NetworkExitAccess{}, "exit_disabled", hub.ErrConflict + } + observation, failureReason := probe.Check(ctx, access) + exit, reason, err := store.RecordNetworkExitCheck(ctx, id, observation, failureReason) + if err != nil { + return hub.NetworkExitAccess{}, reason, err + } + if exit.HealthStatus != "healthy" { + return hub.NetworkExitAccess{}, reason, hub.ErrConflict + } + access, err = store.GetNetworkExitAccess(ctx, id) + return access, reason, err +} + +func listBrowsers(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { envs, err := store.ListEnvs(c.Context()) if err != nil { return hubError(c, err) } - gateways := map[string]hub.Gateway{} - containers := map[string]map[string]containerStatus{} - for _, env := range envs { - if _, ok := gateways[env.Gateway]; ok { - continue - } - gateway, err := store.GetGateway(c.Context(), env.Gateway) - if err != nil { - continue // 环境引用的网关被删除属于异常状态，列表仍要展示该环境 - } - gateways[env.Gateway] = gateway - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodGet, "/v1/browsers", nil, 30*time.Second) - if callErr != nil || status != http.StatusOK { - continue - } - var list []containerStatus - if json.Unmarshal(body, &list) == nil { - byAlias := map[string]containerStatus{} - for _, container := range list { - byAlias[container.Alias] = container - } - containers[env.Gateway] = byAlias - } + containers, gatewayRead, gatewayErrors := gatewayContainerSnapshot(c.Context(), store, envs) + if err := reconcileRuntimeSnapshot(c.Context(), store, probe, resolve, envs, containers, gatewayRead, gatewayErrors); err != nil { + return hubError(c, err) } views := make([]envView, 0, len(envs)) for _, env := range envs { view := envView{Env: env, State: "missing", Status: "网关上不存在容器"} + if environment, contextErr := store.GetEnvironmentContext(c.Context(), env.Alias); contextErr == nil { + view.NetworkExitID, view.BindingVersion = environment.Exit.ID, environment.BindingVersion + view.CleanupPending = environment.RuntimeCleanupPending + view.RecoveryRequired = environment.Exit.ID == "" || environment.RuntimeCleanupPending + } else if !errors.Is(contextErr, hub.ErrNotFound) { + return hubError(c, contextErr) + } if container, ok := containers[env.Gateway][env.Alias]; ok { view.State, view.Status = container.State, container.Status view.ContainerID, view.Endpoint = container.ID, container.Endpoint @@ -302,161 +446,1049 @@ func listBrowsers(store hubStore) fiber.Handler { } } -func createBrowser(store hubStore) fiber.Handler { +func reconcileRuntimeLeases(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error { + runtimeOperations.Lock() + defer runtimeOperations.Unlock() + return reconcileRuntimeLeasesUnlocked(ctx, store, probe, resolve) +} + +func reconcileRuntimeLeasesUnlocked(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) error { + envs, err := store.ListEnvs(ctx) + if err != nil { + return err + } + containers, gatewayRead, gatewayErrors := gatewayContainerSnapshot(ctx, store, envs) + return reconcileRuntimeSnapshot(ctx, store, probe, resolve, envs, containers, gatewayRead, gatewayErrors) +} + +func gatewayContainerSnapshot(ctx context.Context, store hubStore, envs []hub.Env) (map[string]map[string]containerStatus, map[string]bool, map[string]error) { + gateways := map[string]bool{} + containers := map[string]map[string]containerStatus{} + gatewayRead := map[string]bool{} + gatewayErrors := map[string]error{} + for _, env := range envs { + if gateways[env.Gateway] { + continue + } + gateways[env.Gateway] = true + gateway, err := store.GetGateway(ctx, env.Gateway) + if err != nil { + gatewayErrors[env.Gateway] = gatewayUnreachable(err) + continue + } + status, body, callErr := gatewayCall(ctx, gateway, http.MethodGet, "/v1/browsers", nil, 30*time.Second) + if callErr != nil { + gatewayErrors[env.Gateway] = gatewayUnreachable(callErr) + continue + } + if status != http.StatusOK { + gatewayErrors[env.Gateway] = gatewayRejected(status, body) + continue + } + list, parseErr := parseGatewayBrowserList(body) + if parseErr != nil { + gatewayErrors[env.Gateway] = gatewayFailure{status: http.StatusBadGateway, message: "gateway returned an invalid browser list"} + continue + } + byAlias := map[string]containerStatus{} + for _, container := range list { + byAlias[container.Alias] = container + } + containers[env.Gateway] = byAlias + gatewayRead[env.Gateway] = true + } + return containers, gatewayRead, gatewayErrors +} + +func reconcileRuntimeSnapshot(ctx context.Context, store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), envs []hub.Env, + containers map[string]map[string]containerStatus, gatewayRead map[string]bool, gatewayErrors map[string]error) error { + var firstErr error + for _, env := range envs { + if readErr := gatewayErrors[env.Gateway]; readErr != nil { + if firstErr == nil { + firstErr = readErr + } + continue + } + environment, err := store.GetEnvironmentContext(ctx, env.Alias) + if errors.Is(err, hub.ErrNotFound) { + continue + } + if err != nil { + return err + } + if environment.RuntimeCleanupPending { + target, targetErr := store.GetGateway(ctx, environment.Gateway) + if targetErr != nil { + return targetErr + } + if _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment); cleanupErr != nil { + return cleanupErr + } + continue + } + container, found := containers[env.Gateway][env.Alias] + if found && container.State == "running" { + if environment.Exit.ID == "" { + continue + } + auditRecovery := !containerMatchesBinding(container, environment) || !container.ProxyReady + action := actionForEnvironment("reconcile", environment) + if auditRecovery { + if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { + return err + } + } + ready, restoreErr := restoreOrRebuildRuntime(ctx, store, probe, resolve, environment, container) + if auditRecovery { + action.Outcome, action.ReasonCode = "succeeded", "runtime_reconciled" + if !ready { + action.Outcome, action.ReasonCode = "failed", "runtime_unavailable" + } + if restoreErr != nil { + action.Outcome, action.ReasonCode = runtimeRecoveryFailure(ctx, store, environment.Alias, restoreErr) + } + if err := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); err != nil { + return errors.Join(restoreErr, err) + } + } + if restoreErr != nil { + return restoreErr + } + } else if gatewayRead[env.Gateway] { + if err := store.ReleaseRuntime(ctx, env.Alias); err != nil { + return err + } + } + } + return firstErr +} + +func runtimeRecoveryFailure(ctx context.Context, store hubStore, alias string, err error) (string, string) { + var gatewayErr gatewayFailure + if errors.As(err, &gatewayErr) { + return "unknown", "gateway_result_unknown" + } + if environment, contextErr := store.GetEnvironmentContext(ctx, alias); contextErr == nil && environment.RuntimeCleanupPending { + return "unknown", "cleanup_result_unknown" + } + return "failed", "runtime_persistence_failed" +} + +func restoreOrRebuildRuntime(ctx context.Context, store hubStore, probe networkExitProbe, + resolve func(hub.NetworkExitAccess) (string, error), environment hub.EnvironmentContext, container containerStatus) (bool, error) { + if environment.RuntimeCleanupPending { + target, err := store.GetGateway(ctx, environment.Gateway) + if err != nil { + return false, err + } + _, err = removeGatewayRuntime(ctx, store, target, environment) + return false, err + } + access, _, err := verifyNetworkExit(ctx, store, probe, environment.Exit.ID) + if err != nil { + return false, discardRuntime(ctx, store, environment) + } + target, err := store.GetGateway(ctx, environment.Gateway) + if err != nil { + return false, err + } + if containerMatchesBinding(container, environment) && container.ProxyReady { + _, err := store.ActivateRuntime(ctx, environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + return err == nil, err + } + networkExit, err := gatewayNetworkExitFor(access, resolve) + if err != nil { + return false, discardRuntime(ctx, store, environment) + } + if containerMatchesBinding(container, environment) { + status, _, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/proxy", + gatewayProxyPayload(environment, networkExit), 30*time.Second) + if callErr == nil && status == http.StatusNoContent { + container.ProxyReady = true + } + if container.ProxyReady { + _, err := store.ActivateRuntime(ctx, environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + return err == nil, err + } + } + _, removeErr := removeGatewayRuntime(ctx, store, target, environment) + if removeErr != nil { + return false, removeErr + } + imageRef, err := store.ImageRef(ctx, environment.ImageVersion) + if err != nil { + return false, err + } + status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", + gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) + if callErr != nil { + return false, gatewayUnreachable(callErr) + } + if status != http.StatusCreated { + return false, gatewayRejected(status, body) + } + var created struct { + ID string `json:"id"` + } + if json.Unmarshal(body, &created) != nil || created.ID == "" { + return false, errors.New("gateway returned an invalid runtime id") + } + _, err = store.ActivateRuntime(ctx, environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID) + return err == nil, err +} + +func discardRuntime(ctx context.Context, store hubStore, environment hub.EnvironmentContext) error { + target, targetErr := store.GetGateway(ctx, environment.Gateway) + if targetErr != nil { + return targetErr + } + _, err := removeGatewayRuntime(ctx, store, target, environment) + return err +} + +func createBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { input := struct { - Alias string `json:"alias"` - Name string `json:"name"` - Gateway string `json:"gateway"` - ImageVersion string `json:"image_version"` - Fingerprint hub.Fingerprint `json:"fingerprint"` + Alias string `json:"alias"` + Name string `json:"name"` + Gateway string `json:"gateway"` + ImageVersion string `json:"image_version"` + Fingerprint hub.Fingerprint `json:"fingerprint"` + AccountID string `json:"account_id"` + NetworkExitID string `json:"network_exit_id"` }{} if err := decodeHubJSON(c, &input); err != nil { return hubError(c, err) } + if input.Fingerprint.ProxyServer != "" { + return hubError(c, hub.ErrInvalid) + } + input.Fingerprint.DisableNonProxiedUDP = false env := hub.Env{Alias: input.Alias, Name: input.Name, Gateway: input.Gateway, ImageVersion: input.ImageVersion, Fingerprint: input.Fingerprint} if err := env.Fingerprint.Validate(); err != nil { return c.Status(fiber.StatusBadRequest).JSON(map[string]string{"error": err.Error()}) } - gateway, err := store.GetGateway(c.Context(), env.Gateway) + exit, err := store.GetNetworkExit(c.Context(), input.NetworkExitID) if err != nil { return hubError(c, err) } + if exit.HealthStatus != "healthy" { + return hubError(c, hub.ErrConflict) + } + environment, created, err := store.CreateBoundEnv(c.Context(), env, input.AccountID, input.NetworkExitID) + if err != nil { + return hubError(c, err) + } + action := actionForEnvironment("create", environment) + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { + return hubError(c, err) + } + finish := func(outcome, reason string, current hub.EnvironmentContext) error { + action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID + action.BindingVersion = current.BindingVersion + return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) + } + access, reason, err := verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID) + if err != nil { + _ = finish("failed", reason, environment) + return hubError(c, err) + } + gateway, err := store.GetGateway(c.Context(), env.Gateway) + if err != nil { + _ = finish("failed", "gateway_unavailable", environment) + return hubError(c, err) + } + if environment.RuntimeCleanupPending { + if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + environment, err = store.GetEnvironmentContext(c.Context(), env.Alias) + if err != nil { + return hubError(c, err) + } + } imageRef, err := store.ImageRef(c.Context(), env.ImageVersion) if err != nil { + _ = finish("failed", "image_unavailable", environment) return hubError(c, err) } - if err := store.CreateEnv(c.Context(), env); err != nil { - return hubError(c, err) + networkExit, err := gatewayNetworkExitFor(access, resolve) + if err != nil { + _ = finish("failed", "credential_unavailable", environment) + return hubError(c, hub.ErrConflict) } - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(env, imageRef), gatewayLongTimeout) + if !created { + container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, env.Alias) + if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayUnreachable(reconcileErr)) + } + if found && container.State == "running" { + ready, restoreErr := restoreOrRebuildRuntime(c.Context(), store, probe, resolve, environment, container) + if restoreErr != nil { + outcome, reason := runtimeRecoveryFailure(c.Context(), store, environment.Alias, restoreErr) + _ = finish(outcome, reason, environment) + return hubError(c, restoreErr) + } + if !ready { + _ = finish("failed", "runtime_unavailable", environment) + return c.Status(fiber.StatusConflict).JSON(map[string]string{"error": "environment exists but runtime is not running"}) + } + environment, err = store.GetEnvironmentContext(c.Context(), env.Alias) + if err != nil { + return hubError(c, err) + } + if err := finish("succeeded", "environment_reused", environment); err != nil { + return hubError(c, err) + } + return c.JSON(map[string]string{"alias": env.Alias}) + } + } + status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) if callErr != nil || status >= http.StatusInternalServerError { createErr := gatewayRejected(status, body) if callErr != nil { createErr = gatewayUnreachable(callErr) } - exists, reconcileErr := reconcileGatewayBrowser(c.Context(), gateway, env.Alias) - if reconcileErr == nil && exists { + container, exists, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, env.Alias) + if reconcileErr == nil && exists && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { + environment, err = store.ActivateRuntime(c.Context(), env.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + if err != nil { + _ = finish("failed", "runtime_persistence_failed", environment) + return hubError(c, err) + } + if err := finish("succeeded", "gateway_reconciled", environment); err != nil { + return hubError(c, err) + } return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias}) } if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown", environment) return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf( "gateway create result unknown; environment retained for reconciliation: %v", createErr)}) } - _ = store.DeleteEnv(c.Context(), env.Alias) + _ = finish("failed", "gateway_create_failed", environment) return hubError(c, createErr) } if status != http.StatusCreated { - _ = store.DeleteEnv(c.Context(), env.Alias) + _ = finish("failed", "gateway_rejected", environment) return hubError(c, gatewayRejected(status, body)) } + var createdRuntime struct { + ID string `json:"id"` + } + if json.Unmarshal(body, &createdRuntime) != nil || createdRuntime.ID == "" { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway create result unknown; environment retained for reconciliation"}) + } + environment, err = store.ActivateRuntime(c.Context(), env.Alias, createdRuntime.ID, environment.BindingVersion, environment.Exit.ID) + if err != nil { + _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment) + if cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + } else { + _ = finish("failed", "runtime_persistence_failed", environment) + } + return hubError(c, err) + } + if err := finish("succeeded", "environment_created", environment); err != nil { + return hubError(c, err) + } return c.Status(fiber.StatusCreated).JSON(map[string]string{"alias": env.Alias}) } } -func browserAction(store hubStore) fiber.Handler { +func browserAction(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error)) fiber.Handler { return func(c fiber.Ctx) error { switch c.Params("action") { case "start", "stop": - return lifecycleAction(store, c) + return lifecycleAction(store, probe, resolve, c) case "upgrade": - return upgradeBrowser(store, c) + return upgradeBrowser(store, probe, resolve, c) + case "rebind": + return rebindBrowser(store, probe, resolve, c) default: return hubError(c, hub.ErrInvalid) } } } -func lifecycleAction(store hubStore, c fiber.Ctx) error { - env, gateway, err := envWithGateway(c, store, c.Params("alias")) +func lifecycleAction(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { + environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) if err != nil { return hubError(c, err) } - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers/"+env.Alias+"/"+c.Params("action"), nil, 30*time.Second) + actionName := strings.Clone(c.Params("action")) + action := actionForEnvironment(actionName, environment) + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { + return hubError(c, err) + } + finish := func(outcome, reason string, current hub.EnvironmentContext) error { + action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID + action.BindingVersion = current.BindingVersion + return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) + } + if actionName == "start" { + return startBrowser(store, probe, resolve, c, environment, finish) + } + gateway, err := store.GetGateway(c.Context(), environment.Gateway) + if err != nil { + _ = finish("failed", "gateway_unavailable", environment) + return hubError(c, err) + } + if environment.RuntimeCleanupPending { + if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + if err := finish("succeeded", "environment_stopped", environment); err != nil { + return hubError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) + } + status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers/"+environment.Alias+"/"+actionName, nil, 30*time.Second) if callErr != nil { + container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) + if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayUnreachable(callErr)) + } + if !found || container.State != "running" { + if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil { + _ = finish("failed", "runtime_release_failed", environment) + return hubError(c, err) + } + if err := finish("succeeded", "gateway_reconciled", environment); err != nil { + return hubError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) + } + _ = finish("failed", "gateway_action_failed", environment) return hubError(c, gatewayUnreachable(callErr)) } if status != http.StatusNoContent && status != http.StatusNotModified { + _ = finish("failed", "gateway_rejected", environment) return hubError(c, gatewayRejected(status, body)) } + if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil { + _ = finish("failed", "runtime_release_failed", environment) + return hubError(c, err) + } + if err := finish("succeeded", "environment_stopped", environment); err != nil { + return hubError(c, err) + } return c.SendStatus(fiber.StatusNoContent) } -func upgradeBrowser(store hubStore, c fiber.Ctx) error { +func startBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx, + environment hub.EnvironmentContext, finish func(string, string, hub.EnvironmentContext) error) error { + access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID) + if err != nil { + if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + _ = finish("failed", reason, environment) + return hubError(c, err) + } + gateway, err := store.GetGateway(c.Context(), environment.Gateway) + if err != nil { + _ = finish("failed", "gateway_unavailable", environment) + return hubError(c, err) + } + if environment.RuntimeCleanupPending { + if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + environment, err = store.GetEnvironmentContext(c.Context(), environment.Alias) + if err != nil { + return hubError(c, err) + } + } + imageRef, err := store.ImageRef(c.Context(), environment.ImageVersion) + if err != nil { + if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + _ = finish("failed", "image_unavailable", environment) + return hubError(c, err) + } + networkExit, err := gatewayNetworkExitFor(access, resolve) + if err != nil { + if cleanupErr := discardRuntime(c.Context(), store, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + _ = finish("failed", "credential_unavailable", environment) + return hubError(c, hub.ErrConflict) + } + container, found, err := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) + if err != nil { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayUnreachable(err)) + } + if found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { + environment, err = store.ActivateRuntime(c.Context(), environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + if err != nil { + _ = finish("failed", "runtime_persistence_failed", environment) + return hubError(c, err) + } + if err := finish("succeeded", "gateway_reconciled", environment); err != nil { + return hubError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) + } + if found { + if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, removeErr) + } + } else if err := store.ReleaseRuntime(c.Context(), environment.Alias); err != nil { + _ = finish("failed", "runtime_release_failed", environment) + return hubError(c, err) + } + status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", + gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) + if callErr != nil || status >= http.StatusInternalServerError { + container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) + if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}) + } + if !found || container.State != "running" || !container.ProxyReady || !containerMatchesBinding(container, environment) { + _ = finish("failed", "gateway_create_failed", environment) + if callErr != nil { + return hubError(c, gatewayUnreachable(callErr)) + } + return hubError(c, gatewayRejected(status, body)) + } + environment, err = store.ActivateRuntime(c.Context(), environment.Alias, container.ID, environment.BindingVersion, environment.Exit.ID) + } else { + if status != http.StatusCreated { + _ = finish("failed", "gateway_rejected", environment) + return hubError(c, gatewayRejected(status, body)) + } + var created struct { + ID string `json:"id"` + } + if json.Unmarshal(body, &created) != nil || created.ID == "" { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway start result unknown; retry to reconcile"}) + } + environment, err = store.ActivateRuntime(c.Context(), environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID) + } + if err != nil { + _ = finish("failed", "runtime_persistence_failed", environment) + return hubError(c, err) + } + if err := finish("succeeded", "environment_started", environment); err != nil { + return hubError(c, err) + } + return c.SendStatus(fiber.StatusNoContent) +} + +func upgradeBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { var input struct { Version string `json:"version"` } if err := decodeHubJSON(c, &input); err != nil { return hubError(c, err) } - env, gateway, err := envWithGateway(c, store, c.Params("alias")) + environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) if err != nil { return hubError(c, err) } - imageRef, err := store.ImageRef(c.Context(), input.Version) - if err != nil { + var imageRef string + var imageErr error + if !hub.ValidImageVersion(input.Version) { + imageErr = hub.ErrInvalid + } else { + imageRef, imageErr = store.ImageRef(c.Context(), input.Version) + } + action := actionForEnvironment("upgrade", environment) + action.OldImageVersion = environment.ImageVersion + if imageErr != nil { + action.NewImageVersion, action.ReasonCode = "", "upgrade_input_rejected" + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { + return hubError(c, err) + } + action.Outcome = "failed" + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action); err != nil { + return hubError(c, err) + } + return hubError(c, imageErr) + } + action.NewImageVersion = input.Version + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { return hubError(c, err) } + finish := func(outcome, reason string, current hub.EnvironmentContext) error { + action.Outcome, action.ReasonCode, action.RuntimeInstanceID = outcome, reason, current.RuntimeInstanceID + action.BindingVersion = current.BindingVersion + return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) + } + access, reason, err := verifyNetworkExit(c.Context(), store, probe, environment.Exit.ID) + if err != nil { + _ = finish("failed", reason, environment) + return hubError(c, err) + } + gateway, err := store.GetGateway(c.Context(), environment.Gateway) + if err != nil { + _ = finish("failed", "gateway_unavailable", environment) + return hubError(c, err) + } + if environment.RuntimeCleanupPending { + if _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, cleanupErr) + } + environment, err = store.GetEnvironmentContext(c.Context(), environment.Alias) + if err != nil { + return hubError(c, err) + } + } + networkExit, err := gatewayNetworkExitFor(access, resolve) + if err != nil { + _ = finish("failed", "credential_unavailable", environment) + return hubError(c, hub.ErrConflict) + } // 先删容器（保留卷）；404 视为已删除，保证升级可重试。 - status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodDelete, "/v1/browsers/"+env.Alias, nil, 30*time.Second) - if callErr != nil { - return hubError(c, gatewayUnreachable(callErr)) + if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + return hubError(c, removeErr) } - if status != http.StatusNoContent && status != http.StatusNotFound { - return hubError(c, gatewayRejected(status, body)) + if err := store.UpgradeEnv(c.Context(), environment.Alias, input.Version); err != nil { + _ = finish("failed", "persistence_failed", environment) + return hubError(c, err) } - env.ImageVersion = input.Version - status, body, callErr = gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(env, imageRef), gatewayLongTimeout) - if callErr != nil { - return hubError(c, gatewayUnreachable(callErr)) + environment, err = store.GetEnvironmentContext(c.Context(), environment.Alias) + if err != nil { + _ = finish("failed", "persistence_failed", environment) + return hubError(c, err) } - if status != http.StatusCreated { - return hubError(c, gatewayRejected(status, body)) + status, body, callErr := gatewayCall(c.Context(), gateway, http.MethodPost, "/v1/browsers", gatewayCreatePayload(environment, imageRef, networkExit), gatewayLongTimeout) + var createdRuntime struct { + ID string `json:"id"` } - if err := store.UpgradeEnv(c.Context(), env.Alias, input.Version); err != nil { - cleanupStatus, cleanupBody, cleanupErr := gatewayCall(c.Context(), gateway, http.MethodDelete, "/v1/browsers/"+env.Alias, nil, 30*time.Second) - if cleanupErr != nil || (cleanupStatus != http.StatusNoContent && cleanupStatus != http.StatusNotFound) { - if cleanupErr == nil { - cleanupErr = gatewayRejected(cleanupStatus, cleanupBody) - } - return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: fmt.Sprintf( - "persist upgrade: %v; remove uncommitted container: %v", err, cleanupErr)}) + if callErr != nil || status >= http.StatusInternalServerError { + container, found, reconcileErr := reconcileGatewayContainer(c.Context(), gateway, environment.Alias) + if reconcileErr != nil { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}) } + if !found || container.State != "running" || !container.ProxyReady || !containerMatchesBinding(container, environment) { + _ = finish("failed", "gateway_create_failed", environment) + if callErr != nil { + return hubError(c, gatewayUnreachable(callErr)) + } + return hubError(c, gatewayRejected(status, body)) + } + createdRuntime.ID = container.ID + } else { + if status != http.StatusCreated { + _ = finish("failed", "gateway_rejected", environment) + return hubError(c, gatewayRejected(status, body)) + } + if json.Unmarshal(body, &createdRuntime) != nil || createdRuntime.ID == "" { + _ = finish("unknown", "gateway_result_unknown", environment) + return hubError(c, gatewayFailure{status: http.StatusBadGateway, message: "gateway upgrade result unknown; retry to reconcile"}) + } + } + environment, err = store.ActivateRuntime(c.Context(), environment.Alias, createdRuntime.ID, environment.BindingVersion, environment.Exit.ID) + if err != nil { + _, cleanupErr := removeGatewayRuntime(c.Context(), store, gateway, environment) + if cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", environment) + } else { + _ = finish("failed", "runtime_persistence_failed", environment) + } + return hubError(c, err) + } + if err := finish("succeeded", "environment_upgraded", environment); err != nil { return hubError(c, err) } return c.SendStatus(fiber.StatusNoContent) } -func deleteBrowser(store hubStore) fiber.Handler { - return func(c fiber.Ctx) error { - env, gateway, err := envWithGateway(c, store, c.Params("alias")) +type runtimeCreateSpec struct { + imageRef string + networkExit gatewayNetworkExit +} + +func prepareRuntimeCreate(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error), + environment hub.EnvironmentContext, access hub.NetworkExitAccess) (runtimeCreateSpec, error) { + imageRef, err := store.ImageRef(ctx, environment.ImageVersion) + if err != nil { + return runtimeCreateSpec{}, err + } + networkExit, err := gatewayNetworkExitFor(access, resolve) + if err != nil { + return runtimeCreateSpec{}, err + } + return runtimeCreateSpec{imageRef: imageRef, networkExit: networkExit}, nil +} + +func createGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, spec runtimeCreateSpec) (containerStatus, error) { + status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", + gatewayCreatePayload(environment, spec.imageRef, spec.networkExit), gatewayLongTimeout) + if callErr == nil && status == http.StatusCreated { + var created containerStatus + if json.Unmarshal(body, &created) == nil && created.ID != "" { + created.Alias, created.State = environment.Alias, "running" + created.BindingVersion, created.NetworkExitID, created.ProxyReady = environment.BindingVersion, environment.Exit.ID, true + return created, nil + } + } + container, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias) + if reconcileErr == nil && found && container.State == "running" && container.ProxyReady && containerMatchesBinding(container, environment) { + return container, nil + } + if callErr != nil { + return containerStatus{}, gatewayUnreachable(callErr) + } + if status != http.StatusCreated { + return containerStatus{}, gatewayRejected(status, body) + } + if reconcileErr != nil { + return containerStatus{}, gatewayUnreachable(reconcileErr) + } + return containerStatus{}, errors.New("gateway returned an invalid runtime id") +} + +func createStoppedGatewayRuntime(ctx context.Context, target hub.Gateway, environment hub.EnvironmentContext, imageRef string) error { + payload := gatewayCreatePayload(environment, imageRef, gatewayNetworkExit{}) + payload["stopped"] = true + status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers", payload, gatewayLongTimeout) + if callErr == nil && status == http.StatusCreated { + return nil + } + container, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias) + if reconcileErr == nil && found && container.State != "running" && containerMatchesBinding(container, environment) { + return nil + } + if callErr != nil { + return gatewayUnreachable(callErr) + } + if status != http.StatusCreated { + return gatewayRejected(status, body) + } + if reconcileErr != nil { + return gatewayUnreachable(reconcileErr) + } + return errors.New("gateway did not preserve the stopped runtime") +} + +func removeGatewayRuntime(ctx context.Context, store hubStore, target hub.Gateway, environment hub.EnvironmentContext) (bool, error) { + removed := environment.RuntimeCleanupPending + if !environment.RuntimeCleanupPending { + if err := store.SetRuntimeCleanupPending(ctx, environment.Alias, true); err != nil { + return false, err + } + } + for attempt := 0; attempt < 2; attempt++ { + status, body, callErr := gatewayCall(ctx, target, http.MethodDelete, "/v1/browsers/"+environment.Alias, nil, 30*time.Second) + if callErr == nil && (status == http.StatusNoContent || status == http.StatusNotFound) { + return true, store.SetRuntimeCleanupPending(ctx, environment.Alias, false) + } + if callErr == nil && status == http.StatusAccepted { + removed = true + continue + } + _, found, reconcileErr := reconcileGatewayContainer(ctx, target, environment.Alias) + if reconcileErr != nil { + if callErr != nil { + return removed, errors.Join(gatewayUnreachable(callErr), gatewayUnreachable(reconcileErr)) + } + return removed, errors.Join(gatewayRejected(status, body), gatewayUnreachable(reconcileErr)) + } + if found { + if callErr != nil { + return removed, gatewayUnreachable(callErr) + } + return removed, gatewayRejected(status, body) + } + removed = true + if callErr != nil { + return true, gatewayUnreachable(callErr) + } + return true, gatewayRejected(status, body) + } + if removed { + return true, gatewayFailure{status: http.StatusBadGateway, message: "gateway removed the container but network cleanup is pending"} + } + return false, gatewayFailure{status: http.StatusBadGateway, message: "gateway runtime cleanup is pending"} +} + +func restoreRebindRuntime(ctx context.Context, store hubStore, resolve func(hub.NetworkExitAccess) (string, error), + target hub.Gateway, environment hub.EnvironmentContext, previous containerStatus, found bool, prepared *runtimeCreateSpec) (bool, error) { + if !found { + return true, store.ReleaseRuntime(ctx, environment.Alias) + } + if environment.Exit.ID == "" { + if err := store.ReleaseRuntime(ctx, environment.Alias); err != nil { + return false, err + } + if previous.State == "running" { + return false, nil + } + imageRef := "" + if prepared != nil { + imageRef = prepared.imageRef + } else { + var err error + imageRef, err = store.ImageRef(ctx, environment.ImageVersion) + if err != nil { + return false, err + } + } + if err := createStoppedGatewayRuntime(ctx, target, environment, imageRef); err != nil { + _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment) + return false, errors.Join(err, cleanupErr) + } + return true, nil + } + spec := runtimeCreateSpec{} + if prepared != nil { + spec = *prepared + } else { + access, err := store.GetNetworkExitAccess(ctx, environment.Exit.ID) + if err != nil { + return false, err + } + spec, err = prepareRuntimeCreate(ctx, store, resolve, environment, access) + if err != nil { + return false, err + } + } + if err := store.ReleaseRuntime(ctx, environment.Alias); err != nil { + return false, err + } + created, err := createGatewayRuntime(ctx, target, environment, spec) + if err != nil { + _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment) + return false, errors.Join(err, cleanupErr) + } + if previous.State == "running" { + _, err = store.ActivateRuntime(ctx, environment.Alias, created.ID, environment.BindingVersion, environment.Exit.ID) + } else { + status, body, callErr := gatewayCall(ctx, target, http.MethodPost, "/v1/browsers/"+environment.Alias+"/stop", nil, 30*time.Second) + if callErr != nil { + err = gatewayUnreachable(callErr) + } else if status != http.StatusNoContent { + err = gatewayRejected(status, body) + } + } + if err != nil { + _, cleanupErr := removeGatewayRuntime(ctx, store, target, environment) + return false, errors.Join(err, cleanupErr) + } + return true, nil +} + +func rebindRecoveryError(ready bool, err error) error { + if err != nil { + return err + } + if !ready { + return gatewayFailure{status: http.StatusBadGateway, message: "rebind recovery is incomplete; retry reconciliation"} + } + return nil +} + +func runtimeCreateSpecMatches(ctx context.Context, store hubStore, current, previous hub.EnvironmentContext, prepared *runtimeCreateSpec) bool { + if prepared == nil || current.BindingVersion != previous.BindingVersion || current.ImageVersion != previous.ImageVersion || current.Exit.ID != previous.Exit.ID { + return false + } + imageRef, err := store.ImageRef(ctx, current.ImageVersion) + return err == nil && imageRef == prepared.imageRef +} + +func rebindBrowser(store hubStore, probe networkExitProbe, resolve func(hub.NetworkExitAccess) (string, error), c fiber.Ctx) error { + var input struct { + NetworkExitID string `json:"network_exit_id"` + } + if err := decodeHubJSON(c, &input); err != nil { + return hubError(c, err) + } + before, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) + if err != nil { + return hubError(c, err) + } + action := actionForEnvironment("rebind", before) + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { + return hubError(c, err) + } + finish := func(outcome, reason string, current hub.EnvironmentContext) error { + action.Outcome, action.ReasonCode = outcome, reason + action.NetworkExitID, action.BindingVersion, action.RuntimeInstanceID = current.Exit.ID, current.BindingVersion, current.RuntimeInstanceID + return store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) + } + access, reason, err := verifyNetworkExit(c.Context(), store, probe, input.NetworkExitID) + if err != nil { + _ = finish("failed", reason, before) + return hubError(c, err) + } + target, err := store.GetGateway(c.Context(), before.Gateway) + if err != nil { + _ = finish("failed", "gateway_unavailable", before) + return hubError(c, err) + } + if before.RuntimeCleanupPending { + if _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before); cleanupErr != nil { + _ = finish("unknown", "cleanup_result_unknown", before) + return hubError(c, cleanupErr) + } + before, err = store.GetEnvironmentContext(c.Context(), before.Alias) if err != nil { return hubError(c, err) } - status, _, callErr := gatewayCall(c.Context(), gateway, http.MethodDelete, "/v1/browsers/"+env.Alias, nil, 30*time.Second) - if callErr != nil { - return hubError(c, gatewayUnreachable(callErr)) + } + container, found, err := reconcileGatewayContainer(c.Context(), target, before.Alias) + if err != nil { + _ = finish("unknown", "gateway_result_unknown", before) + return hubError(c, gatewayUnreachable(err)) + } + wasRunning := found && container.State == "running" + if err := store.ValidateEnvironmentRebind(c.Context(), before.Alias, input.NetworkExitID, before.BindingVersion); err != nil { + _ = finish("failed", "rebind_not_allowed", before) + return hubError(c, err) + } + var previousSpec *runtimeCreateSpec + if found && before.Exit.ID == "" { + imageRef, imageErr := store.ImageRef(c.Context(), before.ImageVersion) + if imageErr != nil { + _ = finish("failed", "runtime_prepare_failed", before) + return hubError(c, imageErr) } - if status != http.StatusNoContent && status != http.StatusNotFound { - return hubError(c, errors.New("gateway refused container removal")) + previousSpec = &runtimeCreateSpec{imageRef: imageRef} + } else if found { + previousAccess, accessErr := store.GetNetworkExitAccess(c.Context(), before.Exit.ID) + if accessErr != nil { + _ = finish("failed", "credential_unavailable", before) + return hubError(c, accessErr) } - if err := store.DeleteEnv(c.Context(), env.Alias); err != nil { + prepared, prepareErr := prepareRuntimeCreate(c.Context(), store, resolve, before, previousAccess) + if prepareErr != nil { + _ = finish("failed", "runtime_prepare_failed", before) + return hubError(c, prepareErr) + } + previousSpec = &prepared + } + candidate := before + candidate.Exit, candidate.BindingVersion = access.NetworkExit, before.BindingVersion+1 + candidate.RuntimeInstanceID, candidate.RuntimeID = "", "" + var nextSpec runtimeCreateSpec + if wasRunning { + nextSpec, err = prepareRuntimeCreate(c.Context(), store, resolve, candidate, access) + if err != nil { + _ = finish("failed", "runtime_prepare_failed", before) + return hubError(c, err) + } + } + if found { + _, removeErr := removeGatewayRuntime(c.Context(), store, target, before) + if removeErr != nil { + _ = finish("unknown", "gateway_result_unknown", before) + return hubError(c, removeErr) + } + } + var candidateRuntime containerStatus + if wasRunning { + candidateRuntime, err = createGatewayRuntime(c.Context(), target, candidate, nextSpec) + if err != nil { + _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before) + if cleanupErr != nil { + _ = finish("unknown", "rebind_recovery_failed", before) + return hubError(c, errors.Join(err, cleanupErr)) + } + current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias) + ready := false + if contextErr == nil { + ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, container, found, previousSpec) + } + contextErr = rebindRecoveryError(ready, contextErr) + if contextErr != nil { + _ = finish("unknown", "rebind_recovery_failed", before) + return hubError(c, errors.Join(err, contextErr)) + } + _ = finish("failed", "gateway_create_failed", before) + return hubError(c, err) + } + } + after, err := store.RebindEnvironment(c.Context(), before.Alias, input.NetworkExitID, candidateRuntime.ID, before.BindingVersion) + if err != nil { + if candidateRuntime.ID != "" { + _, cleanupErr := removeGatewayRuntime(c.Context(), store, target, before) + if cleanupErr != nil { + _ = finish("unknown", "rebind_recovery_failed", before) + return hubError(c, errors.Join(err, cleanupErr)) + } + } + if found { + current, contextErr := store.GetEnvironmentContext(c.Context(), before.Alias) + ready := false + if contextErr == nil { + prepared := previousSpec + if !runtimeCreateSpecMatches(c.Context(), store, current, before, previousSpec) { + prepared = nil + } + ready, contextErr = restoreRebindRuntime(c.Context(), store, resolve, target, current, container, true, prepared) + } + contextErr = rebindRecoveryError(ready, contextErr) + if contextErr != nil { + _ = finish("unknown", "rebind_recovery_failed", before) + return hubError(c, errors.Join(err, contextErr)) + } + } + _ = finish("failed", "rebind_not_allowed", before) + return hubError(c, err) + } + if err := finish("succeeded", "environment_rebound", after); err != nil { + return hubError(c, err) + } + return c.JSON(after) +} + +func deleteBrowser(store hubStore) fiber.Handler { + return func(c fiber.Ctx) error { + environment, err := store.GetEnvironmentContext(c.Context(), c.Params("alias")) + if err != nil { + return hubError(c, err) + } + action := actionForEnvironment("recycle", environment) + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_requested", action); err != nil { + return hubError(c, err) + } + gateway, err := store.GetGateway(c.Context(), environment.Gateway) + if err != nil { + action.Outcome, action.ReasonCode = "failed", "gateway_unavailable" + _ = store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) + return hubError(c, err) + } + if _, removeErr := removeGatewayRuntime(c.Context(), store, gateway, environment); removeErr != nil { + action.Outcome, action.ReasonCode = "unknown", "cleanup_result_unknown" + _ = store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action) + return hubError(c, removeErr) + } + action.Outcome, action.ReasonCode = "succeeded", "environment_recycled" + if err := store.AppendEnvironmentAction(c.Context(), "environment_action_finished", action); err != nil { return hubError(c, err) } return c.SendStatus(fiber.StatusNoContent) } } -func envWithGateway(c fiber.Ctx, store hubStore, alias string) (hub.Env, hub.Gateway, error) { - env, err := store.GetEnv(c.Context(), alias) - if err != nil { - return hub.Env{}, hub.Gateway{}, err +func actionForEnvironment(actionName string, environment hub.EnvironmentContext) hub.EnvironmentAction { + return hub.EnvironmentAction{ + OperationID: hub.NewOperationID(), Action: actionName, ReasonCode: "action_requested", + AccountID: environment.AccountID, BrowserEnvAlias: environment.Alias, NetworkExitID: environment.Exit.ID, + RuntimeInstanceID: environment.RuntimeInstanceID, BindingVersion: environment.BindingVersion, + OldImageVersion: environment.ImageVersion, NewImageVersion: environment.ImageVersion, } - gateway, err := store.GetGateway(c.Context(), env.Gateway) - if err != nil { - return hub.Env{}, hub.Gateway{}, err - } - return env, gateway, nil } func decodeHubJSON(c fiber.Ctx, destination any) error { diff --git a/cmd/control-plane/hub_test.go b/cmd/control-plane/hub_test.go index ba3e9d3..0ab69ba 100644 --- a/cmd/control-plane/hub_test.go +++ b/cmd/control-plane/hub_test.go @@ -2,31 +2,41 @@ package main import ( "context" + "database/sql" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" + "net/url" + "os" "strings" "sync" "testing" "time" "git.ipao.vip/rogee/creator-hub/internal/hub" + "git.ipao.vip/rogee/creator-hub/internal/phasea" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/adaptor" ) // memoryStore 是 hubStore 的内存桩，记录写入以便断言编排副作用。 type memoryStore struct { - mu sync.Mutex - gateways map[string]hub.Gateway - images map[string]hub.Image - envs map[string]hub.Env - deleted []string - upgraded map[string]string - upgradeErr error - gatewayFn func(name string) (hub.Gateway, error) + mu sync.Mutex + gateways map[string]hub.Gateway + images map[string]hub.Image + envs map[string]hub.Env + exits map[string]hub.NetworkExit + bindings map[string]hub.EnvironmentContext + actions []hub.EnvironmentAction + upgraded map[string]string + upgradeErr error + releaseErr error + cleanupPendingErr error + cleanupPendingErrAfterMutation bool + gatewayFn func(name string) (hub.Gateway, error) } func newMemoryStore() *memoryStore { @@ -34,6 +44,10 @@ func newMemoryStore() *memoryStore { gateways: map[string]hub.Gateway{}, images: map[string]hub.Image{}, envs: map[string]hub.Env{}, + exits: map[string]hub.NetworkExit{ + "exit-1": {ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1}, + }, + bindings: map[string]hub.EnvironmentContext{}, upgraded: map[string]string{}, } } @@ -125,6 +139,9 @@ func (s *memoryStore) UpgradeEnv(_ context.Context, alias, version string) error if _, ok := s.envs[alias]; !ok { return hub.ErrNotFound } + if s.bindings[alias].RuntimeCleanupPending { + return hub.ErrConflict + } if image, exists := s.images[version]; !exists || !image.Enabled { return hub.ErrNotFound } @@ -132,13 +149,190 @@ func (s *memoryStore) UpgradeEnv(_ context.Context, alias, version string) error env := s.envs[alias] env.ImageVersion = version s.envs[alias] = env + bound, ok := s.bindings[alias] + if !ok { + bound = hub.EnvironmentContext{Env: env, AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + bound.Env, bound.BindingVersion = env, bound.BindingVersion+1 + s.bindings[alias] = bound return nil } -func (s *memoryStore) DeleteEnv(_ context.Context, alias string) error { +func (s *memoryStore) CreateNetworkExit(_ context.Context, exit hub.NetworkExit, credentialID string) (hub.NetworkExit, error) { s.mu.Lock() defer s.mu.Unlock() - s.deleted = append(s.deleted, alias) - delete(s.envs, alias) + exit.ID, exit.HealthStatus, exit.Version = "exit-created", "unchecked", 1 + if credentialID != "" { + exit.CredentialReference = &hub.CredentialReference{ID: credentialID, Provider: "os_keyring"} + } + s.exits[exit.ID] = exit + return exit, nil +} +func (s *memoryStore) ListNetworkExits(context.Context) ([]hub.NetworkExit, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := make([]hub.NetworkExit, 0, len(s.exits)) + for _, exit := range s.exits { + result = append(result, exit) + } + return result, nil +} +func (s *memoryStore) GetNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) { + s.mu.Lock() + defer s.mu.Unlock() + exit, ok := s.exits[id] + if !ok { + return hub.NetworkExit{}, hub.ErrNotFound + } + return exit, nil +} +func (s *memoryStore) GetNetworkExitAccess(ctx context.Context, id string) (hub.NetworkExitAccess, error) { + exit, err := s.GetNetworkExit(ctx, id) + return hub.NetworkExitAccess{NetworkExit: exit}, err +} +func (s *memoryStore) RecordNetworkExitCheck(_ context.Context, id string, observation hub.ExitObservation, failure string) (hub.NetworkExit, string, error) { + s.mu.Lock() + defer s.mu.Unlock() + exit, ok := s.exits[id] + if !ok { + return hub.NetworkExit{}, "exit_unavailable", hub.ErrNotFound + } + reason := failure + if reason == "" && exit.ExpectedPublicIP != "" && exit.ExpectedPublicIP != observation.PublicIP { + reason = "exit_ip_drift" + } + if reason == "" && exit.ExpectedRegion != "" && exit.ExpectedRegion != observation.Region { + reason = "exit_region_drift" + } + exit.ObservedPublicIP, exit.ObservedRegion = observation.PublicIP, observation.Region + if reason == "" { + exit.HealthStatus, reason = "healthy", "exit_healthy" + } else { + exit.HealthStatus = "unhealthy" + } + s.exits[id] = exit + return exit, reason, nil +} +func (s *memoryStore) DisableNetworkExit(_ context.Context, id string) (hub.NetworkExit, error) { + s.mu.Lock() + defer s.mu.Unlock() + exit, ok := s.exits[id] + if !ok { + return hub.NetworkExit{}, hub.ErrNotFound + } + exit.HealthStatus = "disabled" + s.exits[id] = exit + return exit, nil +} +func (s *memoryStore) CreateBoundEnv(ctx context.Context, env hub.Env, accountID, exitID string) (hub.EnvironmentContext, bool, error) { + s.mu.Lock() + if existing, ok := s.bindings[env.Alias]; ok { + s.mu.Unlock() + return existing, false, nil + } + s.mu.Unlock() + if err := s.CreateEnv(ctx, env); err != nil { + return hub.EnvironmentContext{}, false, err + } + s.mu.Lock() + defer s.mu.Unlock() + bound := hub.EnvironmentContext{Env: env, AccountID: accountID, BindingID: accountID, BindingVersion: 1, Exit: s.exits[exitID]} + s.bindings[env.Alias] = bound + return bound, true, nil +} +func (s *memoryStore) GetEnvironmentContext(_ context.Context, alias string) (hub.EnvironmentContext, error) { + s.mu.Lock() + defer s.mu.Unlock() + if bound, ok := s.bindings[alias]; ok { + return bound, nil + } + env, ok := s.envs[alias] + if !ok { + return hub.EnvironmentContext{}, hub.ErrNotFound + } + return hub.EnvironmentContext{Env: env, AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]}, nil +} +func (s *memoryStore) ValidateEnvironmentRebind(_ context.Context, alias, exitID string, expectedBindingVersion int64) error { + s.mu.Lock() + defer s.mu.Unlock() + bound, ok := s.bindings[alias] + if !ok { + bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending || s.exits[exitID].HealthStatus != "healthy" { + return hub.ErrConflict + } + return nil +} +func (s *memoryStore) RebindEnvironment(_ context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (hub.EnvironmentContext, error) { + s.mu.Lock() + defer s.mu.Unlock() + bound, ok := s.bindings[alias] + if !ok { + bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + if bound.BindingVersion != expectedBindingVersion || bound.RuntimeCleanupPending { + return hub.EnvironmentContext{}, hub.ErrConflict + } + bound.RuntimeInstanceID, bound.RuntimeID = "", "" + if runtimeID != "" { + bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID + } + bound.Exit, bound.BindingVersion = s.exits[exitID], bound.BindingVersion+1 + s.bindings[alias] = bound + return bound, nil +} +func (s *memoryStore) ActivateRuntime(_ context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (hub.EnvironmentContext, error) { + s.mu.Lock() + defer s.mu.Unlock() + bound, ok := s.bindings[alias] + if !ok { + bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + if bound.BindingVersion != bindingVersion || bound.Exit.ID != exitID || bound.RuntimeCleanupPending { + return hub.EnvironmentContext{}, hub.ErrConflict + } + bound.RuntimeInstanceID, bound.RuntimeID = "runtime-instance", runtimeID + s.bindings[alias] = bound + return bound, nil +} +func (s *memoryStore) ReleaseRuntime(_ context.Context, alias string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.releaseErr != nil { + return s.releaseErr + } + bound, ok := s.bindings[alias] + if !ok { + bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + bound.RuntimeInstanceID, bound.RuntimeID = "", "" + s.bindings[alias] = bound + return nil +} +func (s *memoryStore) SetRuntimeCleanupPending(_ context.Context, alias string, pending bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.cleanupPendingErr != nil && !s.cleanupPendingErrAfterMutation { + return s.cleanupPendingErr + } + if pending && s.releaseErr != nil { + return s.releaseErr + } + bound, ok := s.bindings[alias] + if !ok { + bound = hub.EnvironmentContext{Env: s.envs[alias], AccountID: alias, BindingID: alias, BindingVersion: 1, Exit: s.exits["exit-1"]} + } + bound.RuntimeCleanupPending = pending + if pending { + bound.RuntimeInstanceID, bound.RuntimeID = "", "" + } + s.bindings[alias] = bound + return s.cleanupPendingErr +} +func (s *memoryStore) AppendEnvironmentAction(_ context.Context, _ string, action hub.EnvironmentAction) error { + s.mu.Lock() + defer s.mu.Unlock() + s.actions = append(s.actions, action) return nil } @@ -150,14 +344,54 @@ type recordedRequest struct { // fakeGateway 模拟 docker-gateway：按路由表应答并记录请求。 type fakeGateway struct { - mu sync.Mutex - createOnce sync.Once - token string - requests []recordedRequest - containers []containerStatus - failCreate int // 前 N 次 create 返回失败 - createStarted chan struct{} - releaseCreate <-chan struct{} + mu sync.Mutex + createOnce sync.Once + deleteOnce sync.Once + token string + requests []recordedRequest + containers []containerStatus + failCreate int // 前 N 次 create 返回失败 + failDelete int // 前 N 次 delete 返回 500 且保留容器 + deleteNotFound int + failProxy bool + createStarted chan struct{} + releaseCreate <-chan struct{} + deleteDone chan struct{} + releaseDelete <-chan struct{} + cleanupPending int + disconnectDelete int + disconnectList int + failList int + invalidList int + invalidListBody string + readErrorList int + disconnectListAfterDelete bool +} + +type fakeExitProbe struct { + observation hub.ExitObservation + failure string +} + +func (probe fakeExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) { + if probe.observation.PublicIP == "" && probe.failure == "" { + probe.observation = hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"} + } + return probe.observation, probe.failure +} + +type sequenceExitProbe struct { + calls int + failures []string +} + +func (probe *sequenceExitProbe) Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) { + failure := "" + if probe.calls < len(probe.failures) { + failure = probe.failures[probe.calls] + } + probe.calls++ + return hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, failure } func (g *fakeGateway) handler(t *testing.T) http.Handler { @@ -195,14 +429,122 @@ func (g *fakeGateway) handler(t *testing.T) http.Handler { if g.releaseCreate != nil { <-g.releaseCreate } + state, proxyReady := "running", true + if stopped, _ := body["stopped"].(bool); stopped { + state, proxyReady = "exited", false + } + g.mu.Lock() + g.containers = []containerStatus{{ + ID: "container-id", Alias: body["alias"].(string), State: state, Status: state, ProxyReady: proxyReady, + BindingVersion: int64(body["binding_version"].(float64)), NetworkExitID: body["network_exit_id"].(string), + }} + g.mu.Unlock() response.WriteHeader(http.StatusCreated) _, _ = response.Write([]byte(`{"id":"container-id","alias":"account-a"}`)) case request.Method == http.MethodGet && request.URL.Path == "/v1/browsers": - _ = json.NewEncoder(response).Encode(g.containers) + g.mu.Lock() + if g.failList > 0 { + g.failList-- + g.mu.Unlock() + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(`{"error":"docker unavailable"}`)) + return + } + if g.invalidList > 0 { + g.invalidList-- + body := g.invalidListBody + if body == "" { + body = `{"not":"a browser list"}` + } + g.mu.Unlock() + _, _ = response.Write([]byte(body)) + return + } + if g.readErrorList > 0 { + g.readErrorList-- + g.mu.Unlock() + connection, _, _ := response.(http.Hijacker).Hijack() + _, _ = connection.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n[]")) + _ = connection.Close() + return + } + if g.disconnectList > 0 { + g.disconnectList-- + g.mu.Unlock() + connection, _, _ := response.(http.Hijacker).Hijack() + _ = connection.Close() + return + } + containers := append([]containerStatus{}, g.containers...) + g.mu.Unlock() + _ = json.NewEncoder(response).Encode(containers) case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/v1/browsers/"): + g.mu.Lock() + if g.failDelete > 0 { + g.failDelete-- + g.mu.Unlock() + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(`{"error":"docker delete failed"}`)) + return + } + g.containers = nil + if g.deleteNotFound > 0 { + g.deleteNotFound-- + g.mu.Unlock() + response.WriteHeader(http.StatusNotFound) + return + } + g.mu.Unlock() + if g.deleteDone != nil { + g.deleteOnce.Do(func() { close(g.deleteDone) }) + } + if g.releaseDelete != nil { + <-g.releaseDelete + } + g.mu.Lock() + if g.cleanupPending > 0 { + g.cleanupPending-- + g.mu.Unlock() + response.WriteHeader(http.StatusAccepted) + return + } + if g.disconnectDelete > 0 { + g.disconnectDelete-- + if g.disconnectListAfterDelete { + g.disconnectList++ + } + g.mu.Unlock() + connection, _, _ := response.(http.Hijacker).Hijack() + _ = connection.Close() + return + } + g.mu.Unlock() response.WriteHeader(http.StatusNoContent) - case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"), - request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"): + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/start"): + g.mu.Lock() + if len(g.containers) > 0 { + g.containers[0].State, g.containers[0].Status = "running", "Up" + } + g.mu.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/stop"): + g.mu.Lock() + if len(g.containers) > 0 { + g.containers[0].State, g.containers[0].Status = "exited", "Exited" + } + g.mu.Unlock() + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/proxy"): + g.mu.Lock() + if g.failProxy { + g.mu.Unlock() + response.WriteHeader(http.StatusBadGateway) + return + } + if len(g.containers) > 0 { + g.containers[0].ProxyReady = true + } + g.mu.Unlock() response.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected gateway request %s %s", request.Method, request.URL.Path) @@ -217,6 +559,11 @@ func (g *fakeGateway) recorded() []recordedRequest { } func newTestApp(t *testing.T, store *memoryStore, gateway *fakeGateway) *fiber.App { + return newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) +} + +func newTestAppWithNetwork(t *testing.T, store *memoryStore, gateway *fakeGateway, probe networkExitProbe, + resolve func(hub.NetworkExitAccess) (string, error)) *fiber.App { t.Helper() server := httptest.NewServer(gateway.handler(t)) t.Cleanup(server.Close) @@ -225,7 +572,7 @@ func newTestApp(t *testing.T, store *memoryStore, gateway *fakeGateway) *fiber.A } store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} app := fiber.New() - registerHub(app, store) + registerHubWithNetwork(app, store, probe, resolve) return app } @@ -240,7 +587,53 @@ func do(app *fiber.App, method, path, body string) *httptest.ResponseRecorder { } const createEnvBody = `{"alias":"account-a","name":"店铺一号","gateway":"gw-1","image_version":"148.0.7778.215",` + - `"fingerprint":{"seed":2024,"platform":"windows","timezone":"Asia/Shanghai"}}` + `"fingerprint":{"seed":2024,"platform":"windows","timezone":"Asia/Shanghai"},"account_id":"account-a","network_exit_id":"exit-1"}` + +func TestParseGatewayBrowserListStrict(t *testing.T) { + for _, test := range []struct { + name string + body string + ok bool + }{ + {name: "empty", body: `[]`, ok: true}, + {name: "minimal browser", body: `[{"id":"container-id","alias":"account-a","state":"running"}]`, ok: true}, + {name: "top-level null", body: `null`}, + {name: "null element", body: `[null]`}, + {name: "missing id", body: `[{"alias":"account-a","state":"running"}]`}, + {name: "missing alias", body: `[{"id":"container-id","state":"running"}]`}, + {name: "missing state", body: `[{"id":"container-id","alias":"account-a"}]`}, + {name: "duplicate alias", body: `[{"id":"one","alias":"account-a","state":"running"},{"id":"two","alias":"account-a","state":"exited"}]`}, + } { + t.Run(test.name, func(t *testing.T) { + browsers, err := parseGatewayBrowserList([]byte(test.body)) + if (err == nil) != test.ok { + t.Fatalf("parse result browsers=%#v err=%v", browsers, err) + } + }) + } +} + +func TestGatewayCallPropagatesBodyReadError(t *testing.T) { + gateway := &fakeGateway{token: "unit-test-gateway-token", readErrorList: 1} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + + status, body, err := gatewayCall(context.Background(), hub.Gateway{Endpoint: server.URL, Token: gateway.token}, + http.MethodGet, "/v1/browsers", nil, time.Second) + if status != http.StatusOK || string(body) != "[]" || err == nil { + t.Fatalf("partial response was accepted: status=%d body=%q err=%v", status, body, err) + } +} + +func TestGatewayCreatePayloadStripsLegacyProxyFingerprint(t *testing.T) { + payload := gatewayCreatePayload(hub.EnvironmentContext{Env: hub.Env{Alias: "account-a", Name: "甲", Fingerprint: hub.Fingerprint{ + Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true, + }}, BindingVersion: 1, Exit: hub.NetworkExit{ID: "exit-1"}}, "registry.example/browser:1", gatewayNetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}) + encoded, _ := json.Marshal(payload) + if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") { + t.Fatalf("legacy proxy URI entered the gateway contract: %s", encoded) + } +} func TestCreateBrowserOrchestratesGateway(t *testing.T) { store := newMemoryStore() @@ -265,14 +658,100 @@ func TestCreateBrowserOrchestratesGateway(t *testing.T) { payload["volume"] != "creatorhub-profile-account-a" { t.Fatalf("platform must fully specify the gateway payload: %#v", payload) } + exit := payload["network_exit"].(map[string]any) + if exit["protocol"] != "socks5" || exit["host"] != "127.0.0.1" || exit["port"] != float64(1080) { + t.Fatalf("platform must force the bound exit: %#v", payload) + } cmd := payload["cmd"].([]any) if len(cmd) != 4 || cmd[0] != "--fingerprint=2024" || cmd[1] != "--fingerprint-platform=windows" || cmd[2] != "--timezone=Asia/Shanghai" || cmd[3] != "about:blank" { t.Fatalf("cmd must carry fingerprint args plus start url: %#v", cmd) } + if stored := store.envs["account-a"].Fingerprint; stored.ProxyServer != "" || stored.DisableNonProxiedUDP { + t.Fatalf("persistent fingerprint must not contain proxy material: %#v", stored) + } + if len(store.actions) != 2 || store.actions[0].OperationID != store.actions[1].OperationID || + store.actions[0].Action != "create" || store.actions[1].Outcome != "succeeded" { + t.Fatalf("create must emit a correlated audit pair: %#v", store.actions) + } } -func TestCreateBrowserRollsBackWhenGatewayRejects(t *testing.T) { +func TestExitFailuresStopCreateBeforeGateway(t *testing.T) { + for _, test := range []struct { + name string + probe fakeExitProbe + reason string + }{ + {name: "identity drift", probe: fakeExitProbe{observation: hub.ExitObservation{PublicIP: "203.0.113.11", Region: "test"}}, reason: "exit_ip_drift"}, + {name: "authentication rejected", probe: fakeExitProbe{failure: "proxy_auth_failed"}, reason: "proxy_auth_failed"}, + {name: "proxy disconnected", probe: fakeExitProbe{failure: "proxy_check_failed"}, reason: "proxy_check_failed"}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + store.exits["exit-1"] = hub.NetworkExit{ + ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, + ExpectedPublicIP: "203.0.113.10", HealthStatus: "healthy", Version: 1, + } + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token"} + server := httptest.NewServer(gateway.handler(t)) + defer server.Close() + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: server.URL, Token: gateway.token} + app := fiber.New() + registerHubWithNetwork(app, store, test.probe, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodPost, "/api/browsers", createEnvBody) + if response.Code != http.StatusConflict { + t.Fatalf("expected fail-close 409, got %d: %s", response.Code, response.Body.String()) + } + if len(gateway.recorded()) != 0 { + t.Fatalf("exit failure must stop before any gateway start/create call: %#v", gateway.recorded()) + } + bound, err := store.GetEnvironmentContext(context.Background(), "account-a") + if err != nil || bound.RuntimeInstanceID != "" || store.exits["exit-1"].HealthStatus != "unhealthy" { + t.Fatalf("failed create must retain only the stable inactive binding: %#v err=%v", bound, err) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != test.reason { + t.Fatalf("exit failure must be auditable without secrets: %#v", store.actions) + } + }) + } +} + +func TestRepeatedCreateAndStartReuseStableEnvironment(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token"} + app := newTestApp(t, store, gateway) + + if response := do(app, http.MethodPost, "/api/browsers", createEnvBody); response.Code != http.StatusCreated { + t.Fatalf("initial create failed: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/browsers", createEnvBody); response.Code != http.StatusOK { + t.Fatalf("idempotent create failed: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { + t.Fatalf("idempotent start failed: %d %s", response.Code, response.Body.String()) + } + bound := store.bindings["account-a"] + if bound.Alias != "account-a" || bound.Exit.ID != "exit-1" || bound.RuntimeID != "container-id" { + t.Fatalf("repeated actions changed stable environment identity: %#v", bound) + } + createCalls := 0 + for _, request := range gateway.recorded() { + if request.method == http.MethodPost && request.path == "/v1/browsers" { + createCalls++ + if request.body["volume"] != "creatorhub-profile-account-a" || request.body["network_exit"].(map[string]any)["host"] != "127.0.0.1" { + t.Fatalf("create changed Profile volume or exit: %#v", request.body) + } + } + } + if createCalls != 1 { + t.Fatalf("idempotent create called gateway create %d times", createCalls) + } +} + +func TestCreateBrowserKeepsStableBindingWhenGatewayRejects(t *testing.T) { store := newMemoryStore() _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1} @@ -282,11 +761,8 @@ func TestCreateBrowserRollsBackWhenGatewayRejects(t *testing.T) { if response.Code != http.StatusConflict { t.Fatalf("expected gateway conflict to pass through as 409, got %d: %s", response.Code, response.Body.String()) } - if len(store.deleted) != 1 || store.deleted[0] != "account-a" { - t.Fatalf("env must be rolled back after gateway rejection, deleted=%v", store.deleted) - } - if _, kept := store.envs["account-a"]; kept { - t.Fatal("rolled-back env must not remain in the store") + if _, kept := store.envs["account-a"]; !kept { + t.Fatal("stable environment and Profile anchor must remain retryable") } } @@ -312,7 +788,7 @@ func TestCreateBrowserReconcilesDisconnectedGateway(t *testing.T) { exists := created mu.Unlock() if exists { - _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a"}}) + _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) return } _ = json.NewEncoder(response).Encode([]containerStatus{}) @@ -341,7 +817,7 @@ func TestCreateBrowserReconcilesDisconnectedGateway(t *testing.T) { _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() - registerHub(app, store) + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != test.wantStatus { @@ -350,8 +826,12 @@ func TestCreateBrowserReconcilesDisconnectedGateway(t *testing.T) { if _, err := store.GetEnv(context.Background(), "account-a"); err != nil { t.Fatalf("unknown gateway result must retain the environment for reconciliation: %v", err) } - if len(store.deleted) != 0 { - t.Fatalf("unknown result must not roll back the environment: %v", store.deleted) + wantOutcome := "unknown" + if test.reconcile { + wantOutcome = "succeeded" + } + if len(store.actions) != 2 || store.actions[1].Outcome != wantOutcome { + t.Fatalf("gateway reconciliation outcome must be audited as %s: %#v", wantOutcome, store.actions) } }) } @@ -386,7 +866,7 @@ func TestCreateBrowserReconcilesGatewayBadGateway(t *testing.T) { _ = json.NewEncoder(response).Encode([]containerStatus{}) return } - _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a"}}) + _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}}) })) defer gatewayServer.Close() @@ -394,14 +874,14 @@ func TestCreateBrowserReconcilesGatewayBadGateway(t *testing.T) { _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} app := fiber.New() - registerHub(app, store) + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) response := do(app, http.MethodPost, "/api/browsers", createEnvBody) if response.Code != http.StatusCreated { t.Fatalf("expected reconciled create, status=%d body=%s", response.Code, response.Body.String()) } - if _, err := store.GetEnv(context.Background(), "account-a"); err != nil || len(store.deleted) != 0 { - t.Fatalf("reconciled create must retain DB state: err=%v deleted=%v", err, store.deleted) + if _, err := store.GetEnv(context.Background(), "account-a"); err != nil { + t.Fatalf("reconciled create must retain DB state: err=%v", err) } mu.Lock() gotCalls := listCalls @@ -436,7 +916,7 @@ func TestListBrowsersMergesLiveGatewayState(t *testing.T) { gateway := &fakeGateway{ token: "unit-test-gateway-token", containers: []containerStatus{ - {ID: "id-1", Alias: "account-a", State: "running", Status: "Up", Endpoint: "http://creatorhub-browser-account-a:9222"}, + {ID: "id-1", Alias: "account-a", State: "running", Status: "Up", Endpoint: "http://creatorhub-browser-account-a:9222", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true}, }, } app := newTestApp(t, store, gateway) @@ -459,6 +939,58 @@ func TestListBrowsersMergesLiveGatewayState(t *testing.T) { if byAlias["account-b"].State != "missing" { t.Fatalf("env without container must report missing: %#v", byAlias["account-b"]) } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "id-1" { + t.Fatalf("list reconciliation must heartbeat the running runtime, got %q", runtime) + } +} + +func TestListRestoresProxyAfterGatewayRestartBeforeHeartbeat(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 3, Exit: store.exits["exit-1"], + } + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 3, NetworkExitID: "exit-1", ProxyReady: false, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("gateway restart recovery failed: %d %s", response.Code, response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" { + t.Fatalf("runtime was activated before proxy recovery completed: %q", runtime) + } + requests := gateway.recorded() + if len(requests) != 2 || requests[0].path != "/v1/browsers" || requests[1].path != "/v1/browsers/account-a/proxy" { + t.Fatalf("expected list then proxy recovery without rebuild: %#v", requests) + } + if requests[1].body["binding_version"] != float64(3) || requests[1].body["network_exit_id"] != "exit-1" { + t.Fatalf("proxy recovery did not use the current binding: %#v", requests[1].body) + } +} + +func TestGatewayRestartRebuildsWhenOriginalProxyPortCannotBeRestored(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, Exit: store.exits["exit-1"]} + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", failProxy: true, containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 2, NetworkExitID: "exit-1", + }}} + app := newTestApp(t, store, gateway) + + if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { + t.Fatalf("gateway restart rebuild failed: %d %s", response.Code, response.Body.String()) + } + requests := gateway.recorded() + if len(requests) != 4 || requests[1].path != "/v1/browsers/account-a/proxy" || requests[2].method != http.MethodDelete || requests[3].path != "/v1/browsers" { + t.Fatalf("failed proxy recovery must preserve Profile by rebuilding the container: %#v", requests) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" { + t.Fatalf("rebuilt runtime was not activated: %q", runtime) + } } func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { @@ -484,6 +1016,9 @@ func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { if payload["image"] != "registry.example/browser:144" || payload["volume"] != "creatorhub-profile-account-a" { t.Fatalf("upgrade must reuse the profile volume and switch image: %#v", payload) } + if payload["binding_version"] != float64(2) || payload["network_exit_id"] != "exit-1" { + t.Fatalf("upgrade must create from the committed binding generation: %#v", payload) + } cmd := payload["cmd"].([]any) if cmd[0] != "--fingerprint=2024" || cmd[len(cmd)-1] != "about:blank" { t.Fatalf("upgrade must reuse stored fingerprint params: %#v", cmd) @@ -491,9 +1026,1319 @@ func TestUpgradeBrowserRecreatesWithSameVolumeAndParams(t *testing.T) { if store.upgraded["account-a"] != "144.0.7559.132" || store.envs["account-a"].ImageVersion != "144.0.7559.132" { t.Fatal("image version must be persisted after successful upgrade") } + if len(store.actions) != 2 || store.actions[0].OldImageVersion != "148" || + store.actions[1].NewImageVersion != "144.0.7559.132" || store.actions[1].Outcome != "succeeded" { + t.Fatalf("upgrade must emit image-aware audit evidence: %#v", store.actions) + } } -func TestUpgradeBrowserRemovesUncommittedContainer(t *testing.T) { +func TestUpgradeBrowserUsesCommittedPostgresBinding(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + accountStore, err := phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + if err := accountStore.Close(); err != nil { + t.Fatal(err) + } + store, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + accountStore, err = phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + if err := accountStore.CreateAccount(ctx, phasea.Account{ + ID: "account-a", Platform: "mock", PlatformAccountKey: "account-a", AuthorizationKind: "owned", + CredentialReference: phasea.CredentialReference{ID: "credential-account", Provider: "os_keyring"}, + CredentialKey: "creatorhub/account-a", + }); err != nil { + t.Fatal(err) + } + if err := accountStore.Close(); err != nil { + t.Fatal(err) + } + gateway := &fakeGateway{token: "unit-test-gateway-token"} + gatewayServer := httptest.NewServer(gateway.handler(t)) + t.Cleanup(gatewayServer.Close) + if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil { + t.Fatal(err) + } + for _, image := range []hub.Image{ + {Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}, + {Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}, + } { + if err := store.CreateImage(ctx, image); err != nil { + t.Fatal(err) + } + } + exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080}, "") + if err != nil { + t.Fatal(err) + } + exit, _, err = store.RecordNetworkExitCheck(ctx, exit.ID, hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "") + if err != nil { + t.Fatal(err) + } + before, created, err := store.CreateBoundEnv(ctx, hub.Env{ + Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}, + }, "account-a", exit.ID) + if err != nil || !created { + t.Fatalf("create bound environment: created=%v err=%v", created, err) + } + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"149"}`) + if response.Code != http.StatusNoContent { + t.Fatalf("PostgreSQL-backed upgrade failed: %d: %s", response.Code, response.Body.String()) + } + after, err := store.GetEnvironmentContext(ctx, "account-a") + if err != nil { + t.Fatal(err) + } + if after.ImageVersion != "149" || after.BindingVersion != before.BindingVersion+1 || after.RuntimeID != "container-id" { + t.Fatalf("upgrade did not activate the committed PostgreSQL binding: before=%#v after=%#v", before, after) + } + requests := gateway.recorded() + if len(requests) != 2 || requests[1].body["binding_version"] != float64(after.BindingVersion) || + requests[1].body["network_exit_id"] != after.Exit.ID { + t.Fatalf("gateway labels diverged from the committed binding: %#v", requests) + } + + gateway.mu.Lock() + gateway.failDelete = 1 + gateway.containers = []containerStatus{{ + ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: after.BindingVersion - 1, + NetworkExitID: after.Exit.ID, ProxyReady: true, + }} + gateway.mu.Unlock() + response = do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusBadGateway { + t.Fatalf("PostgreSQL-backed reconcile delete failure returned %d: %s", response.Code, response.Body.String()) + } + released, err := store.GetEnvironmentContext(ctx, "account-a") + if err != nil || released.RuntimeID != "" || released.BindingVersion != after.BindingVersion || !released.RuntimeCleanupPending { + t.Fatalf("delete failure must release the real Store lease without changing binding: %#v err=%v", released, err) + } + response = do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("PostgreSQL-backed cleanup retry returned %d: %s", response.Code, response.Body.String()) + } + released, err = store.GetEnvironmentContext(ctx, "account-a") + if err != nil || released.RuntimeCleanupPending { + t.Fatalf("confirmed cleanup remained pending: %#v err=%v", released, err) + } + if _, err := store.ActivateRuntime(ctx, "account-a", "coherent-container", released.BindingVersion, released.Exit.ID); err != nil { + t.Fatal(err) + } + if err := store.ReleaseRuntime(ctx, "account-a"); err != nil { + t.Fatal(err) + } + gateway.mu.Lock() + gateway.failDelete = 1 + gateway.containers = []containerStatus{{ + ID: "coherent-container", Alias: "account-a", State: "running", BindingVersion: released.BindingVersion, + NetworkExitID: released.Exit.ID, ProxyReady: true, + }} + gateway.mu.Unlock() + response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+released.Exit.ID+`"}`) + if response.Code != http.StatusBadGateway { + t.Fatalf("PostgreSQL-backed rebind delete failure returned %d: %s", response.Code, response.Body.String()) + } + unchanged, err := store.GetEnvironmentContext(ctx, "account-a") + if err != nil || unchanged.BindingVersion != released.BindingVersion || unchanged.Exit.ID != released.Exit.ID || unchanged.RuntimeID != "" || !unchanged.RuntimeCleanupPending { + t.Fatalf("rebind committed before delete confirmation: %#v err=%v", unchanged, err) + } +} + +type postgresRebindFixture struct { + store *hub.Store + db *sql.DB + gateway *fakeGateway + bound hub.EnvironmentContext + exit hub.NetworkExit +} + +type cleanupCommitUnknownStore struct { + hubStore +} + +func (s cleanupCommitUnknownStore) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error { + if err := s.hubStore.SetRuntimeCleanupPending(ctx, alias, pending); err != nil { + return err + } + return errors.New("cleanup commit result unknown") +} + +func newPostgresRebindFixture(t *testing.T, databaseURL string) postgresRebindFixture { + t.Helper() + ctx := context.Background() + databaseURL = isolatedControlPlaneDatabaseURL(t, databaseURL) + accountStore, err := phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + if err := accountStore.Close(); err != nil { + t.Fatal(err) + } + store, err := hub.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + accountStore, err = phasea.Open(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + if err := accountStore.CreateAccount(ctx, phasea.Account{ + ID: "account-a", Platform: "mock", PlatformAccountKey: "account-a", AuthorizationKind: "owned", + CredentialReference: phasea.CredentialReference{ID: "credential-account", Provider: "os_keyring"}, + CredentialKey: "creatorhub/account-a", + }); err != nil { + t.Fatal(err) + } + if err := accountStore.Close(); err != nil { + t.Fatal(err) + } + gateway := &fakeGateway{token: "unit-test-gateway-token"} + gatewayServer := httptest.NewServer(gateway.handler(t)) + t.Cleanup(gatewayServer.Close) + if _, err := store.CreateGateway(ctx, "gw-1", gatewayServer.URL, gateway.token); err != nil { + t.Fatal(err) + } + if err := store.CreateImage(ctx, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}); err != nil { + t.Fatal(err) + } + exit, err := store.CreateNetworkExit(ctx, hub.NetworkExit{Protocol: "http", Host: "proxy.example", Port: 8080}, "") + if err != nil { + t.Fatal(err) + } + exit, _, err = store.RecordNetworkExitCheck(ctx, exit.ID, hub.ExitObservation{PublicIP: "203.0.113.1", Region: "test"}, "") + if err != nil { + t.Fatal(err) + } + bound, created, err := store.CreateBoundEnv(ctx, hub.Env{ + Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}, + }, "account-a", exit.ID) + if err != nil || !created { + t.Fatalf("create bound environment: created=%v err=%v", created, err) + } + db, err := sql.Open("pgx", databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + return postgresRebindFixture{store: store, db: db, gateway: gateway, bound: bound, exit: exit} +} + +func installCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB, condition string) { + t.Helper() + if _, err := db.ExecContext(ctx, ` + CREATE FUNCTION fail_cleanup_transition() RETURNS trigger AS $$ + BEGIN + IF `+condition+` THEN + RAISE EXCEPTION 'injected cleanup state failure'; + END IF; + RETURN NEW; + END + $$ LANGUAGE plpgsql`); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, ` + CREATE TRIGGER fail_cleanup_transition BEFORE UPDATE OF runtime_cleanup_pending ON environment_binding + FOR EACH ROW EXECUTE FUNCTION fail_cleanup_transition()`); err != nil { + t.Fatal(err) + } +} + +func dropCleanupTransitionFailure(t *testing.T, ctx context.Context, db *sql.DB) { + t.Helper() + if _, err := db.ExecContext(ctx, `DROP TRIGGER fail_cleanup_transition ON environment_binding`); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `DROP FUNCTION fail_cleanup_transition()`); err != nil { + t.Fatal(err) + } +} + +func TestPostgresRebindRecoversRealConcurrentRaces(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + for _, test := range []struct { + name string + race string + state string + nullBinding bool + }{ + {name: "active runtime after DELETE", race: "active", state: "running"}, + {name: "executing task after DELETE", race: "executing", state: "running"}, + {name: "binding version after DELETE", race: "version", state: "running"}, + {name: "image upgrade blocked during DELETE", race: "upgrade", state: "running"}, + {name: "stopped container after DELETE", race: "version", state: "exited"}, + {name: "stopped NULL binding after DELETE", race: "version", state: "exited", nullBinding: true}, + } { + t.Run(test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + if test.race == "upgrade" { + if err := fixture.store.CreateImage(ctx, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}); err != nil { + t.Fatal(err) + } + } + if test.nullBinding { + if _, err := fixture.db.ExecContext(ctx, `UPDATE environment_binding SET network_exit_id = NULL WHERE browser_env_alias = 'account-a'`); err != nil { + t.Fatal(err) + } + var err error + fixture.bound, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil { + t.Fatal(err) + } + } + deleteDone, releaseDelete := make(chan struct{}), make(chan struct{}) + fixture.gateway.deleteDone, fixture.gateway.releaseDelete = deleteDone, releaseDelete + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: test.state, ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + app := fiber.New() + registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + server := httptest.NewServer(adaptor.FiberApp(app)) + defer server.Close() + type result struct { + status int + body string + err error + } + done := make(chan result, 1) + go func() { + response, err := server.Client().Post(server.URL+"/api/browsers/account-a/rebind", "application/json", + strings.NewReader(`{"network_exit_id":"`+fixture.exit.ID+`"}`)) + if err != nil { + done <- result{err: err} + return + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + done <- result{status: response.StatusCode, body: string(body)} + }() + select { + case <-deleteDone: + case <-time.After(time.Second): + close(releaseDelete) + t.Fatal("rebind did not reach the DELETE barrier") + } + tx, err := fixture.db.BeginTx(ctx, nil) + upgradeBlocked := false + if err == nil { + switch test.race { + case "active": + _, err = tx.ExecContext(ctx, ` + INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) + VALUES ('runtime-race', 'account-a', 'account-a', 'old-container', now() + interval '1 minute')`) + case "executing": + _, err = tx.ExecContext(ctx, ` + INSERT INTO content_draft (id, account_id, version, content) VALUES ('draft-rebind', 'account-a', 1, 'test'); + INSERT INTO operation_task (id, idempotency_key, account_id, account_version, draft_id, draft_version, state) + SELECT 'task-rebind', 'task-rebind', id, version, 'draft-rebind', 1, 'executing' + FROM social_account WHERE id = 'account-a'`) + case "version": + _, err = tx.ExecContext(ctx, `UPDATE environment_binding SET version = version + 1 WHERE browser_env_alias = 'account-a'`) + case "upgrade": + err = fixture.store.UpgradeEnv(ctx, "account-a", "149") + if errors.Is(err, hub.ErrConflict) { + upgradeBlocked, err = true, nil + } + } + } + if err == nil { + err = tx.Commit() + } else if tx != nil { + _ = tx.Rollback() + } + close(releaseDelete) + response := <-done + if err != nil { + t.Fatal(err) + } + expectedStatus := http.StatusConflict + if test.race == "upgrade" { + expectedStatus = http.StatusOK + } + if response.err != nil || response.status != expectedStatus { + t.Fatalf("rebind race returned status=%d body=%s err=%v", response.status, response.body, response.err) + } + if test.race == "upgrade" && !upgradeBlocked { + t.Fatal("upgrade advanced while runtime cleanup was pending") + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + expectedVersion := fixture.bound.BindingVersion + if test.race == "version" || test.race == "upgrade" { + expectedVersion++ + } + expectedImage := fixture.bound.ImageVersion + if err != nil || after.BindingVersion != expectedVersion || after.Exit.ID != fixture.bound.Exit.ID || after.ImageVersion != expectedImage { + t.Fatalf("failed rebind changed PostgreSQL binding: before=%#v after=%#v err=%v", fixture.bound, after, err) + } + fixture.gateway.mu.Lock() + containers := append([]containerStatus{}, fixture.gateway.containers...) + fixture.gateway.mu.Unlock() + if test.nullBinding { + if after.RuntimeID != "" || len(containers) != 1 || containers[0].State == "running" || + !containerMatchesBinding(containers[0], after) { + t.Fatalf("NULL binding recovery must restore a network-disabled stopped container without a lease: after=%#v containers=%#v", after, containers) + } + } else if test.state == "exited" { + if after.RuntimeID != "" || len(containers) != 1 || containers[0].State != "exited" || !containerMatchesBinding(containers[0], after) { + t.Fatalf("stopped runtime was not restored: after=%#v containers=%#v", after, containers) + } + } else if after.RuntimeID == "" || len(containers) != 1 || containers[0].State != "running" || !containerMatchesBinding(containers[0], after) { + t.Fatalf("running runtime was not restored coherently: after=%#v containers=%#v", after, containers) + } + if test.race == "upgrade" { + requests := fixture.gateway.recorded() + var lastCreate map[string]any + for _, request := range requests { + if request.method == http.MethodPost && request.path == "/v1/browsers" { + lastCreate = request.body + } + } + if lastCreate["image"] != "registry.example/browser:148" || int64(lastCreate["binding_version"].(float64)) != after.BindingVersion { + t.Fatalf("blocked upgrade changed the rebind generation: %#v", lastCreate) + } + } + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event + WHERE event_type = 'environment_action_finished' AND action = 'rebind' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + expectedOutcome, expectedReason := "failed", "rebind_not_allowed" + if test.race == "upgrade" { + expectedOutcome, expectedReason = "succeeded", "environment_rebound" + } + if outcome != expectedOutcome || reason != expectedReason { + t.Fatalf("recovered conflict audit mismatch: outcome=%s reason=%s", outcome, reason) + } + }) + } +} + +func TestPostgresCleanupPendingPersistsAndReconciles(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + for _, test := range []struct { + name string + cleanupPending int + disconnectDelete int + disconnectListAfterDelete bool + }{ + {name: "accepted cleanup pending", cleanupPending: 2}, + {name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true}, + } { + t.Run(test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + fixture.gateway.cleanupPending = test.cleanupPending + fixture.gateway.disconnectDelete = test.disconnectDelete + fixture.gateway.disconnectListAfterDelete = test.disconnectListAfterDelete + app := fiber.New() + registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + target, err := fixture.store.GetGateway(ctx, "gw-1") + if err != nil { + t.Fatal(err) + } + removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) + if !removed || cleanupErr == nil { + t.Fatalf("non-final cleanup lost the removed fact: removed=%v err=%v", removed, cleanupErr) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || !after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion || + after.Exit.ID != fixture.bound.Exit.ID || after.RuntimeID != "" { + t.Fatalf("PostgreSQL did not persist cleanup pending atomically: %#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM environment_binding WHERE browser_env_alias = 'account-a' AND runtime_cleanup_pending`, 1) + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("list reconcile did not confirm cleanup: %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeCleanupPending { + t.Fatalf("confirmed cleanup remained pending: %#v err=%v", after, err) + } + + response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+fixture.exit.ID+`"}`) + if response.Code != http.StatusOK { + t.Fatalf("rebind retry failed after cleanup confirmation: %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeCleanupPending || after.BindingVersion != fixture.bound.BindingVersion+1 || after.RuntimeID != "" { + t.Fatalf("retry did not commit a consistent stopped generation: %#v err=%v", after, err) + } + }) + } +} + +func TestRemoveGatewayRuntimePersistsCleanupBeforeDelete(t *testing.T) { + newRuntime := func() (*memoryStore, *fakeGateway, hub.EnvironmentContext, hub.Gateway) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Gateway: "gw-1", ImageVersion: "148"} + environment := hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + } + store.bindings[environment.Alias] = environment + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: environment.Alias, State: "running", ProxyReady: true, + BindingVersion: environment.BindingVersion, NetworkExitID: environment.Exit.ID, + }}} + server := httptest.NewServer(gateway.handler(t)) + t.Cleanup(server.Close) + return store, gateway, environment, hub.Gateway{Endpoint: server.URL, Token: gateway.token} + } + + t.Run("rollback keeps the lease and skips DELETE", func(t *testing.T) { + store, gateway, environment, target := newRuntime() + store.cleanupPendingErr = errors.New("injected rollback") + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err == nil || removed || len(gateway.recorded()) != 0 { + t.Fatalf("failed pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) + } + after := store.bindings[environment.Alias] + if after.RuntimeCleanupPending || after.RuntimeID != environment.RuntimeID || len(gateway.containers) != 1 { + t.Fatalf("rollback did not preserve the active generation: after=%#v containers=%#v", after, gateway.containers) + } + }) + + t.Run("commit unknown remains retryable", func(t *testing.T) { + store, gateway, environment, target := newRuntime() + store.cleanupPendingErr = errors.New("commit result unknown") + store.cleanupPendingErrAfterMutation = true + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err == nil || removed || len(gateway.recorded()) != 0 { + t.Fatalf("unknown pre-mark result touched the gateway: removed=%v err=%v requests=%#v", removed, err, gateway.recorded()) + } + after := store.bindings[environment.Alias] + if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 1 { + t.Fatalf("committed pre-mark was not retryable: after=%#v containers=%#v", after, gateway.containers) + } + + store.cleanupPendingErr = nil + store.cleanupPendingErrAfterMutation = false + removed, err = removeGatewayRuntime(context.Background(), store, target, after) + if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending || len(gateway.containers) != 0 { + t.Fatalf("retry did not reconcile the committed pre-mark: removed=%v err=%v after=%#v containers=%#v", + removed, err, store.bindings[environment.Alias], gateway.containers) + } + }) + + for _, status := range []int{http.StatusNoContent, http.StatusNotFound} { + t.Run(fmt.Sprintf("clear rollback after %d", status), func(t *testing.T) { + store, gateway, environment, target := newRuntime() + if err := store.SetRuntimeCleanupPending(context.Background(), environment.Alias, true); err != nil { + t.Fatal(err) + } + environment = store.bindings[environment.Alias] + store.cleanupPendingErr = errors.New("injected clear rollback") + if status == http.StatusNotFound { + gateway.deleteNotFound = 1 + } + + removed, err := removeGatewayRuntime(context.Background(), store, target, environment) + if err == nil || !removed { + t.Fatalf("clear rollback lost the delete fact: removed=%v err=%v", removed, err) + } + after := store.bindings[environment.Alias] + if !after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 0 { + t.Fatalf("clear rollback was not retryable: after=%#v containers=%#v", after, gateway.containers) + } + + store.cleanupPendingErr = nil + removed, err = removeGatewayRuntime(context.Background(), store, target, after) + if err != nil || !removed || store.bindings[environment.Alias].RuntimeCleanupPending { + t.Fatalf("clear retry failed: removed=%v err=%v after=%#v", removed, err, store.bindings[environment.Alias]) + } + }) + } +} + +func TestRestoreAndDiscardPreserveGenerationWhenCleanupMarkFails(t *testing.T) { + for _, test := range []struct { + name string + commitKnown bool + outcome string + reason string + }{ + {name: "rollback", commitKnown: true, outcome: "failed", reason: "runtime_persistence_failed"}, + {name: "commit unknown", outcome: "unknown", reason: "cleanup_result_unknown"}, + } { + t.Run("restore "+test.name, func(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + } + store.cleanupPendingErr = errors.New("cleanup state unavailable") + store.cleanupPendingErrAfterMutation = !test.commitKnown + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusInternalServerError { + t.Fatalf("failed cleanup mark returned %d: %s", response.Code, response.Body.String()) + } + after := store.bindings["account-a"] + if test.commitKnown { + if after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + t.Fatalf("rollback changed the old generation: %#v", after) + } + } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("commit-unknown lost its retry marker: %#v", after) + } + if len(gateway.containers) != 1 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { + t.Fatalf("failed cleanup mark touched the old container: containers=%#v requests=%#v", gateway.containers, gateway.recorded()) + } + if len(store.actions) != 2 || store.actions[1].Outcome != test.outcome || store.actions[1].ReasonCode != test.reason { + t.Fatalf("restore audit mismatch: %#v", store.actions) + } + + store.cleanupPendingErr = nil + store.cleanupPendingErrAfterMutation = false + if response = do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { + t.Fatalf("list retry failed: %d: %s", response.Code, response.Body.String()) + } + if test.commitKnown { + after = store.bindings["account-a"] + } else { + if response = do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { + t.Fatalf("start retry failed: %d: %s", response.Code, response.Body.String()) + } + after = store.bindings["account-a"] + } + if after.RuntimeCleanupPending || after.RuntimeID == "" || len(gateway.containers) != 1 || !containerMatchesBinding(gateway.containers[0], after) { + t.Fatalf("retry ended inconsistently: after=%#v containers=%#v", after, gateway.containers) + } + }) + + t.Run("discard "+test.name, func(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + } + store.cleanupPendingErr = errors.New("cleanup state unavailable") + store.cleanupPendingErrAfterMutation = !test.commitKnown + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + }}} + app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "proxy_auth_failed"}, + func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodPost, "/api/browsers/account-a/start", "") + if response.Code != http.StatusInternalServerError { + t.Fatalf("failed discard mark returned %d: %s", response.Code, response.Body.String()) + } + after := store.bindings["account-a"] + if test.commitKnown { + if after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + t.Fatalf("discard rollback changed the old generation: %#v", after) + } + } else if !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("discard commit-unknown lost its retry marker: %#v", after) + } + if len(gateway.containers) != 1 || len(gateway.recorded()) != 0 { + t.Fatalf("failed discard mark touched the gateway: containers=%#v requests=%#v", gateway.containers, gateway.recorded()) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" { + t.Fatalf("discard audit mismatch: %#v", store.actions) + } + + store.cleanupPendingErr = nil + store.cleanupPendingErrAfterMutation = false + response = do(app, http.MethodPost, "/api/browsers/account-a/start", "") + if response.Code != http.StatusConflict { + t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String()) + } + after = store.bindings["account-a"] + if after.RuntimeCleanupPending || after.RuntimeID != "" || len(gateway.containers) != 0 { + t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v", after, gateway.containers) + } + if len(store.actions) != 4 || store.actions[3].Outcome != "failed" || store.actions[3].ReasonCode != "proxy_auth_failed" { + t.Fatalf("discard retry audit mismatch: %#v", store.actions) + } + }) + } +} + +func TestExistingCreateStopsOnGatewayUnknownAndRetriesReuse(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", disconnectList: gatewayReconcileAttempts, containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, BindingVersion: 1, NetworkExitID: "exit-1", + }}} + app := newTestApp(t, store, gateway) + body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}` + + response := do(app, http.MethodPost, "/api/browsers", body) + if response.Code != http.StatusBadGateway { + t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String()) + } + if after := store.bindings["account-a"]; after.RuntimeID != "old-container" || after.RuntimeCleanupPending { + t.Fatalf("gateway unknown changed the existing lease: %#v", after) + } + for _, request := range gateway.recorded() { + if request.method == http.MethodPost { + t.Fatalf("gateway unknown created a duplicate container: %#v", gateway.recorded()) + } + } + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { + t.Fatalf("gateway unknown create audit mismatch: %#v", store.actions) + } + + response = do(app, http.MethodPost, "/api/browsers", body) + if response.Code != http.StatusOK { + t.Fatalf("create retry did not reuse the runtime: %d: %s", response.Code, response.Body.String()) + } + if after := store.bindings["account-a"]; after.RuntimeID != "old-container" || after.RuntimeCleanupPending { + t.Fatalf("create retry changed the matching generation: %#v", after) + } + if len(gateway.containers) != 1 || len(store.actions) != 4 || store.actions[3].Outcome != "succeeded" || store.actions[3].ReasonCode != "environment_reused" { + t.Fatalf("create retry ended inconsistently: containers=%#v actions=%#v", gateway.containers, store.actions) + } +} + +func TestGatewayLookupFailurePreservesLease(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + environment := hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + } + store.bindings[environment.Alias] = environment + store.gatewayFn = func(string) (hub.Gateway, error) { return hub.Gateway{}, errors.New("gateway lookup unavailable") } + + if _, err := restoreOrRebuildRuntime(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }, + environment, containerStatus{ID: "old-container", Alias: environment.Alias, State: "running"}); err == nil { + t.Fatal("restore accepted an unknown gateway") + } + if err := discardRuntime(context.Background(), store, environment); err == nil { + t.Fatal("discard accepted an unknown gateway") + } + if after := store.bindings[environment.Alias]; after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + t.Fatalf("gateway lookup failure changed the old generation: %#v", after) + } +} + +func TestPostgresCleanupPendingTransactionRollbacks(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + + t.Run("pre-mark rollback skips DELETE", func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) + target, err := fixture.store.GetGateway(ctx, "gw-1") + if err != nil { + t.Fatal(err) + } + + removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) + if cleanupErr == nil || removed || len(fixture.gateway.recorded()) != 0 { + t.Fatalf("rolled-back pre-mark touched the gateway: removed=%v err=%v requests=%#v", removed, cleanupErr, fixture.gateway.recorded()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { + t.Fatalf("PostgreSQL rollback did not preserve the active generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + + dropCleanupTransitionFailure(t, ctx, fixture.db) + fixture.gateway.cleanupPending = 2 + removed, cleanupErr = removeGatewayRuntime(ctx, fixture.store, target, after) + if cleanupErr == nil || !removed { + t.Fatalf("202 retry did not retain pending cleanup: removed=%v err=%v", removed, cleanupErr) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 { + t.Fatalf("202 retry was not durable: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + }) + + t.Run("clear rollback remains pending and lifecycle retry recovers", func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + fixture.gateway.deleteNotFound = 1 + installCleanupTransitionFailure(t, ctx, fixture.db, `OLD.runtime_cleanup_pending AND NOT NEW.runtime_cleanup_pending`) + target, err := fixture.store.GetGateway(ctx, "gw-1") + if err != nil { + t.Fatal(err) + } + + removed, cleanupErr := removeGatewayRuntime(ctx, fixture.store, target, fixture.bound) + if cleanupErr == nil || !removed { + t.Fatalf("404 clear rollback lost the delete fact: removed=%v err=%v", removed, cleanupErr) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 { + t.Fatalf("clear rollback was not durably pending: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + + dropCleanupTransitionFailure(t, ctx, fixture.db) + app := fiber.New() + registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { + t.Fatalf("list retry did not confirm cleanup: %d: %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { + t.Fatalf("lifecycle retry did not rebuild runtime: %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + fixture.gateway.mu.Lock() + containers := append([]containerStatus{}, fixture.gateway.containers...) + fixture.gateway.mu.Unlock() + if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { + t.Fatalf("lifecycle retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + } + }) +} + +func TestPostgresCleanupCallChainsPreserveGeneration(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + for _, test := range []struct { + name string + commitUnknown bool + outcome string + reason string + }{ + {name: "rollback", outcome: "failed", reason: "runtime_persistence_failed"}, + {name: "commit unknown", commitUnknown: true, outcome: "unknown", reason: "cleanup_result_unknown"}, + } { + t.Run("restore "+test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion - 1, NetworkExitID: fixture.bound.Exit.ID, + }} + var store hubStore = fixture.store + if test.commitUnknown { + store = cleanupCommitUnknownStore{hubStore: fixture.store} + } else { + installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) + } + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusInternalServerError { + t.Fatalf("restore cleanup failure returned %d: %s", response.Code, response.Body.String()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if test.commitUnknown { + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("restore commit-unknown lost pending state: after=%#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + } else { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + t.Fatalf("restore rollback changed the old generation: after=%#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + } + if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet { + t.Fatalf("restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) + } + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = 'reconcile' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != test.outcome || reason != test.reason { + t.Fatalf("restore audit mismatch: outcome=%s reason=%s", outcome, reason) + } + + if !test.commitUnknown { + dropCleanupTransitionFailure(t, ctx, fixture.db) + } + retryApp := fiber.New() + registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + if response = do(retryApp, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { + t.Fatalf("restore list retry failed: %d: %s", response.Code, response.Body.String()) + } + if test.commitUnknown { + if response = do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { + t.Fatalf("restore start retry failed: %d: %s", response.Code, response.Body.String()) + } + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + fixture.gateway.mu.Lock() + containers := append([]containerStatus{}, fixture.gateway.containers...) + fixture.gateway.mu.Unlock() + if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { + t.Fatalf("restore retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + }) + + t.Run("create restore "+test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion - 1, NetworkExitID: fixture.bound.Exit.ID, + }} + var store hubStore = fixture.store + if test.commitUnknown { + store = cleanupCommitUnknownStore{hubStore: fixture.store} + } else { + installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) + } + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` + + response := do(app, http.MethodPost, "/api/browsers", body) + if response.Code != http.StatusInternalServerError { + t.Fatalf("create restore cleanup failure returned %d: %s", response.Code, response.Body.String()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if test.commitUnknown { + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("create restore commit-unknown lost pending state: after=%#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + } else { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + t.Fatalf("create restore rollback changed the old generation: after=%#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + } + if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 1 || fixture.gateway.recorded()[0].method != http.MethodGet { + t.Fatalf("create restore cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) + } + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = 'create' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != test.outcome || reason != test.reason { + t.Fatalf("create restore audit mismatch: outcome=%s reason=%s", outcome, reason) + } + + if !test.commitUnknown { + dropCleanupTransitionFailure(t, ctx, fixture.db) + } + retryApp := fiber.New() + registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + response = do(retryApp, http.MethodPost, "/api/browsers", body) + if response.Code != http.StatusOK && response.Code != http.StatusCreated { + t.Fatalf("create restore retry failed: %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + fixture.gateway.mu.Lock() + containers := append([]containerStatus{}, fixture.gateway.containers...) + fixture.gateway.mu.Unlock() + if err != nil || after.RuntimeCleanupPending || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { + t.Fatalf("create restore retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + }) + + t.Run("discard "+test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + var store hubStore = fixture.store + if test.commitUnknown { + store = cleanupCommitUnknownStore{hubStore: fixture.store} + } else { + installCleanupTransitionFailure(t, ctx, fixture.db, `NOT OLD.runtime_cleanup_pending AND NEW.runtime_cleanup_pending`) + } + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodPost, "/api/browsers/account-a/start", "") + if response.Code != http.StatusInternalServerError { + t.Fatalf("discard cleanup failure returned %d: %s", response.Code, response.Body.String()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if test.commitUnknown { + if err != nil || !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("discard commit-unknown lost pending state: after=%#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + } else { + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" { + t.Fatalf("discard rollback changed the old generation: after=%#v err=%v", after, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + } + if len(fixture.gateway.containers) != 1 || len(fixture.gateway.recorded()) != 0 { + t.Fatalf("discard cleanup failure touched the container: containers=%#v requests=%#v", fixture.gateway.containers, fixture.gateway.recorded()) + } + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = 'start' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != "unknown" || reason != "cleanup_result_unknown" { + t.Fatalf("discard audit mismatch: outcome=%s reason=%s", outcome, reason) + } + + if !test.commitUnknown { + dropCleanupTransitionFailure(t, ctx, fixture.db) + } + retryApp := fiber.New() + registerHubWithNetwork(retryApp, fixture.store, fakeExitProbe{failure: "proxy_auth_failed"}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + if response = do(retryApp, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusConflict { + t.Fatalf("discard retry returned %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "" || len(fixture.gateway.containers) != 0 { + t.Fatalf("discard retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 0) + }) + } +} + +func TestPostgresGatewayUnknownBlocksListAndCreateUntilRetry(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + for _, test := range []struct { + name string + set func(*fakeGateway, int) + }{ + {name: "disconnect", set: func(gateway *fakeGateway, count int) { gateway.disconnectList = count }}, + {name: "status 500", set: func(gateway *fakeGateway, count int) { gateway.failList = count }}, + {name: "invalid JSON", set: func(gateway *fakeGateway, count int) { gateway.invalidList = count }}, + } { + t.Run("list "+test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + test.set(fixture.gateway, 1) + app := fiber.New() + registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusBadGateway { + t.Fatalf("gateway unknown list returned %d: %s", response.Code, response.Body.String()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { + t.Fatalf("gateway unknown list changed the generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + if response := do(app, http.MethodGet, "/api/browsers", ""); response.Code != http.StatusOK { + t.Fatalf("gateway list retry failed: %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { + t.Fatalf("gateway list retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + }) + + t.Run("create "+test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + test.set(fixture.gateway, gatewayReconcileAttempts) + app := fiber.New() + registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + body := `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` + + if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusBadGateway { + t.Fatalf("gateway unknown create returned %d: %s", response.Code, response.Body.String()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeCleanupPending || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { + t.Fatalf("gateway unknown create changed the generation: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + for _, request := range fixture.gateway.recorded() { + if request.method == http.MethodPost { + t.Fatalf("gateway unknown create sent a duplicate create: %#v", fixture.gateway.recorded()) + } + } + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = 'create' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != "unknown" || reason != "gateway_result_unknown" { + t.Fatalf("gateway unknown create audit mismatch: outcome=%s reason=%s", outcome, reason) + } + + if response := do(app, http.MethodPost, "/api/browsers", body); response.Code != http.StatusOK { + t.Fatalf("gateway create retry failed: %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.RuntimeID != "old-container" || len(fixture.gateway.containers) != 1 { + t.Fatalf("gateway create retry ended inconsistently: after=%#v containers=%#v err=%v", after, fixture.gateway.containers, err) + } + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = 'create' + ORDER BY id DESC LIMIT 1`).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != "succeeded" || reason != "environment_reused" { + t.Fatalf("gateway create retry audit mismatch: outcome=%s reason=%s", outcome, reason) + } + }) + } +} + +func TestPostgresStrictGatewayListBlocksLifecycleUntilRetry(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + for _, test := range []struct { + name, method, path, action, successReason string + configure func(*fakeGateway, int) + active bool + wantStatus int + }{ + {name: "list rejects top-level null", method: http.MethodGet, path: "/api/browsers", active: true, wantStatus: http.StatusOK, + configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `null` }}, + {name: "create rejects null element", method: http.MethodPost, path: "/api/browsers", action: "create", active: true, + wantStatus: http.StatusOK, successReason: "environment_reused", + configure: func(gateway *fakeGateway, count int) { gateway.invalidList, gateway.invalidListBody = count, `[null]` }}, + {name: "start rejects missing fields", method: http.MethodPost, path: "/api/browsers/account-a/start", action: "start", active: true, + wantStatus: http.StatusNoContent, successReason: "gateway_reconciled", + configure: func(gateway *fakeGateway, count int) { + gateway.invalidList, gateway.invalidListBody = count, `[{"alias":"account-a","state":"running"}]` + }}, + {name: "rebind rejects partial body", method: http.MethodPost, path: "/api/browsers/account-a/rebind", action: "rebind", + wantStatus: http.StatusOK, successReason: "environment_rebound", + configure: func(gateway *fakeGateway, count int) { gateway.readErrorList = count }}, + } { + t.Run(test.name, func(t *testing.T) { + fixture := newPostgresRebindFixture(t, databaseURL) + if test.active { + var err error + fixture.bound, err = fixture.store.ActivateRuntime(ctx, "account-a", "old-container", fixture.bound.BindingVersion, fixture.bound.Exit.ID) + if err != nil { + t.Fatal(err) + } + } + fixture.gateway.containers = []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", ProxyReady: true, + BindingVersion: fixture.bound.BindingVersion, NetworkExitID: fixture.bound.Exit.ID, + }} + attempts := gatewayReconcileAttempts + if test.action == "" { + attempts = 1 + } + test.configure(fixture.gateway, attempts) + app := fiber.New() + registerHubWithNetwork(app, fixture.store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + body := "" + if test.action == "create" { + body = `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"` + fixture.exit.ID + `"}` + } else if test.action == "rebind" { + body = `{"network_exit_id":"` + fixture.exit.ID + `"}` + } + + response := do(app, test.method, test.path, body) + if response.Code != http.StatusBadGateway { + t.Fatalf("strict list failure returned %d: %s", response.Code, response.Body.String()) + } + after, err := fixture.store.GetEnvironmentContext(ctx, "account-a") + if err != nil || after.BindingVersion != fixture.bound.BindingVersion || after.Exit.ID != fixture.bound.Exit.ID || + after.RuntimeCleanupPending || after.RuntimeID != fixture.bound.RuntimeID || len(fixture.gateway.containers) != 1 { + t.Fatalf("strict list failure changed the generation: before=%#v after=%#v containers=%#v err=%v", + fixture.bound, after, fixture.gateway.containers, err) + } + for _, request := range fixture.gateway.recorded() { + if request.method != http.MethodGet { + t.Fatalf("strict list failure advanced the lifecycle: %#v", fixture.gateway.recorded()) + } + } + activeCount := 0 + if test.active { + activeCount = 1 + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, activeCount) + if test.action != "" { + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = $1 + ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != "unknown" || reason != "gateway_result_unknown" { + t.Fatalf("strict list audit mismatch: outcome=%s reason=%s", outcome, reason) + } + } + + response = do(app, test.method, test.path, body) + if response.Code != test.wantStatus { + t.Fatalf("strict list retry returned %d: %s", response.Code, response.Body.String()) + } + after, err = fixture.store.GetEnvironmentContext(ctx, "account-a") + fixture.gateway.mu.Lock() + containers := append([]containerStatus{}, fixture.gateway.containers...) + fixture.gateway.mu.Unlock() + wantVersion := fixture.bound.BindingVersion + if test.action == "rebind" { + wantVersion++ + } + if err != nil || after.BindingVersion != wantVersion || after.RuntimeID == "" || len(containers) != 1 || !containerMatchesBinding(containers[0], after) { + t.Fatalf("strict list retry ended inconsistently: after=%#v containers=%#v err=%v", after, containers, err) + } + assertControlPlaneDatabaseCount(t, fixture.db, `SELECT count(*) FROM runtime_instance WHERE binding_id = 'account-a' AND released_at IS NULL`, 1) + if test.action != "" { + var outcome, reason string + if err := fixture.db.QueryRowContext(ctx, ` + SELECT outcome, reason_code FROM audit_event WHERE action = $1 + ORDER BY id DESC LIMIT 1`, test.action).Scan(&outcome, &reason); err != nil { + t.Fatal(err) + } + if outcome != "succeeded" || reason != test.successReason { + t.Fatalf("strict list retry audit mismatch: outcome=%s reason=%s", outcome, reason) + } + } + }) + } +} + +func isolatedControlPlaneDatabaseURL(t *testing.T, databaseURL string) string { + t.Helper() + admin, err := sql.Open("pgx", databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = admin.Close() }) + schema := fmt.Sprintf("creatorhub_hh803_%d", time.Now().UnixNano()) + if _, err := admin.Exec("CREATE SCHEMA " + schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec("DROP SCHEMA " + schema + " CASCADE"); err != nil { + t.Errorf("drop test schema: %v", err) + } + }) + parsed, err := url.Parse(databaseURL) + if err != nil { + t.Fatal(err) + } + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String() +} + +func assertControlPlaneDatabaseCount(t *testing.T, db *sql.DB, query string, want int) { + t.Helper() + var got int + if err := db.QueryRow(query).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("query %q returned %d, want %d", query, got, want) + } +} + +func TestUpgradeBrowserStopsBeforeCreateWhenPersistenceFails(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} store.upgradeErr = hub.ErrNotFound @@ -506,8 +2351,8 @@ func TestUpgradeBrowserRemovesUncommittedContainer(t *testing.T) { t.Fatalf("expected persistence failure, got %d: %s", response.Code, response.Body.String()) } requests := gateway.recorded() - if len(requests) != 3 || requests[2].method != http.MethodDelete || requests[2].path != "/v1/browsers/account-a" { - t.Fatalf("uncommitted upgraded container must be removed: %#v", requests) + if len(requests) != 1 || requests[0].method != http.MethodDelete || requests[0].path != "/v1/browsers/account-a" { + t.Fatalf("failed persistence must abort before creating the upgraded container: %#v", requests) } env, err := store.GetEnv(context.Background(), "account-a") if err != nil || env.ImageVersion != "148" { @@ -515,6 +2360,47 @@ func TestUpgradeBrowserRemovesUncommittedContainer(t *testing.T) { } } +func TestUpgradeStopsBeforeCreateWhenRuntimeReleaseFails(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} + store.releaseErr = errors.New("database unavailable") + _ = store.CreateImage(nil, hub.Image{Version: "144.0.7559.132", ImageRef: "registry.example/browser:144", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token"} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"144.0.7559.132"}`) + if response.Code != http.StatusInternalServerError { + t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String()) + } + requests := gateway.recorded() + if len(requests) != 0 { + t.Fatalf("release failure must abort before delete: %#v", requests) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "cleanup_result_unknown" { + t.Fatalf("release failure must be audited: %#v", store.actions) + } +} + +func TestUpgradeRejectsInvalidVersionWithoutAuditingRawInput(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 2024}} + gateway := &fakeGateway{token: "unit-test-gateway-token"} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/upgrade", `{"version":"http://operator:secret@proxy.example"}`) + if response.Code != http.StatusBadRequest { + t.Fatalf("expected invalid version rejection, got %d: %s", response.Code, response.Body.String()) + } + if len(gateway.recorded()) != 0 || len(store.actions) != 2 { + t.Fatalf("invalid version must be rejected before gateway side effects: requests=%#v actions=%#v", gateway.recorded(), store.actions) + } + for _, action := range store.actions { + if action.NewImageVersion != "" || action.ReasonCode != "upgrade_input_rejected" { + t.Fatalf("raw invalid version reached audit: %#v", store.actions) + } + } +} + func TestImageDisableWaitsForUpgradeCommit(t *testing.T) { releaseCreate := make(chan struct{}) gateway := &fakeGateway{ @@ -537,7 +2423,7 @@ func TestImageDisableWaitsForUpgradeCommit(t *testing.T) { } return c.Next() }) - registerHub(app, store) + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) server := httptest.NewServer(adaptor.FiberApp(app)) defer server.Close() @@ -601,21 +2487,656 @@ func TestImageDisableWaitsForUpgradeCommit(t *testing.T) { } } +func TestListAndHeartbeatWaitForUpgradeCoordination(t *testing.T) { + releaseCreate := make(chan struct{}) + gateway := &fakeGateway{ + token: "unit-test-gateway-token", + createStarted: make(chan struct{}), + releaseCreate: releaseCreate, + } + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + _ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}) + gatewayServer := httptest.NewServer(gateway.handler(t)) + defer gatewayServer.Close() + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: gateway.token} + listArrived := make(chan struct{}) + var listOnce sync.Once + app := fiber.New() + app.Use(func(c fiber.Ctx) error { + if c.Method() == http.MethodGet && c.Path() == "/api/browsers" { + listOnce.Do(func() { close(listArrived) }) + } + return c.Next() + }) + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + server := httptest.NewServer(adaptor.FiberApp(app)) + defer server.Close() + + type result struct { + status int + err error + } + upgradeDone := make(chan result, 1) + go func() { + response, err := server.Client().Post(server.URL+"/api/browsers/account-a/upgrade", "application/json", strings.NewReader(`{"version":"149"}`)) + if err != nil { + upgradeDone <- result{err: err} + return + } + defer response.Body.Close() + upgradeDone <- result{status: response.StatusCode} + }() + select { + case <-gateway.createStarted: + case <-time.After(time.Second): + t.Fatal("upgrade did not reach gateway create") + } + + listDone := make(chan result, 1) + go func() { + response, err := server.Client().Get(server.URL + "/api/browsers") + if err != nil { + listDone <- result{err: err} + return + } + defer response.Body.Close() + listDone <- result{status: response.StatusCode} + }() + <-listArrived + heartbeatDone := make(chan error, 1) + heartbeatStarted := make(chan struct{}) + go func() { + close(heartbeatStarted) + heartbeatDone <- reconcileRuntimeLeases(context.Background(), store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + }() + <-heartbeatStarted + select { + case result := <-listDone: + t.Fatalf("list reconciled a stale snapshot during upgrade: %#v", result) + case err := <-heartbeatDone: + t.Fatalf("heartbeat reconciled a stale snapshot during upgrade: %v", err) + case <-time.After(50 * time.Millisecond): + } + + close(releaseCreate) + if result := <-upgradeDone; result.err != nil || result.status != http.StatusNoContent { + t.Fatalf("upgrade failed: %#v", result) + } + if result := <-listDone; result.err != nil || result.status != http.StatusOK { + t.Fatalf("coordinated list failed: %#v", result) + } + if err := <-heartbeatDone; err != nil { + t.Fatalf("coordinated heartbeat failed: %v", err) + } +} + func TestBrowserActionRoutesStartStopAndRejectsUnknown(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) gateway := &fakeGateway{token: "unit-test-gateway-token"} app := newTestApp(t, store, gateway) if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { t.Fatalf("expected 204 for start, got %d: %s", response.Code, response.Body.String()) } + if response := do(app, http.MethodPost, "/api/browsers/account-a/stop", ""); response.Code != http.StatusNoContent { + t.Fatalf("expected 204 for stop, got %d: %s", response.Code, response.Body.String()) + } if response := do(app, http.MethodPost, "/api/browsers/account-a/pause", ""); response.Code != http.StatusBadRequest { t.Fatalf("expected 400 for unknown action, got %d", response.Code) } + if len(store.actions) != 4 || store.actions[0].Action != "start" || store.actions[1].Outcome != "succeeded" || + store.actions[2].Action != "stop" || store.actions[3].Outcome != "succeeded" { + t.Fatalf("start and stop must each emit an audit pair: %#v", store.actions) + } } -func TestDeleteBrowserKeepsEnvAndContainerInSync(t *testing.T) { +func TestStoppedReconcileReportsRuntimeReleaseFailure(t *testing.T) { + gatewayServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodGet { + _ = json.NewEncoder(response).Encode([]containerStatus{{ID: "container-id", Alias: "account-a", State: "exited"}}) + return + } + connection, _, err := response.(http.Hijacker).Hijack() + if err == nil { + _ = connection.Close() + } + })) + defer gatewayServer.Close() + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id"} + store.releaseErr = errors.New("database unavailable") + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodPost, "/api/browsers/account-a/stop", "") + if response.Code != http.StatusInternalServerError { + t.Fatalf("expected release failure, got %d: %s", response.Code, response.Body.String()) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_release_failed" { + t.Fatalf("stopped reconcile release failure must be audited: %#v", store.actions) + } +} + +func TestRebindRebuildsRunningContainerWithLatestBinding(t *testing.T) { + store := newMemoryStore() + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + } + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) + if response.Code != http.StatusOK { + t.Fatalf("running runtime rebind failed: %d: %s", response.Code, response.Body.String()) + } + bound := store.bindings["account-a"] + if bound.Exit.ID != "exit-2" || bound.BindingVersion != 2 || bound.RuntimeID != "container-id" { + t.Fatalf("running runtime was not rebuilt on the latest binding: %#v", bound) + } + requests := gateway.recorded() + if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete || requests[2].method != http.MethodPost { + t.Fatalf("running rebind must inspect, delete and recreate: %#v", requests) + } + if requests[2].body["binding_version"] != float64(2) || requests[2].body["network_exit_id"] != "exit-2" { + t.Fatalf("recreated runtime did not carry the latest binding CAS: %#v", requests[2].body) + } +} + +func TestSameExitRebindStillRebuildsRunningContainer(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "expired-runtime", RuntimeID: "old-container", + } + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-1"}`) + if response.Code != http.StatusOK { + t.Fatalf("same-exit rebind failed: %d: %s", response.Code, response.Body.String()) + } + bound := store.bindings["account-a"] + if bound.BindingVersion != 2 || bound.RuntimeID != "container-id" { + t.Fatalf("same-exit rebind did not rotate the binding generation and lease: %#v", bound) + } + requests := gateway.recorded() + if len(requests) != 3 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete || requests[2].method != http.MethodPost { + t.Fatalf("same-exit rebind bypassed gateway reconciliation: %#v", requests) + } + if requests[2].body["binding_version"] != float64(2) || requests[2].body["network_exit_id"] != "exit-1" { + t.Fatalf("same-exit rebuild used stale metadata: %#v", requests[2].body) + } +} + +func TestRuntimeReuseRechecksHealthAndDiscardsFailedExit(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{failure: "exit_auth_failed"}, + func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("unhealthy runtime reconciliation failed: %d: %s", response.Code, response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { + t.Fatalf("failed exit remained active: %q", runtime) + } + requests := gateway.recorded() + if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete { + t.Fatalf("failed exit must be rechecked and removed instead of renewed: %#v", requests) + } +} + +func TestReconcileDeleteFailureReleasesLeaseAndAuditsUnknown(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 2, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "stale-container", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{ + ID: "stale-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusBadGateway { + t.Fatalf("delete failure must remain unknown, got %d: %s", response.Code, response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { + t.Fatalf("delete failure retained DB runtime lease: %q", runtime) + } + if len(store.actions) != 2 || store.actions[0].Action != "reconcile" || store.actions[1].Outcome != "unknown" || + store.actions[1].ReasonCode != "gateway_result_unknown" { + t.Fatalf("delete failure was not audited as retryable unknown: %#v", store.actions) + } + if len(gateway.containers) != 1 { + t.Fatal("failed delete unexpectedly removed the gateway container") + } +} + +func TestDisableExitImmediatelyDiscardsRuntime(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "") + if response.Code != http.StatusOK { + t.Fatalf("disable cleanup failed: %d: %s", response.Code, response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { + t.Fatalf("disabled exit remained active: %q", runtime) + } + requests := gateway.recorded() + if len(requests) != 2 || requests[0].method != http.MethodGet || requests[1].method != http.MethodDelete { + t.Fatalf("disable must reconcile and remove the runtime: %#v", requests) + } +} + +func TestDisableExitPropagatesUnknownGatewayReadAndPreservesLease(t *testing.T) { + for _, test := range []struct { + name string + handler http.HandlerFunc + }{ + {name: "disconnect", handler: func(response http.ResponseWriter, _ *http.Request) { + connection, _, err := response.(http.Hijacker).Hijack() + if err == nil { + _ = connection.Close() + } + }}, + {name: "status 500", handler: func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(`{"error":"docker unavailable"}`)) + }}, + {name: "invalid JSON", handler: func(response http.ResponseWriter, _ *http.Request) { + _, _ = response.Write([]byte(`{"not":"a browser list"}`)) + }}, + } { + t.Run(test.name, func(t *testing.T) { + gatewayServer := httptest.NewServer(test.handler) + defer gatewayServer.Close() + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + } + store.gateways["gw-1"] = hub.Gateway{Name: "gw-1", Endpoint: gatewayServer.URL, Token: "unit-test-gateway-token"} + app := fiber.New() + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) + + response := do(app, http.MethodPost, "/api/network-exits/exit-1/disable", "") + if response.Code != http.StatusBadGateway { + t.Fatalf("gateway unknown must not return 200: %d: %s", response.Code, response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "container-id" { + t.Fatalf("gateway unknown changed the unconfirmed runtime lease: %q", runtime) + } + if store.exits["exit-1"].HealthStatus != "disabled" { + t.Fatal("disable state was not retained for retryable reconciliation") + } + }) + } +} + +func TestLegacyNullBindingIsListableAndExplicitlyRecoverable(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{ + Seed: 1, ProxyServer: "http://legacy:secret@proxy.example:8080", DisableNonProxiedUDP: true, + }} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + RuntimeInstanceID: "legacy-runtime", RuntimeID: "legacy-container", + } + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "legacy-container", Alias: "account-a", State: "running", + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodGet, "/api/browsers", "") + if response.Code != http.StatusOK { + t.Fatalf("legacy NULL binding broke browser listing: %d: %s", response.Code, response.Body.String()) + } + var views []envView + if err := json.Unmarshal(response.Body.Bytes(), &views); err != nil || len(views) != 1 || !views[0].RecoveryRequired || views[0].NetworkExitID != "" { + t.Fatalf("legacy recovery state was not visible: %#v err=%v", views, err) + } + + response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-1"}`) + if response.Code != http.StatusOK { + t.Fatalf("legacy explicit recovery failed: %d: %s", response.Code, response.Body.String()) + } + requests := gateway.recorded() + if len(requests) != 4 || requests[2].method != http.MethodDelete || requests[3].method != http.MethodPost { + t.Fatalf("legacy recovery must inspect, remove, then recreate: %#v", requests) + } + encoded, _ := json.Marshal(requests[3].body) + if strings.Contains(string(encoded), "legacy") || strings.Contains(string(encoded), "secret") { + t.Fatalf("legacy Config.Cmd credentials reached the recovered runtime: %s", encoded) + } +} + +func TestRebindDeleteFailureKeepsOriginalBinding(t *testing.T) { + for _, test := range []struct { + name string + binding hub.EnvironmentContext + targetID string + }{ + {name: "running binding", binding: hub.EnvironmentContext{ + AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: hub.NetworkExit{ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy"}, + RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + }, targetID: "exit-2"}, + {name: "legacy NULL binding", binding: hub.EnvironmentContext{ + AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + RuntimeInstanceID: "legacy-runtime", RuntimeID: "legacy-container", + }, targetID: "exit-1"}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + before := test.binding + before.Env = store.envs["account-a"] + store.bindings["account-a"] = before + gateway := &fakeGateway{token: "unit-test-gateway-token", failDelete: 1, containers: []containerStatus{{ + ID: before.RuntimeID, Alias: "account-a", State: "running", BindingVersion: before.BindingVersion, NetworkExitID: before.Exit.ID, ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"`+test.targetID+`"}`) + if response.Code != http.StatusBadGateway { + t.Fatalf("delete failure must remain unknown: %d: %s", response.Code, response.Body.String()) + } + after := store.bindings["account-a"] + if after.BindingVersion != before.BindingVersion || after.Exit.ID != before.Exit.ID || !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("binding committed before old container deletion: before=%#v after=%#v", before, after) + } + if len(gateway.containers) != 1 { + t.Fatalf("non-final delete result lost the existing container: %#v", gateway.containers) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { + t.Fatalf("delete failure was not audited as unknown: %#v", store.actions) + } + }) + } +} + +func TestRebindNetworkCleanupPendingBlocksUntilConfirmed(t *testing.T) { + for _, test := range []struct { + name string + cleanupPending int + disconnectDelete int + disconnectListAfterDelete bool + }{ + {name: "cleanup pending", cleanupPending: 2}, + {name: "disconnect after accepted", cleanupPending: 1, disconnectDelete: 1, disconnectListAfterDelete: true}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container", + } + gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: test.cleanupPending, + disconnectDelete: test.disconnectDelete, disconnectListAfterDelete: test.disconnectListAfterDelete, + containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) + if response.Code != http.StatusBadGateway { + t.Fatalf("network cleanup uncertainty returned %d: %s", response.Code, response.Body.String()) + } + after := store.bindings["account-a"] + if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || !after.RuntimeCleanupPending || after.RuntimeID != "" { + t.Fatalf("network cleanup uncertainty changed the binding: %#v", after) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "unknown" || store.actions[1].ReasonCode != "gateway_result_unknown" { + t.Fatalf("network cleanup uncertainty audit mismatch: %#v", store.actions) + } + if len(gateway.containers) != 0 { + t.Fatalf("container-removed fact was lost: %#v", gateway.containers) + } + + response = do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) + if response.Code != http.StatusOK { + t.Fatalf("confirmed cleanup did not allow retry: %d: %s", response.Code, response.Body.String()) + } + after = store.bindings["account-a"] + if after.RuntimeCleanupPending || after.BindingVersion != 2 || after.Exit.ID != "exit-2" || after.RuntimeID != "" { + t.Fatalf("retry did not commit a clean stopped generation: %#v", after) + } + }) + } +} + +func TestCleanupPendingBlocksEveryLifecyclePath(t *testing.T) { + for _, test := range []struct { + name, method, path, body string + }{ + {name: "list reconcile", method: http.MethodGet, path: "/api/browsers"}, + {name: "create reuse", method: http.MethodPost, path: "/api/browsers", body: `{"alias":"account-a","name":"甲","gateway":"gw-1","image_version":"148","fingerprint":{"seed":1},"account_id":"account-a","network_exit_id":"exit-1"}`}, + {name: "start", method: http.MethodPost, path: "/api/browsers/account-a/start"}, + {name: "upgrade", method: http.MethodPost, path: "/api/browsers/account-a/upgrade", body: `{"version":"149"}`}, + {name: "rebind", method: http.MethodPost, path: "/api/browsers/account-a/rebind", body: `{"network_exit_id":"exit-1"}`}, + {name: "recycle", method: http.MethodDelete, path: "/api/browsers/account-a"}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + RuntimeCleanupPending: true, Exit: store.exits["exit-1"], + } + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + _ = store.CreateImage(nil, hub.Image{Version: "149", ImageRef: "registry.example/browser:149", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", cleanupPending: 2} + app := newTestApp(t, store, gateway) + + response := do(app, test.method, test.path, test.body) + if response.Code != http.StatusBadGateway { + t.Fatalf("pending cleanup returned %d: %s", response.Code, response.Body.String()) + } + after := store.bindings["account-a"] + if !after.RuntimeCleanupPending || after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "" { + t.Fatalf("lifecycle path advanced a pending generation: %#v", after) + } + for _, request := range gateway.recorded() { + if request.method == http.MethodPost && request.path == "/v1/browsers" { + t.Fatalf("lifecycle path created before cleanup confirmation: %#v", gateway.recorded()) + } + } + }) + } +} + +func TestRebindPreparesRunningRuntimeBeforeDelete(t *testing.T) { + for _, test := range []struct { + name string + withImage bool + credential bool + }{ + {name: "image unavailable"}, + {name: "credential unavailable", withImage: true, credential: true}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + if test.withImage { + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + } + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} + if test.credential { + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1, + CredentialReference: &hub.CredentialReference{ID: "credential-exit", Provider: "os_keyring"}} + } + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", + BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container"} + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestAppWithNetwork(t, store, gateway, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { + return "", errors.New("credential unavailable") + }) + + response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) + if response.Code < 400 || len(gateway.recorded()) != 1 || gateway.recorded()[0].method != http.MethodGet { + t.Fatalf("runtime preparation failure touched the old container: status=%d requests=%#v", response.Code, gateway.recorded()) + } + if after := store.bindings["account-a"]; after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID != "old-container" { + t.Fatalf("runtime preparation failure changed state: %#v", after) + } + }) + } +} + +func TestRebindCandidateCreateFailureRestoresOldRuntime(t *testing.T) { + store := newMemoryStore() + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy.example", Port: 8080, HealthStatus: "healthy", Version: 1} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", + BindingVersion: 1, Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "old-container"} + gateway := &fakeGateway{token: "unit-test-gateway-token", failCreate: 1, containers: []containerStatus{{ + ID: "old-container", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", ProxyReady: true, + }}} + app := newTestApp(t, store, gateway) + + response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`) + if response.Code != http.StatusConflict { + t.Fatalf("candidate create failure returned %d: %s", response.Code, response.Body.String()) + } + after := store.bindings["account-a"] + if after.BindingVersion != 1 || after.Exit.ID != "exit-1" || after.RuntimeID == "" || + len(gateway.containers) != 1 || !containerMatchesBinding(gateway.containers[0], after) { + t.Fatalf("candidate create failure did not restore old runtime: after=%#v containers=%#v", after, gateway.containers) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "gateway_create_failed" { + t.Fatalf("candidate create failure audit mismatch: %#v", store.actions) + } +} + +func TestExistingEnvironmentCleanupNeverReturnsReusedSuccess(t *testing.T) { + for _, test := range []struct { + name string + probe *sequenceExitProbe + resolve func(hub.NetworkExitAccess) (string, error) + }{ + {name: "second probe fails", probe: &sequenceExitProbe{failures: []string{"", "exit_auth_failed"}}, + resolve: func(hub.NetworkExitAccess) (string, error) { return "username:password", nil }}, + {name: "second credential restore fails", probe: &sequenceExitProbe{}, resolve: func() func(hub.NetworkExitAccess) (string, error) { + calls := 0 + return func(hub.NetworkExitAccess) (string, error) { + calls++ + if calls == 2 { + return "", errors.New("credential unavailable") + } + return "username:password", nil + } + }()}, + } { + t.Run(test.name, func(t *testing.T) { + store := newMemoryStore() + store.exits["exit-1"] = hub.NetworkExit{ + ID: "exit-1", Protocol: "socks5", Host: "127.0.0.1", Port: 1080, HealthStatus: "healthy", Version: 1, + CredentialReference: &hub.CredentialReference{ID: "credential-exit", Provider: "os_keyring"}, + } + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "店铺一号", Gateway: "gw-1", ImageVersion: "148.0.7778.215", Fingerprint: hub.Fingerprint{Seed: 2024, Platform: "windows", Timezone: "Asia/Shanghai"}} + store.bindings["account-a"] = hub.EnvironmentContext{ + Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, + Exit: store.exits["exit-1"], RuntimeInstanceID: "runtime-instance", RuntimeID: "container-id", + } + _ = store.CreateImage(nil, hub.Image{Version: "148.0.7778.215", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ + ID: "container-id", Alias: "account-a", State: "running", BindingVersion: 1, NetworkExitID: "exit-1", + }}} + app := newTestAppWithNetwork(t, store, gateway, test.probe, test.resolve) + + response := do(app, http.MethodPost, "/api/browsers", createEnvBody) + if response.Code != http.StatusConflict { + t.Fatalf("cleaned runtime must not return reused success: %d: %s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "not running") { + t.Fatalf("cleaned runtime response did not state the stopped result: %s", response.Body.String()) + } + if runtime := store.bindings["account-a"].RuntimeID; runtime != "" { + t.Fatalf("failed recovery retained runtime: %q probe_calls=%d actions=%#v requests=%#v", runtime, test.probe.calls, store.actions, gateway.recorded()) + } + if len(store.actions) != 2 || store.actions[1].Outcome != "failed" || store.actions[1].ReasonCode != "runtime_unavailable" { + t.Fatalf("cleaned runtime was audited as reused success: %#v", store.actions) + } + }) + } +} + +func TestStartRebuildsStoppedContainerAfterRebind(t *testing.T) { + store := newMemoryStore() + store.exits["exit-2"] = hub.NetworkExit{ID: "exit-2", Protocol: "http", Host: "proxy-2.example", Port: 8080, HealthStatus: "healthy", Version: 1} + store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} + store.bindings["account-a"] = hub.EnvironmentContext{Env: store.envs["account-a"], AccountID: "account-a", BindingID: "account-a", BindingVersion: 1, Exit: store.exits["exit-1"]} + _ = store.CreateImage(nil, hub.Image{Version: "148", ImageRef: "registry.example/browser:148", Enabled: true}) + gateway := &fakeGateway{token: "unit-test-gateway-token", containers: []containerStatus{{ID: "old-container", Alias: "account-a", State: "exited"}}} + app := newTestApp(t, store, gateway) + + if response := do(app, http.MethodPost, "/api/browsers/account-a/rebind", `{"network_exit_id":"exit-2"}`); response.Code != http.StatusOK { + t.Fatalf("rebind failed: %d %s", response.Code, response.Body.String()) + } + if response := do(app, http.MethodPost, "/api/browsers/account-a/start", ""); response.Code != http.StatusNoContent { + t.Fatalf("start failed: %d %s", response.Code, response.Body.String()) + } + requests := gateway.recorded() + var createRequest recordedRequest + for _, request := range requests { + if request.method == http.MethodPost && request.path == "/v1/browsers" { + createRequest = request + } + } + if createRequest.body == nil { + t.Fatalf("stopped container was not recreated: %#v", requests) + } + exit := createRequest.body["network_exit"].(map[string]any) + if exit["host"] != "proxy-2.example" { + t.Fatalf("recreated container did not use rebound exit: %#v", createRequest.body) + } +} + +func TestRecycleBrowserKeepsStableEnvironment(t *testing.T) { store := newMemoryStore() store.envs["account-a"] = hub.Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: hub.Fingerprint{Seed: 1}} gateway := &fakeGateway{token: "unit-test-gateway-token"} @@ -625,8 +3146,11 @@ func TestDeleteBrowserKeepsEnvAndContainerInSync(t *testing.T) { t.Fatalf("expected 204, got %d: %s", response.Code, response.Body.String()) } _, kept := store.envs["account-a"] - if len(store.deleted) != 1 || kept { - t.Fatalf("env must be removed with the container, deleted=%v", store.deleted) + if !kept { + t.Fatal("recycle must preserve the stable environment and Profile anchor") + } + if len(store.actions) != 2 || store.actions[0].Action != "recycle" || store.actions[1].Outcome != "succeeded" { + t.Fatalf("recycle must emit an audit pair: %#v", store.actions) } } @@ -648,10 +3172,35 @@ func TestGatewayAndImageCRUDRoutes(t *testing.T) { } } +func TestNetworkExitRoutesAreStrictAndSecretFree(t *testing.T) { + store := newMemoryStore() + gateway := &fakeGateway{token: "unit-test-gateway-token"} + app := newTestApp(t, store, gateway) + + invalid := do(app, http.MethodPost, "/api/network-exits", + `{"protocol":"socks5","host":"proxy.example","port":1080,"credential_reference":{"id":"credential-a","key":"raw-value"}}`) + if invalid.Code != http.StatusBadRequest || len(store.exits) != 1 { + t.Fatalf("raw credential fields must be rejected before persistence: status=%d exits=%#v", invalid.Code, store.exits) + } + created := do(app, http.MethodPost, "/api/network-exits", + `{"protocol":"socks5","host":"proxy.example","port":1080,"credential_reference":{"id":"credential-a"},"expected_public_ip":"203.0.113.1","expected_region":"test"}`) + if created.Code != http.StatusCreated || strings.Contains(created.Body.String(), "raw-value") { + t.Fatalf("unexpected secret-bearing network exit response: status=%d body=%s", created.Code, created.Body.String()) + } + checked := do(app, http.MethodPost, "/api/network-exits/exit-created/check", "") + if checked.Code != http.StatusOK || !strings.Contains(checked.Body.String(), `"health_status":"healthy"`) { + t.Fatalf("network exit check did not record health: status=%d body=%s", checked.Code, checked.Body.String()) + } + disabled := do(app, http.MethodPost, "/api/network-exits/exit-created/disable", "") + if disabled.Code != http.StatusOK || !strings.Contains(disabled.Body.String(), `"health_status":"disabled"`) { + t.Fatalf("network exit disable failed: status=%d body=%s", disabled.Code, disabled.Body.String()) + } +} + func TestCreateImageReturnsJSONOverHTTP(t *testing.T) { store := newMemoryStore() app := fiber.New() - registerHub(app, store) + registerHubWithNetwork(app, store, fakeExitProbe{}, func(hub.NetworkExitAccess) (string, error) { return "", nil }) server := httptest.NewServer(adaptor.FiberApp(app)) defer server.Close() diff --git a/cmd/control-plane/main.go b/cmd/control-plane/main.go index ae37d7c..f931284 100644 --- a/cmd/control-plane/main.go +++ b/cmd/control-plane/main.go @@ -66,15 +66,39 @@ func newCommand() *cobra.Command { } defer hubStore.Close() logStartup(cfg) - return newHandlerWithStores(cfg.webDir, phaseAStore, hubStore).Listen(cfg.listenAddr, fiber.ListenConfig{ + heartbeatContext, stopHeartbeat := context.WithCancel(command.Context()) + heartbeatDone := make(chan struct{}) + go func() { + defer close(heartbeatDone) + runtimeLeaseHeartbeat(heartbeatContext, hubStore) + }() + listenErr := newHandlerWithStores(cfg.webDir, phaseAStore, hubStore).Listen(cfg.listenAddr, fiber.ListenConfig{ GracefulContext: command.Context(), DisableStartupMessage: true, }) + stopHeartbeat() + <-heartbeatDone + return listenErr }, } return command } +func runtimeLeaseHeartbeat(ctx context.Context, store hubStore) { + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := reconcileRuntimeLeases(ctx, store, defaultNetworkExitProbe(), resolveExitCredential); err != nil && ctx.Err() == nil { + logrus.WithField("service", "control-plane").WithError(err).Warn("runtime lease reconciliation failed") + } + } + } +} + func logStartup(cfg config) { logrus.WithFields(logrus.Fields{ "service": "control-plane", diff --git a/cmd/control-plane/network.go b/cmd/control-plane/network.go new file mode 100644 index 0000000..7a8402c --- /dev/null +++ b/cmd/control-plane/network.go @@ -0,0 +1,175 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +const networkExitObservationURL = "https://ipinfo.io/json" + +type networkExitProbe interface { + Check(context.Context, hub.NetworkExitAccess) (hub.ExitObservation, string) +} + +type httpNetworkExitProbe struct { + endpoint string + client *http.Client + resolve func(hub.NetworkExitAccess) (string, error) +} + +func defaultNetworkExitProbe() networkExitProbe { + return httpNetworkExitProbe{endpoint: networkExitObservationURL, client: &http.Client{Timeout: 20 * time.Second}, resolve: resolveExitCredential} +} + +func (probe httpNetworkExitProbe) Check(ctx context.Context, exit hub.NetworkExitAccess) (hub.ExitObservation, string) { + proxyURL := &url.URL{Scheme: exit.Protocol, Host: net.JoinHostPort(exit.Host, fmt.Sprint(exit.Port))} + proxyUsername := "" + if exit.CredentialReference != nil { + secret, err := probe.resolve(exit) + if err != nil { + return hub.ExitObservation{}, "credential_unavailable" + } + username, password, found := strings.Cut(secret, ":") + if !found || username == "" { + return hub.ExitObservation{}, "credential_invalid" + } + proxyUsername = username + proxyURL.User = url.UserPassword(username, password) + } + transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)} + if exit.Protocol == "socks4" { + transport.Proxy = nil + transport.DialContext = socks4DialContext(proxyURL.Host, proxyUsername) + } + defer transport.CloseIdleConnections() + client := *probe.client + client.Transport = transport + request, err := http.NewRequestWithContext(ctx, http.MethodGet, probe.endpoint, nil) + if err != nil { + return hub.ExitObservation{}, "proxy_check_failed" + } + response, err := client.Do(request) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "auth") { + return hub.ExitObservation{}, "proxy_auth_failed" + } + return hub.ExitObservation{}, "proxy_check_failed" + } + defer response.Body.Close() + if response.StatusCode == http.StatusProxyAuthRequired { + return hub.ExitObservation{}, "proxy_auth_failed" + } + if response.StatusCode != http.StatusOK { + return hub.ExitObservation{}, "proxy_check_failed" + } + var observed struct { + IP string `json:"ip"` + Region string `json:"region"` + } + decoder := json.NewDecoder(io.LimitReader(response.Body, 64<<10)) + if err := decoder.Decode(&observed); err != nil || net.ParseIP(observed.IP) == nil || len(observed.Region) > 64 { + return hub.ExitObservation{}, "exit_observation_invalid" + } + return hub.ExitObservation{PublicIP: observed.IP, Region: observed.Region}, "" +} + +func socks4DialContext(proxyAddress, userID string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + connection, err := (&net.Dialer{}).DialContext(ctx, network, proxyAddress) + if err != nil { + return nil, err + } + failed := true + defer func() { + if failed { + _ = connection.Close() + } + }() + host, portText, err := net.SplitHostPort(address) + if err != nil { + return nil, errors.New("invalid SOCKS4 destination") + } + port, err := net.LookupPort("tcp", portText) + if err != nil { + return nil, errors.New("invalid SOCKS4 destination port") + } + request := []byte{4, 1, 0, 0, 0, 0, 0, 1} + binary.BigEndian.PutUint16(request[2:4], uint16(port)) + if ip := net.ParseIP(host).To4(); ip != nil { + copy(request[4:8], ip) + } + request = append(request, userID...) + request = append(request, 0) + if net.ParseIP(host).To4() == nil { + request = append(request, host...) + request = append(request, 0) + } + if deadline, ok := ctx.Deadline(); ok { + _ = connection.SetDeadline(deadline) + } + if _, err := connection.Write(request); err != nil { + return nil, err + } + response := make([]byte, 8) + if _, err := io.ReadFull(connection, response); err != nil || response[1] != 90 { + return nil, errors.New("SOCKS4 proxy rejected connection") + } + _ = connection.SetDeadline(time.Time{}) + failed = false + return connection, nil + } +} + +// Secret managers and keyring bridges inject the referenced value at process start. +// Only the resolved username:password value is kept in the request-local call stack. +func resolveExitCredential(exit hub.NetworkExitAccess) (string, error) { + if exit.CredentialReference == nil || exit.CredentialKey == "" { + return "", errors.New("credential reference unavailable") + } + digest := sha256.Sum256([]byte(exit.CredentialKey)) + name := "CREATORHUB_CREDENTIAL_" + strings.ToUpper(hex.EncodeToString(digest[:])) + value, ok := os.LookupEnv(name) + if !ok || value == "" { + return "", errors.New("credential value unavailable") + } + return value, nil +} + +type gatewayNetworkExit struct { + Protocol string `json:"protocol"` + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` +} + +func gatewayNetworkExitFor(exit hub.NetworkExitAccess, resolve func(hub.NetworkExitAccess) (string, error)) (gatewayNetworkExit, error) { + result := gatewayNetworkExit{Protocol: exit.Protocol, Host: exit.Host, Port: exit.Port} + if exit.CredentialReference == nil { + return result, nil + } + secret, err := resolve(exit) + if err != nil { + return gatewayNetworkExit{}, errors.New("credential unavailable") + } + username, password, found := strings.Cut(secret, ":") + if !found || username == "" { + return gatewayNetworkExit{}, errors.New("credential invalid") + } + result.Username, result.Password = username, password + return result, nil +} diff --git a/cmd/control-plane/network_test.go b/cmd/control-plane/network_test.go new file mode 100644 index 0000000..df02a27 --- /dev/null +++ b/cmd/control-plane/network_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "io" + "net" + "strings" + "testing" + + "git.ipao.vip/rogee/creator-hub/internal/hub" +) + +func TestSOCKS4DialerUsesBoundProxy(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + done := make(chan error, 1) + go func() { + connection, err := listener.Accept() + if err != nil { + done <- err + return + } + defer connection.Close() + header := make([]byte, 8) + if _, err := io.ReadFull(connection, header); err != nil { + done <- err + return + } + user := make([]byte, 0, 16) + for { + var value [1]byte + if _, err := io.ReadFull(connection, value[:]); err != nil { + done <- err + return + } + if value[0] == 0 { + break + } + user = append(user, value[0]) + } + if header[0] != 4 || header[1] != 1 || binary.BigEndian.Uint16(header[2:4]) != 443 || + net.IP(header[4:8]).String() != "203.0.113.1" || string(user) != "operator" { + done <- io.ErrUnexpectedEOF + return + } + _, err = connection.Write([]byte{0, 90, 0, 0, 0, 0, 0, 0}) + done <- err + }() + + connection, err := socks4DialContext(listener.Addr().String(), "operator")(context.Background(), "tcp", "203.0.113.1:443") + if err != nil { + t.Fatal(err) + } + _ = connection.Close() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestGatewayNetworkExitResolvesCredentialWithoutPersistingIt(t *testing.T) { + exit := hub.NetworkExitAccess{NetworkExit: hub.NetworkExit{ + Protocol: "socks5", Host: "proxy.example", Port: 1080, + CredentialReference: &hub.CredentialReference{ID: "credential-a", Provider: "os_keyring"}, + }} + gatewayExit, err := gatewayNetworkExitFor(exit, func(hub.NetworkExitAccess) (string, error) { + return "operator:ephemeral-value", nil + }) + if err != nil || gatewayExit.Username != "operator" || gatewayExit.Password != "ephemeral-value" || gatewayExit.Host != "proxy.example" { + t.Fatalf("credential was not resolved into the request-local gateway payload: %#v err=%v", gatewayExit, err) + } + encoded := string(mustJSON(t, exit.NetworkExit)) + if strings.Contains(encoded, "ephemeral-value") { + t.Fatalf("network exit persistence model contains resolved credential: %s", encoded) + } +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return encoded +} diff --git a/cmd/docker-gateway/main.go b/cmd/docker-gateway/main.go index 650b426..e4e0c0b 100644 --- a/cmd/docker-gateway/main.go +++ b/cmd/docker-gateway/main.go @@ -27,29 +27,33 @@ import ( ) const ( - browserUser = "1000:1000" - browserEntrypoint = "/usr/local/bin/docker-entrypoint.sh" - managedLabel = "io.creatorhub.managed" - idLabel = "io.creatorhub.runtime-id" - nameLabel = "io.creatorhub.display-name" - networkRoleLabel = "io.creatorhub.network-role" - browserNetworkRole = "browser" - controlNetworkName = "creatorhub_control" - namePrefix = "creatorhub-browser-" - pullTimeout = 10 * time.Minute + browserUser = "1000:1000" + browserEntrypoint = "/usr/local/bin/docker-entrypoint.sh" + managedLabel = "io.creatorhub.managed" + idLabel = "io.creatorhub.runtime-id" + nameLabel = "io.creatorhub.display-name" + bindingVersionLabel = "io.creatorhub.binding-version" + networkExitLabel = "io.creatorhub.network-exit-id" + proxyPortLabel = "io.creatorhub.proxy-port" + networkRoleLabel = "io.creatorhub.network-role" + browserNetworkRole = "browser" + controlNetworkName = "creatorhub_control" + namePrefix = "creatorhub-browser-" + pullTimeout = 10 * time.Minute ) var ( - runtimeIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) + runtimeIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) networkNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`) - imageRefPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$`) - volumePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`) + imageRefPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,300}$`) + volumePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`) + exitIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`) ) var ( - errInvalidRuntimeID = errors.New("invalid runtime id") - errUnmanagedContainer = errors.New("refusing to operate on a container not owned by CreatorHub") - errUnauthorized = errors.New("gateway token rejected") + errInvalidRuntimeID = errors.New("invalid runtime id") + errUnmanagedContainer = errors.New("refusing to operate on a container not owned by CreatorHub") + errUnauthorized = errors.New("gateway token rejected") ) type serviceConfig struct { @@ -69,25 +73,42 @@ type dockerClient struct { type gateway struct { docker dockerClient network string + self string token string + proxies *memoryProxyRegistry } // createRequest 全量字段由平台下发；网关不做业务决策，只做输入合法性校验。 type createRequest struct { - Alias string `json:"alias"` - Name string `json:"name"` - Image string `json:"image"` - Cmd []string `json:"cmd"` - Volume string `json:"volume"` + Alias string `json:"alias"` + Name string `json:"name"` + Image string `json:"image"` + Cmd []string `json:"cmd"` + Volume string `json:"volume"` + BindingVersion int64 `json:"binding_version"` + NetworkExitID string `json:"network_exit_id"` + NetworkExit gatewayProxyExit `json:"network_exit"` + Stopped bool `json:"stopped,omitempty"` +} + +type gatewayProxyExit struct { + Protocol string `json:"protocol"` + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password"` } type browser struct { - ID string `json:"id"` - Alias string `json:"alias"` - Name string `json:"name"` - State string `json:"state"` - Status string `json:"status"` - Endpoint string `json:"endpoint"` + ID string `json:"id"` + Alias string `json:"alias"` + Name string `json:"name"` + State string `json:"state"` + Status string `json:"status"` + Endpoint string `json:"endpoint"` + BindingVersion int64 `json:"binding_version"` + NetworkExitID string `json:"network_exit_id"` + ProxyReady bool `json:"proxy_ready"` } func main() { @@ -184,9 +205,6 @@ func run(command *cobra.Command, cfg serviceConfig) error { client: &http.Client{Transport: transport, Timeout: 30 * time.Second}, slow: &http.Client{Transport: transport}, } - if err := docker.ensureBrowserNetwork(cfg.network); err != nil { - return err - } logrus.WithFields(logrus.Fields{ "service": "docker-gateway", "listen_addr": cfg.listenAddr, @@ -199,7 +217,12 @@ func run(command *cobra.Command, cfg serviceConfig) error { } func newGateway(client dockerClient, network, token string) *fiber.App { - api := gateway{docker: client, network: network, token: token} + self, _ := os.Hostname() + return newGatewayWithSelf(client, network, token, self) +} + +func newGatewayWithSelf(client dockerClient, network, token, self string) *fiber.App { + api := gateway{docker: client, network: network, self: self, token: token, proxies: newMemoryProxyRegistry()} app := fiber.New(fiber.Config{ AppName: "CreatorHub Docker gateway", BodyLimit: 1 << 20, @@ -221,6 +244,7 @@ func newGateway(client dockerClient, network, token string) *fiber.App { app.Use("/v1", api.authorize) app.Get("/v1/browsers", api.list) app.Post("/v1/browsers", api.create) + app.Post("/v1/browsers/:id/proxy", api.restoreProxy) app.Post("/v1/browsers/:id/:action", api.changeState) app.Delete("/v1/browsers/:id", api.remove) return app @@ -265,13 +289,18 @@ func (api gateway) list(c fiber.Ctx) error { if name == "" { name = alias } + bindingVersion, _ := strconv.ParseInt(container.Labels[bindingVersionLabel], 10, 64) + proxyPort, _ := strconv.Atoi(container.Labels[proxyPortLabel]) browsers = append(browsers, browser{ - ID: container.ID, - Alias: alias, - Name: name, - State: container.State, - Status: container.Status, - Endpoint: "http://" + namePrefix + alias + ":9222", + ID: container.ID, + Alias: alias, + Name: name, + State: container.State, + Status: container.Status, + Endpoint: "http://" + namePrefix + alias + ":9222", + BindingVersion: bindingVersion, + NetworkExitID: container.Labels[networkExitLabel], + ProxyReady: api.proxies.ready(alias, proxyPort), }) } return writeJSON(c, http.StatusOK, browsers) @@ -282,7 +311,7 @@ func (api gateway) create(c fiber.Ctx) error { decoder := json.NewDecoder(bytes.NewReader(c.Body())) decoder.DisallowUnknownFields() if err := decoder.Decode(&input); err != nil { - return writeError(c, http.StatusBadRequest, errors.New("body must contain only alias, name, image, cmd and volume")) + return writeError(c, http.StatusBadRequest, errors.New("body must contain only alias, name, image, cmd, volume, binding_version, network_exit_id, network_exit and stopped")) } if err := validateCreate(input); err != nil { return writeError(c, http.StatusBadRequest, err) @@ -290,22 +319,48 @@ func (api gateway) create(c fiber.Ctx) error { if err := api.docker.pullIfMissing(c.Context(), input.Image); err != nil { return writeError(c, http.StatusBadGateway, err) } + network, proxyServer, undoProxy := "none", "", func() {} + if !input.Stopped { + var err error + var bindHost string + network, bindHost, err = api.docker.ensureTenantNetwork(api.network, input.Alias, api.self) + if err != nil { + return writeError(c, http.StatusBadGateway, errors.New("configure isolated browser network")) + } + proxyServer, undoProxy, err = api.proxies.configure(input.Alias, bindHost, 0, input.NetworkExit) + if err != nil { + return writeError(c, http.StatusBadGateway, errors.New("configure in-memory proxy")) + } + } + keepProxy := false + defer func() { + if !keepProxy { + undoProxy() + } + }() pidsLimit := int64(512) + cmd := append([]string{}, input.Cmd...) + if !input.Stopped { + cmd = append(cmd[:len(cmd)-1], "--proxy-server="+proxyServer, "--disable-non-proxied-udp", cmd[len(cmd)-1]) + } payload := map[string]any{ "Image": input.Image, "User": browserUser, "Entrypoint": []string{browserEntrypoint}, - "Cmd": input.Cmd, + "Cmd": cmd, "Env": []string{"REMOTE_DEBUGGING_PORT=9222"}, "Labels": map[string]string{ - managedLabel: "true", - idLabel: input.Alias, - nameLabel: input.Name, + managedLabel: "true", + idLabel: input.Alias, + nameLabel: input.Name, + bindingVersionLabel: strconv.FormatInt(input.BindingVersion, 10), + networkExitLabel: input.NetworkExitID, + proxyPortLabel: strconv.Itoa(proxyPort(proxyServer)), }, "ExposedPorts": map[string]any{"9222/tcp": map[string]any{}}, "HostConfig": map[string]any{ - "NetworkMode": api.network, + "NetworkMode": network, "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges"}, @@ -331,7 +386,11 @@ func (api gateway) create(c fiber.Ctx) error { } defer result.Body.Close() if result.StatusCode != http.StatusCreated { - return forwardDockerError(c, result) + status := http.StatusBadGateway + if result.StatusCode == http.StatusConflict { + status = http.StatusConflict + } + return writeError(c, status, errors.New("Docker container creation failed")) } var created struct { ID string `json:"Id"` @@ -343,13 +402,16 @@ func (api gateway) create(c fiber.Ctx) error { } return writeError(c, http.StatusBadGateway, errors.New("Docker returned an invalid container id; container was removed while preserving its Profile volume")) } - if err := api.docker.expect(http.MethodPost, "/containers/"+url.PathEscape(created.ID)+"/start", nil, http.StatusNoContent, http.StatusNotModified); err != nil { - cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(created.ID)+"?force=1&v=0", nil, http.StatusNoContent) - if cleanupErr != nil { - return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and cleanup failed: %w; cleanup: %v", err, cleanupErr)) + if !input.Stopped { + if err := api.docker.expect(http.MethodPost, "/containers/"+url.PathEscape(created.ID)+"/start", nil, http.StatusNoContent, http.StatusNotModified); err != nil { + cleanupErr := api.docker.expect(http.MethodDelete, "/containers/"+url.PathEscape(created.ID)+"?force=1&v=0", nil, http.StatusNoContent) + if cleanupErr != nil { + return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and cleanup failed: %w; cleanup: %v", err, cleanupErr)) + } + return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and was removed while preserving its Profile volume: %w", err)) } - return writeError(c, http.StatusBadGateway, fmt.Errorf("container did not start and was removed while preserving its Profile volume: %w", err)) } + keepProxy = !input.Stopped return writeJSON(c, http.StatusCreated, map[string]string{"id": created.ID, "alias": input.Alias}) } @@ -366,7 +428,11 @@ func validateCreate(input createRequest) error { if !volumePattern.MatchString(input.Volume) { return errors.New("volume must be a valid volume name") } - if len(input.Cmd) == 0 || len(input.Cmd) > 64 { + if input.BindingVersion < 1 || (!input.Stopped && !exitIDPattern.MatchString(input.NetworkExitID)) || + (input.Stopped && (input.NetworkExitID != "" || input.NetworkExit != (gatewayProxyExit{}))) { + return errors.New("binding_version and network_exit_id must identify the current binding") + } + if len(input.Cmd) == 0 || len(input.Cmd) > 64 || input.Cmd[len(input.Cmd)-1] != "about:blank" { return errors.New("cmd must contain 1..64 arguments") } total := 0 @@ -374,14 +440,34 @@ func validateCreate(input createRequest) error { if arg == "" || hasControlRunes(arg) { return errors.New("cmd arguments must be non-empty visible strings") } + if strings.HasPrefix(arg, "--proxy-server") || arg == "--disable-non-proxied-udp" { + return errors.New("proxy arguments are platform-controlled") + } total += len(arg) } if total > 4096 { return errors.New("cmd arguments exceed 4096 characters") } + if input.Stopped { + return nil + } + proxy := input.NetworkExit + if (proxy.Protocol != "http" && proxy.Protocol != "https" && proxy.Protocol != "socks4" && proxy.Protocol != "socks5") || + proxy.Host == "" || len(proxy.Host) > 253 || strings.ContainsAny(proxy.Host, "@/[]?# \t\r\n") || + proxy.Port < 1 || proxy.Port > 65535 || (proxy.Username == "" && proxy.Password != "") || + len(proxy.Username) > 255 || len(proxy.Password) > 255 || + hasControlRunes(proxy.Username) || hasControlRunes(proxy.Password) { + return errors.New("network_exit must contain a valid proxy endpoint and optional credentials") + } return nil } +func proxyPort(proxyServer string) int { + parsed, _ := url.Parse(proxyServer) + port, _ := strconv.Atoi(parsed.Port()) + return port +} + func hasControlRunes(value string) bool { for _, r := range value { if r < 0x20 || r == 0x7f { @@ -415,31 +501,77 @@ func (api gateway) changeState(c fiber.Ctx) error { func (api gateway) remove(c fiber.Ctx) error { id := c.Params("id") - if err := api.requireManaged(id); err != nil { + err := api.requireManaged(id) + if err != nil && !errors.Is(err, os.ErrNotExist) { return writeError(c, statusFor(err), err) } - path := "/containers/" + url.PathEscape(namePrefix+id) + "?force=1&v=0" - if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent); err != nil { - return writeError(c, http.StatusBadGateway, err) + if err == nil { + path := "/containers/" + url.PathEscape(namePrefix+id) + "?force=1&v=0" + if err := api.docker.expect(http.MethodDelete, path, nil, http.StatusNoContent, http.StatusNotFound); err != nil { + return writeError(c, http.StatusBadGateway, err) + } + } + api.proxies.remove(id) + if err := api.docker.removeTenantNetwork(api.network, id, api.self); err != nil { + return c.Status(http.StatusAccepted).JSON(map[string]string{ + "status": "container_removed_network_cleanup_pending", + }) } c.Status(http.StatusNoContent) return nil } +func (api gateway) restoreProxy(c fiber.Ctx) error { + input := struct { + BindingVersion int64 `json:"binding_version"` + NetworkExitID string `json:"network_exit_id"` + NetworkExit gatewayProxyExit `json:"network_exit"` + }{} + decoder := json.NewDecoder(bytes.NewReader(c.Body())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil || input.BindingVersion < 1 || !exitIDPattern.MatchString(input.NetworkExitID) || + validateCreate(createRequest{Alias: c.Params("id"), Name: "x", Image: "x", Cmd: []string{"about:blank"}, Volume: "x", + BindingVersion: input.BindingVersion, NetworkExitID: input.NetworkExitID, NetworkExit: input.NetworkExit}) != nil { + return writeError(c, http.StatusBadRequest, errors.New("invalid proxy recovery request")) + } + labels, err := api.managedLabels(c.Params("id")) + if err != nil { + return writeError(c, statusFor(err), err) + } + version, _ := strconv.ParseInt(labels[bindingVersionLabel], 10, 64) + port, _ := strconv.Atoi(labels[proxyPortLabel]) + if version != input.BindingVersion || labels[networkExitLabel] != input.NetworkExitID || port < 1 { + return writeError(c, http.StatusConflict, errors.New("container binding does not match recovery request")) + } + _, bindHost, err := api.docker.ensureTenantNetwork(api.network, c.Params("id"), api.self) + if err != nil { + return writeError(c, http.StatusBadGateway, errors.New("restore isolated browser network")) + } + if _, _, err := api.proxies.configure(c.Params("id"), bindHost, port, input.NetworkExit); err != nil { + return writeError(c, http.StatusBadGateway, errors.New("restore in-memory proxy")) + } + return c.SendStatus(http.StatusNoContent) +} + func (api gateway) requireManaged(id string) error { + _, err := api.managedLabels(id) + return err +} + +func (api gateway) managedLabels(id string) (map[string]string, error) { if !runtimeIDPattern.MatchString(id) { - return errInvalidRuntimeID + return nil, errInvalidRuntimeID } result, err := api.docker.request(http.MethodGet, "/containers/"+url.PathEscape(namePrefix+id)+"/json", nil) if err != nil { - return err + return nil, err } defer result.Body.Close() if result.StatusCode == http.StatusNotFound { - return os.ErrNotExist + return nil, os.ErrNotExist } if result.StatusCode != http.StatusOK { - return fmt.Errorf("Docker inspect returned %s", result.Status) + return nil, fmt.Errorf("Docker inspect returned %s", result.Status) } var inspected struct { Config struct { @@ -447,12 +579,12 @@ func (api gateway) requireManaged(id string) error { } `json:"Config"` } if err := json.NewDecoder(result.Body).Decode(&inspected); err != nil { - return fmt.Errorf("decode Docker inspect: %w", err) + return nil, fmt.Errorf("decode Docker inspect: %w", err) } if inspected.Config.Labels[managedLabel] != "true" || inspected.Config.Labels[idLabel] != id { - return errUnmanagedContainer + return nil, errUnmanagedContainer } - return nil + return inspected.Config.Labels, nil } // pullIfMissing 在镜像不在本地时从远端仓库拉取；镜像缺失属于可恢复错误，调用方可直接重试。 @@ -541,16 +673,34 @@ func (docker dockerClient) expect(method, path string, payload any, allowed ...i return fmt.Errorf("Docker returned %s: %s", response.Status, strings.TrimSpace(string(message))) } -func (docker dockerClient) ensureBrowserNetwork(name string) error { - if err := validateBrowserNetwork(name); err != nil { - return err +func tenantNetworkName(base, alias string) (string, error) { + name := base + "-" + alias + if !networkNamePattern.MatchString(name) { + return "", errors.New("isolated browser network name is invalid") } - response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil) - if err != nil { - return fmt.Errorf("inspect browser network: %w", err) + return name, nil +} + +func (docker dockerClient) ensureTenantNetwork(base, alias, self string) (string, string, error) { + name, err := tenantNetworkName(base, alias) + if err != nil || self == "" { + return "", "", errors.New("isolated browser network identity is invalid") } - defer response.Body.Close() - if response.StatusCode == http.StatusOK { + inspect := func() (map[string]struct { + Name string `json:"Name"` + IPv4Address string `json:"IPv4Address"` + }, error) { + response, requestErr := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil) + if requestErr != nil { + return nil, requestErr + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return nil, os.ErrNotExist + } + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("inspect isolated browser network returned %s", response.Status) + } var network struct { Name string `json:"Name"` Driver string `json:"Driver"` @@ -558,33 +708,99 @@ func (docker dockerClient) ensureBrowserNetwork(name string) error { Attachable bool `json:"Attachable"` Ingress bool `json:"Ingress"` Labels map[string]string `json:"Labels"` + Containers map[string]struct { + Name string `json:"Name"` + IPv4Address string `json:"IPv4Address"` + } `json:"Containers"` } if err := json.NewDecoder(response.Body).Decode(&network); err != nil { - return fmt.Errorf("decode browser network: %w", err) + return nil, err } if network.Name != name || network.Driver != "bridge" || network.Internal || network.Attachable || network.Ingress || - network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole { - return fmt.Errorf("browser network %q is not a CreatorHub bridge", name) + network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias { + return nil, errors.New("isolated browser network is not owned by this runtime") } + return network.Containers, nil + } + containers, err := inspect() + if errors.Is(err, os.ErrNotExist) { + if err := docker.expect(http.MethodPost, "/networks/create", map[string]any{ + "Name": name, "CheckDuplicate": true, "Driver": "bridge", + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias}, + }, http.StatusCreated); err != nil { + return "", "", err + } + containers = map[string]struct { + Name string `json:"Name"` + IPv4Address string `json:"IPv4Address"` + }{} + } else if err != nil { + return "", "", err + } + findIP := func() string { + for id, container := range containers { + if id == self || strings.HasPrefix(id, self) || strings.HasPrefix(self, id) || container.Name == self { + host, _, _ := net.ParseCIDR(container.IPv4Address) + if host != nil { + return host.String() + } + } + } + return "" + } + if bindHost := findIP(); bindHost != "" { + return name, bindHost, nil + } + if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(name)+"/connect", map[string]any{ + "Container": self, "EndpointConfig": map[string]any{"Aliases": []string{browserProxyHost}}, + }, http.StatusOK); err != nil { + return "", "", err + } + containers, err = inspect() + if err != nil { + return "", "", err + } + if bindHost := findIP(); bindHost != "" { + return name, bindHost, nil + } + return "", "", errors.New("Docker did not assign the gateway an isolated network address") +} + +func (docker dockerClient) removeTenantNetwork(base, alias, self string) error { + name, err := tenantNetworkName(base, alias) + if err != nil { + return err + } + response, err := docker.request(http.MethodGet, "/networks/"+url.PathEscape(name), nil) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { return nil } - if response.StatusCode != http.StatusNotFound { - return fmt.Errorf("inspect browser network returned %s", response.Status) + var network struct { + Name string `json:"Name"` + Labels map[string]string `json:"Labels"` } - return docker.expect(http.MethodPost, "/networks/create", map[string]any{ - "Name": name, - "CheckDuplicate": true, - "Driver": "bridge", - "Labels": map[string]string{ - managedLabel: "true", - networkRoleLabel: browserNetworkRole, - }, - }, http.StatusCreated) + if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&network) != nil || network.Name != name || + network.Labels[managedLabel] != "true" || network.Labels[networkRoleLabel] != browserNetworkRole || network.Labels[idLabel] != alias { + return errors.New("refusing to remove an unowned browser network") + } + if err := docker.expect(http.MethodPost, "/networks/"+url.PathEscape(name)+"/disconnect", map[string]any{ + "Container": self, "Force": true, + }, http.StatusOK, http.StatusNotFound); err != nil { + return fmt.Errorf("disconnect isolated browser network: %w", err) + } + if err := docker.expect(http.MethodDelete, "/networks/"+url.PathEscape(name), nil, http.StatusNoContent, http.StatusNotFound); err != nil { + return fmt.Errorf("remove isolated browser network: %w", err) + } + return nil } func validateBrowserNetwork(name string) error { - if !networkNamePattern.MatchString(name) { - return errors.New("BROWSER_NETWORK is invalid") + if !networkNamePattern.MatchString(name) || len(name) > 31 { + return errors.New("BROWSER_NETWORK must be a valid network prefix of at most 31 characters") } if name == controlNetworkName { return errors.New("BROWSER_NETWORK must not reuse the control network") diff --git a/cmd/docker-gateway/main_test.go b/cmd/docker-gateway/main_test.go index 350a9eb..1ef6ac1 100644 --- a/cmd/docker-gateway/main_test.go +++ b/cmd/docker-gateway/main_test.go @@ -1,10 +1,14 @@ package main import ( + "bytes" "encoding/json" "io" + "net" "net/http" "net/http/httptest" + "os" + "strconv" "strings" "testing" @@ -21,7 +25,27 @@ func authed(method, target string, body io.Reader) *http.Request { } func testDocker(handler http.HandlerFunc) (dockerClient, *httptest.Server) { - server := httptest.NewServer(handler) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if strings.HasPrefix(request.URL.Path, "/networks/creatorhub_browser-") { + if request.Method != http.MethodGet { + if request.Method == http.MethodDelete { + response.WriteHeader(http.StatusNoContent) + } else { + response.WriteHeader(http.StatusOK) + } + return + } + alias := strings.TrimPrefix(request.URL.Path, "/networks/creatorhub_browser-") + self, _ := os.Hostname() + _ = json.NewEncoder(response).Encode(map[string]any{ + "Name": "creatorhub_browser-" + alias, "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: alias}, + "Containers": map[string]any{self: map[string]string{"Name": self, "IPv4Address": "127.0.0.1/8"}}, + }) + return + } + handler(response, request) + })) return dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, server } @@ -35,7 +59,43 @@ func decodeJSONBody(t *testing.T, response *http.Response) map[string]any { } const testCreateBody = `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` + - `"cmd":["--fingerprint=1000","--lang=zh-CN","about:blank"],"volume":"creatorhub-profile-account-a"}` + `"cmd":["--fingerprint=1000","--lang=zh-CN","about:blank"],"volume":"creatorhub-profile-account-a",` + + `"binding_version":1,"network_exit_id":"exit-1",` + + `"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` + +func TestGatewayCreatesNetworkDisabledStoppedRecoveryContainer(t *testing.T) { + created := false + docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + _, _ = response.Write([]byte(`{}`)) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): + var payload map[string]any + _ = json.NewDecoder(request.Body).Decode(&payload) + host := payload["HostConfig"].(map[string]any) + labels := payload["Labels"].(map[string]any) + encoded, _ := json.Marshal(payload["Cmd"]) + if host["NetworkMode"] != "none" || labels[networkExitLabel] != "" || strings.Contains(string(encoded), "proxy") { + t.Fatalf("unsafe stopped recovery payload: %#v", payload) + } + created = true + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"stopped-container"}`)) + default: + t.Fatalf("stopped recovery unexpectedly called Docker %s %s", request.Method, request.URL.String()) + } + }) + defer server.Close() + handler := newGateway(docker, "creatorhub_browser", testToken) + body := `{"alias":"account-a","name":"账号甲","image":"registry.example/browser:1.2.3",` + + `"cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` + + `"binding_version":1,"network_exit_id":"","network_exit":{},"stopped":true}` + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) + if response.Code != http.StatusCreated || !created { + t.Fatalf("stopped recovery create failed: status=%d body=%s", response.Code, response.Body.String()) + } +} func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { var created map[string]any @@ -75,11 +135,12 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { t.Fatalf("runtime identity is not fixed: user=%#v entrypoint=%#v", created["User"], created["Entrypoint"]) } cmd := created["Cmd"].([]any) - if len(cmd) != 3 || cmd[0] != "--fingerprint=1000" || cmd[2] != "about:blank" { + if len(cmd) != 5 || cmd[0] != "--fingerprint=1000" || !strings.HasPrefix(cmd[2].(string), "--proxy-server=http://docker-gateway:") || + cmd[3] != "--disable-non-proxied-udp" || cmd[4] != "about:blank" { t.Fatalf("cmd must be passed through verbatim: %#v", created["Cmd"]) } host := created["HostConfig"].(map[string]any) - if host["NetworkMode"] != "creatorhub_browser" || host["ReadonlyRootfs"] != true { + if host["NetworkMode"] != "creatorhub_browser-account-a" || host["ReadonlyRootfs"] != true { t.Fatalf("missing container isolation: %#v", host) } tmpfs := host["Tmpfs"].(map[string]any) @@ -96,12 +157,51 @@ func TestGatewayCreatesConstrainedBrowserWithPlatformSpec(t *testing.T) { } } +func TestGatewayDockerInspectContainsNoProxyCredentials(t *testing.T) { + var created map[string]any + docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/"): + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create"): + if err := json.NewDecoder(request.Body).Decode(&created); err != nil { + t.Fatal(err) + } + response.WriteHeader(http.StatusCreated) + _, _ = response.Write([]byte(`{"Id":"container-id"}`)) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/container-id/start"): + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + }) + defer server.Close() + + handler := newGateway(docker, "creatorhub_browser", testToken) + body := strings.Replace(testCreateBody, `"protocol":"socks5","host":"proxy.example","port":1080`, + `"protocol":"socks5","host":"proxy.example","port":1080,"username":"operator","password":"ephemeral"`, 1) + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) + if response.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d: %s", response.Code, response.Body.String()) + } + inspect, _ := json.Marshal(created) + for _, secret := range []string{"operator", "ephemeral", "operator:ephemeral@", "proxy.example"} { + if bytes.Contains(inspect, []byte(secret)) { + t.Fatalf("Docker inspect leaked proxy credential %q: %s", secret, inspect) + } + } + if !bytes.Contains(inspect, []byte("--proxy-server=http://docker-gateway:")) { + t.Fatalf("Docker inspect is missing the secret-free proxy configuration: %s", inspect) + } +} + func TestGatewayPullsMissingImageOnCreate(t *testing.T) { tests := []struct { - name string - ref string - fromImage string - tag string + name string + ref string + fromImage string + tag string }{{ name: "tagged ref splits repository and tag", ref: "registry.example/browser:2.0.0", @@ -139,7 +239,9 @@ func TestGatewayPullsMissingImageOnCreate(t *testing.T) { handler := newGateway(docker, "creatorhub_browser", testToken) body := `{"alias":"account-a","name":"账号甲","image":"` + test.ref + - `","cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a"}` + `","cmd":["--fingerprint=1000","about:blank"],"volume":"creatorhub-profile-account-a",` + + `"binding_version":1,"network_exit_id":"exit-1",` + + `"network_exit":{"protocol":"socks5","host":"proxy.example","port":1080}}` response := httptest.NewRecorder() adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) @@ -187,6 +289,7 @@ func TestGatewayRejectsInvalidCreateRequest(t *testing.T) { "invalid image": `{"alias":"account-a","name":"甲","image":"","cmd":["--fingerprint=1"],"volume":"creatorhub-profile-account-a"}`, "empty cmd": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":[],"volume":"creatorhub-profile-account-a"}`, "invalid volume": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1"],"volume":"bad volume!"}`, + "proxy override": `{"alias":"account-a","name":"甲","image":"reg/img:1","cmd":["--fingerprint=1","--proxy-server=http://direct:8080","about:blank"],"volume":"creatorhub-profile-account-a","network_exit":{"protocol":"socks5","host":"proxy","port":1080}}`, } for name, body := range tests { t.Run(name, func(t *testing.T) { @@ -281,6 +384,31 @@ func TestGatewayRemovesFailedContainerAndPreservesProfile(t *testing.T) { } } +func TestGatewayDoesNotEchoProxyCredentialsFromDockerErrors(t *testing.T) { + docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/images/") { + response.WriteHeader(http.StatusOK) + return + } + if request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/containers/create") { + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(`invalid cmd --proxy-server=http://operator:ephemeral@proxy.example:8080`)) + return + } + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path) + }) + defer server.Close() + handler := newGateway(docker, "creatorhub_browser", testToken) + body := strings.Replace(testCreateBody, `"protocol":"socks5","host":"proxy.example","port":1080`, + `"protocol":"http","host":"proxy.example","port":8080,"username":"operator","password":"ephemeral"`, 1) + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers", strings.NewReader(body))) + if response.Code != http.StatusBadGateway || strings.Contains(response.Body.String(), "operator") || + strings.Contains(response.Body.String(), "ephemeral") || strings.Contains(response.Body.String(), "proxy.example") { + t.Fatalf("gateway leaked proxy material: status=%d body=%s", response.Code, response.Body.String()) + } +} + func TestGatewayListsBrowsers(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { if request.Method != http.MethodGet || request.URL.Path != "/containers/json" { @@ -303,6 +431,43 @@ func TestGatewayListsBrowsers(t *testing.T) { } } +func TestGatewayRestartRestoresExistingProxyListener(t *testing.T) { + reserved, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := reserved.Addr().(*net.TCPAddr).Port + _ = reserved.Close() + labels := map[string]string{ + managedLabel: "true", idLabel: "account-a", nameLabel: "账号甲", + bindingVersionLabel: "3", networkExitLabel: "exit-1", proxyPortLabel: strconv.Itoa(port), + } + docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasSuffix(request.URL.Path, "/containers/creatorhub-browser-account-a/json"): + _ = json.NewEncoder(response).Encode(map[string]any{"Config": map[string]any{"Labels": labels}}) + case request.Method == http.MethodGet && request.URL.Path == "/containers/json": + _ = json.NewEncoder(response).Encode([]map[string]any{{"Id": "container-id", "State": "running", "Status": "Up", "Labels": labels}}) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + }) + defer server.Close() + handler := newGateway(docker, "creatorhub_browser", testToken) + recovery := `{"binding_version":3,"network_exit_id":"exit-1","network_exit":{"protocol":"http","host":"127.0.0.1","port":1}}` + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodPost, "/v1/browsers/account-a/proxy", strings.NewReader(recovery))) + if response.Code != http.StatusNoContent { + t.Fatalf("proxy recovery failed: %d %s", response.Code, response.Body.String()) + } + response = httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodGet, "/v1/browsers", nil)) + var browsers []browser + if response.Code != http.StatusOK || json.NewDecoder(response.Body).Decode(&browsers) != nil || len(browsers) != 1 || !browsers[0].ProxyReady { + t.Fatalf("restarted gateway did not report restored proxy: %d %s", response.Code, response.Body.String()) + } +} + func TestGatewayLifecycle(t *testing.T) { tests := []struct { method string @@ -337,6 +502,55 @@ func TestGatewayLifecycle(t *testing.T) { } } +func TestGatewayDeleteDistinguishesContainerRemovalFromNetworkCleanup(t *testing.T) { + containerExists, cleanupFails, containerDeletes := true, true, 0 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/containers/"): + if !containerExists { + response.WriteHeader(http.StatusNotFound) + return + } + _, _ = response.Write([]byte(`{"Config":{"Labels":{"` + managedLabel + `":"true","` + idLabel + `":"account-a"}}}`)) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/containers/"): + containerExists = false + containerDeletes++ + response.WriteHeader(http.StatusNoContent) + case request.Method == http.MethodGet && strings.HasPrefix(request.URL.Path, "/networks/"): + _ = json.NewEncoder(response).Encode(map[string]any{ + "Name": "creatorhub_browser-account-a", + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a"}, + }) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/disconnect"): + if cleanupFails { + response.WriteHeader(http.StatusInternalServerError) + return + } + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodDelete && strings.HasPrefix(request.URL.Path, "/networks/"): + response.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + handler := newGatewayWithSelf(dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()}, + "creatorhub_browser", testToken, "gateway-self") + + response := httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil)) + if response.Code != http.StatusAccepted || containerExists || containerDeletes != 1 { + t.Fatalf("expected definite container removal with pending cleanup, status=%d exists=%v deletes=%d body=%s", + response.Code, containerExists, containerDeletes, response.Body.String()) + } + cleanupFails = false + response = httptest.NewRecorder() + adaptor.FiberApp(handler).ServeHTTP(response, authed(http.MethodDelete, "/v1/browsers/account-a", nil)) + if response.Code != http.StatusNoContent || containerDeletes != 1 { + t.Fatalf("idempotent cleanup retry failed: status=%d deletes=%d body=%s", response.Code, containerDeletes, response.Body.String()) + } +} + func TestGatewayMapsDockerServiceFailureToBadGateway(t *testing.T) { docker, server := testDocker(func(response http.ResponseWriter, _ *http.Request) { http.Error(response, "daemon unavailable", http.StatusInternalServerError) @@ -374,94 +588,39 @@ func TestGatewayRefusesUnmanagedContainer(t *testing.T) { } } -func TestEnsureBrowserNetwork(t *testing.T) { - var created struct { - Name string `json:"Name"` - Driver string `json:"Driver"` - Labels map[string]string `json:"Labels"` - } - docker, server := testDocker(func(response http.ResponseWriter, request *http.Request) { - switch request.Method { - case http.MethodGet: +func TestEnsureTenantNetworkConnectsGatewayOnlyToRuntimeNetwork(t *testing.T) { + created, connected := false, false + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodGet && !created: response.WriteHeader(http.StatusNotFound) - case http.MethodPost: - if err := json.NewDecoder(request.Body).Decode(&created); err != nil { - t.Fatal(err) + case request.Method == http.MethodPost && request.URL.Path == "/networks/create": + var body map[string]any + _ = json.NewDecoder(request.Body).Decode(&body) + labels := body["Labels"].(map[string]any) + if body["Name"] != "creatorhub_browser-account-a" || labels[idLabel] != "account-a" { + t.Fatalf("unexpected isolated network create: %#v", body) } + created = true response.WriteHeader(http.StatusCreated) - } - }) - defer server.Close() - - if err := docker.ensureBrowserNetwork("creatorhub_browser"); err != nil { - t.Fatal(err) - } - if created.Name != "creatorhub_browser" || created.Driver != "bridge" || - created.Labels[managedLabel] != "true" || created.Labels[networkRoleLabel] != browserNetworkRole { - t.Fatalf("network is not a CreatorHub bridge: %#v", created) - } -} - -func TestEnsureBrowserNetworkRejectsUnsafeExistingNetwork(t *testing.T) { - valid := map[string]any{ - "Name": "creatorhub_browser", - "Driver": "bridge", - "Internal": false, - "Attachable": false, - "Ingress": false, - "Labels": map[string]string{ - managedLabel: "true", - networkRoleLabel: browserNetworkRole, - }, - } - validDocker, validServer := testDocker(func(response http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(response).Encode(valid) - }) - defer validServer.Close() - if err := validDocker.ensureBrowserNetwork("creatorhub_browser"); err != nil { - t.Fatalf("expected owned bridge network to be accepted: %v", err) - } - - tests := map[string]func(map[string]any){ - "wrong name": func(network map[string]any) { network["Name"] = "other" }, - "wrong driver": func(network map[string]any) { network["Driver"] = "overlay" }, - "internal": func(network map[string]any) { network["Internal"] = true }, - "attachable": func(network map[string]any) { network["Attachable"] = true }, - "ingress": func(network map[string]any) { network["Ingress"] = true }, - "missing ownership": func(network map[string]any) { - network["Labels"] = map[string]string{networkRoleLabel: browserNetworkRole} - }, - "wrong role": func(network map[string]any) { - network["Labels"] = map[string]string{managedLabel: "true", networkRoleLabel: "control"} - }, - } - - for name, mutate := range tests { - t.Run(name, func(t *testing.T) { - network := make(map[string]any, len(valid)) - for key, value := range valid { - network[key] = value - } - mutate(network) - docker, server := testDocker(func(response http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(response).Encode(network) + case request.Method == http.MethodPost && strings.HasSuffix(request.URL.Path, "/connect"): + connected = true + response.WriteHeader(http.StatusOK) + case request.Method == http.MethodGet: + _ = json.NewEncoder(response).Encode(map[string]any{ + "Name": "creatorhub_browser-account-a", "Driver": "bridge", "Internal": false, "Attachable": false, "Ingress": false, + "Labels": map[string]string{managedLabel: "true", networkRoleLabel: browserNetworkRole, idLabel: "account-a"}, + "Containers": map[string]any{"gateway-id": map[string]string{"Name": "gateway-id", "IPv4Address": "127.0.0.3/8"}}, }) - defer server.Close() - - if err := docker.ensureBrowserNetwork("creatorhub_browser"); err == nil { - t.Fatal("expected unsafe existing network to be rejected") - } - }) - } -} - -func TestEnsureBrowserNetworkRejectsControlNetwork(t *testing.T) { - requested := false - docker, server := testDocker(func(http.ResponseWriter, *http.Request) { requested = true }) + default: + t.Fatalf("unexpected Docker request %s %s", request.Method, request.URL.Path) + } + })) defer server.Close() - - if err := docker.ensureBrowserNetwork(controlNetworkName); err == nil || requested { - t.Fatalf("expected control network to be rejected before Docker request, requested=%v err=%v", requested, err) + docker := dockerClient{baseURL: server.URL, client: server.Client(), slow: server.Client()} + name, bindHost, err := docker.ensureTenantNetwork("creatorhub_browser", "account-a", "gateway-id") + if err != nil || !created || !connected || name != "creatorhub_browser-account-a" || bindHost != "127.0.0.3" { + t.Fatalf("isolated network was not created and connected: name=%q host=%q created=%v connected=%v err=%v", name, bindHost, created, connected, err) } } diff --git a/cmd/docker-gateway/proxy.go b/cmd/docker-gateway/proxy.go new file mode 100644 index 0000000..9d1e3fd --- /dev/null +++ b/cmd/docker-gateway/proxy.go @@ -0,0 +1,362 @@ +package main + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "sync" + "time" +) + +const browserProxyHost = "docker-gateway" + +type memoryProxyRegistry struct { + mu sync.Mutex + proxies map[string]*memoryProxy +} + +type memoryProxy struct { + mu sync.RWMutex + exit gatewayProxyExit + bindHost string + listener net.Listener + server *http.Server + url string +} + +func newMemoryProxyRegistry() *memoryProxyRegistry { + return &memoryProxyRegistry{proxies: map[string]*memoryProxy{}} +} + +func (registry *memoryProxyRegistry) configure(alias, bindHost string, port int, exit gatewayProxyExit) (string, func(), error) { + registry.mu.Lock() + defer registry.mu.Unlock() + if proxy := registry.proxies[alias]; proxy != nil { + if proxy.bindHost == bindHost && (port == 0 || proxy.listener.Addr().(*net.TCPAddr).Port == port) && proxy.exit == exit { + return proxy.url, func() {}, nil + } + delete(registry.proxies, alias) + _ = proxy.server.Close() + } + listener, err := net.Listen("tcp4", net.JoinHostPort(bindHost, strconv.Itoa(port))) + if err != nil { + return "", nil, err + } + actualPort := listener.Addr().(*net.TCPAddr).Port + proxy := &memoryProxy{exit: exit, bindHost: bindHost, listener: listener, url: "http://" + net.JoinHostPort(browserProxyHost, strconv.Itoa(actualPort))} + proxy.server = &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second} + registry.proxies[alias] = proxy + go func() { _ = proxy.server.Serve(listener) }() + undo := func() { + registry.mu.Lock() + defer registry.mu.Unlock() + if registry.proxies[alias] == proxy { + delete(registry.proxies, alias) + _ = proxy.server.Close() + } + } + return proxy.url, undo, nil +} + +func (registry *memoryProxyRegistry) ready(alias string, port int) bool { + registry.mu.Lock() + defer registry.mu.Unlock() + proxy := registry.proxies[alias] + return proxy != nil && proxy.listener.Addr().(*net.TCPAddr).Port == port +} + +func (registry *memoryProxyRegistry) remove(alias string) { + registry.mu.Lock() + proxy := registry.proxies[alias] + delete(registry.proxies, alias) + registry.mu.Unlock() + if proxy != nil { + _ = proxy.server.Close() + } +} + +func (proxy *memoryProxy) ServeHTTP(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodConnect { + proxy.tunnel(response, request) + return + } + proxy.mu.RLock() + exit := proxy.exit + proxy.mu.RUnlock() + transport := &http.Transport{DisableKeepAlives: true} + if exit.Protocol == "http" || exit.Protocol == "https" { + upstream := &url.URL{Scheme: exit.Protocol, Host: net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port))} + if exit.Username != "" { + upstream.User = url.UserPassword(exit.Username, exit.Password) + } + transport.Proxy = http.ProxyURL(upstream) + } else { + transport.DialContext = proxy.dialContext + } + defer transport.CloseIdleConnections() + outbound := request.Clone(request.Context()) + outbound.RequestURI = "" + outbound.Header.Del("Proxy-Authorization") + result, err := transport.RoundTrip(outbound) + if err != nil { + http.Error(response, "proxy connection failed", http.StatusBadGateway) + return + } + defer result.Body.Close() + for key, values := range result.Header { + for _, value := range values { + response.Header().Add(key, value) + } + } + response.WriteHeader(result.StatusCode) + _, _ = io.Copy(response, result.Body) +} + +func (proxy *memoryProxy) tunnel(response http.ResponseWriter, request *http.Request) { + upstream, err := proxy.dialContext(request.Context(), "tcp", request.Host) + if err != nil { + http.Error(response, "proxy connection failed", http.StatusBadGateway) + return + } + client, buffered, err := http.NewResponseController(response).Hijack() + if err != nil { + _ = upstream.Close() + http.Error(response, "proxy tunnel unavailable", http.StatusInternalServerError) + return + } + if _, err := buffered.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil || buffered.Flush() != nil { + _ = client.Close() + _ = upstream.Close() + return + } + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(upstream, client); done <- struct{}{} }() + go func() { _, _ = io.Copy(client, upstream); done <- struct{}{} }() + <-done + _ = client.Close() + _ = upstream.Close() +} + +func (proxy *memoryProxy) dialContext(ctx context.Context, _, target string) (net.Conn, error) { + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + proxy.mu.RLock() + exit := proxy.exit + proxy.mu.RUnlock() + switch exit.Protocol { + case "http", "https": + return dialHTTPProxy(ctx, exit, target) + case "socks4": + return dialSOCKS4Proxy(ctx, exit, target) + case "socks5": + return dialSOCKS5Proxy(ctx, exit, target) + default: + return nil, errors.New("unsupported proxy protocol") + } +} + +func dialHTTPProxy(ctx context.Context, exit gatewayProxyExit, target string) (net.Conn, error) { + address := net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port)) + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address) + if err != nil { + return nil, err + } + if exit.Protocol == "https" { + tlsConnection := tls.Client(connection, &tls.Config{ServerName: exit.Host, MinVersion: tls.VersionTLS12}) + if err := tlsConnection.HandshakeContext(ctx); err != nil { + _ = connection.Close() + return nil, err + } + connection = tlsConnection + } + request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)} + if exit.Username != "" { + request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(exit.Username+":"+exit.Password))) + } + if deadline, ok := ctx.Deadline(); ok { + _ = connection.SetDeadline(deadline) + } + if err := request.Write(connection); err != nil { + _ = connection.Close() + return nil, err + } + result, err := http.ReadResponse(bufio.NewReader(connection), request) + if err != nil { + _ = connection.Close() + return nil, err + } + if result.StatusCode != http.StatusOK { + _ = result.Body.Close() + _ = connection.Close() + return nil, fmt.Errorf("upstream proxy returned %s", result.Status) + } + _ = connection.SetDeadline(time.Time{}) + return connection, nil +} + +func dialSOCKS4Proxy(ctx context.Context, exit gatewayProxyExit, target string) (net.Conn, error) { + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port))) + if err != nil { + return nil, err + } + host, portText, err := net.SplitHostPort(target) + if err != nil { + _ = connection.Close() + return nil, err + } + port, err := strconv.Atoi(portText) + if err != nil || port < 1 || port > 65535 { + _ = connection.Close() + return nil, errors.New("invalid SOCKS4 target") + } + payload := []byte{4, 1, byte(port >> 8), byte(port), 0, 0, 0, 1} + if ip := net.ParseIP(host).To4(); ip != nil { + copy(payload[4:8], ip) + } + payload = append(payload, exit.Username...) + payload = append(payload, 0) + if net.ParseIP(host).To4() == nil { + payload = append(payload, host...) + payload = append(payload, 0) + } + if err := exchangeSOCKS(ctx, connection, payload, 8); err != nil { + return nil, err + } + return connection, nil +} + +func dialSOCKS5Proxy(ctx context.Context, exit gatewayProxyExit, target string) (net.Conn, error) { + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(exit.Host, strconv.Itoa(exit.Port))) + if err != nil { + return nil, err + } + methods := []byte{5, 1, 0} + if exit.Username != "" { + methods = []byte{5, 1, 2} + } + if deadline, ok := ctx.Deadline(); ok { + _ = connection.SetDeadline(deadline) + } + if _, err := connection.Write(methods); err != nil { + _ = connection.Close() + return nil, err + } + selection := make([]byte, 2) + if _, err := io.ReadFull(connection, selection); err != nil || selection[0] != 5 || selection[1] == 0xff { + _ = connection.Close() + return nil, errors.New("SOCKS5 authentication method rejected") + } + if selection[1] == 2 { + if len(exit.Username) > 255 || len(exit.Password) > 255 { + _ = connection.Close() + return nil, errors.New("SOCKS5 credentials too long") + } + auth := append([]byte{1, byte(len(exit.Username))}, exit.Username...) + auth = append(auth, byte(len(exit.Password))) + auth = append(auth, exit.Password...) + if _, err := connection.Write(auth); err != nil { + _ = connection.Close() + return nil, err + } + result := make([]byte, 2) + if _, err := io.ReadFull(connection, result); err != nil || result[1] != 0 { + _ = connection.Close() + return nil, errors.New("SOCKS5 authentication rejected") + } + } else if exit.Username != "" { + _ = connection.Close() + return nil, errors.New("SOCKS5 proxy skipped required authentication") + } + host, portText, err := net.SplitHostPort(target) + if err != nil { + _ = connection.Close() + return nil, err + } + port, err := strconv.Atoi(portText) + if err != nil || port < 1 || port > 65535 { + _ = connection.Close() + return nil, errors.New("invalid SOCKS5 target") + } + request := []byte{5, 1, 0} + if ip := net.ParseIP(host); ip != nil && ip.To4() != nil { + request = append(request, 1) + request = append(request, ip.To4()...) + } else if ip != nil { + request = append(request, 4) + request = append(request, ip.To16()...) + } else { + if len(host) > 255 { + _ = connection.Close() + return nil, errors.New("SOCKS5 target too long") + } + request = append(request, 3, byte(len(host))) + request = append(request, host...) + } + portBytes := make([]byte, 2) + binary.BigEndian.PutUint16(portBytes, uint16(port)) + request = append(request, portBytes...) + if _, err := connection.Write(request); err != nil { + _ = connection.Close() + return nil, err + } + header := make([]byte, 4) + if _, err := io.ReadFull(connection, header); err != nil || header[0] != 5 || header[1] != 0 { + _ = connection.Close() + return nil, errors.New("SOCKS5 proxy rejected connection") + } + addressLength := 0 + switch header[3] { + case 1: + addressLength = 4 + case 4: + addressLength = 16 + case 3: + var length [1]byte + if _, err := io.ReadFull(connection, length[:]); err != nil { + _ = connection.Close() + return nil, err + } + addressLength = int(length[0]) + default: + _ = connection.Close() + return nil, errors.New("invalid SOCKS5 response") + } + if _, err := io.CopyN(io.Discard, connection, int64(addressLength+2)); err != nil { + _ = connection.Close() + return nil, err + } + _ = connection.SetDeadline(time.Time{}) + return connection, nil +} + +func exchangeSOCKS(ctx context.Context, connection net.Conn, request []byte, responseBytes int) error { + if deadline, ok := ctx.Deadline(); ok { + _ = connection.SetDeadline(deadline) + } + if _, err := connection.Write(request); err != nil { + _ = connection.Close() + return err + } + if responseBytes > 0 { + response := make([]byte, responseBytes) + if _, err := io.ReadFull(connection, response); err != nil { + _ = connection.Close() + return err + } + if responseBytes == 8 && response[1] != 90 { + _ = connection.Close() + return errors.New("SOCKS4 proxy rejected connection") + } + } + _ = connection.SetDeadline(time.Time{}) + return nil +} diff --git a/cmd/docker-gateway/proxy_test.go b/cmd/docker-gateway/proxy_test.go new file mode 100644 index 0000000..e53e3d4 --- /dev/null +++ b/cmd/docker-gateway/proxy_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +func TestMemoryProxyUsesSOCKS5Credentials(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + done := make(chan error, 1) + go func() { + connection, err := listener.Accept() + if err != nil { + done <- err + return + } + defer connection.Close() + greeting := make([]byte, 3) + if _, err := io.ReadFull(connection, greeting); err != nil { + done <- err + return + } + _, _ = connection.Write([]byte{5, 2}) + authHeader := make([]byte, 2) + _, _ = io.ReadFull(connection, authHeader) + username := make([]byte, int(authHeader[1])) + _, _ = io.ReadFull(connection, username) + var passwordLength [1]byte + _, _ = io.ReadFull(connection, passwordLength[:]) + password := make([]byte, int(passwordLength[0])) + _, _ = io.ReadFull(connection, password) + if string(username) != "operator" || string(password) != "ephemeral" { + done <- io.ErrUnexpectedEOF + return + } + _, _ = connection.Write([]byte{1, 0}) + requestHeader := make([]byte, 5) + _, _ = io.ReadFull(connection, requestHeader) + host := make([]byte, int(requestHeader[4])) + _, _ = io.ReadFull(connection, host) + port := make([]byte, 2) + _, _ = io.ReadFull(connection, port) + if string(host) != "example.com" || binary.BigEndian.Uint16(port) != 443 { + done <- io.ErrUnexpectedEOF + return + } + if _, err = connection.Write([]byte{5, 0, 0, 1, 127, 0, 0, 1, 0, 0}); err != nil { + done <- err + return + } + var tunneled [1]byte + _, err = io.ReadFull(connection, tunneled[:]) + if err == nil && tunneled[0] != 'x' { + err = io.ErrUnexpectedEOF + } + done <- err + }() + + host, portText, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := net.LookupPort("tcp", portText) + registry := newMemoryProxyRegistry() + proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{ + Protocol: "socks5", Host: host, Port: port, Username: "operator", Password: "ephemeral", + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + parsed, _ := url.Parse(proxyURL) + connection, err := net.Dial("tcp", strings.Replace(parsed.Host, browserProxyHost, "127.0.0.1", 1)) + if err != nil { + t.Fatal(err) + } + if _, err := fmt.Fprint(connection, "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n"); err != nil { + t.Fatal(err) + } + response, err := http.ReadResponse(bufio.NewReader(connection), &http.Request{Method: http.MethodConnect}) + if err != nil || response.StatusCode != http.StatusOK { + t.Fatalf("memory proxy CONNECT failed: response=%v err=%v", response, err) + } + if _, err := connection.Write([]byte{'x'}); err != nil { + t.Fatal(err) + } + _ = connection.Close() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestMemoryProxyUsesAbsoluteFormForHTTPUpstream(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodConnect { + http.Error(response, "CONNECT forbidden", http.StatusMethodNotAllowed) + return + } + if !request.URL.IsAbs() || request.URL.String() != "http://example.com/plain" { + t.Fatalf("expected absolute-form request, got %q", request.URL.String()) + } + if request.Header.Get("Proxy-Authorization") == "" { + t.Fatal("upstream proxy credentials were not applied") + } + _, _ = response.Write([]byte("forwarded")) + })) + defer upstream.Close() + address, _ := url.Parse(upstream.URL) + port, _ := strconv.Atoi(address.Port()) + registry := newMemoryProxyRegistry() + proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{ + Protocol: "http", Host: address.Hostname(), Port: port, Username: "operator", Password: "ephemeral", + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + proxyAddress := strings.Replace(strings.TrimPrefix(proxyURL, "http://"), browserProxyHost, "127.0.0.1", 1) + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(&url.URL{Scheme: "http", Host: proxyAddress})}} + response, err := client.Get("http://example.com/plain") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + if response.StatusCode != http.StatusOK || string(body) != "forwarded" { + t.Fatalf("plain HTTP was not forwarded: status=%d body=%s", response.StatusCode, body) + } +} + +func TestMemoryProxyRejectsCrossAliasAddress(t *testing.T) { + registry := newMemoryProxyRegistry() + proxyURL, cleanup, err := registry.configure("account-a", "127.0.0.1", 0, gatewayProxyExit{Protocol: "http", Host: "127.0.0.1", Port: 1}) + if err != nil { + t.Fatal(err) + } + defer cleanup() + parsed, _ := url.Parse(proxyURL) + if connection, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.2", parsed.Port()), 100*time.Millisecond); err == nil { + _ = connection.Close() + t.Fatal("another tenant address could reach account-a proxy") + } +} diff --git a/compose.yaml b/compose.yaml index 917ddb9..f65fd99 100644 --- a/compose.yaml +++ b/compose.yaml @@ -67,6 +67,5 @@ services: networks: control: name: creatorhub_control - volumes: creatorhub_postgres: diff --git a/docs/architecture/container-control.md b/docs/architecture/container-control.md index 29db0f5..aff8060 100644 --- a/docs/architecture/container-control.md +++ b/docs/architecture/container-control.md @@ -30,7 +30,7 @@ React ──> control-plane ── /api/browsers ──(Bearer token)──> doc 将 socket 以只读文件挂载**不会**限制 Docker API 的写操作；拥有 socket 等价于拥有宿主机 root 权限。因此： -- 只有 `docker-gateway` 挂载 socket，控制面和浏览器容器均不可见；网关只加入 control 网络，浏览器不能连接网关； +- 只有 `docker-gateway` 挂载 socket，控制面和浏览器容器均不可见；网关加入 control 与 browser 网络，浏览器只拿到无凭据的内存转发代理地址，`/v1` 仍必须通过容器内不可见的网关令牌； - 网关只暴露面向领域的路由，不提供通用 Docker 代理；`/v1` 全部接口校验 `Authorization: Bearer `（常数时间比较），令牌由部署者在网关环境变量与平台注册表中保持一致； - 网关固定命令、网络、挂载和资源限制；外部输入是受校验的别名，以及平台下发的镜像引用、启动参数和卷名——镜像引用来自平台维护的版本表，新增/变更由人工在页面审核启用，不再写死在代码中； - 启停和删除前必须同时匹配固定名称前缀及 `io.creatorhub.managed`、`io.creatorhub.runtime-id` 标签； @@ -59,4 +59,10 @@ DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up --build 草稿经 `POST /api/phase-a/confirmations` 显式确认后才可投递到 `/api/phase-a/tasks`。任务由幂等键去重；`POST /api/phase-a/mock/execute` 使用 `FOR UPDATE SKIP LOCKED` 领取一分钟租约，执行前统一核对账号、草稿和确认版本。缺少确认或版本不一致会进入 `needs_confirmation`，暂停账号或 Mock 策略结果会进入 `policy_hold`，不确定结果与过期租约进入 `needs_confirmation`；这些状态都不会自动重试。`GET /api/phase-a/audit` 只导出账号、确认版本、尝试和结果等非秘密证据。 -启动时控制面先应用 Phase A v1，再由 Hub runner 顺序应用 v2、v3；每一步都在事务和 advisory lock 下前向执行。v3 保留旧表、列和历史记录，旧账号回填为 `platform=mock` 并暂停，仅账号 ID 与环境 alias 相同的记录自动建立 binding；其余记录等待显式绑定。本阶段不提供破坏性自动回滚。 +启动时控制面先应用 Phase A v1，再由 Hub runner 顺序应用 v2、v3、v4；每一步都在事务和 advisory lock 下前向执行。v3 保留旧表、列和历史记录，旧账号回填为 `platform=mock` 并暂停，仅账号 ID 与环境 alias 相同的记录自动建立 binding；v4 只追加环境动作审计字段与索引。其余记录等待显式绑定。本阶段不提供破坏性自动回滚。 + +`POST /api/network-exits` 只接受协议、主机、端口、已有 `credential_reference: {id}` 和预期出口身份；新出口为 `unchecked`，由 `POST /api/network-exits/:id/check` 经实际代理链路变为 `healthy` 或 `unhealthy`，`disable` 不可被检查重新启用。credential reference 的 `reference_key` 不出现在 API、日志或审计中；OS Keyring/Secret Manager bridge 在控制面进程启动前注入 `CREATORHUB_CREDENTIAL_`（大写十六进制），值为请求期解析的 `username:password`，控制面不持久化解析值。 + +`POST /api/browsers` 必须同时给出 `account_id` 和 `network_exit_id`。环境创建、启动和升级都会重新检查出口身份，只有 `healthy` 才调用网关；控制面强制下发代理和 `disable_non_proxied_udp`，fingerprint 中的代理字段会被拒绝。显式 `POST /api/browsers/:alias/rebind` 只允许 paused、无 executing task 且无活动 runtime 的账号。`DELETE /api/browsers/:alias` 回收容器但保留稳定 binding、环境和命名 Profile 卷，后续 create 复用它们。create/start/stop/upgrade/recycle 均写共享 operation ID 的 requested/finished 审计对；网关断连且无法调和时 outcome 为 `unknown`。 + +解析后的出口凭据只存在于控制面单次请求和网关内存转发器中；Docker inspect、容器环境、标签、挂载、`Config.Cmd` 与进程参数只包含 `docker-gateway` 的无凭据本地代理地址。stopped 环境启动时先删除旧容器并确认 runtime lease 释放，再按当前 binding 重建；控制面每 20 秒及列表读取时调和网关，续租 running runtime、释放 stopped/missing runtime，过期 lease 也会在绑定事务中回收。 diff --git a/internal/hub/environment.go b/internal/hub/environment.go new file mode 100644 index 0000000..ad3216a --- /dev/null +++ b/internal/hub/environment.go @@ -0,0 +1,662 @@ +package hub + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "net" + "regexp" + "strings" + "time" +) + +var exitIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`) + +type CredentialReference struct { + ID string `json:"id"` + Provider string `json:"provider"` +} + +type NetworkExit struct { + ID string `json:"id"` + Protocol string `json:"protocol"` + Host string `json:"host"` + Port int `json:"port"` + CredentialReference *CredentialReference `json:"credential_reference,omitempty"` + ExpectedPublicIP string `json:"expected_public_ip,omitempty"` + ExpectedRegion string `json:"expected_region,omitempty"` + ObservedPublicIP string `json:"observed_public_ip,omitempty"` + ObservedRegion string `json:"observed_region,omitempty"` + HealthStatus string `json:"health_status"` + LastCheckReason string `json:"last_check_reason,omitempty"` + Version int64 `json:"version"` + LastCheckedAt *time.Time `json:"last_checked_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// NetworkExitAccess is internal-only: reference keys are never serialized or audited. +type NetworkExitAccess struct { + NetworkExit + CredentialKey string `json:"-"` +} + +type ExitObservation struct { + PublicIP string + Region string +} + +type EnvironmentContext struct { + Env + AccountID string `json:"account_id"` + BindingID string `json:"binding_id"` + BindingVersion int64 `json:"binding_version"` + RuntimeCleanupPending bool `json:"runtime_cleanup_pending,omitempty"` + Exit NetworkExit `json:"network_exit"` + RuntimeInstanceID string `json:"runtime_instance_id,omitempty"` + RuntimeID string `json:"runtime_id,omitempty"` +} + +type EnvironmentAction struct { + OperationID string + Action string + AccountID string + BrowserEnvAlias string + NetworkExitID string + RuntimeInstanceID string + BindingVersion int64 + OldImageVersion string + NewImageVersion string + Outcome string + ReasonCode string +} + +func (s *Store) CreateNetworkExit(ctx context.Context, exit NetworkExit, credentialReferenceID string) (NetworkExit, error) { + exit.ID = "exit-" + newHubID() + exit.Protocol, exit.Host = strings.ToLower(strings.TrimSpace(exit.Protocol)), strings.TrimSpace(exit.Host) + exit.ExpectedPublicIP, exit.ExpectedRegion = strings.TrimSpace(exit.ExpectedPublicIP), strings.TrimSpace(exit.ExpectedRegion) + credentialReferenceID = strings.TrimSpace(credentialReferenceID) + if !validNetworkExit(exit) || (credentialReferenceID != "" && !exitIDPattern.MatchString(credentialReferenceID)) { + return NetworkExit{}, ErrInvalid + } + row := s.db.QueryRowContext(ctx, ` + INSERT INTO network_exit (id, protocol, host, port, credential_reference_id, expected_public_ip, expected_region) + VALUES ($1, $2, $3, $4, NULLIF($5, ''), NULLIF($6, '')::inet, $7) + RETURNING id`, exit.ID, exit.Protocol, exit.Host, exit.Port, credentialReferenceID, exit.ExpectedPublicIP, exit.ExpectedRegion) + if err := row.Scan(&exit.ID); err != nil { + return NetworkExit{}, publicDatabaseError(err) + } + return s.GetNetworkExit(ctx, exit.ID) +} + +func validNetworkExit(exit NetworkExit) bool { + if exit.Protocol != "http" && exit.Protocol != "https" && exit.Protocol != "socks4" && exit.Protocol != "socks5" { + return false + } + if !validExitHost(exit.Host) || exit.Port < 1 || exit.Port > 65535 { + return false + } + if exit.ExpectedPublicIP != "" && net.ParseIP(exit.ExpectedPublicIP) == nil { + return false + } + return validOptionalRegion(exit.ExpectedRegion) +} + +func validExitHost(host string) bool { + if host == "" || len(host) > 253 || strings.ContainsAny(host, "@/[]?# \t\r\n") { + return false + } + if net.ParseIP(host) != nil { + return true + } + if strings.HasPrefix(host, ".") || strings.HasSuffix(host, ".") || strings.Contains(host, "..") { + return false + } + for _, label := range strings.Split(host, ".") { + if len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { + return false + } + for _, character := range label { + if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && character != '-' { + return false + } + } + } + return true +} + +func validOptionalRegion(region string) bool { + if len(region) > 64 { + return false + } + for _, character := range region { + if character < 0x20 || character == 0x7f { + return false + } + } + return true +} + +func (s *Store) ListNetworkExits(ctx context.Context) ([]NetworkExit, error) { + rows, err := s.db.QueryContext(ctx, networkExitSelect+` ORDER BY network.created_at, network.id`) + if err != nil { + return nil, errors.New("read network exits") + } + defer rows.Close() + exits := []NetworkExit{} + for rows.Next() { + exit, err := scanNetworkExit(rows) + if err != nil { + return nil, err + } + exits = append(exits, exit) + } + return exits, rows.Err() +} + +func (s *Store) GetNetworkExit(ctx context.Context, id string) (NetworkExit, error) { + if !exitIDPattern.MatchString(id) { + return NetworkExit{}, ErrInvalid + } + return scanNetworkExit(s.db.QueryRowContext(ctx, networkExitSelect+` WHERE network.id = $1`, id)) +} + +func (s *Store) GetNetworkExitAccess(ctx context.Context, id string) (NetworkExitAccess, error) { + exit, err := s.GetNetworkExit(ctx, id) + if err != nil { + return NetworkExitAccess{}, err + } + access := NetworkExitAccess{NetworkExit: exit} + if exit.CredentialReference != nil { + if err := s.db.QueryRowContext(ctx, `SELECT reference_key FROM credential_reference WHERE id = $1`, exit.CredentialReference.ID). + Scan(&access.CredentialKey); err != nil { + return NetworkExitAccess{}, rowError(err) + } + } + return access, nil +} + +const networkExitSelect = ` + SELECT network.id, network.protocol, network.host, network.port, + reference.id, reference.provider, + COALESCE(host(network.expected_public_ip), ''), network.expected_region, + COALESCE(host(network.observed_public_ip), ''), network.observed_region, + network.health_status, COALESCE(network.last_check_reason, ''), network.version, network.last_checked_at, + network.created_at, network.updated_at + FROM network_exit network + LEFT JOIN credential_reference reference ON reference.id = network.credential_reference_id` + +type rowScanner interface{ Scan(...any) error } + +func scanNetworkExit(row rowScanner) (NetworkExit, error) { + var exit NetworkExit + var referenceID, provider sql.NullString + var checked sql.NullTime + if err := row.Scan(&exit.ID, &exit.Protocol, &exit.Host, &exit.Port, &referenceID, &provider, + &exit.ExpectedPublicIP, &exit.ExpectedRegion, &exit.ObservedPublicIP, &exit.ObservedRegion, + &exit.HealthStatus, &exit.LastCheckReason, &exit.Version, &checked, &exit.CreatedAt, &exit.UpdatedAt); err != nil { + return NetworkExit{}, rowError(err) + } + if referenceID.Valid { + exit.CredentialReference = &CredentialReference{ID: referenceID.String, Provider: provider.String} + } + if checked.Valid { + exit.LastCheckedAt = &checked.Time + } + return exit, nil +} + +// RecordNetworkExitCheck stores only observed identity and a stable reason code. +func (s *Store) RecordNetworkExitCheck(ctx context.Context, id string, observation ExitObservation, failureReason string) (NetworkExit, string, error) { + if !exitIDPattern.MatchString(id) || !validOptionalRegion(observation.Region) || + (observation.PublicIP != "" && net.ParseIP(observation.PublicIP) == nil) || !validExitFailureReason(failureReason) { + return NetworkExit{}, "invalid_observation", ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return NetworkExit{}, "persistence_failed", errors.New("begin network exit check") + } + defer tx.Rollback() + var expectedIP, expectedRegion, oldIP, oldRegion, oldStatus string + var version int64 + if err := tx.QueryRowContext(ctx, ` + SELECT COALESCE(host(expected_public_ip), ''), expected_region, + COALESCE(host(observed_public_ip), ''), observed_region, health_status, version + FROM network_exit WHERE id = $1 FOR UPDATE`, id). + Scan(&expectedIP, &expectedRegion, &oldIP, &oldRegion, &oldStatus, &version); err != nil { + return NetworkExit{}, "persistence_failed", rowError(err) + } + if oldStatus == "disabled" { + return NetworkExit{}, "exit_disabled", ErrConflict + } + reason, status := strings.TrimSpace(failureReason), "unhealthy" + if reason == "" && expectedIP != "" && !net.ParseIP(expectedIP).Equal(net.ParseIP(observation.PublicIP)) { + reason = "exit_ip_drift" + } + if reason == "" && expectedRegion != "" && !strings.EqualFold(expectedRegion, observation.Region) { + reason = "exit_region_drift" + } + if reason == "" { + reason, status = "exit_healthy", "healthy" + } + changed := !sameIP(oldIP, observation.PublicIP) || !strings.EqualFold(oldRegion, observation.Region) || oldStatus != status + if changed { + version++ + } + if _, err := tx.ExecContext(ctx, ` + UPDATE network_exit SET observed_public_ip = NULLIF($2, '')::inet, observed_region = $3, + health_status = $4, last_check_reason = $5, version = $6, last_checked_at = now(), updated_at = now() + WHERE id = $1`, id, observation.PublicIP, observation.Region, status, reason, version); err != nil { + return NetworkExit{}, "persistence_failed", errors.New("record network exit check") + } + if changed { + if err := invalidateAccountsForExit(ctx, tx, id); err != nil { + return NetworkExit{}, "persistence_failed", err + } + } + if err := commitHub(tx); err != nil { + return NetworkExit{}, "persistence_failed", err + } + exit, err := s.GetNetworkExit(ctx, id) + return exit, reason, err +} + +func validExitFailureReason(reason string) bool { + switch reason { + case "", "credential_unavailable", "credential_invalid", "proxy_auth_failed", "proxy_check_failed", "exit_observation_invalid": + return true + default: + return false + } +} + +func sameIP(left, right string) bool { + if left == "" || right == "" { + return left == right + } + return net.ParseIP(left).Equal(net.ParseIP(right)) +} + +func (s *Store) DisableNetworkExit(ctx context.Context, id string) (NetworkExit, error) { + if !exitIDPattern.MatchString(id) { + return NetworkExit{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return NetworkExit{}, errors.New("begin network exit disable") + } + defer tx.Rollback() + var oldStatus string + if err := tx.QueryRowContext(ctx, `SELECT health_status FROM network_exit WHERE id = $1 FOR UPDATE`, id).Scan(&oldStatus); err != nil { + return NetworkExit{}, rowError(err) + } + if oldStatus != "disabled" { + if _, err := tx.ExecContext(ctx, ` + UPDATE network_exit SET health_status = 'disabled', last_check_reason = 'exit_disabled', + version = version + 1, updated_at = now() + WHERE id = $1`, id); err != nil { + return NetworkExit{}, errors.New("disable network exit") + } + if err := invalidateAccountsForExit(ctx, tx, id); err != nil { + return NetworkExit{}, err + } + } + if err := commitHub(tx); err != nil { + return NetworkExit{}, err + } + return s.GetNetworkExit(ctx, id) +} + +func invalidateAccountsForExit(ctx context.Context, tx *sql.Tx, exitID string) error { + if _, err := tx.ExecContext(ctx, ` + WITH changed AS ( + UPDATE social_account account SET status = 'paused', paused_at = COALESCE(paused_at, now()), + version = account.version + 1, updated_at = now() + FROM environment_binding binding + WHERE binding.network_exit_id = $1 AND binding.account_id = account.id + RETURNING account.id + ) + UPDATE operation_task task SET state = 'policy_hold', updated_at = now() + FROM changed WHERE task.account_id = changed.id AND task.state = 'queued'`, exitID); err != nil { + return errors.New("invalidate network exit accounts") + } + return nil +} + +func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID string) (EnvironmentContext, bool, error) { + env.Alias, env.Name = strings.TrimSpace(env.Alias), strings.TrimSpace(env.Name) + if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || !aliasPattern.MatchString(accountID) || + !exitIDPattern.MatchString(exitID) || !gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) || + env.Fingerprint.ProxyServer != "" { + return EnvironmentContext{}, false, ErrInvalid + } + if err := env.Fingerprint.Validate(); err != nil { + return EnvironmentContext{}, false, ErrInvalid + } + encoded, _ := json.Marshal(env.Fingerprint) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return EnvironmentContext{}, false, errors.New("begin bound environment create") + } + defer tx.Rollback() + var existingAlias, existingExit string + err = tx.QueryRowContext(ctx, `SELECT browser_env_alias, COALESCE(network_exit_id, '') FROM environment_binding WHERE account_id = $1 FOR UPDATE`, accountID). + Scan(&existingAlias, &existingExit) + if err == nil { + if existingAlias != env.Alias || existingExit != exitID { + return EnvironmentContext{}, false, ErrConflict + } + if err := tx.Commit(); err != nil { + return EnvironmentContext{}, false, errors.New("commit existing environment lookup") + } + context, err := s.GetEnvironmentContext(ctx, env.Alias) + if err != nil || context.Name != env.Name || context.Gateway != env.Gateway || context.ImageVersion != env.ImageVersion || context.Fingerprint != env.Fingerprint { + return EnvironmentContext{}, false, ErrConflict + } + return context, false, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return EnvironmentContext{}, false, publicDatabaseError(err) + } + var created string + if err := tx.QueryRowContext(ctx, ` + INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + SELECT $1, $2, $3, image.version, $5 + FROM browser_image image, social_account account, network_exit network + WHERE image.version = $4 AND image.enabled AND account.id = $6 AND account.status = 'paused' + AND account.authorization_status = 'authorized' AND network.id = $7 AND network.health_status = 'healthy' + RETURNING alias`, env.Alias, env.Name, env.Gateway, env.ImageVersion, encoded, accountID, exitID).Scan(&created); err != nil { + return EnvironmentContext{}, false, rowError(err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO environment_binding (id, account_id, browser_env_alias, network_exit_id) + VALUES ($1, $1, $2, $3)`, accountID, env.Alias, exitID); err != nil { + return EnvironmentContext{}, false, publicDatabaseError(err) + } + if _, err := tx.ExecContext(ctx, `UPDATE social_account SET version = version + 1, updated_at = now() WHERE id = $1`, accountID); err != nil { + return EnvironmentContext{}, false, errors.New("version bound account") + } + if err := commitHub(tx); err != nil { + return EnvironmentContext{}, false, err + } + context, err := s.GetEnvironmentContext(ctx, env.Alias) + return context, true, err +} + +func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (EnvironmentContext, error) { + if !aliasPattern.MatchString(alias) { + return EnvironmentContext{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return EnvironmentContext{}, errors.New("begin environment context read") + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, ` + UPDATE runtime_instance runtime SET released_at = now() + FROM environment_binding binding + WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id + AND runtime.released_at IS NULL AND runtime.lease_until <= now()`, alias); err != nil { + return EnvironmentContext{}, errors.New("expire environment runtime") + } + var result EnvironmentContext + var encoded []byte + var expectedIP, observedIP string + var checked sql.NullTime + var runtimeInstanceID, runtimeID sql.NullString + err = tx.QueryRowContext(ctx, ` + SELECT environment.alias, environment.name, environment.gateway_name, environment.image_version, + environment.fingerprint, environment.created_at, binding.account_id, binding.id, binding.version, + binding.runtime_cleanup_pending, + COALESCE(network.id, ''), COALESCE(network.protocol, ''), COALESCE(network.host, ''), COALESCE(network.port, 0), + COALESCE(host(network.expected_public_ip), ''), COALESCE(network.expected_region, ''), + COALESCE(host(network.observed_public_ip), ''), COALESCE(network.observed_region, ''), + COALESCE(network.health_status, 'unchecked'), COALESCE(network.last_check_reason, ''), + COALESCE(network.version, 0), network.last_checked_at, + COALESCE(network.created_at, to_timestamp(0)), COALESCE(network.updated_at, to_timestamp(0)), + runtime.id, runtime.runtime_id + FROM browser_env environment + JOIN environment_binding binding ON binding.browser_env_alias = environment.alias + LEFT JOIN network_exit network ON network.id = binding.network_exit_id + LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL + WHERE environment.alias = $1`, alias). + Scan(&result.Alias, &result.Name, &result.Gateway, &result.ImageVersion, &encoded, &result.CreatedAt, + &result.AccountID, &result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending, + &result.Exit.ID, &result.Exit.Protocol, &result.Exit.Host, &result.Exit.Port, + &expectedIP, &result.Exit.ExpectedRegion, &observedIP, &result.Exit.ObservedRegion, + &result.Exit.HealthStatus, &result.Exit.LastCheckReason, &result.Exit.Version, &checked, &result.Exit.CreatedAt, &result.Exit.UpdatedAt, + &runtimeInstanceID, &runtimeID) + if err != nil { + return EnvironmentContext{}, rowError(err) + } + if err := json.Unmarshal(encoded, &result.Fingerprint); err != nil { + return EnvironmentContext{}, errors.New("decode bound environment fingerprint") + } + result.Fingerprint.ProxyServer = "" + result.Fingerprint.DisableNonProxiedUDP = false + result.Exit.ExpectedPublicIP, result.Exit.ObservedPublicIP = expectedIP, observedIP + if checked.Valid { + result.Exit.LastCheckedAt = &checked.Time + } + result.RuntimeInstanceID, result.RuntimeID = runtimeInstanceID.String, runtimeID.String + if err := commitHub(tx); err != nil { + return EnvironmentContext{}, err + } + return result, nil +} + +func validateEnvironmentRebind(ctx context.Context, tx *sql.Tx, alias, exitID string, expectedBindingVersion int64) (string, string, error) { + var accountID, bindingID string + var bindingVersion int64 + err := tx.QueryRowContext(ctx, ` + SELECT binding.account_id, binding.id, binding.version + FROM environment_binding binding + JOIN social_account account ON account.id = binding.account_id + WHERE binding.browser_env_alias = $1 AND account.status = 'paused' + AND account.authorization_status = 'authorized' + AND NOT binding.runtime_cleanup_pending + FOR UPDATE OF binding, account`, alias).Scan(&accountID, &bindingID, &bindingVersion) + if errors.Is(err, sql.ErrNoRows) { + return "", "", ErrConflict + } + if err != nil { + return "", "", publicDatabaseError(err) + } + if bindingVersion != expectedBindingVersion { + return "", "", ErrConflict + } + if _, err := tx.ExecContext(ctx, ` + UPDATE runtime_instance SET released_at = now() + WHERE binding_id = $1 AND released_at IS NULL AND lease_until <= now()`, bindingID); err != nil { + return "", "", errors.New("expire runtime before rebind") + } + var allowed bool + if err := tx.QueryRowContext(ctx, ` + SELECT EXISTS (SELECT 1 FROM network_exit WHERE id = $1 AND health_status = 'healthy') + AND NOT EXISTS (SELECT 1 FROM operation_task WHERE account_id = $2 AND state = 'executing') + AND NOT EXISTS (SELECT 1 FROM runtime_instance WHERE binding_id = $3 AND released_at IS NULL)`, + exitID, accountID, bindingID).Scan(&allowed); err != nil { + return "", "", errors.New("check environment rebind") + } + if !allowed { + return "", "", ErrConflict + } + return accountID, bindingID, nil +} + +func (s *Store) ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error { + if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(exitID) || expectedBindingVersion < 1 { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin environment rebind validation") + } + defer tx.Rollback() + if _, _, err := validateEnvironmentRebind(ctx, tx, alias, exitID, expectedBindingVersion); err != nil { + return err + } + if err := commitHub(tx); err != nil { + return err + } + return nil +} + +func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (EnvironmentContext, error) { + if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(exitID) || + (runtimeID != "" && !exitIDPattern.MatchString(runtimeID)) || expectedBindingVersion < 1 { + return EnvironmentContext{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return EnvironmentContext{}, errors.New("begin environment rebind") + } + defer tx.Rollback() + accountID, bindingID, err := validateEnvironmentRebind(ctx, tx, alias, exitID, expectedBindingVersion) + if err != nil { + return EnvironmentContext{}, err + } + if _, err := tx.ExecContext(ctx, `UPDATE environment_binding SET network_exit_id = $2, version = version + 1, updated_at = now() WHERE browser_env_alias = $1`, alias, exitID); err != nil { + return EnvironmentContext{}, errors.New("update environment binding") + } + if _, err := tx.ExecContext(ctx, `UPDATE social_account SET version = version + 1, updated_at = now() WHERE id = $1`, accountID); err != nil { + return EnvironmentContext{}, errors.New("version rebound account") + } + if runtimeID != "" { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) + VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil { + return EnvironmentContext{}, publicDatabaseError(err) + } + } + if err := commitHub(tx); err != nil { + return EnvironmentContext{}, err + } + return s.GetEnvironmentContext(ctx, alias) +} + +func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (EnvironmentContext, error) { + if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || bindingVersion < 1 || !exitIDPattern.MatchString(exitID) { + return EnvironmentContext{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return EnvironmentContext{}, errors.New("begin runtime activation") + } + defer tx.Rollback() + var accountID, bindingID, currentExitID string + var currentBindingVersion int64 + var cleanupPending bool + err = tx.QueryRowContext(ctx, ` + SELECT binding.account_id, binding.id, binding.version, COALESCE(binding.network_exit_id, ''), binding.runtime_cleanup_pending + FROM environment_binding binding + WHERE binding.browser_env_alias = $1 FOR UPDATE OF binding`, alias). + Scan(&accountID, &bindingID, ¤tBindingVersion, ¤tExitID, &cleanupPending) + if err != nil { + return EnvironmentContext{}, rowError(err) + } + if cleanupPending || currentBindingVersion != bindingVersion || currentExitID != exitID { + return EnvironmentContext{}, ErrConflict + } + if _, err := tx.ExecContext(ctx, ` + UPDATE runtime_instance SET released_at = now() + WHERE binding_id = $1 AND released_at IS NULL AND lease_until <= now()`, bindingID); err != nil { + return EnvironmentContext{}, errors.New("expire runtime before activation") + } + var existingInstanceID, existingRuntimeID string + if err := tx.QueryRowContext(ctx, ` + SELECT COALESCE(id, ''), COALESCE(runtime_id, '') FROM runtime_instance + WHERE binding_id = $1 AND released_at IS NULL`, bindingID).Scan(&existingInstanceID, &existingRuntimeID); err != nil && !errors.Is(err, sql.ErrNoRows) { + return EnvironmentContext{}, publicDatabaseError(err) + } + if existingInstanceID != "" && existingRuntimeID != runtimeID { + return EnvironmentContext{}, ErrConflict + } + if existingInstanceID == "" { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until) + VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil { + return EnvironmentContext{}, publicDatabaseError(err) + } + } else if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 minute' WHERE id = $1`, existingInstanceID); err != nil { + return EnvironmentContext{}, errors.New("renew environment runtime") + } + if err := commitHub(tx); err != nil { + return EnvironmentContext{}, err + } + return s.GetEnvironmentContext(ctx, alias) +} + +func (s *Store) ReleaseRuntime(ctx context.Context, alias string) error { + if !aliasPattern.MatchString(alias) { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, ` + UPDATE runtime_instance runtime SET released_at = now() + FROM environment_binding binding + WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id AND runtime.released_at IS NULL`, alias) + if err != nil { + return errors.New("release environment runtime") + } + return nil +} + +func (s *Store) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error { + if !aliasPattern.MatchString(alias) { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin runtime cleanup state update") + } + defer tx.Rollback() + var bindingID string + if err := tx.QueryRowContext(ctx, ` + UPDATE environment_binding SET runtime_cleanup_pending = $2, updated_at = now() + WHERE browser_env_alias = $1 RETURNING id`, alias, pending).Scan(&bindingID); err != nil { + return rowError(err) + } + if pending { + if _, err := tx.ExecContext(ctx, ` + UPDATE runtime_instance SET released_at = now() + WHERE binding_id = $1 AND released_at IS NULL`, bindingID); err != nil { + return errors.New("release runtime for pending cleanup") + } + } + return commitHub(tx) +} + +func (s *Store) AppendEnvironmentAction(ctx context.Context, eventType string, action EnvironmentAction) error { + if (eventType != "environment_action_requested" && eventType != "environment_action_finished") || + !exitIDPattern.MatchString(action.OperationID) || action.Action == "" || action.ReasonCode == "" || + (action.OldImageVersion != "" && !imageVersionPattern.MatchString(action.OldImageVersion)) || + (action.NewImageVersion != "" && !imageVersionPattern.MatchString(action.NewImageVersion)) || + (eventType == "environment_action_finished" && action.Outcome != "succeeded" && action.Outcome != "failed" && action.Outcome != "unknown") { + return ErrInvalid + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO audit_event + (event_type, account_id, browser_env_alias, network_exit_id, runtime_instance_id, + binding_version, actor, reason_code, operation_id, action, outcome, old_image_version, new_image_version) + VALUES ($1, NULLIF($2, ''), NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''), + NULLIF($6, 0), 'local-user', $7, $8, $9, NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, ''))`, + eventType, action.AccountID, action.BrowserEnvAlias, action.NetworkExitID, action.RuntimeInstanceID, + action.BindingVersion, action.ReasonCode, action.OperationID, action.Action, action.Outcome, + action.OldImageVersion, action.NewImageVersion) + if err != nil { + return errors.New("append environment action") + } + return nil +} + +func NewOperationID() string { return "operation-" + newHubID() } + +func newHubID() string { + var value [12]byte + _, _ = rand.Read(value[:]) + return hex.EncodeToString(value[:]) +} diff --git a/internal/hub/migration_test.go b/internal/hub/migration_test.go index 2eb1073..4197556 100644 --- a/internal/hub/migration_test.go +++ b/internal/hub/migration_test.go @@ -29,12 +29,13 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal(err) } defer db.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3)`, 3) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6) assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('social_account', 'browser_env', 'network_exit', 'environment_binding')`, 4) + assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name = 'runtime_cleanup_pending'`, 1) store = openFullyMigratedHub(t, ctx, testURL) store.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3)`, 3) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6) }) t.Run("v1 and v2 data", func(t *testing.T) { @@ -72,7 +73,7 @@ func TestUnifiedAccountMigration(t *testing.T) { INSERT INTO gateway (name, endpoint, token) VALUES ('legacy-gateway', 'http://127.0.0.1:8081', 'legacy-gateway-token'); INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1'); INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) VALUES - ('mapped', 'Mapped', 'legacy-gateway', '1', '{"seed":1}'), + ('mapped', 'Mapped', 'legacy-gateway', '1', '{"seed":1,"proxy_server":"http://legacy:secret@proxy.example:8080","disable_non_proxied_udp":true}'), ('orphan-env', 'Orphan', 'legacy-gateway', '1', '{"seed":2}'); INSERT INTO runtime_instance (id, account_id, runtime_id, lease_until) VALUES ('instance-mapped', 'mapped', 'runtime-mapped', now() + interval '1 hour'), @@ -90,11 +91,22 @@ func TestUnifiedAccountMigration(t *testing.T) { assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE platform = 'mock' AND platform_account_key = id AND status = 'paused'`, 2) assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE profile_id LIKE 'legacy-profile-%'`, 2) assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'mapped' AND browser_env_alias = 'mapped' AND network_exit_id IS NULL`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'mapped' AND NOT runtime_cleanup_pending`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'unbound'`, 0) assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-mapped' AND binding_id = 'mapped'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-unbound' AND binding_id IS NULL`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM audit_event WHERE event_type = 'legacy_event'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'legacy-task' AND state = 'policy_hold'`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND NOT (fingerprint ?| ARRAY['proxy_server', 'disable_non_proxied_udp'])`, 1) + store, err = Open(ctx, testURL) + if err != nil { + t.Fatal(err) + } + legacy, err := store.GetEnvironmentContext(ctx, "mapped") + store.Close() + if err != nil || legacy.Exit.ID != "" || legacy.Fingerprint.ProxyServer != "" || legacy.Fingerprint.DisableNonProxiedUDP { + t.Fatalf("legacy NULL binding must remain visible without persisted proxy credentials: %#v err=%v", legacy, err) + } if _, err := db.Exec(` INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-duplicate', 'os_keyring', 'creatorhub/duplicate'); @@ -154,6 +166,7 @@ func TestUnifiedAccountMigration(t *testing.T) { } store.Close() assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND version = 2 AND image_version = '2'`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE id = 'mapped' AND version = 2`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE id = 'mapped' AND version = 2 AND status = 'paused'`, 1) assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'upgrade-task' AND state = 'policy_hold'`, 1) @@ -162,7 +175,7 @@ func TestUnifiedAccountMigration(t *testing.T) { t.Fatal(err) } store.Close() - assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version = 3`, 1) + assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (3, 4, 5, 6)`, 4) }) } diff --git a/internal/hub/migrations/004_environment_actions.sql b/internal/hub/migrations/004_environment_actions.sql new file mode 100644 index 0000000..1540d22 --- /dev/null +++ b/internal/hub/migrations/004_environment_actions.sql @@ -0,0 +1,12 @@ +ALTER TABLE audit_event + ADD COLUMN operation_id text, + ADD COLUMN action text, + ADD COLUMN outcome text CHECK (outcome IN ('succeeded', 'failed', 'unknown')), + ADD COLUMN old_image_version text, + ADD COLUMN new_image_version text; + +ALTER TABLE network_exit + ADD COLUMN last_check_reason text; + +CREATE INDEX audit_event_operation_id_idx + ON audit_event (operation_id) WHERE operation_id IS NOT NULL; diff --git a/internal/hub/migrations/005_sanitize_legacy_proxy.sql b/internal/hub/migrations/005_sanitize_legacy_proxy.sql new file mode 100644 index 0000000..4bb356c --- /dev/null +++ b/internal/hub/migrations/005_sanitize_legacy_proxy.sql @@ -0,0 +1,3 @@ +UPDATE browser_env +SET fingerprint = fingerprint - 'proxy_server' - 'disable_non_proxied_udp' +WHERE fingerprint ?| ARRAY['proxy_server', 'disable_non_proxied_udp']; diff --git a/internal/hub/migrations/006_runtime_cleanup.sql b/internal/hub/migrations/006_runtime_cleanup.sql new file mode 100644 index 0000000..33c86a2 --- /dev/null +++ b/internal/hub/migrations/006_runtime_cleanup.sql @@ -0,0 +1,2 @@ +ALTER TABLE environment_binding + ADD COLUMN runtime_cleanup_pending boolean NOT NULL DEFAULT false; diff --git a/internal/hub/store.go b/internal/hub/store.go index e521a10..bc33442 100644 --- a/internal/hub/store.go +++ b/internal/hub/store.go @@ -25,12 +25,23 @@ var migration002 string //go:embed migrations/003_unified_accounts.sql var migration003 string +//go:embed migrations/004_environment_actions.sql +var migration004 string + +//go:embed migrations/005_sanitize_legacy_proxy.sql +var migration005 string + +//go:embed migrations/006_runtime_cleanup.sql +var migration006 string + var ( ErrConflict = errors.New("resource conflicts with existing state") ErrInvalid = errors.New("invalid hub input") ErrNotFound = errors.New("resource not found") ) +func ValidImageVersion(version string) bool { return imageVersionPattern.MatchString(version) } + var ( aliasPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) gatewayNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) @@ -107,7 +118,7 @@ func (s *Store) migrate(ctx context.Context) error { for _, migration := range []struct { version int sql string - }{{2, migration002}, {3, migration003}} { + }{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}} { var applied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil { return errors.New("read hub schema migration state") @@ -273,7 +284,8 @@ func (s *Store) CreateEnv(ctx context.Context, env Env) error { env.Alias = strings.TrimSpace(env.Alias) env.Name = strings.TrimSpace(env.Name) if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || - !gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) { + !gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) || + env.Fingerprint.ProxyServer != "" { return ErrInvalid } if err := env.Fingerprint.Validate(); err != nil { @@ -341,6 +353,15 @@ func (s *Store) UpgradeEnv(ctx context.Context, alias, version string) error { return errors.New("begin environment upgrade") } defer tx.Rollback() + var cleanupPending bool + if err := tx.QueryRowContext(ctx, ` + SELECT runtime_cleanup_pending FROM environment_binding + WHERE browser_env_alias = $1 FOR UPDATE`, alias).Scan(&cleanupPending); err != nil && !errors.Is(err, sql.ErrNoRows) { + return publicDatabaseError(err) + } + if cleanupPending { + return ErrConflict + } var updated string if err := tx.QueryRowContext(ctx, ` UPDATE browser_env SET image_version = $2, version = version + 1 @@ -348,6 +369,11 @@ func (s *Store) UpgradeEnv(ctx context.Context, alias, version string) error { RETURNING alias`, alias, version).Scan(&updated); err != nil { return rowError(err) } + if _, err := tx.ExecContext(ctx, ` + UPDATE environment_binding SET version = version + 1, updated_at = now() + WHERE browser_env_alias = $1`, alias); err != nil { + return errors.New("version upgraded environment binding") + } if _, err := tx.ExecContext(ctx, ` WITH changed AS ( UPDATE social_account account @@ -385,6 +411,8 @@ func scanEnv(rows *sql.Rows) (Env, error) { return Env{}, errors.New("decode env fingerprint") } } + env.Fingerprint.ProxyServer = "" + env.Fingerprint.DisableNonProxiedUDP = false return env, nil } diff --git a/internal/hub/store_test.go b/internal/hub/store_test.go index cb7cdb5..5fb18ad 100644 --- a/internal/hub/store_test.go +++ b/internal/hub/store_test.go @@ -113,6 +113,22 @@ func TestStoreValidationRejectsInvalidInputsBeforePersistence(t *testing.T) { if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 0}}); !errors.Is(err, ErrInvalid) { t.Fatalf("expected invalid fingerprint, got %v", err) } + for name, exit := range map[string]NetworkExit{ + "protocol": {Protocol: "direct", Host: "proxy.example", Port: 1080}, + "userinfo": {Protocol: "socks5", Host: "user@proxy.example", Port: 1080}, + "URL host": {Protocol: "socks5", Host: "socks5://proxy.example", Port: 1080}, + "port": {Protocol: "socks5", Host: "proxy.example", Port: 0}, + "ip": {Protocol: "socks5", Host: "proxy.example", Port: 1080, ExpectedPublicIP: "not-an-ip"}, + } { + t.Run("network exit "+name, func(t *testing.T) { + if _, err := store.CreateNetworkExit(ctx, exit, ""); !errors.Is(err, ErrInvalid) { + t.Fatalf("expected invalid network exit, got %v", err) + } + }) + } + if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1, ProxyServer: "socks5://proxy.example:1080"}}); !errors.Is(err, ErrInvalid) { + t.Fatalf("stored fingerprint proxy must be rejected, got %v", err) + } } func TestHubWorkflow(t *testing.T) { @@ -224,3 +240,158 @@ func TestHubWorkflow(t *testing.T) { t.Fatal(err) } } + +func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) { + databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL") + if databaseURL == "" { + t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage") + } + ctx := context.Background() + store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL)) + t.Cleanup(func() { _ = store.Close() }) + if _, err := store.db.ExecContext(ctx, `TRUNCATE audit_event, runtime_instance, environment_binding, network_exit, + social_account, credential_reference, browser_env, browser_image, gateway CASCADE`); err != nil { + t.Fatal(err) + } + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO credential_reference (id, provider, reference_key) + VALUES ('credential-exit', 'os_keyring', 'creatorhub/proxy-main'), + ('credential-account', 'os_keyring', 'creatorhub/account-a'); + INSERT INTO social_account + (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) + VALUES ('account-a', 'credential-account', 'mock', 'account-a', 'owned', 'authorized')`); err != nil { + t.Fatal(err) + } + if _, err := store.CreateGateway(ctx, "gw-main", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil { + t.Fatal(err) + } + if err := store.CreateImage(ctx, Image{Version: "148", ImageRef: "example/browser:148", Enabled: true}); err != nil { + t.Fatal(err) + } + + exit, err := store.CreateNetworkExit(ctx, NetworkExit{ + Protocol: "socks5", Host: "proxy.example", Port: 1080, + ExpectedPublicIP: "203.0.113.10", ExpectedRegion: "test-region", + }, "credential-exit") + if err != nil || exit.HealthStatus != "unchecked" || exit.CredentialReference == nil || exit.CredentialReference.ID != "credential-exit" { + t.Fatalf("unexpected network exit: %#v err=%v", exit, err) + } + exported, _ := json.Marshal(exit) + if strings.Contains(string(exported), "creatorhub/proxy-main") { + t.Fatalf("network exit response leaked a credential reference key: %s", exported) + } + access, err := store.GetNetworkExitAccess(ctx, exit.ID) + if err != nil || access.CredentialKey != "creatorhub/proxy-main" { + t.Fatalf("runtime-only credential resolution data unavailable: %#v err=%v", access, err) + } + + exit, reason, err := store.RecordNetworkExitCheck(ctx, exit.ID, ExitObservation{PublicIP: "203.0.113.11", Region: "test-region"}, "") + if err != nil || exit.HealthStatus != "unhealthy" || reason != "exit_ip_drift" { + t.Fatalf("identity drift must make the exit unhealthy: %#v reason=%s err=%v", exit, reason, err) + } + exit, reason, err = store.RecordNetworkExitCheck(ctx, exit.ID, ExitObservation{PublicIP: "203.0.113.10", Region: "test-region"}, "") + if err != nil || exit.HealthStatus != "healthy" || reason != "exit_healthy" { + t.Fatalf("matching identity must make the exit healthy: %#v reason=%s err=%v", exit, reason, err) + } + + env := Env{Alias: "environment-a", Name: "环境 A", Gateway: "gw-main", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1}} + bound, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID) + if err != nil || !created || bound.AccountID != "account-a" || bound.Exit.ID != exit.ID { + t.Fatalf("create stable binding: %#v created=%v err=%v", bound, created, err) + } + reused, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID) + if err != nil || created || reused.Alias != bound.Alias || reused.BindingID != bound.BindingID { + t.Fatalf("same account must reuse its environment: %#v created=%v err=%v", reused, created, err) + } + active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID) + if err != nil || active.RuntimeInstanceID == "" { + t.Fatalf("activate runtime: %#v err=%v", active, err) + } + + second, err := store.CreateNetworkExit(ctx, NetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}, "") + if err != nil { + t.Fatal(err) + } + second, _, err = store.RecordNetworkExitCheck(ctx, second.ID, ExitObservation{PublicIP: "198.51.100.2", Region: "other"}, "") + if err != nil || second.HealthStatus != "healthy" { + t.Fatalf("prepare second exit: %#v err=%v", second, err) + } + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-account-b', 'os_keyring', 'creatorhub/account-b'); + INSERT INTO social_account + (id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status) + VALUES ('account-b', 'credential-account-b', 'mock', 'account-b', 'owned', 'authorized'); + INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) + VALUES ('environment-b', '环境 B', 'gw-main', '148', '{"seed":2}'); + INSERT INTO environment_binding (id, account_id, browser_env_alias) + VALUES ('binding-b', 'account-b', 'environment-b')`); err != nil { + t.Fatal(err) + } + legacyRebound, err := store.RebindEnvironment(ctx, "environment-b", second.ID, "", 1) + if err != nil || legacyRebound.Exit.ID != second.ID { + t.Fatalf("legacy binding without an exit must support explicit rebind: %#v err=%v", legacyRebound, err) + } + if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID); !errors.Is(err, ErrConflict) { + t.Fatalf("stale binding metadata must not activate a runtime: %v", err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil { + t.Fatal(err) + } + if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID); err != nil { + t.Fatalf("runtime heartbeat failed: %v", err) + } + var renewed bool + if err := store.db.QueryRowContext(ctx, `SELECT lease_until > now() + interval '30 seconds' FROM runtime_instance WHERE id = $1`, active.RuntimeInstanceID).Scan(&renewed); err != nil || !renewed { + t.Fatalf("runtime lease was not renewed: renewed=%v err=%v", renewed, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil { + t.Fatal(err) + } + rebound, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", bound.BindingVersion) + if err != nil || rebound.Exit.ID != second.ID || rebound.BindingVersion != 2 { + t.Fatalf("expired runtime must be transactionally released before rebind: %#v err=%v", rebound, err) + } + if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID); err != nil { + t.Fatalf("activate runtime before same-exit rebind: %v", err) + } + if _, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", rebound.BindingVersion); !errors.Is(err, ErrConflict) { + t.Fatalf("active runtime must block same-exit rebind: %v", err) + } + if err := store.ReleaseRuntime(ctx, env.Alias); err != nil { + t.Fatal(err) + } + rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-container", rebound.BindingVersion) + if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-container" { + t.Fatalf("same-exit rebind must atomically CAS the binding and runtime: %#v err=%v", rebound, err) + } + + action := EnvironmentAction{ + OperationID: NewOperationID(), Action: "start", AccountID: rebound.AccountID, + BrowserEnvAlias: rebound.Alias, NetworkExitID: rebound.Exit.ID, BindingVersion: rebound.BindingVersion, + ReasonCode: "action_requested", + } + if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil { + t.Fatal(err) + } + action.Outcome, action.ReasonCode = "succeeded", "environment_started" + if err := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); err != nil { + t.Fatal(err) + } + assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+action.OperationID+`'`, 2) + invalidAction := action + invalidAction.OperationID = NewOperationID() + invalidAction.NewImageVersion = "http://operator:secret@proxy.example" + if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", invalidAction); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid image version must not reach audit persistence: %v", err) + } + assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+invalidAction.OperationID+`'`, 0) + var auditText string + if err := store.db.QueryRowContext(ctx, `SELECT string_agg(row_to_json(event)::text, '') FROM audit_event event`).Scan(&auditText); err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"creatorhub/proxy-main", "credential-exit", "username", "password"} { + if strings.Contains(auditText, forbidden) { + t.Fatalf("audit leaked sensitive value %q: %s", forbidden, auditText) + } + } +} diff --git a/internal/phasea/store.go b/internal/phasea/store.go index 194428a..5e4876e 100644 --- a/internal/phasea/store.go +++ b/internal/phasea/store.go @@ -108,6 +108,11 @@ type AuditEvent struct { BindingVersion int64 `json:"binding_version,omitempty"` Actor string `json:"actor,omitempty"` ReasonCode string `json:"reason_code,omitempty"` + OperationID string `json:"operation_id,omitempty"` + Action string `json:"action,omitempty"` + Outcome string `json:"outcome,omitempty"` + OldImageVersion string `json:"old_image_version,omitempty"` + NewImageVersion string `json:"new_image_version,omitempty"` Details json.RawMessage `json:"details"` CreatedAt time.Time `json:"created_at"` } @@ -730,6 +735,7 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) { rows, err := s.db.QueryContext(ctx, ` SELECT id, event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id, browser_env_alias, network_exit_id, runtime_instance_id, binding_version, actor, reason_code, + operation_id, action, outcome, old_image_version, new_image_version, details, created_at FROM audit_event ORDER BY id`) if err != nil { @@ -740,10 +746,11 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) { for rows.Next() { var event AuditEvent var accountID, confirmationID, attemptID, taskID, browserEnvAlias, networkExitID sql.NullString - var runtimeInstanceID, actor, reasonCode sql.NullString + var runtimeInstanceID, actor, reasonCode, operationID, action, outcome, oldImage, newImage sql.NullString var confirmationVersion, bindingVersion sql.NullInt64 if err := rows.Scan(&event.ID, &event.EventType, &accountID, &confirmationID, &confirmationVersion, &attemptID, &taskID, &browserEnvAlias, &networkExitID, &runtimeInstanceID, &bindingVersion, &actor, &reasonCode, + &operationID, &action, &outcome, &oldImage, &newImage, &event.Details, &event.CreatedAt); err != nil { return nil, errors.New("decode audit event") } @@ -752,6 +759,8 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) { event.BrowserEnvAlias, event.NetworkExitID = browserEnvAlias.String, networkExitID.String event.RuntimeInstanceID, event.BindingVersion = runtimeInstanceID.String, bindingVersion.Int64 event.Actor, event.ReasonCode = actor.String, reasonCode.String + event.OperationID, event.Action, event.Outcome = operationID.String, action.String, outcome.String + event.OldImageVersion, event.NewImageVersion = oldImage.String, newImage.String events = append(events, event) } return events, rows.Err() diff --git a/internal/phasea/store_test.go b/internal/phasea/store_test.go index 68b820d..a563c3e 100644 --- a/internal/phasea/store_test.go +++ b/internal/phasea/store_test.go @@ -343,7 +343,7 @@ func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) { for _, migrationFile := range []struct { version int name string - }{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}} { + }{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}} { var applied bool if err := store.db.QueryRow(`SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migrationFile.version).Scan(&applied); err != nil { t.Fatal(err) diff --git a/web/src/BrowserList.jsx b/web/src/BrowserList.jsx index 2108e51..2883675 100644 --- a/web/src/BrowserList.jsx +++ b/web/src/BrowserList.jsx @@ -17,7 +17,6 @@ import { MenuItem, Paper, Stack, - Switch, Table, TableBody, TableCell, @@ -54,6 +53,8 @@ const emptyForm = { alias: '', gateway: '', image_version: '', + account_id: '', + network_exit_id: '', seed: '1000', platform: '', platform_version: '', @@ -63,8 +64,6 @@ const emptyForm = { lang: '', accept_lang: '', timezone: '', - proxy_server: '', - disable_non_proxied_udp: false, disable_spoofing: [], } @@ -78,13 +77,11 @@ function buildFingerprint(form) { lang: form.lang, accept_lang: form.accept_lang, timezone: form.timezone, - proxy_server: form.proxy_server, } for (const [key, value] of Object.entries(optionalText)) { if (value !== '') fingerprint[key] = value } if (form.hardware_concurrency !== '') fingerprint.hardware_concurrency = Number(form.hardware_concurrency) - if (form.disable_non_proxied_udp) fingerprint.disable_non_proxied_udp = true if (form.disable_spoofing.length > 0) fingerprint.disable_spoofing = form.disable_spoofing.join(',') return fingerprint } @@ -117,10 +114,14 @@ function Copyable({ value }) { ) } -function CreateForm({ gateways, images, onSubmit, busy }) { +function CreateForm({ gateways, images, accounts, networkExits, onSubmit, busy }) { const [form, setForm] = useState(emptyForm) const [advanced, setAdvanced] = useState(false) const enabledImages = images.filter(image => image.enabled) + const availableAccounts = accounts.filter(account => account.authorization_status === 'authorized' && account.runtime_status === 'paused') + const healthyExits = networkExits.filter(exit => exit.health_status === 'healthy') + const defaultAccountID = availableAccounts[0]?.id ?? '' + const defaultExitID = healthyExits[0]?.id ?? '' const update = (key, value) => setForm(current => ({ ...current, [key]: value })) const toggleSpoofing = option => setForm(current => ({ ...current, @@ -135,10 +136,17 @@ function CreateForm({ gateways, images, onSubmit, busy }) { useEffect(() => { if (form.image_version === '' && enabledImages.length > 0) update('image_version', enabledImages[0].version) }, [enabledImages, form.image_version]) + useEffect(() => { + if (form.account_id === '' && defaultAccountID !== '') update('account_id', defaultAccountID) + }, [defaultAccountID, form.account_id]) + useEffect(() => { + if (form.network_exit_id === '' && defaultExitID !== '') update('network_exit_id', defaultExitID) + }, [defaultExitID, form.network_exit_id]) const seedNumber = Number(form.seed) const valid = form.name.trim() !== '' && aliasPattern.test(form.alias) && form.gateway !== '' && - form.image_version !== '' && Number.isInteger(seedNumber) && seedNumber >= 1 && seedNumber <= 2147483647 + form.image_version !== '' && form.account_id !== '' && form.network_exit_id !== '' && + Number.isInteger(seedNumber) && seedNumber >= 1 && seedNumber <= 2147483647 const submit = event => { event.preventDefault() @@ -148,9 +156,11 @@ function CreateForm({ gateways, images, onSubmit, busy }) { name: form.name.trim(), gateway: form.gateway, image_version: form.image_version, + account_id: form.account_id, + network_exit_id: form.network_exit_id, fingerprint: buildFingerprint(form), }) - setForm(current => ({ ...emptyForm, gateway: current.gateway, image_version: current.image_version })) + setForm(current => ({ ...emptyForm, gateway: current.gateway, image_version: current.image_version, account_id: current.account_id, network_exit_id: current.network_exit_id })) } const label = (htmlFor, text, required = false) => ( @@ -185,6 +195,18 @@ function CreateForm({ gateways, images, onSubmit, busy }) { {label('env-seed', 'Fingerprint Seed', true)} update('seed', event.target.value)} /> + + {label('env-account', '社媒账号', true)} + update('account_id', event.target.value)} disabled={availableAccounts.length === 0} helperText={availableAccounts.length === 0 ? '请先准备已授权且暂停的账号' : ' '}> + {availableAccounts.map(account => {account.id} · {account.platform})} + + + + {label('env-exit', '网络出口', true)} + update('network_exit_id', event.target.value)} disabled={healthyExits.length === 0} helperText={healthyExits.length === 0 ? '请先检查并启用健康出口' : ' '}> + {healthyExits.map(exit => {exit.id} · {exit.protocol}://{exit.host}:{exit.port})} + + } sx={{ mb: 2.5 }}>{message} : null} {isPending ? : null} diff --git a/web/src/BrowserList.test.jsx b/web/src/BrowserList.test.jsx index c525180..0c42e69 100644 --- a/web/src/BrowserList.test.jsx +++ b/web/src/BrowserList.test.jsx @@ -16,6 +16,8 @@ const images = [ { id: '144.0.0.1', version: '144.0.0.1', image_ref: 'reg/img:144', enabled: true }, { id: '139.0.0.1', version: '139.0.0.1', image_ref: 'reg/img:139', enabled: false }, ] +const accounts = [{ id: 'social-a', platform: 'douyin', authorization_status: 'authorized', runtime_status: 'paused' }] +const networkExits = [{ id: 'exit-1', protocol: 'socks5', host: 'proxy.example', port: 1080, health_status: 'healthy' }] function provider(overrides = {}) { return { @@ -23,6 +25,8 @@ function provider(overrides = {}) { if (resource === 'browsers') return Promise.resolve({ data: runtimes, total: runtimes.length }) if (resource === 'gateways') return Promise.resolve({ data: gateways, total: gateways.length }) if (resource === 'browser-images') return Promise.resolve({ data: images, total: images.length }) + if (resource === 'accounts') return Promise.resolve({ data: accounts, total: accounts.length }) + if (resource === 'network-exits') return Promise.resolve({ data: networkExits, total: networkExits.length }) return Promise.reject(new Error(`unsupported ${resource}`)) }), create: vi.fn().mockResolvedValue({ data: { id: 'account-a', alias: 'account-a' } }), @@ -77,7 +81,7 @@ describe('BrowserList', () => { await waitFor(() => expect(dataProvider.browserAction).toHaveBeenCalledWith('account-a', 'stop', undefined)) }) - it('creates an env with chinese name, alias, gateway and enabled image version', async () => { + it('creates an env with an account and healthy network exit without legacy proxy fields', async () => { const dataProvider = provider() render() await screen.findAllByText('店铺一号') @@ -91,8 +95,12 @@ describe('BrowserList', () => { name: '店铺三号', gateway: 'gw-1', image_version: '148.0.0.1', + account_id: 'social-a', + network_exit_id: 'exit-1', fingerprint: { seed: 1000 }, } })) + expect(screen.queryByLabelText('代理 proxy-server')).toBeNull() + expect(screen.queryByLabelText('禁用非代理 UDP')).toBeNull() }) it('upgrades an env through the version dialog', async () => { diff --git a/web/src/dataProvider.js b/web/src/dataProvider.js index c3d3067..8ff39cb 100644 --- a/web/src/dataProvider.js +++ b/web/src/dataProvider.js @@ -21,6 +21,8 @@ const resourcePaths = { browsers: '/browsers', 'browser-images': '/browser-images', gateways: '/gateways', + accounts: '/phase-a/accounts', + 'network-exits': '/network-exits', } export const dataProvider = { @@ -28,7 +30,7 @@ export const dataProvider = { const path = resourcePaths[resource] if (!path) return unsupported(resource, 'getList') const records = await request(path) - return { data: records.map(record => ({ ...record, id: record.alias ?? record.version ?? record.name })), total: records.length } + return { data: records.map(record => ({ ...record, id: record.id ?? record.alias ?? record.version ?? record.name })), total: records.length } }, async create(resource, { data }) { const path = resourcePaths[resource] diff --git a/web/src/dataProvider.test.js b/web/src/dataProvider.test.js index 9845406..3a22f7f 100644 --- a/web/src/dataProvider.test.js +++ b/web/src/dataProvider.test.js @@ -15,6 +15,17 @@ describe('dataProvider', () => { expect(fetch).toHaveBeenCalledWith('/api/browsers', undefined) }) + it.each([ + ['accounts', '/api/phase-a/accounts'], + ['network-exits', '/api/network-exits'], + ])('loads %s for the environment create contract', async (resource, path) => { + const fetch = vi.fn().mockResolvedValue(new Response('[{"id":"record-1"}]', { status: 200 })) + vi.stubGlobal('fetch', fetch) + + await expect(dataProvider.getList(resource)).resolves.toMatchObject({ data: [{ id: 'record-1' }] }) + expect(fetch).toHaveBeenCalledWith(path, undefined) + }) + it.each([ ['start', '/api/browsers/account-a/start', 'POST'], ['stop', '/api/browsers/account-a/stop', 'POST'], diff --git a/web/tests/responsive.e2e.js b/web/tests/responsive.e2e.js index 9efe6fe..e28c11b 100644 --- a/web/tests/responsive.e2e.js +++ b/web/tests/responsive.e2e.js @@ -16,6 +16,8 @@ test('keeps the create form inside a 900px viewport', async ({ page }) => { await page.route('**/api/browsers', route => route.fulfill({ json: [] })) await page.route('**/api/gateways', route => route.fulfill({ json: [] })) await page.route('**/api/browser-images', route => route.fulfill({ json: [] })) + await page.route('**/api/phase-a/accounts', route => route.fulfill({ json: [] })) + await page.route('**/api/network-exits', route => route.fulfill({ json: [] })) await page.setViewportSize({ width: 900, height: 800 }) await page.goto('/') @@ -30,6 +32,10 @@ test('keeps the create form inside a 900px viewport', async ({ page }) => { test('shows the CDP endpoint only in the active branch at 900px and 599px', async ({ page }) => { await page.route('**/api/browsers', route => route.fulfill({ json: [runtime] })) + await page.route('**/api/gateways', route => route.fulfill({ json: [] })) + await page.route('**/api/browser-images', route => route.fulfill({ json: [] })) + await page.route('**/api/phase-a/accounts', route => route.fulfill({ json: [] })) + await page.route('**/api/network-exits', route => route.fulfill({ json: [] })) const endpoint = page.getByText('http://account-a:9222') await page.setViewportSize({ width: 900, height: 800 })