HH-803: add stable network exit orchestration (#19)
This commit is contained in:
@@ -0,0 +1,662 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var exitIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`)
|
||||
|
||||
type CredentialReference struct {
|
||||
ID string `json:"id"`
|
||||
Provider string `json:"provider"`
|
||||
}
|
||||
|
||||
type NetworkExit struct {
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
CredentialReference *CredentialReference `json:"credential_reference,omitempty"`
|
||||
ExpectedPublicIP string `json:"expected_public_ip,omitempty"`
|
||||
ExpectedRegion string `json:"expected_region,omitempty"`
|
||||
ObservedPublicIP string `json:"observed_public_ip,omitempty"`
|
||||
ObservedRegion string `json:"observed_region,omitempty"`
|
||||
HealthStatus string `json:"health_status"`
|
||||
LastCheckReason string `json:"last_check_reason,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
LastCheckedAt *time.Time `json:"last_checked_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// NetworkExitAccess is internal-only: reference keys are never serialized or audited.
|
||||
type NetworkExitAccess struct {
|
||||
NetworkExit
|
||||
CredentialKey string `json:"-"`
|
||||
}
|
||||
|
||||
type ExitObservation struct {
|
||||
PublicIP string
|
||||
Region string
|
||||
}
|
||||
|
||||
type EnvironmentContext struct {
|
||||
Env
|
||||
AccountID string `json:"account_id"`
|
||||
BindingID string `json:"binding_id"`
|
||||
BindingVersion int64 `json:"binding_version"`
|
||||
RuntimeCleanupPending bool `json:"runtime_cleanup_pending,omitempty"`
|
||||
Exit NetworkExit `json:"network_exit"`
|
||||
RuntimeInstanceID string `json:"runtime_instance_id,omitempty"`
|
||||
RuntimeID string `json:"runtime_id,omitempty"`
|
||||
}
|
||||
|
||||
type EnvironmentAction struct {
|
||||
OperationID string
|
||||
Action string
|
||||
AccountID string
|
||||
BrowserEnvAlias string
|
||||
NetworkExitID string
|
||||
RuntimeInstanceID string
|
||||
BindingVersion int64
|
||||
OldImageVersion string
|
||||
NewImageVersion string
|
||||
Outcome string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
func (s *Store) CreateNetworkExit(ctx context.Context, exit NetworkExit, credentialReferenceID string) (NetworkExit, error) {
|
||||
exit.ID = "exit-" + newHubID()
|
||||
exit.Protocol, exit.Host = strings.ToLower(strings.TrimSpace(exit.Protocol)), strings.TrimSpace(exit.Host)
|
||||
exit.ExpectedPublicIP, exit.ExpectedRegion = strings.TrimSpace(exit.ExpectedPublicIP), strings.TrimSpace(exit.ExpectedRegion)
|
||||
credentialReferenceID = strings.TrimSpace(credentialReferenceID)
|
||||
if !validNetworkExit(exit) || (credentialReferenceID != "" && !exitIDPattern.MatchString(credentialReferenceID)) {
|
||||
return NetworkExit{}, ErrInvalid
|
||||
}
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
INSERT INTO network_exit (id, protocol, host, port, credential_reference_id, expected_public_ip, expected_region)
|
||||
VALUES ($1, $2, $3, $4, NULLIF($5, ''), NULLIF($6, '')::inet, $7)
|
||||
RETURNING id`, exit.ID, exit.Protocol, exit.Host, exit.Port, credentialReferenceID, exit.ExpectedPublicIP, exit.ExpectedRegion)
|
||||
if err := row.Scan(&exit.ID); err != nil {
|
||||
return NetworkExit{}, publicDatabaseError(err)
|
||||
}
|
||||
return s.GetNetworkExit(ctx, exit.ID)
|
||||
}
|
||||
|
||||
func validNetworkExit(exit NetworkExit) bool {
|
||||
if exit.Protocol != "http" && exit.Protocol != "https" && exit.Protocol != "socks4" && exit.Protocol != "socks5" {
|
||||
return false
|
||||
}
|
||||
if !validExitHost(exit.Host) || exit.Port < 1 || exit.Port > 65535 {
|
||||
return false
|
||||
}
|
||||
if exit.ExpectedPublicIP != "" && net.ParseIP(exit.ExpectedPublicIP) == nil {
|
||||
return false
|
||||
}
|
||||
return validOptionalRegion(exit.ExpectedRegion)
|
||||
}
|
||||
|
||||
func validExitHost(host string) bool {
|
||||
if host == "" || len(host) > 253 || strings.ContainsAny(host, "@/[]?# \t\r\n") {
|
||||
return false
|
||||
}
|
||||
if net.ParseIP(host) != nil {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(host, ".") || strings.HasSuffix(host, ".") || strings.Contains(host, "..") {
|
||||
return false
|
||||
}
|
||||
for _, label := range strings.Split(host, ".") {
|
||||
if len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
|
||||
return false
|
||||
}
|
||||
for _, character := range label {
|
||||
if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') &&
|
||||
(character < '0' || character > '9') && character != '-' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validOptionalRegion(region string) bool {
|
||||
if len(region) > 64 {
|
||||
return false
|
||||
}
|
||||
for _, character := range region {
|
||||
if character < 0x20 || character == 0x7f {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Store) ListNetworkExits(ctx context.Context) ([]NetworkExit, error) {
|
||||
rows, err := s.db.QueryContext(ctx, networkExitSelect+` ORDER BY network.created_at, network.id`)
|
||||
if err != nil {
|
||||
return nil, errors.New("read network exits")
|
||||
}
|
||||
defer rows.Close()
|
||||
exits := []NetworkExit{}
|
||||
for rows.Next() {
|
||||
exit, err := scanNetworkExit(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exits = append(exits, exit)
|
||||
}
|
||||
return exits, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) GetNetworkExit(ctx context.Context, id string) (NetworkExit, error) {
|
||||
if !exitIDPattern.MatchString(id) {
|
||||
return NetworkExit{}, ErrInvalid
|
||||
}
|
||||
return scanNetworkExit(s.db.QueryRowContext(ctx, networkExitSelect+` WHERE network.id = $1`, id))
|
||||
}
|
||||
|
||||
func (s *Store) GetNetworkExitAccess(ctx context.Context, id string) (NetworkExitAccess, error) {
|
||||
exit, err := s.GetNetworkExit(ctx, id)
|
||||
if err != nil {
|
||||
return NetworkExitAccess{}, err
|
||||
}
|
||||
access := NetworkExitAccess{NetworkExit: exit}
|
||||
if exit.CredentialReference != nil {
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT reference_key FROM credential_reference WHERE id = $1`, exit.CredentialReference.ID).
|
||||
Scan(&access.CredentialKey); err != nil {
|
||||
return NetworkExitAccess{}, rowError(err)
|
||||
}
|
||||
}
|
||||
return access, nil
|
||||
}
|
||||
|
||||
const networkExitSelect = `
|
||||
SELECT network.id, network.protocol, network.host, network.port,
|
||||
reference.id, reference.provider,
|
||||
COALESCE(host(network.expected_public_ip), ''), network.expected_region,
|
||||
COALESCE(host(network.observed_public_ip), ''), network.observed_region,
|
||||
network.health_status, COALESCE(network.last_check_reason, ''), network.version, network.last_checked_at,
|
||||
network.created_at, network.updated_at
|
||||
FROM network_exit network
|
||||
LEFT JOIN credential_reference reference ON reference.id = network.credential_reference_id`
|
||||
|
||||
type rowScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanNetworkExit(row rowScanner) (NetworkExit, error) {
|
||||
var exit NetworkExit
|
||||
var referenceID, provider sql.NullString
|
||||
var checked sql.NullTime
|
||||
if err := row.Scan(&exit.ID, &exit.Protocol, &exit.Host, &exit.Port, &referenceID, &provider,
|
||||
&exit.ExpectedPublicIP, &exit.ExpectedRegion, &exit.ObservedPublicIP, &exit.ObservedRegion,
|
||||
&exit.HealthStatus, &exit.LastCheckReason, &exit.Version, &checked, &exit.CreatedAt, &exit.UpdatedAt); err != nil {
|
||||
return NetworkExit{}, rowError(err)
|
||||
}
|
||||
if referenceID.Valid {
|
||||
exit.CredentialReference = &CredentialReference{ID: referenceID.String, Provider: provider.String}
|
||||
}
|
||||
if checked.Valid {
|
||||
exit.LastCheckedAt = &checked.Time
|
||||
}
|
||||
return exit, nil
|
||||
}
|
||||
|
||||
// RecordNetworkExitCheck stores only observed identity and a stable reason code.
|
||||
func (s *Store) RecordNetworkExitCheck(ctx context.Context, id string, observation ExitObservation, failureReason string) (NetworkExit, string, error) {
|
||||
if !exitIDPattern.MatchString(id) || !validOptionalRegion(observation.Region) ||
|
||||
(observation.PublicIP != "" && net.ParseIP(observation.PublicIP) == nil) || !validExitFailureReason(failureReason) {
|
||||
return NetworkExit{}, "invalid_observation", ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return NetworkExit{}, "persistence_failed", errors.New("begin network exit check")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var expectedIP, expectedRegion, oldIP, oldRegion, oldStatus string
|
||||
var version int64
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT COALESCE(host(expected_public_ip), ''), expected_region,
|
||||
COALESCE(host(observed_public_ip), ''), observed_region, health_status, version
|
||||
FROM network_exit WHERE id = $1 FOR UPDATE`, id).
|
||||
Scan(&expectedIP, &expectedRegion, &oldIP, &oldRegion, &oldStatus, &version); err != nil {
|
||||
return NetworkExit{}, "persistence_failed", rowError(err)
|
||||
}
|
||||
if oldStatus == "disabled" {
|
||||
return NetworkExit{}, "exit_disabled", ErrConflict
|
||||
}
|
||||
reason, status := strings.TrimSpace(failureReason), "unhealthy"
|
||||
if reason == "" && expectedIP != "" && !net.ParseIP(expectedIP).Equal(net.ParseIP(observation.PublicIP)) {
|
||||
reason = "exit_ip_drift"
|
||||
}
|
||||
if reason == "" && expectedRegion != "" && !strings.EqualFold(expectedRegion, observation.Region) {
|
||||
reason = "exit_region_drift"
|
||||
}
|
||||
if reason == "" {
|
||||
reason, status = "exit_healthy", "healthy"
|
||||
}
|
||||
changed := !sameIP(oldIP, observation.PublicIP) || !strings.EqualFold(oldRegion, observation.Region) || oldStatus != status
|
||||
if changed {
|
||||
version++
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE network_exit SET observed_public_ip = NULLIF($2, '')::inet, observed_region = $3,
|
||||
health_status = $4, last_check_reason = $5, version = $6, last_checked_at = now(), updated_at = now()
|
||||
WHERE id = $1`, id, observation.PublicIP, observation.Region, status, reason, version); err != nil {
|
||||
return NetworkExit{}, "persistence_failed", errors.New("record network exit check")
|
||||
}
|
||||
if changed {
|
||||
if err := invalidateAccountsForExit(ctx, tx, id); err != nil {
|
||||
return NetworkExit{}, "persistence_failed", err
|
||||
}
|
||||
}
|
||||
if err := commitHub(tx); err != nil {
|
||||
return NetworkExit{}, "persistence_failed", err
|
||||
}
|
||||
exit, err := s.GetNetworkExit(ctx, id)
|
||||
return exit, reason, err
|
||||
}
|
||||
|
||||
func validExitFailureReason(reason string) bool {
|
||||
switch reason {
|
||||
case "", "credential_unavailable", "credential_invalid", "proxy_auth_failed", "proxy_check_failed", "exit_observation_invalid":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func sameIP(left, right string) bool {
|
||||
if left == "" || right == "" {
|
||||
return left == right
|
||||
}
|
||||
return net.ParseIP(left).Equal(net.ParseIP(right))
|
||||
}
|
||||
|
||||
func (s *Store) DisableNetworkExit(ctx context.Context, id string) (NetworkExit, error) {
|
||||
if !exitIDPattern.MatchString(id) {
|
||||
return NetworkExit{}, ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return NetworkExit{}, errors.New("begin network exit disable")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var oldStatus string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT health_status FROM network_exit WHERE id = $1 FOR UPDATE`, id).Scan(&oldStatus); err != nil {
|
||||
return NetworkExit{}, rowError(err)
|
||||
}
|
||||
if oldStatus != "disabled" {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE network_exit SET health_status = 'disabled', last_check_reason = 'exit_disabled',
|
||||
version = version + 1, updated_at = now()
|
||||
WHERE id = $1`, id); err != nil {
|
||||
return NetworkExit{}, errors.New("disable network exit")
|
||||
}
|
||||
if err := invalidateAccountsForExit(ctx, tx, id); err != nil {
|
||||
return NetworkExit{}, err
|
||||
}
|
||||
}
|
||||
if err := commitHub(tx); err != nil {
|
||||
return NetworkExit{}, err
|
||||
}
|
||||
return s.GetNetworkExit(ctx, id)
|
||||
}
|
||||
|
||||
func invalidateAccountsForExit(ctx context.Context, tx *sql.Tx, exitID string) error {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
WITH changed AS (
|
||||
UPDATE social_account account SET status = 'paused', paused_at = COALESCE(paused_at, now()),
|
||||
version = account.version + 1, updated_at = now()
|
||||
FROM environment_binding binding
|
||||
WHERE binding.network_exit_id = $1 AND binding.account_id = account.id
|
||||
RETURNING account.id
|
||||
)
|
||||
UPDATE operation_task task SET state = 'policy_hold', updated_at = now()
|
||||
FROM changed WHERE task.account_id = changed.id AND task.state = 'queued'`, exitID); err != nil {
|
||||
return errors.New("invalidate network exit accounts")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateBoundEnv(ctx context.Context, env Env, accountID, exitID string) (EnvironmentContext, bool, error) {
|
||||
env.Alias, env.Name = strings.TrimSpace(env.Alias), strings.TrimSpace(env.Name)
|
||||
if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) || !aliasPattern.MatchString(accountID) ||
|
||||
!exitIDPattern.MatchString(exitID) || !gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) ||
|
||||
env.Fingerprint.ProxyServer != "" {
|
||||
return EnvironmentContext{}, false, ErrInvalid
|
||||
}
|
||||
if err := env.Fingerprint.Validate(); err != nil {
|
||||
return EnvironmentContext{}, false, ErrInvalid
|
||||
}
|
||||
encoded, _ := json.Marshal(env.Fingerprint)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, false, errors.New("begin bound environment create")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var existingAlias, existingExit string
|
||||
err = tx.QueryRowContext(ctx, `SELECT browser_env_alias, COALESCE(network_exit_id, '') FROM environment_binding WHERE account_id = $1 FOR UPDATE`, accountID).
|
||||
Scan(&existingAlias, &existingExit)
|
||||
if err == nil {
|
||||
if existingAlias != env.Alias || existingExit != exitID {
|
||||
return EnvironmentContext{}, false, ErrConflict
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return EnvironmentContext{}, false, errors.New("commit existing environment lookup")
|
||||
}
|
||||
context, err := s.GetEnvironmentContext(ctx, env.Alias)
|
||||
if err != nil || context.Name != env.Name || context.Gateway != env.Gateway || context.ImageVersion != env.ImageVersion || context.Fingerprint != env.Fingerprint {
|
||||
return EnvironmentContext{}, false, ErrConflict
|
||||
}
|
||||
return context, false, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return EnvironmentContext{}, false, publicDatabaseError(err)
|
||||
}
|
||||
var created string
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint)
|
||||
SELECT $1, $2, $3, image.version, $5
|
||||
FROM browser_image image, social_account account, network_exit network
|
||||
WHERE image.version = $4 AND image.enabled AND account.id = $6 AND account.status = 'paused'
|
||||
AND account.authorization_status = 'authorized' AND network.id = $7 AND network.health_status = 'healthy'
|
||||
RETURNING alias`, env.Alias, env.Name, env.Gateway, env.ImageVersion, encoded, accountID, exitID).Scan(&created); err != nil {
|
||||
return EnvironmentContext{}, false, rowError(err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO environment_binding (id, account_id, browser_env_alias, network_exit_id)
|
||||
VALUES ($1, $1, $2, $3)`, accountID, env.Alias, exitID); err != nil {
|
||||
return EnvironmentContext{}, false, publicDatabaseError(err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE social_account SET version = version + 1, updated_at = now() WHERE id = $1`, accountID); err != nil {
|
||||
return EnvironmentContext{}, false, errors.New("version bound account")
|
||||
}
|
||||
if err := commitHub(tx); err != nil {
|
||||
return EnvironmentContext{}, false, err
|
||||
}
|
||||
context, err := s.GetEnvironmentContext(ctx, env.Alias)
|
||||
return context, true, err
|
||||
}
|
||||
|
||||
func (s *Store) GetEnvironmentContext(ctx context.Context, alias string) (EnvironmentContext, error) {
|
||||
if !aliasPattern.MatchString(alias) {
|
||||
return EnvironmentContext{}, ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, errors.New("begin environment context read")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE runtime_instance runtime SET released_at = now()
|
||||
FROM environment_binding binding
|
||||
WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id
|
||||
AND runtime.released_at IS NULL AND runtime.lease_until <= now()`, alias); err != nil {
|
||||
return EnvironmentContext{}, errors.New("expire environment runtime")
|
||||
}
|
||||
var result EnvironmentContext
|
||||
var encoded []byte
|
||||
var expectedIP, observedIP string
|
||||
var checked sql.NullTime
|
||||
var runtimeInstanceID, runtimeID sql.NullString
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT environment.alias, environment.name, environment.gateway_name, environment.image_version,
|
||||
environment.fingerprint, environment.created_at, binding.account_id, binding.id, binding.version,
|
||||
binding.runtime_cleanup_pending,
|
||||
COALESCE(network.id, ''), COALESCE(network.protocol, ''), COALESCE(network.host, ''), COALESCE(network.port, 0),
|
||||
COALESCE(host(network.expected_public_ip), ''), COALESCE(network.expected_region, ''),
|
||||
COALESCE(host(network.observed_public_ip), ''), COALESCE(network.observed_region, ''),
|
||||
COALESCE(network.health_status, 'unchecked'), COALESCE(network.last_check_reason, ''),
|
||||
COALESCE(network.version, 0), network.last_checked_at,
|
||||
COALESCE(network.created_at, to_timestamp(0)), COALESCE(network.updated_at, to_timestamp(0)),
|
||||
runtime.id, runtime.runtime_id
|
||||
FROM browser_env environment
|
||||
JOIN environment_binding binding ON binding.browser_env_alias = environment.alias
|
||||
LEFT JOIN network_exit network ON network.id = binding.network_exit_id
|
||||
LEFT JOIN runtime_instance runtime ON runtime.binding_id = binding.id AND runtime.released_at IS NULL
|
||||
WHERE environment.alias = $1`, alias).
|
||||
Scan(&result.Alias, &result.Name, &result.Gateway, &result.ImageVersion, &encoded, &result.CreatedAt,
|
||||
&result.AccountID, &result.BindingID, &result.BindingVersion, &result.RuntimeCleanupPending,
|
||||
&result.Exit.ID, &result.Exit.Protocol, &result.Exit.Host, &result.Exit.Port,
|
||||
&expectedIP, &result.Exit.ExpectedRegion, &observedIP, &result.Exit.ObservedRegion,
|
||||
&result.Exit.HealthStatus, &result.Exit.LastCheckReason, &result.Exit.Version, &checked, &result.Exit.CreatedAt, &result.Exit.UpdatedAt,
|
||||
&runtimeInstanceID, &runtimeID)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, rowError(err)
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &result.Fingerprint); err != nil {
|
||||
return EnvironmentContext{}, errors.New("decode bound environment fingerprint")
|
||||
}
|
||||
result.Fingerprint.ProxyServer = ""
|
||||
result.Fingerprint.DisableNonProxiedUDP = false
|
||||
result.Exit.ExpectedPublicIP, result.Exit.ObservedPublicIP = expectedIP, observedIP
|
||||
if checked.Valid {
|
||||
result.Exit.LastCheckedAt = &checked.Time
|
||||
}
|
||||
result.RuntimeInstanceID, result.RuntimeID = runtimeInstanceID.String, runtimeID.String
|
||||
if err := commitHub(tx); err != nil {
|
||||
return EnvironmentContext{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateEnvironmentRebind(ctx context.Context, tx *sql.Tx, alias, exitID string, expectedBindingVersion int64) (string, string, error) {
|
||||
var accountID, bindingID string
|
||||
var bindingVersion int64
|
||||
err := tx.QueryRowContext(ctx, `
|
||||
SELECT binding.account_id, binding.id, binding.version
|
||||
FROM environment_binding binding
|
||||
JOIN social_account account ON account.id = binding.account_id
|
||||
WHERE binding.browser_env_alias = $1 AND account.status = 'paused'
|
||||
AND account.authorization_status = 'authorized'
|
||||
AND NOT binding.runtime_cleanup_pending
|
||||
FOR UPDATE OF binding, account`, alias).Scan(&accountID, &bindingID, &bindingVersion)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", "", ErrConflict
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", publicDatabaseError(err)
|
||||
}
|
||||
if bindingVersion != expectedBindingVersion {
|
||||
return "", "", ErrConflict
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE runtime_instance SET released_at = now()
|
||||
WHERE binding_id = $1 AND released_at IS NULL AND lease_until <= now()`, bindingID); err != nil {
|
||||
return "", "", errors.New("expire runtime before rebind")
|
||||
}
|
||||
var allowed bool
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT EXISTS (SELECT 1 FROM network_exit WHERE id = $1 AND health_status = 'healthy')
|
||||
AND NOT EXISTS (SELECT 1 FROM operation_task WHERE account_id = $2 AND state = 'executing')
|
||||
AND NOT EXISTS (SELECT 1 FROM runtime_instance WHERE binding_id = $3 AND released_at IS NULL)`,
|
||||
exitID, accountID, bindingID).Scan(&allowed); err != nil {
|
||||
return "", "", errors.New("check environment rebind")
|
||||
}
|
||||
if !allowed {
|
||||
return "", "", ErrConflict
|
||||
}
|
||||
return accountID, bindingID, nil
|
||||
}
|
||||
|
||||
func (s *Store) ValidateEnvironmentRebind(ctx context.Context, alias, exitID string, expectedBindingVersion int64) error {
|
||||
if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(exitID) || expectedBindingVersion < 1 {
|
||||
return ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin environment rebind validation")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, _, err := validateEnvironmentRebind(ctx, tx, alias, exitID, expectedBindingVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := commitHub(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) RebindEnvironment(ctx context.Context, alias, exitID, runtimeID string, expectedBindingVersion int64) (EnvironmentContext, error) {
|
||||
if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(exitID) ||
|
||||
(runtimeID != "" && !exitIDPattern.MatchString(runtimeID)) || expectedBindingVersion < 1 {
|
||||
return EnvironmentContext{}, ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, errors.New("begin environment rebind")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
accountID, bindingID, err := validateEnvironmentRebind(ctx, tx, alias, exitID, expectedBindingVersion)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE environment_binding SET network_exit_id = $2, version = version + 1, updated_at = now() WHERE browser_env_alias = $1`, alias, exitID); err != nil {
|
||||
return EnvironmentContext{}, errors.New("update environment binding")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE social_account SET version = version + 1, updated_at = now() WHERE id = $1`, accountID); err != nil {
|
||||
return EnvironmentContext{}, errors.New("version rebound account")
|
||||
}
|
||||
if runtimeID != "" {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until)
|
||||
VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil {
|
||||
return EnvironmentContext{}, publicDatabaseError(err)
|
||||
}
|
||||
}
|
||||
if err := commitHub(tx); err != nil {
|
||||
return EnvironmentContext{}, err
|
||||
}
|
||||
return s.GetEnvironmentContext(ctx, alias)
|
||||
}
|
||||
|
||||
func (s *Store) ActivateRuntime(ctx context.Context, alias, runtimeID string, bindingVersion int64, exitID string) (EnvironmentContext, error) {
|
||||
if !aliasPattern.MatchString(alias) || !exitIDPattern.MatchString(runtimeID) || bindingVersion < 1 || !exitIDPattern.MatchString(exitID) {
|
||||
return EnvironmentContext{}, ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, errors.New("begin runtime activation")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var accountID, bindingID, currentExitID string
|
||||
var currentBindingVersion int64
|
||||
var cleanupPending bool
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT binding.account_id, binding.id, binding.version, COALESCE(binding.network_exit_id, ''), binding.runtime_cleanup_pending
|
||||
FROM environment_binding binding
|
||||
WHERE binding.browser_env_alias = $1 FOR UPDATE OF binding`, alias).
|
||||
Scan(&accountID, &bindingID, ¤tBindingVersion, ¤tExitID, &cleanupPending)
|
||||
if err != nil {
|
||||
return EnvironmentContext{}, rowError(err)
|
||||
}
|
||||
if cleanupPending || currentBindingVersion != bindingVersion || currentExitID != exitID {
|
||||
return EnvironmentContext{}, ErrConflict
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE runtime_instance SET released_at = now()
|
||||
WHERE binding_id = $1 AND released_at IS NULL AND lease_until <= now()`, bindingID); err != nil {
|
||||
return EnvironmentContext{}, errors.New("expire runtime before activation")
|
||||
}
|
||||
var existingInstanceID, existingRuntimeID string
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT COALESCE(id, ''), COALESCE(runtime_id, '') FROM runtime_instance
|
||||
WHERE binding_id = $1 AND released_at IS NULL`, bindingID).Scan(&existingInstanceID, &existingRuntimeID); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return EnvironmentContext{}, publicDatabaseError(err)
|
||||
}
|
||||
if existingInstanceID != "" && existingRuntimeID != runtimeID {
|
||||
return EnvironmentContext{}, ErrConflict
|
||||
}
|
||||
if existingInstanceID == "" {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO runtime_instance (id, account_id, binding_id, runtime_id, lease_until)
|
||||
VALUES ($1, $2, $3, $4, now() + interval '1 minute')`, "runtime-"+newHubID(), accountID, bindingID, runtimeID); err != nil {
|
||||
return EnvironmentContext{}, publicDatabaseError(err)
|
||||
}
|
||||
} else if _, err := tx.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 minute' WHERE id = $1`, existingInstanceID); err != nil {
|
||||
return EnvironmentContext{}, errors.New("renew environment runtime")
|
||||
}
|
||||
if err := commitHub(tx); err != nil {
|
||||
return EnvironmentContext{}, err
|
||||
}
|
||||
return s.GetEnvironmentContext(ctx, alias)
|
||||
}
|
||||
|
||||
func (s *Store) ReleaseRuntime(ctx context.Context, alias string) error {
|
||||
if !aliasPattern.MatchString(alias) {
|
||||
return ErrInvalid
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
UPDATE runtime_instance runtime SET released_at = now()
|
||||
FROM environment_binding binding
|
||||
WHERE binding.browser_env_alias = $1 AND runtime.binding_id = binding.id AND runtime.released_at IS NULL`, alias)
|
||||
if err != nil {
|
||||
return errors.New("release environment runtime")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) SetRuntimeCleanupPending(ctx context.Context, alias string, pending bool) error {
|
||||
if !aliasPattern.MatchString(alias) {
|
||||
return ErrInvalid
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin runtime cleanup state update")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var bindingID string
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
UPDATE environment_binding SET runtime_cleanup_pending = $2, updated_at = now()
|
||||
WHERE browser_env_alias = $1 RETURNING id`, alias, pending).Scan(&bindingID); err != nil {
|
||||
return rowError(err)
|
||||
}
|
||||
if pending {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE runtime_instance SET released_at = now()
|
||||
WHERE binding_id = $1 AND released_at IS NULL`, bindingID); err != nil {
|
||||
return errors.New("release runtime for pending cleanup")
|
||||
}
|
||||
}
|
||||
return commitHub(tx)
|
||||
}
|
||||
|
||||
func (s *Store) AppendEnvironmentAction(ctx context.Context, eventType string, action EnvironmentAction) error {
|
||||
if (eventType != "environment_action_requested" && eventType != "environment_action_finished") ||
|
||||
!exitIDPattern.MatchString(action.OperationID) || action.Action == "" || action.ReasonCode == "" ||
|
||||
(action.OldImageVersion != "" && !imageVersionPattern.MatchString(action.OldImageVersion)) ||
|
||||
(action.NewImageVersion != "" && !imageVersionPattern.MatchString(action.NewImageVersion)) ||
|
||||
(eventType == "environment_action_finished" && action.Outcome != "succeeded" && action.Outcome != "failed" && action.Outcome != "unknown") {
|
||||
return ErrInvalid
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO audit_event
|
||||
(event_type, account_id, browser_env_alias, network_exit_id, runtime_instance_id,
|
||||
binding_version, actor, reason_code, operation_id, action, outcome, old_image_version, new_image_version)
|
||||
VALUES ($1, NULLIF($2, ''), NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''),
|
||||
NULLIF($6, 0), 'local-user', $7, $8, $9, NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, ''))`,
|
||||
eventType, action.AccountID, action.BrowserEnvAlias, action.NetworkExitID, action.RuntimeInstanceID,
|
||||
action.BindingVersion, action.ReasonCode, action.OperationID, action.Action, action.Outcome,
|
||||
action.OldImageVersion, action.NewImageVersion)
|
||||
if err != nil {
|
||||
return errors.New("append environment action")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewOperationID() string { return "operation-" + newHubID() }
|
||||
|
||||
func newHubID() string {
|
||||
var value [12]byte
|
||||
_, _ = rand.Read(value[:])
|
||||
return hex.EncodeToString(value[:])
|
||||
}
|
||||
@@ -29,12 +29,13 @@ func TestUnifiedAccountMigration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3)`, 3)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('social_account', 'browser_env', 'network_exit', 'environment_binding')`, 4)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'environment_binding' AND column_name = 'runtime_cleanup_pending'`, 1)
|
||||
|
||||
store = openFullyMigratedHub(t, ctx, testURL)
|
||||
store.Close()
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3)`, 3)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (1, 2, 3, 4, 5, 6)`, 6)
|
||||
})
|
||||
|
||||
t.Run("v1 and v2 data", func(t *testing.T) {
|
||||
@@ -72,7 +73,7 @@ func TestUnifiedAccountMigration(t *testing.T) {
|
||||
INSERT INTO gateway (name, endpoint, token) VALUES ('legacy-gateway', 'http://127.0.0.1:8081', 'legacy-gateway-token');
|
||||
INSERT INTO browser_image (version, image_ref) VALUES ('1', 'example/browser:1');
|
||||
INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint) VALUES
|
||||
('mapped', 'Mapped', 'legacy-gateway', '1', '{"seed":1}'),
|
||||
('mapped', 'Mapped', 'legacy-gateway', '1', '{"seed":1,"proxy_server":"http://legacy:secret@proxy.example:8080","disable_non_proxied_udp":true}'),
|
||||
('orphan-env', 'Orphan', 'legacy-gateway', '1', '{"seed":2}');
|
||||
INSERT INTO runtime_instance (id, account_id, runtime_id, lease_until) VALUES
|
||||
('instance-mapped', 'mapped', 'runtime-mapped', now() + interval '1 hour'),
|
||||
@@ -90,11 +91,22 @@ func TestUnifiedAccountMigration(t *testing.T) {
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE platform = 'mock' AND platform_account_key = id AND status = 'paused'`, 2)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE profile_id LIKE 'legacy-profile-%'`, 2)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'mapped' AND browser_env_alias = 'mapped' AND network_exit_id IS NULL`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'mapped' AND NOT runtime_cleanup_pending`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE account_id = 'unbound'`, 0)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-mapped' AND binding_id = 'mapped'`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM runtime_instance WHERE id = 'instance-unbound' AND binding_id IS NULL`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM audit_event WHERE event_type = 'legacy_event'`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'legacy-task' AND state = 'policy_hold'`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND NOT (fingerprint ?| ARRAY['proxy_server', 'disable_non_proxied_udp'])`, 1)
|
||||
store, err = Open(ctx, testURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := store.GetEnvironmentContext(ctx, "mapped")
|
||||
store.Close()
|
||||
if err != nil || legacy.Exit.ID != "" || legacy.Fingerprint.ProxyServer != "" || legacy.Fingerprint.DisableNonProxiedUDP {
|
||||
t.Fatalf("legacy NULL binding must remain visible without persisted proxy credentials: %#v err=%v", legacy, err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-duplicate', 'os_keyring', 'creatorhub/duplicate');
|
||||
@@ -154,6 +166,7 @@ func TestUnifiedAccountMigration(t *testing.T) {
|
||||
}
|
||||
store.Close()
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM browser_env WHERE alias = 'mapped' AND version = 2 AND image_version = '2'`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM environment_binding WHERE id = 'mapped' AND version = 2`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM social_account WHERE id = 'mapped' AND version = 2 AND status = 'paused'`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM operation_task WHERE id = 'upgrade-task' AND state = 'policy_hold'`, 1)
|
||||
|
||||
@@ -162,7 +175,7 @@ func TestUnifiedAccountMigration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Close()
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version = 3`, 1)
|
||||
assertDatabaseCount(t, db, `SELECT count(*) FROM schema_migration WHERE version IN (3, 4, 5, 6)`, 4)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
ALTER TABLE audit_event
|
||||
ADD COLUMN operation_id text,
|
||||
ADD COLUMN action text,
|
||||
ADD COLUMN outcome text CHECK (outcome IN ('succeeded', 'failed', 'unknown')),
|
||||
ADD COLUMN old_image_version text,
|
||||
ADD COLUMN new_image_version text;
|
||||
|
||||
ALTER TABLE network_exit
|
||||
ADD COLUMN last_check_reason text;
|
||||
|
||||
CREATE INDEX audit_event_operation_id_idx
|
||||
ON audit_event (operation_id) WHERE operation_id IS NOT NULL;
|
||||
@@ -0,0 +1,3 @@
|
||||
UPDATE browser_env
|
||||
SET fingerprint = fingerprint - 'proxy_server' - 'disable_non_proxied_udp'
|
||||
WHERE fingerprint ?| ARRAY['proxy_server', 'disable_non_proxied_udp'];
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE environment_binding
|
||||
ADD COLUMN runtime_cleanup_pending boolean NOT NULL DEFAULT false;
|
||||
+30
-2
@@ -25,12 +25,23 @@ var migration002 string
|
||||
//go:embed migrations/003_unified_accounts.sql
|
||||
var migration003 string
|
||||
|
||||
//go:embed migrations/004_environment_actions.sql
|
||||
var migration004 string
|
||||
|
||||
//go:embed migrations/005_sanitize_legacy_proxy.sql
|
||||
var migration005 string
|
||||
|
||||
//go:embed migrations/006_runtime_cleanup.sql
|
||||
var migration006 string
|
||||
|
||||
var (
|
||||
ErrConflict = errors.New("resource conflicts with existing state")
|
||||
ErrInvalid = errors.New("invalid hub input")
|
||||
ErrNotFound = errors.New("resource not found")
|
||||
)
|
||||
|
||||
func ValidImageVersion(version string) bool { return imageVersionPattern.MatchString(version) }
|
||||
|
||||
var (
|
||||
aliasPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
|
||||
gatewayNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
|
||||
@@ -107,7 +118,7 @@ func (s *Store) migrate(ctx context.Context) error {
|
||||
for _, migration := range []struct {
|
||||
version int
|
||||
sql string
|
||||
}{{2, migration002}, {3, migration003}} {
|
||||
}{{2, migration002}, {3, migration003}, {4, migration004}, {5, migration005}, {6, migration006}} {
|
||||
var applied bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migration.version).Scan(&applied); err != nil {
|
||||
return errors.New("read hub schema migration state")
|
||||
@@ -273,7 +284,8 @@ func (s *Store) CreateEnv(ctx context.Context, env Env) error {
|
||||
env.Alias = strings.TrimSpace(env.Alias)
|
||||
env.Name = strings.TrimSpace(env.Name)
|
||||
if !aliasPattern.MatchString(env.Alias) || !validDisplayName(env.Name) ||
|
||||
!gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) {
|
||||
!gatewayNamePattern.MatchString(env.Gateway) || !imageVersionPattern.MatchString(env.ImageVersion) ||
|
||||
env.Fingerprint.ProxyServer != "" {
|
||||
return ErrInvalid
|
||||
}
|
||||
if err := env.Fingerprint.Validate(); err != nil {
|
||||
@@ -341,6 +353,15 @@ func (s *Store) UpgradeEnv(ctx context.Context, alias, version string) error {
|
||||
return errors.New("begin environment upgrade")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var cleanupPending bool
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT runtime_cleanup_pending FROM environment_binding
|
||||
WHERE browser_env_alias = $1 FOR UPDATE`, alias).Scan(&cleanupPending); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return publicDatabaseError(err)
|
||||
}
|
||||
if cleanupPending {
|
||||
return ErrConflict
|
||||
}
|
||||
var updated string
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
UPDATE browser_env SET image_version = $2, version = version + 1
|
||||
@@ -348,6 +369,11 @@ func (s *Store) UpgradeEnv(ctx context.Context, alias, version string) error {
|
||||
RETURNING alias`, alias, version).Scan(&updated); err != nil {
|
||||
return rowError(err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE environment_binding SET version = version + 1, updated_at = now()
|
||||
WHERE browser_env_alias = $1`, alias); err != nil {
|
||||
return errors.New("version upgraded environment binding")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
WITH changed AS (
|
||||
UPDATE social_account account
|
||||
@@ -385,6 +411,8 @@ func scanEnv(rows *sql.Rows) (Env, error) {
|
||||
return Env{}, errors.New("decode env fingerprint")
|
||||
}
|
||||
}
|
||||
env.Fingerprint.ProxyServer = ""
|
||||
env.Fingerprint.DisableNonProxiedUDP = false
|
||||
return env, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -113,6 +113,22 @@ func TestStoreValidationRejectsInvalidInputsBeforePersistence(t *testing.T) {
|
||||
if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 0}}); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("expected invalid fingerprint, got %v", err)
|
||||
}
|
||||
for name, exit := range map[string]NetworkExit{
|
||||
"protocol": {Protocol: "direct", Host: "proxy.example", Port: 1080},
|
||||
"userinfo": {Protocol: "socks5", Host: "user@proxy.example", Port: 1080},
|
||||
"URL host": {Protocol: "socks5", Host: "socks5://proxy.example", Port: 1080},
|
||||
"port": {Protocol: "socks5", Host: "proxy.example", Port: 0},
|
||||
"ip": {Protocol: "socks5", Host: "proxy.example", Port: 1080, ExpectedPublicIP: "not-an-ip"},
|
||||
} {
|
||||
t.Run("network exit "+name, func(t *testing.T) {
|
||||
if _, err := store.CreateNetworkExit(ctx, exit, ""); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("expected invalid network exit, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := store.CreateEnv(ctx, Env{Alias: "account-a", Name: "甲", Gateway: "gw-1", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1, ProxyServer: "socks5://proxy.example:1080"}}); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("stored fingerprint proxy must be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHubWorkflow(t *testing.T) {
|
||||
@@ -224,3 +240,158 @@ func TestHubWorkflow(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkExitBindingRuntimeAndAuditWorkflow(t *testing.T) {
|
||||
databaseURL := os.Getenv("CREATORHUB_POSTGRES_TEST_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("set CREATORHUB_POSTGRES_TEST_URL to run PostgreSQL integration coverage")
|
||||
}
|
||||
ctx := context.Background()
|
||||
store := openFullyMigratedHub(t, ctx, isolatedDatabaseURL(t, databaseURL))
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
if _, err := store.db.ExecContext(ctx, `TRUNCATE audit_event, runtime_instance, environment_binding, network_exit,
|
||||
social_account, credential_reference, browser_env, browser_image, gateway CASCADE`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `
|
||||
INSERT INTO credential_reference (id, provider, reference_key)
|
||||
VALUES ('credential-exit', 'os_keyring', 'creatorhub/proxy-main'),
|
||||
('credential-account', 'os_keyring', 'creatorhub/account-a');
|
||||
INSERT INTO social_account
|
||||
(id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status)
|
||||
VALUES ('account-a', 'credential-account', 'mock', 'account-a', 'owned', 'authorized')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.CreateGateway(ctx, "gw-main", "http://127.0.0.1:8081", "unit-test-gateway-token"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.CreateImage(ctx, Image{Version: "148", ImageRef: "example/browser:148", Enabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
exit, err := store.CreateNetworkExit(ctx, NetworkExit{
|
||||
Protocol: "socks5", Host: "proxy.example", Port: 1080,
|
||||
ExpectedPublicIP: "203.0.113.10", ExpectedRegion: "test-region",
|
||||
}, "credential-exit")
|
||||
if err != nil || exit.HealthStatus != "unchecked" || exit.CredentialReference == nil || exit.CredentialReference.ID != "credential-exit" {
|
||||
t.Fatalf("unexpected network exit: %#v err=%v", exit, err)
|
||||
}
|
||||
exported, _ := json.Marshal(exit)
|
||||
if strings.Contains(string(exported), "creatorhub/proxy-main") {
|
||||
t.Fatalf("network exit response leaked a credential reference key: %s", exported)
|
||||
}
|
||||
access, err := store.GetNetworkExitAccess(ctx, exit.ID)
|
||||
if err != nil || access.CredentialKey != "creatorhub/proxy-main" {
|
||||
t.Fatalf("runtime-only credential resolution data unavailable: %#v err=%v", access, err)
|
||||
}
|
||||
|
||||
exit, reason, err := store.RecordNetworkExitCheck(ctx, exit.ID, ExitObservation{PublicIP: "203.0.113.11", Region: "test-region"}, "")
|
||||
if err != nil || exit.HealthStatus != "unhealthy" || reason != "exit_ip_drift" {
|
||||
t.Fatalf("identity drift must make the exit unhealthy: %#v reason=%s err=%v", exit, reason, err)
|
||||
}
|
||||
exit, reason, err = store.RecordNetworkExitCheck(ctx, exit.ID, ExitObservation{PublicIP: "203.0.113.10", Region: "test-region"}, "")
|
||||
if err != nil || exit.HealthStatus != "healthy" || reason != "exit_healthy" {
|
||||
t.Fatalf("matching identity must make the exit healthy: %#v reason=%s err=%v", exit, reason, err)
|
||||
}
|
||||
|
||||
env := Env{Alias: "environment-a", Name: "环境 A", Gateway: "gw-main", ImageVersion: "148", Fingerprint: Fingerprint{Seed: 1}}
|
||||
bound, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID)
|
||||
if err != nil || !created || bound.AccountID != "account-a" || bound.Exit.ID != exit.ID {
|
||||
t.Fatalf("create stable binding: %#v created=%v err=%v", bound, created, err)
|
||||
}
|
||||
reused, created, err := store.CreateBoundEnv(ctx, env, "account-a", exit.ID)
|
||||
if err != nil || created || reused.Alias != bound.Alias || reused.BindingID != bound.BindingID {
|
||||
t.Fatalf("same account must reuse its environment: %#v created=%v err=%v", reused, created, err)
|
||||
}
|
||||
active, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID)
|
||||
if err != nil || active.RuntimeInstanceID == "" {
|
||||
t.Fatalf("activate runtime: %#v err=%v", active, err)
|
||||
}
|
||||
|
||||
second, err := store.CreateNetworkExit(ctx, NetworkExit{Protocol: "http", Host: "proxy-2.example", Port: 8080}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, _, err = store.RecordNetworkExitCheck(ctx, second.ID, ExitObservation{PublicIP: "198.51.100.2", Region: "other"}, "")
|
||||
if err != nil || second.HealthStatus != "healthy" {
|
||||
t.Fatalf("prepare second exit: %#v err=%v", second, err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `
|
||||
INSERT INTO credential_reference (id, provider, reference_key) VALUES ('credential-account-b', 'os_keyring', 'creatorhub/account-b');
|
||||
INSERT INTO social_account
|
||||
(id, credential_reference_id, platform, platform_account_key, authorization_kind, authorization_status)
|
||||
VALUES ('account-b', 'credential-account-b', 'mock', 'account-b', 'owned', 'authorized');
|
||||
INSERT INTO browser_env (alias, name, gateway_name, image_version, fingerprint)
|
||||
VALUES ('environment-b', '环境 B', 'gw-main', '148', '{"seed":2}');
|
||||
INSERT INTO environment_binding (id, account_id, browser_env_alias)
|
||||
VALUES ('binding-b', 'account-b', 'environment-b')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacyRebound, err := store.RebindEnvironment(ctx, "environment-b", second.ID, "", 1)
|
||||
if err != nil || legacyRebound.Exit.ID != second.ID {
|
||||
t.Fatalf("legacy binding without an exit must support explicit rebind: %#v err=%v", legacyRebound, err)
|
||||
}
|
||||
if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion+1, second.ID); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("stale binding metadata must not activate a runtime: %v", err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() + interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.ActivateRuntime(ctx, env.Alias, "container-a", bound.BindingVersion, bound.Exit.ID); err != nil {
|
||||
t.Fatalf("runtime heartbeat failed: %v", err)
|
||||
}
|
||||
var renewed bool
|
||||
if err := store.db.QueryRowContext(ctx, `SELECT lease_until > now() + interval '30 seconds' FROM runtime_instance WHERE id = $1`, active.RuntimeInstanceID).Scan(&renewed); err != nil || !renewed {
|
||||
t.Fatalf("runtime lease was not renewed: renewed=%v err=%v", renewed, err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE runtime_instance SET lease_until = now() - interval '1 second' WHERE id = $1`, active.RuntimeInstanceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rebound, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", bound.BindingVersion)
|
||||
if err != nil || rebound.Exit.ID != second.ID || rebound.BindingVersion != 2 {
|
||||
t.Fatalf("expired runtime must be transactionally released before rebind: %#v err=%v", rebound, err)
|
||||
}
|
||||
if _, err := store.ActivateRuntime(ctx, env.Alias, "same-exit-container", rebound.BindingVersion, rebound.Exit.ID); err != nil {
|
||||
t.Fatalf("activate runtime before same-exit rebind: %v", err)
|
||||
}
|
||||
if _, err := store.RebindEnvironment(ctx, env.Alias, second.ID, "", rebound.BindingVersion); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("active runtime must block same-exit rebind: %v", err)
|
||||
}
|
||||
if err := store.ReleaseRuntime(ctx, env.Alias); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rebound, err = store.RebindEnvironment(ctx, env.Alias, second.ID, "rebound-container", rebound.BindingVersion)
|
||||
if err != nil || rebound.BindingVersion != 3 || rebound.RuntimeID != "rebound-container" {
|
||||
t.Fatalf("same-exit rebind must atomically CAS the binding and runtime: %#v err=%v", rebound, err)
|
||||
}
|
||||
|
||||
action := EnvironmentAction{
|
||||
OperationID: NewOperationID(), Action: "start", AccountID: rebound.AccountID,
|
||||
BrowserEnvAlias: rebound.Alias, NetworkExitID: rebound.Exit.ID, BindingVersion: rebound.BindingVersion,
|
||||
ReasonCode: "action_requested",
|
||||
}
|
||||
if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", action); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
action.Outcome, action.ReasonCode = "succeeded", "environment_started"
|
||||
if err := store.AppendEnvironmentAction(ctx, "environment_action_finished", action); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+action.OperationID+`'`, 2)
|
||||
invalidAction := action
|
||||
invalidAction.OperationID = NewOperationID()
|
||||
invalidAction.NewImageVersion = "http://operator:secret@proxy.example"
|
||||
if err := store.AppendEnvironmentAction(ctx, "environment_action_requested", invalidAction); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("invalid image version must not reach audit persistence: %v", err)
|
||||
}
|
||||
assertDatabaseCount(t, store.db, `SELECT count(*) FROM audit_event WHERE operation_id = '`+invalidAction.OperationID+`'`, 0)
|
||||
var auditText string
|
||||
if err := store.db.QueryRowContext(ctx, `SELECT string_agg(row_to_json(event)::text, '') FROM audit_event event`).Scan(&auditText); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"creatorhub/proxy-main", "credential-exit", "username", "password"} {
|
||||
if strings.Contains(auditText, forbidden) {
|
||||
t.Fatalf("audit leaked sensitive value %q: %s", forbidden, auditText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +108,11 @@ type AuditEvent struct {
|
||||
BindingVersion int64 `json:"binding_version,omitempty"`
|
||||
Actor string `json:"actor,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
OperationID string `json:"operation_id,omitempty"`
|
||||
Action string `json:"action,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
OldImageVersion string `json:"old_image_version,omitempty"`
|
||||
NewImageVersion string `json:"new_image_version,omitempty"`
|
||||
Details json.RawMessage `json:"details"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
@@ -730,6 +735,7 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, event_type, account_id, confirmation_id, confirmation_version, attempt_id, task_id,
|
||||
browser_env_alias, network_exit_id, runtime_instance_id, binding_version, actor, reason_code,
|
||||
operation_id, action, outcome, old_image_version, new_image_version,
|
||||
details, created_at
|
||||
FROM audit_event ORDER BY id`)
|
||||
if err != nil {
|
||||
@@ -740,10 +746,11 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) {
|
||||
for rows.Next() {
|
||||
var event AuditEvent
|
||||
var accountID, confirmationID, attemptID, taskID, browserEnvAlias, networkExitID sql.NullString
|
||||
var runtimeInstanceID, actor, reasonCode sql.NullString
|
||||
var runtimeInstanceID, actor, reasonCode, operationID, action, outcome, oldImage, newImage sql.NullString
|
||||
var confirmationVersion, bindingVersion sql.NullInt64
|
||||
if err := rows.Scan(&event.ID, &event.EventType, &accountID, &confirmationID, &confirmationVersion, &attemptID, &taskID,
|
||||
&browserEnvAlias, &networkExitID, &runtimeInstanceID, &bindingVersion, &actor, &reasonCode,
|
||||
&operationID, &action, &outcome, &oldImage, &newImage,
|
||||
&event.Details, &event.CreatedAt); err != nil {
|
||||
return nil, errors.New("decode audit event")
|
||||
}
|
||||
@@ -752,6 +759,8 @@ func (s *Store) Audit(ctx context.Context) ([]AuditEvent, error) {
|
||||
event.BrowserEnvAlias, event.NetworkExitID = browserEnvAlias.String, networkExitID.String
|
||||
event.RuntimeInstanceID, event.BindingVersion = runtimeInstanceID.String, bindingVersion.Int64
|
||||
event.Actor, event.ReasonCode = actor.String, reasonCode.String
|
||||
event.OperationID, event.Action, event.Outcome = operationID.String, action.String, outcome.String
|
||||
event.OldImageVersion, event.NewImageVersion = oldImage.String, newImage.String
|
||||
events = append(events, event)
|
||||
}
|
||||
return events, rows.Err()
|
||||
|
||||
@@ -343,7 +343,7 @@ func applyHubMigrationsForPhaseATest(t *testing.T, store *Store) {
|
||||
for _, migrationFile := range []struct {
|
||||
version int
|
||||
name string
|
||||
}{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}} {
|
||||
}{{2, "002_hub.sql"}, {3, "003_unified_accounts.sql"}, {4, "004_environment_actions.sql"}, {5, "005_sanitize_legacy_proxy.sql"}} {
|
||||
var applied bool
|
||||
if err := store.db.QueryRow(`SELECT EXISTS (SELECT 1 FROM schema_migration WHERE version = $1)`, migrationFile.version).Scan(&applied); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user