rogee/any-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Rogee
2025-11-14 12:11:44 +08:00
commit 39ebf61572
88 changed files with 9999 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

374
tests/integration/cache_flow_test.go Normal file
View File

@@ -0,0 +1,374 @@
package integration
import (
"context"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/cache"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/proxy"
"github.com/any-hub/any-hub/internal/server"
)
const (
dockerManifestPath = "/v2/library/cache-flow/manifests/latest"
dockerManifestNoNamespacePath = "/v2/cache-flow/manifests/latest"
)
func TestCacheFlowWithConditionalRequest(t *testing.T) {
upstream := newCacheFlowStub(t, dockerManifestPath)
defer upstream.Close()
storageDir := t.TempDir()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
CacheTTL: config.Duration(30 * time.Second),
StoragePath: storageDir,
},
Hubs: []config.HubConfig{
{
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Upstream: upstream.URL,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(storageDir)
if err != nil {
t.Fatalf("store error: %v", err)
}
client := server.NewUpstreamClient(cfg)
handler := proxy.NewHandler(client, logger, store)
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: handler,
ListenPort: 5000,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
doRequest := func() *http.Response {
req := httptest.NewRequest("GET", "http://docker.hub.local"+dockerManifestPath, nil)
req.Host = "docker.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test error: %v", err)
}
return resp
}
// Miss -> upstream fetch
resp := doRequest()
if resp.StatusCode != fiber.StatusOK {
t.Fatalf("expected status 200, got %d", resp.StatusCode)
}
if hit := resp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected cache miss header, got %s", hit)
}
resp.Body.Close()
// Cache hit with upstream HEAD revalidation
resp2 := doRequest()
if resp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
t.Fatalf("expected cache hit on second request")
}
resp2.Body.Close()
if upstream.hits != 1 {
t.Fatalf("expected single upstream GET, got %d", upstream.hits)
}
if upstream.headHits != 1 {
t.Fatalf("expected single upstream HEAD, got %d", upstream.headHits)
}
// Simulate upstream update and ensure cache refreshes.
upstream.UpdateBody([]byte("upstream v2"))
resp3 := doRequest()
if resp3.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
t.Fatalf("expected refresh when upstream changes")
}
body, _ := io.ReadAll(resp3.Body)
resp3.Body.Close()
if string(body) != "upstream v2" {
t.Fatalf("unexpected body after refresh: %s", string(body))
}
if upstream.hits != 2 {
t.Fatalf("expected upstream GET refresh, got %d hits", upstream.hits)
}
if upstream.headHits != 2 {
t.Fatalf("expected second HEAD before refresh, got %d", upstream.headHits)
}
}
func TestDockerManifestHeadDoesNotOverwriteCache(t *testing.T) {
upstream := newCacheFlowStub(t, dockerManifestPath)
defer upstream.Close()
storageDir := t.TempDir()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
CacheTTL: config.Duration(time.Minute),
StoragePath: storageDir,
},
Hubs: []config.HubConfig{
{
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Upstream: upstream.URL,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(storageDir)
if err != nil {
t.Fatalf("store error: %v", err)
}
client := server.NewUpstreamClient(cfg)
handler := proxy.NewHandler(client, logger, store)
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: handler,
ListenPort: 5000,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
doRequest := func(method string) *http.Response {
req := httptest.NewRequest(method, "http://docker.hub.local"+dockerManifestPath, nil)
req.Host = "docker.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test error: %v", err)
}
return resp
}
resp := doRequest(http.MethodGet)
if resp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
resp.Body.Close()
headResp := doRequest(http.MethodHead)
if headResp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for HEAD, got %d", headResp.StatusCode)
}
headResp.Body.Close()
if upstream.hits != 1 {
t.Fatalf("expected upstream hit only for initial GET, got %d", upstream.hits)
}
if upstream.headHits != 2 {
t.Fatalf("expected two upstream HEAD calls (explicit + revalidation), got %d", upstream.headHits)
}
cachedPath := filepath.Join(storageDir, "docker", "v2", "library", "cache-flow", "manifests", "latest")
info, err := os.Stat(cachedPath)
if err != nil {
t.Fatalf("stat cached manifest: %v", err)
}
if info.Size() == 0 {
t.Fatalf("expected cached manifest to remain non-empty")
}
resp2 := doRequest(http.MethodGet)
body, _ := io.ReadAll(resp2.Body)
resp2.Body.Close()
if string(body) != string(upstream.body) {
t.Fatalf("unexpected cached body after HEAD: %s", string(body))
}
}
func TestDockerNamespaceFallbackAddsLibrary(t *testing.T) {
stub := newCacheFlowStub(t, dockerManifestPath)
defer stub.Close()
storageDir := t.TempDir()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
CacheTTL: config.Duration(30 * time.Second),
StoragePath: storageDir,
},
Hubs: []config.HubConfig{
{
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Upstream: stub.URL,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(storageDir)
if err != nil {
t.Fatalf("store error: %v", err)
}
client := server.NewUpstreamClient(cfg)
handler := proxy.NewHandler(client, logger, store)
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: handler,
ListenPort: 5000,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
req := httptest.NewRequest("GET", "http://docker.hub.local"+dockerManifestNoNamespacePath, nil)
req.Host = "docker.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test failed: %v", err)
}
if resp.StatusCode != fiber.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 200 when fallback applies, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
if stub.hits != 1 {
t.Fatalf("expected single upstream hit, got %d", stub.hits)
}
}
type cacheFlowStub struct {
server *http.Server
listener net.Listener
URL string
mu sync.Mutex
hits int
headHits int
lastRequest *http.Request
body []byte
etag string
lastMod string
}
func newCacheFlowStub(t *testing.T, paths ...string) *cacheFlowStub {
t.Helper()
stub := &cacheFlowStub{
body: []byte("upstream payload"),
etag: `"etag-v1"`,
lastMod: time.Now().UTC().Format(http.TimeFormat),
}
if len(paths) == 0 {
paths = []string{"/pkg"}
}
mux := http.NewServeMux()
for _, p := range paths {
mux.HandleFunc(p, stub.handle)
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Skipf("unable to start stub listener: %v", err)
}
server := &http.Server{Handler: mux}
stub.server = server
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = server.Serve(listener)
}()
return stub
}
func (s *cacheFlowStub) Close() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}
func (s *cacheFlowStub) handle(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
if r.Method == http.MethodHead {
s.headHits++
} else {
s.hits++
}
s.lastRequest = r.Clone(context.Background())
s.mu.Unlock()
if r.Method == http.MethodHead {
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Etag", s.etag)
w.Header().Set("Last-Modified", s.lastMod)
w.WriteHeader(http.StatusOK)
return
}
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Etag", s.etag)
w.Header().Set("Last-Modified", s.lastMod)
_, _ = w.Write(s.body)
}
func (s *cacheFlowStub) UpdateBody(body []byte) {
s.mu.Lock()
defer s.mu.Unlock()
s.body = body
s.lastMod = time.Now().UTC().Format(http.TimeFormat)
}

604
tests/integration/credential_proxy_test.go Normal file
View File

@@ -0,0 +1,604 @@
package integration
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/cache"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/proxy"
"github.com/any-hub/any-hub/internal/server"
)
func TestCredentialProxy(t *testing.T) {
t.Run("fails without credentials", func(t *testing.T) {
stub := newCredentialAuthStub(t, "ci-user", "ci-pass")
defer stub.Close()
app := newCredentialProxyApp(t, stub, false, nil)
resp := performCredentialRequest(t, app)
if resp.StatusCode != http.StatusUnauthorized {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 401 when hub lacks credentials, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
if stub.SuccessCount() != 0 {
t.Fatalf("expected no successful upstream hits without credentials")
}
})
t.Run("succeeds with credentials", func(t *testing.T) {
stub := newCredentialAuthStub(t, "ci-user", "ci-pass")
defer stub.Close()
app := newCredentialProxyApp(t, stub, true, nil)
resp := performCredentialRequest(t, app)
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 200 when credentials configured, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
if stub.SuccessCount() == 0 {
t.Fatalf("expected at least one authorized upstream hit")
}
if last := stub.LastAuthorization(); last != stub.ExpectedAuthorization() {
t.Fatalf("expected upstream to receive header %s, got %s", stub.ExpectedAuthorization(), last)
}
})
t.Run("logs credentialed auth_mode for anonymous client", func(t *testing.T) {
stub := newCredentialAuthStub(t, "ci-user", "ci-pass")
defer stub.Close()
logBuf := &bytes.Buffer{}
app := newCredentialProxyApp(t, stub, true, logBuf)
resp := performCredentialRequest(t, app)
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 200 for first request, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
entry := findLogEntry(t, logBuf, "proxy_complete")
assertLogField(t, entry, "auth_mode", "credentialed")
assertLogField(t, entry, "hub_type", "npm")
assertLogField(t, entry, "cache_hit", false)
assertLogField(t, entry, "upstream_status", float64(200))
logBuf.Reset()
resp2 := performCredentialRequest(t, app)
if resp2.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp2.Body)
t.Fatalf("expected 200 for cache hit, got %d (body=%s)", resp2.StatusCode, string(body))
}
resp2.Body.Close()
entry = findLogEntry(t, logBuf, "proxy_complete")
assertLogField(t, entry, "cache_hit", true)
})
t.Run("retries once when upstream temporarily rejects auth", func(t *testing.T) {
stub := newCredentialAuthStub(t, "ci-user", "ci-pass")
defer stub.Close()
stub.FailNextAuthorizedRequests(1)
app := newCredentialProxyApp(t, stub, true, nil)
resp := performCredentialRequest(t, app)
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 200 after retry, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
if stub.SuccessCount() != 1 {
t.Fatalf("expected single successful upstream call after retry, got %d", stub.SuccessCount())
}
if stub.UnauthorizedCount() != 1 {
t.Fatalf("expected one unauthorized response before retry, got %d", stub.UnauthorizedCount())
}
if stub.TotalRequests() != 2 {
t.Fatalf("expected exactly two upstream attempts, got %d", stub.TotalRequests())
}
})
t.Run("stops after single retry when upstream keeps failing", func(t *testing.T) {
stub := newCredentialAuthStub(t, "ci-user", "ci-pass")
defer stub.Close()
stub.FailNextAuthorizedRequests(2)
app := newCredentialProxyApp(t, stub, true, nil)
resp := performCredentialRequest(t, app)
if resp.StatusCode != http.StatusUnauthorized {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 401 after retry exhaustion, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
if stub.TotalRequests() != 2 {
t.Fatalf("expected two attempts (original + retry), got %d", stub.TotalRequests())
}
})
}
func TestDockerProxyHandlesBearerTokenExchange(t *testing.T) {
stub := newDockerBearerStub(t, "ci-user", "ci-pass")
defer stub.Close()
app := newDockerProxyApp(t, stub)
req := httptest.NewRequest("GET", "http://docker.hub.local/v2/library/alpine/manifests/latest", nil)
req.Host = "docker.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test failed: %v", err)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 200 after token exchange, got %d (body=%s)", resp.StatusCode, string(body))
}
resp.Body.Close()
if stub.TokenHits() != 1 {
t.Fatalf("expected single token request, got %d", stub.TokenHits())
}
if stub.ManifestHits() != 2 {
t.Fatalf("expected manifest retried after token, got %d hits", stub.ManifestHits())
}
expectedBearer := "Bearer " + stub.tokenValue
if stub.ManifestAuth() != expectedBearer {
t.Fatalf("expected manifest Authorization %s, got %s", expectedBearer, stub.ManifestAuth())
}
if stub.TokenAuth() != stub.ExpectedBasic() {
t.Fatalf("expected token endpoint to receive basic auth %s, got %s", stub.ExpectedBasic(), stub.TokenAuth())
}
}
func performCredentialRequest(t *testing.T, app *fiber.App) *http.Response {
t.Helper()
req := httptest.NewRequest("GET", "http://secure.hub.local/private/data", nil)
req.Host = "secure.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test failed: %v", err)
}
return resp
}
func newCredentialProxyApp(t *testing.T, stub *credentialAuthStub, withCredentials bool, logSink io.Writer) *fiber.App {
t.Helper()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
StoragePath: t.TempDir(),
CacheTTL: config.Duration(time.Hour),
MaxMemoryCache: 1,
MaxRetries: 0,
InitialBackoff: config.Duration(time.Second),
UpstreamTimeout: config.Duration(30 * time.Second),
},
Hubs: []config.HubConfig{
{
Name: "secure",
Domain: "secure.hub.local",
Type: "npm",
Upstream: stub.URL,
},
},
}
if withCredentials {
cfg.Hubs[0].Username = stub.username
cfg.Hubs[0].Password = stub.password
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
if logSink != nil {
logger.SetOutput(logSink)
} else {
logger.SetOutput(io.Discard)
}
logger.SetFormatter(&logrus.JSONFormatter{TimestampFormat: time.RFC3339Nano})
store, err := cache.NewStore(cfg.Global.StoragePath)
if err != nil {
t.Fatalf("store error: %v", err)
}
client := server.NewUpstreamClient(cfg)
handler := proxy.NewHandler(client, logger, store)
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: handler,
ListenPort: cfg.Global.ListenPort,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
return app
}
func newDockerProxyApp(t *testing.T, stub *dockerBearerStub) *fiber.App {
t.Helper()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
StoragePath: t.TempDir(),
CacheTTL: config.Duration(time.Hour),
MaxMemoryCache: 1,
MaxRetries: 0,
InitialBackoff: config.Duration(time.Second),
UpstreamTimeout: config.Duration(30 * time.Second),
},
Hubs: []config.HubConfig{
{
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Upstream: stub.URL,
Username: stub.username,
Password: stub.password,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(cfg.Global.StoragePath)
if err != nil {
t.Fatalf("store error: %v", err)
}
client := server.NewUpstreamClient(cfg)
handler := proxy.NewHandler(client, logger, store)
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: handler,
ListenPort: cfg.Global.ListenPort,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
return app
}
type credentialAuthStub struct {
server *http.Server
listener net.Listener
URL string
username string
password string
mu sync.Mutex
lastAuth string
successCount int
unauthCount int
totalRequests int
expectedBasic string
forceFailures int
initialFailure int
}
func newCredentialAuthStub(t *testing.T, username, password string) *credentialAuthStub {
t.Helper()
stub := &credentialAuthStub{
username: username,
password: password,
expectedBasic: "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))),
}
mux := http.NewServeMux()
mux.HandleFunc("/private/data", stub.handle)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Skipf("unable to start upstream stub listener: %v", err)
}
server := &http.Server{Handler: mux}
stub.server = server
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = server.Serve(listener)
}()
return stub
}
func (s *credentialAuthStub) handle(w http.ResponseWriter, r *http.Request) {
auth := r.Header.Get("Authorization")
s.mu.Lock()
s.totalRequests++
s.lastAuth = auth
shouldForceFail := false
if auth == s.expectedBasic && s.forceFailures > 0 {
s.forceFailures--
shouldForceFail = true
}
if auth == s.expectedBasic && !shouldForceFail {
s.successCount++
s.mu.Unlock()
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"status":"ok"}`))
return
}
s.unauthCount++
s.mu.Unlock()
if auth != s.expectedBasic || shouldForceFail {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("missing or invalid auth"))
return
}
}
func (s *credentialAuthStub) Close() {
if s == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}
func (s *credentialAuthStub) LastAuthorization() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.lastAuth
}
func (s *credentialAuthStub) SuccessCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.successCount
}
func (s *credentialAuthStub) ExpectedAuthorization() string {
return s.expectedBasic
}
func (s *credentialAuthStub) UnauthorizedCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.unauthCount
}
func (s *credentialAuthStub) TotalRequests() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.totalRequests
}
func (s *credentialAuthStub) FailNextAuthorizedRequests(n int) {
s.mu.Lock()
s.forceFailures = n
s.mu.Unlock()
}
type dockerBearerStub struct {
server *http.Server
listener net.Listener
URL string
username string
password string
expectedBasic string
tokenValue string
mu sync.Mutex
manifestAuth string
tokenAuth string
manifestHits int
tokenHits int
}
func newDockerBearerStub(t *testing.T, username, password string) *dockerBearerStub {
t.Helper()
stub := &dockerBearerStub{
username: username,
password: password,
expectedBasic: "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))),
tokenValue: "test-token",
}
mux := http.NewServeMux()
mux.HandleFunc("/v2/", stub.handleProbe)
mux.HandleFunc("/v2/library/alpine/manifests/latest", stub.handleManifest)
mux.HandleFunc("/token", stub.handleToken)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Skipf("unable to start docker stub listener: %v", err)
}
server := &http.Server{Handler: mux}
stub.server = server
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = server.Serve(listener)
}()
return stub
}
func (s *dockerBearerStub) handleManifest(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
s.manifestHits++
s.manifestAuth = r.Header.Get("Authorization")
authHeader := fmt.Sprintf(`Bearer realm="%s/token",service="registry.test",scope="repository:library/alpine:pull"`, s.URL)
expectBearer := "Bearer " + s.tokenValue
success := s.manifestAuth == expectBearer
s.mu.Unlock()
if success {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"schemaVersion":2}`))
return
}
w.Header().Set("Www-Authenticate", authHeader)
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("token required"))
}
func (s *dockerBearerStub) handleToken(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
s.tokenHits++
s.tokenAuth = r.Header.Get("Authorization")
service := r.URL.Query().Get("service")
scope := r.URL.Query().Get("scope")
expectAuth := s.expectedBasic
valid := s.tokenAuth == expectAuth && service == "registry.test" && scope == "repository:library/alpine:pull"
s.mu.Unlock()
if !valid {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("invalid credentials"))
return
}
w.Header().Set("Content-Type", "application/json")
resp := map[string]string{"token": s.tokenValue}
data, _ := json.Marshal(resp)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
}
func (s *dockerBearerStub) handleProbe(w http.ResponseWriter, r *http.Request) {
authHeader := fmt.Sprintf(`Bearer realm="%s/token",service="registry.test",scope="repository:library/alpine:pull"`, s.URL)
if r.Header.Get("Authorization") == "Bearer "+s.tokenValue {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
return
}
w.Header().Set("Www-Authenticate", authHeader)
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("probe auth required"))
}
func (s *dockerBearerStub) Close() {
if s == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}
func (s *dockerBearerStub) ManifestAuth() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.manifestAuth
}
func (s *dockerBearerStub) TokenAuth() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.tokenAuth
}
func (s *dockerBearerStub) ManifestHits() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.manifestHits
}
func (s *dockerBearerStub) TokenHits() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.tokenHits
}
func (s *dockerBearerStub) ExpectedBasic() string {
return s.expectedBasic
}
func findLogEntry(t *testing.T, buf *bytes.Buffer, msg string) map[string]any {
t.Helper()
entries := parseLogBuffer(t, buf)
for _, entry := range entries {
if entry["msg"] == msg {
return entry
}
}
t.Fatalf("log entry with msg=%s not found; entries=%v", msg, entries)
return nil
}
func parseLogBuffer(t *testing.T, buf *bytes.Buffer) []map[string]any {
t.Helper()
var result []map[string]any
for {
line, err := buf.ReadBytes('\n')
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("read log buffer: %v", err)
}
if len(bytes.TrimSpace(line)) == 0 {
continue
}
var entry map[string]any
if err := json.Unmarshal(bytes.TrimSpace(line), &entry); err != nil {
t.Fatalf("parse log entry: %v (line=%s)", err, string(line))
}
result = append(result, entry)
}
return result
}
func assertLogField(t *testing.T, entry map[string]any, key string, expected any) {
t.Helper()
value, ok := entry[key]
if !ok {
t.Fatalf("log entry missing %s field: %v", key, entry)
}
if value != expected {
t.Fatalf("log entry %s mismatch: expected %v got %v", key, expected, value)
}
}

116
tests/integration/docker_sample_test.go Normal file
View File

@@ -0,0 +1,116 @@
package integration
import (
"io"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/cache"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/proxy"
"github.com/any-hub/any-hub/internal/server"
)
func TestDockerSampleConfigWithStub(t *testing.T) {
stub := newCacheFlowStub(t, dockerManifestPath)
defer stub.Close()
tempDir := t.TempDir()
storageDir := filepath.Join(tempDir, "storage")
data, err := os.ReadFile(repoPath(t, "configs", "docker.sample.toml"))
if err != nil {
t.Fatalf("read sample config: %v", err)
}
content := strings.ReplaceAll(string(data), "./storage/docker", storageDir)
content = strings.Replace(content, "https://registry-1.docker.io", stub.URL, 1)
tempConfig := filepath.Join(tempDir, "docker.sample.toml")
if err := os.WriteFile(tempConfig, []byte(content), 0o600); err != nil {
t.Fatalf("write temp config: %v", err)
}
cfg, err := config.Load(tempConfig)
if err != nil {
t.Fatalf("config load error: %v", err)
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(cfg.Global.StoragePath)
if err != nil {
t.Fatalf("store error: %v", err)
}
client := server.NewUpstreamClient(cfg)
handler := proxy.NewHandler(client, logger, store)
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: handler,
ListenPort: cfg.Global.ListenPort,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
req := httptest.NewRequest("GET", "http://docker.hub.local"+dockerManifestPath, nil)
req.Host = "docker.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app test error: %v", err)
}
if resp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
if hit := resp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected miss header, got %s", hit)
}
resp.Body.Close()
// second request should hit cache
req2 := httptest.NewRequest("GET", "http://docker.hub.local"+dockerManifestPath, nil)
req2.Host = "docker.hub.local"
resp2, err := app.Test(req2)
if err != nil {
t.Fatalf("app test error: %v", err)
}
if resp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
t.Fatalf("expected cache hit on second request")
}
resp2.Body.Close()
}
func repoPath(t *testing.T, elems ...string) string {
t.Helper()
dir, err := os.Getwd()
if err != nil {
t.Fatalf("getwd: %v", err)
}
for i := 0; i < 10; i++ {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return filepath.Join(append([]string{dir}, elems...)...)
}
parent := filepath.Dir(dir)
if parent == dir {
break
}
dir = parent
}
t.Fatalf("unable to locate repository root from %s", dir)
return ""
}

120
tests/integration/host_routing_test.go Normal file
View File

@@ -0,0 +1,120 @@
package integration
import (
"io"
"net/http/httptest"
"testing"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/server"
)
func TestHostRoutingDistinguishesDomainsOnSinglePort(t *testing.T) {
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
CacheTTL: config.Duration(3600),
},
Hubs: []config.HubConfig{
{
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Upstream: "https://registry-1.docker.io",
},
{
Name: "npm",
Domain: "npm.hub.local",
Type: "npm",
Upstream: "https://registry.npmjs.org",
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("failed to create registry: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
app := newIntegrationApp(t, cfg.Global.ListenPort, logger, registry, &proxyRecorder{})
recorder := app.recorder
req := httptest.NewRequest("GET", "http://docker.hub.local/v2/", nil)
req.Host = "docker.hub.local"
req.Header.Set("Host", "docker.hub.local")
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app test failed: %v", err)
}
if resp.StatusCode != fiber.StatusNoContent {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 204 from docker hub, got %d (body=%s)", resp.StatusCode, string(body))
}
if recorder.routeName != "docker" {
t.Fatalf("expected docker route, got %s", recorder.routeName)
}
req2 := httptest.NewRequest("GET", "http://npm.hub.local/v2/", nil)
req2.Host = "npm.hub.local"
req2.Header.Set("Host", "npm.hub.local")
resp2, err := app.Test(req2)
if err != nil {
t.Fatalf("app test failed: %v", err)
}
if resp2.StatusCode != fiber.StatusNoContent {
body, _ := io.ReadAll(resp2.Body)
t.Fatalf("expected 204 from npm hub, got %d (body=%s)", resp2.StatusCode, string(body))
}
if recorder.routeName != "npm" {
t.Fatalf("expected npm route, got %s", recorder.routeName)
}
req3 := httptest.NewRequest("GET", "http://unknown.hub.local/v2/", nil)
req3.Host = "unknown.hub.local"
resp3, err := app.Test(req3)
if err != nil {
t.Fatalf("app test failed: %v", err)
}
if resp3.StatusCode != fiber.StatusNotFound {
body, _ := io.ReadAll(resp3.Body)
t.Fatalf("expected 404 for unmapped host, got %d (body=%s)", resp3.StatusCode, string(body))
}
}
type integrationApp struct {
*fiber.App
recorder *proxyRecorder
}
func newIntegrationApp(t *testing.T, port int, logger *logrus.Logger, registry *server.HubRegistry, proxy server.ProxyHandler) *integrationApp {
t.Helper()
recorder, ok := proxy.(*proxyRecorder)
if !ok {
recorder = &proxyRecorder{}
}
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
ListenPort: port,
Proxy: recorder,
})
if err != nil {
t.Fatalf("failed to create app: %v", err)
}
return &integrationApp{App: app, recorder: recorder}
}
type proxyRecorder struct {
routeName string
}
func (p *proxyRecorder) Handle(c fiber.Ctx, route *server.HubRoute) error {
p.routeName = route.Config.Name
return c.SendStatus(fiber.StatusNoContent)
}

60
tests/integration/interrupt_test.go Normal file
View File

@@ -0,0 +1,60 @@
package integration
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"testing"
"github.com/any-hub/any-hub/internal/cache"
)
func TestCacheWriteCleanupOnInterruptedStream(t *testing.T) {
tmpDir := t.TempDir()
store, err := cache.NewStore(tmpDir)
if err != nil {
t.Fatalf("store init error: %v", err)
}
loc := cache.Locator{HubName: "docker", Path: "/interrupt/blob.tar"}
reader := &flakyReader{
payload: []byte("partial_data"),
failAfter: 5,
}
if _, err := store.Put(context.Background(), loc, reader, cache.PutOptions{}); err == nil {
t.Fatalf("expected error from interrupted reader")
}
target := filepath.Join(tmpDir, "docker", "interrupt", "blob.tar")
if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("expected no final file, got err=%v", err)
}
pattern := filepath.Join(tmpDir, "docker", "interrupt", ".cache-*")
matches, _ := filepath.Glob(pattern)
if len(matches) != 0 {
t.Fatalf("temporary files should be cleaned up, found %v", matches)
}
}
type flakyReader struct {
payload []byte
failAfter int
readBytes int
}
func (f *flakyReader) Read(p []byte) (int, error) {
if f.readBytes >= f.failAfter {
return 0, io.ErrUnexpectedEOF
}
remaining := f.failAfter - f.readBytes
if remaining > len(p) {
remaining = len(p)
}
copy(p[:remaining], f.payload[f.readBytes:f.readBytes+remaining])
f.readBytes += remaining
return remaining, nil
}

284
tests/integration/upstream_stub_test.go Normal file
View File

@@ -0,0 +1,284 @@
package integration
import (
"bytes"
"context"
"encoding/json"
"io"
"net"
"net/http"
"path"
"sync"
"testing"
"time"
)
type upstreamMode string
const (
upstreamDocker upstreamMode = "docker"
upstreamNPM upstreamMode = "npm"
)
// upstreamStub 暴露简单的 Docker/NPM 上游模拟器，供集成测试复用。
type upstreamStub struct {
server *http.Server
listener net.Listener
URL string
mu sync.Mutex
requests []RecordedRequest
mode upstreamMode
blobBytes []byte
}
// RecordedRequest 捕获每次请求的方法/路径/Host/Headers，便于断言代理行为。
type RecordedRequest struct {
Method string
Path string
Host string
Headers http.Header
Body []byte
}
func newUpstreamStub(t *testing.T, mode upstreamMode) *upstreamStub {
t.Helper()
mux := http.NewServeMux()
stub := &upstreamStub{
mode: mode,
blobBytes: []byte("stub-layer-payload"),
}
switch mode {
case upstreamDocker:
registerDockerHandlers(mux, stub.blobBytes)
case upstreamNPM:
registerNPMHandlers(mux)
default:
t.Fatalf("unsupported stub mode: %s", mode)
}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
stub.recordRequest(r)
mux.ServeHTTP(w, r)
})
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Skipf("unable to start upstream stub listener: %v", err)
}
server := &http.Server{Handler: handler}
stub.server = server
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = server.Serve(listener)
}()
return stub
}
func (s *upstreamStub) Close() {
if s == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}
func (s *upstreamStub) recordRequest(r *http.Request) {
body, _ := io.ReadAll(r.Body)
_ = r.Body.Close()
s.mu.Lock()
s.requests = append(s.requests, RecordedRequest{
Method: r.Method,
Path: r.URL.Path,
Host: r.Host,
Headers: cloneHeader(r.Header),
Body: body,
})
s.mu.Unlock()
r.Body = io.NopCloser(bytes.NewReader(body))
}
func (s *upstreamStub) Requests() []RecordedRequest {
s.mu.Lock()
defer s.mu.Unlock()
result := make([]RecordedRequest, len(s.requests))
copy(result, s.requests)
return result
}
func registerDockerHandlers(mux *http.ServeMux, blob []byte) {
mux.HandleFunc("/v2/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v2/" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"docker":"ok"}`))
})
mux.HandleFunc("/v2/library/sample/manifests/latest", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
resp := map[string]any{
"schemaVersion": 2,
"name": "library/sample",
"tag": "latest",
"layers": []map[string]any{
{
"digest": "sha256:deadbeef",
"size": len(blob),
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
},
},
}
_ = json.NewEncoder(w).Encode(resp)
})
mux.HandleFunc("/v2/library/sample/blobs/", func(w http.ResponseWriter, r *http.Request) {
digest := path.Base(r.URL.Path)
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Docker-Content-Digest", digest)
_, _ = w.Write(blob)
})
}
func registerNPMHandlers(mux *http.ServeMux) {
mux.HandleFunc("/lodash", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
resp := map[string]any{
"name": "lodash",
"dist-tags": map[string]string{
"latest": "4.17.21",
},
"versions": map[string]any{
"4.17.21": map[string]any{
"dist": map[string]any{
"tarball": r.Host + "/lodash/-/lodash-4.17.21.tgz",
},
},
},
}
_ = json.NewEncoder(w).Encode(resp)
})
mux.HandleFunc("/lodash/-/lodash-4.17.21.tgz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write([]byte("tarball-bytes"))
})
}
func cloneHeader(src http.Header) http.Header {
dst := make(http.Header, len(src))
for k, values := range src {
cp := make([]string, len(values))
copy(cp, values)
dst[k] = cp
}
return dst
}
func TestDockerStubServesManifestAndBlob(t *testing.T) {
stub := newUpstreamStub(t, upstreamDocker)
defer stub.Close()
pingResp, err := http.Get(stub.URL + "/v2/")
if err != nil {
t.Fatalf("docker ping failed: %v", err)
}
pingResp.Body.Close()
if pingResp.StatusCode != http.StatusOK {
t.Fatalf("docker ping unexpected status: %d", pingResp.StatusCode)
}
resp, err := http.Get(stub.URL + "/v2/library/sample/manifests/latest")
if err != nil {
t.Fatalf("manifest request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected manifest status: %d", resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
if !bytes.Contains(body, []byte(`"name":"library/sample"`)) {
t.Fatalf("manifest body unexpected: %s", string(body))
}
layerResp, err := http.Get(stub.URL + "/v2/library/sample/blobs/sha256:deadbeef")
if err != nil {
t.Fatalf("layer request failed: %v", err)
}
defer layerResp.Body.Close()
layer, _ := io.ReadAll(layerResp.Body)
if !bytes.Equal(layer, stub.blobBytes) {
t.Fatalf("layer bytes mismatch: %s", string(layer))
}
if got := len(stub.Requests()); got != 3 {
t.Fatalf("expected 3 recorded requests, got %d", got)
}
}
func TestNPMStubServesMetadataAndTarball(t *testing.T) {
stub := newUpstreamStub(t, upstreamNPM)
defer stub.Close()
resp, err := http.Get(stub.URL + "/lodash")
if err != nil {
t.Fatalf("metadata request failed: %v", err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if !bytes.Contains(body, []byte(`"latest":"4.17.21"`)) {
t.Fatalf("metadata unexpected: %s", string(body))
}
tarballResp, err := http.Get(stub.URL + "/lodash/-/lodash-4.17.21.tgz")
if err != nil {
t.Fatalf("tarball request failed: %v", err)
}
defer tarballResp.Body.Close()
data, _ := io.ReadAll(tarballResp.Body)
if !bytes.Equal(data, []byte("tarball-bytes")) {
t.Fatalf("tarball payload mismatch: %s", string(data))
}
if got := len(stub.Requests()); got != 2 {
t.Fatalf("expected 2 recorded requests, got %d", got)
}
}
func TestUpstreamStubSupportsAnonymousCurlHostHeader(t *testing.T) {
stub := newUpstreamStub(t, upstreamDocker)
defer stub.Close()
req, err := http.NewRequest(http.MethodGet, stub.URL+"/v2/", nil)
if err != nil {
t.Fatalf("create request failed: %v", err)
}
req.Host = "docker.hub.local"
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("curl-style request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("expected 200 from curl-style request, got %d body=%s", resp.StatusCode, string(body))
}
if got := stub.Requests(); len(got) != 1 || got[0].Host != "docker.hub.local" {
t.Fatalf("expected recorded host docker.hub.local, got %v", got)
}
}