From 8cff19eccaf2c58b05401e324bf043ccb620767e Mon Sep 17 00:00:00 2001 From: Rogee Date: Mon, 17 Nov 2025 10:16:27 +0800 Subject: [PATCH] add composer --- config.example.toml | 10 + internal/config/loader.go | 7 +- internal/config/modules.go | 1 + internal/config/validation.go | 11 +- internal/hubmodule/composer/module.go | 28 +++ internal/proxy/composer_rewrite.go | 183 +++++++++++++++ internal/proxy/handler.go | 59 +++++ internal/server/router.go | 2 + main.go | 7 +- tests/integration/composer_proxy_test.go | 276 +++++++++++++++++++++++ 10 files changed, 577 insertions(+), 7 deletions(-) create mode 100644 internal/hubmodule/composer/module.go create mode 100644 internal/proxy/composer_rewrite.go create mode 100644 tests/integration/composer_proxy_test.go diff --git a/config.example.toml b/config.example.toml index 9f32f34..6be1da2 100644 --- a/config.example.toml +++ b/config.example.toml @@ -69,3 +69,13 @@ Proxy = "" Username = "" Password = "" Type = "pypi" + +# Composer Repository +[[Hub]] +Domain = "composer.hub.local" +Name = "composer" +Upstream = "https://repo.packagist.org" +Proxy = "" +Username = "" +Password = "" +Type = "composer" diff --git a/internal/config/loader.go b/internal/config/loader.go index b24e5be..d5c3047 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -90,7 +90,12 @@ func applyHubDefaults(h *HubConfig) { h.CacheTTL = Duration(0) } if trimmed := strings.TrimSpace(h.Module); trimmed == "" { - h.Module = hubmodule.DefaultModuleKey() + typeKey := strings.ToLower(strings.TrimSpace(h.Type)) + if meta, ok := hubmodule.Resolve(typeKey); ok { + h.Module = meta.Key + } else { + h.Module = hubmodule.DefaultModuleKey() + } } else { h.Module = strings.ToLower(trimmed) } diff --git a/internal/config/modules.go b/internal/config/modules.go index 1039f2e..a5c39d5 100644 --- a/internal/config/modules.go +++ b/internal/config/modules.go @@ -1,6 +1,7 @@ package config import ( + _ "github.com/any-hub/any-hub/internal/hubmodule/composer" _ "github.com/any-hub/any-hub/internal/hubmodule/docker" _ "github.com/any-hub/any-hub/internal/hubmodule/legacy" _ "github.com/any-hub/any-hub/internal/hubmodule/npm" diff --git a/internal/config/validation.go b/internal/config/validation.go index 7c6cb12..1266bb1 100644 --- a/internal/config/validation.go +++ b/internal/config/validation.go @@ -11,13 +11,14 @@ import ( ) var supportedHubTypes = map[string]struct{}{ - "docker": {}, - "npm": {}, - "go": {}, - "pypi": {}, + "docker": {}, + "npm": {}, + "go": {}, + "pypi": {}, + "composer": {}, } -const supportedHubTypeList = "docker|npm|go|pypi" +const supportedHubTypeList = "docker|npm|go|pypi|composer" // Validate 针对语义级别做进一步校验，防止非法配置启动服务。 func (c *Config) Validate() error { diff --git a/internal/hubmodule/composer/module.go b/internal/hubmodule/composer/module.go new file mode 100644 index 0000000..b7768ce --- /dev/null +++ b/internal/hubmodule/composer/module.go @@ -0,0 +1,28 @@ +// Package composer declares metadata for Composer (PHP) package proxying. +package composer + +import ( + "time" + + "github.com/any-hub/any-hub/internal/hubmodule" +) + +const composerDefaultTTL = 6 * time.Hour + +func init() { + hubmodule.MustRegister(hubmodule.ModuleMetadata{ + Key: "composer", + Description: "Composer packages proxy with metadata+dist caching", + MigrationState: hubmodule.MigrationStateBeta, + SupportedProtocols: []string{ + "composer", + }, + CacheStrategy: hubmodule.CacheStrategyProfile{ + TTLHint: composerDefaultTTL, + ValidationMode: hubmodule.ValidationModeETag, + DiskLayout: "raw_path", + RequiresMetadataFile: false, + SupportsStreamingWrite: true, + }, + }) +} diff --git a/internal/proxy/composer_rewrite.go b/internal/proxy/composer_rewrite.go new file mode 100644 index 0000000..85e0502 --- /dev/null +++ b/internal/proxy/composer_rewrite.go @@ -0,0 +1,183 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/any-hub/any-hub/internal/server" +) + +func (h *Handler) rewriteComposerResponse(route *server.HubRoute, resp *http.Response, path string) (*http.Response, error) { + if resp == nil || route == nil || route.Config.Type != "composer" { + return resp, nil + } + if !isComposerMetadataPath(path) { + return resp, nil + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return resp, err + } + resp.Body.Close() + + rewritten, changed, err := rewriteComposerMetadata(body, route.Config.Domain) + if err != nil { + resp.Body = io.NopCloser(bytes.NewReader(body)) + return resp, err + } + if !changed { + resp.Body = io.NopCloser(bytes.NewReader(body)) + return resp, nil + } + + resp.Body = io.NopCloser(bytes.NewReader(rewritten)) + resp.ContentLength = int64(len(rewritten)) + resp.Header.Set("Content-Length", strconv.Itoa(len(rewritten))) + resp.Header.Set("Content-Type", "application/json") + resp.Header.Del("Content-Encoding") + resp.Header.Del("Etag") + return resp, nil +} + +func rewriteComposerMetadata(body []byte, domain string) ([]byte, bool, error) { + type packagesRoot struct { + Packages map[string]json.RawMessage `json:"packages"` + } + var root packagesRoot + if err := json.Unmarshal(body, &root); err != nil { + return nil, false, err + } + if len(root.Packages) == 0 { + return body, false, nil + } + + changed := false + for name, raw := range root.Packages { + updated, rewritten, err := rewriteComposerPackagesPayload(raw, domain) + if err != nil { + return nil, false, err + } + if rewritten { + root.Packages[name] = updated + changed = true + } + } + if !changed { + return body, false, nil + } + data, err := json.Marshal(root) + if err != nil { + return nil, false, err + } + return data, true, nil +} + +func rewriteComposerPackagesPayload(raw json.RawMessage, domain string) (json.RawMessage, bool, error) { + var asArray []map[string]any + if err := json.Unmarshal(raw, &asArray); err == nil { + rewrote := rewriteComposerVersionSlice(asArray, domain) + if !rewrote { + return raw, false, nil + } + data, err := json.Marshal(asArray) + return data, true, err + } + + var asMap map[string]map[string]any + if err := json.Unmarshal(raw, &asMap); err == nil { + rewrote := rewriteComposerVersionMap(asMap, domain) + if !rewrote { + return raw, false, nil + } + data, err := json.Marshal(asMap) + return data, true, err + } + + return raw, false, nil +} + +func rewriteComposerVersionSlice(items []map[string]any, domain string) bool { + changed := false + for _, entry := range items { + if rewriteComposerVersion(entry, domain) { + changed = true + } + } + return changed +} + +func rewriteComposerVersionMap(items map[string]map[string]any, domain string) bool { + changed := false + for _, entry := range items { + if rewriteComposerVersion(entry, domain) { + changed = true + } + } + return changed +} + +func rewriteComposerVersion(entry map[string]any, domain string) bool { + if entry == nil { + return false + } + distVal, ok := entry["dist"].(map[string]any) + if !ok { + return false + } + urlValue, ok := distVal["url"].(string) + if !ok || urlValue == "" { + return false + } + rewritten := rewriteComposerDistURL(domain, urlValue) + if rewritten == urlValue { + return false + } + distVal["url"] = rewritten + return true +} + +func rewriteComposerDistURL(domain, original string) string { + parsed, err := url.Parse(original) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return original + } + prefix := fmt.Sprintf("/dist/%s/%s", parsed.Scheme, parsed.Host) + newURL := url.URL{ + Scheme: "https", + Host: domain, + Path: prefix + parsed.Path, + RawQuery: parsed.RawQuery, + Fragment: parsed.Fragment, + } + if raw := parsed.RawPath; raw != "" { + newURL.RawPath = prefix + raw + } + return newURL.String() +} + +func isComposerMetadataPath(path string) bool { + switch { + case path == "/packages.json": + return true + case strings.HasPrefix(path, "/p2/"): + return true + case strings.HasPrefix(path, "/p/"): + return true + case strings.HasPrefix(path, "/provider-"): + return true + case strings.HasPrefix(path, "/providers/"): + return true + default: + return false + } +} + +func isComposerDistPath(path string) bool { + return strings.HasPrefix(path, "/dist/") +} diff --git a/internal/proxy/handler.go b/internal/proxy/handler.go index 3300ad8..6df5404 100644 --- a/internal/proxy/handler.go +++ b/internal/proxy/handler.go @@ -200,6 +200,15 @@ func (h *Handler) fetchAndStream( "hub": route.Config.Name, }).Warn("pypi_rewrite_failed") } + } else if route.Config.Type == "composer" { + if rewritten, rewriteErr := h.rewriteComposerResponse(route, resp, requestPath(c)); rewriteErr == nil { + resp = rewritten + } else { + h.logger.WithError(rewriteErr).WithFields(logrus.Fields{ + "action": "composer_rewrite", + "hub": route.Config.Name, + }).Warn("composer_rewrite_failed") + } } defer resp.Body.Close() @@ -446,6 +455,8 @@ func inferCachedContentType(route *server.HubRoute, locator cache.Locator) strin switch { case strings.HasSuffix(clean, ".zip"): return "application/zip" + case strings.HasSuffix(clean, ".json"): + return "application/json" case strings.HasSuffix(clean, ".mod"): return "text/plain" case strings.HasSuffix(clean, ".info"): @@ -624,6 +635,11 @@ func resolveUpstreamURL(route *server.HubRoute, base *url.URL, c fiber.Ctx) *url return filesBase.ResolveReference(relative) } } + if route != nil && route.Config.Type == "composer" && strings.HasPrefix(clean, "/dist/") { + if distTarget, ok := parseComposerDistURL(clean, string(uri.QueryString())); ok { + return distTarget + } + } relative := &url.URL{Path: clean, RawPath: clean} if query := string(uri.QueryString()); query != "" { relative.RawQuery = query @@ -701,6 +717,15 @@ func determineCachePolicy(route *server.HubRoute, locator cache.Locator, method } policy.requireRevalidate = true return policy + case "composer": + if isComposerDistPath(path) { + return policy + } + if isComposerMetadataPath(path) { + policy.requireRevalidate = true + return policy + } + return cachePolicy{} default: return policy } @@ -899,6 +924,38 @@ func applyPyPISimpleFallback(route *server.HubRoute, path string) (string, bool) return "/simple/" + trimmed + "/", true } +func parseComposerDistURL(path string, rawQuery string) (*url.URL, bool) { + if !strings.HasPrefix(path, "/dist/") { + return nil, false + } + trimmed := strings.TrimPrefix(path, "/dist/") + parts := strings.SplitN(trimmed, "/", 3) + if len(parts) < 3 { + return nil, false + } + scheme := parts[0] + host := parts[1] + rest := parts[2] + if scheme == "" || host == "" { + return nil, false + } + if rest == "" { + rest = "/" + } else { + rest = "/" + rest + } + target := &url.URL{ + Scheme: scheme, + Host: host, + Path: rest, + RawPath: rest, + } + if rawQuery != "" { + target.RawQuery = rawQuery + } + return target, true +} + type bearerChallenge struct { Realm string Service string @@ -1119,6 +1176,8 @@ func ensureProxyHubType(route *server.HubRoute) error { return nil case "pypi": return nil + case "composer": + return nil default: return fmt.Errorf("unsupported hub type: %s", route.Config.Type) } diff --git a/internal/server/router.go b/internal/server/router.go index f62eeb2..21e97c8 100644 --- a/internal/server/router.go +++ b/internal/server/router.go @@ -150,6 +150,8 @@ func ensureRouterHubType(route *HubRoute) error { return nil case "pypi": return nil + case "composer": + return nil default: return fmt.Errorf("unsupported hub type: %s", route.Config.Type) } diff --git a/main.go b/main.go index 0cf4987..bf84eed 100644 --- a/main.go +++ b/main.go @@ -134,7 +134,12 @@ func parseCLIFlags(args []string) (cliOptions, error) { }, nil } -func startHTTPServer(cfg *config.Config, registry *server.HubRegistry, proxyHandler server.ProxyHandler, logger *logrus.Logger) error { +func startHTTPServer( + cfg *config.Config, + registry *server.HubRegistry, + proxyHandler server.ProxyHandler, + logger *logrus.Logger, +) error { port := cfg.Global.ListenPort app, err := server.NewApp(server.AppOptions{ Logger: logger, diff --git a/tests/integration/composer_proxy_test.go b/tests/integration/composer_proxy_test.go new file mode 100644 index 0000000..2c2d8c1 --- /dev/null +++ b/tests/integration/composer_proxy_test.go @@ -0,0 +1,276 @@ +package integration + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + "time" + + "github.com/gofiber/fiber/v3" + "github.com/sirupsen/logrus" + + "github.com/any-hub/any-hub/internal/cache" + "github.com/any-hub/any-hub/internal/config" + "github.com/any-hub/any-hub/internal/proxy" + "github.com/any-hub/any-hub/internal/server" +) + +func TestComposerProxyCachesMetadataAndDists(t *testing.T) { + stub := newComposerStub(t) + defer stub.Close() + + storageDir := t.TempDir() + cfg := &config.Config{ + Global: config.GlobalConfig{ + ListenPort: 5000, + CacheTTL: config.Duration(time.Hour), + StoragePath: storageDir, + }, + Hubs: []config.HubConfig{ + { + Name: "composer", + Domain: "composer.hub.local", + Type: "composer", + Upstream: stub.URL, + }, + }, + } + + registry, err := server.NewHubRegistry(cfg) + if err != nil { + t.Fatalf("registry error: %v", err) + } + + logger := logrus.New() + logger.SetOutput(io.Discard) + + store, err := cache.NewStore(storageDir) + if err != nil { + t.Fatalf("store error: %v", err) + } + + app, err := server.NewApp(server.AppOptions{ + Logger: logger, + Registry: registry, + Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store), + ListenPort: 5000, + }) + if err != nil { + t.Fatalf("app error: %v", err) + } + + doRequest := func(path string) *http.Response { + req := httptest.NewRequest("GET", "http://composer.hub.local"+path, nil) + req.Host = "composer.hub.local" + resp, err := app.Test(req) + if err != nil { + t.Fatalf("app.Test error: %v", err) + } + return resp + } + + metaPath := "/p2/example/package.json" + resp := doRequest(metaPath) + if resp.StatusCode != fiber.StatusOK { + t.Fatalf("expected 200 for composer metadata, got %d", resp.StatusCode) + } + if resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("expected metadata content-type json, got %s", resp.Header.Get("Content-Type")) + } + if resp.Header.Get("X-Any-Hub-Cache-Hit") != "false" { + t.Fatalf("expected metadata miss on first request") + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + var meta composerMetadataPayload + if err := json.Unmarshal(body, &meta); err != nil { + t.Fatalf("parse metadata: %v", err) + } + distURL := meta.FindDistURL("example/package") + if distURL == "" { + t.Fatalf("metadata missing dist url: %s", string(body)) + } + parsedDist, err := url.Parse(distURL) + if err != nil { + t.Fatalf("parse dist url: %v", err) + } + if parsedDist.Host != "composer.hub.local" { + t.Fatalf("expected dist url rewritten to proxy host, got %s", parsedDist.Host) + } + + resp2 := doRequest(metaPath) + if resp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" { + t.Fatalf("expected metadata cache hit on second request") + } + resp2.Body.Close() + + distResp := doRequest(parsedDist.RequestURI()) + if distResp.StatusCode != fiber.StatusOK { + t.Fatalf("expected dist 200, got %d", distResp.StatusCode) + } + if distResp.Header.Get("X-Any-Hub-Cache-Hit") != "false" { + t.Fatalf("expected dist miss on first download") + } + distBody, _ := io.ReadAll(distResp.Body) + distResp.Body.Close() + if string(distBody) != stub.DistContent() { + t.Fatalf("unexpected dist body, got %s", string(distBody)) + } + + distResp2 := doRequest(parsedDist.RequestURI()) + if distResp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" { + t.Fatalf("expected cached dist response") + } + distResp2.Body.Close() + + if stub.MetadataHits() != 1 { + t.Fatalf("expected single upstream metadata GET, got %d", stub.MetadataHits()) + } + if stub.DistHits() != 1 { + t.Fatalf("expected single upstream dist GET, got %d", stub.DistHits()) + } +} + +type composerMetadataPayload struct { + Packages map[string][]composerMetadataVersion `json:"packages"` +} + +type composerMetadataVersion struct { + Dist struct { + URL string `json:"url"` + } `json:"dist"` +} + +func (m composerMetadataPayload) FindDistURL(pkg string) string { + versions, ok := m.Packages[pkg] + if !ok || len(versions) == 0 { + return "" + } + return versions[0].Dist.URL +} + +type composerStub struct { + server *http.Server + listener net.Listener + URL string + + mu sync.Mutex + metadataHits int + distHits int + distBody string + metadataBody []byte + metadataPath string + distPath string +} + +func newComposerStub(t *testing.T) *composerStub { + t.Helper() + stub := &composerStub{ + distBody: "zip-bytes", + metadataPath: "/p2/example/package.json", + distPath: "/downloads/example-package-1.0.0.zip", + } + + mux := http.NewServeMux() + mux.HandleFunc("/packages.json", stub.handlePackages) + mux.HandleFunc(stub.metadataPath, stub.handleMetadata) + mux.HandleFunc(stub.distPath, stub.handleDist) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skipf("unable to start composer stub: %v", err) + } + + server := &http.Server{Handler: mux} + stub.server = server + stub.listener = listener + stub.URL = "http://" + listener.Addr().String() + stub.metadataBody = stub.buildMetadata() + + go func() { + _ = server.Serve(listener) + }() + + return stub +} + +func (s *composerStub) buildMetadata() []byte { + payload := map[string]any{ + "packages": map[string][]map[string]any{ + "example/package": { + { + "name": "example/package", + "version": "1.0.0", + "dist": map[string]any{ + "type": "zip", + "url": s.URL + s.distPath, + }, + }, + }, + }, + } + data, _ := json.Marshal(payload) + return data +} + +func (s *composerStub) handlePackages(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"packages":{}}`)) +} + +func (s *composerStub) handleMetadata(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.metadataHits++ + body := s.metadataBody + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) +} + +func (s *composerStub) handleDist(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.distHits++ + body := s.distBody + s.mu.Unlock() + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write([]byte(body)) +} + +func (s *composerStub) MetadataHits() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.metadataHits +} + +func (s *composerStub) DistHits() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.distHits +} + +func (s *composerStub) DistContent() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.distBody +} + +func (s *composerStub) Close() { + if s == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if s.server != nil { + _ = s.server.Shutdown(ctx) + } + if s.listener != nil { + _ = s.listener.Close() + } +}