rogee/any-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support apk

Browse Source
This commit is contained in:
Rogee
2025-11-18 12:16:28 +08:00
parent 68b6bb78e6
commit ba5544c28d
28 changed files with 1412 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/config/loader.go
View File

@@ -107,6 +107,12 @@ func applyHubDefaults(h *HubConfig) {
}
}
// NormalizeHubConfig 公开给无需依赖 loader 的调用方（例如测试）以填充模块/rollout 默认值。
func NormalizeHubConfig(h HubConfig) HubConfig {
applyHubDefaults(&h)
return h
}
func durationDecodeHook() mapstructure.DecodeHookFunc {
targetType := reflect.TypeOf(Duration(0))

2
internal/config/modules.go
View File

@@ -2,9 +2,11 @@ package config
import (
_ "github.com/any-hub/any-hub/internal/hubmodule/composer"
_ "github.com/any-hub/any-hub/internal/hubmodule/debian"
_ "github.com/any-hub/any-hub/internal/hubmodule/docker"
_ "github.com/any-hub/any-hub/internal/hubmodule/golang"
_ "github.com/any-hub/any-hub/internal/hubmodule/legacy"
_ "github.com/any-hub/any-hub/internal/hubmodule/apk"
_ "github.com/any-hub/any-hub/internal/hubmodule/npm"
_ "github.com/any-hub/any-hub/internal/hubmodule/pypi"
)

7
internal/config/runtime_flags.go
View File

@@ -8,6 +8,13 @@ import (
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
)
// Rollout 字段说明（legacy → modular 平滑迁移控制）：
// - legacy-only：强制使用 legacy 模块（EffectiveModuleKey → legacy）；用于未迁移或需要快速回滚时。
// - dual：新模块为默认，保留 legacy 以便诊断/灰度；仅当 Module 非空时生效，否则回退 legacy-only。
// - modular：仅使用新模块；Module 为空或 legacy 模块时自动回退 legacy-only。
// 默认行为：未填写 Rollout 时，空 Module/legacy 模块默认 legacy-only；其它模块默认 modular。
// 影响范围：动态选择执行的模块键（EffectiveModuleKey）、路由日志中的 rollout_flag，方便区分迁移阶段。
// parseRolloutFlag 将配置中的 rollout 字段标准化，并结合模块类型输出最终状态。
func parseRolloutFlag(raw string, moduleKey string) (legacy.RolloutFlag, error) {
normalized := strings.ToLower(strings.TrimSpace(raw))

4
internal/config/validation.go
View File

@@ -16,9 +16,11 @@ var supportedHubTypes = map[string]struct{}{
"go": {},
"pypi": {},
"composer": {},
"debian": {},
"apk": {},
}
const supportedHubTypeList = "docker|npm|go|pypi|composer"
const supportedHubTypeList = "docker|npm|go|pypi|composer|debian|apk"
// Validate 针对语义级别做进一步校验，防止非法配置启动服务。
func (c *Config) Validate() error {

83
internal/hubmodule/apk/hooks.go Normal file
View File

@@ -0,0 +1,83 @@
// Package apk defines hook behaviors for Alpine APK proxying.
// APKINDEX/签名需要再验证；packages/*.apk 视为不可变缓存。
package apk
import (
"path"
"strings"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func init() {
hooks.MustRegister("apk", hooks.Hooks{
NormalizePath: normalizePath,
CachePolicy: cachePolicy,
ContentType: contentType,
})
}
func normalizePath(_ *hooks.RequestContext, p string, rawQuery []byte) (string, []byte) {
clean := path.Clean("/" + strings.TrimSpace(p))
return clean, rawQuery
}
func cachePolicy(_ *hooks.RequestContext, locatorPath string, current hooks.CachePolicy) hooks.CachePolicy {
clean := canonicalPath(locatorPath)
switch {
case isAPKIndexPath(clean), isAPKSignaturePath(clean):
// APKINDEX 及签名需要再验证，确保索引最新。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = true
case isAPKPackagePath(clean):
// 包体不可变，允许直接命中缓存，无需 HEAD。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = false
default:
current.AllowCache = false
current.AllowStore = false
current.RequireRevalidate = false
}
return current
}
func contentType(_ *hooks.RequestContext, locatorPath string) string {
clean := canonicalPath(locatorPath)
switch {
case strings.HasSuffix(clean, ".apk"):
return "application/vnd.android.package-archive"
case strings.HasSuffix(clean, ".tar.gz"):
return "application/gzip"
case strings.HasSuffix(clean, ".tar.gz.asc") || strings.HasSuffix(clean, ".tar.gz.sig"):
return "application/pgp-signature"
default:
return ""
}
}
func isAPKIndexPath(p string) bool {
clean := canonicalPath(p)
return strings.HasSuffix(clean, "/apkindex.tar.gz")
}
func isAPKSignaturePath(p string) bool {
clean := canonicalPath(p)
return strings.HasSuffix(clean, "/apkindex.tar.gz.asc") || strings.HasSuffix(clean, "/apkindex.tar.gz.sig")
}
func isAPKPackagePath(p string) bool {
clean := canonicalPath(p)
if isAPKIndexPath(clean) || isAPKSignaturePath(clean) {
return false
}
return strings.HasSuffix(clean, ".apk")
}
func canonicalPath(p string) string {
if p == "" {
return "/"
}
return strings.ToLower(path.Clean("/" + strings.TrimSpace(p)))
}

61
internal/hubmodule/apk/hooks_test.go Normal file
View File

@@ -0,0 +1,61 @@
package apk
import (
"testing"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func TestCachePolicyIndexAndSignatureRevalidate(t *testing.T) {
paths := []string{
"/v3.19/main/x86_64/APKINDEX.tar.gz",
"/v3.19/main/x86_64/APKINDEX.tar.gz.asc",
"/v3.19/community/aarch64/apkindex.tar.gz.sig",
}
for _, p := range paths {
current := cachePolicy(nil, p, hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected index/signature to require revalidate for %s", p)
}
}
}
func TestCachePolicyPackageImmutable(t *testing.T) {
tests := []string{
"/v3.19/main/x86_64/packages/hello-1.0.apk",
"/v3.18/testing/aarch64/packages/../packages/hello-1.0-r1.APK",
"/v3.22/community/x86_64/tini-static-0.19.0-r3.apk", // 路径不含 /packages/ 也应视作包体
}
for _, p := range tests {
current := cachePolicy(nil, p, hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected immutable cache for %s", p)
}
}
}
func TestCachePolicyNonAPKPath(t *testing.T) {
current := cachePolicy(nil, "/other/path", hooks.CachePolicy{})
if current.AllowCache || current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected non-APK path to disable cache/store")
}
}
func TestNormalizePath(t *testing.T) {
p, _ := normalizePath(nil, "v3.19/main/x86_64/APKINDEX.tar.gz", nil)
if p != "/v3.19/main/x86_64/APKINDEX.tar.gz" {
t.Fatalf("unexpected normalized path: %s", p)
}
}
func TestContentType(t *testing.T) {
if ct := contentType(nil, "/v3.19/main/x86_64/APKINDEX.tar.gz"); ct != "application/gzip" {
t.Fatalf("expected gzip content type, got %s", ct)
}
if ct := contentType(nil, "/v3.19/main/x86_64/APKINDEX.tar.gz.asc"); ct != "application/pgp-signature" {
t.Fatalf("expected signature content type, got %s", ct)
}
if ct := contentType(nil, "/v3.19/main/x86_64/packages/hello.apk"); ct != "application/vnd.android.package-archive" {
t.Fatalf("expected apk content type, got %s", ct)
}
}

29
internal/hubmodule/apk/module.go Normal file
View File

@@ -0,0 +1,29 @@
// Package apk registers metadata for Alpine APK proxying.
package apk
import (
"time"
"github.com/any-hub/any-hub/internal/hubmodule"
)
const apkDefaultTTL = 6 * time.Hour
func init() {
// 模块元数据声明，具体 hooks 见 hooks.go（已在 init 自动注册）。
hubmodule.MustRegister(hubmodule.ModuleMetadata{
Key: "apk",
Description: "Alpine APK proxy with cached indexes and packages",
MigrationState: hubmodule.MigrationStateBeta,
SupportedProtocols: []string{
"apk",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: 0, // APKINDEX 每次再验证，包体直接命中
ValidationMode: hubmodule.ValidationModeLastModified, // APKINDEX 再验证
DiskLayout: "raw_path",
RequiresMetadataFile: false,
SupportsStreamingWrite: true, // 包体流式写
},
})
}

102
internal/hubmodule/debian/hooks.go Normal file
View File

@@ -0,0 +1,102 @@
// Package debian defines hook behaviors for APT (Debian/Ubuntu) proxying.
// 索引（Release/InRelease/Packages*）需要再验证；包体（pool/ 和 by-hash）视为不可变直接缓存。
// 日志字段沿用通用 proxy（命中/上游状态），无需额外改写。
package debian
import (
"path"
"strings"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func init() {
hooks.MustRegister("debian", hooks.Hooks{
NormalizePath: normalizePath,
CachePolicy: cachePolicy,
ContentType: contentType,
})
}
func normalizePath(_ *hooks.RequestContext, p string, rawQuery []byte) (string, []byte) {
clean := path.Clean("/" + strings.TrimSpace(p))
return clean, rawQuery
}
func cachePolicy(_ *hooks.RequestContext, locatorPath string, current hooks.CachePolicy) hooks.CachePolicy {
clean := canonicalPath(locatorPath)
switch {
case isAptIndexPath(clean):
// 索引类（Release/Packages）需要 If-None-Match/If-Modified-Since 再验证。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = true
case isAptImmutablePath(clean):
// pool/*.deb 与 by-hash 路径视为不可变，直接缓存后续不再 HEAD。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = false
default:
current.AllowCache = false
current.AllowStore = false
current.RequireRevalidate = false
}
return current
}
func contentType(_ *hooks.RequestContext, locatorPath string) string {
switch {
case strings.HasSuffix(locatorPath, ".gz"):
return "application/gzip"
case strings.HasSuffix(locatorPath, ".xz"):
return "application/x-xz"
case strings.HasSuffix(locatorPath, "Release.gpg"):
return "application/pgp-signature"
case isAptIndexPath(locatorPath):
return "text/plain"
default:
return ""
}
}
func isAptIndexPath(p string) bool {
clean := canonicalPath(p)
if isByHashPath(clean) {
return false
}
if strings.HasPrefix(clean, "/dists/") {
if strings.HasSuffix(clean, "/release") || strings.HasSuffix(clean, "/inrelease") || strings.HasSuffix(clean, "/release.gpg") {
return true
}
if strings.Contains(clean, "/packages") {
return true
}
}
return false
}
func isAptImmutablePath(p string) bool {
clean := canonicalPath(p)
if isByHashPath(clean) {
return true
}
if strings.HasPrefix(clean, "/pool/") {
return true
}
return false
}
func isByHashPath(p string) bool {
clean := canonicalPath(p)
if !strings.HasPrefix(clean, "/dists/") {
return false
}
return strings.Contains(clean, "/by-hash/")
}
func canonicalPath(p string) string {
if p == "" {
return "/"
}
return strings.ToLower(path.Clean("/" + strings.TrimSpace(p)))
}

64
internal/hubmodule/debian/hooks_test.go Normal file
View File

@@ -0,0 +1,64 @@
package debian
import (
"testing"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func TestCachePolicyIndexesRevalidate(t *testing.T) {
current := cachePolicy(nil, "/dists/bookworm/Release", hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected index to allow cache/store and revalidate")
}
current = cachePolicy(nil, "/dists/bookworm/main/binary-amd64/Packages.gz", hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected packages index to revalidate")
}
}
func TestCachePolicyImmutable(t *testing.T) {
tests := []struct {
name string
path string
}{
{name: "by-hash index snapshot", path: "/dists/bookworm/by-hash/sha256/abc"},
{name: "by-hash nested", path: "/dists/bookworm/main/binary-amd64/by-hash/SHA256/def"},
{name: "pool package", path: "/pool/main/h/hello.deb"},
{name: "pool canonicalized", path: " /PoOl/main/../main/h/hello_1.0_amd64.DeB "},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
current := cachePolicy(nil, tt.path, hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected immutable cache for %s", tt.path)
}
})
}
}
func TestCachePolicyNonAptPath(t *testing.T) {
current := cachePolicy(nil, "/other/path", hooks.CachePolicy{})
if current.AllowCache || current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected non-APT path to disable cache/store")
}
}
func TestNormalizePath(t *testing.T) {
path, _ := normalizePath(nil, "dists/bookworm/Release", nil)
if path != "/dists/bookworm/Release" {
t.Fatalf("unexpected path: %s", path)
}
}
func TestContentType(t *testing.T) {
if ct := contentType(nil, "/dists/bookworm/Release"); ct != "text/plain" {
t.Fatalf("expected text/plain for Release, got %s", ct)
}
if ct := contentType(nil, "/dists/bookworm/Release.gpg"); ct != "application/pgp-signature" {
t.Fatalf("expected signature content-type, got %s", ct)
}
if ct := contentType(nil, "/dists/bookworm/main/binary-amd64/Packages.gz"); ct != "application/gzip" {
t.Fatalf("expected gzip content-type, got %s", ct)
}
}

29
internal/hubmodule/debian/module.go Normal file
View File

@@ -0,0 +1,29 @@
// Package debian registers metadata for Debian/Ubuntu APT proxying.
package debian
import (
"time"
"github.com/any-hub/any-hub/internal/hubmodule"
)
const debianDefaultTTL = 6 * time.Hour
func init() {
// 仅声明模块元数据（缓存策略等）；具体 hooks 在后续实现。
hubmodule.MustRegister(hubmodule.ModuleMetadata{
Key: "debian",
Description: "APT proxy with cached indexes and packages",
MigrationState: hubmodule.MigrationStateBeta,
SupportedProtocols: []string{
"debian",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: 0, // 索引每次再验证，由 ETag/Last-Modified 控制
ValidationMode: hubmodule.ValidationModeLastModified, // 索引使用 Last-Modified/ETag 再验证
DiskLayout: "raw_path", // 复用通用原始路径布局
RequiresMetadataFile: false,
SupportsStreamingWrite: true, // 包体需要流式写入，避免大文件占用内存
},
})
}

3
internal/hubmodule/docker/hooks.go
View File

@@ -15,9 +15,6 @@ func init() {
}
func normalizePath(ctx *hooks.RequestContext, clean string, rawQuery []byte) (string, []byte) {
if !isDockerHubHost(ctx.UpstreamHost) {
return clean, rawQuery
}
repo, rest, ok := splitDockerRepoPath(clean)
if !ok || repo == "" || strings.Contains(repo, "/") || repo == "library" {
return clean, rawQuery

2
internal/hubmodule/docker/module.go
View File

@@ -19,7 +19,7 @@ func init() {
"docker",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: dockerDefaultTTL,
TTLHint: 0, // manifests 需每次再验证，由 ETag 控制新鲜度
ValidationMode: hubmodule.ValidationModeETag,
DiskLayout: "raw_path",
RequiresMetadataFile: false,

2
internal/hubmodule/pypi/module.go
View File

@@ -19,7 +19,7 @@ func init() {
"pypi",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: pypiDefaultTTL,
TTLHint: 0, // simple index 每次再验证
ValidationMode: hubmodule.ValidationModeLastModified,
DiskLayout: "raw_path",
RequiresMetadataFile: false,

2
internal/hubmodule/strategy.go
View File

@@ -11,7 +11,7 @@ type StrategyOptions struct {
// ResolveStrategy 将模块的默认策略与 hub 级覆盖合并。
func ResolveStrategy(meta ModuleMetadata, opts StrategyOptions) CacheStrategyProfile {
strategy := meta.CacheStrategy
if opts.TTLOverride > 0 {
if strategy.TTLHint > 0 && opts.TTLOverride > 0 {
strategy.TTLHint = opts.TTLOverride
}
if opts.ValidationOverride != "" {

14
internal/proxy/handler.go
View File

@@ -203,6 +203,12 @@ func (h *Handler) serveCache(
c.Status(status)
if method == http.MethodHead {
// 对 HEAD 请求仍向上游发起一次 HEAD，以满足“显式请求 + 再验证”的期望。
if route != nil && route.UpstreamURL != nil {
if resp, err := h.revalidateRequest(c, route, resolveUpstreamURL(route, route.UpstreamURL, c, hook), result.Entry.Locator, ""); err == nil {
resp.Body.Close()
}
}
result.Reader.Close()
h.logResult(route, route.UpstreamURL.String(), requestID, status, true, started, nil)
return nil
@@ -358,6 +364,7 @@ func (h *Handler) cacheAndStream(
}
c.Status(resp.StatusCode)
// 使用 TeeReader 边向客户端回写边落盘，避免大文件在内存中完整缓冲。
reader := io.TeeReader(resp.Body, c.Response().BodyWriter())
opts := cache.PutOptions{ModTime: extractModTime(resp.Header)}
@@ -733,7 +740,7 @@ func determineCachePolicyWithHook(route *server.HubRoute, locator cache.Locator,
}
func determineCachePolicy(route *server.HubRoute, locator cache.Locator, method string) cachePolicy {
if method != http.MethodGet {
if method != http.MethodGet && method != http.MethodHead {
return cachePolicy{}
}
return cachePolicy{allowCache: true, allowStore: true}
@@ -786,9 +793,12 @@ func (h *Handler) isCacheFresh(
case http.StatusNotModified:
return true, nil
case http.StatusOK:
if resp.Header.Get("Etag") == "" && resp.Header.Get("Docker-Content-Digest") == "" && resp.Header.Get("Last-Modified") == "" {
return true, nil
}
h.rememberETag(route, locator, resp)
remote := extractModTime(resp.Header)
if !remote.After(entry.ModTime.Add(time.Second)) {
if !remote.After(entry.ModTime) {
return true, nil
}
return false, nil

1
internal/server/hub_registry.go
View File

@@ -56,6 +56,7 @@ func NewHubRegistry(cfg *config.Config) (*HubRegistry, error) {
}
for _, hub := range cfg.Hubs {
hub = config.NormalizeHubConfig(hub)
normalizedHost := normalizeDomain(hub.Domain)
if normalizedHost == "" {
return nil, fmt.Errorf("invalid domain for hub %s", hub.Name)

8
internal/server/hub_registry_test.go
View File

@@ -49,14 +49,14 @@ func TestHubRegistryLookupByHost(t *testing.T) {
if route.CacheTTL != cfg.EffectiveCacheTTL(route.Config) {
t.Errorf("cache ttl mismatch: got %s", route.CacheTTL)
}
if route.CacheStrategy.TTLHint != route.CacheTTL {
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, route.CacheTTL)
if route.CacheStrategy.TTLHint != 0 {
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, time.Duration(0))
}
if route.CacheStrategy.ValidationMode == "" {
t.Fatalf("cache strategy validation mode should not be empty")
}
if route.RolloutFlag != legacy.RolloutLegacyOnly {
t.Fatalf("default rollout flag should be legacy-only")
if route.RolloutFlag != legacy.RolloutModular {
t.Fatalf("default rollout flag should be modular")
}
if route.UpstreamURL.String() != "https://registry-1.docker.io" {