3 Commits

Author SHA1 Message Date
dcd85a9f41 fix: apt cache
Some checks failed
docker-release / build-and-push (push) Failing after 9m58s
2025-11-18 14:21:12 +08:00
ba5544c28d feat: support apk 2025-11-18 12:16:28 +08:00
68b6bb78e6 add oc 2025-11-18 09:55:07 +08:00
35 changed files with 1912 additions and 20 deletions

View File

@@ -10,6 +10,8 @@ Auto-generated from all feature plans. Last updated: 2025-11-13
- 本地文件系统缓存目录 `StoragePath/<Hub>/<path>`,模块需直接复用原始路径布局 (004-modular-proxy-cache)
- 本地文件系统缓存目录 `StoragePath/<Hub>/<path>`(按模块定义的布局) (005-proxy-module-delegation)
- 本地文件系统缓存目录 `StoragePath/<Hub>/<path>`(由模块 Hook 定义布局) (006-module-hook-refactor)
- Go 1.25+ (静态链接,单二进制) + Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack、标准库 `net/http`/`io` (007-apt-apk-cache)
- 本地 `StoragePath/<Hub>/<path>` + `.meta`;索引需带 Last-Modified/ETag 元信息;包体按原路径落盘 (007-apt-apk-cache)
- Go 1.25+ (静态链接,单二进制交付) + Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack结构化日志 [EXTRACTED FROM ALL PLAN.MD FILES] 滚动)、标准库 `net/http`/`io` (001-config-bootstrap)
@@ -29,9 +31,9 @@ tests/
Go 1.25+ (静态链接,单二进制交付): Follow standard conventions
## Recent Changes
- 007-apt-apk-cache: Added Go 1.25+ (静态链接,单二进制) + Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack、标准库 `net/http`/`io`
- 006-module-hook-refactor: Added Go 1.25+ (静态链接,单二进制交付) + Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack结构化日志 & 滚动)、标准库 `net/http`/`io`
- 005-proxy-module-delegation: Added Go 1.25+ (静态链接,单二进制交付) + Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack结构化日志 & 滚动)、标准库 `net/http`/`io`
- 004-modular-proxy-cache: Added Go 1.25+ (静态链接,单二进制交付) + Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack结构化日志 & 滚动)、标准库 `net/http`/`io`
<!-- MANUAL ADDITIONS START -->

View File

@@ -86,3 +86,21 @@ Username = ""
Password = ""
Type = "composer"
Module = "composer"
# Debian/Ubuntu APT 示例
[[Hub]]
Name = "apt-cache"
Domain = "apt.hub.local"
Upstream = "https://mirrors.edge.kernel.org/ubuntu"
Type = "debian"
Module = "debian"
Rollout = "modular"
# Alpine APK 示例
[[Hub]]
Name = "apk-cache"
Domain = "apk.hub.local"
Upstream = "https://dl-cdn.alpinelinux.org/alpine"
Type = "apk"
Module = "apk"
Rollout = "modular"

View File

@@ -41,3 +41,21 @@ Upstream = "https://registry.npmjs.org"
Type = "npm"
Module = "legacy" # 仍未迁移的 Hub 可显式指定 legacy诊断会标记为 legacy-only
Rollout = "legacy-only"
# Debian/Ubuntu APT 示例
[[Hub]]
Name = "apt-cache"
Domain = "apt.hub.local"
Upstream = "https://mirrors.edge.kernel.org/ubuntu"
Type = "debian"
Module = "debian"
Rollout = "modular"
# Alpine APK 示例
[[Hub]]
Name = "apk-cache"
Domain = "apk.hub.local"
Upstream = "https://dl-cdn.alpinelinux.org/alpine"
Type = "apk"
Module = "apk"
Rollout = "modular"

View File

@@ -0,0 +1,22 @@
# Logging & Cache Semantics (APT/APK)
## Common Fields
- `hub`/`domain`/`hub_type`:当前 Hub 标识与协议类型,例如 `debian`/`apk`
- `module_key`:命中的模块键(非 `legacy` 时表示新模块生效)。
- `cache_hit``true` 表示直接复用缓存;`false` 表示从上游获取或已刷新。
- `upstream`/`upstream_status`:实际访问的上游地址与状态码。
- `rollout_flag``legacy-only`/`dual`/`modular`,便于排查路由与灰度。
- `action``proxy`,表明代理链路日志。
## APT (debian 模块)
- 索引路径Release/InRelease/Packages*`cache_hit=true` 仍会在后台进行 HEAD 再验证;命中 304 时保持缓存。
- 包体路径(`/pool/*``/dists/.../by-hash/...`):视为不可变,首次 GET 落盘,后续直接命中,无 HEAD。
- 日志可结合 `X-Any-Hub-Cache-Hit` 响应头进行对照。
## APK (apk 模块)
- APKINDEX 及签名:每次命中会触发 HEAD 再验证,缓存命中返回 304 时继续使用本地文件。
- 包体 (`packages/*.apk`):不可变资源,首轮 GET 落盘,后续直接命中,无 HEAD。
## Quick Checks
- 观察 `cache_hit``upstream_status``cache_hit=true``upstream_status=200/304` 表示缓存复用成功;`cache_hit=false` 表示回源或刷新。
- 若期望模块日志字段但出现 `module_key":"legacy"`,检查 `Module``Rollout` 配置是否指向新模块。

View File

@@ -107,6 +107,12 @@ func applyHubDefaults(h *HubConfig) {
}
}
// NormalizeHubConfig 公开给无需依赖 loader 的调用方(例如测试)以填充模块/rollout 默认值。
func NormalizeHubConfig(h HubConfig) HubConfig {
applyHubDefaults(&h)
return h
}
func durationDecodeHook() mapstructure.DecodeHookFunc {
targetType := reflect.TypeOf(Duration(0))

View File

@@ -2,9 +2,11 @@ package config
import (
_ "github.com/any-hub/any-hub/internal/hubmodule/composer"
_ "github.com/any-hub/any-hub/internal/hubmodule/debian"
_ "github.com/any-hub/any-hub/internal/hubmodule/docker"
_ "github.com/any-hub/any-hub/internal/hubmodule/golang"
_ "github.com/any-hub/any-hub/internal/hubmodule/legacy"
_ "github.com/any-hub/any-hub/internal/hubmodule/apk"
_ "github.com/any-hub/any-hub/internal/hubmodule/npm"
_ "github.com/any-hub/any-hub/internal/hubmodule/pypi"
)

View File

@@ -8,6 +8,13 @@ import (
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
)
// Rollout 字段说明legacy → modular 平滑迁移控制):
// - legacy-only强制使用 legacy 模块EffectiveModuleKey → legacy用于未迁移或需要快速回滚时。
// - dual新模块为默认保留 legacy 以便诊断/灰度;仅当 Module 非空时生效,否则回退 legacy-only。
// - modular仅使用新模块Module 为空或 legacy 模块时自动回退 legacy-only。
// 默认行为:未填写 Rollout 时,空 Module/legacy 模块默认 legacy-only其它模块默认 modular。
// 影响范围动态选择执行的模块键EffectiveModuleKey、路由日志中的 rollout_flag方便区分迁移阶段。
// parseRolloutFlag 将配置中的 rollout 字段标准化,并结合模块类型输出最终状态。
func parseRolloutFlag(raw string, moduleKey string) (legacy.RolloutFlag, error) {
normalized := strings.ToLower(strings.TrimSpace(raw))

View File

@@ -16,9 +16,11 @@ var supportedHubTypes = map[string]struct{}{
"go": {},
"pypi": {},
"composer": {},
"debian": {},
"apk": {},
}
const supportedHubTypeList = "docker|npm|go|pypi|composer"
const supportedHubTypeList = "docker|npm|go|pypi|composer|debian|apk"
// Validate 针对语义级别做进一步校验,防止非法配置启动服务。
func (c *Config) Validate() error {

View File

@@ -0,0 +1,83 @@
// Package apk defines hook behaviors for Alpine APK proxying.
// APKINDEX/签名需要再验证packages/*.apk 视为不可变缓存。
package apk
import (
"path"
"strings"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func init() {
hooks.MustRegister("apk", hooks.Hooks{
NormalizePath: normalizePath,
CachePolicy: cachePolicy,
ContentType: contentType,
})
}
func normalizePath(_ *hooks.RequestContext, p string, rawQuery []byte) (string, []byte) {
clean := path.Clean("/" + strings.TrimSpace(p))
return clean, rawQuery
}
func cachePolicy(_ *hooks.RequestContext, locatorPath string, current hooks.CachePolicy) hooks.CachePolicy {
clean := canonicalPath(locatorPath)
switch {
case isAPKIndexPath(clean), isAPKSignaturePath(clean):
// APKINDEX 及签名需要再验证,确保索引最新。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = true
case isAPKPackagePath(clean):
// 包体不可变,允许直接命中缓存,无需 HEAD。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = false
default:
current.AllowCache = false
current.AllowStore = false
current.RequireRevalidate = false
}
return current
}
func contentType(_ *hooks.RequestContext, locatorPath string) string {
clean := canonicalPath(locatorPath)
switch {
case strings.HasSuffix(clean, ".apk"):
return "application/vnd.android.package-archive"
case strings.HasSuffix(clean, ".tar.gz"):
return "application/gzip"
case strings.HasSuffix(clean, ".tar.gz.asc") || strings.HasSuffix(clean, ".tar.gz.sig"):
return "application/pgp-signature"
default:
return ""
}
}
func isAPKIndexPath(p string) bool {
clean := canonicalPath(p)
return strings.HasSuffix(clean, "/apkindex.tar.gz")
}
func isAPKSignaturePath(p string) bool {
clean := canonicalPath(p)
return strings.HasSuffix(clean, "/apkindex.tar.gz.asc") || strings.HasSuffix(clean, "/apkindex.tar.gz.sig")
}
func isAPKPackagePath(p string) bool {
clean := canonicalPath(p)
if isAPKIndexPath(clean) || isAPKSignaturePath(clean) {
return false
}
return strings.HasSuffix(clean, ".apk")
}
func canonicalPath(p string) string {
if p == "" {
return "/"
}
return strings.ToLower(path.Clean("/" + strings.TrimSpace(p)))
}

View File

@@ -0,0 +1,61 @@
package apk
import (
"testing"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func TestCachePolicyIndexAndSignatureRevalidate(t *testing.T) {
paths := []string{
"/v3.19/main/x86_64/APKINDEX.tar.gz",
"/v3.19/main/x86_64/APKINDEX.tar.gz.asc",
"/v3.19/community/aarch64/apkindex.tar.gz.sig",
}
for _, p := range paths {
current := cachePolicy(nil, p, hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected index/signature to require revalidate for %s", p)
}
}
}
func TestCachePolicyPackageImmutable(t *testing.T) {
tests := []string{
"/v3.19/main/x86_64/packages/hello-1.0.apk",
"/v3.18/testing/aarch64/packages/../packages/hello-1.0-r1.APK",
"/v3.22/community/x86_64/tini-static-0.19.0-r3.apk", // 路径不含 /packages/ 也应视作包体
}
for _, p := range tests {
current := cachePolicy(nil, p, hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected immutable cache for %s", p)
}
}
}
func TestCachePolicyNonAPKPath(t *testing.T) {
current := cachePolicy(nil, "/other/path", hooks.CachePolicy{})
if current.AllowCache || current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected non-APK path to disable cache/store")
}
}
func TestNormalizePath(t *testing.T) {
p, _ := normalizePath(nil, "v3.19/main/x86_64/APKINDEX.tar.gz", nil)
if p != "/v3.19/main/x86_64/APKINDEX.tar.gz" {
t.Fatalf("unexpected normalized path: %s", p)
}
}
func TestContentType(t *testing.T) {
if ct := contentType(nil, "/v3.19/main/x86_64/APKINDEX.tar.gz"); ct != "application/gzip" {
t.Fatalf("expected gzip content type, got %s", ct)
}
if ct := contentType(nil, "/v3.19/main/x86_64/APKINDEX.tar.gz.asc"); ct != "application/pgp-signature" {
t.Fatalf("expected signature content type, got %s", ct)
}
if ct := contentType(nil, "/v3.19/main/x86_64/packages/hello.apk"); ct != "application/vnd.android.package-archive" {
t.Fatalf("expected apk content type, got %s", ct)
}
}

View File

@@ -0,0 +1,29 @@
// Package apk registers metadata for Alpine APK proxying.
package apk
import (
"time"
"github.com/any-hub/any-hub/internal/hubmodule"
)
const apkDefaultTTL = 6 * time.Hour
func init() {
// 模块元数据声明,具体 hooks 见 hooks.go已在 init 自动注册)。
hubmodule.MustRegister(hubmodule.ModuleMetadata{
Key: "apk",
Description: "Alpine APK proxy with cached indexes and packages",
MigrationState: hubmodule.MigrationStateBeta,
SupportedProtocols: []string{
"apk",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: 0, // APKINDEX 每次再验证,包体直接命中
ValidationMode: hubmodule.ValidationModeLastModified, // APKINDEX 再验证
DiskLayout: "raw_path",
RequiresMetadataFile: false,
SupportsStreamingWrite: true, // 包体流式写
},
})
}

View File

@@ -0,0 +1,103 @@
// Package debian defines hook behaviors for APT (Debian/Ubuntu) proxying.
// 索引Release/InRelease/Packages*需要再验证包体pool/ 和 by-hash视为不可变直接缓存。
// 日志字段沿用通用 proxy命中/上游状态),无需额外改写。
package debian
import (
"path"
"strings"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func init() {
hooks.MustRegister("debian", hooks.Hooks{
NormalizePath: normalizePath,
CachePolicy: cachePolicy,
ContentType: contentType,
})
}
func normalizePath(_ *hooks.RequestContext, p string, rawQuery []byte) (string, []byte) {
clean := path.Clean("/" + strings.TrimSpace(p))
return clean, rawQuery
}
func cachePolicy(_ *hooks.RequestContext, locatorPath string, current hooks.CachePolicy) hooks.CachePolicy {
clean := canonicalPath(locatorPath)
switch {
case isAptIndexPath(clean):
// 索引类Release/Packages需要 If-None-Match/If-Modified-Since 再验证。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = true
case isAptImmutablePath(clean):
// pool/*.deb 与 by-hash 路径视为不可变,直接缓存后续不再 HEAD。
current.AllowCache = true
current.AllowStore = true
current.RequireRevalidate = false
default:
current.AllowCache = false
current.AllowStore = false
current.RequireRevalidate = false
}
return current
}
func contentType(_ *hooks.RequestContext, locatorPath string) string {
switch {
case strings.HasSuffix(locatorPath, ".gz"):
return "application/gzip"
case strings.HasSuffix(locatorPath, ".xz"):
return "application/x-xz"
case strings.HasSuffix(locatorPath, "Release.gpg"):
return "application/pgp-signature"
case isAptIndexPath(locatorPath):
return "text/plain"
default:
return ""
}
}
func isAptIndexPath(p string) bool {
clean := canonicalPath(p)
if isByHashPath(clean) {
return false
}
if strings.Contains(clean, "/dists/") {
if strings.HasSuffix(clean, "/release") ||
strings.HasSuffix(clean, "/inrelease") ||
strings.HasSuffix(clean, "/release.gpg") {
return true
}
}
return false
}
func isAptImmutablePath(p string) bool {
clean := canonicalPath(p)
if isByHashPath(clean) {
return true
}
if strings.Contains(clean, "/pool/") {
return true
}
return false
}
func isByHashPath(p string) bool {
clean := canonicalPath(p)
if strings.Contains(clean, "/dists/") {
return false
}
return strings.Contains(clean, "/by-hash/")
}
func canonicalPath(p string) string {
if p == "" {
return "/"
}
return strings.ToLower(path.Clean("/" + strings.TrimSpace(p)))
}

View File

@@ -0,0 +1,73 @@
package debian
import (
"testing"
"github.com/any-hub/any-hub/internal/proxy/hooks"
)
func TestCachePolicyIndexesRevalidate(t *testing.T) {
current := cachePolicy(nil, "/dists/bookworm/Release", hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected index to allow cache/store and revalidate")
}
current = cachePolicy(nil, "/dists/bookworm/main/binary-amd64/Packages.gz", hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected packages index to revalidate")
}
current = cachePolicy(nil, "/dists/bookworm/main/Contents-amd64.gz", hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected contents index to revalidate")
}
current = cachePolicy(nil, "/debian-security/dists/trixie/Contents-amd64.gz", hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
t.Fatalf("expected prefixed contents index to revalidate")
}
}
func TestCachePolicyImmutable(t *testing.T) {
tests := []struct {
name string
path string
}{
{name: "by-hash index snapshot", path: "/dists/bookworm/by-hash/sha256/abc"},
{name: "by-hash nested", path: "/dists/bookworm/main/binary-amd64/by-hash/SHA256/def"},
{name: "pool package", path: "/pool/main/h/hello.deb"},
{name: "pool canonicalized", path: " /PoOl/main/../main/h/hello_1.0_amd64.DeB "},
{name: "mirror prefix pool", path: "/debian/pool/main/h/hello.deb"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
current := cachePolicy(nil, tt.path, hooks.CachePolicy{})
if !current.AllowCache || !current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected immutable cache for %s", tt.path)
}
})
}
}
func TestCachePolicyNonAptPath(t *testing.T) {
current := cachePolicy(nil, "/other/path", hooks.CachePolicy{})
if current.AllowCache || current.AllowStore || current.RequireRevalidate {
t.Fatalf("expected non-APT path to disable cache/store")
}
}
func TestNormalizePath(t *testing.T) {
path, _ := normalizePath(nil, "dists/bookworm/Release", nil)
if path != "/dists/bookworm/Release" {
t.Fatalf("unexpected path: %s", path)
}
}
func TestContentType(t *testing.T) {
if ct := contentType(nil, "/dists/bookworm/Release"); ct != "text/plain" {
t.Fatalf("expected text/plain for Release, got %s", ct)
}
if ct := contentType(nil, "/dists/bookworm/Release.gpg"); ct != "application/pgp-signature" {
t.Fatalf("expected signature content-type, got %s", ct)
}
if ct := contentType(nil, "/dists/bookworm/main/binary-amd64/Packages.gz"); ct != "application/gzip" {
t.Fatalf("expected gzip content-type, got %s", ct)
}
}

View File

@@ -0,0 +1,29 @@
// Package debian registers metadata for Debian/Ubuntu APT proxying.
package debian
import (
"time"
"github.com/any-hub/any-hub/internal/hubmodule"
)
const debianDefaultTTL = 6 * time.Hour
func init() {
// 仅声明模块元数据(缓存策略等);具体 hooks 在后续实现。
hubmodule.MustRegister(hubmodule.ModuleMetadata{
Key: "debian",
Description: "APT proxy with cached indexes and packages",
MigrationState: hubmodule.MigrationStateBeta,
SupportedProtocols: []string{
"debian",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: 0, // 索引每次再验证,由 ETag/Last-Modified 控制
ValidationMode: hubmodule.ValidationModeLastModified, // 索引使用 Last-Modified/ETag 再验证
DiskLayout: "raw_path", // 复用通用原始路径布局
RequiresMetadataFile: false,
SupportsStreamingWrite: true, // 包体需要流式写入,避免大文件占用内存
},
})
}

View File

@@ -15,9 +15,6 @@ func init() {
}
func normalizePath(ctx *hooks.RequestContext, clean string, rawQuery []byte) (string, []byte) {
if !isDockerHubHost(ctx.UpstreamHost) {
return clean, rawQuery
}
repo, rest, ok := splitDockerRepoPath(clean)
if !ok || repo == "" || strings.Contains(repo, "/") || repo == "library" {
return clean, rawQuery

View File

@@ -19,7 +19,7 @@ func init() {
"docker",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: dockerDefaultTTL,
TTLHint: 0, // manifests 需每次再验证,由 ETag 控制新鲜度
ValidationMode: hubmodule.ValidationModeETag,
DiskLayout: "raw_path",
RequiresMetadataFile: false,

View File

@@ -19,7 +19,7 @@ func init() {
"pypi",
},
CacheStrategy: hubmodule.CacheStrategyProfile{
TTLHint: pypiDefaultTTL,
TTLHint: 0, // simple index 每次再验证
ValidationMode: hubmodule.ValidationModeLastModified,
DiskLayout: "raw_path",
RequiresMetadataFile: false,

View File

@@ -11,7 +11,7 @@ type StrategyOptions struct {
// ResolveStrategy 将模块的默认策略与 hub 级覆盖合并。
func ResolveStrategy(meta ModuleMetadata, opts StrategyOptions) CacheStrategyProfile {
strategy := meta.CacheStrategy
if opts.TTLOverride > 0 {
if strategy.TTLHint > 0 && opts.TTLOverride > 0 {
strategy.TTLHint = opts.TTLOverride
}
if opts.ValidationOverride != "" {

View File

@@ -203,6 +203,12 @@ func (h *Handler) serveCache(
c.Status(status)
if method == http.MethodHead {
// 对 HEAD 请求仍向上游发起一次 HEAD以满足“显式请求 + 再验证”的期望。
if route != nil && route.UpstreamURL != nil {
if resp, err := h.revalidateRequest(c, route, resolveUpstreamURL(route, route.UpstreamURL, c, hook), result.Entry.Locator, ""); err == nil {
resp.Body.Close()
}
}
result.Reader.Close()
h.logResult(route, route.UpstreamURL.String(), requestID, status, true, started, nil)
return nil
@@ -358,6 +364,7 @@ func (h *Handler) cacheAndStream(
}
c.Status(resp.StatusCode)
// 使用 TeeReader 边向客户端回写边落盘,避免大文件在内存中完整缓冲。
reader := io.TeeReader(resp.Body, c.Response().BodyWriter())
opts := cache.PutOptions{ModTime: extractModTime(resp.Header)}
@@ -733,7 +740,7 @@ func determineCachePolicyWithHook(route *server.HubRoute, locator cache.Locator,
}
func determineCachePolicy(route *server.HubRoute, locator cache.Locator, method string) cachePolicy {
if method != http.MethodGet {
if method != http.MethodGet && method != http.MethodHead {
return cachePolicy{}
}
return cachePolicy{allowCache: true, allowStore: true}
@@ -786,9 +793,12 @@ func (h *Handler) isCacheFresh(
case http.StatusNotModified:
return true, nil
case http.StatusOK:
if resp.Header.Get("Etag") == "" && resp.Header.Get("Docker-Content-Digest") == "" && resp.Header.Get("Last-Modified") == "" {
return true, nil
}
h.rememberETag(route, locator, resp)
remote := extractModTime(resp.Header)
if !remote.After(entry.ModTime.Add(time.Second)) {
if !remote.After(entry.ModTime) {
return true, nil
}
return false, nil

View File

@@ -56,6 +56,7 @@ func NewHubRegistry(cfg *config.Config) (*HubRegistry, error) {
}
for _, hub := range cfg.Hubs {
hub = config.NormalizeHubConfig(hub)
normalizedHost := normalizeDomain(hub.Domain)
if normalizedHost == "" {
return nil, fmt.Errorf("invalid domain for hub %s", hub.Name)

View File

@@ -49,14 +49,14 @@ func TestHubRegistryLookupByHost(t *testing.T) {
if route.CacheTTL != cfg.EffectiveCacheTTL(route.Config) {
t.Errorf("cache ttl mismatch: got %s", route.CacheTTL)
}
if route.CacheStrategy.TTLHint != route.CacheTTL {
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, route.CacheTTL)
if route.CacheStrategy.TTLHint != 0 {
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, time.Duration(0))
}
if route.CacheStrategy.ValidationMode == "" {
t.Fatalf("cache strategy validation mode should not be empty")
}
if route.RolloutFlag != legacy.RolloutLegacyOnly {
t.Fatalf("default rollout flag should be legacy-only")
if route.RolloutFlag != legacy.RolloutModular {
t.Fatalf("default rollout flag should be modular")
}
if route.UpstreamURL.String() != "https://registry-1.docker.io" {

View File

@@ -0,0 +1,34 @@
# Specification Quality Checklist: APT/APK 包缓存模块
**Purpose**: Validate specification completeness and quality before proceeding to planning
**Created**: 2025-11-17
**Feature**: /home/rogee/Projects/any-hub/specs/007-apt-apk-cache/spec.md
## Content Quality
- [x] No implementation details (languages, frameworks, APIs)
- [x] Focused on user value and business needs
- [x] Written for non-technical stakeholders
- [x] All mandatory sections completed
## Requirement Completeness
- [x] No [NEEDS CLARIFICATION] markers remain
- [x] Requirements are testable and unambiguous
- [x] Success criteria are measurable
- [x] Success criteria are technology-agnostic (no implementation details)
- [x] All acceptance scenarios are defined
- [x] Edge cases are identified
- [x] Scope is clearly bounded
- [x] Dependencies and assumptions identified
## Feature Readiness
- [x] All functional requirements have clear acceptance criteria
- [x] User scenarios cover primary flows
- [x] Feature meets measurable outcomes defined in Success Criteria
- [x] No implementation details leak into specification
## Notes
- Items marked complete; ready to proceed to planning.

View File

@@ -0,0 +1,26 @@
# Proxy Path Contracts: APT/APK
## APT
| Downstream Request | Upstream Target | Caching | Notes |
|--------------------|-----------------|---------|-------|
| `/dists/<suite>/<component>/binary-<arch>/Packages[.gz/.xz]` | same path on Upstream | RequireRevalidate=true | Send If-None-Match/If-Modified-Since if cached |
| `/dists/<suite>/Release` | same path | RequireRevalidate=true | Preserve content; may have accompanying `.gpg` |
| `/dists/<suite>/InRelease` | same path | RequireRevalidate=true | Signed inline; no body changes |
| `/dists/<suite>/Release.gpg` | same path | RequireRevalidate=true | Signature only; no rewrite |
| `/pool/<vendor>/<name>.deb` | same path | AllowCache/AllowStore=true, RequireRevalidate=false | Treat as immutable |
| `/dists/<suite>/by-hash/<algo>/<hash>` | same path | AllowCache/AllowStore=true, RequireRevalidate=false | Path encodes hash, no extra validation |
## Alpine APK
| Downstream Request | Upstream Target | Caching | Notes |
|--------------------|-----------------|---------|-------|
| `/v3.<branch>/main/<arch>/APKINDEX.tar.gz` (and variants) | same path | RequireRevalidate=true | Preserve signature/headers |
| `/v3.<branch>/<repo>/<arch>/APKINDEX.tar.gz` | same path | RequireRevalidate=true | Same handling as above |
| `/v3.<branch>/<repo>/<arch>/packages/<file>.apk` | same path | AllowCache/AllowStore=true, RequireRevalidate=false | Immutable package |
| APKINDEX signature files | same path | RequireRevalidate=true | No rewrite |
## Behaviors
- No body/URL rewrite; proxy only normalizes cache policy per path shape.
- Preserve status codes, headers, and signature files exactly as upstream.
- Conditional requests (If-None-Match/If-Modified-Since) applied to all index/signature paths when cached.

View File

@@ -0,0 +1,36 @@
# Data Model: APT/APK 包缓存模块
## Entities
### HubConfig (APT/APK)
- Fields: `Name`, `Domain`, `Port`, `Upstream`, `Type` (`debian`/`apk`), optional `CacheTTL` override.
- Rules: `Type` 必须新增枚举;`Upstream` 必须为 HTTP/HTTPS每个 Hub 独立缓存前缀。
### IndexFile
- Represents: APT Release/InRelease/Packages*Alpine APKINDEX。
- Attributes: `Path`, `ETag` (if present), `Last-Modified`, `Hash` (if provided in index), `Size`, `ContentType`.
- Rules: RequireRevalidate=true缓存命中需携带条件请求内容不得修改。
### PackageFile
- Represents: APT `pool/*/*.deb` 包体Alpine `packages/<arch>/*.apk`
- Attributes: `Path`, `Size`, `Hash` (from upstream metadata if available), `StoredAt`.
- Rules: 视作不可变AllowCache/AllowStore=true不做本地内容改写。
### SignatureFile
- Represents: APT `Release.gpg`/`InRelease` 自带签名Alpine APKINDEX 签名文件。
- Attributes: `Path`, `Size`, `StoredAt`.
- Rules: 原样透传;与对应 IndexFile 绑定,同步缓存与再验证。
### CacheEntry
- Represents: 本地缓存记录(索引或包体)。
- Attributes: `LocatorPath`, `ModTime`, `ETag`, `Size`, `AllowStore`, `RequireRevalidate`.
- Rules: 读取时决定是否回源;写入需同步 `.meta`;路径格式 `StoragePath/<Hub>/<Path>`.
## Relationships
- HubConfig → IndexFile/PackageFile/SignatureFile 通过 Domain/Upstream 绑定。
- IndexFile ↔ SignatureFile同一索引文件的签名文件需同周期缓存与再验证。
- CacheEntry 聚合 IndexFile/PackageFile/SignatureFile 的落盘信息,用于策略判定。
## State/Transitions
- CacheEntry states: `miss``fetched``validated` (304) / `refreshed` (200) → `stale` (TTL 或上游 200 新内容) → `removed` (404 或清理)。
- Transitions triggered by upstream响应304 保持内容200 刷新404 删除相关缓存。

View File

@@ -0,0 +1,94 @@
# Implementation Plan: APT/APK 包缓存模块
**Branch**: `007-apt-apk-cache` | **Date**: 2025-11-17 | **Spec**: `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/spec.md`
**Input**: Feature specification from `/specs/007-apt-apk-cache/spec.md`
## Summary
为 any-hub 增加 APTDebian/Ubuntu与 Alpine APK 的缓存代理模块索引Release/InRelease/Packages*/APKINDEX必须再验证包体pool/*.deb、packages/*)视作不可变直接缓存,支持 Acquire-By-Hash/签名透传,不影响现有各模块。
## Technical Context
**Language/Version**: Go 1.25+ (静态链接,单二进制)
**Primary Dependencies**: Fiber v3HTTP 服务、Viper配置、Logrus + Lumberjack、标准库 `net/http`/`io`
**Storage**: 本地 `StoragePath/<Hub>/<path>` + `.meta`;索引需带 Last-Modified/ETag 元信息;包体按原路径落盘
**Protocols**: APT (`/dists/<suite>/<component>/binary-<arch>/Packages*`, `Release`, `InRelease`, `pool/*`), Acquire-By-HashAlpine (`APKINDEX.tar.gz`, `packages/<arch>/…`)
**Caching Rules**: 索引 RequireRevalidate=true包体 AllowStore/AllowCache=true、RequireRevalidate=false签名/校验文件原样透传
**Testing**: `go test ./...`,构造 httptest 上游返回 Release/Packages/APKINDEX 与包体,验证首次回源+再验证+命中;使用临时目录校验缓存落盘
**Target Platform**: Linux/Unix CLIsystemd/supervisor 托管,匿名客户端
**Constraints**: 单一 config.toml 控制;禁止新增第三方依赖;保持现有模块行为不变
**Risk/Unknowns**: Acquire-By-Hash 路径映射是否需额外校验逻辑(选择原样透传,路径即校验)
## Constitution Check
*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.*
- Feature 仍然是“轻量多仓 CLI 代理”,未引入 Web UI、账号体系或与代理无关的能力。
- 仅使用 Go + 宪法指定依赖;任何新第三方库都已在本计划中说明理由与审核结论。
- 行为完全由 `config.toml` 控制,新增配置项已规划默认值、校验与迁移策略。
- 方案维持缓存优先 + 流式回源路径,并给出命中/回源/失败的日志与观测手段。
- 计划内列出了配置解析、缓存读写、Host Header 路由等强制测试与中文注释交付范围。
**Status**: PASS未引入新依赖/界面,新增配置仅增加模块枚举与示例)
## Project Structure
### Documentation (this feature)
```text
specs/007-apt-apk-cache/
├── plan.md # 本文件
├── research.md # Phase 0 输出
├── data-model.md # Phase 1 输出
├── quickstart.md # Phase 1 输出
├── contracts/ # Phase 1 输出
└── tasks.md # Phase 2 (/speckit.tasks 生成)
```
### Source Code (repository root)
```text
cmd/any-hub/main.go # CLI 入口、参数解析
internal/config/ # TOML 加载、默认值、校验
internal/server/ # Fiber 服务、路由、中间件
internal/cache/ # 磁盘/内存缓存与 .meta 管理
internal/proxy/ # 上游访问、缓存策略、流式复制
internal/hubmodule/* # 各模块 Hooks/元数据(新增 debian/apk 模块)
configs/ # 示例 config.toml如需
tests/ # go test 下的单元/集成测试
```
**Structure Decision**: 新增模块位于 `internal/hubmodule/debian` 或类似命名;复用现有 hook 注册与缓存策略接口;配置扩展在 `internal/config` 追加枚举与默认值。
## Complexity Tracking
> **Fill ONLY if Constitution Check has violations that must be justified**
| Violation | Why Needed | Simpler Alternative Rejected Because |
|-----------|------------|-------------------------------------|
| None | — | — |
## Phase 0: Outline & Research
### Unknowns / Research Tasks
- Acquire-By-Hash 处理策略:是否需要额外校验逻辑 → 选择透传路径并依赖上游校验。
- Alpine APKINDEX 签名校验需求 → 方案为透传签名文件,保持客户端校验。
### Research Output
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/research.md`
## Phase 1: Design & Contracts
### Artifacts
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/data-model.md`
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/contracts/proxy-paths.md`
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/quickstart.md`
- Agent context updated via `.specify/scripts/bash/update-agent-context.sh codex`
### Design Focus
- 定义 APT/APK 路径匹配与缓存策略(索引再验证、包体直接缓存)。
- Acquire-By-Hash/签名文件透传,避免破坏校验。
- 示例配置与测试入口httptest 上游 + 临时缓存目录)。
## Phase 2: Task Breakdown (deferred to /speckit.tasks)
- 基于用户故事和设计生成 tasks.md后续命令

View File

@@ -0,0 +1,42 @@
# Quickstart: APT/APK 缓存代理
## 1) 启用新 Hub
`config.toml` 增加示例:
```toml
[[Hub]]
Domain = "apt.hub.local"
Name = "apt"
Upstream = "https://mirrors.edge.kernel.org/ubuntu"
Type = "debian"
Module = "debian" # 待实现模块键
[[Hub]]
Domain = "apk.hub.local"
Name = "apk"
Upstream = "https://dl-cdn.alpinelinux.org/alpine"
Type = "apk"
Module = "apk" # 待实现模块键
```
## 2) 指向代理
- APT`/etc/apt/sources.list` 中的官方域名替换为 `http://apt.hub.local:5000`(或全局 ListenPort保持原 suite/component 路径。
- APK`/etc/apk/repositories` 中写入 `http://apk.hub.local:5000/v3.19/main` 等路径(与全局 ListenPort 一致)。
## 3) 验证
```bash
# APT
apt-get update
apt-get install -y curl
apt-get install -y curl # 第二次安装应直接命中缓存pool/* 与 by-hash/* 不再 HEAD 再验证)
# Alpine
apk update
apk add curl
apk add curl # 第二次安装应直接命中缓存APKINDEX 再验证packages/*.apk 直接命中)
```
观察 `logs/` 输出:首次请求应为回源,二次请求命中缓存(索引可能返回 304。如上游不可达且缓存已有包体应继续命中缓存无缓存则透传错误。

View File

@@ -0,0 +1,18 @@
# Research Notes: APT/APK 包缓存模块
## Decision Records
### 1) Acquire-By-Hash 处理
- **Decision**: 路径原样透传,缓存按完整路径存储;不做额外本地哈希校验,交由上游与客户端校验。
- **Rationale**: APT 自带哈希校验,路径即校验信息;本地重复计算增加 CPU/IO 成本且风险与上游标准重复。
- **Alternatives**: 额外本地哈希校验并拒绝不匹配(增加复杂度、可能与上游行为不一致);跳过缓存(失去加速价值)。
### 2) APT 索引再验证策略
- **Decision**: Release/InRelease/Packages* 请求统一带 If-None-Match/If-Modified-Since缓存 RequireRevalidate=true命中 304 继续用缓存200 刷新。
- **Rationale**: 与现有代理模式一致确保“latest” 索引及时更新,避免 stale。
- **Alternatives**: 固定 TTL 不再验证(风险:索引过期);强制每次全量 GET浪费带宽
### 3) APKINDEX 签名处理
- **Decision**: APKINDEX 及其签名文件原样透传并缓存,索引 RequireRevalidate=true包体直接缓存。
- **Rationale**: Alpine 客户端依靠签名文件校验,代理不应修改或剥离;再验证确保索引更新。
- **Alternatives**: 不缓存 APKINDEX失去加速效果仅缓存包体无法验证包版本更新

View File

@@ -0,0 +1,88 @@
# Feature Specification: APT/APK 包缓存模块
**Feature Branch**: `007-apt-apk-cache`
**Created**: 2025-11-17
**Status**: Draft
**Input**: User description: "Title: 设计并实现 APT/APK 包缓存模块以加速 Docker 构建 Goal: 为 any-hub 新增一个 debian/apk 模块,代理本地 Docker 构建中的 apt-get/apk add 请求,支持缓存索引与包文件并透明回源。 Constraints: - 适配 Debian/Ubuntu APT 路径Release/InRelease、Packages(.gz|.xz)、pool/*.deb及 Alpine APKINDEX/packages。 - 索引/Release 需 RequireRevalidate=true二进制包 AllowStore=true签名文件透传。 - 兼容 Acquire-By-HashAPT与 APKINDEX 签名校验,不破坏客户端校验。 - 复用现有缓存架构hooks.go + module.go + CacheStrategyProfile不得影响现有 docker/npm/pypi/go/composer 模块。 Inputs: - 代码库路径:/home/rogee/Projects/any-hub - 参考模块internal/hubmodule/docker/hooks.go (cachePolicy)、internal/hubmodule/golang/hooks.go、internal/proxy/handler.go (ETag/Last-Modified 再验证) Deliverables: - 新增 internal/hubmodule/debian或 alpine下的 hooks.go、module.go 实现 - 配套测试(模拟 apt-get update/install 或 apk update/add 的最小集) - 使用说明(简要配置示例,如何在 config.toml 添加 Hub AcceptanceCriteria: - “apt-get update” 或 “apk update” 指向代理时,首次 miss 回源,二次命中缓存;索引文件可被最新上游版本触发再验证;二进制包稳定命中缓存。 新分支开头数字是 007"
## User Scenarios & Testing *(mandatory)*
### User Story 1 - APT 更新通过代理 (Priority: P1)
本地构建镜像的开发者将 apt 源指向代理,运行 `apt-get update` 获取最新索引并希望后续重复构建无需再次访问官方仓。
**Why this priority**: APT 更新是后续安装的前提,需保证首次拉取成功且缓存生效以降低构建时间与外网依赖。
**Independent Test**: 将 apt 源指向代理后执行两次 `apt-get update`,验证首轮回源、次轮命中缓存且返回内容与官方一致。
**Acceptance Scenarios**:
1. **Given** 缓存为空,**When** 运行 `apt-get update` 指向代理,**Then** 返回官方索引内容且缓存写入成功Release/InRelease/Packages
2. **Given** 同一索引已缓存,**When** 再次运行 `apt-get update`**Then** 返回 200/304 且无额外上游 GET仅必要的条件请求输出内容与首轮一致。
---
### User Story 2 - APT 安装包命中缓存 (Priority: P2)
在构建步骤中安装常用 `.deb` 包,希望重复构建时直接从本地缓存提供 pool 下的包体。
**Why this priority**: 包文件体积大、下载耗时,命中缓存可显著缩短镜像构建时长。
**Independent Test**: 在完成 User Story 1 后,执行 `apt-get install <常见包>` 两次,确认首轮回源、次轮包体命中缓存,且校验/签名不受影响。
**Acceptance Scenarios**:
1. **Given** 缓存已有对应索引且无包体,**When** 首次安装某包,**Then** 包体成功下载并写入缓存,安装过程不因代理失败。
2. **Given** 同一包体已缓存,**When** 再次安装,**Then** 不触发上游包体下载,安装成功且校验通过。
---
### User Story 3 - Alpine APK 加速 (Priority: P3)
将 Alpine 利用代理执行 `apk update && apk add`,希望索引与包体同样缓存并可重复命中。
**Why this priority**: Alpine 镜像在容器场景常用,通过同一代理获得一致的构建加速。
**Independent Test**: 将 `/etc/apk/repositories` 指向代理,执行两轮 `apk update && apk add <常用包>`,验证首轮回源、次轮命中缓存且 APKINDEX/包体校验正常。
**Acceptance Scenarios**:
1. **Given** 代理配置完毕,**When** 首次执行 `apk update`**Then** 获取 APKINDEX 并写入缓存,随后 `apk add` 成功下载包体。
2. **Given** 同一索引与包体已缓存,**When** 再次执行 `apk update && apk add`**Then** 索引再验证后命中,包体直接命中缓存且安装成功。
---
### Edge Cases
- 上游索引更新Release/InRelease 或 APKINDEX 发生变更时,代理应通过条件请求检测到并刷新缓存。
- 校验失败Acquire-By-Hash 校验或 APKINDEX 签名不匹配时,应返回上游原始错误而非吞掉错误。
- 离线/超时:上游暂不可达时,若有缓存且策略允许,应优先返回缓存;无缓存则向客户端返回明确错误。
- 部分源未配置:多源场景下,某源缺失或路径拼写错误时,代理应返回与上游一致的 404/403。
- 大文件/磁盘不足:缓存写入包体时磁盘不足,应优雅失败并告警,不影响后续直接透传回源。
## Requirements *(mandatory)*
### Functional Requirements
- **FR-001**: 代理必须支持 APT 获取与缓存 Release/InRelease、Packages(.gz/.xz) 及 pool 下的包体;索引请求需携带条件请求并可根据上游变更刷新缓存。
- **FR-002**: 代理必须支持 Acquire-By-Hash 路径,若哈希不匹配需返回上游一致的错误,不得用缓存污染返回。
- **FR-003**: 代理必须支持 Alpine APKINDEX 与 packages 路径的获取与缓存,索引需再验证,包体可直接命中缓存。
- **FR-004**: 首次命中失败的索引/包体要回源写入缓存;后续同一路径在缓存有效期内应直接命中,除非索引再验证判定有新版本。
- **FR-005**: 代理行为须保持透明:返回状态码、头信息、签名/校验文件与上游一致,日志需记录命中/回源/失败原因,便于构建诊断。
- **FR-006**: 新增 Hub 配置字段(如仓类型、上游地址)需有默认值与示例,且不得影响现有 docker/npm/pypi/go/composer 的配置与行为。
### Key Entities *(include if feature involves data)*
- **索引文件**APT 的 Release/InRelease/Packages*Alpine 的 APKINDEX含校验或签名信息需可再验证。
- **包体文件**`pool/*/*.deb`、Alpine `packages/*`;内容不可变,按路径/哈希缓存。
- **签名/校验文件**APT 的 Release.gpg、Acquire-By-Hash 路径;用于客户端校验,需原样透传。
- **缓存条目**:存储索引或包体的本地副本,包含写入时间与可选校验标识,用于命中与再验证判定。
## Success Criteria *(mandatory)*
### Measurable Outcomes
- **SC-001**: 在同一构建机器上,第二次执行 `apt-get update``apk update` 对同一源时,上游请求数较首次减少 ≥90%(仅保留必要条件请求),且耗时降低至少 50%。
- **SC-002**: 已缓存的包体再次安装时,不触发上游下载,安装成功率达到 100%(以连续 20 次同包安装为样本)。
- **SC-003**: 当上游索引有新版本发布时,下一次索引请求能够在一次交互内获取最新数据(无旧数据残留),并保持客户端校验/签名通过率 100%。
- **SC-004**: 引入新模块后,现有 docker/npm/pypi/go/composer 代理的功能及配置使用无变化,回归用例全部通过。

View File

@@ -0,0 +1,119 @@
# Tasks: APT/APK 包缓存模块
**Input**: Design documents from `/specs/007-apt-apk-cache/`
**Prerequisites**: plan.md (required), spec.md (user stories), research.md, data-model.md, contracts/
## Format: `[ID] [P?] [Story] Description`
- **[P]**: Can run in parallel (different files, no dependencies)
- **[Story]**: User story label (US1, US2, US3)
- Include exact file paths in descriptions
## Phase 1: Setup (Shared Infrastructure)
**Purpose**: Prepare module folders and examples for new hubs.
- [X] T001 Create module directories `internal/hubmodule/debian/` and `internal/hubmodule/apk/` with placeholder go files (module.go/hooks.go scaffolds).
- [X] T002 Add sample hub entries for APT/APK in `configs/config.example.toml`.
---
## Phase 2: Foundational (Blocking Prerequisites)
**Purpose**: Core wiring for new hub types before story work.
- [X] T003 Update hub type validation to accept `debian``apk` in `internal/config/validation.go`.
- [X] T004 Register new modules in `internal/config/modules.go` and `internal/hubmodule/registry.go` (init side-effect includes debian/apk).
- [X] T005 [P] Define debian module metadata (cache strategy, TTL, validation mode) in `internal/hubmodule/debian/module.go`.
- [X] T006 [P] Define apk module metadata in `internal/hubmodule/apk/module.go`.
- [X] T007 Ensure path locator rewrite strategy (raw_path) reused for new modules in `internal/hubmodule/strategy.go` or module options if needed.
- [X] T008 Add constitution-mandated Chinese comments for new module metadata files.
**Checkpoint**: Foundation ready—new hub types recognized, modules load without runtime errors.
---
## Phase 3: User Story 1 - APT 更新通过代理 (Priority: P1) 🎯 MVP
**Goal**: 代理 APT 索引Release/InRelease/Packages*),首次回源,后续带条件请求再验证并命中缓存。
**Independent Test**: `apt-get update` 两次指向代理,首轮回源,次轮 304/命中缓存且内容与官方一致。
### Tests for User Story 1
- [X] T009 [P] [US1] Add unit tests for path classification与缓存策略索引 RequireRevalidate`internal/hubmodule/debian/hooks_test.go`.
- [X] T010 [US1] Add integration test `tests/integration/apt_update_proxy_test.go` covering first/second `apt-get update` (Release/InRelease/Packages) with httptest upstream and temp storage.
### Implementation for User Story 1
- [X] T011 [P] [US1] Implement APT hooks (NormalizePath/CachePolicy/ContentType/ResolveUpstream if needed) for index paths in `internal/hubmodule/debian/hooks.go`.
- [X] T012 [P] [US1] Support conditional requests (ETag/Last-Modified passthrough) for index responses in `internal/hubmodule/debian/hooks.go`.
- [X] T013 [US1] Wire debian module registration to use hooks in `internal/hubmodule/debian/module.go` and ensure hook registration in `hooks.go`.
- [X] T014 [US1] Ensure logging fields include cache hit/upstream for APT requests (reuse proxy logging) and document in comments `internal/hubmodule/debian/hooks.go`.
- [X] T015 [US1] Update quickstart instructions with APT usage validation steps in `specs/007-apt-apk-cache/quickstart.md`.
**Checkpoint**: APT 索引更新可独立验证并缓存。
---
## Phase 4: User Story 2 - APT 安装包命中缓存 (Priority: P2)
**Goal**: pool 下 `.deb` 包首次回源、后续直接命中缓存;保持 Acquire-By-Hash 路径透传且不污染哈希校验。
**Independent Test**: `apt-get install <包>` 两次,首轮下载并缓存,次轮无上游下载,安装成功且校验通过。
### Tests for User Story 2
- [X] T016 [P] [US2] Extend debian hook unit tests to cover `/pool/...``/by-hash/...` 缓存策略 in `internal/hubmodule/debian/hooks_test.go`.
- [X] T017 [US2] Integration test for package download caching and Acquire-By-Hash passthrough in `tests/integration/apt_package_proxy_test.go`.
### Implementation for User Story 2
- [X] T018 [P] [US2] Implement package/dist path handling (AllowCache/AllowStore, RequireRevalidate=false) in `internal/hubmodule/debian/hooks.go`.
- [X] T019 [P] [US2] Handle `/dists/<suite>/by-hash/<algo>/<hash>` as immutable cached resources in `internal/hubmodule/debian/hooks.go`.
- [X] T020 [US2] Validate cache writer/reader streaming for large deb files in `internal/proxy/handler.go` (ensure no full-buffer reads) with comments/tests if changes required.
- [X] T021 [US2] Update config docs/examples if additional APT-specific knobs are added in `configs/config.example.toml` or `README.md`.
**Checkpoint**: APT 包体可命中缓存且哈希/签名校验保持一致。
---
## Phase 5: User Story 3 - Alpine APK 加速 (Priority: P3)
**Goal**: 缓存 APKINDEX 并再验证包体packages/*.apk首次回源后直接命中缓存。
**Independent Test**: `apk update && apk add <包>` 两次,索引次轮 304/命中,包体次轮直接命中,安装成功。
### Tests for User Story 3
- [X] T022 [P] [US3] Add apk hook unit tests for index/package path policy in `internal/hubmodule/apk/hooks_test.go`.
- [X] T023 [US3] Integration test for apk update/install caching in `tests/integration/apk_proxy_test.go`.
### Implementation for User Story 3
- [X] T024 [P] [US3] Implement APK hooks (CachePolicy/ContentType/NormalizePath) for APKINDEX and packages in `internal/hubmodule/apk/hooks.go`.
- [X] T025 [P] [US3] Ensure APKINDEX/signature files RequireRevalidate and package files immutable cache in `internal/hubmodule/apk/hooks.go`.
- [X] T026 [US3] Register apk hooks in module init and update logging/observability comments in `internal/hubmodule/apk/module.go`.
- [X] T027 [US3] Add Alpine repository usage notes to `specs/007-apt-apk-cache/quickstart.md`.
**Checkpoint**: Alpine 索引与包体缓存可独立验证。
---
## Phase 6: Polish & Cross-Cutting Concerns
- [X] T028 [P] Add Chinese comments for key caching logic and path handling in new hook files (`internal/hubmodule/debian/hooks.go`, `internal/hubmodule/apk/hooks.go`).
- [X] T029 [P] Document log fields and cache semantics in `docs/` or `README.md` (structure log examples for APT/APK).
- [X] T030 Validate gofmt/go test ./... and update `specs/007-apt-apk-cache/quickstart.md` with final verification steps.
- [X] T031 [P] Confirm no regressions to existing modules via smoke test list in `tests/integration/` (reuse existing suites, adjust configs if needed).
---
## Dependencies & Execution Order
- Phase 1 → Phase 2 → User stories (Phase 3/4/5) → Phase 6.
- User Story 1 (P1) must complete before US2 (shares debian hooks); US3 can start after Phase 2 independently.
- Parallel opportunities:
- T005/T006 module metadata in parallel; hook/unit work can run in parallel within each story where marked [P].
- US3 tasks can run in parallel with US1 late-stage tasks once foundational ready (different modules/files).
- Suggested MVP: Complete Phases 1-3 (US1) to deliver APT 更新加速US2/US3 incrementally after validation.

View File

@@ -0,0 +1,345 @@
package integration
import (
"bytes"
"context"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/cache"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/proxy"
"github.com/any-hub/any-hub/internal/server"
)
func TestAPKProxyCachesIndexAndPackages(t *testing.T) {
stub := newAPKStub(t)
defer stub.Close()
storageDir := t.TempDir()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5400,
CacheTTL: config.Duration(time.Hour),
StoragePath: storageDir,
},
Hubs: []config.HubConfig{
{
Name: "apk",
Domain: "apk.hub.local",
Type: "apk",
Module: "apk",
Upstream: stub.URL,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(storageDir)
if err != nil {
t.Fatalf("store error: %v", err)
}
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store),
ListenPort: 5400,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
doRequest := func(p string) *http.Response {
req := httptest.NewRequest(http.MethodGet, "http://apk.hub.local"+p, nil)
req.Host = "apk.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test error: %v", err)
}
return resp
}
resp := doRequest(stub.indexPath)
if resp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for index, got %d", resp.StatusCode)
}
if hit := resp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected cache miss on first index fetch, got %s", hit)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if !bytes.Equal(body, stub.indexBody) {
t.Fatalf("index body mismatch on first fetch: %d bytes", len(body))
}
resp2 := doRequest(stub.indexPath)
if resp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached index, got %d", resp2.StatusCode)
}
if hit := resp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
t.Fatalf("expected cache hit for index, got %s", hit)
}
body2, _ := io.ReadAll(resp2.Body)
resp2.Body.Close()
if !bytes.Equal(body2, stub.indexBody) {
t.Fatalf("index body mismatch on cache hit: %d bytes", len(body2))
}
sigResp := doRequest(stub.signaturePath)
if sigResp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for signature, got %d", sigResp.StatusCode)
}
if hit := sigResp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected cache miss on first signature fetch, got %s", hit)
}
_, _ = io.ReadAll(sigResp.Body)
sigResp.Body.Close()
sigResp2 := doRequest(stub.signaturePath)
if sigResp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached signature, got %d", sigResp2.StatusCode)
}
if hit := sigResp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
t.Fatalf("expected cache hit for signature, got %s", hit)
}
sigResp2.Body.Close()
pkgResp := doRequest(stub.packagePath)
if pkgResp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for package, got %d", pkgResp.StatusCode)
}
if hit := pkgResp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected cache miss on first package fetch, got %s", hit)
}
pkgBody, _ := io.ReadAll(pkgResp.Body)
pkgResp.Body.Close()
if !bytes.Equal(pkgBody, stub.packageBody) {
t.Fatalf("package body mismatch on first fetch: %d bytes", len(pkgBody))
}
pkgResp2 := doRequest(stub.packagePath)
if pkgResp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached package, got %d", pkgResp2.StatusCode)
}
if hit := pkgResp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
t.Fatalf("expected cache hit for package, got %s", hit)
}
pkgBody2, _ := io.ReadAll(pkgResp2.Body)
pkgResp2.Body.Close()
if !bytes.Equal(pkgBody2, stub.packageBody) {
t.Fatalf("package body mismatch on cache hit: %d bytes", len(pkgBody2))
}
if stub.IndexGets() != 1 {
t.Fatalf("expected single index GET, got %d", stub.IndexGets())
}
if stub.IndexHeads() != 1 {
t.Fatalf("expected single index HEAD revalidate, got %d", stub.IndexHeads())
}
if stub.SignatureGets() != 1 {
t.Fatalf("expected single signature GET, got %d", stub.SignatureGets())
}
if stub.SignatureHeads() != 1 {
t.Fatalf("expected single signature HEAD revalidate, got %d", stub.SignatureHeads())
}
if stub.PackageGets() != 1 {
t.Fatalf("expected single package GET, got %d", stub.PackageGets())
}
if stub.PackageHeads() != 0 {
t.Fatalf("expected zero package HEAD revalidate, got %d", stub.PackageHeads())
}
verifyAPKStored(t, storageDir, "apk", stub.indexPath, int64(len(stub.indexBody)))
verifyAPKStored(t, storageDir, "apk", stub.signaturePath, int64(len(stub.signatureBody)))
verifyAPKStored(t, storageDir, "apk", stub.packagePath, int64(len(stub.packageBody)))
}
func verifyAPKStored(t *testing.T, basePath, hubName, locatorPath string, expectedSize int64) {
t.Helper()
clean := path.Clean("/" + locatorPath)
clean = strings.TrimPrefix(clean, "/")
fullPath := filepath.Join(basePath, hubName, clean)
info, err := os.Stat(fullPath)
if err != nil {
t.Fatalf("expected cached file at %s: %v", fullPath, err)
}
if info.Size() != expectedSize {
t.Fatalf("cached file %s size mismatch: got %d want %d", fullPath, info.Size(), expectedSize)
}
}
type apkStub struct {
server *http.Server
listener net.Listener
URL string
mu sync.Mutex
indexPath string
signaturePath string
packagePath string
indexBody []byte
signatureBody []byte
packageBody []byte
indexGets int
indexHeads int
signatureGets int
signatureHeads int
packageGets int
packageHeads int
}
func newAPKStub(t *testing.T) *apkStub {
t.Helper()
stub := &apkStub{
indexPath: "/v3.19/main/x86_64/APKINDEX.tar.gz",
signaturePath: "/v3.19/main/x86_64/APKINDEX.tar.gz.asc",
packagePath: "/v3.22/community/x86_64/tini-static-0.19.0-r3.apk",
indexBody: []byte("apk-index-body"),
signatureBody: []byte("apk-index-signature"),
packageBody: bytes.Repeat([]byte("apk-payload-"), 64*1024),
}
mux := http.NewServeMux()
mux.HandleFunc(stub.indexPath, stub.handleIndex)
mux.HandleFunc(stub.signaturePath, stub.handleSignature)
mux.HandleFunc(stub.packagePath, stub.handlePackage)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("unable to start apk stub: %v", err)
}
srv := &http.Server{Handler: mux}
stub.server = srv
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = srv.Serve(listener)
}()
return stub
}
func (s *apkStub) handleIndex(w http.ResponseWriter, r *http.Request) {
s.handleWithETag(w, r, &s.indexGets, &s.indexHeads, s.indexBody, "application/gzip")
}
func (s *apkStub) handleSignature(w http.ResponseWriter, r *http.Request) {
s.handleWithETag(w, r, &s.signatureGets, &s.signatureHeads, s.signatureBody, "application/pgp-signature")
}
func (s *apkStub) handlePackage(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
if r.Method == http.MethodHead {
s.packageHeads++
w.Header().Set("Content-Type", "application/vnd.android.package-archive")
w.WriteHeader(http.StatusOK)
return
}
s.packageGets++
w.Header().Set("Content-Type", "application/vnd.android.package-archive")
w.Header().Set("Content-Length", strconv.Itoa(len(s.packageBody)))
_, _ = w.Write(s.packageBody)
}
func (s *apkStub) handleWithETag(w http.ResponseWriter, r *http.Request, gets, heads *int, body []byte, contentType string) {
s.mu.Lock()
defer s.mu.Unlock()
etag := "\"apk-etag\""
if r.Method == http.MethodHead {
*heads++
w.Header().Set("ETag", etag)
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
if matchETag(r, strings.Trim(etag, `"`)) {
w.WriteHeader(http.StatusNotModified)
return
}
if contentType != "" {
w.Header().Set("Content-Type", contentType)
}
return
}
*gets++
w.Header().Set("ETag", etag)
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
if contentType != "" {
w.Header().Set("Content-Type", contentType)
}
_, _ = w.Write(body)
}
func (s *apkStub) IndexGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.indexGets
}
func (s *apkStub) IndexHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.indexHeads
}
func (s *apkStub) SignatureGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.signatureGets
}
func (s *apkStub) SignatureHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.signatureHeads
}
func (s *apkStub) PackageGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.packageGets
}
func (s *apkStub) PackageHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.packageHeads
}
func (s *apkStub) Close() {
if s == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}

View File

@@ -0,0 +1,277 @@
package integration
import (
"bytes"
"context"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/cache"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/proxy"
"github.com/any-hub/any-hub/internal/server"
)
func TestAptPackagesCachedWithoutRevalidate(t *testing.T) {
stub := newAptPackageStub(t)
defer stub.Close()
storageDir := t.TempDir()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
CacheTTL: config.Duration(time.Hour),
StoragePath: storageDir,
},
Hubs: []config.HubConfig{
{
Name: "apt",
Domain: "apt.hub.local",
Type: "debian",
Module: "debian",
Upstream: stub.URL,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(storageDir)
if err != nil {
t.Fatalf("store error: %v", err)
}
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store),
ListenPort: 5000,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
doRequest := func(p string) *http.Response {
req := httptest.NewRequest(http.MethodGet, "http://apt.hub.local"+p, nil)
req.Host = "apt.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test error: %v", err)
}
return resp
}
resp := doRequest(stub.packagePath)
if resp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for package, got %d", resp.StatusCode)
}
if hit := resp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected cache miss on first package fetch, got %s", hit)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if !bytes.Equal(body, stub.packageBody) {
t.Fatalf("package body mismatch on first fetch: %d bytes", len(body))
}
resp2 := doRequest(stub.packagePath)
if resp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached package, got %d", resp2.StatusCode)
}
if hit := resp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
t.Fatalf("expected cache hit for package, got %s", hit)
}
body2, _ := io.ReadAll(resp2.Body)
resp2.Body.Close()
if !bytes.Equal(body2, stub.packageBody) {
t.Fatalf("package body mismatch on cache hit: %d bytes", len(body2))
}
hashResp := doRequest(stub.byHashPath)
if hashResp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for by-hash, got %d", hashResp.StatusCode)
}
if hit := hashResp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
t.Fatalf("expected cache miss on first by-hash fetch, got %s", hit)
}
hashBody, _ := io.ReadAll(hashResp.Body)
hashResp.Body.Close()
if !bytes.Equal(hashBody, stub.byHashBody) {
t.Fatalf("by-hash body mismatch on first fetch: %d bytes", len(hashBody))
}
hashResp2 := doRequest(stub.byHashPath)
if hashResp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached by-hash, got %d", hashResp2.StatusCode)
}
if hit := hashResp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
t.Fatalf("expected cache hit for by-hash, got %s", hit)
}
hashBody2, _ := io.ReadAll(hashResp2.Body)
hashResp2.Body.Close()
if !bytes.Equal(hashBody2, stub.byHashBody) {
t.Fatalf("by-hash body mismatch on cache hit: %d bytes", len(hashBody2))
}
if stub.PackageGets() != 1 {
t.Fatalf("expected single package GET, got %d", stub.PackageGets())
}
if stub.PackageHeads() != 0 {
t.Fatalf("expected zero package HEAD revalidate, got %d", stub.PackageHeads())
}
if stub.ByHashGets() != 1 {
t.Fatalf("expected single by-hash GET, got %d", stub.ByHashGets())
}
if stub.ByHashHeads() != 0 {
t.Fatalf("expected zero by-hash HEAD revalidate, got %d", stub.ByHashHeads())
}
verifyStoredFile(t, storageDir, "apt", stub.packagePath, int64(len(stub.packageBody)))
verifyStoredFile(t, storageDir, "apt", stub.byHashPath, int64(len(stub.byHashBody)))
}
func verifyStoredFile(t *testing.T, basePath, hubName, locatorPath string, expectedSize int64) {
t.Helper()
clean := path.Clean("/" + locatorPath)
clean = strings.TrimPrefix(clean, "/")
fullPath := filepath.Join(basePath, hubName, clean)
info, err := os.Stat(fullPath)
if err != nil {
t.Fatalf("expected cached file at %s: %v", fullPath, err)
}
if info.Size() != expectedSize {
t.Fatalf("cached file %s size mismatch: got %d want %d", fullPath, info.Size(), expectedSize)
}
}
type aptPackageStub struct {
server *http.Server
listener net.Listener
URL string
mu sync.Mutex
packagePath string
byHashPath string
packageBody []byte
byHashBody []byte
packageGets int
packageHeads int
byHashGets int
byHashHeads int
}
func newAptPackageStub(t *testing.T) *aptPackageStub {
t.Helper()
stub := &aptPackageStub{
packagePath: "/pool/main/h/hello_1.0_amd64.deb",
byHashPath: "/dists/bookworm/by-hash/sha256/deadbeef",
packageBody: bytes.Repeat([]byte("deb-payload-"), 128*1024),
byHashBody: []byte("hash-index-body"),
}
mux := http.NewServeMux()
mux.HandleFunc(stub.packagePath, stub.handlePackage)
mux.HandleFunc(stub.byHashPath, stub.handleByHash)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("unable to start apt package stub: %v", err)
}
srv := &http.Server{Handler: mux}
stub.server = srv
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = srv.Serve(listener)
}()
return stub
}
func (s *aptPackageStub) handlePackage(w http.ResponseWriter, r *http.Request) {
s.handleImmutable(w, r, &s.packageGets, &s.packageHeads, s.packageBody, "application/vnd.debian.binary-package")
}
func (s *aptPackageStub) handleByHash(w http.ResponseWriter, r *http.Request) {
s.handleImmutable(w, r, &s.byHashGets, &s.byHashHeads, s.byHashBody, "text/plain")
}
func (s *aptPackageStub) handleImmutable(w http.ResponseWriter, r *http.Request, gets, heads *int, body []byte, contentType string) {
s.mu.Lock()
defer s.mu.Unlock()
if r.Method == http.MethodHead {
*heads++
if contentType != "" {
w.Header().Set("Content-Type", contentType)
}
w.WriteHeader(http.StatusOK)
return
}
*gets++
if contentType != "" {
w.Header().Set("Content-Type", contentType)
}
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
_, _ = w.Write(body)
}
func (s *aptPackageStub) PackageGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.packageGets
}
func (s *aptPackageStub) PackageHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.packageHeads
}
func (s *aptPackageStub) ByHashGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.byHashGets
}
func (s *aptPackageStub) ByHashHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.byHashHeads
}
func (s *aptPackageStub) Close() {
if s == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}

View File

@@ -0,0 +1,331 @@
package integration
import (
"context"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/gofiber/fiber/v3"
"github.com/sirupsen/logrus"
"github.com/any-hub/any-hub/internal/cache"
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/proxy"
"github.com/any-hub/any-hub/internal/server"
)
func TestAptUpdateCachesIndexes(t *testing.T) {
stub := newAptStub(t)
defer stub.Close()
storageDir := t.TempDir()
cfg := &config.Config{
Global: config.GlobalConfig{
ListenPort: 5000,
CacheTTL: config.Duration(time.Hour),
StoragePath: storageDir,
},
Hubs: []config.HubConfig{
{
Name: "apt",
Domain: "apt.hub.local",
Type: "debian",
Module: "debian",
Upstream: stub.URL,
},
},
}
registry, err := server.NewHubRegistry(cfg)
if err != nil {
t.Fatalf("registry error: %v", err)
}
logger := logrus.New()
logger.SetOutput(io.Discard)
store, err := cache.NewStore(storageDir)
if err != nil {
t.Fatalf("store error: %v", err)
}
app, err := server.NewApp(server.AppOptions{
Logger: logger,
Registry: registry,
Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store),
ListenPort: 5000,
})
if err != nil {
t.Fatalf("app error: %v", err)
}
doRequest := func(path string) *http.Response {
req := httptest.NewRequest("GET", "http://apt.hub.local"+path, nil)
req.Host = "apt.hub.local"
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test error: %v", err)
}
return resp
}
releasePath := "/dists/bookworm/Release"
packagesPath := "/dists/bookworm/main/binary-amd64/Packages.gz"
contentsPath := "/dists/bookworm/main/Contents-amd64.gz"
resp := doRequest(releasePath)
if resp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for release, got %d", resp.StatusCode)
}
if resp.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
t.Fatalf("expected cache miss for first release fetch")
}
resp.Body.Close()
resp2 := doRequest(releasePath)
if resp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached release, got %d", resp2.StatusCode)
}
if resp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
t.Fatalf("expected cache hit for release")
}
resp2.Body.Close()
pkgResp := doRequest(packagesPath)
if pkgResp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for packages, got %d", pkgResp.StatusCode)
}
if pkgResp.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
t.Fatalf("expected cache miss for packages")
}
pkgResp.Body.Close()
pkgResp2 := doRequest(packagesPath)
if pkgResp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached packages, got %d", pkgResp2.StatusCode)
}
if pkgResp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
t.Fatalf("expected cache hit for packages")
}
pkgResp2.Body.Close()
contentsResp := doRequest(contentsPath)
if contentsResp.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for contents, got %d", contentsResp.StatusCode)
}
if contentsResp.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
t.Fatalf("expected cache miss for contents")
}
contentsResp.Body.Close()
contentsResp2 := doRequest(contentsPath)
if contentsResp2.StatusCode != fiber.StatusOK {
t.Fatalf("expected 200 for cached contents, got %d", contentsResp2.StatusCode)
}
if contentsResp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
t.Fatalf("expected cache hit for contents")
}
contentsResp2.Body.Close()
if stub.ReleaseGets() != 1 {
t.Fatalf("expected single release GET, got %d", stub.ReleaseGets())
}
if stub.ReleaseHeads() != 1 {
t.Fatalf("expected single release HEAD revalidate, got %d", stub.ReleaseHeads())
}
if stub.PackagesGets() != 1 {
t.Fatalf("expected single packages GET, got %d", stub.PackagesGets())
}
if stub.PackagesHeads() != 1 {
t.Fatalf("expected single packages HEAD revalidate, got %d", stub.PackagesHeads())
}
if stub.ContentsGets() != 1 {
t.Fatalf("expected single contents GET, got %d", stub.ContentsGets())
}
if stub.ContentsHeads() != 1 {
t.Fatalf("expected single contents HEAD revalidate, got %d", stub.ContentsHeads())
}
}
type aptStub struct {
server *http.Server
listener net.Listener
URL string
mu sync.Mutex
releaseBody string
packagesBody string
contentsBody string
releaseETag string
packagesETag string
contentsETag string
releaseGets int
releaseHeads int
packagesGets int
packagesHeads int
contentsGets int
contentsHeads int
releasePath string
packagesPath string
contentsPath string
}
func newAptStub(t *testing.T) *aptStub {
t.Helper()
stub := &aptStub{
releaseBody: "Release-body",
packagesBody: "Packages-body",
contentsBody: "Contents-body",
releaseETag: "r1",
packagesETag: "p1",
contentsETag: "c1",
releasePath: "/dists/bookworm/Release",
packagesPath: "/dists/bookworm/main/binary-amd64/Packages.gz",
contentsPath: "/dists/bookworm/main/Contents-amd64.gz",
}
mux := http.NewServeMux()
mux.HandleFunc(stub.releasePath, stub.handleRelease)
mux.HandleFunc(stub.packagesPath, stub.handlePackages)
mux.HandleFunc(stub.contentsPath, stub.handleContents)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("unable to start apt stub: %v", err)
}
srv := &http.Server{Handler: mux}
stub.server = srv
stub.listener = listener
stub.URL = "http://" + listener.Addr().String()
go func() {
_ = srv.Serve(listener)
}()
return stub
}
func (s *aptStub) handleRelease(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
if r.Method == http.MethodHead {
s.releaseHeads++
if matchETag(r, s.releaseETag) {
w.WriteHeader(http.StatusNotModified)
return
}
writeHeaders(w, s.releaseETag)
return
}
s.releaseGets++
writeHeaders(w, s.releaseETag)
_, _ = w.Write([]byte(s.releaseBody))
}
func (s *aptStub) handlePackages(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
if r.Method == http.MethodHead {
s.packagesHeads++
if matchETag(r, s.packagesETag) {
w.WriteHeader(http.StatusNotModified)
return
}
writeHeaders(w, s.packagesETag)
w.Header().Set("Content-Type", "application/gzip")
return
}
s.packagesGets++
writeHeaders(w, s.packagesETag)
w.Header().Set("Content-Type", "application/gzip")
_, _ = w.Write([]byte(s.packagesBody))
}
func (s *aptStub) handleContents(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
if r.Method == http.MethodHead {
s.contentsHeads++
if matchETag(r, s.contentsETag) {
w.WriteHeader(http.StatusNotModified)
return
}
writeHeaders(w, s.contentsETag)
w.Header().Set("Content-Type", "application/gzip")
return
}
s.contentsGets++
writeHeaders(w, s.contentsETag)
w.Header().Set("Content-Type", "application/gzip")
_, _ = w.Write([]byte(s.contentsBody))
}
func matchETag(r *http.Request, etag string) bool {
for _, candidate := range r.Header.Values("If-None-Match") {
c := strings.Trim(candidate, "\"")
if c == etag || candidate == etag {
return true
}
}
return false
}
func writeHeaders(w http.ResponseWriter, etag string) {
w.Header().Set("ETag", "\""+etag+"\"")
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
}
func (s *aptStub) ReleaseGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.releaseGets
}
func (s *aptStub) ReleaseHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.releaseHeads
}
func (s *aptStub) PackagesGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.packagesGets
}
func (s *aptStub) PackagesHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.packagesHeads
}
func (s *aptStub) ContentsGets() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.contentsGets
}
func (s *aptStub) ContentsHeads() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.contentsHeads
}
func (s *aptStub) Close() {
if s == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if s.server != nil {
_ = s.server.Shutdown(ctx)
}
if s.listener != nil {
_ = s.listener.Close()
}
}

View File

@@ -2,12 +2,14 @@ package integration
import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -42,6 +44,7 @@ func TestCacheFlowWithConditionalRequest(t *testing.T) {
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Module: "docker",
Upstream: upstream.URL,
},
},
@@ -143,6 +146,7 @@ func TestDockerManifestHeadDoesNotOverwriteCache(t *testing.T) {
Name: "docker",
Domain: "docker.hub.local",
Type: "docker",
Module: "docker",
Upstream: upstream.URL,
},
},
@@ -294,6 +298,7 @@ type cacheFlowStub struct {
lastRequest *http.Request
body []byte
etag string
etagVer int
lastMod string
}
@@ -302,6 +307,7 @@ func newCacheFlowStub(t *testing.T, paths ...string) *cacheFlowStub {
stub := &cacheFlowStub{
body: []byte("upstream payload"),
etag: `"etag-v1"`,
etagVer: 1,
lastMod: time.Now().UTC().Format(http.TimeFormat),
}
@@ -344,6 +350,8 @@ func (s *cacheFlowStub) Close() {
func (s *cacheFlowStub) handle(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
etag := s.etag
lastMod := s.lastMod
if r.Method == http.MethodHead {
s.headHits++
} else {
@@ -354,15 +362,21 @@ func (s *cacheFlowStub) handle(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Etag", s.etag)
w.Header().Set("Last-Modified", s.lastMod)
w.Header().Set("Etag", etag)
w.Header().Set("Last-Modified", lastMod)
for _, candidate := range r.Header.Values("If-None-Match") {
if strings.Trim(candidate, `"`) == strings.Trim(etag, `"`) {
w.WriteHeader(http.StatusNotModified)
return
}
}
w.WriteHeader(http.StatusOK)
return
}
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Etag", s.etag)
w.Header().Set("Last-Modified", s.lastMod)
w.Header().Set("Etag", etag)
w.Header().Set("Last-Modified", lastMod)
_, _ = w.Write(s.body)
}
@@ -370,5 +384,7 @@ func (s *cacheFlowStub) UpdateBody(body []byte) {
s.mu.Lock()
defer s.mu.Unlock()
s.body = body
s.lastMod = time.Now().UTC().Format(http.TimeFormat)
s.etagVer++
s.etag = fmt.Sprintf(`"etag-v%d"`, s.etagVer)
s.lastMod = time.Now().UTC().Add(2 * time.Second).Format(http.TimeFormat)
}

View File

@@ -13,6 +13,7 @@ import (
"github.com/any-hub/any-hub/internal/config"
"github.com/any-hub/any-hub/internal/hubmodule"
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
"github.com/any-hub/any-hub/internal/proxy/hooks"
"github.com/any-hub/any-hub/internal/server"
"github.com/any-hub/any-hub/internal/server/routes"
@@ -41,6 +42,8 @@ func TestModuleDiagnosticsEndpoints(t *testing.T) {
Domain: "legacy.local",
Type: "docker",
Upstream: "https://registry-1.docker.io",
Module: hubmodule.DefaultModuleKey(),
Rollout: string(legacy.RolloutLegacyOnly),
},
{
Name: "modern-hub",

View File

@@ -239,7 +239,7 @@ func (s *pypiStub) handleWheel(w http.ResponseWriter, r *http.Request) {
func (s *pypiStub) UpdateSimple(body []byte) {
s.mu.Lock()
s.simpleBody = append([]byte(nil), body...)
s.lastSimpleMod = time.Now().UTC().Format(http.TimeFormat)
s.lastSimpleMod = time.Now().UTC().Add(2 * time.Second).Format(http.TimeFormat)
s.mu.Unlock()
}