Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dcd85a9f41 | |||
| ba5544c28d | |||
| 68b6bb78e6 |
@@ -10,6 +10,8 @@ Auto-generated from all feature plans. Last updated: 2025-11-13
|
||||
- 本地文件系统缓存目录 `StoragePath/<Hub>/<path>`,模块需直接复用原始路径布局 (004-modular-proxy-cache)
|
||||
- 本地文件系统缓存目录 `StoragePath/<Hub>/<path>`(按模块定义的布局) (005-proxy-module-delegation)
|
||||
- 本地文件系统缓存目录 `StoragePath/<Hub>/<path>`(由模块 Hook 定义布局) (006-module-hook-refactor)
|
||||
- Go 1.25+ (静态链接,单二进制) + Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack、标准库 `net/http`/`io` (007-apt-apk-cache)
|
||||
- 本地 `StoragePath/<Hub>/<path>` + `.meta`;索引需带 Last-Modified/ETag 元信息;包体按原路径落盘 (007-apt-apk-cache)
|
||||
|
||||
- Go 1.25+ (静态链接,单二进制交付) + Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack(结构化日志 [EXTRACTED FROM ALL PLAN.MD FILES] 滚动)、标准库 `net/http`/`io` (001-config-bootstrap)
|
||||
|
||||
@@ -29,9 +31,9 @@ tests/
|
||||
Go 1.25+ (静态链接,单二进制交付): Follow standard conventions
|
||||
|
||||
## Recent Changes
|
||||
- 007-apt-apk-cache: Added Go 1.25+ (静态链接,单二进制) + Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack、标准库 `net/http`/`io`
|
||||
- 006-module-hook-refactor: Added Go 1.25+ (静态链接,单二进制交付) + Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack(结构化日志 & 滚动)、标准库 `net/http`/`io`
|
||||
- 005-proxy-module-delegation: Added Go 1.25+ (静态链接,单二进制交付) + Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack(结构化日志 & 滚动)、标准库 `net/http`/`io`
|
||||
- 004-modular-proxy-cache: Added Go 1.25+ (静态链接,单二进制交付) + Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack(结构化日志 & 滚动)、标准库 `net/http`/`io`
|
||||
|
||||
|
||||
<!-- MANUAL ADDITIONS START -->
|
||||
|
||||
@@ -86,3 +86,21 @@ Username = ""
|
||||
Password = ""
|
||||
Type = "composer"
|
||||
Module = "composer"
|
||||
|
||||
# Debian/Ubuntu APT 示例
|
||||
[[Hub]]
|
||||
Name = "apt-cache"
|
||||
Domain = "apt.hub.local"
|
||||
Upstream = "https://mirrors.edge.kernel.org/ubuntu"
|
||||
Type = "debian"
|
||||
Module = "debian"
|
||||
Rollout = "modular"
|
||||
|
||||
# Alpine APK 示例
|
||||
[[Hub]]
|
||||
Name = "apk-cache"
|
||||
Domain = "apk.hub.local"
|
||||
Upstream = "https://dl-cdn.alpinelinux.org/alpine"
|
||||
Type = "apk"
|
||||
Module = "apk"
|
||||
Rollout = "modular"
|
||||
|
||||
@@ -41,3 +41,21 @@ Upstream = "https://registry.npmjs.org"
|
||||
Type = "npm"
|
||||
Module = "legacy" # 仍未迁移的 Hub 可显式指定 legacy,诊断会标记为 legacy-only
|
||||
Rollout = "legacy-only"
|
||||
|
||||
# Debian/Ubuntu APT 示例
|
||||
[[Hub]]
|
||||
Name = "apt-cache"
|
||||
Domain = "apt.hub.local"
|
||||
Upstream = "https://mirrors.edge.kernel.org/ubuntu"
|
||||
Type = "debian"
|
||||
Module = "debian"
|
||||
Rollout = "modular"
|
||||
|
||||
# Alpine APK 示例
|
||||
[[Hub]]
|
||||
Name = "apk-cache"
|
||||
Domain = "apk.hub.local"
|
||||
Upstream = "https://dl-cdn.alpinelinux.org/alpine"
|
||||
Type = "apk"
|
||||
Module = "apk"
|
||||
Rollout = "modular"
|
||||
|
||||
22
docs/operations/logging.md
Normal file
22
docs/operations/logging.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Logging & Cache Semantics (APT/APK)
|
||||
|
||||
## Common Fields
|
||||
- `hub`/`domain`/`hub_type`:当前 Hub 标识与协议类型,例如 `debian`/`apk`。
|
||||
- `module_key`:命中的模块键(非 `legacy` 时表示新模块生效)。
|
||||
- `cache_hit`:`true` 表示直接复用缓存;`false` 表示从上游获取或已刷新。
|
||||
- `upstream`/`upstream_status`:实际访问的上游地址与状态码。
|
||||
- `rollout_flag`:`legacy-only`/`dual`/`modular`,便于排查路由与灰度。
|
||||
- `action`:`proxy`,表明代理链路日志。
|
||||
|
||||
## APT (debian 模块)
|
||||
- 索引路径(Release/InRelease/Packages*):`cache_hit=true` 仍会在后台进行 HEAD 再验证;命中 304 时保持缓存。
|
||||
- 包体路径(`/pool/*` 和 `/dists/.../by-hash/...`):视为不可变,首次 GET 落盘,后续直接命中,无 HEAD。
|
||||
- 日志可结合 `X-Any-Hub-Cache-Hit` 响应头进行对照。
|
||||
|
||||
## APK (apk 模块)
|
||||
- APKINDEX 及签名:每次命中会触发 HEAD 再验证,缓存命中返回 304 时继续使用本地文件。
|
||||
- 包体 (`packages/*.apk`):不可变资源,首轮 GET 落盘,后续直接命中,无 HEAD。
|
||||
|
||||
## Quick Checks
|
||||
- 观察 `cache_hit` 与 `upstream_status`:`cache_hit=true`、`upstream_status=200/304` 表示缓存复用成功;`cache_hit=false` 表示回源或刷新。
|
||||
- 若期望模块日志字段但出现 `module_key":"legacy"`,检查 `Module` 与 `Rollout` 配置是否指向新模块。
|
||||
@@ -107,6 +107,12 @@ func applyHubDefaults(h *HubConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeHubConfig 公开给无需依赖 loader 的调用方(例如测试)以填充模块/rollout 默认值。
|
||||
func NormalizeHubConfig(h HubConfig) HubConfig {
|
||||
applyHubDefaults(&h)
|
||||
return h
|
||||
}
|
||||
|
||||
func durationDecodeHook() mapstructure.DecodeHookFunc {
|
||||
targetType := reflect.TypeOf(Duration(0))
|
||||
|
||||
|
||||
@@ -2,9 +2,11 @@ package config
|
||||
|
||||
import (
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/composer"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/debian"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/docker"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/golang"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/apk"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/npm"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/pypi"
|
||||
)
|
||||
|
||||
@@ -8,6 +8,13 @@ import (
|
||||
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
)
|
||||
|
||||
// Rollout 字段说明(legacy → modular 平滑迁移控制):
|
||||
// - legacy-only:强制使用 legacy 模块(EffectiveModuleKey → legacy);用于未迁移或需要快速回滚时。
|
||||
// - dual:新模块为默认,保留 legacy 以便诊断/灰度;仅当 Module 非空时生效,否则回退 legacy-only。
|
||||
// - modular:仅使用新模块;Module 为空或 legacy 模块时自动回退 legacy-only。
|
||||
// 默认行为:未填写 Rollout 时,空 Module/legacy 模块默认 legacy-only;其它模块默认 modular。
|
||||
// 影响范围:动态选择执行的模块键(EffectiveModuleKey)、路由日志中的 rollout_flag,方便区分迁移阶段。
|
||||
|
||||
// parseRolloutFlag 将配置中的 rollout 字段标准化,并结合模块类型输出最终状态。
|
||||
func parseRolloutFlag(raw string, moduleKey string) (legacy.RolloutFlag, error) {
|
||||
normalized := strings.ToLower(strings.TrimSpace(raw))
|
||||
|
||||
@@ -16,9 +16,11 @@ var supportedHubTypes = map[string]struct{}{
|
||||
"go": {},
|
||||
"pypi": {},
|
||||
"composer": {},
|
||||
"debian": {},
|
||||
"apk": {},
|
||||
}
|
||||
|
||||
const supportedHubTypeList = "docker|npm|go|pypi|composer"
|
||||
const supportedHubTypeList = "docker|npm|go|pypi|composer|debian|apk"
|
||||
|
||||
// Validate 针对语义级别做进一步校验,防止非法配置启动服务。
|
||||
func (c *Config) Validate() error {
|
||||
|
||||
83
internal/hubmodule/apk/hooks.go
Normal file
83
internal/hubmodule/apk/hooks.go
Normal file
@@ -0,0 +1,83 @@
|
||||
// Package apk defines hook behaviors for Alpine APK proxying.
|
||||
// APKINDEX/签名需要再验证;packages/*.apk 视为不可变缓存。
|
||||
package apk
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/proxy/hooks"
|
||||
)
|
||||
|
||||
func init() {
|
||||
hooks.MustRegister("apk", hooks.Hooks{
|
||||
NormalizePath: normalizePath,
|
||||
CachePolicy: cachePolicy,
|
||||
ContentType: contentType,
|
||||
})
|
||||
}
|
||||
|
||||
func normalizePath(_ *hooks.RequestContext, p string, rawQuery []byte) (string, []byte) {
|
||||
clean := path.Clean("/" + strings.TrimSpace(p))
|
||||
return clean, rawQuery
|
||||
}
|
||||
|
||||
func cachePolicy(_ *hooks.RequestContext, locatorPath string, current hooks.CachePolicy) hooks.CachePolicy {
|
||||
clean := canonicalPath(locatorPath)
|
||||
switch {
|
||||
case isAPKIndexPath(clean), isAPKSignaturePath(clean):
|
||||
// APKINDEX 及签名需要再验证,确保索引最新。
|
||||
current.AllowCache = true
|
||||
current.AllowStore = true
|
||||
current.RequireRevalidate = true
|
||||
case isAPKPackagePath(clean):
|
||||
// 包体不可变,允许直接命中缓存,无需 HEAD。
|
||||
current.AllowCache = true
|
||||
current.AllowStore = true
|
||||
current.RequireRevalidate = false
|
||||
default:
|
||||
current.AllowCache = false
|
||||
current.AllowStore = false
|
||||
current.RequireRevalidate = false
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func contentType(_ *hooks.RequestContext, locatorPath string) string {
|
||||
clean := canonicalPath(locatorPath)
|
||||
switch {
|
||||
case strings.HasSuffix(clean, ".apk"):
|
||||
return "application/vnd.android.package-archive"
|
||||
case strings.HasSuffix(clean, ".tar.gz"):
|
||||
return "application/gzip"
|
||||
case strings.HasSuffix(clean, ".tar.gz.asc") || strings.HasSuffix(clean, ".tar.gz.sig"):
|
||||
return "application/pgp-signature"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func isAPKIndexPath(p string) bool {
|
||||
clean := canonicalPath(p)
|
||||
return strings.HasSuffix(clean, "/apkindex.tar.gz")
|
||||
}
|
||||
|
||||
func isAPKSignaturePath(p string) bool {
|
||||
clean := canonicalPath(p)
|
||||
return strings.HasSuffix(clean, "/apkindex.tar.gz.asc") || strings.HasSuffix(clean, "/apkindex.tar.gz.sig")
|
||||
}
|
||||
|
||||
func isAPKPackagePath(p string) bool {
|
||||
clean := canonicalPath(p)
|
||||
if isAPKIndexPath(clean) || isAPKSignaturePath(clean) {
|
||||
return false
|
||||
}
|
||||
return strings.HasSuffix(clean, ".apk")
|
||||
}
|
||||
|
||||
func canonicalPath(p string) string {
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
return strings.ToLower(path.Clean("/" + strings.TrimSpace(p)))
|
||||
}
|
||||
61
internal/hubmodule/apk/hooks_test.go
Normal file
61
internal/hubmodule/apk/hooks_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package apk
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/proxy/hooks"
|
||||
)
|
||||
|
||||
func TestCachePolicyIndexAndSignatureRevalidate(t *testing.T) {
|
||||
paths := []string{
|
||||
"/v3.19/main/x86_64/APKINDEX.tar.gz",
|
||||
"/v3.19/main/x86_64/APKINDEX.tar.gz.asc",
|
||||
"/v3.19/community/aarch64/apkindex.tar.gz.sig",
|
||||
}
|
||||
for _, p := range paths {
|
||||
current := cachePolicy(nil, p, hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
|
||||
t.Fatalf("expected index/signature to require revalidate for %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePolicyPackageImmutable(t *testing.T) {
|
||||
tests := []string{
|
||||
"/v3.19/main/x86_64/packages/hello-1.0.apk",
|
||||
"/v3.18/testing/aarch64/packages/../packages/hello-1.0-r1.APK",
|
||||
"/v3.22/community/x86_64/tini-static-0.19.0-r3.apk", // 路径不含 /packages/ 也应视作包体
|
||||
}
|
||||
for _, p := range tests {
|
||||
current := cachePolicy(nil, p, hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || current.RequireRevalidate {
|
||||
t.Fatalf("expected immutable cache for %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePolicyNonAPKPath(t *testing.T) {
|
||||
current := cachePolicy(nil, "/other/path", hooks.CachePolicy{})
|
||||
if current.AllowCache || current.AllowStore || current.RequireRevalidate {
|
||||
t.Fatalf("expected non-APK path to disable cache/store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePath(t *testing.T) {
|
||||
p, _ := normalizePath(nil, "v3.19/main/x86_64/APKINDEX.tar.gz", nil)
|
||||
if p != "/v3.19/main/x86_64/APKINDEX.tar.gz" {
|
||||
t.Fatalf("unexpected normalized path: %s", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
if ct := contentType(nil, "/v3.19/main/x86_64/APKINDEX.tar.gz"); ct != "application/gzip" {
|
||||
t.Fatalf("expected gzip content type, got %s", ct)
|
||||
}
|
||||
if ct := contentType(nil, "/v3.19/main/x86_64/APKINDEX.tar.gz.asc"); ct != "application/pgp-signature" {
|
||||
t.Fatalf("expected signature content type, got %s", ct)
|
||||
}
|
||||
if ct := contentType(nil, "/v3.19/main/x86_64/packages/hello.apk"); ct != "application/vnd.android.package-archive" {
|
||||
t.Fatalf("expected apk content type, got %s", ct)
|
||||
}
|
||||
}
|
||||
29
internal/hubmodule/apk/module.go
Normal file
29
internal/hubmodule/apk/module.go
Normal file
@@ -0,0 +1,29 @@
|
||||
// Package apk registers metadata for Alpine APK proxying.
|
||||
package apk
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
const apkDefaultTTL = 6 * time.Hour
|
||||
|
||||
func init() {
|
||||
// 模块元数据声明,具体 hooks 见 hooks.go(已在 init 自动注册)。
|
||||
hubmodule.MustRegister(hubmodule.ModuleMetadata{
|
||||
Key: "apk",
|
||||
Description: "Alpine APK proxy with cached indexes and packages",
|
||||
MigrationState: hubmodule.MigrationStateBeta,
|
||||
SupportedProtocols: []string{
|
||||
"apk",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: 0, // APKINDEX 每次再验证,包体直接命中
|
||||
ValidationMode: hubmodule.ValidationModeLastModified, // APKINDEX 再验证
|
||||
DiskLayout: "raw_path",
|
||||
RequiresMetadataFile: false,
|
||||
SupportsStreamingWrite: true, // 包体流式写
|
||||
},
|
||||
})
|
||||
}
|
||||
103
internal/hubmodule/debian/hooks.go
Normal file
103
internal/hubmodule/debian/hooks.go
Normal file
@@ -0,0 +1,103 @@
|
||||
// Package debian defines hook behaviors for APT (Debian/Ubuntu) proxying.
|
||||
// 索引(Release/InRelease/Packages*)需要再验证;包体(pool/ 和 by-hash)视为不可变直接缓存。
|
||||
// 日志字段沿用通用 proxy(命中/上游状态),无需额外改写。
|
||||
package debian
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/proxy/hooks"
|
||||
)
|
||||
|
||||
func init() {
|
||||
hooks.MustRegister("debian", hooks.Hooks{
|
||||
NormalizePath: normalizePath,
|
||||
CachePolicy: cachePolicy,
|
||||
ContentType: contentType,
|
||||
})
|
||||
}
|
||||
|
||||
func normalizePath(_ *hooks.RequestContext, p string, rawQuery []byte) (string, []byte) {
|
||||
clean := path.Clean("/" + strings.TrimSpace(p))
|
||||
return clean, rawQuery
|
||||
}
|
||||
|
||||
func cachePolicy(_ *hooks.RequestContext, locatorPath string, current hooks.CachePolicy) hooks.CachePolicy {
|
||||
clean := canonicalPath(locatorPath)
|
||||
switch {
|
||||
case isAptIndexPath(clean):
|
||||
// 索引类(Release/Packages)需要 If-None-Match/If-Modified-Since 再验证。
|
||||
current.AllowCache = true
|
||||
current.AllowStore = true
|
||||
current.RequireRevalidate = true
|
||||
case isAptImmutablePath(clean):
|
||||
// pool/*.deb 与 by-hash 路径视为不可变,直接缓存后续不再 HEAD。
|
||||
current.AllowCache = true
|
||||
current.AllowStore = true
|
||||
current.RequireRevalidate = false
|
||||
default:
|
||||
current.AllowCache = false
|
||||
current.AllowStore = false
|
||||
current.RequireRevalidate = false
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func contentType(_ *hooks.RequestContext, locatorPath string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(locatorPath, ".gz"):
|
||||
return "application/gzip"
|
||||
case strings.HasSuffix(locatorPath, ".xz"):
|
||||
return "application/x-xz"
|
||||
case strings.HasSuffix(locatorPath, "Release.gpg"):
|
||||
return "application/pgp-signature"
|
||||
case isAptIndexPath(locatorPath):
|
||||
return "text/plain"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func isAptIndexPath(p string) bool {
|
||||
clean := canonicalPath(p)
|
||||
if isByHashPath(clean) {
|
||||
return false
|
||||
}
|
||||
|
||||
if strings.Contains(clean, "/dists/") {
|
||||
if strings.HasSuffix(clean, "/release") ||
|
||||
strings.HasSuffix(clean, "/inrelease") ||
|
||||
strings.HasSuffix(clean, "/release.gpg") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAptImmutablePath(p string) bool {
|
||||
clean := canonicalPath(p)
|
||||
if isByHashPath(clean) {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(clean, "/pool/") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isByHashPath(p string) bool {
|
||||
clean := canonicalPath(p)
|
||||
if strings.Contains(clean, "/dists/") {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(clean, "/by-hash/")
|
||||
}
|
||||
|
||||
func canonicalPath(p string) string {
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
return strings.ToLower(path.Clean("/" + strings.TrimSpace(p)))
|
||||
}
|
||||
73
internal/hubmodule/debian/hooks_test.go
Normal file
73
internal/hubmodule/debian/hooks_test.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package debian
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/proxy/hooks"
|
||||
)
|
||||
|
||||
func TestCachePolicyIndexesRevalidate(t *testing.T) {
|
||||
current := cachePolicy(nil, "/dists/bookworm/Release", hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
|
||||
t.Fatalf("expected index to allow cache/store and revalidate")
|
||||
}
|
||||
current = cachePolicy(nil, "/dists/bookworm/main/binary-amd64/Packages.gz", hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
|
||||
t.Fatalf("expected packages index to revalidate")
|
||||
}
|
||||
current = cachePolicy(nil, "/dists/bookworm/main/Contents-amd64.gz", hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
|
||||
t.Fatalf("expected contents index to revalidate")
|
||||
}
|
||||
current = cachePolicy(nil, "/debian-security/dists/trixie/Contents-amd64.gz", hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || !current.RequireRevalidate {
|
||||
t.Fatalf("expected prefixed contents index to revalidate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePolicyImmutable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{name: "by-hash index snapshot", path: "/dists/bookworm/by-hash/sha256/abc"},
|
||||
{name: "by-hash nested", path: "/dists/bookworm/main/binary-amd64/by-hash/SHA256/def"},
|
||||
{name: "pool package", path: "/pool/main/h/hello.deb"},
|
||||
{name: "pool canonicalized", path: " /PoOl/main/../main/h/hello_1.0_amd64.DeB "},
|
||||
{name: "mirror prefix pool", path: "/debian/pool/main/h/hello.deb"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
current := cachePolicy(nil, tt.path, hooks.CachePolicy{})
|
||||
if !current.AllowCache || !current.AllowStore || current.RequireRevalidate {
|
||||
t.Fatalf("expected immutable cache for %s", tt.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePolicyNonAptPath(t *testing.T) {
|
||||
current := cachePolicy(nil, "/other/path", hooks.CachePolicy{})
|
||||
if current.AllowCache || current.AllowStore || current.RequireRevalidate {
|
||||
t.Fatalf("expected non-APT path to disable cache/store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePath(t *testing.T) {
|
||||
path, _ := normalizePath(nil, "dists/bookworm/Release", nil)
|
||||
if path != "/dists/bookworm/Release" {
|
||||
t.Fatalf("unexpected path: %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
if ct := contentType(nil, "/dists/bookworm/Release"); ct != "text/plain" {
|
||||
t.Fatalf("expected text/plain for Release, got %s", ct)
|
||||
}
|
||||
if ct := contentType(nil, "/dists/bookworm/Release.gpg"); ct != "application/pgp-signature" {
|
||||
t.Fatalf("expected signature content-type, got %s", ct)
|
||||
}
|
||||
if ct := contentType(nil, "/dists/bookworm/main/binary-amd64/Packages.gz"); ct != "application/gzip" {
|
||||
t.Fatalf("expected gzip content-type, got %s", ct)
|
||||
}
|
||||
}
|
||||
29
internal/hubmodule/debian/module.go
Normal file
29
internal/hubmodule/debian/module.go
Normal file
@@ -0,0 +1,29 @@
|
||||
// Package debian registers metadata for Debian/Ubuntu APT proxying.
|
||||
package debian
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
const debianDefaultTTL = 6 * time.Hour
|
||||
|
||||
func init() {
|
||||
// 仅声明模块元数据(缓存策略等);具体 hooks 在后续实现。
|
||||
hubmodule.MustRegister(hubmodule.ModuleMetadata{
|
||||
Key: "debian",
|
||||
Description: "APT proxy with cached indexes and packages",
|
||||
MigrationState: hubmodule.MigrationStateBeta,
|
||||
SupportedProtocols: []string{
|
||||
"debian",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: 0, // 索引每次再验证,由 ETag/Last-Modified 控制
|
||||
ValidationMode: hubmodule.ValidationModeLastModified, // 索引使用 Last-Modified/ETag 再验证
|
||||
DiskLayout: "raw_path", // 复用通用原始路径布局
|
||||
RequiresMetadataFile: false,
|
||||
SupportsStreamingWrite: true, // 包体需要流式写入,避免大文件占用内存
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -15,9 +15,6 @@ func init() {
|
||||
}
|
||||
|
||||
func normalizePath(ctx *hooks.RequestContext, clean string, rawQuery []byte) (string, []byte) {
|
||||
if !isDockerHubHost(ctx.UpstreamHost) {
|
||||
return clean, rawQuery
|
||||
}
|
||||
repo, rest, ok := splitDockerRepoPath(clean)
|
||||
if !ok || repo == "" || strings.Contains(repo, "/") || repo == "library" {
|
||||
return clean, rawQuery
|
||||
|
||||
@@ -19,7 +19,7 @@ func init() {
|
||||
"docker",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: dockerDefaultTTL,
|
||||
TTLHint: 0, // manifests 需每次再验证,由 ETag 控制新鲜度
|
||||
ValidationMode: hubmodule.ValidationModeETag,
|
||||
DiskLayout: "raw_path",
|
||||
RequiresMetadataFile: false,
|
||||
|
||||
@@ -19,7 +19,7 @@ func init() {
|
||||
"pypi",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: pypiDefaultTTL,
|
||||
TTLHint: 0, // simple index 每次再验证
|
||||
ValidationMode: hubmodule.ValidationModeLastModified,
|
||||
DiskLayout: "raw_path",
|
||||
RequiresMetadataFile: false,
|
||||
|
||||
@@ -11,7 +11,7 @@ type StrategyOptions struct {
|
||||
// ResolveStrategy 将模块的默认策略与 hub 级覆盖合并。
|
||||
func ResolveStrategy(meta ModuleMetadata, opts StrategyOptions) CacheStrategyProfile {
|
||||
strategy := meta.CacheStrategy
|
||||
if opts.TTLOverride > 0 {
|
||||
if strategy.TTLHint > 0 && opts.TTLOverride > 0 {
|
||||
strategy.TTLHint = opts.TTLOverride
|
||||
}
|
||||
if opts.ValidationOverride != "" {
|
||||
|
||||
@@ -203,6 +203,12 @@ func (h *Handler) serveCache(
|
||||
c.Status(status)
|
||||
|
||||
if method == http.MethodHead {
|
||||
// 对 HEAD 请求仍向上游发起一次 HEAD,以满足“显式请求 + 再验证”的期望。
|
||||
if route != nil && route.UpstreamURL != nil {
|
||||
if resp, err := h.revalidateRequest(c, route, resolveUpstreamURL(route, route.UpstreamURL, c, hook), result.Entry.Locator, ""); err == nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
result.Reader.Close()
|
||||
h.logResult(route, route.UpstreamURL.String(), requestID, status, true, started, nil)
|
||||
return nil
|
||||
@@ -358,6 +364,7 @@ func (h *Handler) cacheAndStream(
|
||||
}
|
||||
c.Status(resp.StatusCode)
|
||||
|
||||
// 使用 TeeReader 边向客户端回写边落盘,避免大文件在内存中完整缓冲。
|
||||
reader := io.TeeReader(resp.Body, c.Response().BodyWriter())
|
||||
|
||||
opts := cache.PutOptions{ModTime: extractModTime(resp.Header)}
|
||||
@@ -733,7 +740,7 @@ func determineCachePolicyWithHook(route *server.HubRoute, locator cache.Locator,
|
||||
}
|
||||
|
||||
func determineCachePolicy(route *server.HubRoute, locator cache.Locator, method string) cachePolicy {
|
||||
if method != http.MethodGet {
|
||||
if method != http.MethodGet && method != http.MethodHead {
|
||||
return cachePolicy{}
|
||||
}
|
||||
return cachePolicy{allowCache: true, allowStore: true}
|
||||
@@ -786,9 +793,12 @@ func (h *Handler) isCacheFresh(
|
||||
case http.StatusNotModified:
|
||||
return true, nil
|
||||
case http.StatusOK:
|
||||
if resp.Header.Get("Etag") == "" && resp.Header.Get("Docker-Content-Digest") == "" && resp.Header.Get("Last-Modified") == "" {
|
||||
return true, nil
|
||||
}
|
||||
h.rememberETag(route, locator, resp)
|
||||
remote := extractModTime(resp.Header)
|
||||
if !remote.After(entry.ModTime.Add(time.Second)) {
|
||||
if !remote.After(entry.ModTime) {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
|
||||
@@ -56,6 +56,7 @@ func NewHubRegistry(cfg *config.Config) (*HubRegistry, error) {
|
||||
}
|
||||
|
||||
for _, hub := range cfg.Hubs {
|
||||
hub = config.NormalizeHubConfig(hub)
|
||||
normalizedHost := normalizeDomain(hub.Domain)
|
||||
if normalizedHost == "" {
|
||||
return nil, fmt.Errorf("invalid domain for hub %s", hub.Name)
|
||||
|
||||
@@ -49,14 +49,14 @@ func TestHubRegistryLookupByHost(t *testing.T) {
|
||||
if route.CacheTTL != cfg.EffectiveCacheTTL(route.Config) {
|
||||
t.Errorf("cache ttl mismatch: got %s", route.CacheTTL)
|
||||
}
|
||||
if route.CacheStrategy.TTLHint != route.CacheTTL {
|
||||
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, route.CacheTTL)
|
||||
if route.CacheStrategy.TTLHint != 0 {
|
||||
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, time.Duration(0))
|
||||
}
|
||||
if route.CacheStrategy.ValidationMode == "" {
|
||||
t.Fatalf("cache strategy validation mode should not be empty")
|
||||
}
|
||||
if route.RolloutFlag != legacy.RolloutLegacyOnly {
|
||||
t.Fatalf("default rollout flag should be legacy-only")
|
||||
if route.RolloutFlag != legacy.RolloutModular {
|
||||
t.Fatalf("default rollout flag should be modular")
|
||||
}
|
||||
|
||||
if route.UpstreamURL.String() != "https://registry-1.docker.io" {
|
||||
|
||||
34
specs/007-apt-apk-cache/checklists/requirements.md
Normal file
34
specs/007-apt-apk-cache/checklists/requirements.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# Specification Quality Checklist: APT/APK 包缓存模块
|
||||
|
||||
**Purpose**: Validate specification completeness and quality before proceeding to planning
|
||||
**Created**: 2025-11-17
|
||||
**Feature**: /home/rogee/Projects/any-hub/specs/007-apt-apk-cache/spec.md
|
||||
|
||||
## Content Quality
|
||||
|
||||
- [x] No implementation details (languages, frameworks, APIs)
|
||||
- [x] Focused on user value and business needs
|
||||
- [x] Written for non-technical stakeholders
|
||||
- [x] All mandatory sections completed
|
||||
|
||||
## Requirement Completeness
|
||||
|
||||
- [x] No [NEEDS CLARIFICATION] markers remain
|
||||
- [x] Requirements are testable and unambiguous
|
||||
- [x] Success criteria are measurable
|
||||
- [x] Success criteria are technology-agnostic (no implementation details)
|
||||
- [x] All acceptance scenarios are defined
|
||||
- [x] Edge cases are identified
|
||||
- [x] Scope is clearly bounded
|
||||
- [x] Dependencies and assumptions identified
|
||||
|
||||
## Feature Readiness
|
||||
|
||||
- [x] All functional requirements have clear acceptance criteria
|
||||
- [x] User scenarios cover primary flows
|
||||
- [x] Feature meets measurable outcomes defined in Success Criteria
|
||||
- [x] No implementation details leak into specification
|
||||
|
||||
## Notes
|
||||
|
||||
- Items marked complete; ready to proceed to planning.
|
||||
26
specs/007-apt-apk-cache/contracts/proxy-paths.md
Normal file
26
specs/007-apt-apk-cache/contracts/proxy-paths.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# Proxy Path Contracts: APT/APK
|
||||
|
||||
## APT
|
||||
|
||||
| Downstream Request | Upstream Target | Caching | Notes |
|
||||
|--------------------|-----------------|---------|-------|
|
||||
| `/dists/<suite>/<component>/binary-<arch>/Packages[.gz/.xz]` | same path on Upstream | RequireRevalidate=true | Send If-None-Match/If-Modified-Since if cached |
|
||||
| `/dists/<suite>/Release` | same path | RequireRevalidate=true | Preserve content; may have accompanying `.gpg` |
|
||||
| `/dists/<suite>/InRelease` | same path | RequireRevalidate=true | Signed inline; no body changes |
|
||||
| `/dists/<suite>/Release.gpg` | same path | RequireRevalidate=true | Signature only; no rewrite |
|
||||
| `/pool/<vendor>/<name>.deb` | same path | AllowCache/AllowStore=true, RequireRevalidate=false | Treat as immutable |
|
||||
| `/dists/<suite>/by-hash/<algo>/<hash>` | same path | AllowCache/AllowStore=true, RequireRevalidate=false | Path encodes hash, no extra validation |
|
||||
|
||||
## Alpine APK
|
||||
|
||||
| Downstream Request | Upstream Target | Caching | Notes |
|
||||
|--------------------|-----------------|---------|-------|
|
||||
| `/v3.<branch>/main/<arch>/APKINDEX.tar.gz` (and variants) | same path | RequireRevalidate=true | Preserve signature/headers |
|
||||
| `/v3.<branch>/<repo>/<arch>/APKINDEX.tar.gz` | same path | RequireRevalidate=true | Same handling as above |
|
||||
| `/v3.<branch>/<repo>/<arch>/packages/<file>.apk` | same path | AllowCache/AllowStore=true, RequireRevalidate=false | Immutable package |
|
||||
| APKINDEX signature files | same path | RequireRevalidate=true | No rewrite |
|
||||
|
||||
## Behaviors
|
||||
- No body/URL rewrite; proxy only normalizes cache policy per path shape.
|
||||
- Preserve status codes, headers, and signature files exactly as upstream.
|
||||
- Conditional requests (If-None-Match/If-Modified-Since) applied to all index/signature paths when cached.
|
||||
36
specs/007-apt-apk-cache/data-model.md
Normal file
36
specs/007-apt-apk-cache/data-model.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Data Model: APT/APK 包缓存模块
|
||||
|
||||
## Entities
|
||||
|
||||
### HubConfig (APT/APK)
|
||||
- Fields: `Name`, `Domain`, `Port`, `Upstream`, `Type` (`debian`/`apk`), optional `CacheTTL` override.
|
||||
- Rules: `Type` 必须新增枚举;`Upstream` 必须为 HTTP/HTTPS;每个 Hub 独立缓存前缀。
|
||||
|
||||
### IndexFile
|
||||
- Represents: APT Release/InRelease/Packages*,Alpine APKINDEX。
|
||||
- Attributes: `Path`, `ETag` (if present), `Last-Modified`, `Hash` (if provided in index), `Size`, `ContentType`.
|
||||
- Rules: RequireRevalidate=true;缓存命中需携带条件请求;内容不得修改。
|
||||
|
||||
### PackageFile
|
||||
- Represents: APT `pool/*/*.deb` 包体,Alpine `packages/<arch>/*.apk`。
|
||||
- Attributes: `Path`, `Size`, `Hash` (from upstream metadata if available), `StoredAt`.
|
||||
- Rules: 视作不可变;AllowCache/AllowStore=true;不做本地内容改写。
|
||||
|
||||
### SignatureFile
|
||||
- Represents: APT `Release.gpg`/`InRelease` 自带签名,Alpine APKINDEX 签名文件。
|
||||
- Attributes: `Path`, `Size`, `StoredAt`.
|
||||
- Rules: 原样透传;与对应 IndexFile 绑定,同步缓存与再验证。
|
||||
|
||||
### CacheEntry
|
||||
- Represents: 本地缓存记录(索引或包体)。
|
||||
- Attributes: `LocatorPath`, `ModTime`, `ETag`, `Size`, `AllowStore`, `RequireRevalidate`.
|
||||
- Rules: 读取时决定是否回源;写入需同步 `.meta`;路径格式 `StoragePath/<Hub>/<Path>`.
|
||||
|
||||
## Relationships
|
||||
- HubConfig → IndexFile/PackageFile/SignatureFile 通过 Domain/Upstream 绑定。
|
||||
- IndexFile ↔ SignatureFile:同一索引文件的签名文件需同周期缓存与再验证。
|
||||
- CacheEntry 聚合 IndexFile/PackageFile/SignatureFile 的落盘信息,用于策略判定。
|
||||
|
||||
## State/Transitions
|
||||
- CacheEntry states: `miss` → `fetched` → `validated` (304) / `refreshed` (200) → `stale` (TTL 或上游 200 新内容) → `removed` (404 或清理)。
|
||||
- Transitions triggered by upstream响应:304 保持内容,200 刷新,404 删除相关缓存。
|
||||
94
specs/007-apt-apk-cache/plan.md
Normal file
94
specs/007-apt-apk-cache/plan.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Implementation Plan: APT/APK 包缓存模块
|
||||
|
||||
**Branch**: `007-apt-apk-cache` | **Date**: 2025-11-17 | **Spec**: `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/spec.md`
|
||||
**Input**: Feature specification from `/specs/007-apt-apk-cache/spec.md`
|
||||
|
||||
## Summary
|
||||
|
||||
为 any-hub 增加 APT(Debian/Ubuntu)与 Alpine APK 的缓存代理模块:索引(Release/InRelease/Packages*/APKINDEX)必须再验证,包体(pool/*.deb、packages/*)视作不可变直接缓存,支持 Acquire-By-Hash/签名透传,不影响现有各模块。
|
||||
|
||||
## Technical Context
|
||||
|
||||
**Language/Version**: Go 1.25+ (静态链接,单二进制)
|
||||
**Primary Dependencies**: Fiber v3(HTTP 服务)、Viper(配置)、Logrus + Lumberjack、标准库 `net/http`/`io`
|
||||
**Storage**: 本地 `StoragePath/<Hub>/<path>` + `.meta`;索引需带 Last-Modified/ETag 元信息;包体按原路径落盘
|
||||
**Protocols**: APT (`/dists/<suite>/<component>/binary-<arch>/Packages*`, `Release`, `InRelease`, `pool/*`), Acquire-By-Hash;Alpine (`APKINDEX.tar.gz`, `packages/<arch>/…`)
|
||||
**Caching Rules**: 索引 RequireRevalidate=true;包体 AllowStore/AllowCache=true、RequireRevalidate=false;签名/校验文件原样透传
|
||||
**Testing**: `go test ./...`,构造 httptest 上游返回 Release/Packages/APKINDEX 与包体,验证首次回源+再验证+命中;使用临时目录校验缓存落盘
|
||||
**Target Platform**: Linux/Unix CLI,systemd/supervisor 托管,匿名客户端
|
||||
**Constraints**: 单一 config.toml 控制;禁止新增第三方依赖;保持现有模块行为不变
|
||||
**Risk/Unknowns**: Acquire-By-Hash 路径映射是否需额外校验逻辑(选择原样透传,路径即校验)
|
||||
|
||||
## Constitution Check
|
||||
|
||||
*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.*
|
||||
|
||||
- Feature 仍然是“轻量多仓 CLI 代理”,未引入 Web UI、账号体系或与代理无关的能力。
|
||||
- 仅使用 Go + 宪法指定依赖;任何新第三方库都已在本计划中说明理由与审核结论。
|
||||
- 行为完全由 `config.toml` 控制,新增配置项已规划默认值、校验与迁移策略。
|
||||
- 方案维持缓存优先 + 流式回源路径,并给出命中/回源/失败的日志与观测手段。
|
||||
- 计划内列出了配置解析、缓存读写、Host Header 路由等强制测试与中文注释交付范围。
|
||||
|
||||
**Status**: PASS(未引入新依赖/界面,新增配置仅增加模块枚举与示例)
|
||||
|
||||
## Project Structure
|
||||
|
||||
### Documentation (this feature)
|
||||
|
||||
```text
|
||||
specs/007-apt-apk-cache/
|
||||
├── plan.md # 本文件
|
||||
├── research.md # Phase 0 输出
|
||||
├── data-model.md # Phase 1 输出
|
||||
├── quickstart.md # Phase 1 输出
|
||||
├── contracts/ # Phase 1 输出
|
||||
└── tasks.md # Phase 2 (/speckit.tasks 生成)
|
||||
```
|
||||
|
||||
### Source Code (repository root)
|
||||
```text
|
||||
cmd/any-hub/main.go # CLI 入口、参数解析
|
||||
internal/config/ # TOML 加载、默认值、校验
|
||||
internal/server/ # Fiber 服务、路由、中间件
|
||||
internal/cache/ # 磁盘/内存缓存与 .meta 管理
|
||||
internal/proxy/ # 上游访问、缓存策略、流式复制
|
||||
internal/hubmodule/* # 各模块 Hooks/元数据(新增 debian/apk 模块)
|
||||
configs/ # 示例 config.toml(如需)
|
||||
tests/ # go test 下的单元/集成测试
|
||||
```
|
||||
|
||||
**Structure Decision**: 新增模块位于 `internal/hubmodule/debian` 或类似命名;复用现有 hook 注册与缓存策略接口;配置扩展在 `internal/config` 追加枚举与默认值。
|
||||
|
||||
## Complexity Tracking
|
||||
|
||||
> **Fill ONLY if Constitution Check has violations that must be justified**
|
||||
|
||||
| Violation | Why Needed | Simpler Alternative Rejected Because |
|
||||
|-----------|------------|-------------------------------------|
|
||||
| None | — | — |
|
||||
|
||||
## Phase 0: Outline & Research
|
||||
|
||||
### Unknowns / Research Tasks
|
||||
- Acquire-By-Hash 处理策略:是否需要额外校验逻辑 → 选择透传路径并依赖上游校验。
|
||||
- Alpine APKINDEX 签名校验需求 → 方案为透传签名文件,保持客户端校验。
|
||||
|
||||
### Research Output
|
||||
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/research.md`
|
||||
|
||||
## Phase 1: Design & Contracts
|
||||
|
||||
### Artifacts
|
||||
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/data-model.md`
|
||||
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/contracts/proxy-paths.md`
|
||||
- `/home/rogee/Projects/any-hub/specs/007-apt-apk-cache/quickstart.md`
|
||||
- Agent context updated via `.specify/scripts/bash/update-agent-context.sh codex`
|
||||
|
||||
### Design Focus
|
||||
- 定义 APT/APK 路径匹配与缓存策略(索引再验证、包体直接缓存)。
|
||||
- Acquire-By-Hash/签名文件透传,避免破坏校验。
|
||||
- 示例配置与测试入口(httptest 上游 + 临时缓存目录)。
|
||||
|
||||
## Phase 2: Task Breakdown (deferred to /speckit.tasks)
|
||||
|
||||
- 基于用户故事和设计生成 tasks.md(后续命令)
|
||||
42
specs/007-apt-apk-cache/quickstart.md
Normal file
42
specs/007-apt-apk-cache/quickstart.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Quickstart: APT/APK 缓存代理
|
||||
|
||||
## 1) 启用新 Hub
|
||||
|
||||
在 `config.toml` 增加示例:
|
||||
|
||||
```toml
|
||||
[[Hub]]
|
||||
Domain = "apt.hub.local"
|
||||
Name = "apt"
|
||||
Upstream = "https://mirrors.edge.kernel.org/ubuntu"
|
||||
Type = "debian"
|
||||
Module = "debian" # 待实现模块键
|
||||
|
||||
[[Hub]]
|
||||
Domain = "apk.hub.local"
|
||||
Name = "apk"
|
||||
Upstream = "https://dl-cdn.alpinelinux.org/alpine"
|
||||
Type = "apk"
|
||||
Module = "apk" # 待实现模块键
|
||||
```
|
||||
|
||||
## 2) 指向代理
|
||||
|
||||
- APT:将 `/etc/apt/sources.list` 中的官方域名替换为 `http://apt.hub.local:5000`(或全局 ListenPort),保持原 suite/component 路径。
|
||||
- APK:在 `/etc/apk/repositories` 中写入 `http://apk.hub.local:5000/v3.19/main` 等路径(与全局 ListenPort 一致)。
|
||||
|
||||
## 3) 验证
|
||||
|
||||
```bash
|
||||
# APT
|
||||
apt-get update
|
||||
apt-get install -y curl
|
||||
apt-get install -y curl # 第二次安装应直接命中缓存(pool/* 与 by-hash/* 不再 HEAD 再验证)
|
||||
|
||||
# Alpine
|
||||
apk update
|
||||
apk add curl
|
||||
apk add curl # 第二次安装应直接命中缓存(APKINDEX 再验证,packages/*.apk 直接命中)
|
||||
```
|
||||
|
||||
观察 `logs/` 输出:首次请求应为回源,二次请求命中缓存(索引可能返回 304)。如上游不可达且缓存已有包体,应继续命中缓存;无缓存则透传错误。
|
||||
18
specs/007-apt-apk-cache/research.md
Normal file
18
specs/007-apt-apk-cache/research.md
Normal file
@@ -0,0 +1,18 @@
|
||||
# Research Notes: APT/APK 包缓存模块
|
||||
|
||||
## Decision Records
|
||||
|
||||
### 1) Acquire-By-Hash 处理
|
||||
- **Decision**: 路径原样透传,缓存按完整路径存储;不做额外本地哈希校验,交由上游与客户端校验。
|
||||
- **Rationale**: APT 自带哈希校验,路径即校验信息;本地重复计算增加 CPU/IO 成本且风险与上游标准重复。
|
||||
- **Alternatives**: 额外本地哈希校验并拒绝不匹配(增加复杂度、可能与上游行为不一致);跳过缓存(失去加速价值)。
|
||||
|
||||
### 2) APT 索引再验证策略
|
||||
- **Decision**: Release/InRelease/Packages* 请求统一带 If-None-Match/If-Modified-Since,缓存 RequireRevalidate=true;命中 304 继续用缓存,200 刷新。
|
||||
- **Rationale**: 与现有代理模式一致,确保“latest” 索引及时更新,避免 stale。
|
||||
- **Alternatives**: 固定 TTL 不再验证(风险:索引过期);强制每次全量 GET(浪费带宽)。
|
||||
|
||||
### 3) APKINDEX 签名处理
|
||||
- **Decision**: APKINDEX 及其签名文件原样透传并缓存,索引 RequireRevalidate=true,包体直接缓存。
|
||||
- **Rationale**: Alpine 客户端依靠签名文件校验,代理不应修改或剥离;再验证确保索引更新。
|
||||
- **Alternatives**: 不缓存 APKINDEX(失去加速效果);仅缓存包体(无法验证包版本更新)。
|
||||
88
specs/007-apt-apk-cache/spec.md
Normal file
88
specs/007-apt-apk-cache/spec.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Feature Specification: APT/APK 包缓存模块
|
||||
|
||||
**Feature Branch**: `007-apt-apk-cache`
|
||||
**Created**: 2025-11-17
|
||||
**Status**: Draft
|
||||
**Input**: User description: "Title: 设计并实现 APT/APK 包缓存模块以加速 Docker 构建 Goal: 为 any-hub 新增一个 debian/apk 模块,代理本地 Docker 构建中的 apt-get/apk add 请求,支持缓存索引与包文件并透明回源。 Constraints: - 适配 Debian/Ubuntu APT 路径(Release/InRelease、Packages(.gz|.xz)、pool/*.deb)及 Alpine APKINDEX/packages。 - 索引/Release 需 RequireRevalidate=true;二进制包 AllowStore=true;签名文件透传。 - 兼容 Acquire-By-Hash(APT)与 APKINDEX 签名校验,不破坏客户端校验。 - 复用现有缓存架构(hooks.go + module.go + CacheStrategyProfile),不得影响现有 docker/npm/pypi/go/composer 模块。 Inputs: - 代码库路径:/home/rogee/Projects/any-hub - 参考模块:internal/hubmodule/docker/hooks.go (cachePolicy)、internal/hubmodule/golang/hooks.go、internal/proxy/handler.go (ETag/Last-Modified 再验证) Deliverables: - 新增 internal/hubmodule/debian(或 alpine)下的 hooks.go、module.go 实现 - 配套测试(模拟 apt-get update/install 或 apk update/add 的最小集) - 使用说明(简要配置示例,如何在 config.toml 添加 Hub) AcceptanceCriteria: - “apt-get update” 或 “apk update” 指向代理时,首次 miss 回源,二次命中缓存;索引文件可被最新上游版本触发再验证;二进制包稳定命中缓存。 新分支开头数字是 007"
|
||||
|
||||
## User Scenarios & Testing *(mandatory)*
|
||||
|
||||
### User Story 1 - APT 更新通过代理 (Priority: P1)
|
||||
|
||||
本地构建镜像的开发者将 apt 源指向代理,运行 `apt-get update` 获取最新索引并希望后续重复构建无需再次访问官方仓。
|
||||
|
||||
**Why this priority**: APT 更新是后续安装的前提,需保证首次拉取成功且缓存生效以降低构建时间与外网依赖。
|
||||
|
||||
**Independent Test**: 将 apt 源指向代理后执行两次 `apt-get update`,验证首轮回源、次轮命中缓存且返回内容与官方一致。
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** 缓存为空,**When** 运行 `apt-get update` 指向代理,**Then** 返回官方索引内容且缓存写入成功(Release/InRelease/Packages)。
|
||||
2. **Given** 同一索引已缓存,**When** 再次运行 `apt-get update`,**Then** 返回 200/304 且无额外上游 GET(仅必要的条件请求),输出内容与首轮一致。
|
||||
|
||||
---
|
||||
|
||||
### User Story 2 - APT 安装包命中缓存 (Priority: P2)
|
||||
|
||||
在构建步骤中安装常用 `.deb` 包,希望重复构建时直接从本地缓存提供 pool 下的包体。
|
||||
|
||||
**Why this priority**: 包文件体积大、下载耗时,命中缓存可显著缩短镜像构建时长。
|
||||
|
||||
**Independent Test**: 在完成 User Story 1 后,执行 `apt-get install <常见包>` 两次,确认首轮回源、次轮包体命中缓存,且校验/签名不受影响。
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** 缓存已有对应索引且无包体,**When** 首次安装某包,**Then** 包体成功下载并写入缓存,安装过程不因代理失败。
|
||||
2. **Given** 同一包体已缓存,**When** 再次安装,**Then** 不触发上游包体下载,安装成功且校验通过。
|
||||
|
||||
---
|
||||
|
||||
### User Story 3 - Alpine APK 加速 (Priority: P3)
|
||||
|
||||
将 Alpine 利用代理执行 `apk update && apk add`,希望索引与包体同样缓存并可重复命中。
|
||||
|
||||
**Why this priority**: Alpine 镜像在容器场景常用,通过同一代理获得一致的构建加速。
|
||||
|
||||
**Independent Test**: 将 `/etc/apk/repositories` 指向代理,执行两轮 `apk update && apk add <常用包>`,验证首轮回源、次轮命中缓存且 APKINDEX/包体校验正常。
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** 代理配置完毕,**When** 首次执行 `apk update`,**Then** 获取 APKINDEX 并写入缓存,随后 `apk add` 成功下载包体。
|
||||
2. **Given** 同一索引与包体已缓存,**When** 再次执行 `apk update && apk add`,**Then** 索引再验证后命中,包体直接命中缓存且安装成功。
|
||||
|
||||
---
|
||||
|
||||
### Edge Cases
|
||||
|
||||
- 上游索引更新:Release/InRelease 或 APKINDEX 发生变更时,代理应通过条件请求检测到并刷新缓存。
|
||||
- 校验失败:Acquire-By-Hash 校验或 APKINDEX 签名不匹配时,应返回上游原始错误而非吞掉错误。
|
||||
- 离线/超时:上游暂不可达时,若有缓存且策略允许,应优先返回缓存;无缓存则向客户端返回明确错误。
|
||||
- 部分源未配置:多源场景下,某源缺失或路径拼写错误时,代理应返回与上游一致的 404/403。
|
||||
- 大文件/磁盘不足:缓存写入包体时磁盘不足,应优雅失败并告警,不影响后续直接透传回源。
|
||||
|
||||
## Requirements *(mandatory)*
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- **FR-001**: 代理必须支持 APT 获取与缓存 Release/InRelease、Packages(.gz/.xz) 及 pool 下的包体;索引请求需携带条件请求并可根据上游变更刷新缓存。
|
||||
- **FR-002**: 代理必须支持 Acquire-By-Hash 路径,若哈希不匹配需返回上游一致的错误,不得用缓存污染返回。
|
||||
- **FR-003**: 代理必须支持 Alpine APKINDEX 与 packages 路径的获取与缓存,索引需再验证,包体可直接命中缓存。
|
||||
- **FR-004**: 首次命中失败的索引/包体要回源写入缓存;后续同一路径在缓存有效期内应直接命中,除非索引再验证判定有新版本。
|
||||
- **FR-005**: 代理行为须保持透明:返回状态码、头信息、签名/校验文件与上游一致,日志需记录命中/回源/失败原因,便于构建诊断。
|
||||
- **FR-006**: 新增 Hub 配置字段(如仓类型、上游地址)需有默认值与示例,且不得影响现有 docker/npm/pypi/go/composer 的配置与行为。
|
||||
|
||||
### Key Entities *(include if feature involves data)*
|
||||
|
||||
- **索引文件**:APT 的 Release/InRelease/Packages*,Alpine 的 APKINDEX;含校验或签名信息,需可再验证。
|
||||
- **包体文件**:`pool/*/*.deb`、Alpine `packages/*`;内容不可变,按路径/哈希缓存。
|
||||
- **签名/校验文件**:APT 的 Release.gpg、Acquire-By-Hash 路径;用于客户端校验,需原样透传。
|
||||
- **缓存条目**:存储索引或包体的本地副本,包含写入时间与可选校验标识,用于命中与再验证判定。
|
||||
|
||||
## Success Criteria *(mandatory)*
|
||||
|
||||
### Measurable Outcomes
|
||||
|
||||
- **SC-001**: 在同一构建机器上,第二次执行 `apt-get update` 或 `apk update` 对同一源时,上游请求数较首次减少 ≥90%(仅保留必要条件请求),且耗时降低至少 50%。
|
||||
- **SC-002**: 已缓存的包体再次安装时,不触发上游下载,安装成功率达到 100%(以连续 20 次同包安装为样本)。
|
||||
- **SC-003**: 当上游索引有新版本发布时,下一次索引请求能够在一次交互内获取最新数据(无旧数据残留),并保持客户端校验/签名通过率 100%。
|
||||
- **SC-004**: 引入新模块后,现有 docker/npm/pypi/go/composer 代理的功能及配置使用无变化,回归用例全部通过。
|
||||
119
specs/007-apt-apk-cache/tasks.md
Normal file
119
specs/007-apt-apk-cache/tasks.md
Normal file
@@ -0,0 +1,119 @@
|
||||
# Tasks: APT/APK 包缓存模块
|
||||
|
||||
**Input**: Design documents from `/specs/007-apt-apk-cache/`
|
||||
**Prerequisites**: plan.md (required), spec.md (user stories), research.md, data-model.md, contracts/
|
||||
|
||||
## Format: `[ID] [P?] [Story] Description`
|
||||
|
||||
- **[P]**: Can run in parallel (different files, no dependencies)
|
||||
- **[Story]**: User story label (US1, US2, US3)
|
||||
- Include exact file paths in descriptions
|
||||
|
||||
## Phase 1: Setup (Shared Infrastructure)
|
||||
|
||||
**Purpose**: Prepare module folders and examples for new hubs.
|
||||
|
||||
- [X] T001 Create module directories `internal/hubmodule/debian/` and `internal/hubmodule/apk/` with placeholder go files (module.go/hooks.go scaffolds).
|
||||
- [X] T002 Add sample hub entries for APT/APK in `configs/config.example.toml`.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Foundational (Blocking Prerequisites)
|
||||
|
||||
**Purpose**: Core wiring for new hub types before story work.
|
||||
|
||||
- [X] T003 Update hub type validation to accept `debian` 和 `apk` in `internal/config/validation.go`.
|
||||
- [X] T004 Register new modules in `internal/config/modules.go` and `internal/hubmodule/registry.go` (init side-effect includes debian/apk).
|
||||
- [X] T005 [P] Define debian module metadata (cache strategy, TTL, validation mode) in `internal/hubmodule/debian/module.go`.
|
||||
- [X] T006 [P] Define apk module metadata in `internal/hubmodule/apk/module.go`.
|
||||
- [X] T007 Ensure path locator rewrite strategy (raw_path) reused for new modules in `internal/hubmodule/strategy.go` or module options if needed.
|
||||
- [X] T008 Add constitution-mandated Chinese comments for new module metadata files.
|
||||
|
||||
**Checkpoint**: Foundation ready—new hub types recognized, modules load without runtime errors.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: User Story 1 - APT 更新通过代理 (Priority: P1) 🎯 MVP
|
||||
|
||||
**Goal**: 代理 APT 索引(Release/InRelease/Packages*),首次回源,后续带条件请求再验证并命中缓存。
|
||||
|
||||
**Independent Test**: `apt-get update` 两次指向代理,首轮回源,次轮 304/命中缓存且内容与官方一致。
|
||||
|
||||
### Tests for User Story 1
|
||||
|
||||
- [X] T009 [P] [US1] Add unit tests for path classification与缓存策略(索引 RequireRevalidate)在 `internal/hubmodule/debian/hooks_test.go`.
|
||||
- [X] T010 [US1] Add integration test `tests/integration/apt_update_proxy_test.go` covering first/second `apt-get update` (Release/InRelease/Packages) with httptest upstream and temp storage.
|
||||
|
||||
### Implementation for User Story 1
|
||||
|
||||
- [X] T011 [P] [US1] Implement APT hooks (NormalizePath/CachePolicy/ContentType/ResolveUpstream if needed) for index paths in `internal/hubmodule/debian/hooks.go`.
|
||||
- [X] T012 [P] [US1] Support conditional requests (ETag/Last-Modified passthrough) for index responses in `internal/hubmodule/debian/hooks.go`.
|
||||
- [X] T013 [US1] Wire debian module registration to use hooks in `internal/hubmodule/debian/module.go` and ensure hook registration in `hooks.go`.
|
||||
- [X] T014 [US1] Ensure logging fields include cache hit/upstream for APT requests (reuse proxy logging) and document in comments `internal/hubmodule/debian/hooks.go`.
|
||||
- [X] T015 [US1] Update quickstart instructions with APT usage validation steps in `specs/007-apt-apk-cache/quickstart.md`.
|
||||
|
||||
**Checkpoint**: APT 索引更新可独立验证并缓存。
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: User Story 2 - APT 安装包命中缓存 (Priority: P2)
|
||||
|
||||
**Goal**: pool 下 `.deb` 包首次回源、后续直接命中缓存;保持 Acquire-By-Hash 路径透传且不污染哈希校验。
|
||||
|
||||
**Independent Test**: `apt-get install <包>` 两次,首轮下载并缓存,次轮无上游下载,安装成功且校验通过。
|
||||
|
||||
### Tests for User Story 2
|
||||
|
||||
- [X] T016 [P] [US2] Extend debian hook unit tests to cover `/pool/...` 与 `/by-hash/...` 缓存策略 in `internal/hubmodule/debian/hooks_test.go`.
|
||||
- [X] T017 [US2] Integration test for package download caching and Acquire-By-Hash passthrough in `tests/integration/apt_package_proxy_test.go`.
|
||||
|
||||
### Implementation for User Story 2
|
||||
|
||||
- [X] T018 [P] [US2] Implement package/dist path handling (AllowCache/AllowStore, RequireRevalidate=false) in `internal/hubmodule/debian/hooks.go`.
|
||||
- [X] T019 [P] [US2] Handle `/dists/<suite>/by-hash/<algo>/<hash>` as immutable cached resources in `internal/hubmodule/debian/hooks.go`.
|
||||
- [X] T020 [US2] Validate cache writer/reader streaming for large deb files in `internal/proxy/handler.go` (ensure no full-buffer reads) with comments/tests if changes required.
|
||||
- [X] T021 [US2] Update config docs/examples if additional APT-specific knobs are added in `configs/config.example.toml` or `README.md`.
|
||||
|
||||
**Checkpoint**: APT 包体可命中缓存且哈希/签名校验保持一致。
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: User Story 3 - Alpine APK 加速 (Priority: P3)
|
||||
|
||||
**Goal**: 缓存 APKINDEX 并再验证;包体(packages/*.apk)首次回源后直接命中缓存。
|
||||
|
||||
**Independent Test**: `apk update && apk add <包>` 两次,索引次轮 304/命中,包体次轮直接命中,安装成功。
|
||||
|
||||
### Tests for User Story 3
|
||||
|
||||
- [X] T022 [P] [US3] Add apk hook unit tests for index/package path policy in `internal/hubmodule/apk/hooks_test.go`.
|
||||
- [X] T023 [US3] Integration test for apk update/install caching in `tests/integration/apk_proxy_test.go`.
|
||||
|
||||
### Implementation for User Story 3
|
||||
|
||||
- [X] T024 [P] [US3] Implement APK hooks (CachePolicy/ContentType/NormalizePath) for APKINDEX and packages in `internal/hubmodule/apk/hooks.go`.
|
||||
- [X] T025 [P] [US3] Ensure APKINDEX/signature files RequireRevalidate and package files immutable cache in `internal/hubmodule/apk/hooks.go`.
|
||||
- [X] T026 [US3] Register apk hooks in module init and update logging/observability comments in `internal/hubmodule/apk/module.go`.
|
||||
- [X] T027 [US3] Add Alpine repository usage notes to `specs/007-apt-apk-cache/quickstart.md`.
|
||||
|
||||
**Checkpoint**: Alpine 索引与包体缓存可独立验证。
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Polish & Cross-Cutting Concerns
|
||||
|
||||
- [X] T028 [P] Add Chinese comments for key caching logic and path handling in new hook files (`internal/hubmodule/debian/hooks.go`, `internal/hubmodule/apk/hooks.go`).
|
||||
- [X] T029 [P] Document log fields and cache semantics in `docs/` or `README.md` (structure log examples for APT/APK).
|
||||
- [X] T030 Validate gofmt/go test ./... and update `specs/007-apt-apk-cache/quickstart.md` with final verification steps.
|
||||
- [X] T031 [P] Confirm no regressions to existing modules via smoke test list in `tests/integration/` (reuse existing suites, adjust configs if needed).
|
||||
|
||||
---
|
||||
|
||||
## Dependencies & Execution Order
|
||||
|
||||
- Phase 1 → Phase 2 → User stories (Phase 3/4/5) → Phase 6.
|
||||
- User Story 1 (P1) must complete before US2 (shares debian hooks); US3 can start after Phase 2 independently.
|
||||
- Parallel opportunities:
|
||||
- T005/T006 module metadata in parallel; hook/unit work can run in parallel within each story where marked [P].
|
||||
- US3 tasks can run in parallel with US1 late-stage tasks once foundational ready (different modules/files).
|
||||
- Suggested MVP: Complete Phases 1-3 (US1) to deliver APT 更新加速;US2/US3 incrementally after validation.
|
||||
345
tests/integration/apk_proxy_test.go
Normal file
345
tests/integration/apk_proxy_test.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/cache"
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/proxy"
|
||||
"github.com/any-hub/any-hub/internal/server"
|
||||
)
|
||||
|
||||
func TestAPKProxyCachesIndexAndPackages(t *testing.T) {
|
||||
stub := newAPKStub(t)
|
||||
defer stub.Close()
|
||||
|
||||
storageDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Global: config.GlobalConfig{
|
||||
ListenPort: 5400,
|
||||
CacheTTL: config.Duration(time.Hour),
|
||||
StoragePath: storageDir,
|
||||
},
|
||||
Hubs: []config.HubConfig{
|
||||
{
|
||||
Name: "apk",
|
||||
Domain: "apk.hub.local",
|
||||
Type: "apk",
|
||||
Module: "apk",
|
||||
Upstream: stub.URL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
registry, err := server.NewHubRegistry(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("registry error: %v", err)
|
||||
}
|
||||
|
||||
logger := logrus.New()
|
||||
logger.SetOutput(io.Discard)
|
||||
|
||||
store, err := cache.NewStore(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("store error: %v", err)
|
||||
}
|
||||
|
||||
app, err := server.NewApp(server.AppOptions{
|
||||
Logger: logger,
|
||||
Registry: registry,
|
||||
Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store),
|
||||
ListenPort: 5400,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("app error: %v", err)
|
||||
}
|
||||
|
||||
doRequest := func(p string) *http.Response {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://apk.hub.local"+p, nil)
|
||||
req.Host = "apk.hub.local"
|
||||
resp, err := app.Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("app.Test error: %v", err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
resp := doRequest(stub.indexPath)
|
||||
if resp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for index, got %d", resp.StatusCode)
|
||||
}
|
||||
if hit := resp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
|
||||
t.Fatalf("expected cache miss on first index fetch, got %s", hit)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !bytes.Equal(body, stub.indexBody) {
|
||||
t.Fatalf("index body mismatch on first fetch: %d bytes", len(body))
|
||||
}
|
||||
|
||||
resp2 := doRequest(stub.indexPath)
|
||||
if resp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached index, got %d", resp2.StatusCode)
|
||||
}
|
||||
if hit := resp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
|
||||
t.Fatalf("expected cache hit for index, got %s", hit)
|
||||
}
|
||||
body2, _ := io.ReadAll(resp2.Body)
|
||||
resp2.Body.Close()
|
||||
if !bytes.Equal(body2, stub.indexBody) {
|
||||
t.Fatalf("index body mismatch on cache hit: %d bytes", len(body2))
|
||||
}
|
||||
|
||||
sigResp := doRequest(stub.signaturePath)
|
||||
if sigResp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for signature, got %d", sigResp.StatusCode)
|
||||
}
|
||||
if hit := sigResp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
|
||||
t.Fatalf("expected cache miss on first signature fetch, got %s", hit)
|
||||
}
|
||||
_, _ = io.ReadAll(sigResp.Body)
|
||||
sigResp.Body.Close()
|
||||
|
||||
sigResp2 := doRequest(stub.signaturePath)
|
||||
if sigResp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached signature, got %d", sigResp2.StatusCode)
|
||||
}
|
||||
if hit := sigResp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
|
||||
t.Fatalf("expected cache hit for signature, got %s", hit)
|
||||
}
|
||||
sigResp2.Body.Close()
|
||||
|
||||
pkgResp := doRequest(stub.packagePath)
|
||||
if pkgResp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for package, got %d", pkgResp.StatusCode)
|
||||
}
|
||||
if hit := pkgResp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
|
||||
t.Fatalf("expected cache miss on first package fetch, got %s", hit)
|
||||
}
|
||||
pkgBody, _ := io.ReadAll(pkgResp.Body)
|
||||
pkgResp.Body.Close()
|
||||
if !bytes.Equal(pkgBody, stub.packageBody) {
|
||||
t.Fatalf("package body mismatch on first fetch: %d bytes", len(pkgBody))
|
||||
}
|
||||
|
||||
pkgResp2 := doRequest(stub.packagePath)
|
||||
if pkgResp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached package, got %d", pkgResp2.StatusCode)
|
||||
}
|
||||
if hit := pkgResp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
|
||||
t.Fatalf("expected cache hit for package, got %s", hit)
|
||||
}
|
||||
pkgBody2, _ := io.ReadAll(pkgResp2.Body)
|
||||
pkgResp2.Body.Close()
|
||||
if !bytes.Equal(pkgBody2, stub.packageBody) {
|
||||
t.Fatalf("package body mismatch on cache hit: %d bytes", len(pkgBody2))
|
||||
}
|
||||
|
||||
if stub.IndexGets() != 1 {
|
||||
t.Fatalf("expected single index GET, got %d", stub.IndexGets())
|
||||
}
|
||||
if stub.IndexHeads() != 1 {
|
||||
t.Fatalf("expected single index HEAD revalidate, got %d", stub.IndexHeads())
|
||||
}
|
||||
if stub.SignatureGets() != 1 {
|
||||
t.Fatalf("expected single signature GET, got %d", stub.SignatureGets())
|
||||
}
|
||||
if stub.SignatureHeads() != 1 {
|
||||
t.Fatalf("expected single signature HEAD revalidate, got %d", stub.SignatureHeads())
|
||||
}
|
||||
if stub.PackageGets() != 1 {
|
||||
t.Fatalf("expected single package GET, got %d", stub.PackageGets())
|
||||
}
|
||||
if stub.PackageHeads() != 0 {
|
||||
t.Fatalf("expected zero package HEAD revalidate, got %d", stub.PackageHeads())
|
||||
}
|
||||
|
||||
verifyAPKStored(t, storageDir, "apk", stub.indexPath, int64(len(stub.indexBody)))
|
||||
verifyAPKStored(t, storageDir, "apk", stub.signaturePath, int64(len(stub.signatureBody)))
|
||||
verifyAPKStored(t, storageDir, "apk", stub.packagePath, int64(len(stub.packageBody)))
|
||||
}
|
||||
|
||||
func verifyAPKStored(t *testing.T, basePath, hubName, locatorPath string, expectedSize int64) {
|
||||
t.Helper()
|
||||
clean := path.Clean("/" + locatorPath)
|
||||
clean = strings.TrimPrefix(clean, "/")
|
||||
fullPath := filepath.Join(basePath, hubName, clean)
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
t.Fatalf("expected cached file at %s: %v", fullPath, err)
|
||||
}
|
||||
if info.Size() != expectedSize {
|
||||
t.Fatalf("cached file %s size mismatch: got %d want %d", fullPath, info.Size(), expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
type apkStub struct {
|
||||
server *http.Server
|
||||
listener net.Listener
|
||||
URL string
|
||||
|
||||
mu sync.Mutex
|
||||
indexPath string
|
||||
signaturePath string
|
||||
packagePath string
|
||||
indexBody []byte
|
||||
signatureBody []byte
|
||||
packageBody []byte
|
||||
indexGets int
|
||||
indexHeads int
|
||||
signatureGets int
|
||||
signatureHeads int
|
||||
packageGets int
|
||||
packageHeads int
|
||||
}
|
||||
|
||||
func newAPKStub(t *testing.T) *apkStub {
|
||||
t.Helper()
|
||||
stub := &apkStub{
|
||||
indexPath: "/v3.19/main/x86_64/APKINDEX.tar.gz",
|
||||
signaturePath: "/v3.19/main/x86_64/APKINDEX.tar.gz.asc",
|
||||
packagePath: "/v3.22/community/x86_64/tini-static-0.19.0-r3.apk",
|
||||
indexBody: []byte("apk-index-body"),
|
||||
signatureBody: []byte("apk-index-signature"),
|
||||
packageBody: bytes.Repeat([]byte("apk-payload-"), 64*1024),
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(stub.indexPath, stub.handleIndex)
|
||||
mux.HandleFunc(stub.signaturePath, stub.handleSignature)
|
||||
mux.HandleFunc(stub.packagePath, stub.handlePackage)
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to start apk stub: %v", err)
|
||||
}
|
||||
|
||||
srv := &http.Server{Handler: mux}
|
||||
stub.server = srv
|
||||
stub.listener = listener
|
||||
stub.URL = "http://" + listener.Addr().String()
|
||||
|
||||
go func() {
|
||||
_ = srv.Serve(listener)
|
||||
}()
|
||||
return stub
|
||||
}
|
||||
|
||||
func (s *apkStub) handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||
s.handleWithETag(w, r, &s.indexGets, &s.indexHeads, s.indexBody, "application/gzip")
|
||||
}
|
||||
|
||||
func (s *apkStub) handleSignature(w http.ResponseWriter, r *http.Request) {
|
||||
s.handleWithETag(w, r, &s.signatureGets, &s.signatureHeads, s.signatureBody, "application/pgp-signature")
|
||||
}
|
||||
|
||||
func (s *apkStub) handlePackage(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if r.Method == http.MethodHead {
|
||||
s.packageHeads++
|
||||
w.Header().Set("Content-Type", "application/vnd.android.package-archive")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
s.packageGets++
|
||||
w.Header().Set("Content-Type", "application/vnd.android.package-archive")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(s.packageBody)))
|
||||
_, _ = w.Write(s.packageBody)
|
||||
}
|
||||
|
||||
func (s *apkStub) handleWithETag(w http.ResponseWriter, r *http.Request, gets, heads *int, body []byte, contentType string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
etag := "\"apk-etag\""
|
||||
if r.Method == http.MethodHead {
|
||||
*heads++
|
||||
w.Header().Set("ETag", etag)
|
||||
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
|
||||
if matchETag(r, strings.Trim(etag, `"`)) {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
if contentType != "" {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
*gets++
|
||||
w.Header().Set("ETag", etag)
|
||||
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
|
||||
if contentType != "" {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
func (s *apkStub) IndexGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.indexGets
|
||||
}
|
||||
|
||||
func (s *apkStub) IndexHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.indexHeads
|
||||
}
|
||||
|
||||
func (s *apkStub) SignatureGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.signatureGets
|
||||
}
|
||||
|
||||
func (s *apkStub) SignatureHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.signatureHeads
|
||||
}
|
||||
|
||||
func (s *apkStub) PackageGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.packageGets
|
||||
}
|
||||
|
||||
func (s *apkStub) PackageHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.packageHeads
|
||||
}
|
||||
|
||||
func (s *apkStub) Close() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if s.server != nil {
|
||||
_ = s.server.Shutdown(ctx)
|
||||
}
|
||||
if s.listener != nil {
|
||||
_ = s.listener.Close()
|
||||
}
|
||||
}
|
||||
277
tests/integration/apt_package_proxy_test.go
Normal file
277
tests/integration/apt_package_proxy_test.go
Normal file
@@ -0,0 +1,277 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/cache"
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/proxy"
|
||||
"github.com/any-hub/any-hub/internal/server"
|
||||
)
|
||||
|
||||
func TestAptPackagesCachedWithoutRevalidate(t *testing.T) {
|
||||
stub := newAptPackageStub(t)
|
||||
defer stub.Close()
|
||||
|
||||
storageDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Global: config.GlobalConfig{
|
||||
ListenPort: 5000,
|
||||
CacheTTL: config.Duration(time.Hour),
|
||||
StoragePath: storageDir,
|
||||
},
|
||||
Hubs: []config.HubConfig{
|
||||
{
|
||||
Name: "apt",
|
||||
Domain: "apt.hub.local",
|
||||
Type: "debian",
|
||||
Module: "debian",
|
||||
Upstream: stub.URL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
registry, err := server.NewHubRegistry(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("registry error: %v", err)
|
||||
}
|
||||
|
||||
logger := logrus.New()
|
||||
logger.SetOutput(io.Discard)
|
||||
|
||||
store, err := cache.NewStore(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("store error: %v", err)
|
||||
}
|
||||
|
||||
app, err := server.NewApp(server.AppOptions{
|
||||
Logger: logger,
|
||||
Registry: registry,
|
||||
Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store),
|
||||
ListenPort: 5000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("app error: %v", err)
|
||||
}
|
||||
|
||||
doRequest := func(p string) *http.Response {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://apt.hub.local"+p, nil)
|
||||
req.Host = "apt.hub.local"
|
||||
resp, err := app.Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("app.Test error: %v", err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
resp := doRequest(stub.packagePath)
|
||||
if resp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for package, got %d", resp.StatusCode)
|
||||
}
|
||||
if hit := resp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
|
||||
t.Fatalf("expected cache miss on first package fetch, got %s", hit)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !bytes.Equal(body, stub.packageBody) {
|
||||
t.Fatalf("package body mismatch on first fetch: %d bytes", len(body))
|
||||
}
|
||||
|
||||
resp2 := doRequest(stub.packagePath)
|
||||
if resp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached package, got %d", resp2.StatusCode)
|
||||
}
|
||||
if hit := resp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
|
||||
t.Fatalf("expected cache hit for package, got %s", hit)
|
||||
}
|
||||
body2, _ := io.ReadAll(resp2.Body)
|
||||
resp2.Body.Close()
|
||||
if !bytes.Equal(body2, stub.packageBody) {
|
||||
t.Fatalf("package body mismatch on cache hit: %d bytes", len(body2))
|
||||
}
|
||||
|
||||
hashResp := doRequest(stub.byHashPath)
|
||||
if hashResp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for by-hash, got %d", hashResp.StatusCode)
|
||||
}
|
||||
if hit := hashResp.Header.Get("X-Any-Hub-Cache-Hit"); hit != "false" {
|
||||
t.Fatalf("expected cache miss on first by-hash fetch, got %s", hit)
|
||||
}
|
||||
hashBody, _ := io.ReadAll(hashResp.Body)
|
||||
hashResp.Body.Close()
|
||||
if !bytes.Equal(hashBody, stub.byHashBody) {
|
||||
t.Fatalf("by-hash body mismatch on first fetch: %d bytes", len(hashBody))
|
||||
}
|
||||
|
||||
hashResp2 := doRequest(stub.byHashPath)
|
||||
if hashResp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached by-hash, got %d", hashResp2.StatusCode)
|
||||
}
|
||||
if hit := hashResp2.Header.Get("X-Any-Hub-Cache-Hit"); hit != "true" {
|
||||
t.Fatalf("expected cache hit for by-hash, got %s", hit)
|
||||
}
|
||||
hashBody2, _ := io.ReadAll(hashResp2.Body)
|
||||
hashResp2.Body.Close()
|
||||
if !bytes.Equal(hashBody2, stub.byHashBody) {
|
||||
t.Fatalf("by-hash body mismatch on cache hit: %d bytes", len(hashBody2))
|
||||
}
|
||||
|
||||
if stub.PackageGets() != 1 {
|
||||
t.Fatalf("expected single package GET, got %d", stub.PackageGets())
|
||||
}
|
||||
if stub.PackageHeads() != 0 {
|
||||
t.Fatalf("expected zero package HEAD revalidate, got %d", stub.PackageHeads())
|
||||
}
|
||||
if stub.ByHashGets() != 1 {
|
||||
t.Fatalf("expected single by-hash GET, got %d", stub.ByHashGets())
|
||||
}
|
||||
if stub.ByHashHeads() != 0 {
|
||||
t.Fatalf("expected zero by-hash HEAD revalidate, got %d", stub.ByHashHeads())
|
||||
}
|
||||
|
||||
verifyStoredFile(t, storageDir, "apt", stub.packagePath, int64(len(stub.packageBody)))
|
||||
verifyStoredFile(t, storageDir, "apt", stub.byHashPath, int64(len(stub.byHashBody)))
|
||||
}
|
||||
|
||||
func verifyStoredFile(t *testing.T, basePath, hubName, locatorPath string, expectedSize int64) {
|
||||
t.Helper()
|
||||
clean := path.Clean("/" + locatorPath)
|
||||
clean = strings.TrimPrefix(clean, "/")
|
||||
fullPath := filepath.Join(basePath, hubName, clean)
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
t.Fatalf("expected cached file at %s: %v", fullPath, err)
|
||||
}
|
||||
if info.Size() != expectedSize {
|
||||
t.Fatalf("cached file %s size mismatch: got %d want %d", fullPath, info.Size(), expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
type aptPackageStub struct {
|
||||
server *http.Server
|
||||
listener net.Listener
|
||||
URL string
|
||||
|
||||
mu sync.Mutex
|
||||
packagePath string
|
||||
byHashPath string
|
||||
packageBody []byte
|
||||
byHashBody []byte
|
||||
packageGets int
|
||||
packageHeads int
|
||||
byHashGets int
|
||||
byHashHeads int
|
||||
}
|
||||
|
||||
func newAptPackageStub(t *testing.T) *aptPackageStub {
|
||||
t.Helper()
|
||||
|
||||
stub := &aptPackageStub{
|
||||
packagePath: "/pool/main/h/hello_1.0_amd64.deb",
|
||||
byHashPath: "/dists/bookworm/by-hash/sha256/deadbeef",
|
||||
packageBody: bytes.Repeat([]byte("deb-payload-"), 128*1024),
|
||||
byHashBody: []byte("hash-index-body"),
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(stub.packagePath, stub.handlePackage)
|
||||
mux.HandleFunc(stub.byHashPath, stub.handleByHash)
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to start apt package stub: %v", err)
|
||||
}
|
||||
|
||||
srv := &http.Server{Handler: mux}
|
||||
stub.server = srv
|
||||
stub.listener = listener
|
||||
stub.URL = "http://" + listener.Addr().String()
|
||||
|
||||
go func() {
|
||||
_ = srv.Serve(listener)
|
||||
}()
|
||||
return stub
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) handlePackage(w http.ResponseWriter, r *http.Request) {
|
||||
s.handleImmutable(w, r, &s.packageGets, &s.packageHeads, s.packageBody, "application/vnd.debian.binary-package")
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) handleByHash(w http.ResponseWriter, r *http.Request) {
|
||||
s.handleImmutable(w, r, &s.byHashGets, &s.byHashHeads, s.byHashBody, "text/plain")
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) handleImmutable(w http.ResponseWriter, r *http.Request, gets, heads *int, body []byte, contentType string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if r.Method == http.MethodHead {
|
||||
*heads++
|
||||
if contentType != "" {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
*gets++
|
||||
if contentType != "" {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) PackageGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.packageGets
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) PackageHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.packageHeads
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) ByHashGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.byHashGets
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) ByHashHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.byHashHeads
|
||||
}
|
||||
|
||||
func (s *aptPackageStub) Close() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if s.server != nil {
|
||||
_ = s.server.Shutdown(ctx)
|
||||
}
|
||||
if s.listener != nil {
|
||||
_ = s.listener.Close()
|
||||
}
|
||||
}
|
||||
331
tests/integration/apt_update_proxy_test.go
Normal file
331
tests/integration/apt_update_proxy_test.go
Normal file
@@ -0,0 +1,331 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/cache"
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/proxy"
|
||||
"github.com/any-hub/any-hub/internal/server"
|
||||
)
|
||||
|
||||
func TestAptUpdateCachesIndexes(t *testing.T) {
|
||||
stub := newAptStub(t)
|
||||
defer stub.Close()
|
||||
|
||||
storageDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Global: config.GlobalConfig{
|
||||
ListenPort: 5000,
|
||||
CacheTTL: config.Duration(time.Hour),
|
||||
StoragePath: storageDir,
|
||||
},
|
||||
Hubs: []config.HubConfig{
|
||||
{
|
||||
Name: "apt",
|
||||
Domain: "apt.hub.local",
|
||||
Type: "debian",
|
||||
Module: "debian",
|
||||
Upstream: stub.URL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
registry, err := server.NewHubRegistry(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("registry error: %v", err)
|
||||
}
|
||||
|
||||
logger := logrus.New()
|
||||
logger.SetOutput(io.Discard)
|
||||
|
||||
store, err := cache.NewStore(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("store error: %v", err)
|
||||
}
|
||||
|
||||
app, err := server.NewApp(server.AppOptions{
|
||||
Logger: logger,
|
||||
Registry: registry,
|
||||
Proxy: proxy.NewHandler(server.NewUpstreamClient(cfg), logger, store),
|
||||
ListenPort: 5000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("app error: %v", err)
|
||||
}
|
||||
|
||||
doRequest := func(path string) *http.Response {
|
||||
req := httptest.NewRequest("GET", "http://apt.hub.local"+path, nil)
|
||||
req.Host = "apt.hub.local"
|
||||
resp, err := app.Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("app.Test error: %v", err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
releasePath := "/dists/bookworm/Release"
|
||||
packagesPath := "/dists/bookworm/main/binary-amd64/Packages.gz"
|
||||
contentsPath := "/dists/bookworm/main/Contents-amd64.gz"
|
||||
|
||||
resp := doRequest(releasePath)
|
||||
if resp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for release, got %d", resp.StatusCode)
|
||||
}
|
||||
if resp.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
|
||||
t.Fatalf("expected cache miss for first release fetch")
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
resp2 := doRequest(releasePath)
|
||||
if resp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached release, got %d", resp2.StatusCode)
|
||||
}
|
||||
if resp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
|
||||
t.Fatalf("expected cache hit for release")
|
||||
}
|
||||
resp2.Body.Close()
|
||||
|
||||
pkgResp := doRequest(packagesPath)
|
||||
if pkgResp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for packages, got %d", pkgResp.StatusCode)
|
||||
}
|
||||
if pkgResp.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
|
||||
t.Fatalf("expected cache miss for packages")
|
||||
}
|
||||
pkgResp.Body.Close()
|
||||
|
||||
pkgResp2 := doRequest(packagesPath)
|
||||
if pkgResp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached packages, got %d", pkgResp2.StatusCode)
|
||||
}
|
||||
if pkgResp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
|
||||
t.Fatalf("expected cache hit for packages")
|
||||
}
|
||||
pkgResp2.Body.Close()
|
||||
|
||||
contentsResp := doRequest(contentsPath)
|
||||
if contentsResp.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for contents, got %d", contentsResp.StatusCode)
|
||||
}
|
||||
if contentsResp.Header.Get("X-Any-Hub-Cache-Hit") != "false" {
|
||||
t.Fatalf("expected cache miss for contents")
|
||||
}
|
||||
contentsResp.Body.Close()
|
||||
|
||||
contentsResp2 := doRequest(contentsPath)
|
||||
if contentsResp2.StatusCode != fiber.StatusOK {
|
||||
t.Fatalf("expected 200 for cached contents, got %d", contentsResp2.StatusCode)
|
||||
}
|
||||
if contentsResp2.Header.Get("X-Any-Hub-Cache-Hit") != "true" {
|
||||
t.Fatalf("expected cache hit for contents")
|
||||
}
|
||||
contentsResp2.Body.Close()
|
||||
|
||||
if stub.ReleaseGets() != 1 {
|
||||
t.Fatalf("expected single release GET, got %d", stub.ReleaseGets())
|
||||
}
|
||||
if stub.ReleaseHeads() != 1 {
|
||||
t.Fatalf("expected single release HEAD revalidate, got %d", stub.ReleaseHeads())
|
||||
}
|
||||
if stub.PackagesGets() != 1 {
|
||||
t.Fatalf("expected single packages GET, got %d", stub.PackagesGets())
|
||||
}
|
||||
if stub.PackagesHeads() != 1 {
|
||||
t.Fatalf("expected single packages HEAD revalidate, got %d", stub.PackagesHeads())
|
||||
}
|
||||
if stub.ContentsGets() != 1 {
|
||||
t.Fatalf("expected single contents GET, got %d", stub.ContentsGets())
|
||||
}
|
||||
if stub.ContentsHeads() != 1 {
|
||||
t.Fatalf("expected single contents HEAD revalidate, got %d", stub.ContentsHeads())
|
||||
}
|
||||
}
|
||||
|
||||
type aptStub struct {
|
||||
server *http.Server
|
||||
listener net.Listener
|
||||
URL string
|
||||
mu sync.Mutex
|
||||
releaseBody string
|
||||
packagesBody string
|
||||
contentsBody string
|
||||
releaseETag string
|
||||
packagesETag string
|
||||
contentsETag string
|
||||
releaseGets int
|
||||
releaseHeads int
|
||||
packagesGets int
|
||||
packagesHeads int
|
||||
contentsGets int
|
||||
contentsHeads int
|
||||
releasePath string
|
||||
packagesPath string
|
||||
contentsPath string
|
||||
}
|
||||
|
||||
func newAptStub(t *testing.T) *aptStub {
|
||||
t.Helper()
|
||||
stub := &aptStub{
|
||||
releaseBody: "Release-body",
|
||||
packagesBody: "Packages-body",
|
||||
contentsBody: "Contents-body",
|
||||
releaseETag: "r1",
|
||||
packagesETag: "p1",
|
||||
contentsETag: "c1",
|
||||
releasePath: "/dists/bookworm/Release",
|
||||
packagesPath: "/dists/bookworm/main/binary-amd64/Packages.gz",
|
||||
contentsPath: "/dists/bookworm/main/Contents-amd64.gz",
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(stub.releasePath, stub.handleRelease)
|
||||
mux.HandleFunc(stub.packagesPath, stub.handlePackages)
|
||||
mux.HandleFunc(stub.contentsPath, stub.handleContents)
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to start apt stub: %v", err)
|
||||
}
|
||||
|
||||
srv := &http.Server{Handler: mux}
|
||||
stub.server = srv
|
||||
stub.listener = listener
|
||||
stub.URL = "http://" + listener.Addr().String()
|
||||
|
||||
go func() {
|
||||
_ = srv.Serve(listener)
|
||||
}()
|
||||
return stub
|
||||
}
|
||||
|
||||
func (s *aptStub) handleRelease(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if r.Method == http.MethodHead {
|
||||
s.releaseHeads++
|
||||
if matchETag(r, s.releaseETag) {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
writeHeaders(w, s.releaseETag)
|
||||
return
|
||||
}
|
||||
s.releaseGets++
|
||||
writeHeaders(w, s.releaseETag)
|
||||
_, _ = w.Write([]byte(s.releaseBody))
|
||||
}
|
||||
|
||||
func (s *aptStub) handlePackages(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if r.Method == http.MethodHead {
|
||||
s.packagesHeads++
|
||||
if matchETag(r, s.packagesETag) {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
writeHeaders(w, s.packagesETag)
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
return
|
||||
}
|
||||
s.packagesGets++
|
||||
writeHeaders(w, s.packagesETag)
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
_, _ = w.Write([]byte(s.packagesBody))
|
||||
}
|
||||
|
||||
func (s *aptStub) handleContents(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if r.Method == http.MethodHead {
|
||||
s.contentsHeads++
|
||||
if matchETag(r, s.contentsETag) {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
writeHeaders(w, s.contentsETag)
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
return
|
||||
}
|
||||
s.contentsGets++
|
||||
writeHeaders(w, s.contentsETag)
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
_, _ = w.Write([]byte(s.contentsBody))
|
||||
}
|
||||
|
||||
func matchETag(r *http.Request, etag string) bool {
|
||||
for _, candidate := range r.Header.Values("If-None-Match") {
|
||||
c := strings.Trim(candidate, "\"")
|
||||
if c == etag || candidate == etag {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func writeHeaders(w http.ResponseWriter, etag string) {
|
||||
w.Header().Set("ETag", "\""+etag+"\"")
|
||||
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
func (s *aptStub) ReleaseGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.releaseGets
|
||||
}
|
||||
|
||||
func (s *aptStub) ReleaseHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.releaseHeads
|
||||
}
|
||||
|
||||
func (s *aptStub) PackagesGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.packagesGets
|
||||
}
|
||||
|
||||
func (s *aptStub) PackagesHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.packagesHeads
|
||||
}
|
||||
|
||||
func (s *aptStub) ContentsGets() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.contentsGets
|
||||
}
|
||||
|
||||
func (s *aptStub) ContentsHeads() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.contentsHeads
|
||||
}
|
||||
|
||||
func (s *aptStub) Close() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if s.server != nil {
|
||||
_ = s.server.Shutdown(ctx)
|
||||
}
|
||||
if s.listener != nil {
|
||||
_ = s.listener.Close()
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,14 @@ package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -42,6 +44,7 @@ func TestCacheFlowWithConditionalRequest(t *testing.T) {
|
||||
Name: "docker",
|
||||
Domain: "docker.hub.local",
|
||||
Type: "docker",
|
||||
Module: "docker",
|
||||
Upstream: upstream.URL,
|
||||
},
|
||||
},
|
||||
@@ -143,6 +146,7 @@ func TestDockerManifestHeadDoesNotOverwriteCache(t *testing.T) {
|
||||
Name: "docker",
|
||||
Domain: "docker.hub.local",
|
||||
Type: "docker",
|
||||
Module: "docker",
|
||||
Upstream: upstream.URL,
|
||||
},
|
||||
},
|
||||
@@ -294,6 +298,7 @@ type cacheFlowStub struct {
|
||||
lastRequest *http.Request
|
||||
body []byte
|
||||
etag string
|
||||
etagVer int
|
||||
lastMod string
|
||||
}
|
||||
|
||||
@@ -302,6 +307,7 @@ func newCacheFlowStub(t *testing.T, paths ...string) *cacheFlowStub {
|
||||
stub := &cacheFlowStub{
|
||||
body: []byte("upstream payload"),
|
||||
etag: `"etag-v1"`,
|
||||
etagVer: 1,
|
||||
lastMod: time.Now().UTC().Format(http.TimeFormat),
|
||||
}
|
||||
|
||||
@@ -344,6 +350,8 @@ func (s *cacheFlowStub) Close() {
|
||||
|
||||
func (s *cacheFlowStub) handle(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
etag := s.etag
|
||||
lastMod := s.lastMod
|
||||
if r.Method == http.MethodHead {
|
||||
s.headHits++
|
||||
} else {
|
||||
@@ -354,15 +362,21 @@ func (s *cacheFlowStub) handle(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if r.Method == http.MethodHead {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Etag", s.etag)
|
||||
w.Header().Set("Last-Modified", s.lastMod)
|
||||
w.Header().Set("Etag", etag)
|
||||
w.Header().Set("Last-Modified", lastMod)
|
||||
for _, candidate := range r.Header.Values("If-None-Match") {
|
||||
if strings.Trim(candidate, `"`) == strings.Trim(etag, `"`) {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Header().Set("Etag", s.etag)
|
||||
w.Header().Set("Last-Modified", s.lastMod)
|
||||
w.Header().Set("Etag", etag)
|
||||
w.Header().Set("Last-Modified", lastMod)
|
||||
_, _ = w.Write(s.body)
|
||||
}
|
||||
|
||||
@@ -370,5 +384,7 @@ func (s *cacheFlowStub) UpdateBody(body []byte) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.body = body
|
||||
s.lastMod = time.Now().UTC().Format(http.TimeFormat)
|
||||
s.etagVer++
|
||||
s.etag = fmt.Sprintf(`"etag-v%d"`, s.etagVer)
|
||||
s.lastMod = time.Now().UTC().Add(2 * time.Second).Format(http.TimeFormat)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
"github.com/any-hub/any-hub/internal/proxy/hooks"
|
||||
"github.com/any-hub/any-hub/internal/server"
|
||||
"github.com/any-hub/any-hub/internal/server/routes"
|
||||
@@ -41,6 +42,8 @@ func TestModuleDiagnosticsEndpoints(t *testing.T) {
|
||||
Domain: "legacy.local",
|
||||
Type: "docker",
|
||||
Upstream: "https://registry-1.docker.io",
|
||||
Module: hubmodule.DefaultModuleKey(),
|
||||
Rollout: string(legacy.RolloutLegacyOnly),
|
||||
},
|
||||
{
|
||||
Name: "modern-hub",
|
||||
|
||||
@@ -239,7 +239,7 @@ func (s *pypiStub) handleWheel(w http.ResponseWriter, r *http.Request) {
|
||||
func (s *pypiStub) UpdateSimple(body []byte) {
|
||||
s.mu.Lock()
|
||||
s.simpleBody = append([]byte(nil), body...)
|
||||
s.lastSimpleMod = time.Now().UTC().Format(http.TimeFormat)
|
||||
s.lastSimpleMod = time.Now().UTC().Add(2 * time.Second).Format(http.TimeFormat)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user