stage 1
This commit is contained in:
13
internal/cache/doc.go
vendored
13
internal/cache/doc.go
vendored
@@ -1,7 +1,10 @@
|
||||
// Package cache defines the disk-backed store responsible for translating hub
|
||||
// requests into StoragePath/<hub>/<path>.body files. The store exposes read/write
|
||||
// primitives with safe semantics (temp file + rename) and surfaces file info
|
||||
// (size, modtime) for higher layers to implement conditional revalidation.
|
||||
// Proxy handlers depend on this package to stream cached responses or trigger
|
||||
// upstream fetches without duplicating filesystem logic.
|
||||
// requests into StoragePath/<hub>/<path> directories that mirror upstream
|
||||
// paths. When a given path also needs to act as the parent of other entries
|
||||
// (例如 npm metadata + tarball目录), the body is stored in a `__content` file
|
||||
// under that directory so两种形态可以共存。The store exposes read/write primitives
|
||||
// with safe semantics (temp file + rename) and surfaces file info (size, modtime)
|
||||
// for higher layers to implement conditional revalidation. Proxy handlers depend
|
||||
// on this package to stream cached responses or trigger upstream fetches without
|
||||
// duplicating filesystem logic.
|
||||
package cache
|
||||
|
||||
157
internal/cache/fs_store.go
vendored
157
internal/cache/fs_store.go
vendored
@@ -15,8 +15,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const cacheFileSuffix = ".body"
|
||||
|
||||
// NewStore 以 basePath 为根目录构建磁盘缓存,整站复用一份实例。
|
||||
func NewStore(basePath string) (Store, error) {
|
||||
if basePath == "" {
|
||||
@@ -58,13 +56,27 @@ func (s *fileStore) Get(ctx context.Context, locator Locator) (*ReadResult, erro
|
||||
default:
|
||||
}
|
||||
|
||||
primary, legacy, err := s.entryPaths(locator)
|
||||
filePath, err := s.entryPath(locator)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filePath, info, f, err := s.openEntryFile(primary, legacy)
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) || isNotDirError(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) || isNotDirError(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -74,11 +86,7 @@ func (s *fileStore) Get(ctx context.Context, locator Locator) (*ReadResult, erro
|
||||
SizeBytes: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
|
||||
return &ReadResult{
|
||||
Entry: entry,
|
||||
Reader: f,
|
||||
}, nil
|
||||
return &ReadResult{Entry: entry, Reader: file}, nil
|
||||
}
|
||||
|
||||
func (s *fileStore) Put(ctx context.Context, locator Locator, body io.Reader, opts PutOptions) (*Entry, error) {
|
||||
@@ -88,12 +96,12 @@ func (s *fileStore) Put(ctx context.Context, locator Locator, body io.Reader, op
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
filePath, legacyPath, err := s.entryPaths(locator)
|
||||
filePath, err := s.entryPath(locator)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := s.ensureDirWithUpgrade(filepath.Dir(filePath)); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -109,12 +117,12 @@ func (s *fileStore) Put(ctx context.Context, locator Locator, body io.Reader, op
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(tempName)
|
||||
_ = os.Remove(tempName)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := os.Rename(tempName, filePath); err != nil {
|
||||
os.Remove(tempName)
|
||||
_ = os.Remove(tempName)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -125,7 +133,6 @@ func (s *fileStore) Put(ctx context.Context, locator Locator, body io.Reader, op
|
||||
if err := os.Chtimes(filePath, modTime, modTime); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = os.Remove(legacyPath)
|
||||
|
||||
entry := Entry{
|
||||
Locator: locator,
|
||||
@@ -143,16 +150,13 @@ func (s *fileStore) Remove(ctx context.Context, locator Locator) error {
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
filePath, legacyPath, err := s.entryPaths(locator)
|
||||
filePath, err := s.entryPath(locator)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(filePath); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(legacyPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -179,7 +183,7 @@ func (s *fileStore) lockEntry(locator Locator) (func(), error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *fileStore) path(locator Locator) (string, error) {
|
||||
func (s *fileStore) entryPath(locator Locator) (string, error) {
|
||||
if locator.HubName == "" {
|
||||
return "", errors.New("hub name required")
|
||||
}
|
||||
@@ -203,121 +207,6 @@ func (s *fileStore) path(locator Locator) (string, error) {
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
func (s *fileStore) entryPaths(locator Locator) (string, string, error) {
|
||||
legacyPath, err := s.path(locator)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return legacyPath + cacheFileSuffix, legacyPath, nil
|
||||
}
|
||||
|
||||
func (s *fileStore) openEntryFile(primaryPath, legacyPath string) (string, fs.FileInfo, *os.File, error) {
|
||||
info, err := os.Stat(primaryPath)
|
||||
if err == nil {
|
||||
if info.IsDir() {
|
||||
return "", nil, nil, ErrNotFound
|
||||
}
|
||||
f, err := os.Open(primaryPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) || isNotDirError(err) {
|
||||
return "", nil, nil, ErrNotFound
|
||||
}
|
||||
return "", nil, nil, err
|
||||
}
|
||||
return primaryPath, info, f, nil
|
||||
}
|
||||
if !errors.Is(err, fs.ErrNotExist) && !isNotDirError(err) {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
info, err = os.Stat(legacyPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) || isNotDirError(err) {
|
||||
return "", nil, nil, ErrNotFound
|
||||
}
|
||||
return "", nil, nil, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return "", nil, nil, ErrNotFound
|
||||
}
|
||||
|
||||
if migrateErr := s.migrateLegacyFile(primaryPath, legacyPath); migrateErr == nil {
|
||||
return s.openEntryFile(primaryPath, legacyPath)
|
||||
}
|
||||
|
||||
f, err := os.Open(legacyPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) || isNotDirError(err) {
|
||||
return "", nil, nil, ErrNotFound
|
||||
}
|
||||
return "", nil, nil, err
|
||||
}
|
||||
return legacyPath, info, f, nil
|
||||
}
|
||||
|
||||
func (s *fileStore) migrateLegacyFile(primaryPath, legacyPath string) error {
|
||||
if legacyPath == "" || primaryPath == legacyPath {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(legacyPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := os.Stat(primaryPath); err == nil {
|
||||
if removeErr := os.Remove(legacyPath); removeErr != nil && !errors.Is(removeErr, fs.ErrNotExist) {
|
||||
return removeErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return os.Rename(legacyPath, primaryPath)
|
||||
}
|
||||
|
||||
func (s *fileStore) ensureDirWithUpgrade(dir string) error {
|
||||
for i := 0; i < 8; i++ {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
if isNotDirError(err) {
|
||||
var pathErr *os.PathError
|
||||
if errors.As(err, &pathErr) {
|
||||
if upgradeErr := s.upgradeLegacyNode(pathErr.Path); upgradeErr != nil {
|
||||
return upgradeErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("ensure cache directory failed for %s", dir)
|
||||
}
|
||||
|
||||
func (s *fileStore) upgradeLegacyNode(conflictPath string) error {
|
||||
if conflictPath == "" {
|
||||
return errors.New("empty conflict path")
|
||||
}
|
||||
rel, err := filepath.Rel(s.basePath, conflictPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(rel, "..") {
|
||||
return fmt.Errorf("conflict path outside storage: %s", conflictPath)
|
||||
}
|
||||
info, err := os.Stat(conflictPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(conflictPath, cacheFileSuffix) {
|
||||
return nil
|
||||
}
|
||||
newPath := conflictPath + cacheFileSuffix
|
||||
if _, err := os.Stat(newPath); err == nil {
|
||||
return os.Remove(conflictPath)
|
||||
}
|
||||
return os.Rename(conflictPath, newPath)
|
||||
}
|
||||
|
||||
func isNotDirError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
|
||||
2
internal/cache/store.go
vendored
2
internal/cache/store.go
vendored
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
// Store 负责管理磁盘缓存的读写。磁盘布局遵循:
|
||||
//
|
||||
// <StoragePath>/<HubName>/<path>.body # 实际正文
|
||||
// <StoragePath>/<HubName>/<path> # 实际正文(与请求路径一致)
|
||||
//
|
||||
// 每个条目仅由正文文件组成,文件的 ModTime/Size 由文件系统提供。
|
||||
type Store interface {
|
||||
|
||||
85
internal/cache/store_test.go
vendored
85
internal/cache/store_test.go
vendored
@@ -7,8 +7,6 @@ import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -42,9 +40,6 @@ func TestStorePutAndGet(t *testing.T) {
|
||||
if !result.Entry.ModTime.Equal(modTime) {
|
||||
t.Fatalf("modtime mismatch: expected %v got %v", modTime, result.Entry.ModTime)
|
||||
}
|
||||
if !strings.HasSuffix(result.Entry.FilePath, cacheFileSuffix) {
|
||||
t.Fatalf("expected cache file suffix %s, got %s", cacheFileSuffix, result.Entry.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreGetMissing(t *testing.T) {
|
||||
@@ -78,11 +73,11 @@ func TestStoreIgnoresDirectories(t *testing.T) {
|
||||
t.Fatalf("unexpected store type %T", store)
|
||||
}
|
||||
|
||||
filePath, err := fs.path(locator)
|
||||
filePath, err := fs.entryPath(locator)
|
||||
if err != nil {
|
||||
t.Fatalf("path error: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filePath+cacheFileSuffix, 0o755); err != nil {
|
||||
if err := os.MkdirAll(filePath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir error: %v", err)
|
||||
}
|
||||
|
||||
@@ -91,82 +86,6 @@ func TestStoreIgnoresDirectories(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreMigratesLegacyEntryOnGet(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
fs, ok := store.(*fileStore)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected store type %T", store)
|
||||
}
|
||||
locator := Locator{HubName: "npm", Path: "/pkg"}
|
||||
legacyPath, err := fs.path(locator)
|
||||
if err != nil {
|
||||
t.Fatalf("path error: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir error: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(legacyPath, []byte("legacy"), 0o644); err != nil {
|
||||
t.Fatalf("write legacy error: %v", err)
|
||||
}
|
||||
|
||||
result, err := store.Get(context.Background(), locator)
|
||||
if err != nil {
|
||||
t.Fatalf("get legacy error: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(result.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("read legacy error: %v", err)
|
||||
}
|
||||
result.Reader.Close()
|
||||
if string(body) != "legacy" {
|
||||
t.Fatalf("unexpected legacy body: %s", string(body))
|
||||
}
|
||||
if !strings.HasSuffix(result.Entry.FilePath, cacheFileSuffix) {
|
||||
t.Fatalf("expected migrated file suffix, got %s", result.Entry.FilePath)
|
||||
}
|
||||
if _, statErr := os.Stat(legacyPath); !errors.Is(statErr, fs.ErrNotExist) {
|
||||
t.Fatalf("expected legacy path removed, got %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreHandlesAncestorFileConflict(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
fs, ok := store.(*fileStore)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected store type %T", store)
|
||||
}
|
||||
metaLocator := Locator{HubName: "npm", Path: "/pkg"}
|
||||
legacyPath, err := fs.path(metaLocator)
|
||||
if err != nil {
|
||||
t.Fatalf("path error: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir error: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(legacyPath, []byte("legacy"), 0o644); err != nil {
|
||||
t.Fatalf("write legacy error: %v", err)
|
||||
}
|
||||
|
||||
tarLocator := Locator{HubName: "npm", Path: "/pkg/-/pkg-1.0.0.tgz"}
|
||||
if _, err := store.Put(context.Background(), tarLocator, bytes.NewReader([]byte("tar")), PutOptions{}); err != nil {
|
||||
t.Fatalf("put tar error: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(legacyPath); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("expected legacy metadata renamed, got %v", err)
|
||||
}
|
||||
if _, err := os.Stat(legacyPath + cacheFileSuffix); err != nil {
|
||||
t.Fatalf("expected migrated legacy cache, got %v", err)
|
||||
}
|
||||
primary, _, err := fs.entryPaths(tarLocator)
|
||||
if err != nil {
|
||||
t.Fatalf("entry path error: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(primary); err != nil {
|
||||
t.Fatalf("expected tar cache file, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// newTestStore returns a Store backed by a temporary directory.
|
||||
func newTestStore(t *testing.T) Store {
|
||||
t.Helper()
|
||||
|
||||
57
internal/cache/writer.go
vendored
Normal file
57
internal/cache/writer.go
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
// ErrStoreUnavailable 表示当前模块未注入缓存存储实例。
|
||||
var ErrStoreUnavailable = errors.New("cache store unavailable")
|
||||
|
||||
// StrategyWriter 注入模块的缓存策略,提供 TTL 决策与写入封装。
|
||||
type StrategyWriter struct {
|
||||
store Store
|
||||
strategy hubmodule.CacheStrategyProfile
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NewStrategyWriter 构造策略感知的写入器,默认使用 time.Now 作为时钟。
|
||||
func NewStrategyWriter(store Store, strategy hubmodule.CacheStrategyProfile) StrategyWriter {
|
||||
return StrategyWriter{
|
||||
store: store,
|
||||
strategy: strategy,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// Enabled 返回当前是否具备缓存写入能力。
|
||||
func (w StrategyWriter) Enabled() bool {
|
||||
return w.store != nil
|
||||
}
|
||||
|
||||
// Put 写入缓存正文,并保持与 Store 相同的语义。
|
||||
func (w StrategyWriter) Put(ctx context.Context, locator Locator, body io.Reader, opts PutOptions) (*Entry, error) {
|
||||
if w.store == nil {
|
||||
return nil, ErrStoreUnavailable
|
||||
}
|
||||
return w.store.Put(ctx, locator, body, opts)
|
||||
}
|
||||
|
||||
// ShouldBypassValidation 根据策略 TTL 判断是否可以直接复用缓存,避免重复 HEAD。
|
||||
func (w StrategyWriter) ShouldBypassValidation(entry Entry) bool {
|
||||
ttl := w.strategy.TTLHint
|
||||
if ttl <= 0 {
|
||||
return false
|
||||
}
|
||||
expireAt := entry.ModTime.Add(ttl)
|
||||
return w.now().Before(expireAt)
|
||||
}
|
||||
|
||||
// SupportsValidation 返回当前策略是否允许通过 HEAD/Etag 等方式再验证。
|
||||
func (w StrategyWriter) SupportsValidation() bool {
|
||||
return w.strategy.ValidationMode != hubmodule.ValidationModeNever
|
||||
}
|
||||
@@ -94,6 +94,9 @@ func applyHubDefaults(h *HubConfig) {
|
||||
} else {
|
||||
h.Module = strings.ToLower(trimmed)
|
||||
}
|
||||
if rollout := strings.TrimSpace(h.Rollout); rollout != "" {
|
||||
h.Rollout = strings.ToLower(rollout)
|
||||
}
|
||||
if h.ValidationMode == "" {
|
||||
h.ValidationMode = string(hubmodule.ValidationModeETag)
|
||||
}
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
package config
|
||||
|
||||
import _ "github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
import (
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/docker"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/npm"
|
||||
_ "github.com/any-hub/any-hub/internal/hubmodule/pypi"
|
||||
)
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
)
|
||||
|
||||
// HubRuntime 将 Hub 配置与模块元数据合并,方便运行时快速取用策略。
|
||||
@@ -9,17 +12,16 @@ type HubRuntime struct {
|
||||
Config HubConfig
|
||||
Module hubmodule.ModuleMetadata
|
||||
CacheStrategy hubmodule.CacheStrategyProfile
|
||||
Rollout legacy.RolloutFlag
|
||||
}
|
||||
|
||||
// BuildHubRuntime 根据 Hub 配置和模块元数据创建运行时描述。
|
||||
func BuildHubRuntime(cfg HubConfig, meta hubmodule.ModuleMetadata) HubRuntime {
|
||||
strategy := hubmodule.ResolveStrategy(meta, hubmodule.StrategyOptions{
|
||||
TTLOverride: cfg.CacheTTL.DurationValue(),
|
||||
ValidationOverride: hubmodule.ValidationMode(cfg.ValidationMode),
|
||||
})
|
||||
// BuildHubRuntime 根据 Hub 配置和模块元数据创建运行时描述,应用最终 TTL 覆盖。
|
||||
func BuildHubRuntime(cfg HubConfig, meta hubmodule.ModuleMetadata, ttl time.Duration, flag legacy.RolloutFlag) HubRuntime {
|
||||
strategy := hubmodule.ResolveStrategy(meta, cfg.StrategyOverrides(ttl))
|
||||
return HubRuntime{
|
||||
Config: cfg,
|
||||
Module: meta,
|
||||
CacheStrategy: strategy,
|
||||
Rollout: flag,
|
||||
}
|
||||
}
|
||||
|
||||
62
internal/config/runtime_flags.go
Normal file
62
internal/config/runtime_flags.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
)
|
||||
|
||||
// parseRolloutFlag 将配置中的 rollout 字段标准化,并结合模块类型输出最终状态。
|
||||
func parseRolloutFlag(raw string, moduleKey string) (legacy.RolloutFlag, error) {
|
||||
normalized := strings.ToLower(strings.TrimSpace(raw))
|
||||
if normalized == "" {
|
||||
return defaultRolloutFlag(moduleKey), nil
|
||||
}
|
||||
|
||||
switch normalized {
|
||||
case string(legacy.RolloutLegacyOnly):
|
||||
return legacy.RolloutLegacyOnly, nil
|
||||
case string(legacy.RolloutDual):
|
||||
if moduleKey == hubmodule.DefaultModuleKey() {
|
||||
return legacy.RolloutLegacyOnly, nil
|
||||
}
|
||||
return legacy.RolloutDual, nil
|
||||
case string(legacy.RolloutModular):
|
||||
if moduleKey == hubmodule.DefaultModuleKey() {
|
||||
return legacy.RolloutLegacyOnly, nil
|
||||
}
|
||||
return legacy.RolloutModular, nil
|
||||
default:
|
||||
return "", fmt.Errorf("不支持的 rollout 值: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRolloutFlag(moduleKey string) legacy.RolloutFlag {
|
||||
if strings.TrimSpace(moduleKey) == "" || moduleKey == hubmodule.DefaultModuleKey() {
|
||||
return legacy.RolloutLegacyOnly
|
||||
}
|
||||
return legacy.RolloutModular
|
||||
}
|
||||
|
||||
// EffectiveModuleKey 根据 rollout 状态计算真实运行的模块。
|
||||
func EffectiveModuleKey(moduleKey string, flag legacy.RolloutFlag) string {
|
||||
if flag == legacy.RolloutLegacyOnly {
|
||||
return hubmodule.DefaultModuleKey()
|
||||
}
|
||||
normalized := strings.ToLower(strings.TrimSpace(moduleKey))
|
||||
if normalized == "" {
|
||||
return hubmodule.DefaultModuleKey()
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
// RolloutFlagValue 返回当前 Hub 的 rollout flag(假定 Validate 已经通过)。
|
||||
func (h HubConfig) RolloutFlagValue() legacy.RolloutFlag {
|
||||
flag := legacy.RolloutFlag(strings.ToLower(strings.TrimSpace(h.Rollout)))
|
||||
if flag == "" {
|
||||
return defaultRolloutFlag(h.Module)
|
||||
}
|
||||
return flag
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
// Duration 提供更灵活的反序列化能力,同时兼容纯秒整数与 Go Duration 字符串。
|
||||
@@ -68,6 +70,7 @@ type HubConfig struct {
|
||||
Proxy string `mapstructure:"Proxy"`
|
||||
Type string `mapstructure:"Type"`
|
||||
Module string `mapstructure:"Module"`
|
||||
Rollout string `mapstructure:"Rollout"`
|
||||
Username string `mapstructure:"Username"`
|
||||
Password string `mapstructure:"Password"`
|
||||
CacheTTL Duration `mapstructure:"CacheTTL"`
|
||||
@@ -105,3 +108,14 @@ func CredentialModes(hubs []HubConfig) []string {
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// StrategyOverrides 将 hub 层的 TTL/Validation 配置映射为模块策略覆盖项。
|
||||
func (h HubConfig) StrategyOverrides(ttl time.Duration) hubmodule.StrategyOptions {
|
||||
opts := hubmodule.StrategyOptions{
|
||||
TTLOverride: ttl,
|
||||
}
|
||||
if mode := strings.TrimSpace(h.ValidationMode); mode != "" {
|
||||
opts.ValidationOverride = hubmodule.ValidationMode(mode)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
@@ -84,6 +84,11 @@ func (c *Config) Validate() error {
|
||||
return newFieldError(hubField(hub.Name, "Module"), fmt.Sprintf("未注册模块: %s", moduleKey))
|
||||
}
|
||||
hub.Module = moduleKey
|
||||
flag, err := parseRolloutFlag(hub.Rollout, hub.Module)
|
||||
if err != nil {
|
||||
return newFieldError(hubField(hub.Name, "Rollout"), err.Error())
|
||||
}
|
||||
hub.Rollout = string(flag)
|
||||
if hub.ValidationMode != "" {
|
||||
mode := strings.ToLower(strings.TrimSpace(hub.ValidationMode))
|
||||
switch mode {
|
||||
|
||||
@@ -15,7 +15,7 @@ internal/hubmodule/
|
||||
## 模块约束
|
||||
- **单一接口**:每个模块需要同时实现代理与缓存接口,避免跨包耦合。
|
||||
- **注册流程**:在模块 `init()` 中调用 `hubmodule.Register(ModuleMetadata{...})`,注册失败必须 panic 以阻止启动。
|
||||
- **缓存布局**:一律使用 `StoragePath/<Hub>/<path>.body`,如需附加目录需在 `ModuleMetadata` 中声明迁移策略。
|
||||
- **缓存布局**:一律使用 `StoragePath/<Hub>/<path>`,即与上游请求完全一致的磁盘路径;当某个路径既要保存正文又要作为子目录父节点时,会在该目录下写入 `__content` 文件以存放正文。
|
||||
- **配置注入**:模块仅通过依赖注入获取 `HubConfigEntry` 和全局参数,禁止直接读取文件或环境变量。
|
||||
- **可观测性**:所有模块必须输出 `module_key`、命中/回源状态等日志字段,并在返回错误时附带 Hub 名称。
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// 模块作者需要:
|
||||
// 1. 在 internal/hubmodule/<module-key>/ 目录下实现代理与缓存接口;
|
||||
// 2. 通过本包暴露的 Register 函数在 init() 中注册模块元数据;
|
||||
// 3. 保证缓存写入仍遵循 StoragePath/<Hub>/<path>.body 布局,并补充中文注释说明实现细节。
|
||||
// 3. 保证缓存写入仍遵循 StoragePath/<Hub>/<path> 原始路径布局,并补充中文注释说明实现细节。
|
||||
//
|
||||
// 该包同时负责提供模块发现、可观测信息以及迁移状态的对外查询能力。
|
||||
package hubmodule
|
||||
|
||||
29
internal/hubmodule/docker/module.go
Normal file
29
internal/hubmodule/docker/module.go
Normal file
@@ -0,0 +1,29 @@
|
||||
// Package docker 定义 Docker Hub 代理模块的元数据与缓存策略描述,供 registry 查表时使用。
|
||||
package docker
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
const dockerDefaultTTL = 12 * time.Hour
|
||||
|
||||
// docker 模块继承 legacy 行为,但声明明确的缓存策略默认值,便于 hub 覆盖。
|
||||
func init() {
|
||||
hubmodule.MustRegister(hubmodule.ModuleMetadata{
|
||||
Key: "docker",
|
||||
Description: "Docker registry module with manifest/blob cache policies",
|
||||
MigrationState: hubmodule.MigrationStateBeta,
|
||||
SupportedProtocols: []string{
|
||||
"docker",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: dockerDefaultTTL,
|
||||
ValidationMode: hubmodule.ValidationModeETag,
|
||||
DiskLayout: "raw_path",
|
||||
RequiresMetadataFile: false,
|
||||
SupportsStreamingWrite: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
package hubmodule
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/cache"
|
||||
)
|
||||
|
||||
// MigrationState 描述模块上线阶段,方便观测端区分 legacy/beta/ga。
|
||||
type MigrationState string
|
||||
@@ -36,9 +40,13 @@ type ModuleMetadata struct {
|
||||
MigrationState MigrationState
|
||||
SupportedProtocols []string
|
||||
CacheStrategy CacheStrategyProfile
|
||||
LocatorRewrite LocatorRewrite
|
||||
}
|
||||
|
||||
// DefaultModuleKey 返回内置 legacy 模块的键值。
|
||||
func DefaultModuleKey() string {
|
||||
return defaultModuleKey
|
||||
}
|
||||
|
||||
// LocatorRewrite 允许模块根据自身协议调整缓存路径,例如将 npm metadata 写入独立文件。
|
||||
type LocatorRewrite func(cache.Locator) cache.Locator
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// Package legacy 提供旧版共享代理+缓存实现的适配器,确保未迁移 Hub 可继续运行。
|
||||
package legacy
|
||||
|
||||
import "github.com/any-hub/any-hub/internal/hubmodule"
|
||||
@@ -12,7 +13,7 @@ func init() {
|
||||
"docker", "npm", "go", "pypi",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
DiskLayout: ".body",
|
||||
DiskLayout: "raw_path",
|
||||
ValidationMode: hubmodule.ValidationModeETag,
|
||||
SupportsStreamingWrite: true,
|
||||
},
|
||||
|
||||
65
internal/hubmodule/legacy/state.go
Normal file
65
internal/hubmodule/legacy/state.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// RolloutFlag 描述 legacy 模块迁移阶段。
|
||||
type RolloutFlag string
|
||||
|
||||
const (
|
||||
RolloutLegacyOnly RolloutFlag = "legacy-only"
|
||||
RolloutDual RolloutFlag = "dual"
|
||||
RolloutModular RolloutFlag = "modular"
|
||||
)
|
||||
|
||||
// AdapterState 记录特定 Hub 在 legacy 适配器中的运行状态。
|
||||
type AdapterState struct {
|
||||
HubName string
|
||||
ModuleKey string
|
||||
Rollout RolloutFlag
|
||||
}
|
||||
|
||||
var (
|
||||
stateMu sync.RWMutex
|
||||
state = make(map[string]AdapterState)
|
||||
)
|
||||
|
||||
// RecordAdapterState 更新指定 Hub 的 rollout 状态,供诊断端和日志使用。
|
||||
func RecordAdapterState(hubName, moduleKey string, flag RolloutFlag) {
|
||||
if hubName == "" {
|
||||
return
|
||||
}
|
||||
key := strings.ToLower(hubName)
|
||||
stateMu.Lock()
|
||||
state[key] = AdapterState{
|
||||
HubName: hubName,
|
||||
ModuleKey: moduleKey,
|
||||
Rollout: flag,
|
||||
}
|
||||
stateMu.Unlock()
|
||||
}
|
||||
|
||||
// SnapshotAdapterStates 返回所有 Hub 的 rollout 状态,按名称排序。
|
||||
func SnapshotAdapterStates() []AdapterState {
|
||||
stateMu.RLock()
|
||||
defer stateMu.RUnlock()
|
||||
|
||||
if len(state) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(state))
|
||||
for k := range state {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
result := make([]AdapterState, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
result = append(result, state[key])
|
||||
}
|
||||
return result
|
||||
}
|
||||
41
internal/hubmodule/npm/locator.go
Normal file
41
internal/hubmodule/npm/locator.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/cache"
|
||||
)
|
||||
|
||||
// rewriteLocator 将 npm metadata JSON 落盘至 package.json,避免与 tarball
|
||||
// 路径的 `/-/` 子目录冲突,同时保持 tarball 使用原始路径。
|
||||
func rewriteLocator(loc cache.Locator) cache.Locator {
|
||||
path := loc.Path
|
||||
if path == "" {
|
||||
return loc
|
||||
}
|
||||
|
||||
var qsSuffix string
|
||||
core := path
|
||||
if idx := strings.Index(core, "/__qs/"); idx >= 0 {
|
||||
qsSuffix = core[idx:]
|
||||
core = core[:idx]
|
||||
}
|
||||
|
||||
if strings.Contains(core, "/-/") {
|
||||
loc.Path = core + qsSuffix
|
||||
return loc
|
||||
}
|
||||
|
||||
clean := strings.TrimSuffix(core, "/")
|
||||
if clean == "" {
|
||||
clean = "/"
|
||||
}
|
||||
|
||||
if clean == "/" {
|
||||
loc.Path = "/package.json" + qsSuffix
|
||||
return loc
|
||||
}
|
||||
|
||||
loc.Path = clean + "/package.json" + qsSuffix
|
||||
return loc
|
||||
}
|
||||
30
internal/hubmodule/npm/module.go
Normal file
30
internal/hubmodule/npm/module.go
Normal file
@@ -0,0 +1,30 @@
|
||||
// Package npm 描述 npm Registry 模块的默认策略与注册逻辑,方便新 Hub 直接启用。
|
||||
package npm
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
const npmDefaultTTL = 30 * time.Minute
|
||||
|
||||
// npm 模块描述 NPM Registry 的默认缓存策略,并允许通过 [[Hub]] 覆盖 TTL/Validation。
|
||||
func init() {
|
||||
hubmodule.MustRegister(hubmodule.ModuleMetadata{
|
||||
Key: "npm",
|
||||
Description: "NPM proxy module with cache strategy overrides for metadata/tarballs",
|
||||
MigrationState: hubmodule.MigrationStateBeta,
|
||||
SupportedProtocols: []string{
|
||||
"npm",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: npmDefaultTTL,
|
||||
ValidationMode: hubmodule.ValidationModeLastModified,
|
||||
DiskLayout: "raw_path",
|
||||
RequiresMetadataFile: false,
|
||||
SupportsStreamingWrite: true,
|
||||
},
|
||||
LocatorRewrite: rewriteLocator,
|
||||
})
|
||||
}
|
||||
52
internal/hubmodule/npm/module_test.go
Normal file
52
internal/hubmodule/npm/module_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
func TestNPMMetadataRegistration(t *testing.T) {
|
||||
meta, ok := hubmodule.Resolve("npm")
|
||||
if !ok {
|
||||
t.Fatalf("npm module not registered")
|
||||
}
|
||||
if meta.Key != "npm" {
|
||||
t.Fatalf("unexpected module key: %s", meta.Key)
|
||||
}
|
||||
if meta.MigrationState == "" {
|
||||
t.Fatalf("migration state must be set")
|
||||
}
|
||||
if len(meta.SupportedProtocols) == 0 {
|
||||
t.Fatalf("supported protocols must not be empty")
|
||||
}
|
||||
if meta.CacheStrategy.TTLHint != npmDefaultTTL {
|
||||
t.Fatalf("expected default ttl %s, got %s", npmDefaultTTL, meta.CacheStrategy.TTLHint)
|
||||
}
|
||||
if meta.CacheStrategy.ValidationMode != hubmodule.ValidationModeLastModified {
|
||||
t.Fatalf("expected validation mode last-modified, got %s", meta.CacheStrategy.ValidationMode)
|
||||
}
|
||||
if !meta.CacheStrategy.SupportsStreamingWrite {
|
||||
t.Fatalf("npm strategy should support streaming writes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNPMStrategyOverrides(t *testing.T) {
|
||||
meta, ok := hubmodule.Resolve("npm")
|
||||
if !ok {
|
||||
t.Fatalf("npm module not registered")
|
||||
}
|
||||
|
||||
overrideTTL := 10 * time.Minute
|
||||
strategy := hubmodule.ResolveStrategy(meta, hubmodule.StrategyOptions{
|
||||
TTLOverride: overrideTTL,
|
||||
ValidationOverride: hubmodule.ValidationModeETag,
|
||||
})
|
||||
if strategy.TTLHint != overrideTTL {
|
||||
t.Fatalf("expected ttl override %s, got %s", overrideTTL, strategy.TTLHint)
|
||||
}
|
||||
if strategy.ValidationMode != hubmodule.ValidationModeETag {
|
||||
t.Fatalf("expected validation mode override to etag, got %s", strategy.ValidationMode)
|
||||
}
|
||||
}
|
||||
29
internal/hubmodule/pypi/module.go
Normal file
29
internal/hubmodule/pypi/module.go
Normal file
@@ -0,0 +1,29 @@
|
||||
// Package pypi 聚焦 PyPI simple index 模块,提供 TTL/验证策略的注册样例。
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
const pypiDefaultTTL = 15 * time.Minute
|
||||
|
||||
// pypi 模块负责 simple index + 分发包的策略声明,默认使用 Last-Modified 校验。
|
||||
func init() {
|
||||
hubmodule.MustRegister(hubmodule.ModuleMetadata{
|
||||
Key: "pypi",
|
||||
Description: "PyPI simple index module with per-hub cache overrides",
|
||||
MigrationState: hubmodule.MigrationStateBeta,
|
||||
SupportedProtocols: []string{
|
||||
"pypi",
|
||||
},
|
||||
CacheStrategy: hubmodule.CacheStrategyProfile{
|
||||
TTLHint: pypiDefaultTTL,
|
||||
ValidationMode: hubmodule.ValidationModeLastModified,
|
||||
DiskLayout: "raw_path",
|
||||
RequiresMetadataFile: false,
|
||||
SupportsStreamingWrite: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -17,5 +17,18 @@ func ResolveStrategy(meta ModuleMetadata, opts StrategyOptions) CacheStrategyPro
|
||||
if opts.ValidationOverride != "" {
|
||||
strategy.ValidationMode = opts.ValidationOverride
|
||||
}
|
||||
return strategy
|
||||
return normalizeStrategy(strategy)
|
||||
}
|
||||
|
||||
func normalizeStrategy(profile CacheStrategyProfile) CacheStrategyProfile {
|
||||
if profile.TTLHint < 0 {
|
||||
profile.TTLHint = 0
|
||||
}
|
||||
if profile.ValidationMode == "" {
|
||||
profile.ValidationMode = ValidationModeETag
|
||||
}
|
||||
if profile.DiskLayout == "" {
|
||||
profile.DiskLayout = "raw_path"
|
||||
}
|
||||
return profile
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
// Package template 提供编写新模块时可复制的骨架示例。
|
||||
package template
|
||||
|
||||
import "github.com/any-hub/any-hub/internal/hubmodule"
|
||||
|
||||
// Package template 提供编写新模块时可复制的骨架示例。
|
||||
//
|
||||
// 使用方式:复制整个目录到 internal/hubmodule/<module-key>/ 并替换字段。
|
||||
// - 将 TemplateModule 重命名为实际模块类型。
|
||||
|
||||
@@ -11,13 +11,14 @@ func BaseFields(action, configPath string) logrus.Fields {
|
||||
}
|
||||
|
||||
// RequestFields 提供 hub/domain/命中状态字段,供代理请求日志复用。
|
||||
func RequestFields(hub, domain, hubType, authMode, moduleKey string, cacheHit bool) logrus.Fields {
|
||||
func RequestFields(hub, domain, hubType, authMode, moduleKey, rolloutFlag string, cacheHit bool) logrus.Fields {
|
||||
return logrus.Fields{
|
||||
"hub": hub,
|
||||
"domain": domain,
|
||||
"hub_type": hubType,
|
||||
"auth_mode": authMode,
|
||||
"cache_hit": cacheHit,
|
||||
"module_key": moduleKey,
|
||||
"hub": hub,
|
||||
"domain": domain,
|
||||
"hub_type": hubType,
|
||||
"auth_mode": authMode,
|
||||
"cache_hit": cacheHit,
|
||||
"module_key": moduleKey,
|
||||
"rollout_flag": rolloutFlag,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ func (h *Handler) Handle(c fiber.Ctx, route *server.HubRoute) error {
|
||||
requestID := server.RequestID(c)
|
||||
locator := buildLocator(route, c)
|
||||
policy := determineCachePolicy(route, locator, c.Method())
|
||||
strategyWriter := cache.NewStrategyWriter(h.store, route.CacheStrategy)
|
||||
|
||||
if err := ensureProxyHubType(route); err != nil {
|
||||
h.logger.WithFields(logrus.Fields{
|
||||
@@ -62,7 +63,7 @@ func (h *Handler) Handle(c fiber.Ctx, route *server.HubRoute) error {
|
||||
}
|
||||
|
||||
var cached *cache.ReadResult
|
||||
if h.store != nil && policy.allowCache {
|
||||
if strategyWriter.Enabled() && policy.allowCache {
|
||||
result, err := h.store.Get(ctx, locator)
|
||||
switch {
|
||||
case err == nil:
|
||||
@@ -79,13 +80,19 @@ func (h *Handler) Handle(c fiber.Ctx, route *server.HubRoute) error {
|
||||
if cached != nil {
|
||||
serve := true
|
||||
if policy.requireRevalidate {
|
||||
fresh, err := h.isCacheFresh(c, route, locator, cached.Entry)
|
||||
if err != nil {
|
||||
h.logger.WithError(err).
|
||||
WithFields(logrus.Fields{"hub": route.Config.Name, "module_key": route.ModuleKey}).
|
||||
Warn("cache_revalidate_failed")
|
||||
serve = false
|
||||
} else if !fresh {
|
||||
if strategyWriter.ShouldBypassValidation(cached.Entry) {
|
||||
serve = true
|
||||
} else if strategyWriter.SupportsValidation() {
|
||||
fresh, err := h.isCacheFresh(c, route, locator, cached.Entry)
|
||||
if err != nil {
|
||||
h.logger.WithError(err).
|
||||
WithFields(logrus.Fields{"hub": route.Config.Name, "module_key": route.ModuleKey}).
|
||||
Warn("cache_revalidate_failed")
|
||||
serve = false
|
||||
} else if !fresh {
|
||||
serve = false
|
||||
}
|
||||
} else {
|
||||
serve = false
|
||||
}
|
||||
}
|
||||
@@ -96,7 +103,7 @@ func (h *Handler) Handle(c fiber.Ctx, route *server.HubRoute) error {
|
||||
cached.Reader.Close()
|
||||
}
|
||||
|
||||
return h.fetchAndStream(c, route, locator, policy, requestID, started, ctx)
|
||||
return h.fetchAndStream(c, route, locator, policy, strategyWriter, requestID, started, ctx)
|
||||
}
|
||||
|
||||
func (h *Handler) serveCache(c fiber.Ctx, route *server.HubRoute, result *cache.ReadResult, requestID string, started time.Time) error {
|
||||
@@ -144,7 +151,7 @@ func (h *Handler) serveCache(c fiber.Ctx, route *server.HubRoute, result *cache.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Handler) fetchAndStream(c fiber.Ctx, route *server.HubRoute, locator cache.Locator, policy cachePolicy, requestID string, started time.Time, ctx context.Context) error {
|
||||
func (h *Handler) fetchAndStream(c fiber.Ctx, route *server.HubRoute, locator cache.Locator, policy cachePolicy, writer cache.StrategyWriter, requestID string, started time.Time, ctx context.Context) error {
|
||||
resp, upstreamURL, err := h.executeRequest(c, route)
|
||||
if err != nil {
|
||||
h.logResult(route, upstreamURL.String(), requestID, 0, false, started, err)
|
||||
@@ -158,17 +165,17 @@ func (h *Handler) fetchAndStream(c fiber.Ctx, route *server.HubRoute, locator ca
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
shouldStore := policy.allowStore && h.store != nil && isCacheableStatus(resp.StatusCode) && c.Method() == http.MethodGet
|
||||
return h.consumeUpstream(c, route, locator, resp, shouldStore, requestID, started, ctx)
|
||||
shouldStore := policy.allowStore && writer.Enabled() && isCacheableStatus(resp.StatusCode) && c.Method() == http.MethodGet
|
||||
return h.consumeUpstream(c, route, locator, resp, shouldStore, writer, requestID, started, ctx)
|
||||
}
|
||||
|
||||
func (h *Handler) consumeUpstream(c fiber.Ctx, route *server.HubRoute, locator cache.Locator, resp *http.Response, shouldStore bool, requestID string, started time.Time, ctx context.Context) error {
|
||||
func (h *Handler) consumeUpstream(c fiber.Ctx, route *server.HubRoute, locator cache.Locator, resp *http.Response, shouldStore bool, writer cache.StrategyWriter, requestID string, started time.Time, ctx context.Context) error {
|
||||
upstreamURL := resp.Request.URL.String()
|
||||
method := c.Method()
|
||||
authFailure := isAuthFailure(resp.StatusCode) && route.Config.HasCredentials()
|
||||
|
||||
if shouldStore {
|
||||
return h.cacheAndStream(c, route, locator, resp, requestID, started, ctx, upstreamURL)
|
||||
return h.cacheAndStream(c, route, locator, resp, writer, requestID, started, ctx, upstreamURL)
|
||||
}
|
||||
|
||||
copyResponseHeaders(c, resp.Header)
|
||||
@@ -196,7 +203,7 @@ func (h *Handler) consumeUpstream(c fiber.Ctx, route *server.HubRoute, locator c
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Handler) cacheAndStream(c fiber.Ctx, route *server.HubRoute, locator cache.Locator, resp *http.Response, requestID string, started time.Time, ctx context.Context, upstreamURL string) error {
|
||||
func (h *Handler) cacheAndStream(c fiber.Ctx, route *server.HubRoute, locator cache.Locator, resp *http.Response, writer cache.StrategyWriter, requestID string, started time.Time, ctx context.Context, upstreamURL string) error {
|
||||
copyResponseHeaders(c, resp.Header)
|
||||
c.Set("X-Any-Hub-Upstream", upstreamURL)
|
||||
c.Set("X-Any-Hub-Cache-Hit", "false")
|
||||
@@ -208,7 +215,7 @@ func (h *Handler) cacheAndStream(c fiber.Ctx, route *server.HubRoute, locator ca
|
||||
reader := io.TeeReader(resp.Body, c.Response().BodyWriter())
|
||||
|
||||
opts := cache.PutOptions{ModTime: extractModTime(resp.Header)}
|
||||
entry, err := h.store.Put(ctx, locator, reader, opts)
|
||||
entry, err := writer.Put(ctx, locator, reader, opts)
|
||||
h.logResult(route, upstreamURL, requestID, resp.StatusCode, false, started, err)
|
||||
if err != nil {
|
||||
return fiber.NewError(fiber.StatusBadGateway, fmt.Sprintf("cache_write_failed: %v", err))
|
||||
@@ -323,7 +330,7 @@ func (h *Handler) writeError(c fiber.Ctx, status int, code string) error {
|
||||
}
|
||||
|
||||
func (h *Handler) logResult(route *server.HubRoute, upstream string, requestID string, status int, cacheHit bool, started time.Time, err error) {
|
||||
fields := logging.RequestFields(route.Config.Name, route.Config.Domain, route.Config.Type, route.Config.AuthMode(), route.ModuleKey, cacheHit)
|
||||
fields := logging.RequestFields(route.Config.Name, route.Config.Domain, route.Config.Type, route.Config.AuthMode(), route.ModuleKey, string(route.RolloutFlag), cacheHit)
|
||||
fields["action"] = "proxy"
|
||||
fields["upstream"] = upstream
|
||||
fields["upstream_status"] = status
|
||||
@@ -399,10 +406,14 @@ func buildLocator(route *server.HubRoute, c fiber.Ctx) cache.Locator {
|
||||
sum := sha1.Sum(query)
|
||||
clean = fmt.Sprintf("%s/__qs/%s", clean, hex.EncodeToString(sum[:]))
|
||||
}
|
||||
return cache.Locator{
|
||||
loc := cache.Locator{
|
||||
HubName: route.Config.Name,
|
||||
Path: clean,
|
||||
}
|
||||
if route.Module.LocatorRewrite != nil {
|
||||
loc = route.Module.LocatorRewrite(loc)
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
func stripQueryMarker(p string) string {
|
||||
@@ -807,7 +818,7 @@ func isAuthFailure(status int) bool {
|
||||
}
|
||||
|
||||
func (h *Handler) logAuthRetry(route *server.HubRoute, upstream string, requestID string, status int) {
|
||||
fields := logging.RequestFields(route.Config.Name, route.Config.Domain, route.Config.Type, route.Config.AuthMode(), route.ModuleKey, false)
|
||||
fields := logging.RequestFields(route.Config.Name, route.Config.Domain, route.Config.Type, route.Config.AuthMode(), route.ModuleKey, string(route.RolloutFlag), false)
|
||||
fields["action"] = "proxy_retry"
|
||||
fields["upstream"] = upstream
|
||||
fields["upstream_status"] = status
|
||||
@@ -819,7 +830,7 @@ func (h *Handler) logAuthRetry(route *server.HubRoute, upstream string, requestI
|
||||
}
|
||||
|
||||
func (h *Handler) logAuthFailure(route *server.HubRoute, upstream string, requestID string, status int) {
|
||||
fields := logging.RequestFields(route.Config.Name, route.Config.Domain, route.Config.Type, route.Config.AuthMode(), route.ModuleKey, false)
|
||||
fields := logging.RequestFields(route.Config.Name, route.Config.Domain, route.Config.Type, route.Config.AuthMode(), route.ModuleKey, string(route.RolloutFlag), false)
|
||||
fields["action"] = "proxy"
|
||||
fields["upstream"] = upstream
|
||||
fields["upstream_status"] = status
|
||||
|
||||
@@ -3,13 +3,12 @@ package server
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
)
|
||||
|
||||
func moduleMetadataForHub(hub config.HubConfig) (hubmodule.ModuleMetadata, error) {
|
||||
if meta, ok := hubmodule.Resolve(hub.Module); ok {
|
||||
func moduleMetadataForKey(key string) (hubmodule.ModuleMetadata, error) {
|
||||
if meta, ok := hubmodule.Resolve(key); ok {
|
||||
return meta, nil
|
||||
}
|
||||
return hubmodule.ModuleMetadata{}, fmt.Errorf("module %s is not registered", hub.Module)
|
||||
return hubmodule.ModuleMetadata{}, fmt.Errorf("module %s is not registered", key)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
)
|
||||
|
||||
// HubRoute 将 Hub 配置与派生属性(如缓存 TTL、解析后的 Upstream/Proxy URL)
|
||||
@@ -28,6 +29,10 @@ type HubRoute struct {
|
||||
// ModuleKey/Module 记录当前 hub 选用的模块及其元数据,便于日志与观测。
|
||||
ModuleKey string
|
||||
Module hubmodule.ModuleMetadata
|
||||
// CacheStrategy 代表模块默认策略与 hub 覆盖后的最终结果。
|
||||
CacheStrategy hubmodule.CacheStrategyProfile
|
||||
// RolloutFlag 反映当前 hub 的 legacy → modular 迁移状态,供日志/诊断使用。
|
||||
RolloutFlag legacy.RolloutFlag
|
||||
}
|
||||
|
||||
// HubRegistry 提供 Host/Host:port 到 HubRoute 的查询能力,所有 Hub 共享同一个监听端口。
|
||||
@@ -100,7 +105,9 @@ func (r *HubRegistry) List() []HubRoute {
|
||||
}
|
||||
|
||||
func buildHubRoute(cfg *config.Config, hub config.HubConfig) (*HubRoute, error) {
|
||||
meta, err := moduleMetadataForHub(hub)
|
||||
flag := hub.RolloutFlagValue()
|
||||
effectiveKey := config.EffectiveModuleKey(hub.Module, flag)
|
||||
meta, err := moduleMetadataForKey(effectiveKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hub %s: %w", hub.Name, err)
|
||||
}
|
||||
@@ -118,14 +125,20 @@ func buildHubRoute(cfg *config.Config, hub config.HubConfig) (*HubRoute, error)
|
||||
}
|
||||
}
|
||||
|
||||
effectiveTTL := cfg.EffectiveCacheTTL(hub)
|
||||
runtime := config.BuildHubRuntime(hub, meta, effectiveTTL, flag)
|
||||
legacy.RecordAdapterState(hub.Name, runtime.Module.Key, flag)
|
||||
|
||||
return &HubRoute{
|
||||
Config: hub,
|
||||
ListenPort: cfg.Global.ListenPort,
|
||||
CacheTTL: cfg.EffectiveCacheTTL(hub),
|
||||
UpstreamURL: upstreamURL,
|
||||
ProxyURL: proxyURL,
|
||||
ModuleKey: meta.Key,
|
||||
Module: meta,
|
||||
Config: hub,
|
||||
ListenPort: cfg.Global.ListenPort,
|
||||
CacheTTL: effectiveTTL,
|
||||
UpstreamURL: upstreamURL,
|
||||
ProxyURL: proxyURL,
|
||||
ModuleKey: runtime.Module.Key,
|
||||
Module: runtime.Module,
|
||||
CacheStrategy: runtime.CacheStrategy,
|
||||
RolloutFlag: runtime.Rollout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/config"
|
||||
"github.com/any-hub/any-hub/internal/hubmodule/legacy"
|
||||
)
|
||||
|
||||
func TestHubRegistryLookupByHost(t *testing.T) {
|
||||
@@ -48,6 +49,15 @@ func TestHubRegistryLookupByHost(t *testing.T) {
|
||||
if route.CacheTTL != cfg.EffectiveCacheTTL(route.Config) {
|
||||
t.Errorf("cache ttl mismatch: got %s", route.CacheTTL)
|
||||
}
|
||||
if route.CacheStrategy.TTLHint != route.CacheTTL {
|
||||
t.Errorf("cache strategy ttl mismatch: %s vs %s", route.CacheStrategy.TTLHint, route.CacheTTL)
|
||||
}
|
||||
if route.CacheStrategy.ValidationMode == "" {
|
||||
t.Fatalf("cache strategy validation mode should not be empty")
|
||||
}
|
||||
if route.RolloutFlag != legacy.RolloutLegacyOnly {
|
||||
t.Fatalf("default rollout flag should be legacy-only")
|
||||
}
|
||||
|
||||
if route.UpstreamURL.String() != "https://registry-1.docker.io" {
|
||||
t.Errorf("unexpected upstream URL: %s", route.UpstreamURL)
|
||||
|
||||
@@ -79,6 +79,10 @@ func requestContextMiddleware(opts AppOptions) fiber.Handler {
|
||||
c.Locals(contextKeyRequestID, reqID)
|
||||
c.Set("X-Request-ID", reqID)
|
||||
|
||||
if isDiagnosticsPath(string(c.Request().URI().Path())) {
|
||||
return c.Next()
|
||||
}
|
||||
|
||||
rawHost := strings.TrimSpace(getHostHeader(c))
|
||||
route, ok := opts.Registry.Lookup(rawHost)
|
||||
if !ok {
|
||||
@@ -153,13 +157,19 @@ func ensureRouterHubType(route *HubRoute) error {
|
||||
|
||||
func renderTypeUnsupported(c fiber.Ctx, logger *logrus.Logger, route *HubRoute, err error) error {
|
||||
fields := logrus.Fields{
|
||||
"action": "hub_type_check",
|
||||
"hub": route.Config.Name,
|
||||
"hub_type": route.Config.Type,
|
||||
"error": "hub_type_unsupported",
|
||||
"action": "hub_type_check",
|
||||
"hub": route.Config.Name,
|
||||
"hub_type": route.Config.Type,
|
||||
"module_key": route.ModuleKey,
|
||||
"rollout_flag": string(route.RolloutFlag),
|
||||
"error": "hub_type_unsupported",
|
||||
}
|
||||
logger.WithFields(fields).Error(err.Error())
|
||||
return c.Status(fiber.StatusNotImplemented).JSON(fiber.Map{
|
||||
"error": "hub_type_unsupported",
|
||||
})
|
||||
}
|
||||
|
||||
func isDiagnosticsPath(path string) bool {
|
||||
return strings.HasPrefix(path, "/-/")
|
||||
}
|
||||
|
||||
114
internal/server/routes/modules.go
Normal file
114
internal/server/routes/modules.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v3"
|
||||
|
||||
"github.com/any-hub/any-hub/internal/hubmodule"
|
||||
"github.com/any-hub/any-hub/internal/server"
|
||||
)
|
||||
|
||||
// RegisterModuleRoutes 暴露 /-/modules 诊断接口,供 SRE 查询模块与 Hub 绑定关系。
|
||||
func RegisterModuleRoutes(app *fiber.App, registry *server.HubRegistry) {
|
||||
if app == nil || registry == nil {
|
||||
return
|
||||
}
|
||||
|
||||
app.Get("/-/modules", func(c fiber.Ctx) error {
|
||||
payload := fiber.Map{
|
||||
"modules": encodeModules(hubmodule.List()),
|
||||
"hubs": encodeHubBindings(registry.List()),
|
||||
}
|
||||
return c.JSON(payload)
|
||||
})
|
||||
|
||||
app.Get("/-/modules/:key", func(c fiber.Ctx) error {
|
||||
key := strings.ToLower(strings.TrimSpace(c.Params("key")))
|
||||
if key == "" {
|
||||
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "module_key_required"})
|
||||
}
|
||||
meta, ok := hubmodule.Resolve(key)
|
||||
if !ok {
|
||||
return c.Status(fiber.StatusNotFound).JSON(fiber.Map{"error": "module_not_found"})
|
||||
}
|
||||
return c.JSON(encodeModule(meta))
|
||||
})
|
||||
}
|
||||
|
||||
type modulePayload struct {
|
||||
Key string `json:"key"`
|
||||
Description string `json:"description"`
|
||||
MigrationState hubmodule.MigrationState `json:"migration_state"`
|
||||
SupportedProtocols []string `json:"supported_protocols"`
|
||||
CacheStrategy cacheStrategyPayload `json:"cache_strategy"`
|
||||
}
|
||||
|
||||
type cacheStrategyPayload struct {
|
||||
TTLSeconds int64 `json:"ttl_seconds"`
|
||||
ValidationMode string `json:"validation_mode"`
|
||||
DiskLayout string `json:"disk_layout"`
|
||||
RequiresMetadataFile bool `json:"requires_metadata_file"`
|
||||
SupportsStreamingWrite bool `json:"supports_streaming_write"`
|
||||
}
|
||||
|
||||
type hubBindingPayload struct {
|
||||
HubName string `json:"hub_name"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
Domain string `json:"domain"`
|
||||
Port int `json:"port"`
|
||||
Rollout string `json:"rollout_flag"`
|
||||
}
|
||||
|
||||
func encodeModules(mods []hubmodule.ModuleMetadata) []modulePayload {
|
||||
if len(mods) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(mods, func(i, j int) bool {
|
||||
return mods[i].Key < mods[j].Key
|
||||
})
|
||||
result := make([]modulePayload, 0, len(mods))
|
||||
for _, meta := range mods {
|
||||
result = append(result, encodeModule(meta))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func encodeModule(meta hubmodule.ModuleMetadata) modulePayload {
|
||||
strategy := meta.CacheStrategy
|
||||
return modulePayload{
|
||||
Key: meta.Key,
|
||||
Description: meta.Description,
|
||||
MigrationState: meta.MigrationState,
|
||||
SupportedProtocols: append([]string(nil), meta.SupportedProtocols...),
|
||||
CacheStrategy: cacheStrategyPayload{
|
||||
TTLSeconds: int64(strategy.TTLHint / time.Second),
|
||||
ValidationMode: string(strategy.ValidationMode),
|
||||
DiskLayout: strategy.DiskLayout,
|
||||
RequiresMetadataFile: strategy.RequiresMetadataFile,
|
||||
SupportsStreamingWrite: strategy.SupportsStreamingWrite,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func encodeHubBindings(routes []server.HubRoute) []hubBindingPayload {
|
||||
if len(routes) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(routes, func(i, j int) bool {
|
||||
return routes[i].Config.Name < routes[j].Config.Name
|
||||
})
|
||||
result := make([]hubBindingPayload, 0, len(routes))
|
||||
for _, route := range routes {
|
||||
result = append(result, hubBindingPayload{
|
||||
HubName: route.Config.Name,
|
||||
ModuleKey: route.ModuleKey,
|
||||
Domain: route.Config.Domain,
|
||||
Port: route.ListenPort,
|
||||
Rollout: string(route.RolloutFlag),
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user