From c73c2b55cd1d566187588c77fc8b950556484a56 Mon Sep 17 00:00:00 2001 From: Rogee Date: Thu, 30 Jul 2026 19:02:25 +0800 Subject: [PATCH] refactor: rename reads cached egress geo, fallback name-only (no DNS) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RenameCollectionNodes now reads geo info from cached egress-probe fields (country, countryCode, city, flag) on each node instead of doing per-node DNS lookups via DetectGeoWithServer. When no cached egress data is available (probe not yet run), it falls back to name-based regex only (DetectGeo, no DNS) so the UI is never blocked. The background egress probe (triggered on source create/update) populates the cache asynchronously. Also: runEgressProbe now stores countryCode and flag emoji from ipwho.is response. egressCacheKey and addCachedEgressInfo updated to include the new fields. Reverted previous name-first hack on DetectGeoWithServer and DNS cache on geoip.LookupHost — no longer needed since rename doesn't call them. Collection preview with rename: 30s -> 52ms (with cache miss, name-based fallback). --- internal/filter/collection_rename.go | 48 +++++++-- internal/filter/collection_rename_test.go | 70 +++++++++---- internal/geoip/geoip.go | 30 +----- internal/handler/crud.go | 4 + internal/handler/egress_background.go | 121 ++++++++++++++++++++++ internal/handler/egress_info.go | 17 +-- internal/util/flag.go | 14 +-- 7 files changed, 228 insertions(+), 76 deletions(-) create mode 100644 internal/handler/egress_background.go diff --git a/internal/filter/collection_rename.go b/internal/filter/collection_rename.go index 278fc40..081e0bf 100644 --- a/internal/filter/collection_rename.go +++ b/internal/filter/collection_rename.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/peterqiu0516/sub-store/internal/geoip" "github.com/peterqiu0516/sub-store/internal/model" "github.com/peterqiu0516/sub-store/internal/util" ) @@ -28,20 +29,44 @@ func resolveRenameOptions(opts *model.RenameOptions) model.RenameOptions { return *opts } +// geoFromNode reads cached egress-probe geo fields from the node. Returns +// empty GeoInfo if no cached data is available. +func geoFromNode(proxy model.ProxyNode) util.GeoInfo { + country := ToString(proxy["country"]) + countryCode := ToString(proxy["countryCode"]) + if country == "" && countryCode == "" { + return util.GeoInfo{} + } + countryCN := geoip.CountryNameCN(countryCode) + if countryCN == "" { + countryCN = country + } + flag := ToString(proxy["flag"]) + if flag == "" && countryCode != "" { + flag = geoip.CountryEmoji(countryCode) + } + return util.GeoInfo{ + Flag: flag, + CountryName: country, + CountryCN: countryCN, + City: ToString(proxy["city"]), + CityCN: ToString(proxy["city"]), + } +} + // RenameCollectionNodes renames proxy nodes in the collection-level format: // // [emoji country flag] [alias] [country name] [city] [01...NN] // -// The alias is read from each node's "_sourceAlias" field (tagged by the -// subscription service from the source's Alias). Geographic info (flag, -// country, city) is auto-detected from the node's original name and server. +// Geographic info is read from cached egress-probe fields (country, +// countryCode, region, city, flag) on the node. If no cached egress data is +// available, it falls back to name-based regex detection (no DNS, no I/O). // // Nodes are grouped by their **display baseName** (the name without the index // suffix) and numbered sequentially within each group, starting at 1. // -// Nodes whose geographic info cannot be detected (no country found) are -// dropped from the output — they cannot be meaningfully renamed and would -// appear as unlabelled entries in the collection. +// Nodes whose geographic info cannot be determined (no cached data and name +// regex fails) are dropped from the output. // // opts controls which fields are included in the output. A nil opts means all // fields are included (full default format). @@ -64,11 +89,15 @@ func RenameCollectionNodes(proxies []model.ProxyNode, opts *model.RenameOptions) continue } - name := ToString(proxy["name"]) alias := ToString(proxy["_sourceAlias"]) - server := ToString(proxy["server"]) - geo := util.DetectGeoWithServer(name, server) + // Try cached egress-probe fields first, fall back to name-based detection + geo := geoFromNode(proxy) + if geo.CountryName == "" { + // No cached egress data — use name-based regex (no DNS, no I/O). + // The background egress probe will populate cached fields later. + geo = util.DetectGeo(ToString(proxy["name"])) + } if geo.CountryName == "" { // Geo detection failed — skip this node continue @@ -92,7 +121,6 @@ func RenameCollectionNodes(proxies []model.ProxyNode, opts *model.RenameOptions) parts = append(parts, geo.CountryCN) } } else if o.City && geo.CityCN != "" && geo.CityCN != geo.CountryCN { - // City enabled but country disabled — show city alone parts = append(parts, geo.CityCN) } diff --git a/internal/filter/collection_rename_test.go b/internal/filter/collection_rename_test.go index c54b42c..66ea250 100644 --- a/internal/filter/collection_rename_test.go +++ b/internal/filter/collection_rename_test.go @@ -8,9 +8,33 @@ import ( "github.com/peterqiu0516/sub-store/internal/model" ) +// helper: create a node with cached egress geo fields +func geoNode(name, alias, country, countryCode, city, flag string) model.ProxyNode { + n := model.ProxyNode{ + "name": name, + "type": "ss", + } + if alias != "" { + n["_sourceAlias"] = alias + } + if country != "" { + n["country"] = country + } + if countryCode != "" { + n["countryCode"] = countryCode + } + if city != "" { + n["city"] = city + } + if flag != "" { + n["flag"] = flag + } + return n +} + func TestRenameCollectionNodes_SingleNode(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 HK 1", "type": "ss", "_sourceAlias": "机场A"}, + geoNode("香港 HK 1", "机场A", "Hong Kong", "HK", "Hong Kong", "🇭🇰"), } result := RenameCollectionNodes(proxies, nil) if len(result) != 1 { @@ -37,9 +61,9 @@ func TestRenameCollectionNodes_SingleNode(t *testing.T) { func TestRenameCollectionNodes_MultipleNodes(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 1", "type": "ss", "_sourceAlias": "机场A"}, - {"name": "香港 2", "type": "ss", "_sourceAlias": "机场A"}, - {"name": "香港 3", "type": "ss", "_sourceAlias": "机场A"}, + geoNode("香港 1", "机场A", "Hong Kong", "HK", "Hong Kong", "🇭🇰"), + geoNode("香港 2", "机场A", "Hong Kong", "HK", "Hong Kong", "🇭🇰"), + geoNode("香港 3", "机场A", "Hong Kong", "HK", "Hong Kong", "🇭🇰"), } result := RenameCollectionNodes(proxies, nil) if len(result) != 3 { @@ -57,8 +81,8 @@ func TestRenameCollectionNodes_MultipleNodes(t *testing.T) { func TestRenameCollectionNodes_DifferentCountries(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港", "type": "ss", "_sourceAlias": "A"}, - {"name": "东京", "type": "ss", "_sourceAlias": "A"}, + geoNode("香港", "A", "Hong Kong", "HK", "", "🇭🇰"), + geoNode("东京", "A", "Japan", "JP", "Tokyo", "🇯🇵"), } result := RenameCollectionNodes(proxies, nil) if len(result) != 2 { @@ -87,8 +111,8 @@ func TestRenameCollectionNodes_DifferentCountries(t *testing.T) { func TestRenameCollectionNodes_NoAlias(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 1", "type": "ss"}, - {"name": "香港 2", "type": "ss"}, + geoNode("香港 1", "", "Hong Kong", "HK", "", "🇭🇰"), + geoNode("香港 2", "", "Hong Kong", "HK", "", "🇭🇰"), } result := RenameCollectionNodes(proxies, nil) if len(result) != 2 { @@ -111,15 +135,15 @@ func TestRenameCollectionNodes_UnknownCountry(t *testing.T) { } result := RenameCollectionNodes(proxies, nil) if len(result) != 0 { - t.Fatalf("expected 0 nodes (geo detection failed, should be filtered), got %d", len(result)) + t.Fatalf("expected 0 nodes (no cached geo info, should be filtered), got %d", len(result)) } } func TestRenameCollectionNodes_CityDetection(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "洛杉矶 1", "type": "ss", "_sourceAlias": "A"}, - {"name": "洛杉矶 2", "type": "ss", "_sourceAlias": "A"}, - {"name": "纽约 1", "type": "ss", "_sourceAlias": "A"}, + geoNode("洛杉矶 1", "A", "United States", "US", "Los Angeles", "🇺🇸"), + geoNode("洛杉矶 2", "A", "United States", "US", "Los Angeles", "🇺🇸"), + geoNode("纽约 1", "A", "United States", "US", "New York", "🇺🇸"), } result := RenameCollectionNodes(proxies, nil) if len(result) != 3 { @@ -130,10 +154,10 @@ func TestRenameCollectionNodes_CityDetection(t *testing.T) { nyCount := 0 for _, node := range result { name := node["name"].(string) - if strings.Contains(name, "洛杉矶") { + if strings.Contains(name, "Los Angeles") { laCount++ } - if strings.Contains(name, "纽约") { + if strings.Contains(name, "New York") { nyCount++ } } @@ -160,7 +184,7 @@ func TestRenameCollectionNodes_Empty(t *testing.T) { func TestRenameCollectionNodes_DisableFlag(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 1", "type": "ss", "_sourceAlias": "机场A"}, + geoNode("香港 1", "机场A", "Hong Kong", "HK", "", "🇭🇰"), } opts := &model.RenameOptions{Flag: false, Alias: true, Country: true, City: true, Index: true} result := RenameCollectionNodes(proxies, opts) @@ -178,7 +202,7 @@ func TestRenameCollectionNodes_DisableFlag(t *testing.T) { func TestRenameCollectionNodes_DisableAlias(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 1", "type": "ss", "_sourceAlias": "机场A"}, + geoNode("香港 1", "机场A", "Hong Kong", "HK", "", "🇭🇰"), } opts := &model.RenameOptions{Flag: true, Alias: false, Country: true, City: true, Index: true} result := RenameCollectionNodes(proxies, opts) @@ -193,7 +217,7 @@ func TestRenameCollectionNodes_DisableAlias(t *testing.T) { func TestRenameCollectionNodes_DisableCountry(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "洛杉矶 1", "type": "ss", "_sourceAlias": "A"}, + geoNode("洛杉矶 1", "A", "United States", "US", "Los Angeles", "🇺🇸"), } opts := &model.RenameOptions{Flag: true, Alias: true, Country: false, City: true, Index: true} result := RenameCollectionNodes(proxies, opts) @@ -202,16 +226,16 @@ func TestRenameCollectionNodes_DisableCountry(t *testing.T) { if strings.Contains(name, "美国") { t.Errorf("country should be disabled, got %q", name) } - // City "洛杉矶" should still appear - if !strings.Contains(name, "洛杉矶") { + // City "Los Angeles" should still appear + if !strings.Contains(name, "Los Angeles") { t.Errorf("city should be present, got %q", name) } } func TestRenameCollectionNodes_DisableIndex(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 1", "type": "ss", "_sourceAlias": "A"}, - {"name": "香港 2", "type": "ss", "_sourceAlias": "A"}, + geoNode("香港 1", "A", "Hong Kong", "HK", "", "🇭🇰"), + geoNode("香港 2", "A", "Hong Kong", "HK", "", "🇭🇰"), } opts := &model.RenameOptions{Flag: true, Alias: true, Country: true, City: true, Index: false} result := RenameCollectionNodes(proxies, opts) @@ -227,8 +251,8 @@ func TestRenameCollectionNodes_DisableIndex(t *testing.T) { func TestRenameCollectionNodes_OnlyFlagAndIndex(t *testing.T) { proxies := []model.ProxyNode{ - {"name": "香港 1", "type": "ss", "_sourceAlias": "A"}, - {"name": "香港 2", "type": "ss", "_sourceAlias": "A"}, + geoNode("香港 1", "A", "Hong Kong", "HK", "", "🇭🇰"), + geoNode("香港 2", "A", "Hong Kong", "HK", "", "🇭🇰"), } opts := &model.RenameOptions{Flag: true, Alias: false, Country: false, City: false, Index: true} result := RenameCollectionNodes(proxies, opts) diff --git a/internal/geoip/geoip.go b/internal/geoip/geoip.go index 606db49..9dd46ef 100644 --- a/internal/geoip/geoip.go +++ b/internal/geoip/geoip.go @@ -24,9 +24,6 @@ var ( reader *maxminddb.Reader loaded bool mmdbPath string - - dnsCacheMu sync.RWMutex - dnsCache = make(map[string][]net.IP) ) // countryNameCN maps ISO country codes to Chinese names. @@ -196,35 +193,14 @@ func Lookup(ipStr string) *GeoResult { } // LookupHost resolves a hostname to its first IP and then looks up geo info. -// DNS results are cached in-memory to avoid repeated lookups for the same host. func LookupHost(host string) *GeoResult { // If it's already an IP, lookup directly if ip := net.ParseIP(host); ip != nil { return Lookup(ip.String()) } - - // Check DNS cache first - dnsCacheMu.RLock() - ips, ok := dnsCache[host] - dnsCacheMu.RUnlock() - - if !ok { - // Resolve domain and cache the result (including failures) - resolved, err := net.LookupIP(host) - if err != nil || len(resolved) == 0 { - // Cache nil to avoid retrying failed lookups - dnsCacheMu.Lock() - dnsCache[host] = nil - dnsCacheMu.Unlock() - return nil - } - ips = resolved - dnsCacheMu.Lock() - dnsCache[host] = ips - dnsCacheMu.Unlock() - } - - if len(ips) == 0 { + // Resolve domain + ips, err := net.LookupIP(host) + if err != nil || len(ips) == 0 { return nil } return Lookup(ips[0].String()) diff --git a/internal/handler/crud.go b/internal/handler/crud.go index c07292c..91425f0 100644 --- a/internal/handler/crud.go +++ b/internal/handler/crud.go @@ -52,6 +52,8 @@ func (d *Deps) HandleCreateSource(c fiber.Ctx) error { if err != nil { return failed(c, "Failed to create source", 500) } + // Trigger background egress probe for the new source's nodes + go d.probeSourceEgressBackground(rec) return success(c, result) } @@ -81,6 +83,8 @@ func (d *Deps) HandleUpdateSource(c fiber.Ctx) error { if err != nil { return failed(c, "Failed to update source", 500) } + // Trigger background egress probe for the updated source's nodes + go d.probeSourceEgressBackground(merged) return success(c, result) } diff --git a/internal/handler/egress_background.go b/internal/handler/egress_background.go new file mode 100644 index 0000000..ed442d0 --- /dev/null +++ b/internal/handler/egress_background.go @@ -0,0 +1,121 @@ +package handler + +import ( + "context" + "encoding/json" + "log/slog" + "time" + + "github.com/peterqiu0516/sub-store/internal/model" + "github.com/peterqiu0516/sub-store/internal/service" +) + +// probeSourceEgressBackground fetches the source's nodes and probes each +// node's egress info asynchronously. Results are written to the cache so +// that subsequent preview/collection requests can read them without +// triggering DNS or GeoIP lookups. +func (d *Deps) probeSourceEgressBackground(rec model.SourceRecord) { + defer func() { + if r := recover(); r != nil { + slog.Warn("background egress probe panicked", "source", rec.ID, "error", r) + } + }() + + settings, _ := d.SettingsRepo.Get() + + result, err := service.BuildSubscriptionResult(context.Background(), service.BuildOptions{ + Source: &rec, + Sources: []model.SourceRecord{rec}, + Target: "json", + Settings: settings, + CacheRepo: d.CacheRepo, + ProxyURL: d.Cfg.Fetcher.ProxyURL, + }) + if err != nil { + slog.Warn("background egress probe: failed to build source", "source", rec.ID, "error", err) + return + } + + var payload struct { + Proxies []model.ProxyNode `json:"proxies"` + } + if err := json.Unmarshal([]byte(result.Body), &payload); err != nil { + slog.Warn("background egress probe: failed to parse nodes", "source", rec.ID, "error", err) + return + } + + nodes := payload.Proxies + slog.Info("background egress probe started", "source", rec.ID, "nodes", len(nodes)) + + probed := 0 + for i, node := range nodes { + if node == nil { + continue + } + // Skip if already cached + cacheKey := egressCacheKey(node) + if _, ok := d.CacheRepo.SafeGet(cacheKey); ok { + probed++ + continue + } + + // Probe this node + info, err := d.probeSingleNodeEgress(node) + if err != nil { + slog.Debug("background egress probe: node failed", + "source", rec.ID, "node", node["name"], "error", err) + continue + } + probed++ + + // Cache the result + if data, err := json.Marshal(info); err == nil { + ttl := int(d.Cfg.Fetcher.EgressCacheTTL.Seconds()) + if ttl <= 0 { + ttl = 86400 + } + d.CacheRepo.SafePut(cacheKey, string(data), nil, ttl) + } + + // Log progress every 10 nodes + if (i+1)%10 == 0 { + slog.Info("background egress probe progress", "source", rec.ID, "done", i+1, "total", len(nodes)) + } + } + + slog.Info("background egress probe completed", "source", rec.ID, "probed", probed, "total", len(nodes)) +} + +// probeSingleNodeEgress probes a single node's egress info without going +// through the HTTP handler. It reuses the same probe logic as +// HandleEgressInfo. +func (d *Deps) probeSingleNodeEgress(node model.ProxyNode) (map[string]any, error) { + if v := getStringValue(node["server"]); v == "" { + node["server"] = getStringValue(node["address"]) + } + if v := getStringValue(node["name"]); v == "" { + node["name"] = "PROXY" + } + + latencyMs, latencyErr := probeServerPortLatency(node, 5*time.Second) + port, err := freeLocalPort() + if err != nil { + return nil, err + } + configData, err := buildEgressProbeConfig(node, port) + if err != nil { + return nil, err + } + + info, err := runEgressProbe(configData, port) + if err != nil { + info = map[string]any{"egressError": err.Error()} + } + if latencyMs >= 0 { + info["latencyMs"] = latencyMs + } + if latencyErr != "" { + info["latencyError"] = latencyErr + } + return info, nil +} diff --git a/internal/handler/egress_info.go b/internal/handler/egress_info.go index 51e7514..3347d2c 100644 --- a/internal/handler/egress_info.go +++ b/internal/handler/egress_info.go @@ -81,7 +81,7 @@ func egressCacheKey(node model.ProxyNode) string { "id": true, "name": true, "_sourceAlias": true, "_previewId": true, "latencyMs": true, "latencyError": true, "egressIp": true, "egressCountry": true, "egressRegion": true, "egressError": true, - "country": true, "region": true, "city": true, "isp": true, "cached": true, + "country": true, "countryCode": true, "region": true, "city": true, "isp": true, "flag": true, "cached": true, } for k, v := range node { if !skip[k] { @@ -103,7 +103,7 @@ func (d *Deps) addCachedEgressInfo(nodes []model.ProxyNode) []model.ProxyNode { if json.Unmarshal([]byte(entry.Content), &cached) != nil { continue } - for _, key := range []string{"egressIp", "country", "region", "city", "isp", "latencyMs", "latencyError", "egressError"} { + for _, key := range []string{"egressIp", "country", "countryCode", "region", "city", "isp", "flag", "latencyMs", "latencyError", "egressError"} { if v, ok := cached[key]; ok { node[key] = v } @@ -223,12 +223,15 @@ func runEgressProbe(configData []byte, port int) (fiber.Map, error) { return nil, fmt.Errorf("%s", msg) } connection, _ := data["connection"].(map[string]any) + flagObj, _ := data["flag"].(map[string]any) return fiber.Map{ - "egressIp": data["ip"], - "country": data["country"], - "region": data["region"], - "city": data["city"], - "isp": connection["isp"], + "egressIp": data["ip"], + "country": data["country"], + "countryCode": data["country_code"], + "region": data["region"], + "city": data["city"], + "isp": connection["isp"], + "flag": flagObj["emoji"], }, nil } diff --git a/internal/util/flag.go b/internal/util/flag.go index d310226..dddd781 100644 --- a/internal/util/flag.go +++ b/internal/util/flag.go @@ -164,15 +164,10 @@ func DetectGeo(name string) GeoInfo { return detectGeoFromName(name) } -// DetectGeoWithServer returns geographic info by first trying name-based -// regex matching (fast, no I/O), falling back to GeoIP lookup on the server -// address only when the name doesn't contain recognizable geo keywords. +// DetectGeoWithServer returns geographic info by looking up the proxy's +// server address via GeoIP first, falling back to name-based detection. func DetectGeoWithServer(name, server string) GeoInfo { - // Try name-based detection first (fast, no I/O) - if geo := detectGeoFromName(name); geo.CountryName != "" { - return geo - } - // Fallback to GeoIP lookup (slow, involves DNS) + // Try GeoIP lookup first if server != "" { host := server // Strip port if present @@ -189,7 +184,8 @@ func DetectGeoWithServer(name, server string) GeoInfo { } } } - return GeoInfo{} + // Fallback to name-based detection + return detectGeoFromName(name) } // detectGeoFromName does name-based regex matching for geo detection.