package service import ( "context" "fmt" "io" "net/http" "strings" "time" "github.com/gochat/gochat/internal/model" "golang.org/x/net/html" ) // captainDocumentCrawlBackendImpl is the production implementation of // CaptainDocumentCrawlBackend. It fetches the document's external URL, // extracts page links for crawl fan-out. type captainDocumentCrawlBackendImpl struct { httpClient *http.Client } // NewCaptainDocumentCrawlBackend creates the production crawl backend. func NewCaptainDocumentCrawlBackend() CaptainDocumentCrawlBackend { return &captainDocumentCrawlBackendImpl{ httpClient: &http.Client{Timeout: 30 * time.Second}, } } func (b *captainDocumentCrawlBackendImpl) CrawlCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentCrawlResult, error) { url := doc.ExternalLink if url == "" { return &CaptainDocumentCrawlResult{ErrorCode: "not_found"}, nil } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err) } req.Header.Set("User-Agent", "GoChat-Captain/1.0") resp, err := b.httpClient.Do(req) if err != nil { return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) } body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max if err != nil { return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err) } links := extractPageLinks(string(body), url) return &CaptainDocumentCrawlResult{ PageLinks: links, }, nil } // extractPageLinks parses an HTML document and returns all unique absolute // href links found in tags. func extractPageLinks(htmlStr, baseURL string) []string { doc, err := html.Parse(strings.NewReader(htmlStr)) if err != nil { return nil } base := baseURL // Remove trailing slash for consistent prefix matching base = strings.TrimSuffix(base, "/") var links []string var walk func(*html.Node) walk = func(n *html.Node) { if n.Type == html.ElementNode && n.Data == "a" { for _, attr := range n.Attr { if attr.Key == "href" { href := strings.TrimSpace(attr.Val) if href == "" || strings.HasPrefix(href, "#") { continue } absolute := resolveURL(href, base) if absolute != "" { links = append(links, absolute) } } } } for c := n.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(doc) // Deduplicate seen := make(map[string]struct{}, len(links)) unique := make([]string, 0, len(links)) for _, l := range links { if _, ok := seen[l]; ok { continue } seen[l] = struct{}{} unique = append(unique, l) } return unique } // resolveURL converts a relative URL to an absolute URL using the base URL. func resolveURL(href, base string) string { href = strings.TrimSpace(href) if href == "" { return "" } // Already absolute if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { return href } // Protocol-relative if strings.HasPrefix(href, "//") { return "https:" + href } // Absolute path if strings.HasPrefix(href, "/") { // Extract scheme://host from base idx := strings.Index(base, "://") if idx < 0 { return "" } hostEnd := strings.IndexByte(base[idx+3:], '/') if hostEnd < 0 { return base + href } return base[:idx+3+hostEnd] + href } // Relative path return base + "/" + href }