140 lines
3.6 KiB
Go
140 lines
3.6 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gochat/gochat/internal/model"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
// captainDocumentCrawlBackendImpl is the production implementation of
|
|
// CaptainDocumentCrawlBackend. It fetches the document's external URL,
|
|
// extracts page links for crawl fan-out.
|
|
type captainDocumentCrawlBackendImpl struct {
|
|
httpClient *http.Client
|
|
}
|
|
|
|
// NewCaptainDocumentCrawlBackend creates the production crawl backend.
|
|
func NewCaptainDocumentCrawlBackend() CaptainDocumentCrawlBackend {
|
|
return &captainDocumentCrawlBackendImpl{
|
|
httpClient: &http.Client{Timeout: 30 * time.Second},
|
|
}
|
|
}
|
|
|
|
func (b *captainDocumentCrawlBackendImpl) CrawlCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentCrawlResult, error) {
|
|
url := doc.ExternalLink
|
|
if url == "" {
|
|
return &CaptainDocumentCrawlResult{ErrorCode: "not_found"}, nil
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", "GoChat-Captain/1.0")
|
|
|
|
resp, err := b.httpClient.Do(req)
|
|
if err != nil {
|
|
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
|
|
}
|
|
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
|
|
if err != nil {
|
|
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
|
|
}
|
|
|
|
links := extractPageLinks(string(body), url)
|
|
return &CaptainDocumentCrawlResult{
|
|
PageLinks: links,
|
|
}, nil
|
|
}
|
|
|
|
// extractPageLinks parses an HTML document and returns all unique absolute
|
|
// href links found in <a> tags.
|
|
func extractPageLinks(htmlStr, baseURL string) []string {
|
|
doc, err := html.Parse(strings.NewReader(htmlStr))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
base := baseURL
|
|
// Remove trailing slash for consistent prefix matching
|
|
base = strings.TrimSuffix(base, "/")
|
|
|
|
var links []string
|
|
var walk func(*html.Node)
|
|
walk = func(n *html.Node) {
|
|
if n.Type == html.ElementNode && n.Data == "a" {
|
|
for _, attr := range n.Attr {
|
|
if attr.Key == "href" {
|
|
href := strings.TrimSpace(attr.Val)
|
|
if href == "" || strings.HasPrefix(href, "#") {
|
|
continue
|
|
}
|
|
absolute := resolveURL(href, base)
|
|
if absolute != "" {
|
|
links = append(links, absolute)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(doc)
|
|
|
|
// Deduplicate
|
|
seen := make(map[string]struct{}, len(links))
|
|
unique := make([]string, 0, len(links))
|
|
for _, l := range links {
|
|
if _, ok := seen[l]; ok {
|
|
continue
|
|
}
|
|
seen[l] = struct{}{}
|
|
unique = append(unique, l)
|
|
}
|
|
return unique
|
|
}
|
|
|
|
// resolveURL converts a relative URL to an absolute URL using the base URL.
|
|
func resolveURL(href, base string) string {
|
|
href = strings.TrimSpace(href)
|
|
if href == "" {
|
|
return ""
|
|
}
|
|
// Already absolute
|
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
|
return href
|
|
}
|
|
// Protocol-relative
|
|
if strings.HasPrefix(href, "//") {
|
|
return "https:" + href
|
|
}
|
|
// Absolute path
|
|
if strings.HasPrefix(href, "/") {
|
|
// Extract scheme://host from base
|
|
idx := strings.Index(base, "://")
|
|
if idx < 0 {
|
|
return ""
|
|
}
|
|
hostEnd := strings.IndexByte(base[idx+3:], '/')
|
|
if hostEnd < 0 {
|
|
return base + href
|
|
}
|
|
return base[:idx+3+hostEnd] + href
|
|
}
|
|
// Relative path
|
|
return base + "/" + href
|
|
}
|