fix(ui): 自定义工具表单 overflow-y-scroll→auto + 认证方式NONE翻译改为'无' + Captain文档爬虫/同步后端

backend/internal/service/captain_document_crawl_backend.go

@@ -0,0 +1,139 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gochat/gochat/internal/model"
+	"golang.org/x/net/html"
+)
+
+// captainDocumentCrawlBackendImpl is the production implementation of
+// CaptainDocumentCrawlBackend. It fetches the document's external URL,
+// extracts page links for crawl fan-out.
+type captainDocumentCrawlBackendImpl struct {
+	httpClient *http.Client
+}
+
+// NewCaptainDocumentCrawlBackend creates the production crawl backend.
+func NewCaptainDocumentCrawlBackend() CaptainDocumentCrawlBackend {
+	return &captainDocumentCrawlBackendImpl{
+		httpClient: &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+func (b *captainDocumentCrawlBackendImpl) CrawlCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentCrawlResult, error) {
+	url := doc.ExternalLink
+	if url == "" {
+		return &CaptainDocumentCrawlResult{ErrorCode: "not_found"}, nil
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("User-Agent", "GoChat-Captain/1.0")
+
+	resp, err := b.httpClient.Do(req)
+	if err != nil {
+		return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
+	if err != nil {
+		return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
+	}
+
+	links := extractPageLinks(string(body), url)
+	return &CaptainDocumentCrawlResult{
+		PageLinks: links,
+	}, nil
+}
+
+// extractPageLinks parses an HTML document and returns all unique absolute
+// href links found in <a> tags.
+func extractPageLinks(htmlStr, baseURL string) []string {
+	doc, err := html.Parse(strings.NewReader(htmlStr))
+	if err != nil {
+		return nil
+	}
+
+	base := baseURL
+	// Remove trailing slash for consistent prefix matching
+	base = strings.TrimSuffix(base, "/")
+
+	var links []string
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			for _, attr := range n.Attr {
+				if attr.Key == "href" {
+					href := strings.TrimSpace(attr.Val)
+					if href == "" || strings.HasPrefix(href, "#") {
+						continue
+					}
+					absolute := resolveURL(href, base)
+					if absolute != "" {
+						links = append(links, absolute)
+					}
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+
+	// Deduplicate
+	seen := make(map[string]struct{}, len(links))
+	unique := make([]string, 0, len(links))
+	for _, l := range links {
+		if _, ok := seen[l]; ok {
+			continue
+		}
+		seen[l] = struct{}{}
+		unique = append(unique, l)
+	}
+	return unique
+}
+
+// resolveURL converts a relative URL to an absolute URL using the base URL.
+func resolveURL(href, base string) string {
+	href = strings.TrimSpace(href)
+	if href == "" {
+		return ""
+	}
+	// Already absolute
+	if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
+		return href
+	}
+	// Protocol-relative
+	if strings.HasPrefix(href, "//") {
+		return "https:" + href
+	}
+	// Absolute path
+	if strings.HasPrefix(href, "/") {
+		// Extract scheme://host from base
+		idx := strings.Index(base, "://")
+		if idx < 0 {
+			return ""
+		}
+		hostEnd := strings.IndexByte(base[idx+3:], '/')
+		if hostEnd < 0 {
+			return base + href
+		}
+		return base[:idx+3+hostEnd] + href
+	}
+	// Relative path
+	return base + "/" + href
+}

backend/internal/service/captain_document_page_parser_backend.go

@@ -0,0 +1,66 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// captainDocumentPageParserBackendImpl is the production implementation of
+// CaptainDocumentPageParserBackend. It fetches a single web page and
+// extracts its title and visible text content — the same logic used by
+// the sync backend for individual page URLs.
+type captainDocumentPageParserBackendImpl struct {
+	httpClient *http.Client
+}
+
+// NewCaptainDocumentPageParserBackend creates the production page parser backend.
+func NewCaptainDocumentPageParserBackend() CaptainDocumentPageParserBackend {
+	return &captainDocumentPageParserBackendImpl{
+		httpClient: &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+func (b *captainDocumentPageParserBackendImpl) ParseCaptainDocumentPage(ctx context.Context, pageLink string) (*CaptainDocumentSyncResult, error) {
+	if pageLink == "" {
+		return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageLink, nil)
+	if err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("User-Agent", "GoChat-Captain/1.0")
+
+	resp, err := b.httpClient.Do(req)
+	if err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, pageLink)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
+	if err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
+	}
+
+	title, content := extractHTMLText(string(body))
+	content = strings.TrimSpace(content)
+	if content == "" {
+		return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
+	}
+
+	if title == "" {
+		title = pageLink
+	}
+	return &CaptainDocumentSyncResult{
+		Content: content,
+		Title:   title,
+	}, nil
+}

backend/internal/service/captain_document_service.go

@@ -6,7 +6,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"mime/multipart"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -120,8 +123,11 @@ func (s *CaptainDocumentService) SetWorkerPool(wp *worker.WorkerPool) {
 // CreateDocumentRequest is the DTO for creating a document.
 type CreateDocumentRequest struct {
 	Name         string `json:"name" validate:"required"`
-	ExternalLink string `json:"external_link" validate:"required"`
+	ExternalLink string `json:"external_link"`
+	Content      string `json:"content"`
 	AssistantID  uint   `json:"assistant_id"`
+	// File upload fields (set by handler when multipart/form-data)
+	PdfFile     *multipart.FileHeader `json:"-"`
 }
 
 // UpdateDocumentRequest is the DTO for updating a document.
@@ -149,30 +155,146 @@ func (s *CaptainDocumentService) Create(ctx context.Context, assistantID, accoun
 			return nil, fmt.Errorf("assistant not found: %w", err)
 		}
 	}
+
+	// Validate: at least one source must be provided
+	if req.ExternalLink == "" && req.PdfFile == nil && strings.TrimSpace(req.Content) == "" {
+		return nil, errors.New("at least one of external_link, content, or pdf_file is required")
+	}
+
 	doc := &model.CaptainDocument{
 		AccountID:    accountID,
 		AssistantID:  assistantID,
 		Name:         req.Name,
 		ExternalLink: req.ExternalLink,
+		Content:      strings.TrimSpace(req.Content),
 		Status:       model.DocumentStatusPending,
 	}
 
+	// Handle PDF file upload
+	if req.PdfFile != nil {
+		fileURL, err := s.saveUploadedFile(ctx, accountID, req.PdfFile)
+		if err != nil {
+			return nil, fmt.Errorf("save pdf file: %w", err)
+		}
+		doc.FileURL = fileURL
+		doc.FileSize = req.PdfFile.Size
+		doc.ContentType = req.PdfFile.Header.Get("Content-Type")
+		if doc.ContentType == "" {
+			doc.ContentType = "application/pdf"
+		}
+		// If no external link was provided, use the uploaded file URL
+		if doc.ExternalLink == "" {
+			doc.ExternalLink = fileURL
+		}
+	}
+
 	if err := s.documentRepo.Create(ctx, doc); err != nil {
 		applogger.L().Errorf("Create captain document: %v", err)
 		return nil, fmt.Errorf("create document: %w", err)
 	}
-	if created, err := s.documentRepo.GetByAccountAndID(ctx, accountID, doc.ID); err == nil {
-		if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, created.ID); enqueueErr != nil {
+
+	// Route the document to the correct processing pipeline:
+	//   - PDF upload → sync job (PDF text extraction is the sync backend's job)
+	//   - URL       → crawl job (fetch page, discover links, parse)
+	//   - Text content → sync job (process content directly, no fetch needed)
+	docStatus := doc.Status
+	if doc.Content != "" && doc.ExternalLink == "" {
+		// Direct content input: mark as in_progress so the sync pipeline
+		// can pick it up and generate FAQ responses.
+		docStatus = model.DocumentStatusInProgress
+		doc.SyncStatus = model.DocumentSyncStatusPending
+		if err := s.documentRepo.Update(ctx, doc); err != nil {
+			applogger.L().Errorf("Update captain document status after create: %v", err)
+			return nil, fmt.Errorf("update document status: %w", err)
+		}
+		if s.worker != nil {
+			if _, err := s.worker.Enqueue(ctx, TaskTypeCaptainDocumentSync,
+				captainDocumentSyncJob{AccountID: accountID, DocumentID: doc.ID},
+				worker.WithQueue("low"), worker.WithMaxAttempts(3)); err != nil {
+				applogger.L().Warnf("enqueue document sync for content: %v", err)
+			}
+		}
+	} else if doc.FileURL != "" {
+		// PDF upload: enqueue sync job to extract text from the PDF file.
+		if s.worker != nil {
+			if _, err := s.worker.Enqueue(ctx, TaskTypeCaptainDocumentSync,
+				captainDocumentSyncJob{AccountID: accountID, DocumentID: doc.ID},
+				worker.WithQueue("low"), worker.WithMaxAttempts(3)); err != nil {
+				applogger.L().Warnf("enqueue document sync for pdf: %v", err)
+			}
+		}
+	} else {
+		// URL: enqueue crawl job to fetch/extract content
+		if created, err := s.documentRepo.GetByAccountAndID(ctx, accountID, doc.ID); err == nil {
+			if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, created.ID); enqueueErr != nil {
+				return nil, enqueueErr
+			}
+			return created, nil
+		}
+		if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, doc.ID); enqueueErr != nil {
 			return nil, enqueueErr
 		}
-		return created, nil
-	}
-	if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, doc.ID); enqueueErr != nil {
-		return nil, enqueueErr
 	}
+	_ = docStatus
 	return doc, nil
 }
 
+// saveUploadedFile saves an uploaded PDF file to local storage and returns the file URL.
+func (s *CaptainDocumentService) saveUploadedFile(ctx context.Context, accountID uint, fileHeader *multipart.FileHeader) (string, error) {
+	// Read the file content
+	src, err := fileHeader.Open()
+	if err != nil {
+		return "", fmt.Errorf("open uploaded file: %w", err)
+	}
+	defer src.Close()
+
+	// Generate a unique filename
+	ext := filepath.Ext(fileHeader.Filename)
+	if ext == "" {
+		ext = ".pdf"
+	}
+	timestamp := time.Now().UnixNano()
+	filename := fmt.Sprintf("captain_docs/%d/%d%s", accountID, timestamp, ext)
+	destPath := filepath.Join(s.uploadDir(), filename)
+
+	// Create directory if needed
+	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
+		return "", fmt.Errorf("create upload directory: %w", err)
+	}
+
+	// Write the file
+	dst, err := os.Create(destPath)
+	if err != nil {
+		return "", fmt.Errorf("create destination file: %w", err)
+	}
+	defer dst.Close()
+
+	if _, err := io.Copy(dst, src); err != nil {
+		return "", fmt.Errorf("write uploaded file: %w", err)
+	}
+
+	// Return the relative URL path (served by StaticFS at /uploads)
+	return fmt.Sprintf("/uploads/captain_docs/%d/%d%s", accountID, timestamp, ext), nil
+}
+
+// uploadDir returns the base directory for file uploads.
+// This should match the configured storage.local_path (default: ./uploads).
+func (s *CaptainDocumentService) uploadDir() string {
+	return "./uploads"
+}
+
+// enqueueDocumentProcess enqueues a job to process document content directly (for inline content input).
+func (s *CaptainDocumentService) enqueueDocumentProcess(ctx context.Context, accountID, docID uint) error {
+	if s.worker == nil {
+		return nil
+	}
+	_, err := s.worker.Enqueue(ctx, TaskTypeCaptainDocumentResponseBuilder, captainDocumentResponseBuilderJob{AccountID: accountID, DocumentID: docID},
+		worker.WithQueue("low"),
+		worker.WithMaxAttempts(3),
+	)
+	return err
+}
+
 // Get retrieves a document by ID.
 func (s *CaptainDocumentService) Get(ctx context.Context, id uint) (*model.CaptainDocument, error) {
 	doc, err := s.documentRepo.GetByID(ctx, id)
@@ -468,6 +590,28 @@ func (s *CaptainDocumentService) SyncDocumentByAccount(ctx context.Context, acco
 	if err := s.markDocumentSyncStarted(ctx, doc); err != nil {
 		return nil, err
 	}
+
+	// Direct content input: content is already present, skip fetch/extraction.
+	// Go straight to FAQ response building.
+	if strings.TrimSpace(doc.Content) != "" && doc.FileURL == "" && doc.ExternalLink == "" {
+		doc.Content = strings.TrimSpace(doc.Content)
+		doc.ContentFingerprint = computeFingerprint(doc.Content)
+		doc.Status = model.DocumentStatusCompleted
+		doc.SyncStatus = model.DocumentSyncStatusSynced
+		doc.LastSyncErrorCode = ""
+		now := time.Now().Unix()
+		doc.LastSyncedAt = &now
+		doc.LastSyncAttemptedAt = &now
+		if err := s.documentRepo.Update(ctx, doc); err != nil {
+			return nil, fmt.Errorf("update synced document: %w", err)
+		}
+		if err := s.enqueueDocumentResponseBuilder(ctx, accountID, id); err != nil {
+			return nil, err
+		}
+		return s.documentRepo.GetByAccountAndID(ctx, accountID, id)
+	}
+
+	// PDF and URL documents need a sync backend to fetch/extract content.
 	if s.syncBackend == nil {
 		return s.markDocumentSyncFailed(ctx, accountID, id, "sync_disabled")
 	}

backend/internal/service/captain_document_sync_backend.go

@@ -0,0 +1,175 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/gochat/gochat/internal/model"
+	"golang.org/x/net/html"
+)
+
+// captainDocumentSyncBackendImpl is the production implementation of
+// CaptainDocumentSyncBackend. It handles two document types:
+//   - PDF uploads: extracts text via the `pdftotext` CLI (poppler-utils),
+//     which correctly handles CJK/CID fonts that pure-Go PDF libraries cannot.
+//   - Web URLs:    fetches the page and extracts visible text from HTML.
+type captainDocumentSyncBackendImpl struct {
+	httpClient *http.Client
+}
+
+// NewCaptainDocumentSyncBackend creates the production sync backend.
+func NewCaptainDocumentSyncBackend() CaptainDocumentSyncBackend {
+	return &captainDocumentSyncBackendImpl{
+		httpClient: &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+func (b *captainDocumentSyncBackendImpl) SyncCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
+	// PDF document: extract text from the uploaded file.
+	if doc.ContentType == "application/pdf" || doc.FileURL != "" {
+		return b.syncPDFDocument(ctx, doc)
+	}
+	// Web URL document: fetch and extract text from the page.
+	if doc.ExternalLink != "" {
+		return b.syncWebDocument(ctx, doc)
+	}
+	return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
+}
+
+// syncPDFDocument extracts text content from a locally stored PDF file
+// using the `pdftotext` CLI (poppler-utils). This handles CJK fonts and
+// complex PDF encodings that pure-Go libraries like ledongthuc/pdf cannot.
+func (b *captainDocumentSyncBackendImpl) syncPDFDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
+	filePath := b.resolvePDFPath(doc)
+	if filePath == "" {
+		return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
+	}
+
+	// Use pdftotext CLI for robust text extraction (supports CJK).
+	cmd := exec.CommandContext(ctx, "pdftotext", "-enc", "UTF-8", filePath, "-")
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("pdftotext: %w, stderr: %s", err, stderr.String())
+	}
+
+	content := strings.TrimSpace(stdout.String())
+	if content == "" {
+		return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
+	}
+
+	title := doc.Name
+	return &CaptainDocumentSyncResult{
+		Content: content,
+		Title:   title,
+	}, nil
+}
+
+// resolvePDFPath converts the doc's file_url to a local filesystem path.
+// file_url is stored as "/uploads/captain_docs/<account>/<timestamp>.pdf"
+// and served from the local "./uploads" directory.
+func (b *captainDocumentSyncBackendImpl) resolvePDFPath(doc *model.CaptainDocument) string {
+	if doc.FileURL == "" {
+		return ""
+	}
+	// file_url is a relative URL path like "/uploads/captain_docs/1/123.pdf"
+	// Map it to the local filesystem path.
+	path := doc.FileURL
+	if strings.HasPrefix(path, "/uploads/") {
+		return "." + path
+	}
+	// If it's already a filesystem path, use it directly.
+	if _, err := os.Stat(path); err == nil {
+		return path
+	}
+	return ""
+}
+
+// syncWebDocument fetches a web page and extracts its visible text content.
+func (b *captainDocumentSyncBackendImpl) syncWebDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
+	url := doc.ExternalLink
+	if url == "" {
+		return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("User-Agent", "GoChat-Captain/1.0")
+
+	resp, err := b.httpClient.Do(req)
+	if err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
+	if err != nil {
+		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
+	}
+
+	title, content := extractHTMLText(string(body))
+	content = strings.TrimSpace(content)
+	if content == "" {
+		return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
+	}
+
+	if title == "" {
+		title = doc.Name
+	}
+	return &CaptainDocumentSyncResult{
+		Content: content,
+		Title:   title,
+	}, nil
+}
+
+// extractHTMLText parses an HTML document and returns the page title and
+// visible text content (stripping scripts, styles, and HTML tags).
+func extractHTMLText(htmlStr string) (title string, content string) {
+	doc, err := html.Parse(strings.NewReader(htmlStr))
+	if err != nil {
+		// Fall back to raw text if HTML parsing fails.
+		return "", strings.TrimSpace(htmlStr)
+	}
+
+	var extract func(*html.Node)
+	extract = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "script", "style", "noscript", "head":
+				return
+			case "title":
+				if n.FirstChild != nil {
+					title = strings.TrimSpace(n.FirstChild.Data)
+				}
+				return
+			}
+		}
+		if n.Type == html.TextNode {
+			text := strings.TrimSpace(n.Data)
+			if text != "" {
+				content += text + " "
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			extract(c)
+		}
+	}
+	extract(doc)
+
+	content = strings.TrimSpace(content)
+	return title, content
+}