gochat/backend/internal/service/captain_document_sync_backend.go

package service

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/gochat/gochat/internal/model"
	"golang.org/x/net/html"
)

// captainDocumentSyncBackendImpl is the production implementation of
// CaptainDocumentSyncBackend. It handles two document types:
//   - PDF uploads: extracts text via the `pdftotext` CLI (poppler-utils),
//     which correctly handles CJK/CID fonts that pure-Go PDF libraries cannot.
//   - Web URLs:    fetches the page and extracts visible text from HTML.
type captainDocumentSyncBackendImpl struct {
	httpClient *http.Client
}

// NewCaptainDocumentSyncBackend creates the production sync backend.
func NewCaptainDocumentSyncBackend() CaptainDocumentSyncBackend {
	return &captainDocumentSyncBackendImpl{
		httpClient: &http.Client{Timeout: 30 * time.Second},
	}
}

func (b *captainDocumentSyncBackendImpl) SyncCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
	// PDF document: extract text from the uploaded file.
	if doc.ContentType == "application/pdf" || doc.FileURL != "" {
		return b.syncPDFDocument(ctx, doc)
	}
	// Web URL document: fetch and extract text from the page.
	if doc.ExternalLink != "" {
		return b.syncWebDocument(ctx, doc)
	}
	return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
}

// syncPDFDocument extracts text content from a locally stored PDF file
// using the `pdftotext` CLI (poppler-utils). This handles CJK fonts and
// complex PDF encodings that pure-Go libraries like ledongthuc/pdf cannot.
func (b *captainDocumentSyncBackendImpl) syncPDFDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
	filePath := b.resolvePDFPath(doc)
	if filePath == "" {
		return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
	}

	// Use pdftotext CLI for robust text extraction (supports CJK).
	cmd := exec.CommandContext(ctx, "pdftotext", "-enc", "UTF-8", filePath, "-")
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("pdftotext: %w, stderr: %s", err, stderr.String())
	}

	content := strings.TrimSpace(stdout.String())
	if content == "" {
		return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
	}

	title := doc.Name
	return &CaptainDocumentSyncResult{
		Content: content,
		Title:   title,
	}, nil
}

// resolvePDFPath converts the doc's file_url to a local filesystem path.
// file_url is stored as "/uploads/captain_docs/<account>/<timestamp>.pdf"
// and served from the local "./uploads" directory.
func (b *captainDocumentSyncBackendImpl) resolvePDFPath(doc *model.CaptainDocument) string {
	if doc.FileURL == "" {
		return ""
	}
	// file_url is a relative URL path like "/uploads/captain_docs/1/123.pdf"
	// Map it to the local filesystem path.
	path := doc.FileURL
	if strings.HasPrefix(path, "/uploads/") {
		return "." + path
	}
	// If it's already a filesystem path, use it directly.
	if _, err := os.Stat(path); err == nil {
		return path
	}
	return ""
}

// syncWebDocument fetches a web page and extracts its visible text content.
func (b *captainDocumentSyncBackendImpl) syncWebDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
	url := doc.ExternalLink
	if url == "" {
		return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
	}
	req.Header.Set("User-Agent", "GoChat-Captain/1.0")

	resp, err := b.httpClient.Do(req)
	if err != nil {
		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
	}

	body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
	if err != nil {
		return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
	}

	title, content := extractHTMLText(string(body))
	content = strings.TrimSpace(content)
	if content == "" {
		return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
	}

	if title == "" {
		title = doc.Name
	}
	return &CaptainDocumentSyncResult{
		Content: content,
		Title:   title,
	}, nil
}

// extractHTMLText parses an HTML document and returns the page title and
// visible text content (stripping scripts, styles, and HTML tags).
func extractHTMLText(htmlStr string) (title string, content string) {
	doc, err := html.Parse(strings.NewReader(htmlStr))
	if err != nil {
		// Fall back to raw text if HTML parsing fails.
		return "", strings.TrimSpace(htmlStr)
	}

	var extract func(*html.Node)
	extract = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.Data {
			case "script", "style", "noscript", "head":
				return
			case "title":
				if n.FirstChild != nil {
					title = strings.TrimSpace(n.FirstChild.Data)
				}
				return
			}
		}
		if n.Type == html.TextNode {
			text := strings.TrimSpace(n.Data)
			if text != "" {
				content += text + " "
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			extract(c)
		}
	}
	extract(doc)

	content = strings.TrimSpace(content)
	return title, content
}