fix(ui): 自定义工具表单 overflow-y-scroll→auto + 认证方式NONE翻译改为'无' + Captain文档爬虫/同步后端
This commit is contained in:
Rogee
+5
-8
@@ -1,6 +1,6 @@
|
||||
module github.com/gochat/gochat
|
||||
|
||||
go 1.24.0
|
||||
go 1.24.1
|
||||
|
||||
toolchain go1.24.4
|
||||
|
||||
@@ -9,8 +9,6 @@ require (
|
||||
github.com/ThreeDotsLabs/watermill-redisstream v1.4.5
|
||||
github.com/alicebob/miniredis/v2 v2.38.0
|
||||
github.com/cloudwego/eino v0.9.12
|
||||
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260707093537-4921614ba2e8
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.1.13
|
||||
github.com/emersion/go-imap v1.2.1
|
||||
github.com/emersion/go-message v0.18.2
|
||||
github.com/fsnotify/fsnotify v1.7.0
|
||||
@@ -45,22 +43,22 @@ require (
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/bytedance/gopkg v0.1.3 // indirect
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/eino-contrib/jsonschema v1.0.3 // indirect
|
||||
github.com/evanphx/json-patch v0.5.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
||||
github.com/go-openapi/spec v0.20.4 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/go-sql-driver/mysql v1.8.1 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/goph/emperror v0.17.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/meguminnnnnnnnn/go-openai v0.1.2 // indirect
|
||||
github.com/nikolalohinski/gonja v1.5.3 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
|
||||
github.com/smarty/assertions v1.16.0 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/yargevad/filepathx v1.0.0 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
@@ -70,7 +68,6 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute/metadata v0.8.0 // indirect
|
||||
github.com/Rican7/retry v0.3.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/bytedance/sonic v1.15.0 // indirect
|
||||
@@ -126,7 +123,7 @@ require (
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/net v0.47.0
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA=
|
||||
cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
|
||||
@@ -39,8 +37,6 @@ github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqR
|
||||
github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
|
||||
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
|
||||
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
|
||||
github.com/bytedance/mockey v1.3.0 h1:ONLRdvhqmCfr9rTasUB8ZKCfvbdD2tohOg4u+4Q/ed0=
|
||||
github.com/bytedance/mockey v1.3.0/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY=
|
||||
github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE=
|
||||
github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k=
|
||||
github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE=
|
||||
@@ -52,12 +48,6 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI
|
||||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||
github.com/cloudwego/eino v0.9.12 h1:mHAMo5k7GdvnVD8Lc2sLyfpkxEm0S/y3PkEMhsSYt78=
|
||||
github.com/cloudwego/eino v0.9.12/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ=
|
||||
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260707093537-4921614ba2e8 h1:lLdhCtKZhun/8flC+RqFEN9y7EmoSUxnm5T5h/apCgI=
|
||||
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260707093537-4921614ba2e8/go.mod h1:zyPrZT2bO6LyRJgVksQowR18jVgyLSvqK93hnO53/Lc=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.1.13 h1:5XHRTiTD5bt9KQrMHcfvuWNklEC3tpm3XHejdozt9vM=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.1.13/go.mod h1:mgIoqYYOc0eECCqvLbEYpOJrQNTNxkwXzSJzFU+v5sQ=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 h1:EeVcR1TslRA2IdNW1h/2LaGbPlffwGhQm99jM3zWZiI=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17/go.mod h1:Zkcx6DPTR2NfWmtSXbhItswGw6hqUezNPhNcke0pOG8=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
@@ -91,8 +81,6 @@ github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0
|
||||
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
|
||||
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
|
||||
github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
|
||||
github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
|
||||
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
@@ -182,7 +170,6 @@ github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
|
||||
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
@@ -223,8 +210,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/meguminnnnnnnnn/go-openai v0.1.2 h1:iXombGGjqjBrmE9WaSidUhhi3YQhf42QTHvHLMkgvCA=
|
||||
github.com/meguminnnnnnnnn/go-openai v0.1.2/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
@@ -348,8 +333,6 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
||||
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
|
||||
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
|
||||
@@ -580,6 +580,9 @@ func Bootstrap(env string) (*App, error) {
|
||||
captainAssistantService := service.NewCaptainAssistantService(captainAssistantRepo, captainInboxRepo, captainDocumentRepo, captainAssistantResponseRepo, llmProvider, rdb)
|
||||
captainDocumentService := service.NewCaptainDocumentService(captainDocumentRepo, llmProvider, captainAssistantRepo)
|
||||
captainDocumentService.SetResponseRepo(captainAssistantResponseRepo)
|
||||
captainDocumentService.SetSyncBackend(service.NewCaptainDocumentSyncBackend())
|
||||
captainDocumentService.SetCrawlBackend(service.NewCaptainDocumentCrawlBackend())
|
||||
captainDocumentService.SetPageParserBackend(service.NewCaptainDocumentPageParserBackend())
|
||||
captainDocumentService.SetWorkerPool(workerPool)
|
||||
if _, err := service.EnqueueCaptainDocumentScheduleSyncs(context.Background(), workerPool, time.Now()); err != nil {
|
||||
applogger.L().Warnf("failed to enqueue initial Captain document sync scheduler: %v", err)
|
||||
|
||||
@@ -3,6 +3,7 @@ package v1
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
@@ -24,6 +25,7 @@ func NewCaptainDocumentHandler(svc *service.CaptainDocumentService) *CaptainDocu
|
||||
|
||||
// Create creates a new captain document.
|
||||
// POST /api/v1/accounts/:account_id/captain_assistants/:assistant_id/documents
|
||||
// Supports both JSON (for URL/content input) and multipart/form-data (for PDF upload).
|
||||
func (h *CaptainDocumentHandler) Create(c *gin.Context) {
|
||||
accountID := parseAccountIDParam(c)
|
||||
if accountID == 0 {
|
||||
@@ -32,10 +34,31 @@ func (h *CaptainDocumentHandler) Create(c *gin.Context) {
|
||||
}
|
||||
|
||||
var req service.CreateDocumentRequest
|
||||
if err := bindNestedJSONPayload(c, "document", &req); err != nil {
|
||||
response.AbortWithStatusError(c, http.StatusBadRequest, response.ErrValidation, err.Error())
|
||||
return
|
||||
|
||||
contentType := c.GetHeader("Content-Type")
|
||||
if strings.Contains(contentType, "multipart/form-data") {
|
||||
// Multipart form-data: PDF file upload or form fields
|
||||
req.Name = c.PostForm("document[name]")
|
||||
req.ExternalLink = c.PostForm("document[external_link]")
|
||||
req.Content = c.PostForm("document[content]")
|
||||
if assistantIDStr := c.PostForm("document[assistant_id]"); assistantIDStr != "" {
|
||||
if id, err := strconv.ParseUint(assistantIDStr, 10, 64); err == nil {
|
||||
req.AssistantID = uint(id)
|
||||
}
|
||||
}
|
||||
|
||||
// Check for PDF file
|
||||
if fileHeader, err := c.FormFile("document[pdf_file]"); err == nil {
|
||||
req.PdfFile = fileHeader
|
||||
}
|
||||
} else {
|
||||
// JSON request
|
||||
if err := bindNestedJSONPayload(c, "document", &req); err != nil {
|
||||
response.AbortWithStatusError(c, http.StatusBadRequest, response.ErrValidation, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
assistantID := req.AssistantID
|
||||
if assistantID == 0 {
|
||||
assistantID, _ = parseUintParam(c, "assistant_id")
|
||||
@@ -48,7 +71,7 @@ func (h *CaptainDocumentHandler) Create(c *gin.Context) {
|
||||
doc, err := h.svc.Create(c.Request.Context(), assistantID, accountID, &req)
|
||||
if err != nil {
|
||||
applogger.L().Errorf("Create captain document: %v", err)
|
||||
response.AbortWithStatusError(c, http.StatusUnprocessableEntity, response.ErrValidation, "failed to create document")
|
||||
response.AbortWithStatusError(c, http.StatusUnprocessableEntity, response.ErrValidation, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -212,16 +235,18 @@ func captainDocumentPayload(doc *model.CaptainDocument) gin.H {
|
||||
if syncStatus == model.DocumentSyncStatusPending {
|
||||
syncStatus = "syncing"
|
||||
}
|
||||
pdfDocument := doc.ContentType == "application/pdf" || doc.FileURL != ""
|
||||
payload := gin.H{
|
||||
"account_id": doc.AccountID,
|
||||
"assistant": captainDocumentAssistantPayload(doc),
|
||||
"content": doc.Content,
|
||||
"content_type": "text/html",
|
||||
"content_type": doc.ContentType,
|
||||
"created_at": doc.CreatedAt.Unix(),
|
||||
"external_link": doc.ExternalLink,
|
||||
"display_url": doc.ExternalLink,
|
||||
"file_size": 0,
|
||||
"pdf_document": false,
|
||||
"file_size": doc.FileSize,
|
||||
"file_url": doc.FileURL,
|
||||
"pdf_document": pdfDocument,
|
||||
"id": doc.ID,
|
||||
"name": doc.Name,
|
||||
"status": status,
|
||||
|
||||
@@ -182,7 +182,7 @@ type CaptainDocument struct {
|
||||
AccountID uint `gorm:"index;not null" json:"account_id"`
|
||||
AssistantID uint `gorm:"index;not null" json:"assistant_id"`
|
||||
Name string `gorm:"size:255" json:"name"`
|
||||
ExternalLink string `gorm:"size:2048;not null" json:"external_link"`
|
||||
ExternalLink string `gorm:"size:2048" json:"external_link"`
|
||||
Content string `gorm:"type:text" json:"content,omitempty"`
|
||||
ContentFingerprint string `gorm:"size:64" json:"content_fingerprint,omitempty"`
|
||||
Status DocumentStatus `gorm:"size:50;default:in_progress;not null" json:"status"`
|
||||
@@ -192,6 +192,11 @@ type CaptainDocument struct {
|
||||
LastSyncErrorCode string `gorm:"size:50" json:"last_sync_error_code,omitempty"`
|
||||
Metadata json.RawMessage `gorm:"type:jsonb;serializer:json" json:"metadata,omitempty"`
|
||||
|
||||
// File upload metadata (PDF support)
|
||||
FileSize int64 `gorm:"default:0" json:"file_size,omitempty"`
|
||||
ContentType string `gorm:"size:100" json:"content_type,omitempty"`
|
||||
FileURL string `gorm:"size:2048" json:"file_url,omitempty"`
|
||||
|
||||
// Relationships
|
||||
Assistant CaptainAssistant `gorm:"foreignKey:AssistantID" json:"assistant,omitempty"`
|
||||
Responses []CaptainAssistantResponse `gorm:"foreignKey:DocumentableID" json:"responses,omitempty"`
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
package pgvector
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Vector represents a vector for similarity search.
|
||||
// Stub implementation for building without PostgreSQL/pgvector extension.
|
||||
type Vector []float32
|
||||
@@ -17,4 +26,89 @@ func (v Vector) String() string {
|
||||
// Dimensions returns the number of dimensions in the vector.
|
||||
func (v Vector) Dimensions() int {
|
||||
return len(v)
|
||||
}
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner so GORM/database/sql can scan vector columns
|
||||
// from PostgreSQL into the Vector type.
|
||||
//
|
||||
// PostgreSQL pgvector returns the vector as a string like "[0.1,0.2,0.3]"
|
||||
// via text protocol, or as raw bytes via binary protocol.
|
||||
// We handle both cases.
|
||||
func (v *Vector) Scan(src any) error {
|
||||
if src == nil {
|
||||
*v = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
switch val := src.(type) {
|
||||
case string:
|
||||
return v.parseVectorString(val)
|
||||
case []byte:
|
||||
// Try text protocol first: "[0.1,0.2,...]"
|
||||
s := string(val)
|
||||
if strings.HasPrefix(s, "[") {
|
||||
return v.parseVectorString(s)
|
||||
}
|
||||
// Binary protocol: pgvector binary format
|
||||
// Header: 2 bytes unused (version), 2 bytes dim count, then dim*4 bytes float32 big-endian
|
||||
return v.parseVectorBinary(val)
|
||||
default:
|
||||
return fmt.Errorf("pgvector: cannot scan %T into Vector", src)
|
||||
}
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer so GORM/database/sql can use Vector
|
||||
// in parameter binding. Returns the PostgreSQL text representation.
|
||||
func (v Vector) Value() (driver.Value, error) {
|
||||
if len(v) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
strs := make([]string, len(v))
|
||||
for i, f := range v {
|
||||
strs[i] = strconv.FormatFloat(float64(f), 'f', -1, 32)
|
||||
}
|
||||
return "[" + strings.Join(strs, ",") + "]", nil
|
||||
}
|
||||
|
||||
// parseVectorString parses a PostgreSQL vector text representation "[0.1,0.2,0.3]".
|
||||
func (v *Vector) parseVectorString(s string) error {
|
||||
s = strings.TrimSpace(s)
|
||||
s = strings.TrimPrefix(s, "[")
|
||||
s = strings.TrimSuffix(s, "]")
|
||||
if s == "" {
|
||||
*v = Vector{}
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
result := make(Vector, len(parts))
|
||||
for i, part := range parts {
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(part), 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pgvector: parse float %q: %w", part, err)
|
||||
}
|
||||
result[i] = float32(f)
|
||||
}
|
||||
*v = result
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseVectorBinary parses pgvector binary format.
|
||||
// Format: 2 bytes (version, currently 0), 2 bytes (dims), then dims * 4 bytes float32.
|
||||
func (v *Vector) parseVectorBinary(data []byte) error {
|
||||
if len(data) < 4 {
|
||||
return fmt.Errorf("pgvector: binary vector too short (%d bytes)", len(data))
|
||||
}
|
||||
dims := int(binary.BigEndian.Uint16(data[2:4]))
|
||||
expected := 4 + dims*4
|
||||
if len(data) < expected {
|
||||
return fmt.Errorf("pgvector: binary vector expected %d bytes, got %d", expected, len(data))
|
||||
}
|
||||
result := make(Vector, dims)
|
||||
for i := 0; i < dims; i++ {
|
||||
offset := 4 + i*4
|
||||
bits := binary.BigEndian.Uint32(data[offset : offset+4])
|
||||
result[i] = math.Float32frombits(bits)
|
||||
}
|
||||
*v = result
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -84,9 +84,14 @@ func (r *CaptainDocumentRepo) ListByAccount(ctx context.Context, accountID uint,
|
||||
}
|
||||
switch filters.Source {
|
||||
case "web":
|
||||
db = db.Where("external_link <> ''")
|
||||
// Web documents: have external_link but not a PDF.
|
||||
db = db.Where("(content_type IS NULL OR content_type = '' OR content_type != 'application/pdf') AND (file_url = '' OR file_url IS NULL) AND external_link <> ''")
|
||||
case "pdf":
|
||||
db = db.Where("external_link = ''")
|
||||
// PDF documents: identified by content_type or file_url.
|
||||
db = db.Where("content_type = 'application/pdf' OR (file_url IS NOT NULL AND file_url <> '')")
|
||||
case "text":
|
||||
// Text documents: inline content, no external link and no file.
|
||||
db = db.Where("(external_link = '' OR external_link IS NULL) AND (file_url = '' OR file_url IS NULL) AND content <> ''")
|
||||
}
|
||||
switch filters.Filter {
|
||||
case "syncing":
|
||||
@@ -95,6 +100,11 @@ func (r *CaptainDocumentRepo) ListByAccount(ctx context.Context, accountID uint,
|
||||
db = db.Where("sync_status = ?", model.DocumentSyncStatusFailed)
|
||||
case "synced":
|
||||
db = db.Where("sync_status = ?", model.DocumentSyncStatusSynced)
|
||||
case "stale":
|
||||
// Documents that need updating: never synced or explicitly marked stale,
|
||||
// excluding those currently syncing.
|
||||
db = db.Where("(last_synced_at IS NULL OR sync_status = ?) AND sync_status != ?",
|
||||
model.DocumentSyncStatusStale, model.DocumentSyncStatusPending)
|
||||
}
|
||||
if filters.SearchKey != "" {
|
||||
query := "%" + strings.ToLower(filters.SearchKey) + "%"
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// captainDocumentCrawlBackendImpl is the production implementation of
|
||||
// CaptainDocumentCrawlBackend. It fetches the document's external URL,
|
||||
// extracts page links for crawl fan-out.
|
||||
type captainDocumentCrawlBackendImpl struct {
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewCaptainDocumentCrawlBackend creates the production crawl backend.
|
||||
func NewCaptainDocumentCrawlBackend() CaptainDocumentCrawlBackend {
|
||||
return &captainDocumentCrawlBackendImpl{
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (b *captainDocumentCrawlBackendImpl) CrawlCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentCrawlResult, error) {
|
||||
url := doc.ExternalLink
|
||||
if url == "" {
|
||||
return &CaptainDocumentCrawlResult{ErrorCode: "not_found"}, nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "GoChat-Captain/1.0")
|
||||
|
||||
resp, err := b.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
|
||||
if err != nil {
|
||||
return &CaptainDocumentCrawlResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
|
||||
links := extractPageLinks(string(body), url)
|
||||
return &CaptainDocumentCrawlResult{
|
||||
PageLinks: links,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// extractPageLinks parses an HTML document and returns all unique absolute
|
||||
// href links found in <a> tags.
|
||||
func extractPageLinks(htmlStr, baseURL string) []string {
|
||||
doc, err := html.Parse(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
base := baseURL
|
||||
// Remove trailing slash for consistent prefix matching
|
||||
base = strings.TrimSuffix(base, "/")
|
||||
|
||||
var links []string
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode && n.Data == "a" {
|
||||
for _, attr := range n.Attr {
|
||||
if attr.Key == "href" {
|
||||
href := strings.TrimSpace(attr.Val)
|
||||
if href == "" || strings.HasPrefix(href, "#") {
|
||||
continue
|
||||
}
|
||||
absolute := resolveURL(href, base)
|
||||
if absolute != "" {
|
||||
links = append(links, absolute)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
|
||||
// Deduplicate
|
||||
seen := make(map[string]struct{}, len(links))
|
||||
unique := make([]string, 0, len(links))
|
||||
for _, l := range links {
|
||||
if _, ok := seen[l]; ok {
|
||||
continue
|
||||
}
|
||||
seen[l] = struct{}{}
|
||||
unique = append(unique, l)
|
||||
}
|
||||
return unique
|
||||
}
|
||||
|
||||
// resolveURL converts a relative URL to an absolute URL using the base URL.
|
||||
func resolveURL(href, base string) string {
|
||||
href = strings.TrimSpace(href)
|
||||
if href == "" {
|
||||
return ""
|
||||
}
|
||||
// Already absolute
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
// Protocol-relative
|
||||
if strings.HasPrefix(href, "//") {
|
||||
return "https:" + href
|
||||
}
|
||||
// Absolute path
|
||||
if strings.HasPrefix(href, "/") {
|
||||
// Extract scheme://host from base
|
||||
idx := strings.Index(base, "://")
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
hostEnd := strings.IndexByte(base[idx+3:], '/')
|
||||
if hostEnd < 0 {
|
||||
return base + href
|
||||
}
|
||||
return base[:idx+3+hostEnd] + href
|
||||
}
|
||||
// Relative path
|
||||
return base + "/" + href
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// captainDocumentPageParserBackendImpl is the production implementation of
|
||||
// CaptainDocumentPageParserBackend. It fetches a single web page and
|
||||
// extracts its title and visible text content — the same logic used by
|
||||
// the sync backend for individual page URLs.
|
||||
type captainDocumentPageParserBackendImpl struct {
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewCaptainDocumentPageParserBackend creates the production page parser backend.
|
||||
func NewCaptainDocumentPageParserBackend() CaptainDocumentPageParserBackend {
|
||||
return &captainDocumentPageParserBackendImpl{
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (b *captainDocumentPageParserBackendImpl) ParseCaptainDocumentPage(ctx context.Context, pageLink string) (*CaptainDocumentSyncResult, error) {
|
||||
if pageLink == "" {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageLink, nil)
|
||||
if err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "GoChat-Captain/1.0")
|
||||
|
||||
resp, err := b.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, pageLink)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
|
||||
if err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
|
||||
title, content := extractHTMLText(string(body))
|
||||
content = strings.TrimSpace(content)
|
||||
if content == "" {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
|
||||
}
|
||||
|
||||
if title == "" {
|
||||
title = pageLink
|
||||
}
|
||||
return &CaptainDocumentSyncResult{
|
||||
Content: content,
|
||||
Title: title,
|
||||
}, nil
|
||||
}
|
||||
@@ -6,7 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -120,8 +123,11 @@ func (s *CaptainDocumentService) SetWorkerPool(wp *worker.WorkerPool) {
|
||||
// CreateDocumentRequest is the DTO for creating a document.
|
||||
type CreateDocumentRequest struct {
|
||||
Name string `json:"name" validate:"required"`
|
||||
ExternalLink string `json:"external_link" validate:"required"`
|
||||
ExternalLink string `json:"external_link"`
|
||||
Content string `json:"content"`
|
||||
AssistantID uint `json:"assistant_id"`
|
||||
// File upload fields (set by handler when multipart/form-data)
|
||||
PdfFile *multipart.FileHeader `json:"-"`
|
||||
}
|
||||
|
||||
// UpdateDocumentRequest is the DTO for updating a document.
|
||||
@@ -149,30 +155,146 @@ func (s *CaptainDocumentService) Create(ctx context.Context, assistantID, accoun
|
||||
return nil, fmt.Errorf("assistant not found: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate: at least one source must be provided
|
||||
if req.ExternalLink == "" && req.PdfFile == nil && strings.TrimSpace(req.Content) == "" {
|
||||
return nil, errors.New("at least one of external_link, content, or pdf_file is required")
|
||||
}
|
||||
|
||||
doc := &model.CaptainDocument{
|
||||
AccountID: accountID,
|
||||
AssistantID: assistantID,
|
||||
Name: req.Name,
|
||||
ExternalLink: req.ExternalLink,
|
||||
Content: strings.TrimSpace(req.Content),
|
||||
Status: model.DocumentStatusPending,
|
||||
}
|
||||
|
||||
// Handle PDF file upload
|
||||
if req.PdfFile != nil {
|
||||
fileURL, err := s.saveUploadedFile(ctx, accountID, req.PdfFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("save pdf file: %w", err)
|
||||
}
|
||||
doc.FileURL = fileURL
|
||||
doc.FileSize = req.PdfFile.Size
|
||||
doc.ContentType = req.PdfFile.Header.Get("Content-Type")
|
||||
if doc.ContentType == "" {
|
||||
doc.ContentType = "application/pdf"
|
||||
}
|
||||
// If no external link was provided, use the uploaded file URL
|
||||
if doc.ExternalLink == "" {
|
||||
doc.ExternalLink = fileURL
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.documentRepo.Create(ctx, doc); err != nil {
|
||||
applogger.L().Errorf("Create captain document: %v", err)
|
||||
return nil, fmt.Errorf("create document: %w", err)
|
||||
}
|
||||
if created, err := s.documentRepo.GetByAccountAndID(ctx, accountID, doc.ID); err == nil {
|
||||
if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, created.ID); enqueueErr != nil {
|
||||
|
||||
// Route the document to the correct processing pipeline:
|
||||
// - PDF upload → sync job (PDF text extraction is the sync backend's job)
|
||||
// - URL → crawl job (fetch page, discover links, parse)
|
||||
// - Text content → sync job (process content directly, no fetch needed)
|
||||
docStatus := doc.Status
|
||||
if doc.Content != "" && doc.ExternalLink == "" {
|
||||
// Direct content input: mark as in_progress so the sync pipeline
|
||||
// can pick it up and generate FAQ responses.
|
||||
docStatus = model.DocumentStatusInProgress
|
||||
doc.SyncStatus = model.DocumentSyncStatusPending
|
||||
if err := s.documentRepo.Update(ctx, doc); err != nil {
|
||||
applogger.L().Errorf("Update captain document status after create: %v", err)
|
||||
return nil, fmt.Errorf("update document status: %w", err)
|
||||
}
|
||||
if s.worker != nil {
|
||||
if _, err := s.worker.Enqueue(ctx, TaskTypeCaptainDocumentSync,
|
||||
captainDocumentSyncJob{AccountID: accountID, DocumentID: doc.ID},
|
||||
worker.WithQueue("low"), worker.WithMaxAttempts(3)); err != nil {
|
||||
applogger.L().Warnf("enqueue document sync for content: %v", err)
|
||||
}
|
||||
}
|
||||
} else if doc.FileURL != "" {
|
||||
// PDF upload: enqueue sync job to extract text from the PDF file.
|
||||
if s.worker != nil {
|
||||
if _, err := s.worker.Enqueue(ctx, TaskTypeCaptainDocumentSync,
|
||||
captainDocumentSyncJob{AccountID: accountID, DocumentID: doc.ID},
|
||||
worker.WithQueue("low"), worker.WithMaxAttempts(3)); err != nil {
|
||||
applogger.L().Warnf("enqueue document sync for pdf: %v", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// URL: enqueue crawl job to fetch/extract content
|
||||
if created, err := s.documentRepo.GetByAccountAndID(ctx, accountID, doc.ID); err == nil {
|
||||
if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, created.ID); enqueueErr != nil {
|
||||
return nil, enqueueErr
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, doc.ID); enqueueErr != nil {
|
||||
return nil, enqueueErr
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
if enqueueErr := s.enqueueDocumentCrawl(ctx, accountID, doc.ID); enqueueErr != nil {
|
||||
return nil, enqueueErr
|
||||
}
|
||||
_ = docStatus
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
// saveUploadedFile saves an uploaded PDF file to local storage and returns the file URL.
|
||||
func (s *CaptainDocumentService) saveUploadedFile(ctx context.Context, accountID uint, fileHeader *multipart.FileHeader) (string, error) {
|
||||
// Read the file content
|
||||
src, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("open uploaded file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Generate a unique filename
|
||||
ext := filepath.Ext(fileHeader.Filename)
|
||||
if ext == "" {
|
||||
ext = ".pdf"
|
||||
}
|
||||
timestamp := time.Now().UnixNano()
|
||||
filename := fmt.Sprintf("captain_docs/%d/%d%s", accountID, timestamp, ext)
|
||||
destPath := filepath.Join(s.uploadDir(), filename)
|
||||
|
||||
// Create directory if needed
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
|
||||
return "", fmt.Errorf("create upload directory: %w", err)
|
||||
}
|
||||
|
||||
// Write the file
|
||||
dst, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create destination file: %w", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
return "", fmt.Errorf("write uploaded file: %w", err)
|
||||
}
|
||||
|
||||
// Return the relative URL path (served by StaticFS at /uploads)
|
||||
return fmt.Sprintf("/uploads/captain_docs/%d/%d%s", accountID, timestamp, ext), nil
|
||||
}
|
||||
|
||||
// uploadDir returns the base directory for file uploads.
|
||||
// This should match the configured storage.local_path (default: ./uploads).
|
||||
func (s *CaptainDocumentService) uploadDir() string {
|
||||
return "./uploads"
|
||||
}
|
||||
|
||||
// enqueueDocumentProcess enqueues a job to process document content directly (for inline content input).
|
||||
func (s *CaptainDocumentService) enqueueDocumentProcess(ctx context.Context, accountID, docID uint) error {
|
||||
if s.worker == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := s.worker.Enqueue(ctx, TaskTypeCaptainDocumentResponseBuilder, captainDocumentResponseBuilderJob{AccountID: accountID, DocumentID: docID},
|
||||
worker.WithQueue("low"),
|
||||
worker.WithMaxAttempts(3),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get retrieves a document by ID.
|
||||
func (s *CaptainDocumentService) Get(ctx context.Context, id uint) (*model.CaptainDocument, error) {
|
||||
doc, err := s.documentRepo.GetByID(ctx, id)
|
||||
@@ -468,6 +590,28 @@ func (s *CaptainDocumentService) SyncDocumentByAccount(ctx context.Context, acco
|
||||
if err := s.markDocumentSyncStarted(ctx, doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Direct content input: content is already present, skip fetch/extraction.
|
||||
// Go straight to FAQ response building.
|
||||
if strings.TrimSpace(doc.Content) != "" && doc.FileURL == "" && doc.ExternalLink == "" {
|
||||
doc.Content = strings.TrimSpace(doc.Content)
|
||||
doc.ContentFingerprint = computeFingerprint(doc.Content)
|
||||
doc.Status = model.DocumentStatusCompleted
|
||||
doc.SyncStatus = model.DocumentSyncStatusSynced
|
||||
doc.LastSyncErrorCode = ""
|
||||
now := time.Now().Unix()
|
||||
doc.LastSyncedAt = &now
|
||||
doc.LastSyncAttemptedAt = &now
|
||||
if err := s.documentRepo.Update(ctx, doc); err != nil {
|
||||
return nil, fmt.Errorf("update synced document: %w", err)
|
||||
}
|
||||
if err := s.enqueueDocumentResponseBuilder(ctx, accountID, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.documentRepo.GetByAccountAndID(ctx, accountID, id)
|
||||
}
|
||||
|
||||
// PDF and URL documents need a sync backend to fetch/extract content.
|
||||
if s.syncBackend == nil {
|
||||
return s.markDocumentSyncFailed(ctx, accountID, id, "sync_disabled")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// captainDocumentSyncBackendImpl is the production implementation of
|
||||
// CaptainDocumentSyncBackend. It handles two document types:
|
||||
// - PDF uploads: extracts text via the `pdftotext` CLI (poppler-utils),
|
||||
// which correctly handles CJK/CID fonts that pure-Go PDF libraries cannot.
|
||||
// - Web URLs: fetches the page and extracts visible text from HTML.
|
||||
type captainDocumentSyncBackendImpl struct {
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewCaptainDocumentSyncBackend creates the production sync backend.
|
||||
func NewCaptainDocumentSyncBackend() CaptainDocumentSyncBackend {
|
||||
return &captainDocumentSyncBackendImpl{
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (b *captainDocumentSyncBackendImpl) SyncCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
|
||||
// PDF document: extract text from the uploaded file.
|
||||
if doc.ContentType == "application/pdf" || doc.FileURL != "" {
|
||||
return b.syncPDFDocument(ctx, doc)
|
||||
}
|
||||
// Web URL document: fetch and extract text from the page.
|
||||
if doc.ExternalLink != "" {
|
||||
return b.syncWebDocument(ctx, doc)
|
||||
}
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
|
||||
}
|
||||
|
||||
// syncPDFDocument extracts text content from a locally stored PDF file
|
||||
// using the `pdftotext` CLI (poppler-utils). This handles CJK fonts and
|
||||
// complex PDF encodings that pure-Go libraries like ledongthuc/pdf cannot.
|
||||
func (b *captainDocumentSyncBackendImpl) syncPDFDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
|
||||
filePath := b.resolvePDFPath(doc)
|
||||
if filePath == "" {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
|
||||
}
|
||||
|
||||
// Use pdftotext CLI for robust text extraction (supports CJK).
|
||||
cmd := exec.CommandContext(ctx, "pdftotext", "-enc", "UTF-8", filePath, "-")
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("pdftotext: %w, stderr: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
content := strings.TrimSpace(stdout.String())
|
||||
if content == "" {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
|
||||
}
|
||||
|
||||
title := doc.Name
|
||||
return &CaptainDocumentSyncResult{
|
||||
Content: content,
|
||||
Title: title,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// resolvePDFPath converts the doc's file_url to a local filesystem path.
|
||||
// file_url is stored as "/uploads/captain_docs/<account>/<timestamp>.pdf"
|
||||
// and served from the local "./uploads" directory.
|
||||
func (b *captainDocumentSyncBackendImpl) resolvePDFPath(doc *model.CaptainDocument) string {
|
||||
if doc.FileURL == "" {
|
||||
return ""
|
||||
}
|
||||
// file_url is a relative URL path like "/uploads/captain_docs/1/123.pdf"
|
||||
// Map it to the local filesystem path.
|
||||
path := doc.FileURL
|
||||
if strings.HasPrefix(path, "/uploads/") {
|
||||
return "." + path
|
||||
}
|
||||
// If it's already a filesystem path, use it directly.
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// syncWebDocument fetches a web page and extracts its visible text content.
|
||||
func (b *captainDocumentSyncBackendImpl) syncWebDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) {
|
||||
url := doc.ExternalLink
|
||||
if url == "" {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "not_found"}, nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "GoChat-Captain/1.0")
|
||||
|
||||
resp, err := b.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("fetch url: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5MB max
|
||||
if err != nil {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "fetch_failed"}, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
|
||||
title, content := extractHTMLText(string(body))
|
||||
content = strings.TrimSpace(content)
|
||||
if content == "" {
|
||||
return &CaptainDocumentSyncResult{ErrorCode: "content_empty"}, nil
|
||||
}
|
||||
|
||||
if title == "" {
|
||||
title = doc.Name
|
||||
}
|
||||
return &CaptainDocumentSyncResult{
|
||||
Content: content,
|
||||
Title: title,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// extractHTMLText parses an HTML document and returns the page title and
|
||||
// visible text content (stripping scripts, styles, and HTML tags).
|
||||
func extractHTMLText(htmlStr string) (title string, content string) {
|
||||
doc, err := html.Parse(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
// Fall back to raw text if HTML parsing fails.
|
||||
return "", strings.TrimSpace(htmlStr)
|
||||
}
|
||||
|
||||
var extract func(*html.Node)
|
||||
extract = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "script", "style", "noscript", "head":
|
||||
return
|
||||
case "title":
|
||||
if n.FirstChild != nil {
|
||||
title = strings.TrimSpace(n.FirstChild.Data)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
if n.Type == html.TextNode {
|
||||
text := strings.TrimSpace(n.Data)
|
||||
if text != "" {
|
||||
content += text + " "
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
extract(c)
|
||||
}
|
||||
}
|
||||
extract(doc)
|
||||
|
||||
content = strings.TrimSpace(content)
|
||||
return title, content
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE captain_documents DROP COLUMN IF EXISTS file_size;
|
||||
ALTER TABLE captain_documents DROP COLUMN IF EXISTS content_type;
|
||||
ALTER TABLE captain_documents DROP COLUMN IF EXISTS file_url;
|
||||
ALTER TABLE captain_documents ALTER COLUMN external_link SET NOT NULL;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Add file upload metadata fields to captain_documents for PDF support
|
||||
ALTER TABLE captain_documents ADD COLUMN IF NOT EXISTS file_size BIGINT DEFAULT 0;
|
||||
ALTER TABLE captain_documents ADD COLUMN IF NOT EXISTS content_type VARCHAR(100) DEFAULT '';
|
||||
ALTER TABLE captain_documents ADD COLUMN IF NOT EXISTS file_url VARCHAR(2048) DEFAULT '';
|
||||
|
||||
-- Allow external_link to be nullable (for direct content input documents)
|
||||
ALTER TABLE captain_documents ALTER COLUMN external_link DROP NOT NULL;
|
||||
Binary file not shown.
@@ -25,13 +25,11 @@ const emitChange = () => emit('change');
|
||||
|
||||
const handleSourceSelect = sourceKey => {
|
||||
source.value = sourceKey;
|
||||
if (sourceKey !== 'web') status.value = null;
|
||||
emitChange();
|
||||
};
|
||||
|
||||
const handleStatusSelect = statusKey => {
|
||||
status.value = statusKey;
|
||||
if (statusKey) source.value = 'web';
|
||||
emitChange();
|
||||
};
|
||||
|
||||
|
||||
+2
-3
@@ -28,6 +28,7 @@ const MENU_CONFIG = [
|
||||
{ labelKey: 'SOURCE.ALL', value: 'all', icon: 'i-lucide-files' },
|
||||
{ labelKey: 'SOURCE.WEB', value: 'web', icon: 'i-lucide-link' },
|
||||
{ labelKey: 'SOURCE.PDF', value: 'pdf', icon: 'i-lucide-file-text' },
|
||||
{ labelKey: 'SOURCE.TEXT', value: 'text', icon: 'i-lucide-type' },
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -78,9 +79,7 @@ const MENU_CONFIG = [
|
||||
];
|
||||
|
||||
const filterMenus = computed(() =>
|
||||
MENU_CONFIG.filter(
|
||||
menu => !(menu.key === 'status' && props.activeSourceFilter === 'pdf')
|
||||
).map(menu => {
|
||||
MENU_CONFIG.map(menu => {
|
||||
const active = props[menu.activeKey];
|
||||
const items = menu.options.map(opt => ({
|
||||
label: t(`CAPTAIN.DOCUMENTS.FILTERS.${opt.labelKey}`),
|
||||
|
||||
+1
-1
@@ -179,7 +179,7 @@ const handleTest = async () => {
|
||||
|
||||
<template>
|
||||
<form
|
||||
class="flex flex-col px-4 -mx-4 gap-4 max-h-[calc(100vh-200px)] overflow-y-scroll"
|
||||
class="flex flex-col px-4 -mx-4 gap-4 max-h-[calc(100vh-200px)] overflow-y-auto"
|
||||
@submit.prevent="handleSubmit"
|
||||
>
|
||||
<Input
|
||||
|
||||
+35
-1
@@ -30,6 +30,7 @@ const formState = {
|
||||
const initialState = {
|
||||
name: '',
|
||||
url: '',
|
||||
content: '',
|
||||
documentType: 'url',
|
||||
pdfFile: null,
|
||||
};
|
||||
@@ -46,11 +47,18 @@ const validationRules = {
|
||||
pdfFile: {
|
||||
required: requiredIf(() => state.documentType === 'pdf'),
|
||||
},
|
||||
content: {
|
||||
required: requiredIf(() => state.documentType === 'content'),
|
||||
minLength: requiredIf(
|
||||
() => state.documentType === 'content' && minLength(10)
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
const documentTypeOptions = [
|
||||
{ value: 'url', label: t('CAPTAIN.DOCUMENTS.FORM.TYPE.URL') },
|
||||
{ value: 'pdf', label: t('CAPTAIN.DOCUMENTS.FORM.TYPE.PDF') },
|
||||
{ value: 'content', label: t('CAPTAIN.DOCUMENTS.FORM.TYPE.CONTENT') },
|
||||
];
|
||||
|
||||
const v$ = useVuelidate(validationRules, state);
|
||||
@@ -58,6 +66,7 @@ const v$ = useVuelidate(validationRules, state);
|
||||
const isLoading = computed(() => formState.uiFlags.value.creatingItem);
|
||||
|
||||
const hasPdfFileError = computed(() => v$.value.pdfFile.$error);
|
||||
const hasContentError = computed(() => v$.value.content.$error);
|
||||
|
||||
const getErrorMessage = (field, errorKey) => {
|
||||
return v$.value[field].$error
|
||||
@@ -68,6 +77,7 @@ const getErrorMessage = (field, errorKey) => {
|
||||
const formErrors = computed(() => ({
|
||||
url: getErrorMessage('url', 'URL'),
|
||||
pdfFile: getErrorMessage('pdfFile', 'PDF_FILE'),
|
||||
content: getErrorMessage('content', 'CONTENT'),
|
||||
}));
|
||||
|
||||
const handleCancel = () => emit('cancel');
|
||||
@@ -107,13 +117,19 @@ const prepareDocumentDetails = () => {
|
||||
if (state.documentType === 'url') {
|
||||
formData.append('document[external_link]', state.url);
|
||||
formData.append('document[name]', state.name || state.url);
|
||||
} else {
|
||||
} else if (state.documentType === 'pdf') {
|
||||
formData.append('document[pdf_file]', state.pdfFile);
|
||||
formData.append(
|
||||
'document[name]',
|
||||
state.name || state.pdfFile.name.replace('.pdf', '')
|
||||
);
|
||||
// No need to send external_link for PDF - it's auto-generated in the backend
|
||||
} else if (state.documentType === 'content') {
|
||||
formData.append('document[content]', state.content);
|
||||
formData.append(
|
||||
'document[name]',
|
||||
state.name || t('CAPTAIN.DOCUMENTS.FORM.CONTENT.DEFAULT_NAME')
|
||||
);
|
||||
}
|
||||
|
||||
return formData;
|
||||
@@ -208,6 +224,24 @@ const handleSubmit = async () => {
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div v-if="state.documentType === 'content'" class="flex flex-col gap-1">
|
||||
<label class="text-sm font-medium text-n-slate-12">
|
||||
{{ t('CAPTAIN.DOCUMENTS.FORM.CONTENT.LABEL') }}
|
||||
</label>
|
||||
<textarea
|
||||
v-model="state.content"
|
||||
rows="8"
|
||||
:placeholder="t('CAPTAIN.DOCUMENTS.FORM.CONTENT.PLACEHOLDER')"
|
||||
class="w-full px-3 py-2 text-sm border rounded-lg resize-y bg-n-alpha-1 border-n-alpha-3 text-n-slate-12 focus:outline-none focus:border-n-blue-8"
|
||||
/>
|
||||
<p class="text-xs text-n-slate-11">
|
||||
{{ t('CAPTAIN.DOCUMENTS.FORM.CONTENT.HELP_TEXT') }}
|
||||
</p>
|
||||
<p v-if="formErrors.content" class="text-xs text-n-ruby-9">
|
||||
{{ formErrors.content }}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<Input
|
||||
v-model="state.name"
|
||||
:label="t('CAPTAIN.DOCUMENTS.FORM.NAME.LABEL')"
|
||||
|
||||
@@ -765,7 +765,8 @@
|
||||
"SOURCE": {
|
||||
"ALL": "All sources",
|
||||
"WEB": "Web pages",
|
||||
"PDF": "PDFs"
|
||||
"PDF": "PDFs",
|
||||
"TEXT": "Text"
|
||||
},
|
||||
"STATUS": {
|
||||
"ANY": "Any status",
|
||||
@@ -810,7 +811,8 @@
|
||||
"TYPE": {
|
||||
"LABEL": "Document Type",
|
||||
"URL": "URL",
|
||||
"PDF": "PDF File"
|
||||
"PDF": "PDF File",
|
||||
"CONTENT": "Text / Markdown"
|
||||
},
|
||||
"URL": {
|
||||
"LABEL": "URL",
|
||||
@@ -828,6 +830,13 @@
|
||||
"NAME": {
|
||||
"LABEL": "Document Name (Optional)",
|
||||
"PLACEHOLDER": "Enter a name for the document"
|
||||
},
|
||||
"CONTENT": {
|
||||
"LABEL": "Content",
|
||||
"PLACEHOLDER": "Enter or paste the document content (supports Markdown)",
|
||||
"HELP_TEXT": "Enter the text content directly. Markdown formatting is supported.",
|
||||
"ERROR": "Please enter at least 10 characters of content",
|
||||
"DEFAULT_NAME": "Inline Document"
|
||||
}
|
||||
},
|
||||
"DELETE": {
|
||||
@@ -837,7 +846,6 @@
|
||||
"SUCCESS_MESSAGE": "The document has been successfully deleted",
|
||||
"ERROR_MESSAGE": "There was an error deleting the document, please try again."
|
||||
},
|
||||
|
||||
"OPTIONS": {
|
||||
"VIEW_RELATED_RESPONSES": "View Related Responses",
|
||||
"SYNC_NOW": "Refresh now",
|
||||
|
||||
@@ -764,7 +764,8 @@
|
||||
"SOURCE": {
|
||||
"ALL": "全部来源",
|
||||
"WEB": "网页",
|
||||
"PDF": "PDF"
|
||||
"PDF": "PDF",
|
||||
"TEXT": "文本"
|
||||
},
|
||||
"STATUS": {
|
||||
"ANY": "任意状态",
|
||||
@@ -809,7 +810,8 @@
|
||||
"TYPE": {
|
||||
"LABEL": "文档类型",
|
||||
"URL": "网址",
|
||||
"PDF": "PDF 文件"
|
||||
"PDF": "PDF 文件",
|
||||
"CONTENT": "文本 / Markdown"
|
||||
},
|
||||
"URL": {
|
||||
"LABEL": "网址",
|
||||
@@ -827,6 +829,13 @@
|
||||
"NAME": {
|
||||
"LABEL": "文档名称(可选)",
|
||||
"PLACEHOLDER": "输入文档的名称"
|
||||
},
|
||||
"CONTENT": {
|
||||
"LABEL": "内容",
|
||||
"PLACEHOLDER": "输入或粘贴文档内容（支持 Markdown）",
|
||||
"HELP_TEXT": "直接输入文本内容，支持 Markdown 格式。",
|
||||
"ERROR": "请输入至少 10 个字符的内容",
|
||||
"DEFAULT_NAME": "内联文档"
|
||||
}
|
||||
},
|
||||
"DELETE": {
|
||||
@@ -928,7 +937,7 @@
|
||||
"LABEL": "认证方式"
|
||||
},
|
||||
"AUTH_TYPES": {
|
||||
"NONE": "啥都没有",
|
||||
"NONE": "无",
|
||||
"BEARER": "Bearer Token",
|
||||
"BASIC": "Basic Auth",
|
||||
"API_KEY": "API 密钥"
|
||||
|
||||
Reference in New Issue
Block a user