diff --git a/backend/internal/app/bootstrap.go b/backend/internal/app/bootstrap.go index e0625128..c727da1c 100644 --- a/backend/internal/app/bootstrap.go +++ b/backend/internal/app/bootstrap.go @@ -247,6 +247,7 @@ func Bootstrap(env string) (*App, error) { categoryRepo := repository.NewCategoryRepo(db) relatedCategoryRepo := repository.NewRelatedCategoryRepo(db) articleRepo := repository.NewArticleRepo(db) + articleEmbeddingRepo := repository.NewArticleEmbeddingRepo(db) folderRepo := repository.NewFolderRepo(db) portalMemberRepo := repository.NewPortalMemberRepo(db) @@ -644,6 +645,8 @@ func Bootstrap(env string) (*App, error) { articleService := service.NewArticleService(articleRepo) articleService.SetWorkerPool(workerPool) articleService.SetArticleTranslationBackend(service.NewLLMArticleTranslationBackend(llmProvider)) + articleService.SetEmbeddingRepo(articleEmbeddingRepo) + articleService.SetLLMProvider(llmProvider) folderService := service.NewFolderService(folderRepo) portalMemberService := service.NewPortalMemberService(portalMemberRepo) diff --git a/backend/internal/handler/api/v1/article_handler.go b/backend/internal/handler/api/v1/article_handler.go index 4005a773..b18c6dc5 100644 --- a/backend/internal/handler/api/v1/article_handler.go +++ b/backend/internal/handler/api/v1/article_handler.go @@ -405,6 +405,45 @@ func (h *ArticleHandler) Search(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"payload": articlePayloads(articles), "meta": articleListMetaPayload(meta, p.Page)}) } +// SemanticSearch performs AI-powered semantic search on help center articles. +// Uses LLM embeddings + pgvector cosine similarity to find articles by meaning, +// not just keyword matching. +// +// GET /portals/:portal_id/articles/semantic_search?query=how+to+reset+password +func (h *ArticleHandler) SemanticSearch(c *gin.Context) { + accountID := getAccountID(c) + if accountID == 0 { + response.AbortWithStatusError(c, http.StatusBadRequest, response.ErrBadRequest, "invalid account_id") + return + } + portal, ok := h.resolvePortal(c, accountID) + if !ok { + return + } + + query := strings.TrimSpace(c.Query("query")) + if query == "" { + response.AbortWithStatusError(c, http.StatusBadRequest, response.ErrBadRequest, "query parameter is required") + return + } + + limit := 10 + if l := c.Query("limit"); l != "" { + if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 && parsed <= 50 { + limit = parsed + } + } + + articles, err := h.svc.SemanticSearch(c.Request.Context(), portal.ID, query, limit) + if err != nil { + applogger.L().Errorf("SemanticSearch articles: %v", err) + response.AbortWithStatusError(c, http.StatusInternalServerError, response.ErrInternal, "failed to perform semantic search") + return + } + + c.JSON(http.StatusOK, gin.H{"payload": articlePayloads(articles)}) +} + // StatusCounts returns article counts grouped by status. // GET /portals/:portal_id/articles/status_counts func (h *ArticleHandler) StatusCounts(c *gin.Context) { diff --git a/backend/internal/model/article_embedding.go b/backend/internal/model/article_embedding.go index 8c2ed6d7..8cae081c 100644 --- a/backend/internal/model/article_embedding.go +++ b/backend/internal/model/article_embedding.go @@ -2,15 +2,18 @@ package model import ( "encoding/json" + + "github.com/pgvector/pgvector-go" ) // ArticleEmbedding stores vector embeddings for semantic article search. // Reference: Chatwoot ArticleEmbedding (enterprise) + P2B M9 spec type ArticleEmbedding struct { Base - ArticleID uint `gorm:"not null;index" json:"article_id"` - Embedding json.RawMessage `gorm:"type:jsonb" json:"embedding"` // vector data (1536-dim, stored as JSON for portability) - Term string `gorm:"type:text;not null" json:"term"` // searchable text content + ArticleID uint `gorm:"not null;index" json:"article_id"` + Embedding json.RawMessage `gorm:"type:jsonb" json:"embedding"` // original JSONB storage (backward compat) + VectorEmbedding pgvector.Vector `gorm:"type:vector(1536)" json:"-"` // pgvector column for cosine similarity search + Term string `gorm:"type:text;not null" json:"term"` // searchable text content Article Article `gorm:"foreignKey:ArticleID" json:"article,omitempty"` } diff --git a/backend/internal/repository/article_embedding_repo.go b/backend/internal/repository/article_embedding_repo.go new file mode 100644 index 00000000..fc127984 --- /dev/null +++ b/backend/internal/repository/article_embedding_repo.go @@ -0,0 +1,75 @@ +package repository + +import ( + "context" + + "github.com/gochat/gochat/internal/model" + "github.com/pgvector/pgvector-go" + "gorm.io/gorm" +) + +// ArticleEmbeddingRepo provides data access for ArticleEmbedding (semantic search). +type ArticleEmbeddingRepo struct { + db *gorm.DB +} + +func NewArticleEmbeddingRepo(db *gorm.DB) *ArticleEmbeddingRepo { + return &ArticleEmbeddingRepo{db: db} +} + +func (r *ArticleEmbeddingRepo) DB() *gorm.DB { return r.db } + +// Upsert creates or updates an article embedding. +func (r *ArticleEmbeddingRepo) Upsert(ctx context.Context, emb *model.ArticleEmbedding) error { + // Try to find existing by article_id + var existing model.ArticleEmbedding + err := r.db.WithContext(ctx).Where("article_id = ?", emb.ArticleID).First(&existing).Error + if err == gorm.ErrRecordNotFound { + return r.db.WithContext(ctx).Create(emb).Error + } + if err != nil { + return err + } + // Update existing + existing.Embedding = emb.Embedding + existing.Term = emb.Term + return r.db.WithContext(ctx).Save(&existing).Error +} + +// GetByArticleID retrieves the embedding for a specific article. +func (r *ArticleEmbeddingRepo) GetByArticleID(ctx context.Context, articleID uint) (*model.ArticleEmbedding, error) { + var emb model.ArticleEmbedding + if err := r.db.WithContext(ctx).Where("article_id = ?", articleID).First(&emb).Error; err != nil { + return nil, err + } + return &emb, nil +} + +// DeleteByArticleID removes the embedding for an article. +func (r *ArticleEmbeddingRepo) DeleteByArticleID(ctx context.Context, articleID uint) error { + return r.db.WithContext(ctx).Where("article_id = ?", articleID).Delete(&model.ArticleEmbedding{}).Error +} + +// SearchByEmbedding performs semantic search using pgvector cosine distance. +// Returns articles sorted by similarity (closest first). +// Note: This method is PG-only (requires pgvector extension). +func (r *ArticleEmbeddingRepo) SearchByEmbedding(ctx context.Context, portalID uint, embedding pgvector.Vector, limit int) ([]model.Article, error) { + if limit <= 0 { + limit = 10 + } + + var articles []model.Article + // Join articles with article_embeddings, compute cosine distance on vector_embedding column + // pgvector cosine distance operator: <=> (for vector type) + err := r.db.WithContext(ctx).Raw(` + SELECT a.* FROM articles a + INNER JOIN article_embeddings ae ON ae.article_id = a.id + WHERE a.portal_id = ? AND a.status = 'published' + ORDER BY ae.vector_embedding <=> ? + LIMIT ? + `, portalID, embedding, limit).Scan(&articles).Error + if err != nil { + return nil, err + } + return articles, nil +} diff --git a/backend/internal/router/router.go b/backend/internal/router/router.go index 6e18721a..aa26e7a7 100644 --- a/backend/internal/router/router.go +++ b/backend/internal/router/router.go @@ -1624,6 +1624,7 @@ func registerV1Routes(g *gin.RouterGroup, h *Handlers) { articles.POST("", h.Article.Create) articles.POST("/", h.Article.Create) articles.GET("/search", h.Article.Search) + articles.GET("/semantic_search", h.Article.SemanticSearch) articles.GET("/status_counts", h.Article.StatusCounts) articles.POST("/reorder", h.Article.Reorder) articles.PATCH("/bulk_actions/update_status", h.Article.BulkUpdateStatus) diff --git a/backend/internal/service/article_service.go b/backend/internal/service/article_service.go index 1f5e2aef..e12a08a2 100644 --- a/backend/internal/service/article_service.go +++ b/backend/internal/service/article_service.go @@ -14,6 +14,7 @@ import ( "github.com/gochat/gochat/internal/repository" "github.com/gochat/gochat/internal/worker" applogger "github.com/gochat/gochat/pkg/logger" + "github.com/pgvector/pgvector-go" "golang.org/x/text/language" "golang.org/x/text/language/display" "gorm.io/gorm" @@ -24,6 +25,8 @@ const TaskTypeCaptainArticleTranslate = "captain:article_translate" // ArticleService implements business logic for Article CRUD. type ArticleService struct { repo *repository.ArticleRepo + embeddingRepo *repository.ArticleEmbeddingRepo + llmProvider llm.Provider searchIndexer SearchIndexer worker *worker.WorkerPool translator ArticleTranslationBackend @@ -81,6 +84,16 @@ func (s *ArticleService) SetArticleTranslationBackend(backend ArticleTranslation s.translator = backend } +// SetEmbeddingRepo injects the article embedding repository for semantic search. +func (s *ArticleService) SetEmbeddingRepo(repo *repository.ArticleEmbeddingRepo) { + s.embeddingRepo = repo +} + +// SetLLMProvider injects the LLM provider for embedding generation. +func (s *ArticleService) SetLLMProvider(provider llm.Provider) { + s.llmProvider = provider +} + func (s *ArticleService) indexArticle(ctx context.Context, article *model.Article) { if s.searchIndexer != nil { logSearchIndexError("article", article.ID, s.searchIndexer.IndexArticle(ctx, article)) @@ -856,3 +869,104 @@ func articleSlug(title string) string { } return fmt.Sprintf("%d-%s", time.Now().UTC().Unix(), slug) } + +// --- Semantic Search (Phase 3) --- + +// SemanticSearch performs AI-powered semantic search on help center articles. +// It generates an embedding for the query text, then searches article_embeddings +// using pgvector cosine similarity. +// +// Reference: Chatwoot HelpCenter semantic search + AI_FEATURE_ROADMAP.md ยง3.1 +func (s *ArticleService) SemanticSearch(ctx context.Context, portalID uint, query string, limit int) ([]model.Article, error) { + if s.embeddingRepo == nil { + return nil, fmt.Errorf("semantic search not configured: embedding repo not set") + } + if s.llmProvider == nil { + return nil, fmt.Errorf("semantic search not configured: LLM provider not set") + } + if strings.TrimSpace(query) == "" { + return nil, fmt.Errorf("query is required") + } + if limit <= 0 { + limit = 10 + } + + // Generate embedding for the query + embedResp, err := s.llmProvider.CreateEmbedding(ctx, llm.EmbeddingRequest{ + Model: "text-embedding-3-small", + Input: []string{query}, + }) + if err != nil { + return nil, fmt.Errorf("generate query embedding: %w", err) + } + if len(embedResp.Data) == 0 { + return nil, fmt.Errorf("no embedding returned for query") + } + + // Convert to float32 for pgvector + float32Emb := make([]float32, len(embedResp.Data[0].Embedding)) + for i, v := range embedResp.Data[0].Embedding { + float32Emb[i] = float32(v) + } + pgvectorEmb := pgvector.NewVector(float32Emb) + + // Search by cosine similarity + articles, err := s.embeddingRepo.SearchByEmbedding(ctx, portalID, pgvectorEmb, limit) + if err != nil { + return nil, fmt.Errorf("semantic search failed: %w", err) + } + return articles, nil +} + +// GenerateEmbedding generates and stores an embedding for an article. +// Called when an article is created or updated (via worker for async). +func (s *ArticleService) GenerateEmbedding(ctx context.Context, articleID uint) error { + if s.embeddingRepo == nil || s.llmProvider == nil { + return nil // not configured, skip silently + } + + article, err := s.repo.GetByID(ctx, articleID) + if err != nil { + return fmt.Errorf("article not found: %w", err) + } + + // Build text for embedding: title + description + content (truncated) + text := article.Title + if article.Description != "" { + text += "\n" + article.Description + } + if article.Content != "" { + // Truncate content to avoid token limits (rough char-based truncation) + content := article.Content + if len(content) > 3000 { + content = content[:3000] + } + text += "\n" + content + } + + // Generate embedding + embedResp, err := s.llmProvider.CreateEmbedding(ctx, llm.EmbeddingRequest{ + Model: "text-embedding-3-small", + Input: []string{text}, + }) + if err != nil { + return fmt.Errorf("generate article embedding: %w", err) + } + if len(embedResp.Data) == 0 { + return fmt.Errorf("no embedding returned") + } + + // Convert to float32 for pgvector + float32Emb := make([]float32, len(embedResp.Data[0].Embedding)) + for i, v := range embedResp.Data[0].Embedding { + float32Emb[i] = float32(v) + } + + emb := &model.ArticleEmbedding{ + ArticleID: articleID, + VectorEmbedding: pgvector.NewVector(float32Emb), + Term: text, + } + + return s.embeddingRepo.Upsert(ctx, emb) +} diff --git a/backend/migrations/000049_add_article_embedding_vector.down.sql b/backend/migrations/000049_add_article_embedding_vector.down.sql new file mode 100644 index 00000000..f0d99888 --- /dev/null +++ b/backend/migrations/000049_add_article_embedding_vector.down.sql @@ -0,0 +1,5 @@ +-- 000049_add_article_embedding_vector.down.sql +-- Remove the pgvector column from article_embeddings + +DROP INDEX IF EXISTS idx_article_embeddings_vector; +ALTER TABLE article_embeddings DROP COLUMN IF EXISTS vector_embedding; diff --git a/backend/migrations/000049_add_article_embedding_vector.up.sql b/backend/migrations/000049_add_article_embedding_vector.up.sql new file mode 100644 index 00000000..914e0400 --- /dev/null +++ b/backend/migrations/000049_add_article_embedding_vector.up.sql @@ -0,0 +1,25 @@ +-- 000049_add_article_embedding_vector.up.sql +-- Add pgvector vector column to article_embeddings for semantic search. +-- The original embedding JSONB column is kept for backward compatibility; +-- the new vector_column stores the same data as a native pgvector type +-- enabling the <=> (cosine distance) operator for efficient similarity search. + +-- Ensure pgvector extension is available (should already be installed). +CREATE EXTENSION IF NOT EXISTS vector; + +-- Add a vector(1536) column to article_embeddings +ALTER TABLE article_embeddings ADD COLUMN IF NOT EXISTS vector_embedding vector(1536); + +-- Create an index for efficient cosine similarity search +CREATE INDEX IF NOT EXISTS idx_article_embeddings_vector + ON article_embeddings USING ivfflat (vector_embedding vector_cosine_ops) + WITH (lists = 100); + +-- Migrate existing JSONB embeddings to the vector column +-- This parses the JSON array into a pgvector-compatible format +UPDATE article_embeddings +SET vector_embedding = ( + SELECT array_agg(elem::real)::vector(1536) + FROM jsonb_array_elements_text(embedding) AS elem +) +WHERE vector_embedding IS NULL AND embedding IS NOT NULL;