feat(search): add meilisearch engine foundation

2026-06-04 19:56:29 +08:00
parent db0bd5b0f6
commit 3fc4859e20
13 changed files with 1515 additions and 289 deletions
@@ -0,0 +1,266 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gochat/gochat/internal/app"
+	"github.com/gochat/gochat/internal/config"
+	"github.com/gochat/gochat/internal/model"
+	"github.com/gochat/gochat/internal/repository"
+	"github.com/gochat/gochat/internal/search"
+	applogger "github.com/gochat/gochat/pkg/logger"
+	"gorm.io/gorm"
+)
+
+func main() {
+	var accountID uint
+	var batchSize int
+	var types string
+	var bootstrap bool
+	flag.UintVar(&accountID, "account", 0, "account ID to reindex; 0 means all accounts")
+	flag.IntVar(&batchSize, "batch", 500, "documents per indexing batch")
+	flag.StringVar(&types, "types", "all", "comma-separated types: conversation,message,contact,company,article,help_center,all")
+	flag.BoolVar(&bootstrap, "bootstrap", true, "create/update Meilisearch indexes and settings before indexing")
+	flag.Parse()
+
+	if batchSize < 1 {
+		fmt.Fprintln(os.Stderr, "batch must be >= 1")
+		os.Exit(1)
+	}
+
+	cfg, err := config.Load()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "config load: %v\n", err)
+		os.Exit(1)
+	}
+	_ = applogger.Init(applogger.Config{Level: cfg.Log.Level, Format: cfg.Log.Format, Output: "stdout", ErrorOutput: "stderr"})
+
+	db, err := app.NewDatabase(&cfg.Database, cfg.Log.Level)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "database: %v\n", err)
+		os.Exit(1)
+	}
+
+	searchRepo := repository.NewSearchRepo(db)
+	engine, err := search.NewSearchEngine(search.EngineConfig{
+		Engine:         cfg.Search.Engine,
+		Host:           cfg.Search.Host,
+		APIKey:         cfg.Search.APIKey,
+		IndexPrefix:    cfg.Search.IndexPrefix,
+		TimeoutSeconds: cfg.Search.TimeoutSeconds,
+	}, searchRepo)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "search engine: %v\n", err)
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+	if bootstrap {
+		if err := engine.Bootstrap(ctx); err != nil {
+			fmt.Fprintf(os.Stderr, "bootstrap search indexes: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
+	selected := parseTypes(types)
+	stats := map[search.SearchResultType]int{}
+	if selected[search.ResultTypeConversation] {
+		stats[search.ResultTypeConversation], err = reindexConversations(ctx, engine, db, accountID, batchSize)
+		fatalIf(err)
+	}
+	if selected[search.ResultTypeMessage] {
+		stats[search.ResultTypeMessage], err = reindexMessages(ctx, engine, db, accountID, batchSize)
+		fatalIf(err)
+	}
+	if selected[search.ResultTypeContact] {
+		stats[search.ResultTypeContact], err = reindexContacts(ctx, engine, db, accountID, batchSize)
+		fatalIf(err)
+	}
+	if selected[search.ResultTypeCompany] {
+		stats[search.ResultTypeCompany], err = reindexCompanies(ctx, engine, db, accountID, batchSize)
+		fatalIf(err)
+	}
+	if selected[search.ResultTypeArticle] || selected[search.ResultTypeHelpCenter] {
+		stats[search.ResultTypeArticle], err = reindexArticles(ctx, engine, db, accountID, batchSize)
+		fatalIf(err)
+	}
+
+	fmt.Println("Search reindex complete")
+	for docType, count := range stats {
+		fmt.Printf("%s: %d\n", docType, count)
+	}
+}
+
+func parseTypes(raw string) map[search.SearchResultType]bool {
+	selected := map[search.SearchResultType]bool{}
+	for _, part := range strings.Split(raw, ",") {
+		switch search.SearchResultType(strings.TrimSpace(part)) {
+		case "", "all":
+			selected[search.ResultTypeConversation] = true
+			selected[search.ResultTypeMessage] = true
+			selected[search.ResultTypeContact] = true
+			selected[search.ResultTypeCompany] = true
+			selected[search.ResultTypeArticle] = true
+			selected[search.ResultTypeHelpCenter] = true
+		case search.ResultTypeConversation:
+			selected[search.ResultTypeConversation] = true
+		case search.ResultTypeMessage:
+			selected[search.ResultTypeMessage] = true
+		case search.ResultTypeContact:
+			selected[search.ResultTypeContact] = true
+		case search.ResultTypeCompany:
+			selected[search.ResultTypeCompany] = true
+		case search.ResultTypeArticle:
+			selected[search.ResultTypeArticle] = true
+		case search.ResultTypeHelpCenter:
+			selected[search.ResultTypeHelpCenter] = true
+		}
+	}
+	return selected
+}
+
+func reindexConversations(ctx context.Context, engine search.SearchEngine, db *gorm.DB, accountID uint, batchSize int) (int, error) {
+	var count int
+	var lastID uint
+	for {
+		var rows []model.Conversation
+		q := db.WithContext(ctx).Where("id > ?", lastID).Order("id ASC").Limit(batchSize)
+		if accountID != 0 {
+			q = q.Where("account_id = ?", accountID)
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return count, err
+		}
+		if len(rows) == 0 {
+			return count, nil
+		}
+		docs := make([]search.SearchDocument, 0, len(rows))
+		for _, row := range rows {
+			docs = append(docs, search.ConversationDocument(row))
+			lastID = row.ID
+		}
+		if err := engine.IndexBatch(ctx, docs); err != nil {
+			return count, err
+		}
+		count += len(rows)
+	}
+}
+
+func reindexMessages(ctx context.Context, engine search.SearchEngine, db *gorm.DB, accountID uint, batchSize int) (int, error) {
+	var count int
+	var lastID uint
+	for {
+		var rows []model.Message
+		q := db.WithContext(ctx).Where("id > ?", lastID).Order("id ASC").Limit(batchSize)
+		if accountID != 0 {
+			q = q.Where("account_id = ?", accountID)
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return count, err
+		}
+		if len(rows) == 0 {
+			return count, nil
+		}
+		docs := make([]search.SearchDocument, 0, len(rows))
+		for _, row := range rows {
+			docs = append(docs, search.MessageDocument(row))
+			lastID = row.ID
+		}
+		if err := engine.IndexBatch(ctx, docs); err != nil {
+			return count, err
+		}
+		count += len(rows)
+	}
+}
+
+func reindexContacts(ctx context.Context, engine search.SearchEngine, db *gorm.DB, accountID uint, batchSize int) (int, error) {
+	var count int
+	var lastID uint
+	for {
+		var rows []model.Contact
+		q := db.WithContext(ctx).Where("id > ?", lastID).Order("id ASC").Limit(batchSize)
+		if accountID != 0 {
+			q = q.Where("account_id = ?", accountID)
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return count, err
+		}
+		if len(rows) == 0 {
+			return count, nil
+		}
+		docs := make([]search.SearchDocument, 0, len(rows))
+		for _, row := range rows {
+			docs = append(docs, search.ContactDocument(row))
+			lastID = row.ID
+		}
+		if err := engine.IndexBatch(ctx, docs); err != nil {
+			return count, err
+		}
+		count += len(rows)
+	}
+}
+
+func reindexCompanies(ctx context.Context, engine search.SearchEngine, db *gorm.DB, accountID uint, batchSize int) (int, error) {
+	var count int
+	var lastID uint
+	for {
+		var rows []model.Company
+		q := db.WithContext(ctx).Where("id > ?", lastID).Order("id ASC").Limit(batchSize)
+		if accountID != 0 {
+			q = q.Where("account_id = ?", accountID)
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return count, err
+		}
+		if len(rows) == 0 {
+			return count, nil
+		}
+		docs := make([]search.SearchDocument, 0, len(rows))
+		for _, row := range rows {
+			docs = append(docs, search.CompanyDocument(row))
+			lastID = row.ID
+		}
+		if err := engine.IndexBatch(ctx, docs); err != nil {
+			return count, err
+		}
+		count += len(rows)
+	}
+}
+
+func reindexArticles(ctx context.Context, engine search.SearchEngine, db *gorm.DB, accountID uint, batchSize int) (int, error) {
+	var count int
+	var lastID uint
+	for {
+		var rows []model.Article
+		q := db.WithContext(ctx).Where("id > ?", lastID).Order("id ASC").Limit(batchSize)
+		if accountID != 0 {
+			q = q.Where("account_id = ?", accountID)
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return count, err
+		}
+		if len(rows) == 0 {
+			return count, nil
+		}
+		docs := make([]search.SearchDocument, 0, len(rows))
+		for _, row := range rows {
+			docs = append(docs, search.ArticleDocument(row))
+			lastID = row.ID
+		}
+		if err := engine.IndexBatch(ctx, docs); err != nil {
+			return count, err
+		}
+		count += len(rows)
+	}
+}
+
+func fatalIf(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}