diff --git a/backend/internal/service/copilot_context_service.go b/backend/internal/service/copilot_context_service.go index e0b61e20..a29a93c2 100644 --- a/backend/internal/service/copilot_context_service.go +++ b/backend/internal/service/copilot_context_service.go @@ -60,8 +60,8 @@ type ContextMessage struct { // GetCurrentViewingContext retrieves and formats the conversation context // that the agent is currently viewing, to inject into Copilot LLM calls. func (s *CopilotContextService) GetCurrentViewingContext(ctx context.Context, accountID, conversationID uint) (*ConversationContext, error) { - // Get conversation messages (last N messages for context) - messages, _, err := s.messageRepo.FindByConversation(ctx, conversationID, 0, 20) + // Get conversation messages (fetch up to 100, then truncate by token budget) + messages, _, err := s.messageRepo.FindByConversation(ctx, conversationID, 0, 100) if err != nil { applogger.L().Errorf("GetCurrentViewingContext FindByConversation: %v", err) return nil, fmt.Errorf("retrieve conversation messages: %w", err) @@ -88,20 +88,33 @@ func (s *CopilotContextService) GetCurrentViewingContext(ctx context.Context, ac } } - // Format messages for LLM context + // Format messages for LLM context with token-budgeted truncation + // (keeps most recent messages, drops older ones to fit token budget) + convMsgs := make([]ConversationMessage, 0, len(messages)) for _, m := range messages { - role := "customer" - if m.MessageType == "outgoing" { - role = "agent" - } // Skip activity/template messages if m.MessageType == "activity" || m.ContentType != "text" { continue } + convMsgs = append(convMsgs, ConversationMessage{ + Content: m.Content, + MessageType: m.MessageType, + }) + } + + truncated, dropped := BuildContextWindow(convMsgs, 4096) + if dropped > 0 { + applogger.L().Infof("GetCurrentViewingContext: truncated %d older messages to fit token budget", dropped) + } + + for _, msg := range truncated { + role := "customer" + if msg.Role == "assistant" { + role = "agent" + } context.Messages = append(context.Messages, ContextMessage{ - Role: role, - Content: m.Content, - Timestamp: m.CreatedAt.Format("2006-01-02 15:04:05"), + Role: role, + Content: msg.Content, }) } diff --git a/backend/internal/service/token_estimator.go b/backend/internal/service/token_estimator.go new file mode 100644 index 00000000..45f6d2bc --- /dev/null +++ b/backend/internal/service/token_estimator.go @@ -0,0 +1,136 @@ +package service + +import ( + "strings" +) + +// TokenEstimator provides approximate token counting for LLM context management. +// Uses the industry-standard heuristic of ~4 characters per token for English text. +// This is not exact but sufficient for context window truncation decisions. +// +// Reference: AI_FEATURE_ROADMAP.md §3.4 — token counting + sliding window +type TokenEstimator struct{} + +// NewTokenEstimator creates a new TokenEstimator. +func NewTokenEstimator() *TokenEstimator { + return &TokenEstimator{} +} + +// EstimateText returns an approximate token count for the given text. +// Heuristic: ~4 characters per token (works well for English; slightly +// underestimates for CJK text which is ~2 chars/token). +func (e *TokenEstimator) EstimateText(text string) int { + if text == "" { + return 0 + } + // Count characters (not bytes) for better accuracy with Unicode + charCount := len([]rune(text)) + return (charCount + 3) / 4 // ceiling division +} + +// EstimateMessages returns the total approximate token count for a list of LLM messages. +// Each message has overhead (~4 tokens for role + delimiters). +func (e *TokenEstimator) EstimateMessages(messages []llmMessageWithContent) int { + total := 0 + for _, msg := range messages { + total += 4 // role + delimiter overhead + total += e.EstimateText(msg.Content) + } + total += 2 // priming tokens + return total +} + +// llmMessageWithContent is a minimal interface for messages with content. +type llmMessageWithContent struct { + Role string + Content string +} + +// TruncateMessages applies a sliding window to fit messages within a token budget. +// It keeps the most recent messages (by index) and drops older ones until +// the total estimated tokens fit within maxTokens. +// +// If the most recent single message exceeds maxTokens, it is truncated to fit. +// +// Returns the truncated message list and the number of messages dropped. +func (e *TokenEstimator) TruncateMessages(messages []llmMessageWithContent, maxTokens int) ([]llmMessageWithContent, int) { + if maxTokens <= 0 { + return messages, 0 + } + + // Calculate total tokens + totalTokens := e.EstimateMessages(messages) + if totalTokens <= maxTokens { + return messages, 0 + } + + // Drop oldest messages until we fit + dropped := 0 + for len(messages) > 1 { + messages = messages[1:] + dropped++ + totalTokens = e.EstimateMessages(messages) + if totalTokens <= maxTokens { + return messages, dropped + } + } + + // If single message still too long, truncate its content + if len(messages) == 1 { + overhead := 6 // role + delimiters + priming + availableTokens := maxTokens - overhead + if availableTokens > 0 { + maxChars := availableTokens * 4 + content := messages[0].Content + if len([]rune(content)) > maxChars { + runes := []rune(content) + messages[0].Content = string(runes[:maxChars]) + "..." + } + } + } + + return messages, dropped +} + +// BuildContextWindow retrieves messages and applies token-budgeted truncation. +// This is the main entry point for CopilotContextService and other services +// that need to build LLM context from conversation history. +// +// Parameters: +// - rawMessages: all messages from the conversation (oldest first) +// - maxContextTokens: token budget for the message context (default 4096 if <= 0) +// +// Returns the truncated messages (most recent kept) formatted as role/content pairs. +func BuildContextWindow(rawMessages []ConversationMessage, maxContextTokens int) ([]llmMessageWithContent, int) { + if maxContextTokens <= 0 { + maxContextTokens = 4096 + } + + // Convert to estimator format + msgs := make([]llmMessageWithContent, 0, len(rawMessages)) + for _, m := range rawMessages { + content := strings.TrimSpace(m.Content) + if content == "" { + continue + } + // Skip activity/template messages + if m.MessageType == "activity" { + continue + } + role := "user" + if m.MessageType == "outgoing" { + role = "assistant" + } + msgs = append(msgs, llmMessageWithContent{Role: role, Content: content}) + } + + estimator := NewTokenEstimator() + return estimator.TruncateMessages(msgs, maxContextTokens) +} + +// ConversationMessage is a minimal message representation for context building. +// Used to avoid importing model.Message in the estimator (keeps it testable). +type ConversationMessage struct { + Content string + MessageType string // "incoming", "outgoing", "activity" +}