diff --git a/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md b/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md index be3b5cce..c27ef8ae 100644 --- a/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md +++ b/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md @@ -16,9 +16,9 @@ Build GoChat as a Go backend that can directly reuse the frontend from `referenc ## Current Baseline -- Latest implementation checkpoint: `feat(captain): align playground fallback`. -- Latest documentation checkpoint: this checkpoint, recorded with the B11.3b Captain playground fallback implementation. -- Worktree status at this implementation checkpoint: B11.1a aligns Captain assistant CRUD/tools/inbox bindings; B11.1b aligns Captain scenarios and custom tools; B11.1c aligns Captain documents, assistant responses, bulk actions, and custom-tool test payloads; B11.2 aligns Copilot thread/message create/list/get/delete payloads, account/user scoping, and no-LLM fallback persistence; B11.3a aligns Captain preferences show/update payloads and account-level model/feature storage; B11.3b aligns Captain playground request/response payloads, account scoping, v2 history handling, and no-LLM fallback. Next active slice is B11.3c document sync/indexing and Meilisearch/embedding gates, followed by B11.3d Copilot tasks/tool-call depth, B11.3e streaming/realtime compatibility, and B12 reused frontend smoke. +- Latest implementation checkpoint: `feat(captain): gate document sync backend`. +- Latest documentation checkpoint: this checkpoint, recorded with the B11.3c Captain document sync backend gate implementation. +- Worktree status at this implementation checkpoint: B11.1a aligns Captain assistant CRUD/tools/inbox bindings; B11.1b aligns Captain scenarios and custom tools; B11.1c aligns Captain documents, assistant responses, bulk actions, and custom-tool test payloads; B11.2 aligns Copilot thread/message create/list/get/delete payloads, account/user scoping, and no-LLM fallback persistence; B11.3a aligns Captain preferences show/update payloads and account-level model/feature storage; B11.3b aligns Captain playground request/response payloads, account scoping, v2 history handling, and no-LLM fallback; B11.3c adds the fakeable Captain document sync backend gate with disabled, failed, and fake-success states. Next active slice is B11.3d Copilot tasks/tool-call depth, followed by B11.3e streaming/realtime compatibility and B12 reused frontend smoke. - `go test ./...` passes. - Route dump succeeds with `TOTAL: 830` after adding the Chatwoot-compatible applied-SLA index route. - Route parity artifacts now exist under `docs/parity/` and are generated by `cmd/route_parity`. @@ -44,10 +44,9 @@ Next ordered checkpoints: | Order | Slice | Required outcome | Primary verification | | --- | --- | --- | --- | -| 1 | B11.3c | Captain document sync has observable Meilisearch/embedding gates and fakeable indexing boundaries. | Document sync/indexing fixtures for disabled, failure, and fake-success paths. | -| 2 | B11.3d | Copilot tasks, suggestions, and tool-call records persist enough frontend state for reloads. | Copilot task/tool-call handler/service fixtures and serializer tests. | -| 3 | B11.3e | Streaming/realtime paths have frontend-compatible non-streaming or disabled fallbacks until true streaming is wired. | Deterministic response tests and event/push payload tests. | -| 4 | B12 | Reused Chatwoot frontend smoke runs repeatably against GoChat. | Checked smoke command plus gap report under `docs/parity/`. | +| 1 | B11.3d | Copilot tasks, suggestions, and tool-call records persist enough frontend state for reloads. | Copilot task/tool-call handler/service fixtures and serializer tests. | +| 2 | B11.3e | Streaming/realtime paths have frontend-compatible non-streaming or disabled fallbacks until true streaming is wired. | Deterministic response tests and event/push payload tests. | +| 3 | B12 | Reused Chatwoot frontend smoke runs repeatably against GoChat. | Checked smoke command plus gap report under `docs/parity/`. | ## Execution Snapshot @@ -57,7 +56,7 @@ Next ordered checkpoints: | Phase 1 | Meilisearch search engine | Review | B6 payload parity, optional live gate, and DB-fallback hardening are implemented; an actual live Meilisearch run is optional and environment-dependent | | Phase 2 | Route and controller parity audit | Doing | Ruby/Bundler unavailable, so Chatwoot route extraction currently uses static `routes.rb` fallback | | Phase 3 | Data and serializer parity | Doing | JSON fixture coverage is partial and still endpoint-family based | -| Phase 4 | Enterprise feature completion | Doing | B7, B8, B9, and B10 are in Review; B11 Captain resources, preferences, playground, and Copilot thread/message payloads are landed, while Copilot tasks/tool-call depth and document/LLM/Meilisearch gates remain active | +| Phase 4 | Enterprise feature completion | Doing | B7, B8, B9, and B10 are in Review; B11 Captain resources, preferences, playground, document sync gates, and Copilot thread/message payloads are landed, while Copilot tasks/tool-call depth and streaming remain active | | Phase 5 | Background jobs and integrations | Planned | durable worker choice and job parity are open | | Phase 6 | Core placeholder burn-down | Doing | account/contact/conversation/message/inbox placeholder groups remain broad | | Phase 7 | Verification harness | Planned | search live gate and reused-frontend smoke harness are not complete | @@ -155,14 +154,15 @@ This ledger records the committed parity checkpoints that future slices should b | `feat(captain): align preferences payloads` | Starts B11.3a by replacing the legacy local `captain_preferences` CRUD response with Chatwoot's account-level Captain preferences contract: `GET/PUT /captain/preferences` now return raw `{ providers, models, features }`, `captain_models` and `captain_features` persist on accounts, updates merge with existing account settings, invalid model selections return `422`, and update is administrator-gated. Route artifacts were regenerated after removing frontend-unused POST/DELETE preference routes and adding exact no-trailing-slash GET/PUT paths. | `go test ./internal/handler/api/v1 -run 'CaptainPreference' -count=1`; `go test ./internal/service -run 'CaptainPreference' -count=1`; `go test ./internal/handler/api/v1 -run 'Captain\|Copilot' -count=1`; `go test ./internal/service -run 'Captain\|Copilot' -count=1`; `go test ./internal/handler/api/v1 -count=1`; `go test ./internal/service -count=1`; `go test ./internal/model -count=1`; `go test ./internal/router -count=1`; `go test ./cmd/migrate -count=1`; `go run ./cmd/dump_routes > docs/parity/gochat_routes.txt`; `go run ./cmd/route_parity`; escalated full `go test ./...`; `git diff --check`. | Continue B11.3b with document sync/indexing, Meilisearch/embedding gates, remaining Copilot task/tool-call/playground depth, streaming/realtime compatibility, and reused frontend smoke coverage. | | `docs: land captain copilot completion plan` | Documentation-only handoff checkpoint that breaks the remaining B11.3 work into explicit Captain playground, document sync/indexing, Copilot task/tool-call, streaming/realtime, and B12 smoke slices. It also records the reference files, required payload contracts, fallback behavior, verification commands, and status gates so no Hermes-era plan remains implicit. | `git diff --check`; no Go code changed. | Start B11.3b Captain playground implementation from the recorded contract. | | `feat(captain): align playground fallback` | Completes B11.3b Captain assistant playground parity: `POST /captain/assistants/:assistant_id/playground` now accepts the Chatwoot frontend top-level `message_content` and `message_history` payload, enforces account-scoped assistant lookup, returns raw legacy `{ content }` or v2 `{ response }` payloads, appends the current user message once for `captain_integration_v2`, preserves history ordering through the LLM boundary, and returns a deterministic raw disabled response when no LLM provider is configured. | `go test ./internal/handler/api/v1 -run CaptainAssistant -count=1`; `go test ./internal/service -run Captain -count=1`; `go test ./internal/handler/api/v1 -run 'Captain\|Copilot' -count=1`; `go test ./internal/service -run 'Captain\|Copilot' -count=1`; `go test ./internal/handler/api/v1 -count=1`; `go test ./internal/service -count=1`; `go test ./...`; `git diff --check`. Focused handler fixtures cover raw no-envelope fallback, no-history default, cross-account rejection, v2 append/no-duplicate history behavior, and fake LLM request capture. | Continue B11.3c with Captain document sync/indexing and Meilisearch/embedding gates. | +| `feat(captain): gate document sync backend` | Completes B11.3c's first document sync gate: Captain document sync now has a fakeable `CaptainDocumentSyncBackend` boundary and account-scoped `SyncDocumentByAccount` worker entry that marks disabled config as `sync_disabled`, records backend errors as `sync_error`, preserves backend error codes such as `access_denied`, stores successful content/title/fingerprint updates as `synced`, and avoids external HTTP/LLM calls in default tests. | `go test ./internal/service -run CaptainDocument -count=1`; `go test ./internal/service -run 'Captain\|Copilot' -count=1`; `go test ./internal/handler/api/v1 -run 'Captain\|Copilot' -count=1`; `go test ./internal/service -count=1`; `go test ./internal/handler/api/v1 -count=1`; `go test ./...`; `git diff --check`. Focused service fixtures cover disabled, fake success, backend error, backend error-code, content fingerprint normalization, and account scoping. | Continue B11.3d with Copilot task/tool-call persistence and suggestion payload depth. | ## Next Slice Contract -Completed implementation slice: B11.3b now aligns Captain assistant playground payloads and fallback behavior with the reused Chatwoot frontend, building on B11.1a-B11.3a Captain/Copilot resource contracts. +Completed implementation slice: B11.3c now adds Captain document sync backend gates and observable disabled/failure/success states, building on B11.1a-B11.3b Captain/Copilot resource contracts. Completed documentation slice: the remaining development plan is now fully landed in this tracker. B11.3 is split into concrete Captain playground, document sync/indexing, Copilot task/tool-call, and streaming/realtime checkpoints. B12 has a repeatable smoke-report contract. B9.3 delayed/durable worker scheduling and B9.4 macro attachment depth remain named Phase 5 follow-ups. -Next implementation slice: start B11.3c Captain document sync/indexing and Meilisearch/embedding gates. Do not expand to Copilot tasks until disabled config, failed sync observability, fake successful indexing, and account-scoped document lookup are covered by tests. +Next implementation slice: start B11.3d Copilot task/tool-call persistence and suggestion payload depth. Do not expand to streaming until rewrite/summarize/reply suggestion/label suggestion/follow-up request payloads, raw disabled states, and reloadable task/tool-call serializers are covered by tests. | Step | Required result | Reference source | Verification | | --- | --- | --- | --- | @@ -188,7 +188,7 @@ Next implementation slice: start B11.3c Captain document sync/indexing and Meili | N20 | Keep B11.2 Copilot thread/message resources as current Copilot baseline. | `copilot_threads_controller.rb`, `copilot_messages_controller.rb`, Copilot Jbuilder partials, and dashboard `copilotThreads.js`/`copilotMessages.js` clients. | Done by `feat(copilot): align thread message payloads`; threads/messages persist account/user-scoped Chatwoot payloads and no-LLM fallback assistant messages. | | N21 | Keep B11.3a Captain preferences as current account-level Captain config baseline. | `preferences_controller.rb`, `CaptainFeaturable`, `config/llm.yml`, dashboard `preferences.js` API/store. | Done by `feat(captain): align preferences payloads`; raw model/provider/feature payloads, merge update behavior, account storage, admin gate, and exact preference routes are covered. | | N22 | Keep B11.3b Captain assistant playground payload parity as current playground baseline. | `reference/chatwoot/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb#playground`, dashboard `api/captain/assistant.js`, assistant playground component/store. | Done by `feat(captain): align playground fallback`; top-level `message_content` and `message_history` are accepted, account-scoped assistant lookup is enforced, legacy `{ content }` and v2 `{ response }` shapes are covered, and no-LLM fallback returns raw frontend-compatible JSON without 500. | -| N23 | Implement B11.3c Captain document sync/indexing gates. | Captain document sync service/jobs, document controller `sync`, existing Meilisearch engine, and any local embedding boundary. | Disabled config, fake successful indexing, failed sync metadata, and account-scoped document lookup are covered without opening external network connections in default tests. | +| N23 | Keep B11.3c Captain document sync/indexing gates as current document-sync baseline. | Captain document sync service/jobs, document controller `sync`, existing Meilisearch engine, and any local embedding boundary. | Done by `feat(captain): gate document sync backend`; disabled config, fake successful indexing, failed sync metadata, fingerprint normalization, and account-scoped document lookup are covered without opening external network connections in default tests. | | N24 | Implement B11.3d Copilot task/tool-call persistence. | `resource :tasks` routes, Copilot/Captain task services, dashboard Copilot clients, current local `copilot_*` models. | Rewrite/summarize/reply suggestion/label suggestion/follow-up requests persist task/tool-call state or return documented raw disabled states; serializers survive frontend reload. | | N25 | Implement B11.3e streaming/realtime compatibility. | Chatwoot Copilot/Captain streaming, push/event payloads, current Go channel dispatcher. | If streaming is not implemented, the endpoint returns deterministic non-streaming payloads and records the fallback; no route returns placeholder success or local envelope. | | N26 | Start B12 smoke harness after B11.3 has a tested disabled/external-provider story. | `reference/chatwoot` frontend boot scripts and GoChat dev/test boot flow. | A checked command and `docs/parity/frontend_smoke_report.md` record core and enterprise smoke status. | @@ -787,7 +787,7 @@ B11 Captain/Copilot breakdown: | B11.2 | Align Copilot thread/message APIs, account/user scoping, assistant scope, frontend payloads, and no-LLM fallback persistence. | `copilot_threads_controller.rb`, `copilot_messages_controller.rb`, Copilot Jbuilder partials, dashboard `copilotThreads.js` and `copilotMessages.js`. | Handler tests cover thread/message create/list/get/delete, nested message ordering, assistant/account isolation, user isolation, and deterministic disabled LLM fallback. | Done by `feat(copilot): align thread message payloads` | | B11.3a | Align Captain preferences show/update payloads, account-level model/feature storage, and admin update gate. | `preferences_controller.rb`, `CaptainFeaturable`, `config/llm.yml`, dashboard `preferences.js` API/store. | Handler tests cover raw `{ providers, models, features }`, default selected models, merge update behavior, account persistence, invalid model rejection, admin gate, and exact preference routes. | Done by `feat(captain): align preferences payloads` | | B11.3b | Align Captain assistant playground request/response behavior and no-LLM fallback. | `assistants_controller.rb#playground`, `Captain::Llm::AssistantChatService`, `AgentRunnerService`, dashboard `api/captain/assistant.js`, playground component/store. | Handler/service tests cover top-level `message_content`, `message_history`, account-scoped assistant lookup, no-history payloads, no duplicate latest user message for v2 behavior if enabled, raw `{ content }` or `{ response }` response shape, and safe no-LLM fallback. | Done by `feat(captain): align playground fallback` | -| B11.3c | Implement document sync/indexing gates for Meilisearch or the chosen embedding/search backend. | Captain document sync/indexing services, document controller `sync`, existing Meilisearch engine, local embedding/config boundaries. | Tests cover disabled config, failed sync observability, successful fake backend indexing, account-scoped document lookup, and no external network call in default test mode. | Todo | +| B11.3c | Implement document sync/indexing gates for Meilisearch or the chosen embedding/search backend. | Captain document sync/indexing services, document controller `sync`, existing Meilisearch engine, local embedding/config boundaries. | Tests cover disabled config, failed sync observability, successful fake backend indexing, account-scoped document lookup, and no external network call in default test mode. | Done by `feat(captain): gate document sync backend` | | B11.3d | Finish Copilot task, suggestion, and tool-call persistence depth. | `resource :tasks` routes in `reference/chatwoot/config/routes.rb:62-89`, Copilot/Captain task services, dashboard Copilot clients. | Tests cover rewrite/summarize/reply suggestion/label suggestion/follow-up request payloads, persisted task/tool-call records, reloadable serializers, account/user scoping, and raw disabled state when provider config is missing. | Todo | | B11.3e | Add streaming/realtime compatibility or deterministic non-streaming fallback. | Chatwoot Captain/Copilot streaming paths, push payloads, current Go dispatcher/realtime boundaries. | Tests prove endpoints never return placeholder success/local envelopes; non-streaming fallback payloads are deterministic and frontend-compatible until true streaming is implemented. | Todo | @@ -865,6 +865,16 @@ B11.3c-B11.3e follow-up contracts: | B11.3d Copilot tasks/tool-calls | Rewrite, summarize, reply suggestion, label suggestion, and follow-up task routes must accept Chatwoot request payloads, persist enough task/tool-call output for reloads, and serialize raw frontend payloads. | B11 remains `Doing` until task/tool-call fixtures cover account/user scoping and no-provider fallback. | | B11.3e streaming/realtime | Streaming can be deferred only with a deterministic non-streaming response or explicit disabled state. Realtime push payloads should use the same thread/message serializer already landed in B11.2. | B11 moves to `Review` only after every streaming/realtime endpoint has a tested frontend-compatible fallback. | +B11.3c current checkpoint: + +- Reference files inspected for this slice: `reference/chatwoot/enterprise/app/controllers/api/v1/accounts/captain/documents_controller.rb#sync`, `Captain::Documents::SyncService`, `Captain::Documents::PerformSyncJob`, `Captain::Document`, and document Jbuilder payload fields. +- `CaptainDocumentService` now exposes a fakeable `CaptainDocumentSyncBackend` boundary and `SyncDocumentByAccount` worker-style entry point, separate from the controller's immediate `202 Accepted` queue marker. +- Missing sync backend marks the document `failed` with `last_sync_error_code=sync_disabled`, making disabled external configuration observable instead of silently attempting HTTP/LLM work. +- Backend execution failures mark `sync_error`; backend-supplied codes such as `access_denied` are preserved for frontend/reporting visibility. +- Fake successful sync updates content, title, normalized content fingerprint, `last_synced_at`, `last_sync_attempted_at`, `sync_status=synced`, and clears the previous error code. +- Account-scoped document lookup prevents another account from invoking sync work against a document primary key. +- Default tests do not open external sockets or call an LLM provider; fake backend tests cover success/failure states. + B11 landing rules: | Area | Landing requirement | Done signal | @@ -1283,7 +1293,7 @@ Enterprise tracking table: | P4.3 | Custom roles/permissions | `internal/model/custom_role.go`, `internal/service/custom_role_service.go`, `internal/middleware/role_check.go`, `internal/handler/api/v1/custom_role_handler.go` | Permission arrays, raw admin payloads, administrator gates, account-user resolution, profile permissions, and delete nullification are implemented; keep in Review until frontend smoke covers enterprise settings. | Review | | P4.4 | Agent capacity | `internal/model/agent_capacity_policy.go`, `internal/service/agent_capacity_policy_service.go`, `internal/handler/api/v1/agent_capacity_handler.go`, `internal/autoassignment/*` | First align AgentCapacityPolicy API, serializers, nested users, and `InboxCapacityLimit` data contracts; then enforce capacity in manual/auto assignment. | Review | | P4.5 | Inbox limits | `internal/model/agent_capacity_policy.go` `InboxCapacityLimit`, plus legacy `internal/model/inbox_limit.go`, `internal/service/inbox_limit_service.go`, `internal/repository/inbox_limit_repo.go`, `internal/handler/api/v1/inbox_limit_handler.go` | Chatwoot enterprise nested `agent_capacity_policies/:policy_id/inbox_limits`, assignment-path enforcement, and account-level `usage_limits[:inboxes]` create-path enforcement are covered. | Review | -| P4.6 | Captain/Copilot | `internal/model/captain_models.go`, `internal/model/copilot_models.go`, `internal/service/captain_*`, `internal/service/copilot_*`, `internal/handler/api/v1/captain_*`, `internal/handler/api/v1/copilot_handler.go` | Captain resource payloads through B11.1c, Copilot thread/message payloads through B11.2, Captain preferences through B11.3a, and Captain playground through B11.3b are aligned; complete B11.3c document/LLM/Meilisearch gates, B11.3d Copilot task/tool-call depth, and B11.3e streaming compatibility. | Doing | +| P4.6 | Captain/Copilot | `internal/model/captain_models.go`, `internal/model/copilot_models.go`, `internal/service/captain_*`, `internal/service/copilot_*`, `internal/handler/api/v1/captain_*`, `internal/handler/api/v1/copilot_handler.go` | Captain resource payloads through B11.1c, Copilot thread/message payloads through B11.2, Captain preferences through B11.3a, Captain playground through B11.3b, and document sync gates through B11.3c are aligned; complete B11.3d Copilot task/tool-call depth and B11.3e streaming compatibility. | Doing | | P4.7 | CSAT | `internal/csat/*`, `internal/automation/csat_survey_*`, `internal/handler/api/v1/csat_*`, `internal/service/csat_metrics_service.go` | Account-side list/metrics/review-note payloads, public update depth, resolve-triggered generic survey send, and download CSV are aligned; channel-specific template send hooks remain Phase 5 integration work. | Review | | P4.8 | Automation and macros | `internal/automation/*`, `internal/handler/api/v1/automation_rule_handler.go`, `internal/handler/api/v1/macro_handler.go` | Automation-rule CRUD/listener/log/external-action delivery and macro CRUD/display-ID execution parity are implemented; durable delayed actions and deeper macro attachment/file parity remain B9.3/B9.4 follow-ups. | Review | | P4.9 | Assignment policies | `internal/autoassignment/*`, `internal/automation/agent_bot_rule_listener.go` | Match Chatwoot assignment policy behavior and availability/capacity rules. | Doing | @@ -1300,7 +1310,7 @@ Enterprise work package breakdown: | Audit | Audit model parity, mutating action coverage, request metadata, filters/pagination, admin endpoint payloads. | List/admin payload parity is covered by `feat(audit): align chatwoot audit log payloads`; named representative writer coverage is covered by `feat(audit): record enterprise mutations` and `feat(audit): cover operational mutations`. | Review | | Custom roles | Permission-key parity, account-user role resolution, policy middleware, create/update/delete behavior. | Permission arrays, raw admin payloads, admin gates, RBAC/account-scope/profile resolution, and delete nullification are covered by `feat(custom-roles): align chatwoot permissions`. | Review | | Inbox limits | Account/inbox limit models, create/update enforcement, UI-readable limit responses, admin overrides. | Chatwoot nested capacity-limit API, assignment-path enforcement, and account-level create-limit `402` behavior are covered by B5/B7/B10.4. | Review | -| Captain/Copilot | Assistants, inbox bindings, scenarios, responses, documents, tools, preferences, copilot threads/messages, tasks, streaming/tool-call behavior. | Assistant CRUD/tools/inbox binding fixtures are covered by `feat(captain): align assistant resources`; scenario/custom-tool fixtures are covered by `feat(captain): align scenario and tool payloads`; document/assistant-response/bulk/custom-tool-test fixtures are covered by `feat(captain): align document response actions`; Copilot thread/message fixtures are covered by `feat(copilot): align thread message payloads`; preferences fixtures are covered by `feat(captain): align preferences payloads`; playground fixtures are covered by `feat(captain): align playground fallback`; B11.3c-B11.3e now track external LLM/document gates, remaining Copilot task/tool-call behavior, streaming fallback, and frontend smoke screens. | Doing | +| Captain/Copilot | Assistants, inbox bindings, scenarios, responses, documents, tools, preferences, copilot threads/messages, tasks, streaming/tool-call behavior. | Assistant CRUD/tools/inbox binding fixtures are covered by `feat(captain): align assistant resources`; scenario/custom-tool fixtures are covered by `feat(captain): align scenario and tool payloads`; document/assistant-response/bulk/custom-tool-test fixtures are covered by `feat(captain): align document response actions`; Copilot thread/message fixtures are covered by `feat(copilot): align thread message payloads`; preferences fixtures are covered by `feat(captain): align preferences payloads`; playground fixtures are covered by `feat(captain): align playground fallback`; document sync gate fixtures are covered by `feat(captain): gate document sync backend`; B11.3d-B11.3e now track remaining Copilot task/tool-call behavior, streaming fallback, and frontend smoke screens. | Doing | Enterprise acceptance gates: @@ -1541,3 +1551,4 @@ Verification milestone gates: - 2026-06-05: B11.2 Copilot thread/message checkpoint prepared as `feat(copilot): align thread message payloads`; Copilot threads now accept Chatwoot `{ message, assistant_id, conversation_id }`, validate account-scoped assistants, create initial user plus safe no-LLM assistant messages, and return raw thread serializers. Nested Copilot messages now list/create raw message payloads with embedded thread push data, current account/user scoping, ascending message order, and no local envelopes. Focused Copilot handler/service/repository tests, Captain/Copilot handler/service tests, handler/service/repository package tests, escalated full `go test ./...`, and `git diff --check` passed. Next slice is B11.3 document sync/indexing, Meilisearch/embedding gates, Copilot tasks/preferences/tool-call depth, and streaming/realtime compatibility. - 2026-06-05: B11.3a Captain preferences checkpoint prepared as `feat(captain): align preferences payloads`; Captain preferences now use Chatwoot's raw `{ providers, models, features }` show/update payload, persist `captain_models`/`captain_features` on accounts, merge partial preference updates, reject invalid model selections with `422`, gate updates to administrators, and expose exact no-trailing-slash GET/PUT preference routes. Focused CaptainPreference handler/service tests, Captain/Copilot handler/service tests, handler/service/model/router package tests, migration tests, route dump/parity regeneration, escalated full `go test ./...`, and `git diff --check` passed. Next slice is B11.3b Captain playground parity. - 2026-06-05: B11.3b Captain playground checkpoint prepared as `feat(captain): align playground fallback`; playground now accepts Chatwoot top-level `message_content/message_history`, scopes assistant lookup to the account, returns raw legacy `{ content }` or v2 `{ response }`, appends current v2 user messages without duplication, preserves history into the fakeable LLM boundary, and returns deterministic no-LLM fallback JSON instead of 500. Focused CaptainAssistant/Captain tests, handler/service package tests, full `go test ./...`, and `git diff --check` passed. Next slice is B11.3c document sync/indexing and Meilisearch/embedding gates. +- 2026-06-05: B11.3c Captain document sync gate checkpoint prepared as `feat(captain): gate document sync backend`; document sync now has a fakeable backend boundary and account-scoped worker entry, disabled config is recorded as `sync_disabled`, backend failures become `sync_error` or preserve explicit backend error codes, fake success updates content/title/fingerprint/timestamps as `synced`, and default tests avoid external network/LLM calls. Focused Captain/Copilot tests, handler/service package tests, full `go test ./...`, and `git diff --check` passed. Next slice is B11.3d Copilot task/tool-call persistence. diff --git a/internal/service/captain_document_service.go b/internal/service/captain_document_service.go index 756471fa..d6d31777 100644 --- a/internal/service/captain_document_service.go +++ b/internal/service/captain_document_service.go @@ -21,6 +21,17 @@ type CaptainDocumentService struct { documentRepo *repository.CaptainDocumentRepo assistantRepo *repository.CaptainAssistantRepo llmProvider llm.Provider + syncBackend CaptainDocumentSyncBackend +} + +type CaptainDocumentSyncBackend interface { + SyncCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) +} + +type CaptainDocumentSyncResult struct { + Content string + Title string + ErrorCode string } // NewCaptainDocumentService creates a new CaptainDocumentService. @@ -39,6 +50,10 @@ func NewCaptainDocumentService( return s } +func (s *CaptainDocumentService) SetSyncBackend(syncBackend CaptainDocumentSyncBackend) { + s.syncBackend = syncBackend +} + // --- Request DTOs --- // CreateDocumentRequest is the DTO for creating a document. @@ -201,6 +216,79 @@ func (s *CaptainDocumentService) MarkSyncing(ctx context.Context, accountID, id return s.documentRepo.GetByAccountAndID(ctx, accountID, id) } +func (s *CaptainDocumentService) SyncDocumentByAccount(ctx context.Context, accountID, id uint) (*model.CaptainDocument, error) { + doc, err := s.documentRepo.GetByAccountAndID(ctx, accountID, id) + if err != nil { + return nil, fmt.Errorf("document not found: %w", err) + } + if err := s.markDocumentSyncStarted(ctx, doc); err != nil { + return nil, err + } + if s.syncBackend == nil { + return s.markDocumentSyncFailed(ctx, accountID, id, "sync_disabled") + } + + result, err := s.syncBackend.SyncCaptainDocument(ctx, doc) + if err != nil { + updated, markErr := s.markDocumentSyncFailed(ctx, accountID, id, "sync_error") + if markErr != nil { + return nil, markErr + } + return updated, fmt.Errorf("sync document: %w", err) + } + if result == nil { + return s.markDocumentSyncFailed(ctx, accountID, id, "sync_error") + } + if result.ErrorCode != "" { + return s.markDocumentSyncFailed(ctx, accountID, id, result.ErrorCode) + } + if strings.TrimSpace(result.Content) == "" { + return s.markDocumentSyncFailed(ctx, accountID, id, "content_empty") + } + + doc.Content = strings.TrimSpace(result.Content) + if strings.TrimSpace(result.Title) != "" { + doc.Name = strings.TrimSpace(result.Title) + } + doc.ContentFingerprint = computeFingerprint(doc.Content) + doc.Status = model.DocumentStatusCompleted + doc.SyncStatus = model.DocumentSyncStatusSynced + doc.LastSyncErrorCode = "" + now := time.Now().Unix() + doc.LastSyncedAt = &now + doc.LastSyncAttemptedAt = &now + if err := s.documentRepo.Update(ctx, doc); err != nil { + return nil, fmt.Errorf("update synced document: %w", err) + } + return s.documentRepo.GetByAccountAndID(ctx, accountID, id) +} + +func (s *CaptainDocumentService) markDocumentSyncStarted(ctx context.Context, doc *model.CaptainDocument) error { + now := time.Now().Unix() + doc.LastSyncAttemptedAt = &now + doc.SyncStatus = model.DocumentSyncStatusPending + doc.LastSyncErrorCode = "" + if err := s.documentRepo.Update(ctx, doc); err != nil { + return fmt.Errorf("mark document syncing: %w", err) + } + return nil +} + +func (s *CaptainDocumentService) markDocumentSyncFailed(ctx context.Context, accountID, id uint, code string) (*model.CaptainDocument, error) { + doc, err := s.documentRepo.GetByAccountAndID(ctx, accountID, id) + if err != nil { + return nil, fmt.Errorf("document not found: %w", err) + } + now := time.Now().Unix() + doc.SyncStatus = model.DocumentSyncStatusFailed + doc.LastSyncAttemptedAt = &now + doc.LastSyncErrorCode = code + if err := s.documentRepo.Update(ctx, doc); err != nil { + return nil, fmt.Errorf("mark document sync failed: %w", err) + } + return s.documentRepo.GetByAccountAndID(ctx, accountID, id) +} + // --- Document Processing --- // ProcessDocument extracts content and generates embedding for a document. @@ -339,6 +427,7 @@ func (s *CaptainDocumentService) fetchContent(ctx context.Context, url string) ( // computeFingerprint generates a SHA256 hash of content for deduplication. func computeFingerprint(content string) string { h := sha256.New() - h.Write([]byte(content)) + normalized := strings.Join(strings.Fields(content), " ") + h.Write([]byte(normalized)) return fmt.Sprintf("%x", h.Sum(nil)) } diff --git a/internal/service/captain_document_service_test.go b/internal/service/captain_document_service_test.go new file mode 100644 index 00000000..52157acf --- /dev/null +++ b/internal/service/captain_document_service_test.go @@ -0,0 +1,119 @@ +package service + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/gochat/gochat/internal/model" + "github.com/gochat/gochat/internal/repository" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func setupCaptainDocumentServiceTest(t *testing.T) (*gorm.DB, *CaptainDocumentService) { + t.Helper() + dbName := fmt.Sprintf("file:%s?mode=memory&cache=private", t.Name()) + db, err := gorm.Open(sqlite.Open(dbName), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.Account{}, &model.CaptainAssistant{}, &model.CaptainDocument{})) + t.Cleanup(func() { + sqlDB, _ := db.DB() + sqlDB.Close() + }) + documentRepo := repository.NewCaptainDocumentRepo(db) + assistantRepo := repository.NewCaptainAssistantRepo(db) + return db, NewCaptainDocumentService(documentRepo, nil, assistantRepo) +} + +func seedCaptainDocumentSyncFixture(t *testing.T, db *gorm.DB) (*model.Account, *model.Account, *model.CaptainDocument) { + t.Helper() + account := &model.Account{Name: "Captain Org", Active: true} + otherAccount := &model.Account{Name: "Other Org", Active: true} + require.NoError(t, db.Create(account).Error) + require.NoError(t, db.Create(otherAccount).Error) + assistant := &model.CaptainAssistant{AccountID: account.ID, Name: "Fin", Description: "Support", Status: model.AssistantStatusActive} + require.NoError(t, db.Create(assistant).Error) + doc := &model.CaptainDocument{ + AccountID: account.ID, + AssistantID: assistant.ID, + Name: "Help", + ExternalLink: "https://example.com/help", + Status: model.DocumentStatusCompleted, + SyncStatus: model.DocumentSyncStatusSynced, + } + require.NoError(t, db.Create(doc).Error) + return account, otherAccount, doc +} + +func TestCaptainDocumentService_SyncDocumentByAccountDisabledMarksFailed(t *testing.T) { + db, svc := setupCaptainDocumentServiceTest(t) + account, _, doc := seedCaptainDocumentSyncFixture(t, db) + + updated, err := svc.SyncDocumentByAccount(context.Background(), account.ID, doc.ID) + require.NoError(t, err) + assert.Equal(t, model.DocumentSyncStatusFailed, updated.SyncStatus) + assert.Equal(t, "sync_disabled", updated.LastSyncErrorCode) + assert.NotNil(t, updated.LastSyncAttemptedAt) +} + +func TestCaptainDocumentService_SyncDocumentByAccountUsesFakeBackend(t *testing.T) { + db, svc := setupCaptainDocumentServiceTest(t) + account, _, doc := seedCaptainDocumentSyncFixture(t, db) + backend := &captainDocumentFakeSyncBackend{result: &CaptainDocumentSyncResult{Title: "Fresh Help", Content: "hello\n\nworld"}} + svc.SetSyncBackend(backend) + + updated, err := svc.SyncDocumentByAccount(context.Background(), account.ID, doc.ID) + require.NoError(t, err) + assert.Equal(t, doc.ID, backend.documentID) + assert.Equal(t, "Fresh Help", updated.Name) + assert.Equal(t, "hello\n\nworld", updated.Content) + assert.Equal(t, computeFingerprint("hello world"), updated.ContentFingerprint) + assert.Equal(t, model.DocumentStatusCompleted, updated.Status) + assert.Equal(t, model.DocumentSyncStatusSynced, updated.SyncStatus) + assert.Empty(t, updated.LastSyncErrorCode) + assert.NotNil(t, updated.LastSyncedAt) +} + +func TestCaptainDocumentService_SyncDocumentByAccountRecordsBackendFailures(t *testing.T) { + db, svc := setupCaptainDocumentServiceTest(t) + account, _, doc := seedCaptainDocumentSyncFixture(t, db) + svc.SetSyncBackend(&captainDocumentFakeSyncBackend{err: errors.New("boom")}) + + updated, err := svc.SyncDocumentByAccount(context.Background(), account.ID, doc.ID) + require.Error(t, err) + assert.Equal(t, model.DocumentSyncStatusFailed, updated.SyncStatus) + assert.Equal(t, "sync_error", updated.LastSyncErrorCode) + + svc.SetSyncBackend(&captainDocumentFakeSyncBackend{result: &CaptainDocumentSyncResult{ErrorCode: "access_denied"}}) + updated, err = svc.SyncDocumentByAccount(context.Background(), account.ID, doc.ID) + require.NoError(t, err) + assert.Equal(t, model.DocumentSyncStatusFailed, updated.SyncStatus) + assert.Equal(t, "access_denied", updated.LastSyncErrorCode) +} + +func TestCaptainDocumentService_SyncDocumentByAccountScopesDocument(t *testing.T) { + db, svc := setupCaptainDocumentServiceTest(t) + _, otherAccount, doc := seedCaptainDocumentSyncFixture(t, db) + svc.SetSyncBackend(&captainDocumentFakeSyncBackend{result: &CaptainDocumentSyncResult{Content: "fresh"}}) + + _, err := svc.SyncDocumentByAccount(context.Background(), otherAccount.ID, doc.ID) + require.Error(t, err) +} + +type captainDocumentFakeSyncBackend struct { + result *CaptainDocumentSyncResult + err error + documentID uint +} + +func (b *captainDocumentFakeSyncBackend) SyncCaptainDocument(ctx context.Context, doc *model.CaptainDocument) (*CaptainDocumentSyncResult, error) { + b.documentID = doc.ID + if b.err != nil { + return nil, b.err + } + return b.result, nil +}