feat(messages): queue delivery statuses
This commit is contained in:
@@ -16,12 +16,12 @@ Build GoChat as a Go backend that can directly reuse the frontend from `referenc
|
||||
|
||||
## Current Baseline
|
||||
|
||||
- Current tracking checkpoint: 2026-06-05 after `cda7d02 feat(conversations): queue bulk actions`, with this checkpoint prepared as `feat(messages): queue send replies`.
|
||||
- Latest implementation checkpoint: this checkpoint, prepared as `feat(messages): queue send replies`.
|
||||
- Current tracking checkpoint: 2026-06-05 after `1a4b4a2 feat(messages): queue send replies`, with this checkpoint prepared as `feat(messages): queue delivery statuses`.
|
||||
- Latest implementation checkpoint: this checkpoint, prepared as `feat(messages): queue delivery statuses`.
|
||||
- Latest documentation-only checkpoint: this checkpoint, prepared as `docs: land active parity tracking board`; this document is now the active follow-up plan and supersedes `.hermes/plans/*`.
|
||||
- Worktree status at this implementation checkpoint: B11.1a aligns Captain assistant CRUD/tools/inbox bindings; B11.1b aligns Captain scenarios and custom tools; B11.1c aligns Captain documents, assistant responses, bulk actions, and custom-tool test payloads; B11.2 aligns Copilot thread/message create/list/get/delete payloads, account/user scoping, and no-LLM fallback persistence; B11.3a aligns Captain preferences show/update payloads and account-level model/feature storage; B11.3b aligns Captain playground request/response payloads, account scoping, v2 history handling, and no-LLM fallback; B11.3c adds the fakeable Captain document sync backend gate with disabled, failed, and fake-success states; B11.3d aligns Captain task request/response payloads, no-provider disabled states, follow-up context, suggestion persistence, and Copilot message tool-call key validation; B11.3e aligns Captain stream DTOs/disabled SSE fallbacks and Copilot push-event payload shapes; B12.1 adds the reusable GoChat server/seed entrypoint plus a Meilisearch-first reused Chatwoot frontend smoke harness and report; B12.2a adds API smoke assertions for auth/profile, inbox, conversation/messages, contact/company, widget config/message, and public CSAT; B12.2b adds a zero-dependency Chrome DevTools browser smoke that loads the reused Chatwoot login and dashboard entrypoints through Vite and checks browser auth/dashboard API requests; B12.3a adds enterprise API smoke assertions for SLA reports/download, CSAT reports/download, automation/macros, audit/custom roles, capacity, Captain, and Copilot; B12.3b adds reused-frontend enterprise browser route navigation for SLA, CSAT, automation, macros, audit logs, custom roles, capacity, Captain, and Copilot request coverage; P5.1 adds the PostgreSQL-backed durable `background_jobs` model/migration plus WorkerPool enqueue, schedule, retry/backoff, dead-letter, idempotency, stale-lock recovery, and focused tests; P5.2 wires `channel.Dispatcher` and `dispatch.EventDispatcher` async paths into durable event jobs with worker replay tests; P5.3 queues Meilisearch write-side index/delete jobs for conversations, messages, contacts, companies, and articles while keeping search reads Meilisearch-first; P5.4 queues automation webhook and email transcript side effects as durable jobs while preserving fakeable delivery boundaries; P5.5 queues Chatwoot-style macro execute fan-out through durable `automation:macro_execution` jobs; P5.6 queues resolve-triggered CSAT survey sends and WhatsApp/Twilio CSAT template creation through durable jobs; P5.7 queues Chatwoot enterprise SLA account scans and applied-SLA evaluation jobs through the durable worker; P5.8 queues Chatwoot-style contact export artifact generation through durable `contact:export` jobs; P5.10 now queues Chatwoot `SendReplyJob`-style outbound message delivery through durable `message:send_reply` jobs; P5.11 queues Captain document sync requests through durable `captain:document_sync` jobs; P5.12 queues scheduled item fan-out, one-off campaigns, snoozed conversation reopening, account auto-resolution, widget/public message status updates, and account conversation bulk actions through durable jobs. Next active implementation slice is P5.10 provider delivery-status job depth, followed by P5.9 inbound webhook deferral, remaining P5.11 Captain/Copilot jobs, and P5.13 analytics aggregation.
|
||||
- Worktree status at this implementation checkpoint: B11.1a aligns Captain assistant CRUD/tools/inbox bindings; B11.1b aligns Captain scenarios and custom tools; B11.1c aligns Captain documents, assistant responses, bulk actions, and custom-tool test payloads; B11.2 aligns Copilot thread/message create/list/get/delete payloads, account/user scoping, and no-LLM fallback persistence; B11.3a aligns Captain preferences show/update payloads and account-level model/feature storage; B11.3b aligns Captain playground request/response payloads, account scoping, v2 history handling, and no-LLM fallback; B11.3c adds the fakeable Captain document sync backend gate with disabled, failed, and fake-success states; B11.3d aligns Captain task request/response payloads, no-provider disabled states, follow-up context, suggestion persistence, and Copilot message tool-call key validation; B11.3e aligns Captain stream DTOs/disabled SSE fallbacks and Copilot push-event payload shapes; B12.1 adds the reusable GoChat server/seed entrypoint plus a Meilisearch-first reused Chatwoot frontend smoke harness and report; B12.2a adds API smoke assertions for auth/profile, inbox, conversation/messages, contact/company, widget config/message, and public CSAT; B12.2b adds a zero-dependency Chrome DevTools browser smoke that loads the reused Chatwoot login and dashboard entrypoints through Vite and checks browser auth/dashboard API requests; B12.3a adds enterprise API smoke assertions for SLA reports/download, CSAT reports/download, automation/macros, audit/custom roles, capacity, Captain, and Copilot; B12.3b adds reused-frontend enterprise browser route navigation for SLA, CSAT, automation, macros, audit logs, custom roles, capacity, Captain, and Copilot request coverage; P5.1 adds the PostgreSQL-backed durable `background_jobs` model/migration plus WorkerPool enqueue, schedule, retry/backoff, dead-letter, idempotency, stale-lock recovery, and focused tests; P5.2 wires `channel.Dispatcher` and `dispatch.EventDispatcher` async paths into durable event jobs with worker replay tests; P5.3 queues Meilisearch write-side index/delete jobs for conversations, messages, contacts, companies, and articles while keeping search reads Meilisearch-first; P5.4 queues automation webhook and email transcript side effects as durable jobs while preserving fakeable delivery boundaries; P5.5 queues Chatwoot-style macro execute fan-out through durable `automation:macro_execution` jobs; P5.6 queues resolve-triggered CSAT survey sends and WhatsApp/Twilio CSAT template creation through durable jobs; P5.7 queues Chatwoot enterprise SLA account scans and applied-SLA evaluation jobs through the durable worker; P5.8 queues Chatwoot-style contact export artifact generation through durable `contact:export` jobs; P5.10 now queues Chatwoot `SendReplyJob`-style outbound message delivery through durable `message:send_reply` jobs and provider delivery-status/read-receipt updates through durable webhook status jobs; P5.11 queues Captain document sync requests through durable `captain:document_sync` jobs; P5.12 queues scheduled item fan-out, one-off campaigns, snoozed conversation reopening, account auto-resolution, widget/public message status updates, and account conversation bulk actions through durable jobs. Next active implementation slice is P5.9 inbound webhook deferral, followed by remaining P5.11 Captain/Copilot jobs and P5.13 analytics aggregation.
|
||||
- `go test ./...` passes.
|
||||
- Route dump succeeds with `TOTAL: 830` after adding the Chatwoot-compatible applied-SLA index route.
|
||||
- Route dump succeeds with `TOTAL: 832` after adding the Chatwoot-compatible Twilio delivery-status route plus the legacy namespaced alias.
|
||||
- Route parity artifacts now exist under `docs/parity/` and are generated by `cmd/route_parity`.
|
||||
- Tracked frontend-critical route audit covers 277 Chatwoot routes: 270 exact, 0 method-compatible, 7 parameter-compatible, 0 missing. The 7 parameter-compatible routes are Gin-internal parameter-name differences for nested AgentCapacityPolicy users/inbox limits; the external URL shape is equivalent.
|
||||
- `/api/v1/widget` stubs are burned down and public inbox/contact/conversation/message core flows are backed by real handlers.
|
||||
@@ -45,18 +45,17 @@ Next ordered checkpoints:
|
||||
|
||||
| Order | Slice | Required outcome | Primary verification |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | P5.10 outbound jobs | Queue Chatwoot `SendReplyJob`-style outbound provider delivery and provider delivery-status updates. | Focused message delivery worker tests, provider fake tests, and `go test ./...`. |
|
||||
| 2 | P5.9 inbound jobs | Move provider webhook persistence/dispatch that Chatwoot defers into durable jobs while keeping HTTP acknowledgement behavior compatible. | Provider webhook handler tests plus worker replay/idempotency tests. |
|
||||
| 3 | P5.11 remaining Captain/Copilot jobs | Queue Captain crawl/response/embedding and Copilot response work behind fakeable provider gates. | Captain/Copilot worker tests and disabled/failure-state fixtures. |
|
||||
| 4 | P5.13 analytics aggregation | Replace frontend-visible placeholder report values with scheduled or cached real aggregations. | Report service/handler fixtures and freshness/idempotency tests. |
|
||||
| 5 | Phase 2/3 audit pass | Route/controller/serializer drift found by B12 or new reference inspection is captured as named slices, not free-form TODOs. | Regenerated route parity artifacts and fixture-backed serializer tests. |
|
||||
| 6 | Phase 6 placeholder burn-down | Remaining account/contact/conversation/message/inbox placeholder handlers are either real Chatwoot-compatible flows or explicitly tracked as unsupported reference gaps. | `rg` placeholder audit, route smoke, and endpoint-family tests. |
|
||||
| 1 | P5.9 inbound jobs | Move provider webhook persistence/dispatch that Chatwoot defers into durable jobs while keeping HTTP acknowledgement behavior compatible. | Provider webhook handler tests plus worker replay/idempotency tests. |
|
||||
| 2 | P5.11 remaining Captain/Copilot jobs | Queue Captain crawl/response/embedding and Copilot response work behind fakeable provider gates. | Captain/Copilot worker tests and disabled/failure-state fixtures. |
|
||||
| 3 | P5.13 analytics aggregation | Replace frontend-visible placeholder report values with scheduled or cached real aggregations. | Report service/handler fixtures and freshness/idempotency tests. |
|
||||
| 4 | Phase 2/3 audit pass | Route/controller/serializer drift found by B12 or new reference inspection is captured as named slices, not free-form TODOs. | Regenerated route parity artifacts and fixture-backed serializer tests. |
|
||||
| 5 | Phase 6 placeholder burn-down | Remaining account/contact/conversation/message/inbox placeholder handlers are either real Chatwoot-compatible flows or explicitly tracked as unsupported reference gaps. | `rg` placeholder audit, route smoke, and endpoint-family tests. |
|
||||
|
||||
## Handoff Contract
|
||||
|
||||
This checkpoint is intended to make the development plan complete enough to track without reading Hermes notes first.
|
||||
|
||||
- The next active implementation slice is Phase 5 remaining job parity, starting with P5.10 outbound message delivery. B12 has repeatable API/browser/enterprise smoke harnesses in Review; optional live failures should be converted into named slices instead of blocking Phase 5 job work.
|
||||
- The next active implementation slice is Phase 5 remaining job parity, starting with P5.9 inbound webhook deferral. B12 has repeatable API/browser/enterprise smoke harnesses in Review; optional live failures should be converted into named slices instead of blocking Phase 5 job work.
|
||||
- The Hermes search plan is fully represented by Phase 1/B6. Future search changes must be Meilisearch-first and must not reintroduce production DB fallback.
|
||||
- The Hermes automation/macro/CSAT plan is fully represented by B8/B9 and Phase 5. Durable delayed execution and channel-specific template delivery remain visible Phase 5 work, not hidden Done work.
|
||||
- Enterprise scope is fixed: SLA, Audit, CustomRole, AgentCapacity, Captain/Copilot, CSAT, InboxLimit, automation, macros, assignment policies, and related limits/workflows are in scope; SSO/SAML/LDAP/OIDC are out of scope.
|
||||
@@ -80,7 +79,7 @@ Open work after the current checkpoint:
|
||||
| Phase 2 | Route and controller parity audit | Doing | Ruby/Bundler unavailable, so Chatwoot route extraction currently uses static `routes.rb` fallback |
|
||||
| Phase 3 | Data and serializer parity | Doing | JSON fixture coverage is partial and still endpoint-family based |
|
||||
| Phase 4 | Enterprise feature completion | Doing | B7, B8, B9, B10, and B11 are in Review; B12 reused frontend smoke harnesses exist and optional live runs can expose follow-up slices |
|
||||
| Phase 5 | Background jobs and integrations | Doing | P5.1/P5.2/P5.3/P5.4/P5.5/P5.6/P5.7/P5.8/P5.11/P5.12 durable worker, event dispatch, search indexing, automation delivery, macro, CSAT survey/template, SLA scan, contact export, Captain document sync, conversation maintenance, message status update, and account bulk-action cores are in Review; remaining Captain/Copilot/provider/outbound job integration remains open |
|
||||
| Phase 5 | Background jobs and integrations | Doing | P5.1/P5.2/P5.3/P5.4/P5.5/P5.6/P5.7/P5.8/P5.10/P5.11/P5.12 durable worker, event dispatch, search indexing, automation delivery, macro, CSAT survey/template, SLA scan, contact export, outbound/provider delivery status, Captain document sync, conversation maintenance, message status update, and account bulk-action cores are in Review; remaining inbound webhook, Captain/Copilot, and analytics aggregation jobs remain open |
|
||||
| Phase 6 | Core placeholder burn-down | Doing | account/contact/conversation/message/inbox placeholder groups remain broad |
|
||||
| Phase 7 | Verification harness | Review | B12.1 boot/readiness, B12.2a API assertions, B12.2b browser smoke harness, B12.3a enterprise API assertions, and B12.3b enterprise browser route navigation exist; optional live Meilisearch/full-browser runs remain environment-dependent |
|
||||
|
||||
@@ -90,13 +89,12 @@ This table is the shortest authoritative handoff view. If an older lower section
|
||||
|
||||
| Priority | Workstream | Current state | Next checkpoint | Commit close rule |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | P5.10 outbound message delivery | SendReplyJob-style durable provider delivery is implemented; provider delivery-status job depth remains open. | Queue provider delivery-status updates where Chatwoot defers them and keep retry/status transitions idempotent. | Commit code, tests, P5.10 table row, commit ledger row, and progress log together. |
|
||||
| 2 | P5.9 inbound webhook deferral | Provider ingress routes verify, persist, and dispatch synchronously. | Preserve provider HTTP ack behavior while moving Chatwoot-deferred persistence/dispatch into retryable jobs. | Provider HTTP tests plus worker replay/idempotency tests must pass. |
|
||||
| 3 | P5.11 Captain/Copilot jobs | Captain document sync is queued; crawl/response/embedding/Copilot response jobs remain open. | Add fakeable durable jobs for remaining Captain/Copilot async work and disabled-provider states. | Focused Captain/Copilot worker tests plus full `go test ./...`. |
|
||||
| 4 | P5.13 reports/analytics | Frontend smoke harness exists; `analytics_service` still has placeholder aggregation paths. | Replace frontend-visible report placeholders with real scheduled/cached aggregations. | Report fixtures verify values, cache/freshness behavior, and no hidden placeholder JSON. |
|
||||
| 5 | Phase 2/3 drift | Tracked route parity is 0 missing for the current critical set; serializer fixtures remain partial. | Expand route/serializer fixtures when smoke or reference inspection exposes drift. | Regenerate parity artifacts and add endpoint-family fixture tests. |
|
||||
| 6 | Phase 6 placeholder audit | Widget/public/webhook critical placeholders are burned down; account/contact/conversation/message/inbox audit remains broad. | Run a fresh placeholder audit and assign every frontend-reachable stub to a tracked owner. | `rg` audit result is recorded and no reused-frontend blocker is ownerless. |
|
||||
| 7 | B12 optional live smoke | API/browser/enterprise smoke commands are checked in; live runs need PostgreSQL, Redis, Meilisearch, Vite, and Chrome. | Run full live smoke when environment is available and map failures to the board. | `docs/parity/frontend_smoke_report.md` records pass/fail and linked owners. |
|
||||
| 1 | P5.9 inbound webhook deferral | Provider ingress routes verify, persist, and dispatch synchronously. | Preserve provider HTTP ack behavior while moving Chatwoot-deferred persistence/dispatch into retryable jobs. | Provider HTTP tests plus worker replay/idempotency tests must pass. |
|
||||
| 2 | P5.11 Captain/Copilot jobs | Captain document sync is queued; crawl/response/embedding/Copilot response jobs remain open. | Add fakeable durable jobs for remaining Captain/Copilot async work and disabled-provider states. | Focused Captain/Copilot worker tests plus full `go test ./...`. |
|
||||
| 3 | P5.13 reports/analytics | Frontend smoke harness exists; `analytics_service` still has placeholder aggregation paths. | Replace frontend-visible report placeholders with real scheduled/cached aggregations. | Report fixtures verify values, cache/freshness behavior, and no hidden placeholder JSON. |
|
||||
| 4 | Phase 2/3 drift | Tracked route parity is 0 missing for the current critical set; serializer fixtures remain partial. | Expand route/serializer fixtures when smoke or reference inspection exposes drift. | Regenerate parity artifacts and add endpoint-family fixture tests. |
|
||||
| 5 | Phase 6 placeholder audit | Widget/public/webhook critical placeholders are burned down; account/contact/conversation/message/inbox audit remains broad. | Run a fresh placeholder audit and assign every frontend-reachable stub to a tracked owner. | `rg` audit result is recorded and no reused-frontend blocker is ownerless. |
|
||||
| 6 | B12 optional live smoke | API/browser/enterprise smoke commands are checked in; live runs need PostgreSQL, Redis, Meilisearch, Vite, and Chrome. | Run full live smoke when environment is available and map failures to the board. | `docs/parity/frontend_smoke_report.md` records pass/fail and linked owners. |
|
||||
|
||||
## Tracking Artifacts
|
||||
|
||||
@@ -117,7 +115,8 @@ This ledger records the committed parity checkpoints that future slices should b
|
||||
|
||||
| Commit | Scope | Verification summary | Follow-up state |
|
||||
| --- | --- | --- | --- |
|
||||
| `feat(messages): queue send replies` | Advances P5.10 with a durable Chatwoot `SendReplyJob` equivalent for outbound message delivery. `MessageService.Create` now enqueues `message:send_reply` on the `high` queue when a WorkerPool is configured, keeps the legacy `message.outgoing` dispatch fallback only for no-worker construction, and worker replay reloads message/conversation/inbox/contact, skips already-sent/private/voice-call/non-outgoing messages, calls the registered channel provider, persists `source_id`, and records retryable provider failures as `failed` with `content_attributes.external_error`. | `go test ./internal/service -run 'MessageDeliveryWorker\|MessageService' -count=1`; `go test ./internal/service ./internal/worker ./internal/app -count=1`; `go test ./...`; `git diff --check`; full verification recorded in the P5.10 section. | Move P5.10 send-reply core to Review; continue P5.10 provider delivery-status job depth, then P5.9 inbound webhook deferral, P5.11 Captain/Copilot remaining jobs, and P5.13 analytics aggregation. |
|
||||
| `feat(messages): queue delivery statuses` | Completes P5.10 provider delivery-status depth. `IncomingPersister` can now enqueue direct message status updates as `webhook:message_status_update` jobs and contact-scoped read receipts as `webhook:contact_messages_status_update` jobs on the `low` queue; worker replay reloads the inbox, applies sent/delivered/read/failed transitions without downgrading already-read messages, upserts `delivery_statuses`, dispatches status events, and records provider failure details in `content_attributes.external_error`. Twilio delivery callbacks now return Chatwoot-style `204`, include the exact `/twilio/delivery_status` route plus the legacy `/webhooks/twilio/delivery_status` alias, resolve channels by `MessagingServiceSid` or `AccountSid`/`From`, and enqueue through the same durable path when the WorkerPool is wired. | `go test ./internal/handler/webhook -run 'IncomingPersister.*Status\|StatusJob\|TwilioDeliveryStatus' -count=1`; `go test ./internal/handler/webhook ./internal/service ./internal/worker ./internal/app -count=1`; `go run ./cmd/dump_routes > docs/parity/gochat_routes.txt`; `go run ./cmd/route_parity`; `go test ./...`; `git diff --check`; full verification recorded in the P5.10 section. | Move P5.10 to Review; next active slice is P5.9 inbound webhook deferral, then remaining P5.11 Captain/Copilot jobs and P5.13 analytics aggregation. |
|
||||
| `feat(messages): queue send replies` | Advances P5.10 with a durable Chatwoot `SendReplyJob` equivalent for outbound message delivery. `MessageService.Create` now enqueues `message:send_reply` on the `high` queue when a WorkerPool is configured, keeps the legacy `message.outgoing` dispatch fallback only for no-worker construction, and worker replay reloads message/conversation/inbox/contact, skips already-sent/private/voice-call/non-outgoing messages, calls the registered channel provider, persists `source_id`, and records retryable provider failures as `failed` with `content_attributes.external_error`. | `go test ./internal/service -run 'MessageDeliveryWorker\|MessageService' -count=1`; `go test ./internal/service ./internal/worker ./internal/app -count=1`; `go test ./...`; `git diff --check`; full verification recorded in the P5.10 section. | Send-reply core moved to Review; provider delivery-status depth is completed by `feat(messages): queue delivery statuses`. |
|
||||
| `docs: land active parity tracking board` | Documentation-only checkpoint that reconciles the current state after `cda7d02`. Adds the front-loaded Active Remaining Work Board, makes Phase 5 the active slice, records P5.10/P5.9/P5.11/P5.13 as the next implementation order, and updates stale B12/M6/M7 tracker rows to Review/Doing. | `git diff --check`; documentation-only change. | Start P5.10 outbound message delivery from the new active board. |
|
||||
| `feat(conversations): queue bulk actions` | Advances P5.12 with a durable Chatwoot `BulkActionsJob` equivalent for account conversation bulk actions. `POST /bulk_actions` now accepts frontend Chatwoot payloads without `action_name`, including `fields`, nested `labels.add/remove`, `snoozed_until`, and display-ID `ids`; when a WorkerPool is configured it enqueues `conversation:bulk_action` on the `medium` queue and returns empty `200 OK`. Worker replay scopes by account and display IDs, merges labels, applies status/team/assignee/snooze updates, and leaves same-display conversations in other accounts untouched. | `go test ./internal/service -run 'ConversationMaintenanceJobs.*BulkAction\|ConversationMaintenanceJobs' -count=1`; `go test ./internal/handler/api/v1 -run 'BulkActionHandler' -count=1`; `go test ./internal/service ./internal/handler/api/v1 ./internal/worker ./internal/app -count=1`; `go test ./...`; `git diff --check`; full verification recorded in the P5.12 section. | Move P5.12 account bulk actions to Review; continue remaining Captain/Copilot jobs, provider webhook/outbound delivery, inbound webhook job deferral, and analytics aggregation. |
|
||||
| `feat(conversations): queue message status updates` | Advances P5.12 with a durable Chatwoot `Conversations::UpdateMessageStatusJob` equivalent. Widget and public inbox `update_last_seen` paths now enqueue `conversation:update_message_status` on the `deferred` queue when a WorkerPool is configured. Worker replay validates `read`/`delivered`, skips missing conversations, and updates only non-incoming messages created at or before the viewing timestamp while preserving already-read and post-cutoff messages. | `go test ./internal/service -run 'ConversationMaintenanceJobs\|WidgetService_UpdateLastSeenQueuesMessageStatusJob' -count=1`; `go test ./internal/service ./internal/worker ./internal/app -count=1`; `go test ./...`; `git diff --check`; full verification recorded in the P5.12 section. | Move P5.12 update-message-status depth to Review; continue durable bulk-action fan-out, remaining Captain/Copilot jobs, provider webhook/outbound delivery, and analytics aggregation. |
|
||||
@@ -1609,7 +1608,7 @@ Reference job families to keep in scope:
|
||||
Known hotspots:
|
||||
|
||||
- `internal/worker/worker.go` has the committed P5.1 durable core and is already wired to dispatcher, search indexing, automation delivery, macro fan-out, CSAT survey/template sends, SLA scans, contact exports, Captain document sync, and conversation maintenance jobs.
|
||||
- `internal/service/message_delivery_worker.go`, provider delivery-status webhook services, and message status handlers remain the P5.10 focus. SendReplyJob-style outbound delivery is queued; provider delivery-status job depth remains open.
|
||||
- `internal/service/message_delivery_worker.go` and `internal/handler/webhook/incoming_persister_jobs.go` now cover P5.10 SendReplyJob-style outbound delivery plus provider delivery-status/read-receipt jobs.
|
||||
- `internal/handler/webhook/*` and provider services have synchronous verified ingress/persistence; P5.9 must move Chatwoot-deferred webhook work behind durable jobs without changing provider acknowledgement semantics.
|
||||
- `internal/service/captain_*` and `internal/service/copilot_*` have document sync job coverage only; P5.11 still needs crawl/response/embedding/Copilot response jobs.
|
||||
- `internal/service/analytics_service.go` has placeholder analytics/report paths and is the owner for P5.13 aggregation work.
|
||||
@@ -1619,7 +1618,7 @@ Checklist:
|
||||
- [x] Add `background_jobs` persistence with job type, payload, queue, status, attempt counters, scheduled/locked timestamps, idempotency key, last error, and completion timestamps.
|
||||
- [x] Replace `internal/worker/worker.go` placeholder with enqueue, schedule, perform, retry/backoff, dead-letter, and graceful shutdown behavior.
|
||||
- [x] Map the committed Chatwoot job/listener families to Go worker responsibilities in the tracking table.
|
||||
- [ ] Finish durable job dispatch for outbound provider delivery/status, inbound provider webhook deferral, remaining Captain/Copilot jobs, delayed automation actions, and report aggregation.
|
||||
- [ ] Finish durable job dispatch for inbound provider webhook deferral, remaining Captain/Copilot jobs, delayed automation actions, and report aggregation.
|
||||
- [ ] Add retry and failure logging for the remaining external provider calls.
|
||||
- [ ] Add tests for the remaining enqueueing, uniqueness/idempotency, delayed execution, retries, worker restart pickup, and fakeable external side effects.
|
||||
|
||||
@@ -1641,7 +1640,7 @@ Tracking table:
|
||||
| P5.7 | Queue SLA account-wide scans and applied-SLA processing. | `enterprise/app/jobs/sla/trigger_slas_for_accounts_job.rb`, `process_account_applied_slas_job.rb`, `process_applied_sla_job.rb` | `internal/service/applied_sla_service.go`, SLA event services | Scheduler enqueues account scans and per-SLA processing; threshold transitions remain idempotent and notification fan-out is retryable. | Review by `feat(sla): queue applied sla scans` |
|
||||
| P5.8 | Queue contact export artifact generation and completion notification/email. | `account/contacts_export_job.rb` | contact export service and mailer boundary | Export API returns immediately, artifact generation is durable, completion notification/email is fakeable, and download remains stable after restart. | Review by `feat(crm): queue contact exports` |
|
||||
| P5.9 | Queue inbound provider webhook processing where Chatwoot defers work. | `webhooks/*_events_job.rb`, `webhooks/*_delivery_job.rb` | webhook handlers, provider services | Provider HTTP ack behavior remains Chatwoot-compatible while persistence/dispatch runs through retryable jobs with signature validation preserved. | Todo |
|
||||
| P5.10 | Queue outbound message delivery and delivery-status updates. | `send_reply_job.rb`, provider delivery/status jobs | message send/channel services, delivery status handler | Outgoing message creation and provider delivery are separated; retries update message/delivery status exactly once. | Doing: send-reply core Review by `feat(messages): queue send replies`; provider delivery-status jobs Todo |
|
||||
| P5.10 | Queue outbound message delivery and delivery-status updates. | `send_reply_job.rb`, provider delivery/status jobs | message send/channel services, delivery status handler | Outgoing message creation and provider delivery are separated; retries update message/delivery status exactly once. | Review by `feat(messages): queue send replies` and `feat(messages): queue delivery statuses` |
|
||||
| P5.11 | Queue Captain document sync, crawl, response building, embeddings, and Copilot responses. | Captain document/crawl/response/embedding/Copilot jobs | `internal/service/captain_document_service.go`, Captain/Copilot services | Existing fakeable disabled/failure gates run under durable jobs; document statuses and Copilot message persistence survive worker restart. | Doing: Captain document sync Review by `feat(captain): queue document syncs`; crawl/response/embedding/Copilot response jobs Todo |
|
||||
| P5.12 | Queue conversation maintenance jobs. | `trigger_scheduled_items_job.rb`, `campaigns/trigger_oneoff_campaign_job.rb`, `conversations/resolution_job.rb`, `reopen_snoozed_conversations_job.rb`, `update_message_status_job.rb`, `bulk_actions_job.rb` | conversation service/handlers | Auto-resolution, snooze reopen, status updates, and bulk actions are scheduled/retryable with idempotent tests. | Review by `feat(conversations): queue maintenance jobs`, `feat(conversations): queue message status updates`, and `feat(conversations): queue bulk actions` |
|
||||
| P5.13 | Replace placeholder analytics/report builders that need background aggregation. | reporting jobs/services and report controllers | `internal/service/analytics_service.go`, reporting services | Frontend-visible reports no longer use placeholder values; any expensive aggregation is scheduled or cached with freshness rules. | Todo |
|
||||
@@ -1652,7 +1651,7 @@ P5.1 current checkpoint:
|
||||
- Replaced the `WorkerPool` stub with enqueue/schedule APIs, handler registration, `ProcessOne`, start/stop worker loops, PostgreSQL `FOR UPDATE SKIP LOCKED` claiming, retry/backoff, dead-letter state, and stale-lock recovery for worker restart pickup.
|
||||
- Kept older `NewWorkerPool()` construction as a no-op-compatible path while adding `NewWorkerPoolWithOptions(db, ...)` for durable wiring.
|
||||
- Added focused tests proving enqueue/idempotency, due job completion, retry to dead-letter, queue/schedule filtering, and stale running job requeue.
|
||||
- Remaining Phase 5 work: delayed automation scheduled-item execution, inbound provider webhook deferral, outbound provider delivery/status jobs, Captain crawl/response/embedding/Copilot response jobs, and analytics/report aggregation.
|
||||
- Remaining Phase 5 work: delayed automation scheduled-item execution, inbound provider webhook deferral, Captain crawl/response/embedding/Copilot response jobs, and analytics/report aggregation.
|
||||
|
||||
P5.1 verification:
|
||||
|
||||
@@ -1668,7 +1667,7 @@ P5.2 current checkpoint:
|
||||
- `dispatch.EventDispatcher` can now attach a `WorkerPool`; sync listeners still execute inline, while async listeners and `DispatchAsync` subscribers enqueue durable per-listener `event:listener_dispatch` jobs.
|
||||
- Worker replay resolves the listener by name and invokes `OnEvent` with the serialized event payload. Missing listeners or malformed payloads fail the job, preserving retry/dead-letter visibility.
|
||||
- `WorkerPool` default queue handling was broadened to process all queues unless explicitly filtered, so `events` jobs are processed by the default worker.
|
||||
- Remaining integration work is feature-specific: automation webhook/transcript actions, delayed automation/macros, CSAT sends/templates, Meilisearch indexing, SLA scans, Captain/Copilot, contact exports, and provider delivery must register producers/handlers on this durable path.
|
||||
- Remaining integration work is feature-specific: delayed automation/macros, inbound provider webhook deferral, Captain/Copilot, and analytics aggregation must register producers/handlers on this durable path.
|
||||
|
||||
P5.2 verification:
|
||||
|
||||
@@ -1739,7 +1738,7 @@ P5.7 current checkpoint:
|
||||
- `sla:process_account` mirrors `Sla::ProcessAccountAppliedSlasJob`: it finds active and active_with_misses AppliedSLA rows for the account and queues `sla:process_applied` jobs.
|
||||
- `sla:process_applied` mirrors `Sla::ProcessAppliedSlaJob`: it calls the existing `AppliedSlaService.Evaluate`, preserving idempotent miss events, notification fan-out, retry/backoff, and dead-letter visibility through `background_jobs`.
|
||||
- Bootstrap registers the SLA job handlers and enqueues the initial root scan when the app boots.
|
||||
- Remaining Phase 5 work: delayed automation scheduled-item execution, WhatsApp/Twilio CSAT templates, contact export artifacts, provider webhooks/outbound delivery/status jobs, Captain/Copilot durable jobs, and conversation maintenance.
|
||||
- Remaining Phase 5 work: delayed automation scheduled-item execution, inbound provider webhook deferral, Captain/Copilot durable jobs, and analytics aggregation.
|
||||
|
||||
P5.7 verification:
|
||||
|
||||
@@ -1757,7 +1756,7 @@ P5.8 current checkpoint:
|
||||
- Completed export jobs are idempotent: a duplicate replay of an already-completed export is a no-op, so completion notifications and mail are not duplicated.
|
||||
- Failed worker execution records the export error and leaves the background job retry/dead-letter state observable through `background_jobs`.
|
||||
- No-worker construction still uses the synchronous fallback for focused tests and local paths that do not start the durable worker.
|
||||
- Remaining Phase 5 work: delayed automation scheduled-item execution, WhatsApp/Twilio CSAT templates, provider webhooks/outbound delivery/status jobs, Captain/Copilot durable jobs, and conversation maintenance.
|
||||
- Remaining Phase 5 work: delayed automation scheduled-item execution, inbound provider webhook deferral, Captain/Copilot durable jobs, and analytics aggregation.
|
||||
|
||||
P5.8 verification:
|
||||
|
||||
@@ -1776,13 +1775,21 @@ P5.10 current checkpoint:
|
||||
- Successful provider sends persist the returned external message ID into `messages.source_id` and clear any stale `external_error` from `content_attributes` while keeping status `sent`.
|
||||
- Provider failures mark the message `failed`, persist `content_attributes.external_error`, and return an error so durable retry/backoff/dead-letter state remains observable in `background_jobs`.
|
||||
- Bootstrap wires `MessageService.SetWorkerPool(workerPool)`, so production outgoing replies are replayable after process restart.
|
||||
- Remaining P5.10 work: queue provider delivery-status updates where Chatwoot defers status callbacks, then prove read/delivered/failed transitions are idempotent and do not downgrade `read` to `delivered`.
|
||||
- `IncomingPersister.SetWorkerPool` now registers provider delivery-status jobs: direct receipts enqueue `webhook:message_status_update`, contact-scoped read receipts enqueue `webhook:contact_messages_status_update`, and worker replay reloads the inbox before applying status transitions.
|
||||
- Delivery-status replay upserts `delivery_statuses`, dispatches `message.status_updated`, records failed provider details in `content_attributes.external_error`, clears stale external errors on non-failed updates, and prevents stale `sent`/`delivered` receipts from downgrading already-`read` messages.
|
||||
- Twilio status callbacks now match the reference `Twilio::DeliveryStatusController` acknowledgement with `204 No Content`, support the exact `/twilio/delivery_status` route plus the legacy `/webhooks/twilio/delivery_status` alias, resolve channels by `MessagingServiceSid` or `AccountSid`/`From`, map `undelivered` to `failed`, and persist Twilio `ErrorCode`/`ErrorMessage` as `external_error`.
|
||||
- Bootstrap wires Facebook/Instagram, WhatsApp, TikTok, and Twilio webhook persisters to the WorkerPool, so provider status callbacks are replayable after process restart. No-worker construction still performs status updates inline for focused unit tests.
|
||||
- Remaining P5.10 work: none known for the named outbound delivery/status row; follow-up provider ingress persistence deferral is tracked under P5.9.
|
||||
|
||||
P5.10 verification:
|
||||
|
||||
```bash
|
||||
env GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go test ./internal/service -run 'MessageDeliveryWorker\|MessageService' -count=1
|
||||
env GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go test ./internal/handler/webhook -run 'IncomingPersister.*Status\|StatusJob\|TwilioDeliveryStatus' -count=1
|
||||
env GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go test ./internal/service ./internal/worker ./internal/app -count=1
|
||||
env GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go test ./internal/handler/webhook ./internal/service ./internal/worker ./internal/app -count=1
|
||||
env GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go run ./cmd/dump_routes > docs/parity/gochat_routes.txt
|
||||
env GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go run ./cmd/route_parity
|
||||
env TMPDIR=/home/rogee/Projects/gochat/.tmp/test-tmp GOCACHE=/tmp/gochat-gocache GOMODCACHE=/tmp/gochat-gomodcache go test ./...
|
||||
git diff --check
|
||||
```
|
||||
@@ -1817,7 +1824,7 @@ P5.12 current checkpoint:
|
||||
- Message-status replay validates only `read` and `delivered`, no-ops missing conversations, and updates non-incoming `sent`/`delivered` messages created before the viewer timestamp while skipping incoming, already-read, and post-cutoff messages.
|
||||
- `conversation:bulk_action` now mirrors Chatwoot `BulkActionsJob` on the `medium` queue for account conversation bulk actions. The account `POST /bulk_actions` handler accepts the reused frontend payload shape with `fields`, nested `labels.add/remove`, `snoozed_until`, and display-ID `ids`, enqueues the job when a WorkerPool is configured, and returns empty `200 OK` like Chatwoot.
|
||||
- Bulk-action replay scopes records by account and conversation display IDs, merges label adds/removals without duplicating labels, applies status/team/assignee/snooze fields, and leaves same-display conversations in other accounts untouched.
|
||||
- Remaining P5.12 work: none known for the named maintenance jobs in this row; follow-up provider/outbound jobs remain P5.9/P5.10.
|
||||
- Remaining P5.12 work: none known for the named maintenance jobs in this row; follow-up provider ingress jobs remain P5.9.
|
||||
|
||||
P5.12 verification:
|
||||
|
||||
@@ -1932,7 +1939,8 @@ Verification milestone gates:
|
||||
|
||||
## Progress Log
|
||||
|
||||
- 2026-06-05: P5.10 durable send-reply checkpoint prepared as `feat(messages): queue send replies`; outgoing message creation now enqueues `message:send_reply` on the `high` queue when a WorkerPool is configured, worker replay reloads account-scoped message/conversation/inbox/contact records, skips duplicate/private/voice-call/non-deliverable messages, calls the registered channel provider, persists `source_id`, and records retryable provider failures as message `failed` with `content_attributes.external_error`. Focused service/worker/app tests passed. Next slice is P5.10 provider delivery-status job depth, then P5.9 inbound webhook deferral.
|
||||
- 2026-06-05: P5.10 provider delivery-status checkpoint prepared as `feat(messages): queue delivery statuses`; provider direct status callbacks now enqueue `webhook:message_status_update`, contact-scoped read receipts enqueue `webhook:contact_messages_status_update`, worker replay upserts delivery status rows, records failed provider `external_error`, preserves monotonic sent/delivered/read transitions, and Twilio status callbacks now support the exact Chatwoot `/twilio/delivery_status` route with `204 No Content`. Focused webhook tests, route generation, full `go test ./...`, and `git diff --check` passed. Next slice is P5.9 inbound webhook deferral.
|
||||
- 2026-06-05: P5.10 durable send-reply checkpoint prepared as `feat(messages): queue send replies`; outgoing message creation now enqueues `message:send_reply` on the `high` queue when a WorkerPool is configured, worker replay reloads account-scoped message/conversation/inbox/contact records, skips duplicate/private/voice-call/non-deliverable messages, calls the registered channel provider, persists `source_id`, and records retryable provider failures as message `failed` with `content_attributes.external_error`. Focused service/worker/app tests passed; provider delivery-status depth is completed by the next P5.10 checkpoint.
|
||||
- 2026-06-05: Active parity tracker checkpoint prepared as `docs: land active parity tracking board`; the plan now front-loads the current authoritative remaining-work board, points the next implementation slice at P5.10 outbound message delivery, records P5.9 inbound job deferral, P5.11 remaining Captain/Copilot jobs, P5.13 analytics aggregation, Phase 2/3 drift, Phase 6 placeholder audit, and B12 optional live smoke as tracked owners, and reconciles stale B12/M6/M7 status rows. Documentation-only checkpoint; `git diff --check` passed.
|
||||
- 2026-06-04: Baseline stabilized and committed as `42cdab8 chore: stabilize chatwoot parity baseline`; `go test ./...` passed and route dump reported `TOTAL: 704`.
|
||||
- 2026-06-04: Phase 1 search foundation added: Meilisearch config/env defaults, `SearchEngine` contract, Meilisearch HTTP wrapper with bootstrap/settings, DB fallback adapter, document builders, reindex command, and no-live-Meilisearch tests. Verified `go test ./...` in unsandboxed mode because miniredis/httptest need local sockets; route dump still reports `TOTAL: 704`.
|
||||
|
||||
@@ -735,6 +735,7 @@ POST /public/api/v1/inboxes/:inbox_id/contacts/:contact_id/conversations/:conver
|
||||
POST /public/api/v1/inboxes/:inbox_id/contacts/:contact_id/conversations/:conversation_id/toggle_status
|
||||
POST /public/api/v1/inboxes/:inbox_id/contacts/:contact_id/conversations/:conversation_id/toggle_typing
|
||||
POST /public/api/v1/inboxes/:inbox_id/contacts/:contact_id/conversations/:conversation_id/update_last_seen
|
||||
POST /twilio/delivery_status
|
||||
POST /webhooks/instagram
|
||||
POST /webhooks/line/:line_channel_id
|
||||
POST /webhooks/shopify
|
||||
@@ -742,6 +743,7 @@ POST /webhooks/sms/:phone_number
|
||||
POST /webhooks/telegram/:bot_token
|
||||
POST /webhooks/tiktok
|
||||
POST /webhooks/tiktok/:business_id
|
||||
POST /webhooks/twilio/delivery_status
|
||||
POST /webhooks/twilio/sms/:phone_number
|
||||
POST /webhooks/twilio/status/:phone_number
|
||||
POST /webhooks/twitter
|
||||
@@ -828,4 +830,4 @@ PUT /public/api/v1/csat_survey/:id
|
||||
PUT /public/api/v1/inboxes/:inbox_id/contacts/:contact_id
|
||||
PUT /public/api/v1/inboxes/:inbox_id/contacts/:contact_id/conversations/:conversation_id/messages/:message_id
|
||||
PUT /widget/direct_uploads/:upload_uuid
|
||||
TOTAL: 830
|
||||
TOTAL: 832
|
||||
|
||||
@@ -403,6 +403,7 @@ func Bootstrap(env string) (*App, error) {
|
||||
|
||||
// Create Facebook webhook handler (Gin HTTP handler for FB/IG webhook endpoints)
|
||||
facebookWebhookHandler := webhook.NewFacebookWebhookHandler(fbProvider, igProvider, db, channelDispatcher)
|
||||
facebookWebhookHandler.WithWorkerPool(workerPool)
|
||||
shopifyWebhookHandler := webhook.NewShopifyWebhookHandler(db, "")
|
||||
|
||||
// Create Telegram webhook handler (Gin HTTP handler for Telegram webhook endpoint)
|
||||
@@ -426,6 +427,7 @@ func Bootstrap(env string) (*App, error) {
|
||||
// Create WhatsApp webhook handler (Gin HTTP handler for WA webhook endpoints)
|
||||
waWebhook := whatsappchannel.NewWebhookHandler(waProvider)
|
||||
whatsappWebhookHandler := webhook.NewWhatsAppWebhookHandler(waProvider, waWebhook, db, channelDispatcher)
|
||||
whatsappWebhookHandler.WithWorkerPool(workerPool)
|
||||
|
||||
// Step 8d: Wire TikTok channel provider (Business API)
|
||||
// TikTok requires service/repo/pipeline deps (like WhatsApp), so wire here in bootstrap.
|
||||
@@ -442,6 +444,7 @@ func Bootstrap(env string) (*App, error) {
|
||||
// Create TikTok webhook handler (Gin HTTP handler for TikTok webhook endpoints)
|
||||
ttWebhook := tiktokchannel.NewWebhookHandler(ttService, ttPipeline)
|
||||
tiktokWebhookHandler := webhook.NewTikTokWebhookHandler(ttWebhook, ttPipeline, db, channelDispatcher)
|
||||
tiktokWebhookHandler.WithWorkerPool(workerPool)
|
||||
|
||||
// Step 8e: Wire LINE channel provider (Messaging API)
|
||||
// LINE requires service/repo/pipeline deps (like WhatsApp), so wire here in bootstrap.
|
||||
@@ -474,6 +477,7 @@ func Bootstrap(env string) (*App, error) {
|
||||
// Create Twilio webhook handler (Gin HTTP handler for Twilio SMS webhook endpoints)
|
||||
twilioWebhook := twiliochannel.NewWebhookHandler(twilioPipeline, twilioService)
|
||||
twilioWebhookHandler := webhook.NewTwilioWebhookHandler(twilioWebhook, db, channelDispatcher)
|
||||
twilioWebhookHandler.WithWorkerPool(workerPool)
|
||||
|
||||
// Step 8g: Wire Email channel provider (IMAP/SMTP)
|
||||
// Email requires service/repo/pipeline deps, so wire here in bootstrap.
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
fbchannel "github.com/gochat/gochat/internal/channel/facebook"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
channelmodel "github.com/gochat/gochat/internal/model/channel"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
applogger "github.com/gochat/gochat/pkg/logger"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -68,6 +69,13 @@ func NewFacebookWebhookHandler(
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FacebookWebhookHandler) WithWorkerPool(wp *worker.WorkerPool) *FacebookWebhookHandler {
|
||||
if h != nil && h.persister != nil {
|
||||
h.persister.SetWorkerPool(wp)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// HandleFacebookVerification handles the GET webhook verification request from Facebook.
|
||||
// URL pattern: /webhooks/facebook/:inbox_id
|
||||
// Method: GET
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/gochat/gochat/internal/channel"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
)
|
||||
|
||||
// IncomingPersister is the durable boundary after provider-specific webhook parsing.
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
type IncomingPersister struct {
|
||||
db *gorm.DB
|
||||
dispatcher *channel.Dispatcher
|
||||
worker *worker.WorkerPool
|
||||
}
|
||||
|
||||
type IncomingPersistResult struct {
|
||||
@@ -31,6 +33,15 @@ type IncomingPersistResult struct {
|
||||
ConversationCreated bool
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) SetWorkerPool(wp *worker.WorkerPool) *IncomingPersister {
|
||||
if p == nil || wp == nil {
|
||||
return p
|
||||
}
|
||||
p.worker = wp
|
||||
p.registerJobs(wp)
|
||||
return p
|
||||
}
|
||||
|
||||
func NewIncomingPersister(db *gorm.DB, dispatcher ...*channel.Dispatcher) *IncomingPersister {
|
||||
if db == nil {
|
||||
return nil
|
||||
@@ -102,6 +113,21 @@ func (p *IncomingPersister) PersistIncoming(ctx context.Context, inbox *model.In
|
||||
|
||||
// UpdateMessageStatus applies provider delivery/read/failed receipts to existing messages.
|
||||
func (p *IncomingPersister) UpdateMessageStatus(ctx context.Context, inbox *model.Inbox, sourceID string, status model.MessageStatus, occurredAt *time.Time) error {
|
||||
return p.UpdateMessageStatusWithError(ctx, inbox, sourceID, status, occurredAt, "")
|
||||
}
|
||||
|
||||
// UpdateMessageStatusWithError applies provider receipts and records provider failure details.
|
||||
func (p *IncomingPersister) UpdateMessageStatusWithError(ctx context.Context, inbox *model.Inbox, sourceID string, status model.MessageStatus, occurredAt *time.Time, externalError string) error {
|
||||
if p == nil || p.db == nil || inbox == nil || sourceID == "" {
|
||||
return nil
|
||||
}
|
||||
if p.worker != nil {
|
||||
return p.enqueueMessageStatusUpdate(ctx, inbox.ID, sourceID, status, occurredAt, externalError)
|
||||
}
|
||||
return p.performMessageStatusUpdate(ctx, inbox, sourceID, status, occurredAt, externalError)
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) performMessageStatusUpdate(ctx context.Context, inbox *model.Inbox, sourceID string, status model.MessageStatus, occurredAt *time.Time, externalError string) error {
|
||||
if p == nil || p.db == nil || inbox == nil || sourceID == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -114,15 +140,27 @@ func (p *IncomingPersister) UpdateMessageStatus(ctx context.Context, inbox *mode
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&message).Update("status", string(status)).Error; err != nil {
|
||||
if !validProviderMessageStatusTransition(model.MessageStatus(message.Status), status) {
|
||||
return nil
|
||||
}
|
||||
updates := map[string]any{
|
||||
"status": string(status),
|
||||
"content_attributes": setProviderMessageExternalError(message.ContentAttributes, status, externalError),
|
||||
}
|
||||
if err := tx.Model(&message).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
message.Status = string(status)
|
||||
if message.SenderID == nil {
|
||||
message.ContentAttributes = updates["content_attributes"].(datatypes.JSON)
|
||||
contactID, err := p.deliveryStatusContactID(ctx, tx, &message)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if contactID == 0 {
|
||||
p.dispatchMessageStatusEvent(ctx, inbox, &message)
|
||||
return nil
|
||||
}
|
||||
if err := p.upsertDeliveryStatus(ctx, tx, &message, *message.SenderID, status, occurredAt); err != nil {
|
||||
if err := p.upsertDeliveryStatus(ctx, tx, &message, contactID, status, occurredAt); err != nil {
|
||||
return err
|
||||
}
|
||||
p.dispatchMessageStatusEvent(ctx, inbox, &message)
|
||||
@@ -135,6 +173,16 @@ func (p *IncomingPersister) UpdateContactConversationMessagesStatus(ctx context.
|
||||
if p == nil || p.db == nil || inbox == nil || contactSourceID == "" {
|
||||
return nil
|
||||
}
|
||||
if p.worker != nil {
|
||||
return p.enqueueContactMessagesStatusUpdate(ctx, inbox.ID, contactSourceID, status, occurredAt)
|
||||
}
|
||||
return p.performContactMessagesStatusUpdate(ctx, inbox, contactSourceID, status, occurredAt)
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) performContactMessagesStatusUpdate(ctx context.Context, inbox *model.Inbox, contactSourceID string, status model.MessageStatus, occurredAt *time.Time) error {
|
||||
if p == nil || p.db == nil || inbox == nil || contactSourceID == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var contactInbox model.ContactInbox
|
||||
@@ -157,17 +205,46 @@ func (p *IncomingPersister) UpdateContactConversationMessagesStatus(ctx context.
|
||||
if err := query.Select("messages.*").Find(&messages).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := query.Update("status", string(status)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range messages {
|
||||
if !validProviderMessageStatusTransition(model.MessageStatus(messages[i].Status), status) {
|
||||
continue
|
||||
}
|
||||
if err := tx.Model(&messages[i]).Updates(map[string]any{
|
||||
"status": string(status),
|
||||
"content_attributes": setProviderMessageExternalError(messages[i].ContentAttributes, status, ""),
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
messages[i].Status = string(status)
|
||||
if err := p.upsertDeliveryStatus(ctx, tx, &messages[i], contactInbox.ContactID, status, occurredAt); err != nil {
|
||||
return err
|
||||
}
|
||||
p.dispatchMessageStatusEvent(ctx, inbox, &messages[i])
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) deliveryStatusContactID(ctx context.Context, tx *gorm.DB, message *model.Message) (uint, error) {
|
||||
if message == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if message.ConversationID != 0 {
|
||||
var conversation model.Conversation
|
||||
if err := tx.WithContext(ctx).Select("id", "contact_id").First(&conversation, message.ConversationID).Error; err != nil {
|
||||
if err != gorm.ErrRecordNotFound {
|
||||
return 0, err
|
||||
}
|
||||
} else if conversation.ContactID != 0 {
|
||||
return conversation.ContactID, nil
|
||||
}
|
||||
}
|
||||
if message.SenderID != nil {
|
||||
return *message.SenderID, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) upsertDeliveryStatus(ctx context.Context, tx *gorm.DB, message *model.Message, contactID uint, status model.MessageStatus, occurredAt *time.Time) error {
|
||||
var delivery model.DeliveryStatus
|
||||
err := tx.WithContext(ctx).Where("message_id = ? AND contact_id = ?", message.ID, contactID).First(&delivery).Error
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
)
|
||||
|
||||
const (
|
||||
TaskTypeProviderMessageStatusUpdate = "webhook:message_status_update"
|
||||
TaskTypeProviderContactMessagesStatusUpdate = "webhook:contact_messages_status_update"
|
||||
)
|
||||
|
||||
type providerMessageStatusUpdateJob struct {
|
||||
InboxID uint `json:"inbox_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Status model.MessageStatus `json:"status"`
|
||||
OccurredAtUnixNano int64 `json:"occurred_at_unix_nano,omitempty"`
|
||||
ExternalError string `json:"external_error,omitempty"`
|
||||
}
|
||||
|
||||
type providerContactMessagesStatusUpdateJob struct {
|
||||
InboxID uint `json:"inbox_id"`
|
||||
ContactSourceID string `json:"contact_source_id"`
|
||||
Status model.MessageStatus `json:"status"`
|
||||
OccurredAtUnixNano int64 `json:"occurred_at_unix_nano,omitempty"`
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) registerJobs(wp *worker.WorkerPool) {
|
||||
if p == nil || wp == nil {
|
||||
return
|
||||
}
|
||||
wp.Register(TaskTypeProviderMessageStatusUpdate, p.performMessageStatusUpdateJob)
|
||||
wp.Register(TaskTypeProviderContactMessagesStatusUpdate, p.performContactMessagesStatusUpdateJob)
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) enqueueMessageStatusUpdate(ctx context.Context, inboxID uint, sourceID string, status model.MessageStatus, occurredAt *time.Time, externalError string) error {
|
||||
if p == nil || p.worker == nil || inboxID == 0 || sourceID == "" {
|
||||
return nil
|
||||
}
|
||||
if !validProviderMessageStatus(status) {
|
||||
return nil
|
||||
}
|
||||
payload := providerMessageStatusUpdateJob{
|
||||
InboxID: inboxID,
|
||||
SourceID: sourceID,
|
||||
Status: status,
|
||||
OccurredAtUnixNano: timeToUnixNano(occurredAt),
|
||||
ExternalError: strings.TrimSpace(externalError),
|
||||
}
|
||||
_, err := p.worker.Enqueue(ctx, TaskTypeProviderMessageStatusUpdate, payload,
|
||||
worker.WithQueue("low"),
|
||||
worker.WithMaxAttempts(3),
|
||||
worker.WithIdempotencyKey(fmt.Sprintf("webhook:message_status:%d:%s:%s:%d", inboxID, sourceID, status, payload.OccurredAtUnixNano)),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) enqueueContactMessagesStatusUpdate(ctx context.Context, inboxID uint, contactSourceID string, status model.MessageStatus, occurredAt *time.Time) error {
|
||||
if p == nil || p.worker == nil || inboxID == 0 || contactSourceID == "" {
|
||||
return nil
|
||||
}
|
||||
if !validProviderMessageStatus(status) {
|
||||
return nil
|
||||
}
|
||||
payload := providerContactMessagesStatusUpdateJob{
|
||||
InboxID: inboxID,
|
||||
ContactSourceID: contactSourceID,
|
||||
Status: status,
|
||||
OccurredAtUnixNano: timeToUnixNano(occurredAt),
|
||||
}
|
||||
_, err := p.worker.Enqueue(ctx, TaskTypeProviderContactMessagesStatusUpdate, payload,
|
||||
worker.WithQueue("low"),
|
||||
worker.WithMaxAttempts(3),
|
||||
worker.WithIdempotencyKey(fmt.Sprintf("webhook:contact_messages_status:%d:%s:%s:%d", inboxID, contactSourceID, status, payload.OccurredAtUnixNano)),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) performMessageStatusUpdateJob(ctx context.Context, job *model.BackgroundJob) error {
|
||||
var payload providerMessageStatusUpdateJob
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil {
|
||||
return fmt.Errorf("unmarshal provider message status job: %w", err)
|
||||
}
|
||||
if payload.InboxID == 0 || payload.SourceID == "" || !validProviderMessageStatus(payload.Status) {
|
||||
return fmt.Errorf("invalid provider message status job payload: %#v", payload)
|
||||
}
|
||||
inbox, err := p.loadInboxForStatusJob(ctx, payload.InboxID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inbox == nil {
|
||||
return nil
|
||||
}
|
||||
return p.performMessageStatusUpdate(ctx, inbox, payload.SourceID, payload.Status, unixNanoToTime(payload.OccurredAtUnixNano), payload.ExternalError)
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) performContactMessagesStatusUpdateJob(ctx context.Context, job *model.BackgroundJob) error {
|
||||
var payload providerContactMessagesStatusUpdateJob
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil {
|
||||
return fmt.Errorf("unmarshal provider contact messages status job: %w", err)
|
||||
}
|
||||
if payload.InboxID == 0 || payload.ContactSourceID == "" || !validProviderMessageStatus(payload.Status) {
|
||||
return fmt.Errorf("invalid provider contact messages status job payload: %#v", payload)
|
||||
}
|
||||
inbox, err := p.loadInboxForStatusJob(ctx, payload.InboxID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inbox == nil {
|
||||
return nil
|
||||
}
|
||||
return p.performContactMessagesStatusUpdate(ctx, inbox, payload.ContactSourceID, payload.Status, unixNanoToTime(payload.OccurredAtUnixNano))
|
||||
}
|
||||
|
||||
func (p *IncomingPersister) loadInboxForStatusJob(ctx context.Context, inboxID uint) (*model.Inbox, error) {
|
||||
var inbox model.Inbox
|
||||
if err := p.db.WithContext(ctx).First(&inbox, inboxID).Error; err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("load status inbox %d: %w", inboxID, err)
|
||||
}
|
||||
return &inbox, nil
|
||||
}
|
||||
|
||||
func validProviderMessageStatus(status model.MessageStatus) bool {
|
||||
switch status {
|
||||
case model.MessageStatusSent, model.MessageStatusDelivered, model.MessageStatusRead, model.MessageStatusFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validProviderMessageStatusTransition(current, next model.MessageStatus) bool {
|
||||
if !validProviderMessageStatus(next) || current == next {
|
||||
return validProviderMessageStatus(next)
|
||||
}
|
||||
if next == model.MessageStatusFailed || current == model.MessageStatusFailed {
|
||||
return true
|
||||
}
|
||||
return providerMessageStatusRank(next) >= providerMessageStatusRank(current)
|
||||
}
|
||||
|
||||
func providerMessageStatusRank(status model.MessageStatus) int {
|
||||
switch status {
|
||||
case model.MessageStatusSent:
|
||||
return 1
|
||||
case model.MessageStatusDelivered:
|
||||
return 2
|
||||
case model.MessageStatusRead:
|
||||
return 3
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func setProviderMessageExternalError(attrs datatypes.JSON, status model.MessageStatus, externalError string) datatypes.JSON {
|
||||
obj := map[string]any{}
|
||||
if len(attrs) > 0 {
|
||||
_ = json.Unmarshal(attrs, &obj)
|
||||
}
|
||||
if status == model.MessageStatusFailed && strings.TrimSpace(externalError) != "" {
|
||||
obj["external_error"] = strings.TrimSpace(externalError)
|
||||
} else {
|
||||
delete(obj, "external_error")
|
||||
}
|
||||
bytes, _ := json.Marshal(obj)
|
||||
return datatypes.JSON(bytes)
|
||||
}
|
||||
|
||||
func timeToUnixNano(t *time.Time) int64 {
|
||||
if t == nil || t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UTC().UnixNano()
|
||||
}
|
||||
|
||||
func unixNanoToTime(value int64) *time.Time {
|
||||
if value <= 0 {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(0, value).UTC()
|
||||
return &t
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
tiktokchannel "github.com/gochat/gochat/internal/channel/tiktok"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
channelmodel "github.com/gochat/gochat/internal/model/channel"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
applogger "github.com/gochat/gochat/pkg/logger"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -46,6 +47,13 @@ func NewTikTokWebhookHandler(tiktokWebhook *tiktokchannel.WebhookHandler, pipeli
|
||||
}
|
||||
}
|
||||
|
||||
func (h *TikTokWebhookHandler) WithWorkerPool(wp *worker.WorkerPool) *TikTokWebhookHandler {
|
||||
if h != nil && h.persister != nil {
|
||||
h.persister.SetWorkerPool(wp)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// HandleTikTokWebhook processes incoming TikTok webhook HTTP requests.
|
||||
func (h *TikTokWebhookHandler) HandleTikTokWebhook(c *gin.Context) {
|
||||
body, err := io.ReadAll(c.Request.Body)
|
||||
|
||||
@@ -13,11 +13,13 @@ package webhook
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/gochat/gochat/internal/channel"
|
||||
twiliochannel "github.com/gochat/gochat/internal/channel/twilio"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
channelmodel "github.com/gochat/gochat/internal/model/channel"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
applogger "github.com/gochat/gochat/pkg/logger"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -31,6 +33,13 @@ type TwilioWebhookHandler struct {
|
||||
persister *IncomingPersister
|
||||
}
|
||||
|
||||
func (h *TwilioWebhookHandler) WithWorkerPool(wp *worker.WorkerPool) *TwilioWebhookHandler {
|
||||
if h != nil && h.persister != nil {
|
||||
h.persister.SetWorkerPool(wp)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// NewTwilioWebhookHandler creates a Twilio SMS webhook handler for Gin integration.
|
||||
func NewTwilioWebhookHandler(twilioWebhook *twiliochannel.WebhookHandler, db *gorm.DB, dispatcher ...*channel.Dispatcher) *TwilioWebhookHandler {
|
||||
return &TwilioWebhookHandler{
|
||||
@@ -74,33 +83,65 @@ func (h *TwilioWebhookHandler) HandleTwilioInboundSMS(c *gin.Context) {
|
||||
// HandleTwilioDeliveryStatus processes a Twilio delivery status callback.
|
||||
func (h *TwilioWebhookHandler) HandleTwilioDeliveryStatus(c *gin.Context) {
|
||||
phoneNumber := c.Param("phone_number")
|
||||
if phoneNumber == "" {
|
||||
applogger.L().Warn("Twilio status webhook: missing phone_number in path")
|
||||
c.Status(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
// Lookup inbox from database
|
||||
inbox, err := h.lookupInboxByPhoneNumber(phoneNumber)
|
||||
if err != nil {
|
||||
applogger.L().Warnf("Twilio status webhook: inbox lookup failed for phone_number %s: %v", phoneNumber, err)
|
||||
c.Status(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
if err := c.Request.ParseForm(); err != nil {
|
||||
applogger.L().Errorf("Twilio status webhook: parse form failed for inbox %d: %v", inbox.ID, err)
|
||||
c.Status(http.StatusOK)
|
||||
applogger.L().Errorf("Twilio status webhook: parse form failed: %v", err)
|
||||
c.Status(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
var inbox *model.Inbox
|
||||
var err error
|
||||
if phoneNumber != "" {
|
||||
inbox, err = h.lookupInboxByPhoneNumber(phoneNumber)
|
||||
} else {
|
||||
inbox, err = h.lookupDeliveryStatusInbox(c.Request.Form)
|
||||
}
|
||||
if err != nil {
|
||||
applogger.L().Warnf("Twilio status webhook: inbox lookup failed: %v", err)
|
||||
c.Status(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
messageSID := c.Request.FormValue("MessageSid")
|
||||
messageStatus := c.Request.FormValue("MessageStatus")
|
||||
if mapped, ok := mapTwilioMessageStatus(messageStatus); ok {
|
||||
if err := h.persister.UpdateMessageStatus(c.Request.Context(), inbox, messageSID, mapped, nil); err != nil {
|
||||
if err := h.persister.UpdateMessageStatusWithError(c.Request.Context(), inbox, messageSID, mapped, nil, twilioExternalError(c.Request.FormValue("ErrorCode"), c.Request.FormValue("ErrorMessage"), messageStatus)); err != nil {
|
||||
applogger.L().Errorf("Twilio status webhook: status persistence failed for inbox %d sid=%s status=%s: %v", inbox.ID, messageSID, messageStatus, err)
|
||||
}
|
||||
}
|
||||
c.Status(http.StatusOK)
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func twilioExternalError(errorCode, errorMessage, status string) string {
|
||||
if errorCode == "" || (status != "failed" && status != "undelivered") {
|
||||
return ""
|
||||
}
|
||||
if errorMessage != "" {
|
||||
return fmt.Sprintf("%s - %s", errorCode, errorMessage)
|
||||
}
|
||||
return fmt.Sprintf("Twilio delivery failed with error code %s", errorCode)
|
||||
}
|
||||
|
||||
func (h *TwilioWebhookHandler) lookupDeliveryStatusInbox(params url.Values) (*model.Inbox, error) {
|
||||
if h.db == nil {
|
||||
return nil, fmt.Errorf("twilio webhook database is not configured")
|
||||
}
|
||||
var twilioChannel channelmodel.ChannelTwilioSMS
|
||||
query := h.db
|
||||
if sid := params.Get("MessagingServiceSid"); sid != "" {
|
||||
query = query.Where(&channelmodel.ChannelTwilioSMS{MessagingServiceSID: sid})
|
||||
} else if accountSID, from := params.Get("AccountSid"), params.Get("From"); accountSID != "" && from != "" {
|
||||
query = query.Where(&channelmodel.ChannelTwilioSMS{AccountSID: accountSID, PhoneNumber: from})
|
||||
} else {
|
||||
return nil, fmt.Errorf("delivery status missing MessagingServiceSid or AccountSid/From")
|
||||
}
|
||||
if err := query.First(&twilioChannel).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var inbox model.Inbox
|
||||
if err := h.db.Where("id = ? AND channel_type IN ?", twilioChannel.InboxID, []string{"twilio_sms", "sms"}).First(&inbox).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &inbox, nil
|
||||
}
|
||||
|
||||
func mapTwilioMessageStatus(status string) (model.MessageStatus, bool) {
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
whatsappchannel "github.com/gochat/gochat/internal/channel/whatsapp"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
channelmodel "github.com/gochat/gochat/internal/model/channel"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -62,6 +63,7 @@ func newWebhookLookupTestDB(t *testing.T) *gorm.DB {
|
||||
&model.Conversation{},
|
||||
&model.Message{},
|
||||
&model.DeliveryStatus{},
|
||||
&model.BackgroundJob{},
|
||||
&automation.AutomationRule{},
|
||||
&automation.AutomationExecution{},
|
||||
&channelmodel.ChannelTelegram{},
|
||||
@@ -167,6 +169,193 @@ func TestIncomingPersisterUpdatesMessageStatus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncomingPersisterQueuesMessageStatusUpdateWithWorker(t *testing.T) {
|
||||
db := newWebhookLookupTestDB(t)
|
||||
inbox := seedWebhookInbox(t, db, "telegram")
|
||||
dispatcher := channel.NewDispatcher()
|
||||
listener := &recordingListener{}
|
||||
dispatcher.Register(listener)
|
||||
wp := worker.NewWorkerPoolWithOptions(db, worker.WithQueues("low"))
|
||||
persister := NewIncomingPersister(db, dispatcher).SetWorkerPool(wp)
|
||||
|
||||
msg := &channel.IncomingMessage{
|
||||
ChannelType: channel.ChannelTelegram,
|
||||
SourceID: "tg-status-job-1",
|
||||
SenderID: "tg-user-status-job",
|
||||
SenderName: "Status Worker User",
|
||||
SenderType: channel.SenderContact,
|
||||
Content: "status me async",
|
||||
ContentType: channel.ContentText,
|
||||
InboxID: inbox.ID,
|
||||
AccountID: inbox.AccountID,
|
||||
}
|
||||
result, err := persister.PersistIncoming(t.Context(), &inbox, msg)
|
||||
if err != nil {
|
||||
t.Fatalf("persist incoming: %v", err)
|
||||
}
|
||||
|
||||
occurredAt := time.Now().UTC().Add(-time.Minute)
|
||||
if err := persister.UpdateMessageStatus(t.Context(), &inbox, "tg-status-job-1", model.MessageStatusDelivered, &occurredAt); err != nil {
|
||||
t.Fatalf("enqueue status: %v", err)
|
||||
}
|
||||
var queued model.BackgroundJob
|
||||
if err := db.Where("job_type = ? AND queue = ? AND status = ?", TaskTypeProviderMessageStatusUpdate, "low", model.BackgroundJobStatusQueued).First(&queued).Error; err != nil {
|
||||
t.Fatalf("expected queued status job: %v", err)
|
||||
}
|
||||
var before model.Message
|
||||
if err := db.First(&before, result.Message.ID).Error; err != nil {
|
||||
t.Fatalf("load before message: %v", err)
|
||||
}
|
||||
if before.Status != string(model.MessageStatusSent) {
|
||||
t.Fatalf("expected status to remain sent before worker, got %s", before.Status)
|
||||
}
|
||||
|
||||
processed, err := wp.ProcessOne(t.Context())
|
||||
if err != nil || !processed {
|
||||
t.Fatalf("process status job processed=%v err=%v", processed, err)
|
||||
}
|
||||
var updated model.Message
|
||||
if err := db.First(&updated, result.Message.ID).Error; err != nil {
|
||||
t.Fatalf("load updated message: %v", err)
|
||||
}
|
||||
if updated.Status != string(model.MessageStatusDelivered) {
|
||||
t.Fatalf("expected delivered status, got %s", updated.Status)
|
||||
}
|
||||
var delivery model.DeliveryStatus
|
||||
if err := db.Where("message_id = ? AND contact_id = ?", updated.ID, result.Contact.ID).First(&delivery).Error; err != nil {
|
||||
t.Fatalf("expected delivery status: %v", err)
|
||||
}
|
||||
if delivery.Status != model.MessageStatusDelivered || delivery.DeliveredAt == nil {
|
||||
t.Fatalf("expected delivered timestamp, got %#v", delivery)
|
||||
}
|
||||
if !listenerSaw(listener, channel.EventMessageStatusUpdated) {
|
||||
t.Fatalf("expected message.status_updated event, got %#v", listener.events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncomingPersisterStatusJobDoesNotDowngradeRead(t *testing.T) {
|
||||
db := newWebhookLookupTestDB(t)
|
||||
inbox := seedWebhookInbox(t, db, "telegram")
|
||||
contact := model.Contact{AccountID: inbox.AccountID, Name: "Read User"}
|
||||
if err := db.Create(&contact).Error; err != nil {
|
||||
t.Fatalf("create contact: %v", err)
|
||||
}
|
||||
conversation := model.Conversation{AccountID: inbox.AccountID, InboxID: inbox.ID, ContactID: contact.ID, Status: string(model.ConversationStatusOpen), ChannelType: inbox.ChannelType, Channel: inbox.ChannelType}
|
||||
if err := db.Create(&conversation).Error; err != nil {
|
||||
t.Fatalf("create conversation: %v", err)
|
||||
}
|
||||
message := model.Message{ConversationID: conversation.ID, AccountID: inbox.AccountID, InboxID: inbox.ID, SenderID: &contact.ID, SenderType: string(model.SenderTypeContact), Content: "already read", ContentType: string(model.MessageContentTypeText), MessageType: string(model.MessageTypeOutgoing), Status: string(model.MessageStatusRead), SourceID: "provider-read-1"}
|
||||
if err := db.Create(&message).Error; err != nil {
|
||||
t.Fatalf("create message: %v", err)
|
||||
}
|
||||
wp := worker.NewWorkerPoolWithOptions(db, worker.WithQueues("low"))
|
||||
persister := NewIncomingPersister(db).SetWorkerPool(wp)
|
||||
|
||||
if err := persister.UpdateMessageStatus(t.Context(), &inbox, "provider-read-1", model.MessageStatusDelivered, nil); err != nil {
|
||||
t.Fatalf("enqueue downgrade: %v", err)
|
||||
}
|
||||
processed, err := wp.ProcessOne(t.Context())
|
||||
if err != nil || !processed {
|
||||
t.Fatalf("process downgrade job processed=%v err=%v", processed, err)
|
||||
}
|
||||
var updated model.Message
|
||||
if err := db.First(&updated, message.ID).Error; err != nil {
|
||||
t.Fatalf("load message: %v", err)
|
||||
}
|
||||
if updated.Status != string(model.MessageStatusRead) {
|
||||
t.Fatalf("expected read to remain read, got %s", updated.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncomingPersisterStatusJobStoresExternalError(t *testing.T) {
|
||||
db := newWebhookLookupTestDB(t)
|
||||
inbox := seedWebhookInbox(t, db, "twilio_sms")
|
||||
contact := model.Contact{AccountID: inbox.AccountID, Name: "SMS User"}
|
||||
if err := db.Create(&contact).Error; err != nil {
|
||||
t.Fatalf("create contact: %v", err)
|
||||
}
|
||||
conversation := model.Conversation{AccountID: inbox.AccountID, InboxID: inbox.ID, ContactID: contact.ID, Status: string(model.ConversationStatusOpen), ChannelType: inbox.ChannelType, Channel: inbox.ChannelType}
|
||||
if err := db.Create(&conversation).Error; err != nil {
|
||||
t.Fatalf("create conversation: %v", err)
|
||||
}
|
||||
message := model.Message{ConversationID: conversation.ID, AccountID: inbox.AccountID, InboxID: inbox.ID, SenderID: &contact.ID, SenderType: string(model.SenderTypeContact), Content: "failed", ContentType: string(model.MessageContentTypeText), MessageType: string(model.MessageTypeOutgoing), Status: string(model.MessageStatusSent), SourceID: "SMFAIL1"}
|
||||
if err := db.Create(&message).Error; err != nil {
|
||||
t.Fatalf("create message: %v", err)
|
||||
}
|
||||
wp := worker.NewWorkerPoolWithOptions(db, worker.WithQueues("low"))
|
||||
persister := NewIncomingPersister(db).SetWorkerPool(wp)
|
||||
|
||||
if err := persister.UpdateMessageStatusWithError(t.Context(), &inbox, "SMFAIL1", model.MessageStatusFailed, nil, "30003 - unreachable handset"); err != nil {
|
||||
t.Fatalf("enqueue failure: %v", err)
|
||||
}
|
||||
processed, err := wp.ProcessOne(t.Context())
|
||||
if err != nil || !processed {
|
||||
t.Fatalf("process failure job processed=%v err=%v", processed, err)
|
||||
}
|
||||
var updated model.Message
|
||||
if err := db.First(&updated, message.ID).Error; err != nil {
|
||||
t.Fatalf("load failed message: %v", err)
|
||||
}
|
||||
if updated.Status != string(model.MessageStatusFailed) {
|
||||
t.Fatalf("expected failed status, got %s", updated.Status)
|
||||
}
|
||||
attrs := map[string]any{}
|
||||
if err := json.Unmarshal(updated.ContentAttributes, &attrs); err != nil {
|
||||
t.Fatalf("unmarshal content attrs: %v", err)
|
||||
}
|
||||
if attrs["external_error"] != "30003 - unreachable handset" {
|
||||
t.Fatalf("expected external_error, got %#v", attrs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncomingPersisterQueuesContactMessagesStatusUpdateWithWorker(t *testing.T) {
|
||||
db := newWebhookLookupTestDB(t)
|
||||
inbox := seedWebhookInbox(t, db, "facebook")
|
||||
contact := model.Contact{AccountID: inbox.AccountID, Name: "Meta User"}
|
||||
if err := db.Create(&contact).Error; err != nil {
|
||||
t.Fatalf("create contact: %v", err)
|
||||
}
|
||||
contactInbox := model.ContactInbox{ContactID: contact.ID, InboxID: inbox.ID, SourceID: "meta-user-1", PubsubToken: "pub-meta"}
|
||||
if err := db.Create(&contactInbox).Error; err != nil {
|
||||
t.Fatalf("create contact inbox: %v", err)
|
||||
}
|
||||
conversation := model.Conversation{AccountID: inbox.AccountID, InboxID: inbox.ID, ContactID: contact.ID, ContactInboxID: &contactInbox.ID, Status: string(model.ConversationStatusOpen), ChannelType: inbox.ChannelType, Channel: inbox.ChannelType}
|
||||
if err := db.Create(&conversation).Error; err != nil {
|
||||
t.Fatalf("create conversation: %v", err)
|
||||
}
|
||||
cutoff := time.Now().UTC()
|
||||
beforeSent := createWebhookStatusMessage(t, db, inbox, conversation, "sent-before", model.MessageStatusSent, cutoff.Add(-time.Minute))
|
||||
beforeDelivered := createWebhookStatusMessage(t, db, inbox, conversation, "delivered-before", model.MessageStatusDelivered, cutoff.Add(-30*time.Second))
|
||||
alreadyRead := createWebhookStatusMessage(t, db, inbox, conversation, "read-before", model.MessageStatusRead, cutoff.Add(-20*time.Second))
|
||||
afterCutoff := createWebhookStatusMessage(t, db, inbox, conversation, "sent-after", model.MessageStatusSent, cutoff.Add(time.Minute))
|
||||
wp := worker.NewWorkerPoolWithOptions(db, worker.WithQueues("low"))
|
||||
persister := NewIncomingPersister(db).SetWorkerPool(wp)
|
||||
|
||||
if err := persister.UpdateContactConversationMessagesStatus(t.Context(), &inbox, "meta-user-1", model.MessageStatusRead, &cutoff); err != nil {
|
||||
t.Fatalf("enqueue contact read: %v", err)
|
||||
}
|
||||
var queued model.BackgroundJob
|
||||
if err := db.Where("job_type = ? AND queue = ?", TaskTypeProviderContactMessagesStatusUpdate, "low").First(&queued).Error; err != nil {
|
||||
t.Fatalf("expected queued contact status job: %v", err)
|
||||
}
|
||||
processed, err := wp.ProcessOne(t.Context())
|
||||
if err != nil || !processed {
|
||||
t.Fatalf("process contact status job processed=%v err=%v", processed, err)
|
||||
}
|
||||
assertWebhookMessageStatus(t, db, beforeSent.ID, model.MessageStatusRead)
|
||||
assertWebhookMessageStatus(t, db, beforeDelivered.ID, model.MessageStatusRead)
|
||||
assertWebhookMessageStatus(t, db, alreadyRead.ID, model.MessageStatusRead)
|
||||
assertWebhookMessageStatus(t, db, afterCutoff.ID, model.MessageStatusSent)
|
||||
|
||||
var deliveryCount int64
|
||||
if err := db.Model(&model.DeliveryStatus{}).Where("contact_id = ? AND status = ?", contact.ID, model.MessageStatusRead).Count(&deliveryCount).Error; err != nil {
|
||||
t.Fatalf("count delivery statuses: %v", err)
|
||||
}
|
||||
if deliveryCount != 3 {
|
||||
t.Fatalf("expected 3 read delivery statuses, got %d", deliveryCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncomingPersisterCreatesConversationMessageAndDedupes(t *testing.T) {
|
||||
db := newWebhookLookupTestDB(t)
|
||||
inbox := seedWebhookInbox(t, db, "telegram")
|
||||
@@ -239,6 +428,37 @@ func listenerSaw(listener *recordingListener, eventType channel.EventType) bool
|
||||
return false
|
||||
}
|
||||
|
||||
func createWebhookStatusMessage(t *testing.T, db *gorm.DB, inbox model.Inbox, conversation model.Conversation, sourceID string, status model.MessageStatus, createdAt time.Time) model.Message {
|
||||
t.Helper()
|
||||
message := model.Message{
|
||||
ConversationID: conversation.ID,
|
||||
AccountID: inbox.AccountID,
|
||||
InboxID: inbox.ID,
|
||||
Content: sourceID,
|
||||
ContentType: string(model.MessageContentTypeText),
|
||||
MessageType: string(model.MessageTypeOutgoing),
|
||||
Status: string(status),
|
||||
SourceID: sourceID,
|
||||
}
|
||||
message.CreatedAt = createdAt
|
||||
message.UpdatedAt = createdAt
|
||||
if err := db.Create(&message).Error; err != nil {
|
||||
t.Fatalf("create status message: %v", err)
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func assertWebhookMessageStatus(t *testing.T, db *gorm.DB, messageID uint, want model.MessageStatus) {
|
||||
t.Helper()
|
||||
var message model.Message
|
||||
if err := db.First(&message, messageID).Error; err != nil {
|
||||
t.Fatalf("load message %d: %v", messageID, err)
|
||||
}
|
||||
if message.Status != string(want) {
|
||||
t.Fatalf("message %d expected %s, got %s", messageID, want, message.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func shopifyHMAC(secret string, body []byte) string {
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
mac.Write(body)
|
||||
@@ -524,8 +744,8 @@ func TestTwilioDeliveryStatusUpdatesExistingMessage(t *testing.T) {
|
||||
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("expected 204, got %d", w.Code)
|
||||
}
|
||||
var updated model.Message
|
||||
if err := db.First(&updated, message.ID).Error; err != nil {
|
||||
@@ -536,6 +756,76 @@ func TestTwilioDeliveryStatusUpdatesExistingMessage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwilioDeliveryStatusQueuesExactRouteWithWorker(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db := newWebhookLookupTestDB(t)
|
||||
inbox := seedWebhookInbox(t, db, "twilio_sms")
|
||||
channelRecord := channelmodel.ChannelTwilioSMS{AccountID: 1, InboxID: inbox.ID, AccountSID: "AC123", PhoneNumber: "+15551234567", MessagingServiceSID: "MG123"}
|
||||
if err := db.Create(&channelRecord).Error; err != nil {
|
||||
t.Fatalf("create twilio channel: %v", err)
|
||||
}
|
||||
contact := model.Contact{AccountID: inbox.AccountID, Name: "SMS Contact", Identifier: "+15550001111"}
|
||||
if err := db.Create(&contact).Error; err != nil {
|
||||
t.Fatalf("create contact: %v", err)
|
||||
}
|
||||
conversation := model.Conversation{AccountID: inbox.AccountID, InboxID: inbox.ID, ContactID: contact.ID, Status: string(model.ConversationStatusOpen), ChannelType: inbox.ChannelType, Channel: inbox.ChannelType}
|
||||
if err := db.Create(&conversation).Error; err != nil {
|
||||
t.Fatalf("create conversation: %v", err)
|
||||
}
|
||||
message := model.Message{ConversationID: conversation.ID, AccountID: inbox.AccountID, InboxID: inbox.ID, SenderID: &contact.ID, SenderType: string(model.SenderTypeContact), Content: "out", ContentType: string(model.MessageContentTypeText), MessageType: string(model.MessageTypeOutgoing), Status: string(model.MessageStatusSent), SourceID: "SMFAILED"}
|
||||
if err := db.Create(&message).Error; err != nil {
|
||||
t.Fatalf("create message: %v", err)
|
||||
}
|
||||
wp := worker.NewWorkerPoolWithOptions(db, worker.WithQueues("low"))
|
||||
h := NewTwilioWebhookHandler(nil, db).WithWorkerPool(wp)
|
||||
r := gin.New()
|
||||
r.POST("/twilio/delivery_status", h.HandleTwilioDeliveryStatus)
|
||||
form := url.Values{}
|
||||
form.Set("MessagingServiceSid", "MG123")
|
||||
form.Set("MessageSid", "SMFAILED")
|
||||
form.Set("MessageStatus", "undelivered")
|
||||
form.Set("ErrorCode", "30003")
|
||||
form.Set("ErrorMessage", "Unreachable handset")
|
||||
req := httptest.NewRequest(http.MethodPost, "/twilio/delivery_status", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("expected 204, got %d", w.Code)
|
||||
}
|
||||
var queued model.BackgroundJob
|
||||
if err := db.Where("job_type = ? AND queue = ? AND status = ?", TaskTypeProviderMessageStatusUpdate, "low", model.BackgroundJobStatusQueued).First(&queued).Error; err != nil {
|
||||
t.Fatalf("expected queued twilio status job: %v", err)
|
||||
}
|
||||
var before model.Message
|
||||
if err := db.First(&before, message.ID).Error; err != nil {
|
||||
t.Fatalf("load before message: %v", err)
|
||||
}
|
||||
if before.Status != string(model.MessageStatusSent) {
|
||||
t.Fatalf("expected message to remain sent before worker, got %s", before.Status)
|
||||
}
|
||||
processed, err := wp.ProcessOne(t.Context())
|
||||
if err != nil || !processed {
|
||||
t.Fatalf("process twilio status job processed=%v err=%v", processed, err)
|
||||
}
|
||||
var updated model.Message
|
||||
if err := db.First(&updated, message.ID).Error; err != nil {
|
||||
t.Fatalf("load updated message: %v", err)
|
||||
}
|
||||
if updated.Status != string(model.MessageStatusFailed) {
|
||||
t.Fatalf("expected failed status, got %s", updated.Status)
|
||||
}
|
||||
attrs := map[string]any{}
|
||||
if err := json.Unmarshal(updated.ContentAttributes, &attrs); err != nil {
|
||||
t.Fatalf("unmarshal attrs: %v", err)
|
||||
}
|
||||
if attrs["external_error"] != "30003 - Unreachable handset" {
|
||||
t.Fatalf("expected twilio external error, got %#v", attrs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatsAppWebhookPersistsIncomingMessage(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db := newWebhookLookupTestDB(t)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/gochat/gochat/internal/channel"
|
||||
"github.com/gochat/gochat/internal/channel/whatsapp"
|
||||
"github.com/gochat/gochat/internal/model"
|
||||
"github.com/gochat/gochat/internal/worker"
|
||||
)
|
||||
|
||||
// WhatsAppWebhookHandler is a Gin adapter that wraps the WhatsApp
|
||||
@@ -70,6 +71,13 @@ func NewWhatsAppWebhookHandler(provider *whatsapp.WhatsAppProvider, waWebhook *w
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *WhatsAppWebhookHandler) WithWorkerPool(wp *worker.WorkerPool) *WhatsAppWebhookHandler {
|
||||
if h != nil && h.persister != nil {
|
||||
h.persister.SetWorkerPool(wp)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
type whatsAppPersisterAdapter struct {
|
||||
persister *IncomingPersister
|
||||
}
|
||||
|
||||
@@ -395,6 +395,20 @@ func RegisterRoutes(
|
||||
}
|
||||
handlers.TwilioWebhook.HandleTwilioDeliveryStatus(c)
|
||||
})
|
||||
twilioGroup.POST("/delivery_status", func(c *gin.Context) {
|
||||
if handlers == nil || handlers.TwilioWebhook == nil {
|
||||
chatwootParityStub(c)
|
||||
return
|
||||
}
|
||||
handlers.TwilioWebhook.HandleTwilioDeliveryStatus(c)
|
||||
})
|
||||
engine.POST("/twilio/delivery_status", func(c *gin.Context) {
|
||||
if handlers == nil || handlers.TwilioWebhook == nil {
|
||||
chatwootParityStub(c)
|
||||
return
|
||||
}
|
||||
handlers.TwilioWebhook.HandleTwilioDeliveryStatus(c)
|
||||
})
|
||||
|
||||
// Twitter webhook — Account Activity API CRC validation + event processing
|
||||
// GET: CRC challenge response (crc_token query param)
|
||||
|
||||
Reference in New Issue
Block a user