From 781ed352dd6c8641f5c4ae5252a682dce7803b6b Mon Sep 17 00:00:00 2001 From: Rogee Date: Sun, 7 Jun 2026 05:11:35 +0800 Subject: [PATCH] feat(contacts): align filter payloads --- docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md | 14 +- internal/handler/api/v1/contact_handler.go | 4 + .../api/v1/contact_handler_crud_test.go | 117 +++++ internal/repository/contact_repo.go | 420 +++++++++++++++++- 4 files changed, 543 insertions(+), 12 deletions(-) diff --git a/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md b/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md index 6bd5a5d2..7f5c967c 100644 --- a/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md +++ b/docs/CHATWOOT_PARITY_DEVELOPMENT_PLAN.md @@ -49,12 +49,12 @@ Hermes task landing checklist: ## Current Baseline -- Current tracking checkpoint: 2026-06-07 P3.67 conversation filter standard-operator parity, prepared as `feat(conversations): align standard filters`. -- Latest implementation checkpoint: this checkpoint, prepared as `feat(conversations): align standard filters`. -- Latest documentation/tooling checkpoint: this tracker update records Chatwoot `Conversations::FilterService` standard labels/presence/date operator behavior. No route artifacts change. +- Current tracking checkpoint: 2026-06-07 P3.68 contact filter payload parity, prepared as `feat(contacts): align filter payloads`. +- Latest implementation checkpoint: this checkpoint, prepared as `feat(contacts): align filter payloads`. +- Latest documentation/tooling checkpoint: this tracker update records Chatwoot `Contacts::FilterService` payload behavior for standard, additional, label, date, and custom contact attributes. No route artifacts change. - Plan landing status: complete for the current known Hermes plans and user-confirmed scope. Future work should update this file directly instead of opening a parallel tracker. -- Worktree status at this implementation checkpoint: conversation filter payload conditions now support the remaining Chatwoot standard conversation filter operators for `assignee_id`, `inbox_id`, `team_id`, `campaign_id`, `labels`, `display_id`, `created_at`, and `last_activity_at`, while retaining P3.66 additional/custom attributes, P3.65 query-operator chains, P3.64 status/priority/display-id basics, and filtered counts. P3.63 search message-order parity, P3.62 trimmed search payload fields, P3.61 sender-name, P3.60 numeric message-type parity, and prior report/profile/channel/contact checkpoints remain implemented. Live API/browser/enterprise smoke still needs the full PostgreSQL/Redis/Meilisearch/GoChat/Vite/Chrome stack. -- Next executable implementation checkpoint: continue Phase 2/3 drift audit for remaining conversation filter message/contact custom-attribute drift or the next reused-frontend mismatch, or run B12 live smoke when the full PostgreSQL/Redis/Meilisearch/GoChat/Vite/Chrome stack is available. Re-run Phase 6 placeholder audit after future route/smoke changes. +- Worktree status at this implementation checkpoint: contact filter payload conditions now follow Chatwoot `Contacts::FilterService` for `name`, `email`, `phone_number`, `identifier`, `blocked`, additional attributes `country_code`, `city`, `company_name`, contact labels, `created_at`, `last_activity_at`, query-operator validation, and account-scoped contact custom attributes with null-inclusive `not_equal_to`. Existing P3.64-P3.67 conversation filter payload parity, P3.63 search message-order parity, P3.62 trimmed search payload fields, P3.61 sender-name, P3.60 numeric message-type parity, and prior report/profile/channel/contact checkpoints remain implemented. Live API/browser/enterprise smoke still needs the full PostgreSQL/Redis/Meilisearch/GoChat/Vite/Chrome stack. +- Next executable implementation checkpoint: continue Phase 2/3 drift audit for remaining message filters, conversation permission-filter depth, contact filter frontend/reference drift such as frontend-only `referer`, or the next reused-frontend mismatch; or run B12 live smoke when the full PostgreSQL/Redis/Meilisearch/GoChat/Vite/Chrome stack is available. Re-run Phase 6 placeholder audit after future route/smoke changes. - `go test ./...` passes when run outside the restricted socket sandbox for the latest implementation baseline; the latest docs/tooling checkpoint verified `scripts/parity_frontend_smoke.sh --check` with workspace-local temp/cache dirs after `/tmp` was full. - Route dump succeeds with `967` registered routes after profile MFA route tracking. - Route parity artifacts now exist under `docs/parity/` and are generated by `cmd/route_parity`. @@ -156,6 +156,7 @@ This table is the shortest authoritative handoff view. If an older lower section | Priority | Workstream | Current state | Next checkpoint | Commit close rule | | --- | --- | --- | --- | --- | +| 0 | P3.68 contact filter payload parity | Implemented for reused CRM contact advanced filters: `POST /contacts/filter` now accepts Chatwoot `{ payload: [...] }` conditions for standard contact fields, additional contact attributes, labels, dates, query operators, and account-scoped contact custom attributes. | Keep in Review; reopen from B12 CRM filter smoke or fresh reference evidence for frontend-only `referer`, exact PostgreSQL JSON/date casting drift, or contact `base_relation` CRM v2 resolved-contact scope. | Focused contact filter handler tests, contact repository/service/handler regression tests, full `go test ./...`, and `git diff --check` must pass. No route artifacts change. | | 0 | P3.67 conversation filter standard-operator parity | Implemented for reused dashboard advanced conversation filters: `{ payload: [...] }` now covers Chatwoot standard conversation operators for labels, presence checks, display-id contains, and date comparisons/days-before on `created_at` and `last_activity_at`. | Keep in Review; reopen from B12 filter smoke or fresh reference evidence for message filters, contact custom attributes inside conversation filters, or full `Conversations::PermissionFilterService` depth. | Focused conversation filter handler tests, full `go test ./...`, and `git diff --check` must pass. No route artifacts change. | | 0 | P3.66 conversation filter attribute parity | Implemented for reused dashboard advanced conversation filters: `{ payload: [...] }` now supports Chatwoot conversation additional attributes from `filter_keys.yml` and account-scoped conversation custom attributes, including text/list/link case handling and `not_equal_to` matching rows where the custom attribute is absent. | Keep in Review; reopen from B12 filter smoke or fresh reference evidence for message filters, contact custom attributes inside conversation filters, or full `Conversations::PermissionFilterService` depth. | Focused conversation filter handler tests, full `go test ./...`, and `git diff --check` must pass. No route artifacts change. | | 0 | P3.65 conversation filter query-operator parity | Implemented for reused dashboard advanced conversation filters: `{ payload: [...] }` conditions now honor `query_operator` chains (`AND`/`OR`) using the same SQL precedence as Chatwoot's concatenated filter query, and invalid query operators return raw `422 { error }`. | Keep in Review; reopen from B12 filter smoke or fresh reference evidence for message filters, contact custom attributes inside conversation filters, or full `Conversations::PermissionFilterService` depth. | Focused conversation filter handler tests, full `go test ./...`, and `git diff --check` must pass. No route artifacts change. | @@ -230,6 +231,7 @@ These rows are the executable development plan from this point forward. A checkp | ID | Owner files | Reference files | Work to land | Exit gate | | --- | --- | --- | --- | --- | +| P3.68 contact filter payload parity | `internal/repository/contact_repo.go`, `internal/handler/api/v1/contact_handler.go`, `internal/handler/api/v1/contact_handler_crud_test.go` | `reference/chatwoot/app/services/contacts/filter_service.rb`, `reference/chatwoot/app/services/filter_service.rb`, `reference/chatwoot/app/helpers/filters/filter_helper.rb`, `reference/chatwoot/app/services/filters/custom_attribute_filter_helper.rb`, `reference/chatwoot/lib/filters/filter_keys.yml`, `reference/chatwoot/app/javascript/dashboard/components-next/filter/contactProvider.js` | Contact filter now accepts the reused CRM `{ payload: [...] }` body, validates `query_operator`, returns raw `422 { error }` for payload validation failures, normalizes phone numbers and case-insensitive values like Chatwoot, supports standard contact fields, additional attributes `country_code`, `city`, `company_name`, labels, date comparisons/days-before, and account-scoped contact custom attributes with null-inclusive `not_equal_to`. Backend reference excludes frontend-only `referer`, so it remains a tracked drift candidate rather than a silently accepted local extension. | Review by `feat(contacts): align filter payloads`; focused contact filter handler tests assert standard/additional filters, labels plus date filters, custom-attribute `not_equal_to` null inclusion, and invalid attribute errors; full `go test ./...` and `git diff --check`. No route artifacts change. | | P3.67 conversation filter standard-operator parity | `internal/service/conversation_service.go`, `internal/handler/api/v1/conversation_handler_crud_test.go` | `reference/chatwoot/lib/filters/filter_keys.yml`, `reference/chatwoot/app/services/filter_service.rb#filter_operation`, `reference/chatwoot/app/services/filter_service.rb#tag_filter_query`, `reference/chatwoot/app/services/filter_service.rb#lt_gt_filter_query`, `reference/chatwoot/app/services/conversations/filter_service.rb` | Conversation filter payload now supports Chatwoot standard filter operators beyond the P3.64 basics: `assignee_id`, `inbox_id`, `team_id`, and `campaign_id` support equality plus present/absent checks; `display_id` supports contains/does-not-contain through text casting; `labels` uses account tag joins with equal/not-equal/present/absent semantics; `created_at` and `last_activity_at` support greater-than, less-than, and days-before. | Review by `feat(conversations): align standard filters`; focused filter handler tests assert presence filters, label joins, display-id contains with date filters, last-activity days-before, full `go test ./...`, and `git diff --check`. No route artifacts change. | | P3.66 conversation filter attribute parity | `internal/service/conversation_service.go`, `internal/handler/api/v1/conversation_handler_crud_test.go` | `reference/chatwoot/lib/filters/filter_keys.yml`, `reference/chatwoot/app/helpers/filters/filter_helper.rb#handle_additional_attributes`, `reference/chatwoot/app/services/filters/custom_attribute_filter_helper.rb`, `reference/chatwoot/app/services/filter_service.rb#filter_operation`, `reference/chatwoot/app/services/conversations/filter_service.rb` | Conversation filter payload now supports additional-attribute keys `browser_language`, `conversation_language`, `referer`, and `mail_subject` with Chatwoot operator sets, dialect-aware JSON extraction, case-insensitive contains checks, and account-scoped conversation custom attribute definitions. Custom `not_equal_to` includes conversations where the custom attribute key is absent, matching Chatwoot's appended `IS NULL` behavior. | Review by `feat(conversations): align filter attributes`; focused filter handler tests assert additional-attribute equality, additional-attribute contains, and custom-attribute `not_equal_to` null inclusion; full `go test ./...` and `git diff --check` must pass. No route artifacts change. | | P3.65 conversation filter query-operator parity | `internal/service/conversation_service.go`, conversation filter handler tests | `reference/chatwoot/app/services/filter_service.rb#query_builder`, `reference/chatwoot/app/helpers/filters/filter_helper.rb#validate_single_condition`, `reference/chatwoot/spec/services/conversations/filter_service_frontend_alignment_spec.rb` | Conversation filter payload conditions now append `query_operator` into one SQL predicate chain, matching Chatwoot's `A AND B OR C` / `A OR B AND C` frontend alignment behavior and returning raw `422 { error: 'Query operator must be either "AND" or "OR".' }` for invalid operators. | Review by `feat(conversations): align filter query operators`; focused filter handler tests assert OR/AND precedence and invalid query-operator error; full `go test ./...` and `git diff --check` must pass. No route artifacts change. | @@ -322,6 +324,7 @@ This ledger records the committed parity checkpoints that future slices should b | Commit | Scope | Verification summary | Follow-up state | | --- | --- | --- | --- | +| `feat(contacts): align filter payloads` | Advances P3.68 with Chatwoot contact filter payload parity. `POST /contacts/filter` now understands `{ payload: [...] }` conditions for standard fields, additional attributes, labels, dates, query-operator chains, and account-scoped contact custom attributes; payload errors return raw `422 { error }`, and contact equality uses scalar `=`/`!=` semantics matching `Contacts::FilterService`. | `go test ./internal/handler/api/v1 -run 'ContactHandlerCRUDTestSuite/TestFilter' -count=1`; `go test ./internal/repository ./internal/service ./internal/handler/api/v1 -run 'Contact.*(Filter\|List\|Search\|Export\|Label)\|ContactHandlerCRUDTestSuite/Test(Filter\|Labels\|List\|Search\|Export)\|ContactService\|ContactRepo' -count=1`; `go test ./...`; `git diff --check`. No route artifacts change. | Move P3.68 to Review; continue Phase 2/3 drift audit for frontend-only `referer`, message filters, conversation permission-filter depth, Phase 6 placeholder audit, or B12 live smoke. | | `feat(conversations): align standard filters` | Advances P3.67 with Chatwoot conversation filter standard-operator parity. Payload filters now cover presence operators for assignee/inbox/team/campaign fields, display-id contains/does-not-contain through text casts, account tag joins for labels, and date/days-before comparisons for `created_at` plus Unix-second `last_activity_at`. | `go test ./internal/handler/api/v1 -run 'ConversationCrudTestSuite/TestFilter' -count=1`; `go test ./internal/repository ./internal/service ./internal/handler/api/v1 -run 'ConversationRepo_(GetMeta\|ListForFinder)\|Conversation.*(Meta\|List\|Search\|Filter)\|ConversationCrudTestSuite/Test(Filter\|Search\|List)\|ConversationService' -count=1`; `go test ./...`; `git diff --check`. No route artifacts change. | Move P3.67 to Review; continue Phase 2/3 drift audit for message filters, contact custom attributes inside conversation filters, Phase 6 placeholder audit, or B12 live smoke. | | `feat(conversations): align filter attributes` | Advances P3.66 with Chatwoot conversation filter additional/custom attribute parity. Payload filters now cover `browser_language`, `conversation_language`, `referer`, and `mail_subject` through JSON extraction and Chatwoot operator sets, and custom conversation attributes are resolved through account-scoped definitions with text/list/link lowercasing plus null-inclusive `not_equal_to`. | `go test ./internal/handler/api/v1 -run 'ConversationCrudTestSuite/TestFilter' -count=1`; `go test ./internal/repository ./internal/service ./internal/handler/api/v1 -run 'ConversationRepo_(GetMeta\|ListForFinder)\|Conversation.*(Meta\|List\|Search\|Filter)\|ConversationCrudTestSuite/Test(Filter\|Search\|List)\|ConversationService' -count=1`; `go test ./...`; `git diff --check`. No route artifacts change. | Move P3.66 to Review; continue Phase 2/3 drift audit for message filters, contact custom attributes inside conversation filters, Phase 6 placeholder audit, or B12 live smoke. | | `feat(conversations): align filter query operators` | Advances P3.65 with Chatwoot conversation filter frontend alignment. Payload conditions now form a single SQL predicate chain with `AND`/`OR` operators and database precedence, so `A OR B AND C` and `A AND B OR C` behave like the reference `FilterService#query_builder`; invalid query operators return raw `422 { error }`. | `go test ./internal/handler/api/v1 -run 'ConversationCrudTestSuite/TestFilter' -count=1`; `go test ./internal/repository ./internal/service ./internal/handler/api/v1 -run 'ConversationRepo_(GetMeta\|ListForFinder)\|Conversation.*(Meta\|List\|Search\|Filter)\|ConversationCrudTestSuite/Test(Filter\|Search\|List)\|ConversationService' -count=1`; `go test ./...`; `git diff --check`. No route artifacts change. | Move P3.65 to Review; continue Phase 2/3 drift audit for labels/dates/assignee/inbox/team/message filter payload depth, Phase 6 placeholder audit, or B12 live smoke. | @@ -2649,3 +2652,4 @@ Verification milestone gates: - 2026-06-07: P3.65 conversation filter query-operator checkpoint prepared as `feat(conversations): align filter query operators`; audited Chatwoot `FilterService#query_builder`, `Filters::FilterHelper#validate_single_condition`, and frontend alignment specs. GoChat filter payload chains now honor `AND`/`OR` query operators with SQL precedence and return raw invalid-query-operator errors. Focused filter handler tests passed; full `go test ./...` and `git diff --check` passed. No route artifacts change. - 2026-06-07: P3.66 conversation filter attribute checkpoint prepared as `feat(conversations): align filter attributes`; audited Chatwoot `filter_keys.yml`, `Filters::FilterHelper#handle_additional_attributes`, `Filters::CustomAttributeFilterHelper`, `FilterService#filter_operation`, and `Conversations::FilterService`. GoChat payload filters now support conversation additional attributes (`browser_language`, `conversation_language`, `referer`, `mail_subject`) and account-scoped conversation custom attributes, including case-normalized text/list/link comparisons and `not_equal_to` matching absent custom-attribute keys. Focused filter handler tests passed; full `go test ./...` and `git diff --check` passed. No route artifacts change. - 2026-06-07: P3.67 conversation filter standard-operator checkpoint prepared as `feat(conversations): align standard filters`; audited Chatwoot `filter_keys.yml`, `FilterService#tag_filter_query`, `#filter_operation`, `#lt_gt_filter_query`, and `Conversations::FilterService`. GoChat payload filters now support presence operators for assignee/inbox/team/campaign fields, label equal/not-equal/present/absent through account tag joins, `display_id` contains/does-not-contain through text casts, and date/days-before comparisons for `created_at` and Unix-second `last_activity_at`. Focused filter handler tests passed; full `go test ./...` and `git diff --check` passed. No route artifacts change. +- 2026-06-07: P3.68 contact filter payload checkpoint prepared as `feat(contacts): align filter payloads`; audited Chatwoot `Contacts::FilterService`, base `FilterService`, `Filters::FilterHelper`, `CustomAttributeFilterHelper`, contact `filter_keys.yml`, and reused `contactProvider.js`. GoChat contact filters now accept frontend `{ payload: [...] }` bodies for standard fields, additional attributes, labels, dates, query operators, and account-scoped contact custom attributes; errors return raw `422 { error }`, phone/case normalization follows the reference, and contact equality uses scalar `=`/`!=`. Focused contact filter tests passed; full `go test ./...` and `git diff --check` passed. No route artifacts change. diff --git a/internal/handler/api/v1/contact_handler.go b/internal/handler/api/v1/contact_handler.go index 9d3fb70f..fef9afe8 100644 --- a/internal/handler/api/v1/contact_handler.go +++ b/internal/handler/api/v1/contact_handler.go @@ -1080,6 +1080,10 @@ func (h *ContactHandler) Filter(c *gin.Context) { offset := (page - 1) * perPage contacts, total, svcErr := h.svc.Filter(c.Request.Context(), accountID, params, offset, perPage) if svcErr != nil { + if len(params.Payload) > 0 { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": svcErr.Error()}) + return + } c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "failed to filter contacts"}) return } diff --git a/internal/handler/api/v1/contact_handler_crud_test.go b/internal/handler/api/v1/contact_handler_crud_test.go index 95c99fbd..3ca707b5 100644 --- a/internal/handler/api/v1/contact_handler_crud_test.go +++ b/internal/handler/api/v1/contact_handler_crud_test.go @@ -12,6 +12,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "gorm.io/datatypes" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" @@ -52,6 +53,7 @@ func (s *ContactHandlerCRUDTestSuite) SetupSuite() { &model.Contact{}, &model.Tag{}, &model.ContactLabel{}, + &model.CustomAttributeDefinition{}, &model.ContactExport{}, &model.DataImport{}, &model.Notification{}, @@ -89,6 +91,7 @@ func (s *ContactHandlerCRUDTestSuite) SetupSuite() { s.router.GET("/api/v1/accounts/:id/contacts", s.handler.List) s.router.GET("/api/v1/accounts/:id/contacts/search", s.handler.Search) s.router.GET("/api/v1/accounts/:id/contacts/active", s.handler.Active) + s.router.POST("/api/v1/accounts/:id/contacts/filter", s.handler.Filter) s.router.POST("/api/v1/accounts/:id/contacts/export", s.handler.ExportRequest) s.router.GET("/api/v1/accounts/:id/contacts/export/:export_id/download", s.handler.DownloadExport) s.router.POST("/api/v1/accounts/:id/contacts/import", s.handler.Import) @@ -137,6 +140,7 @@ func (s *ContactHandlerCRUDTestSuite) SetupSuite() { func (s *ContactHandlerCRUDTestSuite) SetupTest() { s.db.Exec("DELETE FROM contact_notes") s.db.Exec("DELETE FROM contact_labels") + s.db.Exec("DELETE FROM custom_attribute_definitions") s.db.Exec("DELETE FROM tags") s.db.Exec("DELETE FROM contact_exports") s.db.Exec("DELETE FROM data_imports") @@ -854,6 +858,119 @@ func (s *ContactHandlerCRUDTestSuite) TestLabels_UpdateListAndFilter() { s.Equal(float64(s.contact.ID), payload[0].(map[string]interface{})["id"]) } +func (s *ContactHandlerCRUDTestSuite) TestFilter_ChatwootPayloadStandardAndAdditionalAttributes() { + s.contact.PhoneNumber = "+1234567890" + s.contact.AdditionalAttributes = datatypes.JSON(`{"country_code":"uk","city":"London"}`) + s.Require().NoError(s.db.Save(s.contact).Error) + + other := &model.Contact{AccountID: s.account.ID, Name: "Other User", Email: "other@example.com", PhoneNumber: "+1987654321", AdditionalAttributes: datatypes.JSON(`{"country_code":"gr","city":"Athens"}`)} + s.Require().NoError(s.db.Create(other).Error) + + body, _ := json.Marshal(map[string]interface{}{ + "payload": []map[string]interface{}{ + {"attribute_key": "email", "filter_operator": "contains", "values": []string{"JANE"}, "query_operator": "AND"}, + {"attribute_key": "phone_number", "filter_operator": "equal_to", "values": []string{"1234567890"}, "query_operator": "AND"}, + {"attribute_key": "country_code", "filter_operator": "equal_to", "values": []string{"UK"}}, + }, + }) + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", fmt.Sprintf("/api/v1/accounts/%d/contacts/filter", s.account.ID), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + s.router.ServeHTTP(w, req) + + s.Equal(http.StatusOK, w.Code) + var resp map[string]interface{} + s.NoError(json.Unmarshal(w.Body.Bytes(), &resp)) + payload := resp["payload"].([]interface{}) + s.Len(payload, 1) + s.Equal(float64(s.contact.ID), payload[0].(map[string]interface{})["id"]) +} + +func (s *ContactHandlerCRUDTestSuite) TestFilter_ChatwootPayloadLabelsAndDate() { + tag := &model.Tag{AccountID: s.account.ID, Name: "support", Color: "#1f93ff"} + s.Require().NoError(s.db.Create(tag).Error) + s.Require().NoError(s.db.Create(&model.ContactLabel{AccountID: s.account.ID, ContactID: s.contact.ID, TagID: tag.ID}).Error) + recentActivity := time.Now().UTC().AddDate(0, 0, -1).Unix() + oldActivity := time.Now().UTC().AddDate(0, 0, -8).Unix() + s.contact.LastActivityAt = &oldActivity + s.Require().NoError(s.db.Save(s.contact).Error) + + other := &model.Contact{AccountID: s.account.ID, Name: "Other User", LastActivityAt: &recentActivity} + s.Require().NoError(s.db.Create(other).Error) + + body, _ := json.Marshal(map[string]interface{}{ + "payload": []map[string]interface{}{ + {"attribute_key": "labels", "filter_operator": "is_present", "values": []string{}, "query_operator": "AND"}, + {"attribute_key": "last_activity_at", "filter_operator": "days_before", "values": []string{"3"}}, + }, + }) + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", fmt.Sprintf("/api/v1/accounts/%d/contacts/filter", s.account.ID), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + s.router.ServeHTTP(w, req) + + s.Equal(http.StatusOK, w.Code) + var resp map[string]interface{} + s.NoError(json.Unmarshal(w.Body.Bytes(), &resp)) + payload := resp["payload"].([]interface{}) + s.Len(payload, 1) + s.Equal(float64(s.contact.ID), payload[0].(map[string]interface{})["id"]) +} + +func (s *ContactHandlerCRUDTestSuite) TestFilter_ChatwootPayloadCustomAttributeNotEqualIncludesNull() { + def := &model.CustomAttributeDefinition{ + AccountID: s.account.ID, + AttributeName: "customer_type", + AttributeDisplayName: "Customer type", + AttributeType: "list", + AttributeModel: "contact_attribute", + AttributeValues: datatypes.JSON(`["platinum","regular"]`), + } + s.Require().NoError(s.db.Create(def).Error) + s.contact.CustomAttributes = datatypes.JSON(`{"customer_type":"platinum"}`) + s.Require().NoError(s.db.Save(s.contact).Error) + + regular := &model.Contact{AccountID: s.account.ID, Name: "Regular User", CustomAttributes: datatypes.JSON(`{"customer_type":"regular"}`)} + s.Require().NoError(s.db.Create(regular).Error) + missing := &model.Contact{AccountID: s.account.ID, Name: "Missing User", CustomAttributes: datatypes.JSON(`{}`)} + s.Require().NoError(s.db.Create(missing).Error) + + body, _ := json.Marshal(map[string]interface{}{ + "payload": []map[string]interface{}{ + {"attribute_key": "customer_type", "custom_attribute_type": "contact_attribute", "filter_operator": "not_equal_to", "values": []string{"platinum"}}, + }, + }) + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", fmt.Sprintf("/api/v1/accounts/%d/contacts/filter", s.account.ID), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + s.router.ServeHTTP(w, req) + + s.Equal(http.StatusOK, w.Code) + var resp map[string]interface{} + s.NoError(json.Unmarshal(w.Body.Bytes(), &resp)) + payload := resp["payload"].([]interface{}) + s.Len(payload, 2) + ids := []float64{payload[0].(map[string]interface{})["id"].(float64), payload[1].(map[string]interface{})["id"].(float64)} + s.ElementsMatch([]float64{float64(regular.ID), float64(missing.ID)}, ids) +} + +func (s *ContactHandlerCRUDTestSuite) TestFilter_ChatwootPayloadInvalidAttribute() { + body, _ := json.Marshal(map[string]interface{}{ + "payload": []map[string]interface{}{ + {"attribute_key": "unknown", "filter_operator": "equal_to", "values": []string{"x"}}, + }, + }) + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", fmt.Sprintf("/api/v1/accounts/%d/contacts/filter", s.account.ID), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + s.router.ServeHTTP(w, req) + + s.Equal(http.StatusUnprocessableEntity, w.Code) + var resp map[string]string + s.NoError(json.Unmarshal(w.Body.Bytes(), &resp)) + s.Contains(resp["error"], "Invalid attribute key - [unknown]") +} + func (s *ContactHandlerCRUDTestSuite) TestDestroyCustomAttributes_SelectedKeysPayload() { attrs := model.JSONMap{"tier": "gold", "plan": "pro", "vip": true} s.contact.CustomAttributes = model.ToDatatypesJSON(&attrs) diff --git a/internal/repository/contact_repo.go b/internal/repository/contact_repo.go index 1b94e743..78da6c80 100644 --- a/internal/repository/contact_repo.go +++ b/internal/repository/contact_repo.go @@ -2,6 +2,9 @@ package repository import ( "context" + "errors" + "fmt" + "strconv" "strings" "time" @@ -317,13 +320,22 @@ func (r *ContactRepo) DeleteCustomAttributes(ctx context.Context, id uint) error // ContactFilterParams holds filter criteria for the POST /contacts/filter endpoint. // Reference: Chatwoot ContactFilterService#perform type ContactFilterParams struct { - Sort string `json:"sort,omitempty" form:"sort"` - ContactType string `json:"contact_type,omitempty" form:"contact_type"` - ContactSource string `json:"contact_source,omitempty" form:"contact_source"` - InboxID *uint `json:"inbox_id,omitempty" form:"inbox_id"` - Labels string `json:"labels,omitempty" form:"labels"` - Status string `json:"status,omitempty" form:"status"` // active, inactive - UpdatedWithin *int `json:"updated_within,omitempty" form:"updated_within"` // seconds + Sort string `json:"sort,omitempty" form:"sort"` + ContactType string `json:"contact_type,omitempty" form:"contact_type"` + ContactSource string `json:"contact_source,omitempty" form:"contact_source"` + InboxID *uint `json:"inbox_id,omitempty" form:"inbox_id"` + Labels string `json:"labels,omitempty" form:"labels"` + Status string `json:"status,omitempty" form:"status"` // active, inactive + UpdatedWithin *int `json:"updated_within,omitempty" form:"updated_within"` // seconds + Payload []ContactFilterCondition `json:"payload,omitempty" form:"-"` +} + +type ContactFilterCondition struct { + AttributeKey string `json:"attribute_key"` + FilterOperator string `json:"filter_operator"` + Values []any `json:"values"` + QueryOperator string `json:"query_operator,omitempty"` + CustomAttributeType string `json:"custom_attribute_type,omitempty"` } // Filter retrieves contacts matching advanced filter criteria with pagination. @@ -335,6 +347,13 @@ func (r *ContactRepo) Filter(ctx context.Context, accountID uint, params Contact q := r.db.WithContext(ctx).Model(&model.Contact{}).Where("contacts.account_id = ?", accountID) usesDistinct := false + if len(params.Payload) > 0 { + advancedQuery, err := r.applyContactFilterPayload(ctx, q, accountID, params.Payload) + if err != nil { + return nil, 0, err + } + q = advancedQuery + } // Apply contact type filter if params.ContactType != "" { @@ -389,3 +408,390 @@ func (r *ContactRepo) Filter(ctx context.Context, accountID uint, params Contact Find(&contacts).Error return contacts, total, err } + +func (r *ContactRepo) applyContactFilterPayload(ctx context.Context, query *gorm.DB, accountID uint, payload []ContactFilterCondition) (*gorm.DB, error) { + clauses := make([]string, 0, len(payload)*2) + args := make([]any, 0, len(payload)) + for _, condition := range payload { + queryOperator := strings.ToUpper(strings.TrimSpace(condition.QueryOperator)) + if queryOperator != "" && queryOperator != "AND" && queryOperator != "OR" { + return nil, fmt.Errorf("Query operator must be either \"AND\" or \"OR\".") + } + + clause, clauseArgs, err := r.contactFilterClause(ctx, query, accountID, condition) + if err != nil { + return nil, err + } + if len(clauses) > 0 && !contactFilterIsQueryOperator(clauses[len(clauses)-1]) { + clauses = append(clauses, "AND") + } + clauses = append(clauses, clause) + args = append(args, clauseArgs...) + if queryOperator != "" { + clauses = append(clauses, queryOperator) + } + } + if len(clauses) == 0 { + return query, nil + } + if contactFilterIsQueryOperator(clauses[len(clauses)-1]) { + clauses = clauses[:len(clauses)-1] + } + return query.Where(strings.Join(clauses, " "), args...), nil +} + +func (r *ContactRepo) contactFilterClause(ctx context.Context, query *gorm.DB, accountID uint, condition ContactFilterCondition) (string, []any, error) { + attribute := strings.TrimSpace(condition.AttributeKey) + operator := strings.TrimSpace(condition.FilterOperator) + values := contactFilterStringValues(condition.Values) + values = contactFilterNormalizeValues(attribute, values) + + if attribute == "labels" { + return contactFilterLabelsClause(accountID, operator, values) + } + if attribute == "created_at" || attribute == "last_activity_at" { + return contactFilterDateClause(attribute, operator, values) + } + if additional, ok := contactFilterAdditionalAttribute(attribute); ok { + expr, exprArgs := contactFilterJSONExtract(query, "contacts.additional_attributes", attribute) + if additional.caseInsensitive { + expr = "LOWER(" + expr + ")" + } + return contactFilterBuildClause(attribute, expr, exprArgs, operator, values, additional.allowedOperators, false) + } + if column, allowedOperators, ok := contactFilterColumn(attribute); ok { + return contactFilterBuildClause(attribute, column, nil, operator, values, allowedOperators, false) + } + + def, defErr := r.findContactFilterCustomAttributeDefinition(ctx, accountID, attribute, condition.CustomAttributeType) + if defErr != nil { + return "", nil, defErr + } + if def == nil { + return "", nil, fmt.Errorf("Invalid attribute key - [%s]", attribute) + } + expr, exprArgs := contactFilterJSONExtract(query, "contacts.custom_attributes", attribute) + return contactFilterCustomAttributeClause(attribute, expr, exprArgs, operator, values, def.AttributeType) +} + +func contactFilterColumn(attribute string) (string, []string, bool) { + switch attribute { + case "name", "email": + return "LOWER(contacts." + attribute + ")", []string{"equal_to", "not_equal_to", "contains", "does_not_contain"}, true + case "phone_number": + return "contacts.phone_number", []string{"equal_to", "not_equal_to", "contains", "does_not_contain", "starts_with"}, true + case "identifier": + return "LOWER(contacts.identifier)", []string{"equal_to", "not_equal_to"}, true + case "blocked": + return "contacts.blocked", []string{"equal_to", "not_equal_to"}, true + default: + return "", nil, false + } +} + +type contactAdditionalFilter struct { + allowedOperators []string + caseInsensitive bool +} + +func contactFilterAdditionalAttribute(attribute string) (contactAdditionalFilter, bool) { + switch attribute { + case "country_code": + return contactAdditionalFilter{allowedOperators: []string{"equal_to", "not_equal_to"}, caseInsensitive: true}, true + case "city", "company_name": + return contactAdditionalFilter{allowedOperators: []string{"equal_to", "not_equal_to", "contains", "does_not_contain"}, caseInsensitive: true}, true + default: + return contactAdditionalFilter{}, false + } +} + +func (r *ContactRepo) findContactFilterCustomAttributeDefinition(ctx context.Context, accountID uint, attribute, customAttributeType string) (*model.CustomAttributeDefinition, error) { + attributeModel := strings.TrimSpace(customAttributeType) + if attributeModel == "" || attributeModel == "contact" { + attributeModel = "contact_attribute" + } + if attributeModel != "contact_attribute" { + return nil, fmt.Errorf("Invalid attribute key - [%s]", attribute) + } + var def model.CustomAttributeDefinition + err := r.db.WithContext(ctx). + Where("account_id = ? AND attribute_name = ? AND attribute_model IN ?", accountID, attribute, []string{"contact_attribute", "contact"}). + First(&def).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &def, nil +} + +func contactFilterLabelsClause(accountID uint, operator string, values []string) (string, []any, error) { + allowedOperators := []string{"equal_to", "not_equal_to", "is_present", "is_not_present"} + if !contactFilterOperatorAllowed(operator, allowedOperators) { + return "", nil, fmt.Errorf("Invalid operator. The allowed operators for labels are [%s].", strings.Join(allowedOperators, ",")) + } + if operator != "is_present" && operator != "is_not_present" && len(values) == 0 { + return "", nil, fmt.Errorf("Invalid value for labels") + } + base := "SELECT 1 FROM contact_labels JOIN tags ON tags.id = contact_labels.tag_id WHERE contact_labels.contact_id = contacts.id AND contact_labels.account_id = ? AND tags.account_id = ?" + args := []any{accountID, accountID} + switch operator { + case "equal_to": + args = append(args, values) + return "EXISTS (" + base + " AND tags.name IN ?)", args, nil + case "not_equal_to": + args = append(args, values) + return "NOT EXISTS (" + base + " AND tags.name IN ?)", args, nil + case "is_present": + return "EXISTS (" + base + ")", args, nil + case "is_not_present": + return "NOT EXISTS (" + base + ")", args, nil + default: + return "", nil, fmt.Errorf("Invalid operator. The allowed operators for labels are [%s].", strings.Join(allowedOperators, ",")) + } +} + +func contactFilterDateClause(attribute, operator string, values []string) (string, []any, error) { + allowedOperators := []string{"is_greater_than", "is_less_than", "days_before"} + if !contactFilterOperatorAllowed(operator, allowedOperators) { + return "", nil, fmt.Errorf("Invalid operator. The allowed operators for %s are [%s].", attribute, strings.Join(allowedOperators, ",")) + } + if len(values) == 0 { + return "", nil, fmt.Errorf("Invalid value for %s", attribute) + } + value, err := contactFilterDateValue(attribute, operator, values[0]) + if err != nil { + return "", nil, fmt.Errorf("Invalid value for %s", attribute) + } + comparison := ">" + if operator == "is_less_than" || operator == "days_before" { + comparison = "<" + } + if attribute == "last_activity_at" { + return "contacts.last_activity_at " + comparison + " ?", []any{value.Unix()}, nil + } + return "contacts.created_at " + comparison + " ?", []any{value}, nil +} + +func contactFilterDateValue(attribute, operator, value string) (time.Time, error) { + if operator == "days_before" { + days, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil { + return time.Time{}, err + } + return time.Now().UTC().Truncate(24*time.Hour).AddDate(0, 0, -days), nil + } + if parsed, err := time.Parse(time.RFC3339, value); err == nil { + return parsed, nil + } + if parsed, err := time.Parse("2006-01-02", value); err == nil { + return parsed, nil + } + if attribute == "last_activity_at" { + seconds, err := strconv.ParseInt(value, 10, 64) + if err == nil { + return time.Unix(seconds, 0).UTC(), nil + } + } + return time.Time{}, fmt.Errorf("invalid date") +} + +func contactFilterBuildClause(attribute, expression string, expressionArgs []any, operator string, values []string, allowedOperators []string, includeNullForNotEqual bool) (string, []any, error) { + if !contactFilterOperatorAllowed(operator, allowedOperators) { + return "", nil, fmt.Errorf("Invalid operator. The allowed operators for %s are [%s].", attribute, strings.Join(allowedOperators, ",")) + } + if len(values) == 0 { + return "", nil, fmt.Errorf("Invalid value for %s", attribute) + } + args := append([]any{}, expressionArgs...) + switch operator { + case "equal_to": + args = append(args, contactFilterCoerceValue(attribute, values[0])) + return expression + " = ?", args, nil + case "not_equal_to": + args = append(args, contactFilterCoerceValue(attribute, values[0])) + if includeNullForNotEqual { + args = append(args, expressionArgs...) + return "(" + expression + " != ? OR " + expression + " IS NULL)", args, nil + } + return expression + " != ?", args, nil + case "contains", "does_not_contain", "starts_with": + clause, likeArgs := contactFilterLikeClause(expression, expressionArgs, values, operator) + return clause, likeArgs, nil + default: + return "", nil, fmt.Errorf("Invalid operator. The allowed operators for %s are [%s].", attribute, strings.Join(allowedOperators, ",")) + } +} + +func contactFilterLikeClause(expression string, expressionArgs []any, values []string, operator string) (string, []any) { + clauses := make([]string, 0, len(values)) + args := make([]any, 0, len(values)*(len(expressionArgs)+1)) + negate := operator == "does_not_contain" + likeOperator := "LIKE" + joiner := " OR " + if negate { + likeOperator = "NOT LIKE" + joiner = " AND " + } + for _, value := range values { + pattern := "%" + strings.ToLower(value) + "%" + if operator == "starts_with" { + pattern = strings.ToLower(value) + "%" + } + clauses = append(clauses, "LOWER("+expression+") "+likeOperator+" ?") + args = append(args, expressionArgs...) + args = append(args, pattern) + } + return "(" + strings.Join(clauses, joiner) + ")", args +} + +func contactFilterJSONExtract(query *gorm.DB, column, key string) (string, []any) { + dialect := "" + if query != nil && query.Dialector != nil { + dialect = query.Dialector.Name() + } + switch dialect { + case "sqlite": + return "json_extract(" + column + ", ?)", []any{"$." + key} + case "mysql": + return "JSON_UNQUOTE(JSON_EXTRACT(" + column + ", ?))", []any{"$." + key} + default: + return column + " ->> ?", []any{key} + } +} + +func contactFilterCustomAttributeOperators(attributeType string) []string { + switch strings.TrimSpace(attributeType) { + case "text", "link", "list", "": + return []string{"equal_to", "not_equal_to", "contains", "does_not_contain"} + case "date", "number": + return []string{"equal_to", "not_equal_to", "is_greater_than", "is_less_than", "days_before"} + default: + return []string{"equal_to", "not_equal_to"} + } +} + +func contactFilterCustomAttributeClause(attribute, expression string, expressionArgs []any, operator string, values []string, attributeType string) (string, []any, error) { + allowedOperators := contactFilterCustomAttributeOperators(attributeType) + if !contactFilterOperatorAllowed(operator, allowedOperators) { + return "", nil, fmt.Errorf("Invalid operator. The allowed operators for %s are [%s].", attribute, strings.Join(allowedOperators, ",")) + } + if operator == "is_present" { + return expression + " IS NOT NULL", append([]any{}, expressionArgs...), nil + } + if operator == "is_not_present" { + return expression + " IS NULL", append([]any{}, expressionArgs...), nil + } + if len(values) == 0 { + return "", nil, fmt.Errorf("Invalid value for %s", attribute) + } + + switch strings.TrimSpace(attributeType) { + case "date": + value, err := contactFilterDateValue(attribute, operator, values[0]) + if err != nil { + return "", nil, fmt.Errorf("Invalid value for %s", attribute) + } + comparison := contactFilterComparisonOperator(operator) + if comparison == "" { + return contactFilterBuildClause(attribute, expression, expressionArgs, operator, values, allowedOperators, true) + } + args := append([]any{}, expressionArgs...) + args = append(args, value.Format("2006-01-02")) + return expression + " " + comparison + " ?", args, nil + case "number": + comparison := contactFilterComparisonOperator(operator) + if comparison == "" { + return contactFilterBuildClause(attribute, expression, expressionArgs, operator, values, allowedOperators, true) + } + args := append([]any{}, expressionArgs...) + args = append(args, values[0]) + return "CAST(" + expression + " AS NUMERIC) " + comparison + " ?", args, nil + default: + expression = contactFilterCustomAttributeExpression(expression, attributeType) + return contactFilterBuildClause(attribute, expression, expressionArgs, operator, contactFilterCustomAttributeValues(values, attributeType), allowedOperators, true) + } +} + +func contactFilterComparisonOperator(operator string) string { + switch operator { + case "is_greater_than": + return ">" + case "is_less_than", "days_before": + return "<" + default: + return "" + } +} + +func contactFilterCustomAttributeValues(values []string, attributeType string) []string { + switch strings.TrimSpace(attributeType) { + case "text", "link", "list", "": + lowered := make([]string, 0, len(values)) + for _, value := range values { + lowered = append(lowered, strings.ToLower(value)) + } + return lowered + default: + return values + } +} + +func contactFilterCustomAttributeExpression(expression, attributeType string) string { + switch strings.TrimSpace(attributeType) { + case "text", "link", "list", "": + return "LOWER(" + expression + ")" + default: + return expression + } +} + +func contactFilterNormalizeValues(attribute string, values []string) []string { + normalized := make([]string, 0, len(values)) + for _, value := range values { + switch attribute { + case "phone_number": + trimmed := strings.TrimPrefix(value, "+") + if trimmed != "" { + normalized = append(normalized, "+"+trimmed) + } + case "name", "email", "identifier", "country_code", "city", "company_name": + normalized = append(normalized, strings.ToLower(value)) + default: + normalized = append(normalized, value) + } + } + return normalized +} + +func contactFilterCoerceValue(attribute string, value string) any { + if attribute != "blocked" { + return value + } + return value == "true" || value == "1" +} + +func contactFilterOperatorAllowed(operator string, allowed []string) bool { + for _, value := range allowed { + if operator == value { + return true + } + } + return false +} + +func contactFilterIsQueryOperator(value string) bool { + return value == "AND" || value == "OR" +} + +func contactFilterStringValues(values []any) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + text := strings.TrimSpace(fmt.Sprintf("%v", value)) + if text != "" { + result = append(result, text) + } + } + return result +}