This commit is contained in:
+170
-14
@@ -30,7 +30,7 @@ var (
|
||||
)
|
||||
|
||||
const (
|
||||
accountSchemaVersion = "1"
|
||||
accountSchemaVersion = "2"
|
||||
catalogSchemaVersion = "1"
|
||||
)
|
||||
|
||||
@@ -94,17 +94,27 @@ type MessageRecord struct {
|
||||
PayloadHash string
|
||||
}
|
||||
|
||||
type ContactRecord struct {
|
||||
ChatID string
|
||||
ChatType string
|
||||
DisplayName string
|
||||
Remark string
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
type IngestBatch struct {
|
||||
BatchID string
|
||||
SourceGeneration string
|
||||
StreamKey string
|
||||
Sequence int64
|
||||
CursorStart string
|
||||
CursorEnd string
|
||||
PayloadHash string
|
||||
CoverageState string
|
||||
Conversations []ConversationRecord
|
||||
Messages []MessageRecord
|
||||
BatchID string
|
||||
SourceGeneration string
|
||||
StreamKey string
|
||||
Sequence int64
|
||||
CursorStart string
|
||||
CursorEnd string
|
||||
PayloadHash string
|
||||
CoverageState string
|
||||
Conversations []ConversationRecord
|
||||
Messages []MessageRecord
|
||||
Contacts []ContactRecord
|
||||
ContactsSnapshotID string
|
||||
}
|
||||
|
||||
type BatchApplyResult struct {
|
||||
@@ -662,6 +672,25 @@ func (s *AccountStore) ApplyBatch(ctx context.Context, batch IngestBatch) (Batch
|
||||
return rollback(fmt.Errorf("write message: %w", err))
|
||||
}
|
||||
}
|
||||
for _, contact := range batch.Contacts {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO contacts (chat_id, chat_type, display_name, remark, observed_at, snapshot_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(chat_id) DO UPDATE SET
|
||||
chat_type = excluded.chat_type,
|
||||
display_name = excluded.display_name,
|
||||
remark = excluded.remark,
|
||||
observed_at = excluded.observed_at,
|
||||
snapshot_id = excluded.snapshot_id`, contact.ChatID, contact.ChatType,
|
||||
contact.DisplayName, contact.Remark, formatTime(ptrTime(contact.ObservedAt)), batch.ContactsSnapshotID); err != nil {
|
||||
return rollback(fmt.Errorf("write contact: %w", err))
|
||||
}
|
||||
}
|
||||
if batch.StreamKey == "contacts" && batch.CoverageState == "complete" {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM contacts WHERE snapshot_id <> ?`, batch.ContactsSnapshotID); err != nil {
|
||||
return rollback(fmt.Errorf("replace contacts snapshot: %w", err))
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO sync_state (stream_key, source_generation, confirmed_sequence, confirmed_cursor, coverage_state, last_success_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
@@ -798,6 +827,78 @@ func (s *AccountStore) QueryConversations(ctx context.Context, limit int, cursor
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (s *AccountStore) QueryContacts(ctx context.Context, contains string, groupsOnly bool, limit, offset int) ([]ContactRecord, bool, error) {
|
||||
if limit < 1 || limit > 200 || offset < 0 || offset > 1_000_000 {
|
||||
return nil, false, errors.New("invalid contacts pagination")
|
||||
}
|
||||
scopes, _, err := s.manager.activeScopes(ctx, s.accountID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
allChats := slices.ContainsFunc(scopes, func(scope ReportingScope) bool {
|
||||
return scope.ChatID == "*" && scopeAllows(scopes, "*", "contacts")
|
||||
})
|
||||
allowedChats := make([]string, 0, len(scopes))
|
||||
seenChats := make(map[string]struct{}, len(scopes))
|
||||
for _, scope := range scopes {
|
||||
if scope.ChatID == "*" || !scopeAllows(scopes, scope.ChatID, "contacts") {
|
||||
continue
|
||||
}
|
||||
if _, exists := seenChats[scope.ChatID]; exists {
|
||||
continue
|
||||
}
|
||||
seenChats[scope.ChatID] = struct{}{}
|
||||
allowedChats = append(allowedChats, scope.ChatID)
|
||||
}
|
||||
if !allChats && len(allowedChats) == 0 {
|
||||
return nil, false, ErrAccountNotAuthorized
|
||||
}
|
||||
|
||||
query := `SELECT chat_id, chat_type, display_name, remark, observed_at FROM contacts`
|
||||
conditions := make([]string, 0, 3)
|
||||
args := make([]any, 0, len(allowedChats)+4)
|
||||
if !allChats {
|
||||
conditions = append(conditions, `chat_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(allowedChats)), ",")+")")
|
||||
for _, chatID := range allowedChats {
|
||||
args = append(args, chatID)
|
||||
}
|
||||
}
|
||||
if groupsOnly {
|
||||
conditions = append(conditions, `chat_type = 'Group'`)
|
||||
} else {
|
||||
conditions = append(conditions, `chat_type = 'Private'`)
|
||||
}
|
||||
if contains != "" {
|
||||
conditions = append(conditions, `instr(lower(chat_id || ' ' || display_name || ' ' || remark), lower(?)) > 0`)
|
||||
args = append(args, contains)
|
||||
}
|
||||
query += ` WHERE ` + strings.Join(conditions, ` AND `) + ` ORDER BY lower(CASE WHEN remark <> '' THEN remark WHEN display_name <> '' THEN display_name ELSE chat_id END), chat_id LIMIT ? OFFSET ?`
|
||||
args = append(args, limit+1, offset)
|
||||
rows, err := s.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("query contacts: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var records []ContactRecord
|
||||
for rows.Next() {
|
||||
var item ContactRecord
|
||||
var observed string
|
||||
if err := rows.Scan(&item.ChatID, &item.ChatType, &item.DisplayName, &item.Remark, &observed); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
item.ObservedAt = parseTime(observed)
|
||||
records = append(records, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hasMore := len(records) > limit
|
||||
if hasMore {
|
||||
records = records[:limit]
|
||||
}
|
||||
return records, hasMore, nil
|
||||
}
|
||||
|
||||
type AccountSyncStatus struct {
|
||||
StreamKey string
|
||||
SourceGeneration string
|
||||
@@ -932,6 +1033,31 @@ func validateAccountSchema(ctx context.Context, db *sql.DB) error {
|
||||
if err := db.QueryRowContext(ctx, `SELECT value FROM schema_meta WHERE key = 'schema_version'`).Scan(&version); err != nil {
|
||||
return fmt.Errorf("read account schema version: %w", err)
|
||||
}
|
||||
if version == "1" {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`CREATE TABLE IF NOT EXISTS contacts (
|
||||
chat_id TEXT PRIMARY KEY,
|
||||
chat_type TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
remark TEXT NOT NULL,
|
||||
observed_at TEXT NOT NULL,
|
||||
snapshot_id TEXT NOT NULL
|
||||
)`); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return fmt.Errorf("migrate account contacts schema: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`UPDATE schema_meta SET value = ? WHERE key = 'schema_version'`, accountSchemaVersion); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return fmt.Errorf("update account schema version: %w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
version = accountSchemaVersion
|
||||
}
|
||||
if version != accountSchemaVersion {
|
||||
return fmt.Errorf("unsupported account schema version %q", version)
|
||||
}
|
||||
@@ -984,7 +1110,15 @@ func createAccountShard(path string) error {
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_time ON messages(chat_id, source_time DESC, message_id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_observed_at ON messages(observed_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_conversations_activity ON conversations(last_activity_at DESC, observed_at DESC, chat_id);
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
CREATE TABLE IF NOT EXISTS contacts (
|
||||
chat_id TEXT PRIMARY KEY,
|
||||
chat_type TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
remark TEXT NOT NULL,
|
||||
observed_at TEXT NOT NULL,
|
||||
snapshot_id TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
stream_key TEXT PRIMARY KEY,
|
||||
source_generation TEXT NOT NULL,
|
||||
confirmed_sequence INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -1007,7 +1141,7 @@ func createAccountShard(path string) error {
|
||||
received_at TEXT NOT NULL,
|
||||
confirmed_at TEXT
|
||||
);
|
||||
INSERT INTO schema_meta(key, value) VALUES ('schema_version', '1')
|
||||
INSERT INTO schema_meta(key, value) VALUES ('schema_version', '2')
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value;
|
||||
`)
|
||||
if err != nil {
|
||||
@@ -1075,6 +1209,15 @@ func validateBatch(batch IngestBatch) error {
|
||||
if strings.TrimSpace(batch.BatchID) == "" || strings.TrimSpace(batch.SourceGeneration) == "" || strings.TrimSpace(batch.StreamKey) == "" || batch.Sequence < 1 || strings.TrimSpace(batch.PayloadHash) == "" {
|
||||
return errors.New("batch id, source generation, stream key, positive sequence, and payload hash are required")
|
||||
}
|
||||
if batch.StreamKey != "messages" && batch.StreamKey != "contacts" {
|
||||
return errors.New("unsupported data stream")
|
||||
}
|
||||
if batch.StreamKey == "contacts" && (strings.TrimSpace(batch.ContactsSnapshotID) == "" || len(batch.Conversations) > 0 || len(batch.Messages) > 0) {
|
||||
return errors.New("contacts batches require a snapshot id and cannot contain messages or conversations")
|
||||
}
|
||||
if batch.StreamKey == "messages" && (len(batch.Contacts) > 0 || batch.ContactsSnapshotID != "") {
|
||||
return errors.New("message batches cannot contain contacts")
|
||||
}
|
||||
for _, conversation := range batch.Conversations {
|
||||
if strings.TrimSpace(conversation.ChatID) == "" || strings.TrimSpace(conversation.Source) == "" || conversation.ObservedAt.IsZero() {
|
||||
return errors.New("conversation identity and observation time are required")
|
||||
@@ -1085,11 +1228,16 @@ func validateBatch(batch IngestBatch) error {
|
||||
return errors.New("message identity, payload hash, source time, and observation time are required")
|
||||
}
|
||||
}
|
||||
for _, contact := range batch.Contacts {
|
||||
if strings.TrimSpace(contact.ChatID) == "" || len(contact.ChatID) > 512 || len(contact.DisplayName) > 2048 || len(contact.Remark) > 2048 || (contact.ChatType != string(ChatPrivate) && contact.ChatType != string(ChatGroup)) || contact.ObservedAt.IsZero() {
|
||||
return errors.New("contact identity, type, and observation time are required")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func authorizeBatch(batch IngestBatch, scopes []ReportingScope) error {
|
||||
if len(scopes) == 0 && (len(batch.Conversations) > 0 || len(batch.Messages) > 0) {
|
||||
if len(scopes) == 0 && (len(batch.Conversations) > 0 || len(batch.Messages) > 0 || len(batch.Contacts) > 0) {
|
||||
return ErrAccountNotAuthorized
|
||||
}
|
||||
for _, conversation := range batch.Conversations {
|
||||
@@ -1102,6 +1250,11 @@ func authorizeBatch(batch IngestBatch, scopes []ReportingScope) error {
|
||||
return ErrAccountNotAuthorized
|
||||
}
|
||||
}
|
||||
for _, contact := range batch.Contacts {
|
||||
if !scopeAllows(scopes, contact.ChatID, "contacts") {
|
||||
return ErrAccountNotAuthorized
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1176,6 +1329,9 @@ func estimateBatchBytes(batch IngestBatch) int64 {
|
||||
for _, item := range batch.Messages {
|
||||
total += int64(len(item.MessageID) + len(item.ChatID) + len(item.SourceMessageID) + len(item.Direction) + len(item.MessageType) + len(item.Text) + len(item.SourceVersion) + len(item.PayloadHash) + 192)
|
||||
}
|
||||
for _, item := range batch.Contacts {
|
||||
total += int64(len(item.ChatID) + len(item.ChatType) + len(item.DisplayName) + len(item.Remark) + 128)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestContactSnapshotsReplaceOnlyAfterCompleteSync(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
manager, err := OpenAccountStoreManager(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer manager.Close()
|
||||
store, err := manager.RegisterAccount(ctx, AccountRegistration{
|
||||
AccountID: "account-contacts", StableIdentity: "wechat-contacts", SourceNodeID: "node-contacts",
|
||||
SourceGeneration: "generation-contacts", Verified: true, AuthorizationVersion: 1,
|
||||
ReportingScopes: []ReportingScope{{ChatID: "*", DataType: "*", ConfigVersion: 1}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = store.Close() }()
|
||||
|
||||
// Simulate an existing v1 shard so opening it exercises the contacts-table migration.
|
||||
if _, err := store.db.ExecContext(ctx, `DROP TABLE contacts`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE schema_meta SET value = '1' WHERE key = 'schema_version'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = manager.OpenAccount(ctx, "account-contacts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
private := ContactRecord{ChatID: "wxid-contact", ChatType: string(ChatPrivate), DisplayName: "Contact", Remark: "Local", ObservedAt: now}
|
||||
group := ContactRecord{ChatID: "group@chatroom", ChatType: string(ChatGroup), DisplayName: "Group", ObservedAt: now}
|
||||
apply := func(sequence int64, snapshotID, coverage string, contacts ...ContactRecord) error {
|
||||
_, err := store.ApplyBatch(ctx, IngestBatch{
|
||||
BatchID: "contacts-" + snapshotID, SourceGeneration: "generation-contacts", StreamKey: "contacts",
|
||||
Sequence: sequence, CursorStart: "0", CursorEnd: "0", PayloadHash: "hash-" + snapshotID,
|
||||
CoverageState: coverage, ContactsSnapshotID: snapshotID, Contacts: contacts,
|
||||
})
|
||||
return err
|
||||
}
|
||||
if err := apply(1, "snapshot-1", "complete", private, group); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if items, _, err := store.QueryContacts(ctx, "", true, 10, 0); err != nil || len(items) != 1 || items[0].ChatID != group.ChatID {
|
||||
t.Fatalf("unexpected cached groups: %v %+v", err, items)
|
||||
}
|
||||
if err := apply(2, "snapshot-2", "partial", private); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if items, _, err := store.QueryContacts(ctx, "", true, 10, 0); err != nil || len(items) != 1 {
|
||||
t.Fatalf("partial snapshot removed prior contacts: %v %+v", err, items)
|
||||
}
|
||||
if err := apply(3, "snapshot-3", "complete", private); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if items, _, err := store.QueryContacts(ctx, "", true, 10, 0); err != nil || len(items) != 0 {
|
||||
t.Fatalf("complete snapshot did not remove stale contacts: %v %+v", err, items)
|
||||
}
|
||||
}
|
||||
+118
-16
@@ -6,24 +6,27 @@ import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maxDataBatchBytes = 512 * 1024
|
||||
|
||||
type dataBatchRequest struct {
|
||||
NodeID string `json:"node_id"`
|
||||
AccountID string `json:"account_id"`
|
||||
BatchID string `json:"batch_id"`
|
||||
SourceGeneration string `json:"source_generation"`
|
||||
StreamKey string `json:"stream_key"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
CursorStart string `json:"cursor_start"`
|
||||
CursorEnd string `json:"cursor_end"`
|
||||
PayloadHash string `json:"payload_hash"`
|
||||
CoverageState string `json:"coverage_state"`
|
||||
Conversations []dataConversationRequest `json:"conversations"`
|
||||
Messages []dataMessageRequest `json:"messages"`
|
||||
NodeID string `json:"node_id"`
|
||||
AccountID string `json:"account_id"`
|
||||
BatchID string `json:"batch_id"`
|
||||
SourceGeneration string `json:"source_generation"`
|
||||
StreamKey string `json:"stream_key"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
CursorStart string `json:"cursor_start"`
|
||||
CursorEnd string `json:"cursor_end"`
|
||||
PayloadHash string `json:"payload_hash"`
|
||||
CoverageState string `json:"coverage_state"`
|
||||
Conversations []dataConversationRequest `json:"conversations"`
|
||||
Messages []dataMessageRequest `json:"messages"`
|
||||
Contacts []dataContactRequest `json:"contacts"`
|
||||
ContactsSnapshotID string `json:"contacts_snapshot_id"`
|
||||
}
|
||||
|
||||
type dataConversationRequest struct {
|
||||
@@ -50,6 +53,14 @@ type dataMessageRequest struct {
|
||||
PayloadHash string `json:"payload_hash"`
|
||||
}
|
||||
|
||||
type dataContactRequest struct {
|
||||
ChatID string `json:"chat_id"`
|
||||
ChatType ChatType `json:"chat_type"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Remark string `json:"remark"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
}
|
||||
|
||||
type dataBatchAck struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
@@ -80,6 +91,14 @@ type dataConversationView struct {
|
||||
DirectoryState string `json:"directory_state"`
|
||||
}
|
||||
|
||||
type dataContactView struct {
|
||||
ChatID string `json:"id"`
|
||||
ChatType string `json:"type"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Remark string `json:"remark"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
}
|
||||
|
||||
type dataMessageView struct {
|
||||
MessageID string `json:"message_id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
@@ -166,7 +185,14 @@ func (s *Server) nodeDataRoute(w http.ResponseWriter, r *http.Request, nodeID st
|
||||
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
||||
}
|
||||
defer store.Close()
|
||||
status, err := store.GetSyncStatus(r.Context(), "messages")
|
||||
streamKey := r.URL.Query().Get("stream_key")
|
||||
if streamKey == "" {
|
||||
streamKey = "messages"
|
||||
}
|
||||
if streamKey != "messages" && streamKey != "contacts" {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidStream", message: "The sync stream is invalid."}
|
||||
}
|
||||
status, err := store.GetSyncStatus(r.Context(), streamKey)
|
||||
if err != nil {
|
||||
return requestError{status: http.StatusInternalServerError, code: "DataStatusFailed", message: "The platform sync status query failed."}
|
||||
}
|
||||
@@ -183,7 +209,7 @@ func (s *Server) ingestDataBatch(w http.ResponseWriter, r *http.Request, correla
|
||||
if err := decodeJSON(r, &request, maxDataBatchBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
if request.NodeID != nodeID || request.AccountID == "" || request.BatchID == "" || request.SourceGeneration == "" || request.StreamKey != "messages" || request.Sequence < 1 || request.PayloadHash == "" || len(request.Messages) > 5000 || len(request.Conversations) > 1000 {
|
||||
if request.NodeID != nodeID || request.AccountID == "" || request.BatchID == "" || request.SourceGeneration == "" || (request.StreamKey != "messages" && request.StreamKey != "contacts") || request.Sequence < 1 || request.PayloadHash == "" || len(request.Messages) > 5000 || len(request.Conversations) > 1000 || len(request.Contacts) > 1000 || request.StreamKey == "contacts" && (request.ContactsSnapshotID == "" || len(request.Messages) > 0 || len(request.Conversations) > 0) || request.StreamKey == "messages" && (len(request.Contacts) > 0 || request.ContactsSnapshotID != "") {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidDataBatch", message: "The data batch is invalid."}
|
||||
}
|
||||
var account AccountSummary
|
||||
@@ -214,6 +240,7 @@ func (s *Server) ingestDataBatch(w http.ResponseWriter, r *http.Request, correla
|
||||
BatchID: request.BatchID, SourceGeneration: request.SourceGeneration, StreamKey: request.StreamKey,
|
||||
Sequence: request.Sequence, CursorStart: request.CursorStart, CursorEnd: request.CursorEnd,
|
||||
PayloadHash: request.PayloadHash, CoverageState: request.CoverageState,
|
||||
ContactsSnapshotID: request.ContactsSnapshotID,
|
||||
}
|
||||
for _, conversation := range request.Conversations {
|
||||
if !validChatType(conversation.ChatType) || conversation.ObservedAt.IsZero() {
|
||||
@@ -236,6 +263,15 @@ func (s *Server) ingestDataBatch(w http.ResponseWriter, r *http.Request, correla
|
||||
PayloadHash: message.PayloadHash,
|
||||
})
|
||||
}
|
||||
for _, contact := range request.Contacts {
|
||||
if !validChatType(contact.ChatType) || strings.TrimSpace(contact.ChatID) == "" || len(contact.ChatID) > 512 || len(contact.DisplayName) > 2048 || len(contact.Remark) > 2048 || contact.ObservedAt.IsZero() {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidDataBatch", message: "A contact record is invalid."}
|
||||
}
|
||||
batch.Contacts = append(batch.Contacts, ContactRecord{
|
||||
ChatID: contact.ChatID, ChatType: string(contact.ChatType), DisplayName: contact.DisplayName,
|
||||
Remark: contact.Remark, ObservedAt: contact.ObservedAt,
|
||||
})
|
||||
}
|
||||
result, err := store.ApplyBatch(r.Context(), batch)
|
||||
if err != nil {
|
||||
switch {
|
||||
@@ -282,6 +318,11 @@ func (s *Server) dataRoute(w http.ResponseWriter, r *http.Request, correlationID
|
||||
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
||||
}
|
||||
return s.listStoredConversations(w, r, accountID)
|
||||
case "contacts":
|
||||
if r.Method != http.MethodGet {
|
||||
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
||||
}
|
||||
return s.listStoredContacts(w, r, accountID)
|
||||
case "messages":
|
||||
if r.Method != http.MethodGet {
|
||||
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
||||
@@ -354,6 +395,52 @@ func (s *Server) listStoredConversations(w http.ResponseWriter, r *http.Request,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listStoredContacts(w http.ResponseWriter, r *http.Request, accountID string) error {
|
||||
limit := queryLimit(r.URL.Query().Get("limit"))
|
||||
offset := 0
|
||||
if raw := r.URL.Query().Get("offset"); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil || parsed < 0 || parsed > 1_000_000 {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidOffset", message: "The contact offset is invalid."}
|
||||
}
|
||||
offset = parsed
|
||||
}
|
||||
groupsOnly := false
|
||||
if raw := r.URL.Query().Get("groups_only"); raw != "" {
|
||||
parsed, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidGroupFilter", message: "groups_only must be a boolean."}
|
||||
}
|
||||
groupsOnly = parsed
|
||||
}
|
||||
contains := r.URL.Query().Get("contains")
|
||||
if len(contains) > 200 {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidQuery", message: "The contact query is too long."}
|
||||
}
|
||||
store, err := s.accountStores.OpenAccount(r.Context(), accountID)
|
||||
if err != nil {
|
||||
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
||||
}
|
||||
defer store.Close()
|
||||
items, hasMore, err := store.QueryContacts(r.Context(), contains, groupsOnly, limit, offset)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAccountNotAuthorized) {
|
||||
return requestError{status: http.StatusForbidden, code: "DataNotAuthorized", message: "The account has no active contact reporting scope."}
|
||||
}
|
||||
return requestError{status: http.StatusInternalServerError, code: "DataQueryFailed", message: "The platform contact query failed."}
|
||||
}
|
||||
status, err := store.GetSyncStatus(r.Context(), "contacts")
|
||||
if err != nil {
|
||||
return requestError{status: http.StatusInternalServerError, code: "DataStatusFailed", message: "The platform sync status query failed."}
|
||||
}
|
||||
views := make([]dataContactView, 0, len(items))
|
||||
for _, item := range items {
|
||||
views = append(views, dataContactView{item.ChatID, item.ChatType, item.DisplayName, item.Remark, item.ObservedAt})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"items": views, "limit": limit, "offset": offset, "has_more": hasMore, "sync": syncView(status)})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listStoredMessages(w http.ResponseWriter, r *http.Request, accountID string) error {
|
||||
chatID := r.URL.Query().Get("chat_id")
|
||||
if !validIdentifier(chatID, 512) {
|
||||
@@ -435,6 +522,21 @@ func (s *Server) revokeDataAuthorization(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
|
||||
func (s *Server) requestDataRefresh(w http.ResponseWriter, r *http.Request, accountID, username, correlationID string) error {
|
||||
streamKey := "messages"
|
||||
if r.ContentLength > 0 || strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
var request struct {
|
||||
StreamKey string `json:"stream_key"`
|
||||
}
|
||||
if err := decodeJSON(r, &request, 8*1024); err != nil {
|
||||
return err
|
||||
}
|
||||
if request.StreamKey != "" {
|
||||
streamKey = request.StreamKey
|
||||
}
|
||||
}
|
||||
if streamKey != "messages" && streamKey != "contacts" {
|
||||
return requestError{status: http.StatusBadRequest, code: "InvalidStream", message: "The sync stream is invalid."}
|
||||
}
|
||||
var nodeID string
|
||||
if err := s.store.Read(func(state PersistedState) error {
|
||||
for id, node := range state.Nodes {
|
||||
@@ -455,6 +557,6 @@ func (s *Server) requestDataRefresh(w http.ResponseWriter, r *http.Request, acco
|
||||
if nodeID == "" {
|
||||
return requestError{status: http.StatusConflict, code: "NoAuthorizedNode", message: "No verified node is available for this account."}
|
||||
}
|
||||
payload := json.RawMessage(`{"stream_key":"messages","reason":"web-refresh"}`)
|
||||
return s.createTaskSubmission(w, TaskSubmission{NodeID: nodeID, AccountID: accountID, Kind: "sync-data", IdempotencyKey: "sync:" + accountID + ":" + strconv.FormatInt(time.Now().UTC().Unix()/5, 10), Payload: payload}, username, correlationID)
|
||||
payload, _ := json.Marshal(map[string]string{"stream_key": streamKey, "reason": "web-refresh"})
|
||||
return s.createTaskSubmission(w, TaskSubmission{NodeID: nodeID, AccountID: accountID, Kind: "sync-data", IdempotencyKey: "sync:" + accountID + ":" + streamKey + ":" + strconv.FormatInt(time.Now().UTC().Unix()/5, 10), Payload: payload}, username, correlationID)
|
||||
}
|
||||
|
||||
@@ -92,7 +92,29 @@ func TestDataBatchRoundTripAndWebQueriesUseAccountShard(t *testing.T) {
|
||||
t.Fatalf("unexpected message body: %+v", messageBody)
|
||||
}
|
||||
|
||||
refresh := doJSON(t, client, http.MethodPost, httpServer.URL+"/v1/data/accounts/account-a/refresh", webAuth, nil)
|
||||
contactBatch := dataBatchRequest{
|
||||
NodeID: "node-a", AccountID: "account-a", BatchID: "contacts-1", SourceGeneration: "account-a", StreamKey: "contacts", Sequence: 1,
|
||||
CursorStart: "{}", CursorEnd: "{}", PayloadHash: "contacts-hash", CoverageState: "complete", ContactsSnapshotID: "snapshot-1",
|
||||
Contacts: []dataContactRequest{{ChatID: "chat-a", ChatType: ChatPrivate, DisplayName: "测试联系人", Remark: "本地备注", ObservedAt: now}},
|
||||
}
|
||||
contactPost := doJSON(t, client, http.MethodPost, httpServer.URL+"/v1/data/batches", "Bearer secret-a", contactBatch)
|
||||
if contactPost.Code != http.StatusOK {
|
||||
t.Fatalf("contact batch status = %d: %s", contactPost.Code, contactPost.Body.String())
|
||||
}
|
||||
contacts := doJSON(t, client, http.MethodGet, httpServer.URL+"/v1/data/accounts/account-a/contacts?groups_only=false&limit=20", webAuth, nil)
|
||||
if contacts.Code != http.StatusOK {
|
||||
t.Fatalf("contact query status = %d: %s", contacts.Code, contacts.Body.String())
|
||||
}
|
||||
var contactBody struct {
|
||||
Items []dataContactView `json:"items"`
|
||||
Sync dataSyncView `json:"sync"`
|
||||
}
|
||||
decodeBody(t, contacts, &contactBody)
|
||||
if len(contactBody.Items) != 1 || contactBody.Items[0].ChatID != "chat-a" || contactBody.Sync.State != "complete" {
|
||||
t.Fatalf("unexpected contact body: %+v", contactBody)
|
||||
}
|
||||
|
||||
refresh := doJSON(t, client, http.MethodPost, httpServer.URL+"/v1/data/accounts/account-a/refresh", webAuth, map[string]string{"stream_key": "contacts"})
|
||||
if refresh.Code != http.StatusAccepted {
|
||||
t.Fatalf("refresh status = %d: %s", refresh.Code, refresh.Body.String())
|
||||
}
|
||||
|
||||
@@ -981,7 +981,7 @@ func validTaskPayload(kind string, payload jsonRaw) bool {
|
||||
StreamKey string `json:"stream_key"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
return decodeRaw(payload, &value) && value.StreamKey == "messages" && (value.Reason == "" || validIdentifier(value.Reason, 80))
|
||||
return decodeRaw(payload, &value) && (value.StreamKey == "messages" || value.StreamKey == "contacts") && (value.Reason == "" || validIdentifier(value.Reason, 80))
|
||||
}
|
||||
if !isReadTaskKind(kind) || len(payload) == 0 || len(payload) > 64*1024 || !json.Valid(payload) {
|
||||
return false
|
||||
|
||||
+40
File diff suppressed because one or more lines are too long
-40
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -6,7 +6,7 @@
|
||||
<meta name="theme-color" content="#f6f7fb" />
|
||||
<meta name="description" content="WxAgent 普通用户工作台" />
|
||||
<title>WxAgent 工作台</title>
|
||||
<script type="module" crossorigin src="/assets/index-PxzqqEA7.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-BS_ZFWFu.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-6ql7JGil.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -476,25 +476,47 @@ function ContactsView({ token, client, onSettings }) {
|
||||
const [groupsOnly, setGroupsOnly] = useState(false);
|
||||
const [query, setQuery] = useState("");
|
||||
const [contacts, setContacts] = useState([]);
|
||||
const [syncInfo, setSyncInfo] = useState(null);
|
||||
const [hasMore, setHasMore] = useState(false);
|
||||
const [syncing, setSyncing] = useState(false);
|
||||
const [syncNotice, setSyncNotice] = useState("");
|
||||
const [state, setState] = useState({ loading: false, error: "" });
|
||||
const requestRef = useRef(0);
|
||||
const load = useCallback(async () => {
|
||||
const load = useCallback(async (append = false) => {
|
||||
if (!clientId || !accountId) return;
|
||||
const requestId = ++requestRef.current;
|
||||
setState({ loading: true, error: "" });
|
||||
try {
|
||||
const content = await readWithTask("/v1/reads/contacts", { node_id: clientId, account_id: accountId, limit: 200, groups_only: groupsOnly, contains: query }, token);
|
||||
const params = new URLSearchParams({ limit: "200", offset: String(append ? contacts.length : 0), groups_only: String(groupsOnly) });
|
||||
if (query.trim()) params.set("contains", query.trim());
|
||||
const content = await api(dataPath(accountId, "contacts", `?${params.toString()}`), { token });
|
||||
if (requestId !== requestRef.current) return;
|
||||
setContacts(normalizeContacts(content));
|
||||
const nextContacts = normalizeContacts(content);
|
||||
setContacts((current) => append ? [...current, ...nextContacts] : nextContacts);
|
||||
setHasMore(Boolean(content.has_more));
|
||||
setSyncInfo(content.sync || null);
|
||||
setState({ loading: false, error: "" });
|
||||
} catch (reason) {
|
||||
if (requestId === requestRef.current) setState({ loading: false, error: reason.message });
|
||||
}
|
||||
}, [accountId, clientId, groupsOnly, query, token]);
|
||||
useEffect(() => { setContacts([]); if (clientId && accountId) load(); }, [clientId, accountId, groupsOnly]);
|
||||
}, [accountId, clientId, contacts.length, groupsOnly, query, token]);
|
||||
const syncContacts = async () => {
|
||||
if (!clientId || !accountId || syncing) return;
|
||||
setSyncing(true);
|
||||
setSyncNotice("");
|
||||
try {
|
||||
await api(dataPath(accountId, "refresh"), { token, method: "POST", body: { stream_key: "contacts" } });
|
||||
setSyncNotice("通讯录同步任务已提交;完成后点击“刷新缓存”查看最新数据。");
|
||||
} catch (reason) {
|
||||
setState((current) => ({ ...current, error: reason.message }));
|
||||
} finally {
|
||||
setSyncing(false);
|
||||
}
|
||||
};
|
||||
useEffect(() => { setContacts([]); setHasMore(false); setSyncInfo(null); setSyncNotice(""); if (clientId && accountId) load(); }, [clientId, accountId, groupsOnly]);
|
||||
if (!client) return <ContextEmpty onSettings={onSettings} />;
|
||||
const visible = contacts.filter((contact) => !query || `${contact.title} ${contact.id} ${contact.detail}`.toLowerCase().includes(query.toLowerCase()));
|
||||
return <section className="workspace-panel"><div className="section-heading"><div><h2>通讯录</h2><p>只读查看当前 Client 已授权的联系人和群聊。</p></div><div className="heading-actions"><label className="check-filter"><input type="checkbox" checked={groupsOnly} onChange={(event) => setGroupsOnly(event.target.checked)} />只看群聊</label><button className="secondary-button" disabled title="添加好友尚未通过真机验收">添加好友</button></div></div><div className="toolbar"><label className="search-box wide"><Icon name="search" size={16} /><input value={query} onChange={(event) => setQuery(event.target.value)} onKeyDown={(event) => event.key === "Enter" && load()} placeholder="搜索联系人或群聊" /></label><button className="secondary-button" onClick={load} disabled={state.loading}>{state.loading ? "读取中…" : "刷新"}</button></div>{state.loading ? <LoadingState text="正在读取通讯录…" /> : state.error ? <ReadError error={state.error} onRetry={load} /> : visible.length === 0 ? <Empty text="暂无可见联系人或群聊。" /> : <div className="contact-list">{visible.map((contact) => <div className="contact-row" key={contact.id}><span className="contact-avatar">{contact.title.slice(0, 1).toUpperCase()}</span><div><strong>{contact.title}</strong><small>{contact.type === "Group" ? "群聊" : "联系人"} · {shortId(contact.id, 28)}{contact.detail ? ` · ${contact.detail}` : ""}</small></div><button className="text-button" disabled title="写操作尚未通过真机验收">添加好友</button></div>)}</div>}</section>;
|
||||
return <section className="workspace-panel"><div className="section-heading"><div><h2>通讯录</h2><p>平台缓存 · {syncInfo?.state || "尚未同步"} · 最近同步 {formatTime(syncInfo?.last_success_at)}</p></div><div className="heading-actions"><label className="check-filter"><input type="checkbox" checked={groupsOnly} onChange={(event) => setGroupsOnly(event.target.checked)} />只看群聊</label><button className="secondary-button" onClick={syncContacts} disabled={syncing || state.loading}>{syncing ? "提交中…" : "同步通讯录"}</button><button className="secondary-button" disabled title="添加好友尚未通过真机验收">添加好友</button></div></div><div className="toolbar"><label className="search-box wide"><Icon name="search" size={16} /><input value={query} onChange={(event) => setQuery(event.target.value)} onKeyDown={(event) => event.key === "Enter" && load()} placeholder="搜索联系人或群聊" /></label><button className="secondary-button" onClick={load} disabled={state.loading}>{state.loading ? "读取中…" : "刷新缓存"}</button><button className="secondary-button" onClick={() => load(true)} disabled={state.loading || !hasMore}>加载更多</button>{syncNotice && <span className="read-only-note" role="status">{syncNotice}</span>}</div>{state.loading ? <LoadingState text="正在读取通讯录…" /> : state.error ? <ReadError error={state.error} onRetry={load} /> : visible.length === 0 ? <Empty text={syncInfo?.state === "unknown" ? "平台尚未缓存通讯录,请点击“同步通讯录”。" : "暂无可见联系人或群聊。"} /> : <div className="contact-list">{visible.map((contact) => <div className="contact-row" key={contact.id}><span className="contact-avatar">{contact.title.slice(0, 1).toUpperCase()}</span><div><strong>{contact.title}</strong><small>{contact.type === "Group" ? "群聊" : "联系人"} · {shortId(contact.id, 28)}{contact.detail ? ` · ${contact.detail}` : ""}</small></div><button className="text-button" disabled title="写操作尚未通过真机验收">添加好友</button></div>)}</div>}</section>;
|
||||
}
|
||||
|
||||
function TasksView({ tasks, selectedClientId }) {
|
||||
|
||||
@@ -168,12 +168,15 @@ public sealed class RemoteControlClient : IDisposable
|
||||
|
||||
public Task<RemoteDataSyncStatus> GetDataSyncStatusAsync(
|
||||
string accountId,
|
||||
CancellationToken cancellationToken = default)
|
||||
CancellationToken cancellationToken = default,
|
||||
string streamKey = "messages")
|
||||
{
|
||||
RemoteAgentOptions.ValidateIdentifier(accountId, "accountId", 200);
|
||||
if (streamKey is not ("messages" or "contacts"))
|
||||
throw new ArgumentOutOfRangeException(nameof(streamKey));
|
||||
return SendAuthenticatedAsync<RemoteDataSyncStatus>(
|
||||
HttpMethod.Get,
|
||||
$"/v1/nodes/{Escape(_options.NodeId!)}/data/accounts/{Escape(accountId)}/sync-status",
|
||||
$"/v1/nodes/{Escape(_options.NodeId!)}/data/accounts/{Escape(accountId)}/sync-status?stream_key={Escape(streamKey)}",
|
||||
null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
@@ -24,7 +24,21 @@ public sealed record RemoteSyncBatch(
|
||||
[property: JsonPropertyName("payload_hash")] string PayloadHash,
|
||||
[property: JsonPropertyName("coverage_state")] string CoverageState,
|
||||
[property: JsonPropertyName("conversations")] IReadOnlyList<RemoteSyncConversation> Conversations,
|
||||
[property: JsonPropertyName("messages")] IReadOnlyList<RemoteSyncMessage> Messages);
|
||||
[property: JsonPropertyName("messages")] IReadOnlyList<RemoteSyncMessage> Messages)
|
||||
{
|
||||
[JsonPropertyName("contacts")]
|
||||
public IReadOnlyList<RemoteSyncContact> Contacts { get; init; } = [];
|
||||
|
||||
[JsonPropertyName("contacts_snapshot_id")]
|
||||
public string? ContactsSnapshotId { get; init; }
|
||||
}
|
||||
|
||||
public sealed record RemoteSyncContact(
|
||||
[property: JsonPropertyName("chat_id")] string ChatId,
|
||||
[property: JsonPropertyName("chat_type")] ReportingChatType ChatType,
|
||||
[property: JsonPropertyName("display_name")] string DisplayName,
|
||||
[property: JsonPropertyName("remark")] string Remark,
|
||||
[property: JsonPropertyName("observed_at")] DateTimeOffset ObservedAt);
|
||||
|
||||
public sealed record RemoteSyncConversation(
|
||||
[property: JsonPropertyName("chat_id")] string ChatId,
|
||||
@@ -276,14 +290,19 @@ public static class RemoteDataBatchAuthorization
|
||||
ReportingAuthorization.Check(config, batch.AccountId, conversation.ChatId, conversation.ChatType, ReportingDataType.TaskResult).Allowed).ToArray();
|
||||
var messages = batch.Messages.Where(message =>
|
||||
ReportingAuthorization.Check(config, batch.AccountId, message.ChatId, message.ChatType, ReportingDataType.Message).Allowed).ToArray();
|
||||
var dropped = conversations.Length != batch.Conversations.Count || messages.Length != batch.Messages.Count;
|
||||
var contacts = batch.Contacts.Where(contact =>
|
||||
ReportingAuthorization.Check(config, batch.AccountId, contact.ChatId, contact.ChatType, ReportingDataType.TaskResult).Allowed).ToArray();
|
||||
var dropped = conversations.Length != batch.Conversations.Count || messages.Length != batch.Messages.Count || contacts.Length != batch.Contacts.Count;
|
||||
reason = dropped ? "SomeRecordsDroppedByReporting" : "Authorized";
|
||||
if (conversations.Length == 0 && messages.Length == 0 && (batch.Conversations.Count > 0 || batch.Messages.Count > 0))
|
||||
if (conversations.Length == 0 && messages.Length == 0 && contacts.Length == 0 && (batch.Conversations.Count > 0 || batch.Messages.Count > 0 || batch.Contacts.Count > 0))
|
||||
{
|
||||
reason = "AllRecordsDroppedByReporting";
|
||||
return null;
|
||||
}
|
||||
return dropped ? batch with { CoverageState = "partial", Conversations = conversations, Messages = messages } : batch;
|
||||
if (!dropped)
|
||||
return batch;
|
||||
var filteredBatch = batch with { CoverageState = "partial", Conversations = conversations, Messages = messages, Contacts = contacts };
|
||||
return filteredBatch with { PayloadHash = ComputePayloadHash(filteredBatch) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,7 +459,21 @@ public sealed class RemoteDataBatchQueue
|
||||
{
|
||||
foreach (var value in new[] { batch.NodeId, batch.AccountId, batch.BatchId, batch.SourceGeneration, batch.StreamKey, batch.PayloadHash })
|
||||
RemoteAgentOptions.ValidateIdentifier(value, "sync batch field", 512);
|
||||
if (batch.Sequence < 1 || batch.Conversations.Count + batch.Messages.Count > RemoteDataProtocol.MaxBatchMessages)
|
||||
if (batch.Sequence < 1 || batch.Conversations.Count + batch.Messages.Count > RemoteDataProtocol.MaxBatchMessages
|
||||
|| batch.Contacts.Count > RemoteDataProtocol.MaxBatchItems
|
||||
|| batch.StreamKey is not ("messages" or "contacts")
|
||||
|| batch.StreamKey == "contacts" && (string.IsNullOrWhiteSpace(batch.ContactsSnapshotId) || batch.Conversations.Count > 0 || batch.Messages.Count > 0)
|
||||
|| batch.StreamKey == "messages" && (batch.Contacts.Count > 0 || batch.ContactsSnapshotId is not null))
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The sync batch sequence or item count is invalid.");
|
||||
if (batch.ContactsSnapshotId is not null)
|
||||
RemoteAgentOptions.ValidateIdentifier(batch.ContactsSnapshotId, "contacts snapshot", 512);
|
||||
foreach (var contact in batch.Contacts)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(contact.ChatId) || contact.ChatId.Length > 512
|
||||
|| contact.DisplayName.Length > 2048 || contact.Remark.Length > 2048
|
||||
|| contact.ChatType is not (ReportingChatType.Group or ReportingChatType.Private)
|
||||
|| contact.ObservedAt == default)
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The contact record is invalid.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,7 +136,7 @@ public sealed class RemoteAgentHostedService(
|
||||
var tasks = await client.PollTasksAsync(accountId, stoppingToken, waitSeconds: 5);
|
||||
foreach (var task in tasks)
|
||||
{
|
||||
await ProcessTaskAsync(client, remote, reporting, ledger, task, stoppingToken);
|
||||
await ProcessTaskAsync(client, remote, reporting, ledger, task, dataQueue, syncState, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,40 +284,47 @@ public sealed class RemoteAgentHostedService(
|
||||
{
|
||||
foreach (var account in reporting.Accounts.Where(item => item.Enabled))
|
||||
{
|
||||
var local = state.GetLatest(account.AccountId, "messages");
|
||||
RemoteDataSyncStatus remote;
|
||||
try
|
||||
var revoked = false;
|
||||
foreach (var streamKey in new[] { "messages", "contacts" })
|
||||
{
|
||||
remote = await client.GetDataSyncStatusAsync(account.AccountId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (RemoteClientException exception) when (exception.StatusCode == 404 && exception.Code == "NotFound")
|
||||
{
|
||||
// Older control planes do not expose the reconciliation endpoint; retain the legacy ACK path.
|
||||
continue;
|
||||
}
|
||||
catch (RemoteClientException exception) when (exception.StatusCode == 403 && exception.Code == "AccountNotAuthorized")
|
||||
{
|
||||
var droppedOnRevoke = queue.DropAllForAccount(account.AccountId);
|
||||
blockedAccountIds.Add(account.AccountId);
|
||||
logger.LogWarning(
|
||||
"Data sync authorization is revoked; accountId={AccountId}; pending content was discarded and collection is paused; droppedPendingBatches={DroppedPendingBatches}.",
|
||||
account.AccountId, droppedOnRevoke);
|
||||
continue;
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
// A disconnected platform must not prevent local DB collection.
|
||||
return;
|
||||
}
|
||||
var local = state.GetLatest(account.AccountId, streamKey);
|
||||
RemoteDataSyncStatus remote;
|
||||
try
|
||||
{
|
||||
remote = await client.GetDataSyncStatusAsync(account.AccountId, cancellationToken, streamKey).ConfigureAwait(false);
|
||||
}
|
||||
catch (RemoteClientException exception) when (exception.StatusCode == 404 && exception.Code == "NotFound")
|
||||
{
|
||||
// Older control planes do not expose the reconciliation endpoint; retain the legacy ACK path.
|
||||
continue;
|
||||
}
|
||||
catch (RemoteClientException exception) when (exception.StatusCode == 403 && exception.Code == "AccountNotAuthorized")
|
||||
{
|
||||
var droppedOnRevoke = queue.DropAllForAccount(account.AccountId);
|
||||
blockedAccountIds.Add(account.AccountId);
|
||||
logger.LogWarning(
|
||||
"Data sync authorization is revoked; accountId={AccountId}; pending content was discarded and collection is paused; droppedPendingBatches={DroppedPendingBatches}.",
|
||||
account.AccountId, droppedOnRevoke);
|
||||
revoked = true;
|
||||
break;
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
// A disconnected platform must not prevent local DB collection.
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockedAccountIds.Remove(account.AccountId))
|
||||
logger.LogInformation("Data sync authorization was restored; accountId={AccountId}; collection is resumed.", account.AccountId);
|
||||
if (!state.Reconcile(remote, account.AccountId, "messages"))
|
||||
if (blockedAccountIds.Remove(account.AccountId))
|
||||
logger.LogInformation("Data sync authorization was restored; accountId={AccountId}; collection is resumed.", account.AccountId);
|
||||
if (!state.Reconcile(remote, account.AccountId, streamKey))
|
||||
continue;
|
||||
var dropped = queue.DropForAccount(account.AccountId, streamKey);
|
||||
logger.LogWarning(
|
||||
"Data sync checkpoint reconciled with the platform; accountId={AccountId}; stream={StreamKey}; localSequence={LocalSequence}; platformSequence={PlatformSequence}; droppedPendingBatches={DroppedPendingBatches}.",
|
||||
account.AccountId, streamKey, local.ConfirmedSequence, remote.ConfirmedSequence, dropped);
|
||||
}
|
||||
if (revoked)
|
||||
continue;
|
||||
var dropped = queue.DropForAccount(account.AccountId, "messages");
|
||||
logger.LogWarning(
|
||||
"Data sync checkpoint reconciled with the platform; accountId={AccountId}; localSequence={LocalSequence}; platformSequence={PlatformSequence}; droppedPendingBatches={DroppedPendingBatches}.",
|
||||
account.AccountId, local.ConfirmedSequence, remote.ConfirmedSequence, dropped);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,6 +354,8 @@ public sealed class RemoteAgentHostedService(
|
||||
ReportingConfig reporting,
|
||||
RemoteTaskLedger ledger,
|
||||
RemoteTaskEnvelope task,
|
||||
RemoteDataBatchQueue? dataQueue,
|
||||
RemoteDataSyncStateStore? syncState,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (task.Status is RemoteTaskStatus.Accepted or RemoteTaskStatus.Running)
|
||||
@@ -415,10 +424,56 @@ public sealed class RemoteAgentHostedService(
|
||||
var renewed = await client.RenewTaskAsync(started, cancellationToken);
|
||||
if (task.Kind == "sync-data")
|
||||
{
|
||||
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
var streamKey = task.Payload.ValueKind == JsonValueKind.Object
|
||||
&& task.Payload.TryGetProperty("stream_key", out var streamElement)
|
||||
&& streamElement.ValueKind == JsonValueKind.String
|
||||
? streamElement.GetString() ?? "messages"
|
||||
: "messages";
|
||||
if (streamKey == "contacts")
|
||||
{
|
||||
try
|
||||
{
|
||||
if (dataQueue is null || syncState is null)
|
||||
throw new ServiceException("SyncUnavailable", 503, "The contacts sync queue is not available.");
|
||||
var queued = await QueueContactsSnapshotAsync(remote.NodeId!, reporting, task.AccountId, dataQueue, syncState, cancellationToken);
|
||||
var message = queued ? "通讯录同步已加入节点本地队列。" : "已有通讯录同步待上传,本次请求未重复排队。";
|
||||
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
RemoteTaskStatus.Succeeded, null, message, false, null, Guid.NewGuid().ToString("N"));
|
||||
ledger.Complete(result);
|
||||
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (ServiceException exception)
|
||||
{
|
||||
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
RemoteTaskStatus.Failed, exception.Code, "通讯录同步失败。", false, null, Guid.NewGuid().ToString("N"));
|
||||
ledger.Complete(result);
|
||||
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
|
||||
}
|
||||
catch (WxAgentException exception)
|
||||
{
|
||||
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
RemoteTaskStatus.Failed, exception.Code.ToString(), "通讯录同步失败。", false, null, Guid.NewGuid().ToString("N"));
|
||||
ledger.Complete(result);
|
||||
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (streamKey != "messages")
|
||||
{
|
||||
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
RemoteTaskStatus.Failed, "InvalidTaskPayload", "The sync stream is unsupported.", false, null, Guid.NewGuid().ToString("N"));
|
||||
ledger.Complete(result);
|
||||
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
|
||||
return;
|
||||
}
|
||||
var messagesResult = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
RemoteTaskStatus.Succeeded, null, "后台数据同步已受理;采集器将在下一轮同步周期执行。", false, null, Guid.NewGuid().ToString("N"));
|
||||
ledger.Complete(result);
|
||||
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
|
||||
ledger.Complete(messagesResult);
|
||||
await ReportResultAsync(client, ledger, messagesResult, reporting, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -600,6 +655,105 @@ public sealed class RemoteAgentHostedService(
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> QueueContactsSnapshotAsync(
|
||||
string nodeId,
|
||||
ReportingConfig reporting,
|
||||
string accountId,
|
||||
RemoteDataBatchQueue dataQueue,
|
||||
RemoteDataSyncStateStore syncState,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (dataQueue.Pending().Any(batch => batch.AccountId == accountId && batch.StreamKey == "contacts"))
|
||||
return false;
|
||||
|
||||
var scopes = AuthorizedChatScopes(reporting, accountId);
|
||||
RequireScopes(reporting, accountId, scopes);
|
||||
var authorizedContacts = await ReadContactsForScopesAsync(accountId, scopes, cancellationToken);
|
||||
var authorizedById = authorizedContacts.ToDictionary(contact => contact.Id, StringComparer.Ordinal);
|
||||
var groupIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
for (var offset = 0; ;)
|
||||
{
|
||||
var page = await backend.ContactsAsync(accountId, null, true, 200, offset, cancellationToken);
|
||||
foreach (var group in page.Items)
|
||||
{
|
||||
if (authorizedById.ContainsKey(group.Id))
|
||||
groupIds.Add(group.Id);
|
||||
}
|
||||
if (!page.HasMore)
|
||||
break;
|
||||
var nextOffset = page.NextOffset ?? offset + page.Items.Count;
|
||||
if (nextOffset <= offset)
|
||||
throw new ServiceException("InvalidPagination", 502, "The contacts page did not advance.");
|
||||
offset = nextOffset;
|
||||
}
|
||||
|
||||
var observedAt = DateTimeOffset.UtcNow;
|
||||
var contacts = authorizedContacts
|
||||
.Select(contact => new RemoteSyncContact(
|
||||
contact.Id,
|
||||
groupIds.Contains(contact.Id) ? ReportingChatType.Group : ReportingChatType.Private,
|
||||
contact.DisplayName ?? string.Empty,
|
||||
contact.Remark ?? string.Empty,
|
||||
observedAt))
|
||||
.Where(contact => scopes.Any(scope => scope.ChatType == contact.ChatType
|
||||
&& (scope.ChatId == "*" || scope.ChatId == contact.ChatId)))
|
||||
.GroupBy(contact => contact.ChatId, StringComparer.Ordinal)
|
||||
.Select(group => group.First())
|
||||
.OrderBy(contact => contact.ChatType)
|
||||
.ThenBy(contact => contact.ChatId, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
const string streamKey = "contacts";
|
||||
var checkpoint = syncState.GetLatest(accountId, streamKey);
|
||||
var sourceGeneration = syncState.GetLatest(accountId, "messages").SourceGeneration;
|
||||
if (string.IsNullOrWhiteSpace(sourceGeneration))
|
||||
sourceGeneration = checkpoint.SourceGeneration;
|
||||
if (string.IsNullOrWhiteSpace(sourceGeneration))
|
||||
throw new ServiceException("SyncUnavailable", 503, "The account data source has not been initialized.");
|
||||
var sequence = checkpoint.SourceGeneration == sourceGeneration ? checkpoint.ConfirmedSequence + 1 : 1;
|
||||
var snapshotId = Guid.NewGuid().ToString("N");
|
||||
var pendingContacts = new List<RemoteSyncContact>();
|
||||
|
||||
RemoteSyncBatch CreateBatch(IReadOnlyList<RemoteSyncContact> items, bool complete)
|
||||
{
|
||||
var batch = new RemoteSyncBatch(
|
||||
nodeId, accountId, Guid.NewGuid().ToString("N"), sourceGeneration, streamKey, sequence,
|
||||
"{}", "{}", string.Empty, complete ? "complete" : "partial", [], [])
|
||||
{
|
||||
Contacts = items,
|
||||
ContactsSnapshotId = snapshotId
|
||||
};
|
||||
return batch with { PayloadHash = RemoteDataBatchAuthorization.ComputePayloadHash(batch) };
|
||||
}
|
||||
|
||||
void Enqueue(bool complete)
|
||||
{
|
||||
var batch = CreateBatch(pendingContacts.ToArray(), complete);
|
||||
if (JsonSerializer.SerializeToUtf8Bytes(batch, RemoteJson.Options).Length > RemoteDataProtocol.MaxBatchBytes - 2048)
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "A contacts sync batch exceeds the size limit.");
|
||||
var result = dataQueue.Enqueue(reporting, batch);
|
||||
if (!result.Accepted)
|
||||
throw new ServiceException(result.Reason ?? "SyncQueueFull", 503, "The contacts sync batch was not queued.");
|
||||
sequence++;
|
||||
pendingContacts.Clear();
|
||||
}
|
||||
|
||||
foreach (var contact in contacts)
|
||||
{
|
||||
pendingContacts.Add(contact);
|
||||
if (pendingContacts.Count <= RemoteDataProtocol.MaxBatchItems
|
||||
&& JsonSerializer.SerializeToUtf8Bytes(CreateBatch(pendingContacts, false), RemoteJson.Options).Length <= RemoteDataProtocol.MaxBatchBytes - 2048)
|
||||
continue;
|
||||
pendingContacts.RemoveAt(pendingContacts.Count - 1);
|
||||
if (pendingContacts.Count == 0)
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "A contact record exceeds the sync batch size limit.");
|
||||
Enqueue(false);
|
||||
pendingContacts.Add(contact);
|
||||
}
|
||||
Enqueue(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<ContactInfo>> ReadContactsForScopesAsync(
|
||||
string accountId, IReadOnlyList<RemoteReportingScope> scopes, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -160,4 +160,29 @@ public sealed class RemoteDataSyncTests
|
||||
private static RemoteSyncMessage Message(string messageId, string chatId) => new(
|
||||
messageId, chatId, ReportingChatType.Private, "source-" + messageId, "incoming", "text", "hello",
|
||||
DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, "wx-1", "payload-" + messageId);
|
||||
|
||||
[Fact]
|
||||
public void ContactBatches_FilterUnauthorizedContactsAndRehash()
|
||||
{
|
||||
var config = ConfigFor("chat-a");
|
||||
var batch = new RemoteSyncBatch(
|
||||
"node-a", "account-a", "contacts-1", "generation-a", "contacts", 1, "{}", "{}", string.Empty, "complete", [], [])
|
||||
{
|
||||
ContactsSnapshotId = "snapshot-1",
|
||||
Contacts =
|
||||
[
|
||||
new RemoteSyncContact("chat-a", ReportingChatType.Private, "Allowed", "", DateTimeOffset.UtcNow),
|
||||
new RemoteSyncContact("chat-b", ReportingChatType.Private, "Filtered", "", DateTimeOffset.UtcNow)
|
||||
]
|
||||
};
|
||||
|
||||
var filtered = RemoteDataBatchAuthorization.Filter(config, batch, out var reason);
|
||||
|
||||
Assert.NotNull(filtered);
|
||||
Assert.Equal("SomeRecordsDroppedByReporting", reason);
|
||||
Assert.Equal("partial", filtered!.CoverageState);
|
||||
Assert.Single(filtered.Contacts);
|
||||
Assert.Equal("chat-a", filtered.Contacts[0].ChatId);
|
||||
Assert.Equal(RemoteDataBatchAuthorization.ComputePayloadHash(filtered), filtered.PayloadHash);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user