422 lines
19 KiB
Go
422 lines
19 KiB
Go
package controlplane
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
const maxDataBatchBytes = 512 * 1024
|
|
|
|
type dataBatchRequest struct {
|
|
NodeID string `json:"node_id"`
|
|
AccountID string `json:"account_id"`
|
|
BatchID string `json:"batch_id"`
|
|
SourceGeneration string `json:"source_generation"`
|
|
StreamKey string `json:"stream_key"`
|
|
Sequence int64 `json:"sequence"`
|
|
CursorStart string `json:"cursor_start"`
|
|
CursorEnd string `json:"cursor_end"`
|
|
PayloadHash string `json:"payload_hash"`
|
|
CoverageState string `json:"coverage_state"`
|
|
Conversations []dataConversationRequest `json:"conversations"`
|
|
Messages []dataMessageRequest `json:"messages"`
|
|
}
|
|
|
|
type dataConversationRequest struct {
|
|
ChatID string `json:"chat_id"`
|
|
ChatType ChatType `json:"chat_type"`
|
|
Title string `json:"title"`
|
|
LastActivityAt *time.Time `json:"last_activity_at"`
|
|
Source string `json:"source"`
|
|
ObservedAt time.Time `json:"observed_at"`
|
|
DirectoryState string `json:"directory_state"`
|
|
}
|
|
|
|
type dataMessageRequest struct {
|
|
MessageID string `json:"message_id"`
|
|
ChatID string `json:"chat_id"`
|
|
ChatType ChatType `json:"chat_type"`
|
|
SourceMessageID string `json:"source_message_id"`
|
|
Direction string `json:"direction"`
|
|
MessageType string `json:"message_type"`
|
|
Text string `json:"text"`
|
|
SourceTime time.Time `json:"source_time"`
|
|
ObservedAt time.Time `json:"observed_at"`
|
|
SourceVersion string `json:"source_version"`
|
|
PayloadHash string `json:"payload_hash"`
|
|
}
|
|
|
|
type dataBatchAck struct {
|
|
Accepted bool `json:"accepted"`
|
|
Duplicate bool `json:"duplicate"`
|
|
BatchID string `json:"batch_id"`
|
|
ConfirmedSequence int64 `json:"confirmed_sequence"`
|
|
ConfirmedCursor string `json:"confirmed_cursor"`
|
|
Reason string `json:"reason,omitempty"`
|
|
}
|
|
|
|
type dataSyncView struct {
|
|
State string `json:"state"`
|
|
Source string `json:"source"`
|
|
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
|
ConfirmedSequence int64 `json:"confirmed_sequence"`
|
|
ConfirmedCursor string `json:"confirmed_cursor,omitempty"`
|
|
BacklogCount *int64 `json:"backlog_count"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
ErrorMessage string `json:"error_message,omitempty"`
|
|
}
|
|
|
|
type dataConversationView struct {
|
|
ChatID string `json:"chat_id"`
|
|
ChatType string `json:"chat_type"`
|
|
Title string `json:"title"`
|
|
LastActivityAt *time.Time `json:"last_activity_at,omitempty"`
|
|
Source string `json:"source"`
|
|
ObservedAt time.Time `json:"observed_at"`
|
|
DirectoryState string `json:"directory_state"`
|
|
}
|
|
|
|
type dataMessageView struct {
|
|
MessageID string `json:"message_id"`
|
|
ChatID string `json:"chat_id"`
|
|
SourceMessageID string `json:"source_message_id"`
|
|
Direction string `json:"direction"`
|
|
MessageType string `json:"message_type"`
|
|
Text string `json:"text"`
|
|
SourceTime time.Time `json:"source_time"`
|
|
ObservedAt time.Time `json:"observed_at"`
|
|
SourceVersion string `json:"source_version"`
|
|
}
|
|
|
|
func (s *Server) registerDataAccounts(nodeID string, registration NodeRegistration) error {
|
|
for _, account := range registration.Accounts {
|
|
if !account.Verified {
|
|
continue
|
|
}
|
|
scopes := make([]ReportingScope, 0, len(account.AllowedChats))
|
|
for _, chat := range account.AllowedChats {
|
|
dataType := "read"
|
|
if chat.ChatID == "" || !validChatType(chat.ChatType) {
|
|
continue
|
|
}
|
|
scopes = append(scopes, ReportingScope{ChatID: chat.ChatID, DataType: dataType, ConfigVersion: int(registration.ReportingConfigVersion)})
|
|
}
|
|
store, err := s.accountStores.RegisterAccount(context.Background(), AccountRegistration{
|
|
AccountID: account.AccountID,
|
|
StableIdentity: account.AccountID,
|
|
SourceNodeID: nodeID,
|
|
SourceGeneration: account.AccountID,
|
|
Verified: true,
|
|
AuthorizationVersion: int(registration.ReportingConfigVersion),
|
|
ReportingScopes: scopes,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_ = store.Close()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type nodeDataSyncStatusView struct {
|
|
State string `json:"state"`
|
|
SourceGeneration string `json:"source_generation"`
|
|
ConfirmedSequence int64 `json:"confirmed_sequence"`
|
|
ConfirmedCursor string `json:"confirmed_cursor"`
|
|
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
|
}
|
|
|
|
func (s *Server) nodeDataRoute(w http.ResponseWriter, r *http.Request, nodeID string, parts []string, _ string) error {
|
|
if len(parts) != 3 || parts[0] != "accounts" || parts[2] != "sync-status" || r.Method != http.MethodGet || !validIdentifier(parts[1], 200) {
|
|
return requestError{status: http.StatusNotFound, code: "NotFound", message: "Resource was not found."}
|
|
}
|
|
accountID := parts[1]
|
|
var authorized bool
|
|
if err := s.store.Read(func(state PersistedState) error {
|
|
node, ok := state.Nodes[nodeID]
|
|
if !ok {
|
|
return requestError{status: http.StatusConflict, code: "NodeNotRegistered", message: "Register the node before reading sync status."}
|
|
}
|
|
authorized = nodeHasAccount(node, accountID)
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if !authorized {
|
|
return requestError{status: http.StatusForbidden, code: "AccountNotAuthorized", message: "The node is not authorized for this account."}
|
|
}
|
|
store, err := s.accountStores.OpenAccount(r.Context(), accountID)
|
|
if err != nil {
|
|
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
|
}
|
|
defer store.Close()
|
|
status, err := store.GetSyncStatus(r.Context(), "messages")
|
|
if err != nil {
|
|
return requestError{status: http.StatusInternalServerError, code: "DataStatusFailed", message: "The platform sync status query failed."}
|
|
}
|
|
writeJSON(w, http.StatusOK, nodeDataSyncStatusView{State: status.CoverageState, SourceGeneration: status.SourceGeneration, ConfirmedSequence: status.ConfirmedSequence, ConfirmedCursor: status.ConfirmedCursor, LastSuccessAt: status.LastSuccessAt})
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) ingestDataBatch(w http.ResponseWriter, r *http.Request, correlationID string) error {
|
|
nodeID, err := s.authenticateNode(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var request dataBatchRequest
|
|
if err := decodeJSON(r, &request, maxDataBatchBytes); err != nil {
|
|
return err
|
|
}
|
|
if request.NodeID != nodeID || request.AccountID == "" || request.BatchID == "" || request.SourceGeneration == "" || request.StreamKey != "messages" || request.Sequence < 1 || request.PayloadHash == "" || len(request.Messages) > 5000 || len(request.Conversations) > 1000 {
|
|
return requestError{status: http.StatusBadRequest, code: "InvalidDataBatch", message: "The data batch is invalid."}
|
|
}
|
|
var account AccountSummary
|
|
if err := s.store.Read(func(state PersistedState) error {
|
|
node, ok := state.Nodes[nodeID]
|
|
if !ok {
|
|
return requestError{status: http.StatusConflict, code: "NodeNotRegistered", message: "Register the node before sending data."}
|
|
}
|
|
for _, candidate := range node.Accounts {
|
|
if candidate.AccountID == request.AccountID {
|
|
account = candidate
|
|
return nil
|
|
}
|
|
}
|
|
return requestError{status: http.StatusForbidden, code: "AccountNotAuthorized", message: "The node is not authorized for this account."}
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if !account.Verified {
|
|
return requestError{status: http.StatusForbidden, code: "AccountNotVerified", message: "The account identity is not verified."}
|
|
}
|
|
store, err := s.accountStores.OpenAccount(r.Context(), request.AccountID)
|
|
if err != nil {
|
|
return requestError{status: http.StatusForbidden, code: "AccountDataUnavailable", message: "The account data store is not available."}
|
|
}
|
|
defer store.Close()
|
|
batch := IngestBatch{
|
|
BatchID: request.BatchID, SourceGeneration: request.SourceGeneration, StreamKey: request.StreamKey,
|
|
Sequence: request.Sequence, CursorStart: request.CursorStart, CursorEnd: request.CursorEnd,
|
|
PayloadHash: request.PayloadHash, CoverageState: request.CoverageState,
|
|
}
|
|
for _, conversation := range request.Conversations {
|
|
if !validChatType(conversation.ChatType) || conversation.ObservedAt.IsZero() {
|
|
return requestError{status: http.StatusBadRequest, code: "InvalidDataBatch", message: "A conversation record is invalid."}
|
|
}
|
|
batch.Conversations = append(batch.Conversations, ConversationRecord{
|
|
ChatID: conversation.ChatID, ChatType: string(conversation.ChatType), Title: conversation.Title,
|
|
LastActivityAt: conversation.LastActivityAt, Source: conversation.Source,
|
|
ObservedAt: conversation.ObservedAt, DirectoryState: conversation.DirectoryState,
|
|
})
|
|
}
|
|
for _, message := range request.Messages {
|
|
if !validChatType(message.ChatType) || message.SourceTime.IsZero() || message.ObservedAt.IsZero() || message.MessageID == "" || message.ChatID == "" || message.PayloadHash == "" {
|
|
return requestError{status: http.StatusBadRequest, code: "InvalidDataBatch", message: "A message record is invalid."}
|
|
}
|
|
batch.Messages = append(batch.Messages, MessageRecord{
|
|
MessageID: message.MessageID, ChatID: message.ChatID, SourceMessageID: message.SourceMessageID,
|
|
Direction: message.Direction, MessageType: message.MessageType, Text: message.Text,
|
|
SourceTime: message.SourceTime, ObservedAt: message.ObservedAt, SourceVersion: message.SourceVersion,
|
|
PayloadHash: message.PayloadHash,
|
|
})
|
|
}
|
|
result, err := store.ApplyBatch(r.Context(), batch)
|
|
if err != nil {
|
|
switch {
|
|
case errors.Is(err, ErrAccountNotAuthorized):
|
|
return requestError{status: http.StatusForbidden, code: "DataNotAuthorized", message: "The data batch contains data outside the active reporting scopes."}
|
|
case errors.Is(err, ErrBatchConflict):
|
|
return requestError{status: http.StatusConflict, code: "BatchConflict", message: "The batch id or payload conflicts with an existing batch."}
|
|
case errors.Is(err, ErrBatchSequenceGap):
|
|
return requestError{status: http.StatusConflict, code: "BatchSequenceGap", message: "The batch sequence has a gap; resend from the confirmed cursor."}
|
|
case errors.Is(err, ErrBatchSequenceConflict):
|
|
return requestError{status: http.StatusConflict, code: "BatchSequenceConflict", message: "The batch sequence is older than confirmed progress."}
|
|
case errors.Is(err, ErrAccountBindingConflict):
|
|
return requestError{status: http.StatusConflict, code: "AccountBindingConflict", message: "The source generation does not match the verified account binding."}
|
|
case errors.Is(err, ErrAccountCapacityExceeded):
|
|
return requestError{status: http.StatusInsufficientStorage, code: "AccountCapacityExceeded", message: "The account data capacity budget has been reached."}
|
|
default:
|
|
return requestError{status: http.StatusInternalServerError, code: "DataBatchFailed", message: "The platform could not persist the data batch."}
|
|
}
|
|
}
|
|
writeJSON(w, http.StatusOK, dataBatchAck{Accepted: true, Duplicate: result.Duplicate, BatchID: request.BatchID, ConfirmedSequence: result.ConfirmedSequence, ConfirmedCursor: result.ConfirmedCursor, Reason: "Persisted"})
|
|
return nil
|
|
}
|
|
|
|
func validChatType(chatType ChatType) bool {
|
|
return chatType == ChatGroup || chatType == ChatPrivate
|
|
}
|
|
|
|
func (s *Server) dataRoute(w http.ResponseWriter, r *http.Request, correlationID string) error {
|
|
username, err := s.authenticateWeb(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parts := pathParts(r.URL.Path)
|
|
if len(parts) < 5 || parts[0] != "v1" || parts[1] != "data" || parts[2] != "accounts" || !validIdentifier(parts[3], 200) {
|
|
return requestError{status: http.StatusNotFound, code: "NotFound", message: "Resource was not found."}
|
|
}
|
|
accountID := parts[3]
|
|
if len(parts) != 5 {
|
|
return requestError{status: http.StatusNotFound, code: "NotFound", message: "Resource was not found."}
|
|
}
|
|
switch parts[4] {
|
|
case "conversations":
|
|
if r.Method != http.MethodGet {
|
|
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
|
}
|
|
return s.listStoredConversations(w, r, accountID)
|
|
case "messages":
|
|
if r.Method != http.MethodGet {
|
|
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
|
}
|
|
return s.listStoredMessages(w, r, accountID)
|
|
case "sync-status":
|
|
if r.Method != http.MethodGet {
|
|
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
|
}
|
|
return s.storedSyncStatus(w, r, accountID)
|
|
case "refresh":
|
|
if r.Method != http.MethodPost {
|
|
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
|
}
|
|
return s.requestDataRefresh(w, r, accountID, username, correlationID)
|
|
case "revoke":
|
|
if r.Method != http.MethodPost {
|
|
return requestError{status: http.StatusMethodNotAllowed, code: "MethodNotAllowed", message: "Method is not allowed."}
|
|
}
|
|
return s.revokeDataAuthorization(w, r, accountID, username, correlationID)
|
|
default:
|
|
return requestError{status: http.StatusNotFound, code: "NotFound", message: "Resource was not found."}
|
|
}
|
|
}
|
|
|
|
func (s *Server) listStoredConversations(w http.ResponseWriter, r *http.Request, accountID string) error {
|
|
limit, offset := queryLimit(r.URL.Query().Get("limit")), queryOffset(r.URL.Query().Get("offset"))
|
|
store, err := s.accountStores.OpenAccount(r.Context(), accountID)
|
|
if err != nil {
|
|
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
|
}
|
|
defer store.Close()
|
|
items, err := store.QueryConversations(r.Context(), limit, offset)
|
|
if err != nil {
|
|
if errors.Is(err, ErrAccountNotAuthorized) {
|
|
return requestError{status: http.StatusForbidden, code: "DataNotAuthorized", message: "The account has no active conversation reporting scope."}
|
|
}
|
|
return requestError{status: http.StatusInternalServerError, code: "DataQueryFailed", message: "The platform data query failed."}
|
|
}
|
|
status, err := store.GetSyncStatus(r.Context(), "messages")
|
|
if err != nil {
|
|
return requestError{status: http.StatusInternalServerError, code: "DataStatusFailed", message: "The platform sync status query failed."}
|
|
}
|
|
views := make([]dataConversationView, 0, len(items))
|
|
for _, item := range items {
|
|
views = append(views, dataConversationView{item.ChatID, item.ChatType, item.Title, item.LastActivityAt, item.Source, item.ObservedAt, item.DirectoryState})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"items": views, "limit": limit, "offset": offset, "has_more": len(views) == limit, "next_offset": offset + len(views), "sync": syncView(status)})
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) listStoredMessages(w http.ResponseWriter, r *http.Request, accountID string) error {
|
|
chatID := r.URL.Query().Get("chat_id")
|
|
if !validIdentifier(chatID, 512) {
|
|
return requestError{status: http.StatusBadRequest, code: "InvalidChat", message: "chat_id is required."}
|
|
}
|
|
limit, offset := queryLimit(r.URL.Query().Get("limit")), queryOffset(r.URL.Query().Get("offset"))
|
|
store, err := s.accountStores.OpenAccount(r.Context(), accountID)
|
|
if err != nil {
|
|
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
|
}
|
|
defer store.Close()
|
|
allowed, err := store.AuthorizeChat(r.Context(), chatID, "messages")
|
|
if err != nil || !allowed {
|
|
return requestError{status: http.StatusForbidden, code: "DataNotAuthorized", message: "The chat is not in the active reporting scope."}
|
|
}
|
|
items, err := store.QueryMessages(r.Context(), chatID, limit, offset)
|
|
if err != nil {
|
|
return requestError{status: http.StatusInternalServerError, code: "DataQueryFailed", message: "The platform data query failed."}
|
|
}
|
|
status, err := store.GetSyncStatus(r.Context(), "messages")
|
|
if err != nil {
|
|
return requestError{status: http.StatusInternalServerError, code: "DataStatusFailed", message: "The platform sync status query failed."}
|
|
}
|
|
views := make([]dataMessageView, 0, len(items))
|
|
for _, item := range items {
|
|
views = append(views, dataMessageView{item.MessageID, item.ChatID, item.SourceMessageID, item.Direction, item.MessageType, item.Text, item.SourceTime, item.ObservedAt, item.SourceVersion})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"items": views, "limit": limit, "offset": offset, "has_more": len(views) == limit, "next_offset": offset + len(views), "sync": syncView(status)})
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) storedSyncStatus(w http.ResponseWriter, r *http.Request, accountID string) error {
|
|
store, err := s.accountStores.OpenAccount(r.Context(), accountID)
|
|
if err != nil {
|
|
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
|
}
|
|
defer store.Close()
|
|
status, err := store.GetSyncStatus(r.Context(), "messages")
|
|
if err != nil {
|
|
return requestError{status: http.StatusInternalServerError, code: "DataStatusFailed", message: "The platform sync status query failed."}
|
|
}
|
|
writeJSON(w, http.StatusOK, syncView(status))
|
|
return nil
|
|
}
|
|
|
|
func syncView(status AccountSyncStatus) dataSyncView {
|
|
return dataSyncView{State: status.CoverageState, Source: "platform-cache", LastSuccessAt: status.LastSuccessAt, ConfirmedSequence: status.ConfirmedSequence, ConfirmedCursor: status.ConfirmedCursor, BacklogCount: nil, ErrorCode: status.ErrorCode, ErrorMessage: status.ErrorMessage}
|
|
}
|
|
|
|
func (s *Server) revokeDataAuthorization(w http.ResponseWriter, r *http.Request, accountID, username, correlationID string) error {
|
|
if err := s.accountStores.SetScopes(r.Context(), accountID, nil); err != nil {
|
|
if errors.Is(err, ErrAccountStoreNotFound) {
|
|
return requestError{status: http.StatusNotFound, code: "AccountDataNotFound", message: "No platform data is available for this account."}
|
|
}
|
|
return requestError{status: http.StatusInternalServerError, code: "AuthorizationRevokeFailed", message: "The account authorization could not be revoked."}
|
|
}
|
|
if err := s.appendAudit("user:"+username, "data.authorization.revoke", accountID, correlationID, "success"); err != nil {
|
|
return err
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"account_id": accountID, "revoked": true})
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) requestDataRefresh(w http.ResponseWriter, r *http.Request, accountID, username, correlationID string) error {
|
|
var nodeID string
|
|
if err := s.store.Read(func(state PersistedState) error {
|
|
for id, node := range state.Nodes {
|
|
for _, account := range node.Accounts {
|
|
if account.AccountID == accountID && account.Verified {
|
|
nodeID = id
|
|
break
|
|
}
|
|
}
|
|
if nodeID != "" {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if nodeID == "" {
|
|
return requestError{status: http.StatusConflict, code: "NoAuthorizedNode", message: "No verified node is available for this account."}
|
|
}
|
|
payload := json.RawMessage(`{"stream_key":"messages","reason":"web-refresh"}`)
|
|
return s.createTaskSubmission(w, TaskSubmission{NodeID: nodeID, AccountID: accountID, Kind: "sync-data", IdempotencyKey: "sync:" + accountID + ":" + strconv.FormatInt(time.Now().UTC().Unix()/5, 10), Payload: payload}, username, correlationID)
|
|
}
|
|
|
|
func queryOffset(raw string) int {
|
|
if raw == "" {
|
|
return 0
|
|
}
|
|
value, err := strconv.Atoi(raw)
|
|
if err != nil || value < 0 || value > 10_000_000 {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|