using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; namespace WxAgent.Core; public static class RemoteDataProtocol { public const int MaxBatchBytes = 512 * 1024; public const int MaxBatchItems = 1000; public const int MaxBatchMessages = 5000; } public sealed record RemoteSyncBatch( [property: JsonPropertyName("node_id")] string NodeId, [property: JsonPropertyName("account_id")] string AccountId, [property: JsonPropertyName("batch_id")] string BatchId, [property: JsonPropertyName("source_generation")] string SourceGeneration, [property: JsonPropertyName("stream_key")] string StreamKey, [property: JsonPropertyName("sequence")] long Sequence, [property: JsonPropertyName("cursor_start")] string CursorStart, [property: JsonPropertyName("cursor_end")] string CursorEnd, [property: JsonPropertyName("payload_hash")] string PayloadHash, [property: JsonPropertyName("coverage_state")] string CoverageState, [property: JsonPropertyName("conversations")] IReadOnlyList Conversations, [property: JsonPropertyName("messages")] IReadOnlyList Messages); public sealed record RemoteSyncConversation( [property: JsonPropertyName("chat_id")] string ChatId, [property: JsonPropertyName("chat_type")] ReportingChatType ChatType, [property: JsonPropertyName("title")] string Title, [property: JsonPropertyName("last_activity_at")] DateTimeOffset? LastActivityAt, [property: JsonPropertyName("source")] string Source, [property: JsonPropertyName("observed_at")] DateTimeOffset ObservedAt, [property: JsonPropertyName("directory_state")] string DirectoryState); public sealed record RemoteSyncMessage( [property: JsonPropertyName("message_id")] string MessageId, [property: JsonPropertyName("chat_id")] string ChatId, [property: JsonPropertyName("chat_type")] ReportingChatType ChatType, [property: JsonPropertyName("source_message_id")] string SourceMessageId, [property: JsonPropertyName("direction")] string Direction, [property: JsonPropertyName("message_type")] string MessageType, [property: JsonPropertyName("text")] string Text, [property: JsonPropertyName("source_time")] DateTimeOffset SourceTime, [property: JsonPropertyName("observed_at")] DateTimeOffset ObservedAt, [property: JsonPropertyName("source_version")] string SourceVersion, [property: JsonPropertyName("payload_hash")] string PayloadHash); public sealed record RemoteDataBatchAck( [property: JsonPropertyName("accepted")] bool Accepted, [property: JsonPropertyName("duplicate")] bool Duplicate, [property: JsonPropertyName("batch_id")] string BatchId, [property: JsonPropertyName("confirmed_sequence")] long ConfirmedSequence, [property: JsonPropertyName("confirmed_cursor")] string ConfirmedCursor, [property: JsonPropertyName("reason")] string? Reason = null); public sealed record RemoteDataBatchEnqueueResult(bool Accepted, bool Duplicate, string Reason, RemoteSyncBatch? Batch); public sealed record RemoteDataCollection( string SourceGeneration, IReadOnlyDictionary NextCursors, IReadOnlyList Conversations, IReadOnlyList Messages, bool IsComplete, bool HasNewItems, IReadOnlyList UnavailableSources); public interface IRemoteDataCollector { Task CollectAsync( string accountId, IReadOnlyList scopes, RemoteDataSyncCheckpoint checkpoint, CancellationToken cancellationToken); } public sealed record RemoteDataSyncCheckpoint( string AccountId, string StreamKey, string SourceGeneration, long ConfirmedSequence, IReadOnlyDictionary ConfirmedCursors, DateTimeOffset? LastCollectedAt, DateTimeOffset? LastConfirmedAt, string CoverageState, IReadOnlyList UnavailableSources); public sealed class RemoteDataSyncStateStore { private sealed class StateFile { [JsonPropertyName("items")] public List Items { get; set; } = []; } private readonly string path; private readonly object gate = new(); private readonly Dictionary items; public RemoteDataSyncStateStore(string path) { this.path = Path.GetFullPath(path); items = Load(this.path).ToDictionary(item => Key(item.AccountId, item.StreamKey), StringComparer.Ordinal); } public RemoteDataSyncCheckpoint Get(string accountId, string streamKey, string sourceGeneration) { lock (gate) { if (!items.TryGetValue(Key(accountId, streamKey), out var current) || !string.Equals(current.SourceGeneration, sourceGeneration, StringComparison.Ordinal)) { return NewCheckpoint(accountId, streamKey, sourceGeneration); } return current; } } public RemoteDataSyncCheckpoint GetLatest(string accountId, string streamKey) { lock (gate) { return items.TryGetValue(Key(accountId, streamKey), out var current) ? current : NewCheckpoint(accountId, streamKey, string.Empty); } } public void MarkCollected(RemoteSyncBatch batch, IReadOnlyList unavailableSources) { lock (gate) { var key = Key(batch.AccountId, batch.StreamKey); var current = items.TryGetValue(key, out var existing) ? existing : new RemoteDataSyncCheckpoint(batch.AccountId, batch.StreamKey, batch.SourceGeneration, 0, new Dictionary(), null, null, "unknown", []); items[key] = current with { SourceGeneration = batch.SourceGeneration, LastCollectedAt = DateTimeOffset.UtcNow, CoverageState = batch.CoverageState, UnavailableSources = unavailableSources.ToArray() }; SaveLocked(); } } public void MarkConfirmed(RemoteSyncBatch batch) { lock (gate) { var key = Key(batch.AccountId, batch.StreamKey); var current = items.TryGetValue(key, out var existing) ? existing : NewCheckpoint(batch.AccountId, batch.StreamKey, batch.SourceGeneration); var cursors = ParseCursors(batch.CursorEnd); items[key] = current with { SourceGeneration = batch.SourceGeneration, ConfirmedSequence = batch.Sequence, ConfirmedCursors = cursors, LastConfirmedAt = DateTimeOffset.UtcNow, CoverageState = batch.CoverageState, UnavailableSources = [] }; SaveLocked(); } } public bool Reconcile(RemoteDataSyncStatus remote, string accountId, string streamKey) { lock (gate) { var key = Key(accountId, streamKey); var current = items.TryGetValue(key, out var existing) ? existing : NewCheckpoint(accountId, streamKey, string.Empty); var cursors = ParseCursors(remote.ConfirmedCursor); var unchanged = string.Equals(current.SourceGeneration, remote.SourceGeneration, StringComparison.Ordinal) && current.ConfirmedSequence == remote.ConfirmedSequence && current.ConfirmedCursors.OrderBy(item => item.Key).SequenceEqual(cursors.OrderBy(item => item.Key)); // A zero/zero response is an empty platform checkpoint, not evidence // that a locally collected but not-yet-ACKed batch should be dropped. if (unchanged || current.ConfirmedSequence == 0 && remote.ConfirmedSequence == 0 && current.ConfirmedCursors.Count == 0 && cursors.Count == 0) return false; items[key] = new RemoteDataSyncCheckpoint( accountId, streamKey, remote.SourceGeneration, remote.ConfirmedSequence, cursors, null, remote.LastSuccessAt, remote.State, []); SaveLocked(); return true; } } private void SaveLocked() { var directory = Path.GetDirectoryName(path) ?? AppContext.BaseDirectory; Directory.CreateDirectory(directory); var temporary = path + ".tmp-" + Guid.NewGuid().ToString("N"); try { var state = new StateFile { Items = items.Values.OrderBy(item => item.AccountId).ThenBy(item => item.StreamKey).ToList() }; File.WriteAllText(temporary, JsonSerializer.Serialize(state, RemoteJson.Options), Encoding.UTF8); if (!OperatingSystem.IsWindows()) { try { File.SetUnixFileMode(temporary, UnixFileMode.UserRead | UnixFileMode.UserWrite); } catch (PlatformNotSupportedException) { } } if (OperatingSystem.IsWindows() && File.Exists(path)) File.Replace(temporary, path, null); else File.Move(temporary, path, true); } finally { if (File.Exists(temporary)) File.Delete(temporary); } } private static IReadOnlyList Load(string path) { if (!File.Exists(path)) return []; try { return JsonSerializer.Deserialize(File.ReadAllText(path), RemoteJson.Options)?.Items ?? []; } catch (Exception exception) when (exception is IOException or JsonException or NotSupportedException) { throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The data sync state could not be loaded; synchronization is blocked.", exception); } } private static Dictionary ParseCursors(string value) { if (string.IsNullOrWhiteSpace(value)) return new(StringComparer.Ordinal); try { return JsonSerializer.Deserialize>(value, RemoteJson.Options) ?? new(StringComparer.Ordinal); } catch (JsonException exception) { throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The confirmed sync cursor is invalid.", exception); } } private static RemoteDataSyncCheckpoint NewCheckpoint(string accountId, string streamKey, string sourceGeneration) => new(accountId, streamKey, sourceGeneration, 0, new Dictionary(), null, null, "unknown", []); private static string Key(string accountId, string streamKey) => $"{accountId}\u001f{streamKey}"; } public static class RemoteDataBatchAuthorization { public static string ComputePayloadHash(RemoteSyncBatch batch) { var canonical = batch with { PayloadHash = string.Empty }; return Convert.ToHexString(SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(canonical, RemoteJson.Options))).ToLowerInvariant(); } public static RemoteSyncBatch? Filter(ReportingConfig? config, RemoteSyncBatch batch, out string reason) { if (config is null || !config.Enabled || config.FindAccount(batch.AccountId) is not { Enabled: true }) { reason = "AccountNotAuthorized"; return null; } var conversations = batch.Conversations.Where(conversation => ReportingAuthorization.Check(config, batch.AccountId, conversation.ChatId, conversation.ChatType, ReportingDataType.TaskResult).Allowed).ToArray(); var messages = batch.Messages.Where(message => ReportingAuthorization.Check(config, batch.AccountId, message.ChatId, message.ChatType, ReportingDataType.Message).Allowed).ToArray(); var dropped = conversations.Length != batch.Conversations.Count || messages.Length != batch.Messages.Count; reason = dropped ? "SomeRecordsDroppedByReporting" : "Authorized"; if (conversations.Length == 0 && messages.Length == 0 && (batch.Conversations.Count > 0 || batch.Messages.Count > 0)) { reason = "AllRecordsDroppedByReporting"; return null; } return dropped ? batch with { CoverageState = "partial", Conversations = conversations, Messages = messages } : batch; } } public sealed class RemoteDataBatchQueue { private sealed class QueueState { [JsonPropertyName("items")] public List Items { get; set; } = []; } private readonly string path; private readonly int maxItems; private readonly long maxBytes; private readonly object gate = new(); private QueueState state; public RemoteDataBatchQueue(string path, int maxItems = RemoteDataProtocol.MaxBatchItems, long maxBytes = 16 * 1024 * 1024) { if (maxItems is < 1 or > 100_000) throw new ArgumentOutOfRangeException(nameof(maxItems)); if (maxBytes is < 1 or > 512L * 1024 * 1024) throw new ArgumentOutOfRangeException(nameof(maxBytes)); this.path = Path.GetFullPath(path); this.maxItems = maxItems; this.maxBytes = maxBytes; state = Load(this.path); } public int PendingCount { get { lock (gate) return state.Items.Count; } } public long PendingBytes { get { lock (gate) return state.Items.Sum(SerializedSize); } } public RemoteDataBatchEnqueueResult Enqueue(ReportingConfig config, RemoteSyncBatch batch) { ValidateBatch(batch); var filtered = RemoteDataBatchAuthorization.Filter(config, batch, out var reason); if (filtered is null) return new RemoteDataBatchEnqueueResult(false, false, reason, null); var size = SerializedSize(filtered); if (size > RemoteDataProtocol.MaxBatchBytes) throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The sync batch is too large."); lock (gate) { var existing = state.Items.FirstOrDefault(item => item.BatchId == filtered.BatchId); if (existing is not null) { if (!string.Equals(existing.PayloadHash, filtered.PayloadHash, StringComparison.Ordinal)) throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The sync batch id was reused with a different payload."); return new RemoteDataBatchEnqueueResult(true, true, "AlreadyQueued", existing); } if (state.Items.Count >= maxItems || state.Items.Sum(SerializedSize) + size > maxBytes) return new RemoteDataBatchEnqueueResult(false, false, "SyncQueueFull", null); state.Items.Add(filtered); SaveLocked(); return new RemoteDataBatchEnqueueResult(true, false, reason, filtered); } } public IReadOnlyList Pending() { lock (gate) return state.Items.ToArray(); } public int DropForAccount(string accountId, string streamKey) { lock (gate) { var removed = state.Items.RemoveAll(item => string.Equals(item.AccountId, accountId, StringComparison.Ordinal) && string.Equals(item.StreamKey, streamKey, StringComparison.Ordinal)); if (removed > 0) SaveLocked(); return removed; } } public int DropAllForAccount(string accountId) { lock (gate) { var removed = state.Items.RemoveAll(item => string.Equals(item.AccountId, accountId, StringComparison.Ordinal)); if (removed > 0) SaveLocked(); return removed; } } public bool MarkConfirmed(RemoteDataBatchAck acknowledgement) { lock (gate) { var index = state.Items.FindIndex(item => item.BatchId == acknowledgement.BatchId); if (index < 0) return false; if (!acknowledgement.Accepted || !string.Equals(state.Items[index].CursorEnd, acknowledgement.ConfirmedCursor, StringComparison.Ordinal)) return false; state.Items.RemoveAt(index); SaveLocked(); return true; } } public bool Drop(string batchId) { lock (gate) { var removed = state.Items.RemoveAll(item => item.BatchId == batchId) > 0; if (removed) SaveLocked(); return removed; } } private void SaveLocked() { var directory = Path.GetDirectoryName(path) ?? AppContext.BaseDirectory; Directory.CreateDirectory(directory); var temporary = path + ".tmp-" + Guid.NewGuid().ToString("N"); try { File.WriteAllText(temporary, JsonSerializer.Serialize(state, RemoteJson.Options), Encoding.UTF8); if (!OperatingSystem.IsWindows()) { try { File.SetUnixFileMode(temporary, UnixFileMode.UserRead | UnixFileMode.UserWrite); } catch (PlatformNotSupportedException) { } } if (OperatingSystem.IsWindows() && File.Exists(path)) File.Replace(temporary, path, null); else File.Move(temporary, path, true); } finally { if (File.Exists(temporary)) File.Delete(temporary); } } private static QueueState Load(string path) { if (!File.Exists(path)) return new QueueState(); try { var result = JsonSerializer.Deserialize(File.ReadAllText(path), RemoteJson.Options) ?? new QueueState(); result.Items ??= []; return result; } catch (Exception exception) when (exception is IOException or JsonException or NotSupportedException or ArgumentException) { throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The data sync queue could not be loaded; synchronization is blocked.", exception); } } private static long SerializedSize(RemoteSyncBatch batch) => JsonSerializer.SerializeToUtf8Bytes(batch, RemoteJson.Options).LongLength; private static void ValidateBatch(RemoteSyncBatch batch) { foreach (var value in new[] { batch.NodeId, batch.AccountId, batch.BatchId, batch.SourceGeneration, batch.StreamKey, batch.PayloadHash }) RemoteAgentOptions.ValidateIdentifier(value, "sync batch field", 512); if (batch.Sequence < 1 || batch.Conversations.Count + batch.Messages.Count > RemoteDataProtocol.MaxBatchMessages) throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The sync batch sequence or item count is invalid."); } }