feat: add account-scoped data synchronization
This commit is contained in:
@@ -255,7 +255,15 @@ public sealed record RemoteAccountSummary(
|
||||
[property: JsonPropertyName("active")] bool Active,
|
||||
[property: JsonPropertyName("verified")] bool Verified,
|
||||
[property: JsonPropertyName("allowed_group_count")] int AllowedGroupCount,
|
||||
[property: JsonPropertyName("allowed_private_count")] int AllowedPrivateCount);
|
||||
[property: JsonPropertyName("allowed_private_count")] int AllowedPrivateCount)
|
||||
{
|
||||
[JsonPropertyName("allowed_chats")]
|
||||
public IReadOnlyList<RemoteAllowedChatSummary> AllowedChats { get; init; } = [];
|
||||
}
|
||||
|
||||
public sealed record RemoteAllowedChatSummary(
|
||||
[property: JsonPropertyName("chat_id")] string ChatId,
|
||||
[property: JsonPropertyName("chat_type")] ReportingChatType ChatType);
|
||||
|
||||
public sealed record RemoteNodeRegistrationResponse(
|
||||
[property: JsonPropertyName("node_id")] string NodeId,
|
||||
@@ -264,6 +272,13 @@ public sealed record RemoteNodeRegistrationResponse(
|
||||
[property: JsonPropertyName("correlation_id")] string CorrelationId,
|
||||
[property: JsonPropertyName("connection_id")] string? ConnectionId = null);
|
||||
|
||||
public sealed record RemoteDataSyncStatus(
|
||||
[property: JsonPropertyName("state")] string State,
|
||||
[property: JsonPropertyName("source_generation")] string SourceGeneration,
|
||||
[property: JsonPropertyName("confirmed_sequence")] long ConfirmedSequence,
|
||||
[property: JsonPropertyName("confirmed_cursor")] string ConfirmedCursor,
|
||||
[property: JsonPropertyName("last_success_at")] DateTimeOffset? LastSuccessAt = null);
|
||||
|
||||
public sealed record RemoteHeartbeatResponse(
|
||||
[property: JsonPropertyName("node_id")] string NodeId,
|
||||
[property: JsonPropertyName("status")] RemoteNodeStatus Status,
|
||||
|
||||
@@ -166,6 +166,52 @@ public sealed class RemoteControlClient : IDisposable
|
||||
return sent;
|
||||
}
|
||||
|
||||
public Task<RemoteDataSyncStatus> GetDataSyncStatusAsync(
|
||||
string accountId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
RemoteAgentOptions.ValidateIdentifier(accountId, "accountId", 200);
|
||||
return SendAuthenticatedAsync<RemoteDataSyncStatus>(
|
||||
HttpMethod.Get,
|
||||
$"/v1/nodes/{Escape(_options.NodeId!)}/data/accounts/{Escape(accountId)}/sync-status",
|
||||
null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<RemoteDataBatchAck> SubmitDataBatchAsync(
|
||||
ReportingConfig reportingConfig,
|
||||
RemoteSyncBatch batch,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var filtered = RemoteDataBatchAuthorization.Filter(reportingConfig, batch, out var reason);
|
||||
if (filtered is null)
|
||||
return new RemoteDataBatchAck(false, false, batch.BatchId, 0, string.Empty, reason);
|
||||
return await SendAuthenticatedAsync<RemoteDataBatchAck>(HttpMethod.Post, "/v1/data/batches", filtered,
|
||||
cancellationToken, RemoteDataProtocol.MaxBatchBytes).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<RemoteSyncBatch>> FlushDataBatchesAsync(
|
||||
RemoteDataBatchQueue queue,
|
||||
ReportingConfig reportingConfig,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
EnsureAuthenticated();
|
||||
var confirmed = new List<RemoteSyncBatch>();
|
||||
foreach (var batch in queue.Pending())
|
||||
{
|
||||
var filtered = RemoteDataBatchAuthorization.Filter(reportingConfig, batch, out _);
|
||||
if (filtered is null)
|
||||
{
|
||||
queue.Drop(batch.BatchId);
|
||||
continue;
|
||||
}
|
||||
var acknowledgement = await SubmitDataBatchAsync(reportingConfig, filtered, cancellationToken).ConfigureAwait(false);
|
||||
if (acknowledgement.Accepted && queue.MarkConfirmed(acknowledgement))
|
||||
confirmed.Add(filtered);
|
||||
}
|
||||
return confirmed;
|
||||
}
|
||||
|
||||
public Task<RemoteTaskEnvelope> AcknowledgeCancellationAsync(
|
||||
RemoteTaskEnvelope task,
|
||||
RemoteTaskStatus status,
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace WxAgent.Core;
|
||||
|
||||
public static class RemoteDataProtocol
|
||||
{
|
||||
public const int MaxBatchBytes = 512 * 1024;
|
||||
public const int MaxBatchItems = 1000;
|
||||
public const int MaxBatchMessages = 5000;
|
||||
}
|
||||
|
||||
public sealed record RemoteSyncBatch(
|
||||
[property: JsonPropertyName("node_id")] string NodeId,
|
||||
[property: JsonPropertyName("account_id")] string AccountId,
|
||||
[property: JsonPropertyName("batch_id")] string BatchId,
|
||||
[property: JsonPropertyName("source_generation")] string SourceGeneration,
|
||||
[property: JsonPropertyName("stream_key")] string StreamKey,
|
||||
[property: JsonPropertyName("sequence")] long Sequence,
|
||||
[property: JsonPropertyName("cursor_start")] string CursorStart,
|
||||
[property: JsonPropertyName("cursor_end")] string CursorEnd,
|
||||
[property: JsonPropertyName("payload_hash")] string PayloadHash,
|
||||
[property: JsonPropertyName("coverage_state")] string CoverageState,
|
||||
[property: JsonPropertyName("conversations")] IReadOnlyList<RemoteSyncConversation> Conversations,
|
||||
[property: JsonPropertyName("messages")] IReadOnlyList<RemoteSyncMessage> Messages);
|
||||
|
||||
public sealed record RemoteSyncConversation(
|
||||
[property: JsonPropertyName("chat_id")] string ChatId,
|
||||
[property: JsonPropertyName("chat_type")] ReportingChatType ChatType,
|
||||
[property: JsonPropertyName("title")] string Title,
|
||||
[property: JsonPropertyName("last_activity_at")] DateTimeOffset? LastActivityAt,
|
||||
[property: JsonPropertyName("source")] string Source,
|
||||
[property: JsonPropertyName("observed_at")] DateTimeOffset ObservedAt,
|
||||
[property: JsonPropertyName("directory_state")] string DirectoryState);
|
||||
|
||||
public sealed record RemoteSyncMessage(
|
||||
[property: JsonPropertyName("message_id")] string MessageId,
|
||||
[property: JsonPropertyName("chat_id")] string ChatId,
|
||||
[property: JsonPropertyName("chat_type")] ReportingChatType ChatType,
|
||||
[property: JsonPropertyName("source_message_id")] string SourceMessageId,
|
||||
[property: JsonPropertyName("direction")] string Direction,
|
||||
[property: JsonPropertyName("message_type")] string MessageType,
|
||||
[property: JsonPropertyName("text")] string Text,
|
||||
[property: JsonPropertyName("source_time")] DateTimeOffset SourceTime,
|
||||
[property: JsonPropertyName("observed_at")] DateTimeOffset ObservedAt,
|
||||
[property: JsonPropertyName("source_version")] string SourceVersion,
|
||||
[property: JsonPropertyName("payload_hash")] string PayloadHash);
|
||||
|
||||
public sealed record RemoteDataBatchAck(
|
||||
[property: JsonPropertyName("accepted")] bool Accepted,
|
||||
[property: JsonPropertyName("duplicate")] bool Duplicate,
|
||||
[property: JsonPropertyName("batch_id")] string BatchId,
|
||||
[property: JsonPropertyName("confirmed_sequence")] long ConfirmedSequence,
|
||||
[property: JsonPropertyName("confirmed_cursor")] string ConfirmedCursor,
|
||||
[property: JsonPropertyName("reason")] string? Reason = null);
|
||||
|
||||
public sealed record RemoteDataBatchEnqueueResult(bool Accepted, bool Duplicate, string Reason, RemoteSyncBatch? Batch);
|
||||
|
||||
public sealed record RemoteDataCollection(
|
||||
string SourceGeneration,
|
||||
IReadOnlyDictionary<string, long> NextCursors,
|
||||
IReadOnlyList<RemoteSyncConversation> Conversations,
|
||||
IReadOnlyList<RemoteSyncMessage> Messages,
|
||||
bool IsComplete,
|
||||
bool HasNewItems,
|
||||
IReadOnlyList<string> UnavailableSources);
|
||||
|
||||
public interface IRemoteDataCollector
|
||||
{
|
||||
Task<RemoteDataCollection> CollectAsync(
|
||||
string accountId,
|
||||
IReadOnlyList<RemoteReportingScope> scopes,
|
||||
RemoteDataSyncCheckpoint checkpoint,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record RemoteDataSyncCheckpoint(
|
||||
string AccountId,
|
||||
string StreamKey,
|
||||
string SourceGeneration,
|
||||
long ConfirmedSequence,
|
||||
IReadOnlyDictionary<string, long> ConfirmedCursors,
|
||||
DateTimeOffset? LastCollectedAt,
|
||||
DateTimeOffset? LastConfirmedAt,
|
||||
string CoverageState,
|
||||
IReadOnlyList<string> UnavailableSources);
|
||||
|
||||
public sealed class RemoteDataSyncStateStore
|
||||
{
|
||||
private sealed class StateFile
|
||||
{
|
||||
[JsonPropertyName("items")]
|
||||
public List<RemoteDataSyncCheckpoint> Items { get; set; } = [];
|
||||
}
|
||||
|
||||
private readonly string path;
|
||||
private readonly object gate = new();
|
||||
private readonly Dictionary<string, RemoteDataSyncCheckpoint> items;
|
||||
|
||||
public RemoteDataSyncStateStore(string path)
|
||||
{
|
||||
this.path = Path.GetFullPath(path);
|
||||
items = Load(this.path).ToDictionary(item => Key(item.AccountId, item.StreamKey), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public RemoteDataSyncCheckpoint Get(string accountId, string streamKey, string sourceGeneration)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (!items.TryGetValue(Key(accountId, streamKey), out var current)
|
||||
|| !string.Equals(current.SourceGeneration, sourceGeneration, StringComparison.Ordinal))
|
||||
{
|
||||
return NewCheckpoint(accountId, streamKey, sourceGeneration);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
}
|
||||
|
||||
public RemoteDataSyncCheckpoint GetLatest(string accountId, string streamKey)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
return items.TryGetValue(Key(accountId, streamKey), out var current)
|
||||
? current
|
||||
: NewCheckpoint(accountId, streamKey, string.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkCollected(RemoteSyncBatch batch, IReadOnlyList<string> unavailableSources)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
var key = Key(batch.AccountId, batch.StreamKey);
|
||||
var current = items.TryGetValue(key, out var existing)
|
||||
? existing
|
||||
: new RemoteDataSyncCheckpoint(batch.AccountId, batch.StreamKey, batch.SourceGeneration, 0, new Dictionary<string, long>(), null, null, "unknown", []);
|
||||
items[key] = current with
|
||||
{
|
||||
SourceGeneration = batch.SourceGeneration,
|
||||
LastCollectedAt = DateTimeOffset.UtcNow,
|
||||
CoverageState = batch.CoverageState,
|
||||
UnavailableSources = unavailableSources.ToArray()
|
||||
};
|
||||
SaveLocked();
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkConfirmed(RemoteSyncBatch batch)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
var key = Key(batch.AccountId, batch.StreamKey);
|
||||
var current = items.TryGetValue(key, out var existing)
|
||||
? existing
|
||||
: NewCheckpoint(batch.AccountId, batch.StreamKey, batch.SourceGeneration);
|
||||
var cursors = ParseCursors(batch.CursorEnd);
|
||||
items[key] = current with
|
||||
{
|
||||
SourceGeneration = batch.SourceGeneration,
|
||||
ConfirmedSequence = batch.Sequence,
|
||||
ConfirmedCursors = cursors,
|
||||
LastConfirmedAt = DateTimeOffset.UtcNow,
|
||||
CoverageState = batch.CoverageState,
|
||||
UnavailableSources = []
|
||||
};
|
||||
SaveLocked();
|
||||
}
|
||||
}
|
||||
|
||||
public bool Reconcile(RemoteDataSyncStatus remote, string accountId, string streamKey)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
var key = Key(accountId, streamKey);
|
||||
var current = items.TryGetValue(key, out var existing)
|
||||
? existing
|
||||
: NewCheckpoint(accountId, streamKey, string.Empty);
|
||||
var cursors = ParseCursors(remote.ConfirmedCursor);
|
||||
var unchanged = string.Equals(current.SourceGeneration, remote.SourceGeneration, StringComparison.Ordinal)
|
||||
&& current.ConfirmedSequence == remote.ConfirmedSequence
|
||||
&& current.ConfirmedCursors.OrderBy(item => item.Key).SequenceEqual(cursors.OrderBy(item => item.Key));
|
||||
// A zero/zero response is an empty platform checkpoint, not evidence
|
||||
// that a locally collected but not-yet-ACKed batch should be dropped.
|
||||
if (unchanged || current.ConfirmedSequence == 0 && remote.ConfirmedSequence == 0 && current.ConfirmedCursors.Count == 0 && cursors.Count == 0)
|
||||
return false;
|
||||
items[key] = new RemoteDataSyncCheckpoint(
|
||||
accountId,
|
||||
streamKey,
|
||||
remote.SourceGeneration,
|
||||
remote.ConfirmedSequence,
|
||||
cursors,
|
||||
null,
|
||||
remote.LastSuccessAt,
|
||||
remote.State,
|
||||
[]);
|
||||
SaveLocked();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveLocked()
|
||||
{
|
||||
var directory = Path.GetDirectoryName(path) ?? AppContext.BaseDirectory;
|
||||
Directory.CreateDirectory(directory);
|
||||
var temporary = path + ".tmp-" + Guid.NewGuid().ToString("N");
|
||||
try
|
||||
{
|
||||
var state = new StateFile { Items = items.Values.OrderBy(item => item.AccountId).ThenBy(item => item.StreamKey).ToList() };
|
||||
File.WriteAllText(temporary, JsonSerializer.Serialize(state, RemoteJson.Options), Encoding.UTF8);
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
try { File.SetUnixFileMode(temporary, UnixFileMode.UserRead | UnixFileMode.UserWrite); }
|
||||
catch (PlatformNotSupportedException) { }
|
||||
}
|
||||
if (OperatingSystem.IsWindows() && File.Exists(path)) File.Replace(temporary, path, null);
|
||||
else File.Move(temporary, path, true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temporary)) File.Delete(temporary);
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<RemoteDataSyncCheckpoint> Load(string path)
|
||||
{
|
||||
if (!File.Exists(path)) return [];
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<StateFile>(File.ReadAllText(path), RemoteJson.Options)?.Items ?? [];
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or JsonException or NotSupportedException)
|
||||
{
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The data sync state could not be loaded; synchronization is blocked.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<string, long> ParseCursors(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
return new(StringComparer.Ordinal);
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<Dictionary<string, long>>(value, RemoteJson.Options) ?? new(StringComparer.Ordinal);
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The confirmed sync cursor is invalid.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static RemoteDataSyncCheckpoint NewCheckpoint(string accountId, string streamKey, string sourceGeneration) =>
|
||||
new(accountId, streamKey, sourceGeneration, 0, new Dictionary<string, long>(), null, null, "unknown", []);
|
||||
|
||||
private static string Key(string accountId, string streamKey) => $"{accountId}\u001f{streamKey}";
|
||||
}
|
||||
|
||||
public static class RemoteDataBatchAuthorization
|
||||
{
|
||||
public static string ComputePayloadHash(RemoteSyncBatch batch)
|
||||
{
|
||||
var canonical = batch with { PayloadHash = string.Empty };
|
||||
return Convert.ToHexString(SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(canonical, RemoteJson.Options))).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public static RemoteSyncBatch? Filter(ReportingConfig? config, RemoteSyncBatch batch, out string reason)
|
||||
{
|
||||
if (config is null || !config.Enabled || config.FindAccount(batch.AccountId) is not { Enabled: true })
|
||||
{
|
||||
reason = "AccountNotAuthorized";
|
||||
return null;
|
||||
}
|
||||
|
||||
var conversations = batch.Conversations.Where(conversation =>
|
||||
ReportingAuthorization.Check(config, batch.AccountId, conversation.ChatId, conversation.ChatType, ReportingDataType.TaskResult).Allowed).ToArray();
|
||||
var messages = batch.Messages.Where(message =>
|
||||
ReportingAuthorization.Check(config, batch.AccountId, message.ChatId, message.ChatType, ReportingDataType.Message).Allowed).ToArray();
|
||||
var dropped = conversations.Length != batch.Conversations.Count || messages.Length != batch.Messages.Count;
|
||||
reason = dropped ? "SomeRecordsDroppedByReporting" : "Authorized";
|
||||
if (conversations.Length == 0 && messages.Length == 0 && (batch.Conversations.Count > 0 || batch.Messages.Count > 0))
|
||||
{
|
||||
reason = "AllRecordsDroppedByReporting";
|
||||
return null;
|
||||
}
|
||||
return dropped ? batch with { CoverageState = "partial", Conversations = conversations, Messages = messages } : batch;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class RemoteDataBatchQueue
|
||||
{
|
||||
private sealed class QueueState
|
||||
{
|
||||
[JsonPropertyName("items")]
|
||||
public List<RemoteSyncBatch> Items { get; set; } = [];
|
||||
}
|
||||
|
||||
private readonly string path;
|
||||
private readonly int maxItems;
|
||||
private readonly long maxBytes;
|
||||
private readonly object gate = new();
|
||||
private QueueState state;
|
||||
|
||||
public RemoteDataBatchQueue(string path, int maxItems = RemoteDataProtocol.MaxBatchItems, long maxBytes = 16 * 1024 * 1024)
|
||||
{
|
||||
if (maxItems is < 1 or > 100_000) throw new ArgumentOutOfRangeException(nameof(maxItems));
|
||||
if (maxBytes is < 1 or > 512L * 1024 * 1024) throw new ArgumentOutOfRangeException(nameof(maxBytes));
|
||||
this.path = Path.GetFullPath(path);
|
||||
this.maxItems = maxItems;
|
||||
this.maxBytes = maxBytes;
|
||||
state = Load(this.path);
|
||||
}
|
||||
|
||||
public int PendingCount { get { lock (gate) return state.Items.Count; } }
|
||||
|
||||
public long PendingBytes
|
||||
{
|
||||
get { lock (gate) return state.Items.Sum(SerializedSize); }
|
||||
}
|
||||
|
||||
public RemoteDataBatchEnqueueResult Enqueue(ReportingConfig config, RemoteSyncBatch batch)
|
||||
{
|
||||
ValidateBatch(batch);
|
||||
var filtered = RemoteDataBatchAuthorization.Filter(config, batch, out var reason);
|
||||
if (filtered is null)
|
||||
return new RemoteDataBatchEnqueueResult(false, false, reason, null);
|
||||
var size = SerializedSize(filtered);
|
||||
if (size > RemoteDataProtocol.MaxBatchBytes)
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The sync batch is too large.");
|
||||
|
||||
lock (gate)
|
||||
{
|
||||
var existing = state.Items.FirstOrDefault(item => item.BatchId == filtered.BatchId);
|
||||
if (existing is not null)
|
||||
{
|
||||
if (!string.Equals(existing.PayloadHash, filtered.PayloadHash, StringComparison.Ordinal))
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The sync batch id was reused with a different payload.");
|
||||
return new RemoteDataBatchEnqueueResult(true, true, "AlreadyQueued", existing);
|
||||
}
|
||||
if (state.Items.Count >= maxItems || state.Items.Sum(SerializedSize) + size > maxBytes)
|
||||
return new RemoteDataBatchEnqueueResult(false, false, "SyncQueueFull", null);
|
||||
state.Items.Add(filtered);
|
||||
SaveLocked();
|
||||
return new RemoteDataBatchEnqueueResult(true, false, reason, filtered);
|
||||
}
|
||||
}
|
||||
|
||||
public IReadOnlyList<RemoteSyncBatch> Pending()
|
||||
{
|
||||
lock (gate) return state.Items.ToArray();
|
||||
}
|
||||
|
||||
public int DropForAccount(string accountId, string streamKey)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
var removed = state.Items.RemoveAll(item => string.Equals(item.AccountId, accountId, StringComparison.Ordinal)
|
||||
&& string.Equals(item.StreamKey, streamKey, StringComparison.Ordinal));
|
||||
if (removed > 0)
|
||||
SaveLocked();
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
||||
public bool MarkConfirmed(RemoteDataBatchAck acknowledgement)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
var index = state.Items.FindIndex(item => item.BatchId == acknowledgement.BatchId);
|
||||
if (index < 0) return false;
|
||||
if (!acknowledgement.Accepted || !string.Equals(state.Items[index].CursorEnd, acknowledgement.ConfirmedCursor, StringComparison.Ordinal))
|
||||
return false;
|
||||
state.Items.RemoveAt(index);
|
||||
SaveLocked();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Drop(string batchId)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
var removed = state.Items.RemoveAll(item => item.BatchId == batchId) > 0;
|
||||
if (removed) SaveLocked();
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveLocked()
|
||||
{
|
||||
var directory = Path.GetDirectoryName(path) ?? AppContext.BaseDirectory;
|
||||
Directory.CreateDirectory(directory);
|
||||
var temporary = path + ".tmp-" + Guid.NewGuid().ToString("N");
|
||||
try
|
||||
{
|
||||
File.WriteAllText(temporary, JsonSerializer.Serialize(state, RemoteJson.Options), Encoding.UTF8);
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
try { File.SetUnixFileMode(temporary, UnixFileMode.UserRead | UnixFileMode.UserWrite); }
|
||||
catch (PlatformNotSupportedException) { }
|
||||
}
|
||||
if (OperatingSystem.IsWindows() && File.Exists(path)) File.Replace(temporary, path, null);
|
||||
else File.Move(temporary, path, true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temporary)) File.Delete(temporary);
|
||||
}
|
||||
}
|
||||
|
||||
private static QueueState Load(string path)
|
||||
{
|
||||
if (!File.Exists(path)) return new QueueState();
|
||||
try
|
||||
{
|
||||
var result = JsonSerializer.Deserialize<QueueState>(File.ReadAllText(path), RemoteJson.Options) ?? new QueueState();
|
||||
result.Items ??= [];
|
||||
return result;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or JsonException or NotSupportedException or ArgumentException)
|
||||
{
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidOperationState, "The data sync queue could not be loaded; synchronization is blocked.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static long SerializedSize(RemoteSyncBatch batch) => JsonSerializer.SerializeToUtf8Bytes(batch, RemoteJson.Options).LongLength;
|
||||
|
||||
private static void ValidateBatch(RemoteSyncBatch batch)
|
||||
{
|
||||
foreach (var value in new[] { batch.NodeId, batch.AccountId, batch.BatchId, batch.SourceGeneration, batch.StreamKey, batch.PayloadHash })
|
||||
RemoteAgentOptions.ValidateIdentifier(value, "sync batch field", 512);
|
||||
if (batch.Sequence < 1 || batch.Conversations.Count + batch.Messages.Count > RemoteDataProtocol.MaxBatchMessages)
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The sync batch sequence or item count is invalid.");
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,12 @@ using WxAgent.Windows;
|
||||
|
||||
namespace WxAgent.Host;
|
||||
|
||||
public sealed class WindowsAgentBackend(AccountBindingStore bindings, ServiceOptions options) : IAgentBackend, IAgentEventSource
|
||||
public sealed class WindowsAgentBackend(AccountBindingStore bindings, ServiceOptions options) : IAgentBackend, IAgentEventSource, IRemoteDataCollector
|
||||
{
|
||||
private readonly bool validationOperationsEnabled = options.EnableValidationOperations;
|
||||
private readonly bool listenerEventsEnabled = options.EnableListenerEvents;
|
||||
private readonly SemaphoreSlim bindingGate = new(1, 1);
|
||||
private readonly DatabaseMessageSyncCollector databaseSyncCollector = new(null, options.DataSyncBatchLimit, options.DataSyncOverlapRows);
|
||||
|
||||
public IReadOnlyList<AgentCapability> Capabilities =>
|
||||
[
|
||||
@@ -439,6 +440,13 @@ public sealed class WindowsAgentBackend(AccountBindingStore bindings, ServiceOpt
|
||||
return new Page<GroupMemberInfo>(items, limit, offset, page.HasMore, page.NextOffset);
|
||||
}
|
||||
|
||||
public Task<RemoteDataCollection> CollectAsync(
|
||||
string accountId,
|
||||
IReadOnlyList<RemoteReportingScope> scopes,
|
||||
RemoteDataSyncCheckpoint checkpoint,
|
||||
CancellationToken cancellationToken) =>
|
||||
databaseSyncCollector.CollectAsync(accountId, scopes, checkpoint, cancellationToken);
|
||||
|
||||
public async Task<IReadOnlyList<DatabaseMessageInfo>> DatabaseMessagesAsync(string accountId, string chatId, int limit, long? localId, CancellationToken cancellationToken)
|
||||
{
|
||||
var account = await DatabaseKeyStore.LoadAsync(null, cancellationToken);
|
||||
|
||||
@@ -1,6 +1,25 @@
|
||||
namespace WxAgent.Service;
|
||||
|
||||
public sealed record Page<T>(IReadOnlyList<T> Items, int Limit, int Offset, bool HasMore, int? NextOffset);
|
||||
public static class ReadCoverageStates
|
||||
{
|
||||
public const string Complete = "complete";
|
||||
public const string Partial = "partial";
|
||||
public const string Unknown = "unknown";
|
||||
}
|
||||
|
||||
public sealed record ReadCoverage(
|
||||
string State,
|
||||
string Source,
|
||||
int ObservedCount,
|
||||
int AuthorizedScopeCount,
|
||||
int MatchedScopeCount,
|
||||
DateTimeOffset ObservedAt,
|
||||
string? ErrorCode = null);
|
||||
|
||||
public sealed record Page<T>(IReadOnlyList<T> Items, int Limit, int Offset, bool HasMore, int? NextOffset)
|
||||
{
|
||||
public ReadCoverage? Coverage { get; init; }
|
||||
}
|
||||
public sealed record AccountInfo(string AccountId, string? DisplayName, string? WechatId, string? Region, string DataFingerprint, bool IsUiBindingKnown, AccountBinding? Binding = null, string BindingStatus = "Unbound");
|
||||
public sealed record SessionInfo(string Name, string AutomationId, bool IsCurrent);
|
||||
public sealed record MessageInfo(string Fingerprint, string Type, string? Sender, string? Summary, string? Content);
|
||||
|
||||
@@ -10,7 +10,8 @@ public sealed class RemoteAgentHostedService(
|
||||
ServiceOptions options,
|
||||
IAgentBackend backend,
|
||||
ILogger<RemoteAgentHostedService> logger,
|
||||
RemoteEventQueue? remoteQueue = null) : BackgroundService
|
||||
RemoteEventQueue? remoteQueue = null,
|
||||
IRemoteDataCollector? dataCollector = null) : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan HeartbeatInterval = TimeSpan.FromSeconds(10);
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(2);
|
||||
@@ -41,6 +42,17 @@ public sealed class RemoteAgentHostedService(
|
||||
using var client = new RemoteControlClient(remote);
|
||||
var ledger = new RemoteTaskLedger(Path.Combine(options.DataDirectory, "remote-task-ledger.json"));
|
||||
var eventQueue = remoteQueue ?? new RemoteEventQueue(Path.Combine(options.DataDirectory, "remote-event-queue.json"));
|
||||
var dataQueue = options.EnableDataSync
|
||||
? new RemoteDataBatchQueue(Path.Combine(options.DataDirectory, "remote-data-queue.json"), options.DataSyncQueueMaxItems, options.DataSyncQueueMaxBytes)
|
||||
: null;
|
||||
var syncState = options.EnableDataSync
|
||||
? new RemoteDataSyncStateStore(Path.Combine(options.DataDirectory, "remote-data-sync-state.json"))
|
||||
: null;
|
||||
var lastDataSyncAt = DateTimeOffset.MinValue;
|
||||
string? registeredActiveAccountId = null;
|
||||
bool? registeredActiveAccountVerified = null;
|
||||
long registeredReportingConfigVersion = -1;
|
||||
ReportingConfig? activeReporting = null;
|
||||
var retry = RetryDelay;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -54,16 +66,43 @@ public sealed class RemoteAgentHostedService(
|
||||
continue;
|
||||
}
|
||||
var reporting = runtime.Reporting;
|
||||
activeReporting = reporting;
|
||||
var snapshot = await ReadSnapshotAsync(remote, stoppingToken);
|
||||
if (client.AuthState != RemoteAuthState.Authenticated)
|
||||
var registrationNeedsRefresh = client.AuthState != RemoteAuthState.Authenticated
|
||||
|| !string.Equals(registeredActiveAccountId, snapshot.ActiveAccountId, StringComparison.Ordinal)
|
||||
|| registeredActiveAccountVerified != snapshot.ActiveAccountVerified
|
||||
|| registeredReportingConfigVersion != reporting.ConfigVersion;
|
||||
if (registrationNeedsRefresh)
|
||||
{
|
||||
await client.RegisterAsync(CreateRegistration(remote, reporting, snapshot), stoppingToken);
|
||||
registeredActiveAccountId = snapshot.ActiveAccountId;
|
||||
registeredActiveAccountVerified = snapshot.ActiveAccountVerified;
|
||||
registeredReportingConfigVersion = reporting.ConfigVersion;
|
||||
retry = RetryDelay;
|
||||
}
|
||||
|
||||
await client.HeartbeatAsync(CreateHeartbeat(remote, reporting, snapshot, null), stoppingToken);
|
||||
await ReplayUnreportedResultsAsync(client, ledger, reporting, stoppingToken);
|
||||
await client.FlushEventsAsync(eventQueue, reporting, stoppingToken);
|
||||
if (dataQueue is not null && syncState is not null && dataCollector is not null
|
||||
&& DateTimeOffset.UtcNow - lastDataSyncAt >= TimeSpan.FromSeconds(options.DataSyncIntervalSeconds))
|
||||
{
|
||||
await ReconcileDataStatusAsync(client, reporting, dataQueue, syncState, stoppingToken);
|
||||
try
|
||||
{
|
||||
var confirmedBatches = await client.FlushDataBatchesAsync(dataQueue, reporting, stoppingToken);
|
||||
foreach (var confirmedBatch in confirmedBatches)
|
||||
syncState.MarkConfirmed(confirmedBatch);
|
||||
}
|
||||
catch (Exception exception) when (exception is HttpRequestException or RemoteClientException)
|
||||
{
|
||||
logger.LogWarning("Data sync delivery deferred; type={ExceptionType}.", exception.GetType().Name);
|
||||
}
|
||||
// Collection must continue while the platform is unreachable; the durable queue is
|
||||
// the offline buffer and will be flushed on the next successful connection.
|
||||
await CollectDataBatchesAsync(remote, reporting, dataCollector, dataQueue, syncState, stoppingToken);
|
||||
lastDataSyncAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
var pollAccountIds = reporting.Accounts
|
||||
.Where(account => account.Enabled)
|
||||
.Select(account => account.AccountId)
|
||||
@@ -94,18 +133,120 @@ public sealed class RemoteAgentHostedService(
|
||||
{
|
||||
logger.LogWarning("Remote control-plane request failed; code={Code}; status={StatusCode}; correlationId={CorrelationId}.",
|
||||
exception.Code, exception.StatusCode, exception.CorrelationId);
|
||||
await TryCollectOfflineDataBatchesAsync(remote, activeReporting, dataCollector, dataQueue, syncState, stoppingToken);
|
||||
await DelayAsync(retry, stoppingToken);
|
||||
retry = TimeSpan.FromSeconds(Math.Min(retry.TotalSeconds * 2, 30));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning("Remote agent cycle failed; type={ExceptionType}.", exception.GetType().Name);
|
||||
await TryCollectOfflineDataBatchesAsync(remote, activeReporting, dataCollector, dataQueue, syncState, stoppingToken);
|
||||
await DelayAsync(retry, stoppingToken);
|
||||
retry = TimeSpan.FromSeconds(Math.Min(retry.TotalSeconds * 2, 30));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CollectDataBatchesAsync(
|
||||
RemoteAgentOptions remote,
|
||||
ReportingConfig reporting,
|
||||
IRemoteDataCollector collector,
|
||||
RemoteDataBatchQueue queue,
|
||||
RemoteDataSyncStateStore state,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var account in reporting.Accounts.Where(item => item.Enabled))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var scopes = AuthorizedChatScopes(reporting, account.AccountId);
|
||||
if (scopes.Count == 0 || queue.Pending().Any(batch => string.Equals(batch.AccountId, account.AccountId, StringComparison.Ordinal)
|
||||
&& string.Equals(batch.StreamKey, "messages", StringComparison.Ordinal)))
|
||||
continue;
|
||||
|
||||
var checkpoint = state.GetLatest(account.AccountId, "messages");
|
||||
var collection = await collector.CollectAsync(account.AccountId, scopes, checkpoint, cancellationToken).ConfigureAwait(false);
|
||||
var sourceChanged = !string.Equals(checkpoint.SourceGeneration, collection.SourceGeneration, StringComparison.Ordinal);
|
||||
var cursorStart = sourceChanged ? new Dictionary<string, long>(StringComparer.Ordinal) : checkpoint.ConfirmedCursors;
|
||||
var cursorChanged = !cursorStart.OrderBy(item => item.Key).SequenceEqual(collection.NextCursors.OrderBy(item => item.Key));
|
||||
if (!sourceChanged && !cursorChanged && !collection.HasNewItems && checkpoint.ConfirmedSequence > 0)
|
||||
continue;
|
||||
|
||||
var sequence = sourceChanged ? 1 : checkpoint.ConfirmedSequence + 1;
|
||||
var batch = new RemoteSyncBatch(
|
||||
remote.NodeId!, account.AccountId, Guid.NewGuid().ToString("N"), collection.SourceGeneration,
|
||||
"messages", sequence, SerializeCursors(cursorStart), SerializeCursors(collection.NextCursors),
|
||||
string.Empty, collection.IsComplete ? "complete" : "partial", collection.Conversations, collection.Messages);
|
||||
batch = batch with { PayloadHash = RemoteDataBatchAuthorization.ComputePayloadHash(batch) };
|
||||
var queued = queue.Enqueue(reporting, batch);
|
||||
if (queued.Accepted && queued.Batch is not null)
|
||||
state.MarkCollected(queued.Batch, collection.UnavailableSources);
|
||||
if (!collection.IsComplete)
|
||||
logger.LogWarning("Data sync is partial; accountId={AccountId}; unavailableSources={UnavailableSourceCount}.", account.AccountId, collection.UnavailableSources.Count);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryCollectOfflineDataBatchesAsync(
|
||||
RemoteAgentOptions remote,
|
||||
ReportingConfig? reporting,
|
||||
IRemoteDataCollector? collector,
|
||||
RemoteDataBatchQueue? queue,
|
||||
RemoteDataSyncStateStore? state,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (reporting is null || collector is null || queue is null || state is null)
|
||||
return;
|
||||
try
|
||||
{
|
||||
await CollectDataBatchesAsync(remote, reporting, collector, queue, state, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning("Offline data collection deferred; type={ExceptionType}.", exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReconcileDataStatusAsync(
|
||||
RemoteControlClient client,
|
||||
ReportingConfig reporting,
|
||||
RemoteDataBatchQueue queue,
|
||||
RemoteDataSyncStateStore state,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var account in reporting.Accounts.Where(item => item.Enabled))
|
||||
{
|
||||
var local = state.GetLatest(account.AccountId, "messages");
|
||||
RemoteDataSyncStatus remote;
|
||||
try
|
||||
{
|
||||
remote = await client.GetDataSyncStatusAsync(account.AccountId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (RemoteClientException exception) when (exception.StatusCode == 404 && exception.Code == "NotFound")
|
||||
{
|
||||
// Older control planes do not expose the reconciliation endpoint; retain the legacy ACK path.
|
||||
continue;
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
// A disconnected platform must not prevent local DB collection.
|
||||
return;
|
||||
}
|
||||
|
||||
if (!state.Reconcile(remote, account.AccountId, "messages"))
|
||||
continue;
|
||||
var dropped = queue.DropForAccount(account.AccountId, "messages");
|
||||
logger.LogWarning(
|
||||
"Data sync checkpoint reconciled with the platform; accountId={AccountId}; localSequence={LocalSequence}; platformSequence={PlatformSequence}; droppedPendingBatches={DroppedPendingBatches}.",
|
||||
account.AccountId, local.ConfirmedSequence, remote.ConfirmedSequence, dropped);
|
||||
}
|
||||
}
|
||||
|
||||
private static string SerializeCursors(IReadOnlyDictionary<string, long> cursors) =>
|
||||
JsonSerializer.Serialize(cursors.OrderBy(item => item.Key).ToDictionary(item => item.Key, item => item.Value), RemoteJson.Options);
|
||||
|
||||
private async Task ProcessTaskAsync(
|
||||
RemoteControlClient client,
|
||||
RemoteAgentOptions remote,
|
||||
@@ -178,6 +319,15 @@ public sealed class RemoteAgentHostedService(
|
||||
}
|
||||
|
||||
var renewed = await client.RenewTaskAsync(started, cancellationToken);
|
||||
if (task.Kind == "sync-data")
|
||||
{
|
||||
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
|
||||
RemoteTaskStatus.Succeeded, null, "后台数据同步已受理；采集器将在下一轮同步周期执行。", false, null, Guid.NewGuid().ToString("N"));
|
||||
ledger.Complete(result);
|
||||
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (task.Kind == "send-text")
|
||||
{
|
||||
if (!TryReadSendTextPayload(task.Payload, out var targetId, out var text))
|
||||
@@ -271,7 +421,17 @@ public sealed class RemoteAgentHostedService(
|
||||
var visibleSessions = sessions
|
||||
.Where(session => scopes.Any(scope => SessionMatchesScope(session, scope, contacts)))
|
||||
.ToArray();
|
||||
var page = visibleSessions.ToPage(payload.Limit, payload.Offset);
|
||||
var matchedScopeCount = scopes.Count(scope => sessions.Any(session => SessionMatchesScope(session, scope, contacts)));
|
||||
var coverage = CreateCoverage(
|
||||
matchedScopeCount == scopes.Count
|
||||
? ReadCoverageStates.Complete
|
||||
: matchedScopeCount == 0 ? ReadCoverageStates.Unknown : ReadCoverageStates.Partial,
|
||||
"uia-session-list",
|
||||
sessions.Count,
|
||||
scopes.Count,
|
||||
matchedScopeCount,
|
||||
matchedScopeCount == 0 ? "ScopeIdentityNotObserved" : null);
|
||||
var page = visibleSessions.ToPage(payload.Limit, payload.Offset) with { Coverage = coverage };
|
||||
return new RemoteReadExecution(JsonSerializer.SerializeToElement(page, RemoteJson.Options), scopes);
|
||||
}
|
||||
case "read-contacts":
|
||||
@@ -294,7 +454,17 @@ public sealed class RemoteAgentHostedService(
|
||||
}
|
||||
var allowed = scopes.Select(scope => scope.ChatId).ToHashSet(StringComparer.Ordinal);
|
||||
var items = all.Where(contact => allowed.Contains(contact.Id)).ToArray();
|
||||
var resultPage = items.ToPage(payload.Limit, payload.Offset);
|
||||
var matchedScopeCount = scopes.Count(scope => all.Any(contact => string.Equals(contact.Id, scope.ChatId, StringComparison.Ordinal)));
|
||||
var coverage = CreateCoverage(
|
||||
matchedScopeCount == scopes.Count
|
||||
? ReadCoverageStates.Complete
|
||||
: matchedScopeCount == 0 ? ReadCoverageStates.Unknown : ReadCoverageStates.Partial,
|
||||
"uia-contact-list",
|
||||
all.Count,
|
||||
scopes.Count,
|
||||
matchedScopeCount,
|
||||
matchedScopeCount == 0 ? "ScopeIdentityNotObserved" : null);
|
||||
var resultPage = items.ToPage(payload.Limit, payload.Offset) with { Coverage = coverage };
|
||||
return new RemoteReadExecution(JsonSerializer.SerializeToElement(resultPage, RemoteJson.Options), scopes);
|
||||
}
|
||||
case "read-messages":
|
||||
@@ -326,7 +496,8 @@ public sealed class RemoteAgentHostedService(
|
||||
if (candidateSessions.Length != 1)
|
||||
throw new ServiceException("ChatIdentityUnconfirmed", 409, "The requested chat identity is not uniquely visible.");
|
||||
var messages = await backend.MessagesAsync(task.AccountId, candidateSessions[0].AutomationId, payload.IncludeContent, cancellationToken);
|
||||
var page = messages.ToPage(payload.Limit, payload.Offset);
|
||||
var coverage = CreateCoverage(ReadCoverageStates.Complete, "uia-session-messages", messages.Count, 1, 1);
|
||||
var page = messages.ToPage(payload.Limit, payload.Offset) with { Coverage = coverage };
|
||||
return new RemoteReadExecution(JsonSerializer.SerializeToElement(page, RemoteJson.Options), scopes);
|
||||
}
|
||||
default:
|
||||
@@ -405,6 +576,15 @@ public sealed class RemoteAgentHostedService(
|
||||
}
|
||||
}
|
||||
|
||||
private static ReadCoverage CreateCoverage(
|
||||
string state,
|
||||
string source,
|
||||
int observedCount,
|
||||
int authorizedScopeCount,
|
||||
int matchedScopeCount,
|
||||
string? errorCode = null) =>
|
||||
new(state, source, observedCount, authorizedScopeCount, matchedScopeCount, DateTimeOffset.UtcNow, errorCode);
|
||||
|
||||
private static void ValidatePage(int limit, int offset)
|
||||
{
|
||||
if (limit is < 1 or > 200 || offset < 0)
|
||||
@@ -527,12 +707,19 @@ public sealed class RemoteAgentHostedService(
|
||||
.Select(identity =>
|
||||
{
|
||||
var account = reporting.FindAccount(identity.AccountId);
|
||||
return new RemoteAccountSummary(
|
||||
var summary = new RemoteAccountSummary(
|
||||
identity.AccountId,
|
||||
string.Equals(identity.AccountId, snapshot.ActiveAccountId, StringComparison.Ordinal),
|
||||
identity.Verified,
|
||||
account?.AllowedChats.Count(chat => chat.Type == ReportingChatType.Group && chat.Enabled && chat.IdentityVerified) ?? 0,
|
||||
account?.AllowedChats.Count(chat => chat.Type == ReportingChatType.Private && chat.Enabled && chat.IdentityVerified) ?? 0);
|
||||
return summary with
|
||||
{
|
||||
AllowedChats = account?.AllowedChats
|
||||
.Where(chat => chat.Enabled && chat.IdentityVerified)
|
||||
.Select(chat => new RemoteAllowedChatSummary(chat.ChatId, chat.Type))
|
||||
.ToArray() ?? []
|
||||
};
|
||||
}).ToArray());
|
||||
|
||||
private static RemoteHeartbeat CreateHeartbeat(RemoteAgentOptions remote, ReportingConfig reporting, BackendSnapshot snapshot, string? errorCode) =>
|
||||
|
||||
@@ -45,6 +45,8 @@ public static class ServiceHost
|
||||
builder.Services.ConfigureHttpJsonOptions(o => o.SerializerOptions.UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow);
|
||||
builder.Services.AddSingleton(options);
|
||||
builder.Services.AddSingleton(backend);
|
||||
if (backend is IRemoteDataCollector dataCollector)
|
||||
builder.Services.AddSingleton<IRemoteDataCollector>(dataCollector);
|
||||
builder.Services.AddSingleton<ServiceSecurity>();
|
||||
builder.Services.AddSingleton<OperationStore>();
|
||||
builder.Services.AddSingleton<EventHub>();
|
||||
|
||||
@@ -27,6 +27,13 @@ public sealed class ServiceOptions
|
||||
public bool EnableValidationOperations { get; init; }
|
||||
public bool EnableListenerEvents { get; init; }
|
||||
public bool PreventAutoLock { get; init; }
|
||||
// Database-backed platform sync is opt-in until its source/account mapping has passed shadow validation.
|
||||
public bool EnableDataSync { get; init; }
|
||||
public int DataSyncIntervalSeconds { get; init; } = 5;
|
||||
public int DataSyncBatchLimit { get; init; } = 100;
|
||||
public int DataSyncOverlapRows { get; init; } = 1;
|
||||
public int DataSyncQueueMaxItems { get; init; } = 1000;
|
||||
public long DataSyncQueueMaxBytes { get; init; } = 16 * 1024 * 1024;
|
||||
|
||||
// Kept only so older service.json files can be loaded and rewritten by the tray.
|
||||
[JsonIgnore]
|
||||
@@ -48,6 +55,14 @@ public sealed class ServiceOptions
|
||||
throw new ArgumentException("External HTTP requires a concrete listen IP; do not use 0.0.0.0 or ::.");
|
||||
if (AccessToken is not null && !IsValidAccessToken(AccessToken))
|
||||
throw new ArgumentException("AccessToken must be non-empty and contain no whitespace.");
|
||||
if (DataSyncIntervalSeconds is < 1 or > 300)
|
||||
throw new ArgumentException("DataSyncIntervalSeconds must be between 1 and 300.");
|
||||
if (DataSyncBatchLimit is < 1 or > 500)
|
||||
throw new ArgumentException("DataSyncBatchLimit must be between 1 and 500.");
|
||||
if (DataSyncOverlapRows is < 0 or > 100)
|
||||
throw new ArgumentException("DataSyncOverlapRows must be between 0 and 100.");
|
||||
if (DataSyncQueueMaxItems is < 1 or > 100_000 || DataSyncQueueMaxBytes is < 1 or > 512L * 1024 * 1024)
|
||||
throw new ArgumentException("Data sync queue limits are out of range.");
|
||||
try
|
||||
{
|
||||
Remote?.Validate();
|
||||
|
||||
@@ -174,7 +174,14 @@ internal sealed class TrayApplicationContext : ApplicationContext
|
||||
Reporting = source.Reporting,
|
||||
RemoteConfigurationFile = source.RemoteConfigurationFile,
|
||||
EnableValidationOperations = source.EnableValidationOperations,
|
||||
PreventAutoLock = source.PreventAutoLock
|
||||
EnableListenerEvents = source.EnableListenerEvents,
|
||||
PreventAutoLock = source.PreventAutoLock,
|
||||
EnableDataSync = source.EnableDataSync,
|
||||
DataSyncIntervalSeconds = source.DataSyncIntervalSeconds,
|
||||
DataSyncBatchLimit = source.DataSyncBatchLimit,
|
||||
DataSyncOverlapRows = source.DataSyncOverlapRows,
|
||||
DataSyncQueueMaxItems = source.DataSyncQueueMaxItems,
|
||||
DataSyncQueueMaxBytes = source.DataSyncQueueMaxBytes
|
||||
};
|
||||
|
||||
private ServiceOptions ReadOptions() =>
|
||||
@@ -313,7 +320,13 @@ internal sealed class TrayApplicationContext : ApplicationContext
|
||||
RemoteConfigurationFile = null,
|
||||
EnableValidationOperations = source.EnableValidationOperations,
|
||||
EnableListenerEvents = source.EnableListenerEvents,
|
||||
PreventAutoLock = source.PreventAutoLock
|
||||
PreventAutoLock = source.PreventAutoLock,
|
||||
EnableDataSync = source.EnableDataSync,
|
||||
DataSyncIntervalSeconds = source.DataSyncIntervalSeconds,
|
||||
DataSyncBatchLimit = source.DataSyncBatchLimit,
|
||||
DataSyncOverlapRows = source.DataSyncOverlapRows,
|
||||
DataSyncQueueMaxItems = source.DataSyncQueueMaxItems,
|
||||
DataSyncQueueMaxBytes = source.DataSyncQueueMaxBytes
|
||||
};
|
||||
|
||||
private void SetStatus(string message)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using WxAgent.Core;
|
||||
|
||||
namespace WxAgent.Windows;
|
||||
|
||||
/// <summary>
|
||||
/// Collects authorized messages directly from verified read-only database keys.
|
||||
/// It never opens a WeChat window, changes the active account, or writes to the
|
||||
/// source database; unavailable shards are returned as an explicit partial result.
|
||||
/// </summary>
|
||||
public sealed class DatabaseMessageSyncCollector : IRemoteDataCollector
|
||||
{
|
||||
private readonly string? keyFile;
|
||||
private readonly int perChatLimit;
|
||||
private readonly int overlapRows;
|
||||
|
||||
public DatabaseMessageSyncCollector(string? keyFile = null, int perChatLimit = 100, int overlapRows = 1)
|
||||
{
|
||||
if (perChatLimit is < 1 or > 500) throw new ArgumentOutOfRangeException(nameof(perChatLimit));
|
||||
if (overlapRows is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(overlapRows));
|
||||
this.keyFile = keyFile;
|
||||
this.perChatLimit = perChatLimit;
|
||||
this.overlapRows = overlapRows;
|
||||
}
|
||||
|
||||
public async Task<RemoteDataCollection> CollectAsync(
|
||||
string accountId,
|
||||
IReadOnlyList<RemoteReportingScope> scopes,
|
||||
RemoteDataSyncCheckpoint checkpoint,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(accountId))
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "A database sync account id is required.");
|
||||
var accounts = await DatabaseKeyStore.LoadAsync(keyFile, cancellationToken).ConfigureAwait(false);
|
||||
var account = accounts.SingleOrDefault(item => string.Equals(item.AccountRootFingerprint, accountId, StringComparison.OrdinalIgnoreCase));
|
||||
if (account is null)
|
||||
throw new WxAgentException(WxAgentErrorCode.DatabaseKeyNotFound, "The verified database account does not match the requested sync account.");
|
||||
if (account.Databases.Any(database => database.Confidence != KeyBindingConfidence.PageHmacVerified))
|
||||
throw new WxAgentException(WxAgentErrorCode.DatabaseOpenFailed, "The account contains a database key without page-1 HMAC verification.");
|
||||
|
||||
// The verified account fingerprint is the stable source generation for v1;
|
||||
// a future key-cache version can deliberately rotate it for a rebind.
|
||||
var sourceGeneration = account.AccountRootFingerprint;
|
||||
var cursors = string.Equals(checkpoint.SourceGeneration, sourceGeneration, StringComparison.Ordinal)
|
||||
? checkpoint.ConfirmedCursors
|
||||
: new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
var chatScopes = scopes.DistinctBy(scope => $"{scope.ChatType}:{scope.ChatId}").ToArray();
|
||||
var chatIds = chatScopes.Select(scope => scope.ChatId).Distinct(StringComparer.Ordinal).ToArray();
|
||||
var page = await WechatMessageDbReader.ReadIncrementalAsync(
|
||||
account.AccountRootPath, account.Databases, chatIds, cursors, perChatLimit, cancellationToken, overlapRows).ConfigureAwait(false);
|
||||
var scopeByChat = chatScopes
|
||||
.GroupBy(scope => scope.ChatId, StringComparer.Ordinal)
|
||||
.ToDictionary(group => group.Key, group => group.First(), StringComparer.Ordinal);
|
||||
var observedAt = DateTimeOffset.UtcNow;
|
||||
var conversations = chatScopes.Select(scope => new RemoteSyncConversation(
|
||||
scope.ChatId,
|
||||
scope.ChatType,
|
||||
scope.ChatId,
|
||||
null,
|
||||
"authorized-db-scope",
|
||||
observedAt,
|
||||
"observed")).ToArray();
|
||||
var messages = page.Messages.Select(message =>
|
||||
{
|
||||
var scope = scopeByChat.GetValueOrDefault(message.ChatId)
|
||||
?? new RemoteReportingScope(message.ChatId, ReportingChatType.Private);
|
||||
var sourceMessageId = message.LocalId.ToString(CultureInfo.InvariantCulture);
|
||||
var messageId = message.ServerId > 0
|
||||
? $"{message.ChatId}:server:{message.ServerId.ToString(CultureInfo.InvariantCulture)}"
|
||||
: $"{message.ChatId}:local:{sourceMessageId}";
|
||||
var text = message.Content ?? string.Empty;
|
||||
return new RemoteSyncMessage(
|
||||
messageId,
|
||||
message.ChatId,
|
||||
scope.ChatType,
|
||||
sourceMessageId,
|
||||
message.IsSelf == true ? "outgoing" : "incoming",
|
||||
message.Type.ToString(CultureInfo.InvariantCulture),
|
||||
text,
|
||||
message.Timestamp,
|
||||
observedAt,
|
||||
account.WechatVersion ?? "unknown",
|
||||
HashMessage(messageId, text, message.Timestamp));
|
||||
}).ToArray();
|
||||
return new RemoteDataCollection(sourceGeneration, page.NextCursors, conversations, messages, page.IsComplete, page.NewMessageCount > 0, page.UnavailableDatabases);
|
||||
}
|
||||
|
||||
private static string HashMessage(string messageId, string text, DateTimeOffset sourceTime)
|
||||
{
|
||||
var canonical = JsonSerializer.Serialize(new { messageId, text, sourceTime = sourceTime.ToUniversalTime().ToString("O") });
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,13 @@ using WxAgent.Core;
|
||||
|
||||
namespace WxAgent.Windows;
|
||||
|
||||
public sealed record DbMessageIncrementalPage(
|
||||
IReadOnlyList<DbMessage> Messages,
|
||||
IReadOnlyDictionary<string, long> NextCursors,
|
||||
IReadOnlyList<string> UnavailableDatabases,
|
||||
bool IsComplete,
|
||||
int NewMessageCount);
|
||||
|
||||
/// <summary>Reads chat messages from the account's SQLCipher message databases using cached verified keys.</summary>
|
||||
public static class WechatMessageDbReader
|
||||
{
|
||||
@@ -14,7 +21,7 @@ public static class WechatMessageDbReader
|
||||
if (limit is < 1 or > 500) throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "Message limit must be between 1 and 500.");
|
||||
|
||||
// AccountRootPath points at the account's db_storage folder; the directory above it is <wxid>_<random>.
|
||||
var accountRootDirectoryName = Path.GetFileName(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(accountRootPath))));
|
||||
var accountRootDirectoryName = Path.GetFileName(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(accountRootPath)))) ?? string.Empty;
|
||||
|
||||
var tableName = WechatDbMessage.TableNameFor(chatId);
|
||||
if (!tableName.StartsWith("Msg_", StringComparison.Ordinal) || tableName.Length != 36 || !tableName[4..].All(Uri.IsHexDigit))
|
||||
@@ -39,6 +46,119 @@ public static class WechatMessageDbReader
|
||||
.Where(sender => !string.IsNullOrEmpty(sender)).Cast<string>().Distinct(StringComparer.Ordinal).ToArray();
|
||||
var contacts = await ReadContactsAsync(accountRootPath, databases, senders, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return BuildMessages(rows, chatId, accountRootDirectoryName, contacts);
|
||||
}
|
||||
|
||||
public static async Task<DbMessageIncrementalPage> ReadIncrementalAsync(
|
||||
string accountRootPath,
|
||||
IReadOnlyList<DatabaseKeyEvidence> databases,
|
||||
IReadOnlyList<string> chatIds,
|
||||
IReadOnlyDictionary<string, long> cursors,
|
||||
int perChatLimit,
|
||||
CancellationToken cancellationToken,
|
||||
int overlapRows = 1)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(accountRootPath);
|
||||
if (chatIds.Count == 0) return new DbMessageIncrementalPage([], cursors, [], true, 0);
|
||||
if (perChatLimit is < 1 or > 500) throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "Incremental message limit must be between 1 and 500.");
|
||||
|
||||
var accountRootDirectoryName = Path.GetFileName(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(accountRootPath)))) ?? string.Empty;
|
||||
var candidates = databases
|
||||
.Where(database => database.RelativePath.StartsWith("message/", StringComparison.OrdinalIgnoreCase)
|
||||
&& database.RelativePath.EndsWith(".db", StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(database => database.RelativePath, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
if (candidates.Length == 0)
|
||||
return new DbMessageIncrementalPage([], cursors, ["message/*.db"], false, 0);
|
||||
|
||||
var rowsByChat = new List<(string ChatId, string RelativePath, IReadOnlyList<IReadOnlyDictionary<string, object?>> Rows)>();
|
||||
var unavailable = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var chatId in chatIds.Distinct(StringComparer.Ordinal))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var tableName = WechatDbMessage.TableNameFor(chatId);
|
||||
if (!tableName.StartsWith("Msg_", StringComparison.Ordinal) || tableName.Length != 36 || !tableName[4..].All(Uri.IsHexDigit))
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "Message table name could not be derived from the chat id.");
|
||||
foreach (var database in candidates)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = CursorKey(chatId, database.RelativePath);
|
||||
var cursor = cursors.GetValueOrDefault(key);
|
||||
var queryCursor = cursor > 0 ? Math.Max(0, cursor - Math.Max(0, overlapRows)) : 0;
|
||||
var path = DatabasePath(accountRootPath, database.RelativePath);
|
||||
try
|
||||
{
|
||||
var tableRows = await SqlCipherDatabaseReader.QueryRowsAsync(path, database.EncKey,
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = $name;",
|
||||
[new KeyValuePair<string, object?>("$name", tableName)], cancellationToken).ConfigureAwait(false);
|
||||
if (tableRows.Count == 0) continue;
|
||||
var sql = "SELECT m.local_id, m.server_id, m.local_type, m.create_time, " +
|
||||
"hex(m.message_content) AS hex_content, m.WCDB_CT_message_content AS is_compressed, n.user_name AS sender_wxid " +
|
||||
$"FROM \"{tableName}\" m LEFT JOIN Name2Id n ON m.real_sender_id = n.rowid " +
|
||||
$"WHERE m.local_id {(cursor > 0 ? ">=" : ">")} $afterLocalId ORDER BY m.local_id ASC LIMIT $limit;";
|
||||
var rows = await SqlCipherDatabaseReader.QueryRowsAsync(path, database.EncKey, sql,
|
||||
[new KeyValuePair<string, object?>("$afterLocalId", queryCursor), new KeyValuePair<string, object?>("$limit", (long)perChatLimit)], cancellationToken).ConfigureAwait(false);
|
||||
rowsByChat.Add((chatId, database.RelativePath, rows));
|
||||
}
|
||||
catch (OperationCanceledException) { throw; }
|
||||
catch (WxAgentException)
|
||||
{
|
||||
unavailable.Add(database.RelativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var allSenders = rowsByChat.SelectMany(item => item.Rows)
|
||||
.Select(row => row.GetValueOrDefault("sender_wxid") as string)
|
||||
.Where(sender => !string.IsNullOrWhiteSpace(sender)).Cast<string>()
|
||||
.Distinct(StringComparer.Ordinal).ToArray();
|
||||
IReadOnlyDictionary<string, WechatContactName> contacts;
|
||||
try
|
||||
{
|
||||
contacts = await ReadContactsAsync(accountRootPath, databases, allSenders, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (WxAgentException)
|
||||
{
|
||||
contacts = new Dictionary<string, WechatContactName>(StringComparer.Ordinal);
|
||||
unavailable.Add("contact/contact.db");
|
||||
}
|
||||
|
||||
var nextCursors = new Dictionary<string, long>(cursors, StringComparer.Ordinal);
|
||||
var messages = new List<DbMessage>();
|
||||
var newMessageCount = 0;
|
||||
foreach (var group in rowsByChat)
|
||||
{
|
||||
messages.AddRange(BuildMessages(group.Rows, group.ChatId, accountRootDirectoryName, contacts));
|
||||
var cursorKey = CursorKey(group.ChatId, group.RelativePath);
|
||||
var previousCursor = cursors.GetValueOrDefault(cursorKey);
|
||||
newMessageCount += group.Rows.Count(row => Convert.ToInt64(row.GetValueOrDefault("local_id") ?? 0L, CultureInfo.InvariantCulture) > previousCursor);
|
||||
if (group.Rows.Count > 0)
|
||||
{
|
||||
var next = group.Rows.Max(row => Convert.ToInt64(row.GetValueOrDefault("local_id") ?? 0L, CultureInfo.InvariantCulture));
|
||||
nextCursors[CursorKey(group.ChatId, group.RelativePath)] = Math.Max(nextCursors.GetValueOrDefault(CursorKey(group.ChatId, group.RelativePath)), next);
|
||||
}
|
||||
}
|
||||
return new DbMessageIncrementalPage(messages.OrderBy(message => message.Timestamp).ThenBy(message => message.LocalId).ToArray(), nextCursors, unavailable.OrderBy(item => item, StringComparer.OrdinalIgnoreCase).ToArray(), unavailable.Count == 0, newMessageCount);
|
||||
}
|
||||
|
||||
public static string CursorKey(string chatId, string databaseRelativePath) => $"{chatId}\u001f{databaseRelativePath.Replace('\\', '/')}";
|
||||
|
||||
private static string DatabasePath(string accountRootPath, string relativePath)
|
||||
{
|
||||
var root = Path.GetFullPath(accountRootPath).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
|
||||
var relative = relativePath.Replace('/', Path.DirectorySeparatorChar);
|
||||
var path = Path.GetFullPath(Path.Combine(root, relative));
|
||||
if (Path.IsPathRooted(relative) || !path.StartsWith(root, StringComparison.OrdinalIgnoreCase))
|
||||
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The database path must remain inside the selected account root.");
|
||||
return path;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<DbMessage> BuildMessages(
|
||||
IReadOnlyList<IReadOnlyDictionary<string, object?>> rows,
|
||||
string chatId,
|
||||
string accountRootDirectoryName,
|
||||
IReadOnlyDictionary<string, WechatContactName> contacts)
|
||||
{
|
||||
var messages = new List<DbMessage>(rows.Count);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
@@ -59,7 +179,6 @@ public static class WechatMessageDbReader
|
||||
DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(row.GetValueOrDefault("create_time") ?? 0L, CultureInfo.InvariantCulture)),
|
||||
WechatDbMessage.IsSelf(senderWxId, accountRootDirectoryName)));
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user