rogee/wx-win-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add account-scoped data synchronization

Browse Source
This commit is contained in:
rogee
2026-09-22 09:58:58 +08:00
parent ab9ff389f2
commit 72e040546a
37 changed files with 4252 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node-agent/WxAgent.Windows/DatabaseMessageSyncCollector.cs
+96
View File
@@ -0,0 +1,96 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using WxAgent.Core;
namespace WxAgent.Windows;
/// <summary>
/// Collects authorized messages directly from verified read-only database keys.
/// It never opens a WeChat window, changes the active account, or writes to the
/// source database; unavailable shards are returned as an explicit partial result.
/// </summary>
public sealed class DatabaseMessageSyncCollector : IRemoteDataCollector
{
private readonly string? keyFile;
private readonly int perChatLimit;
private readonly int overlapRows;
public DatabaseMessageSyncCollector(string? keyFile = null, int perChatLimit = 100, int overlapRows = 1)
{
if (perChatLimit is < 1 or > 500) throw new ArgumentOutOfRangeException(nameof(perChatLimit));
if (overlapRows is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(overlapRows));
this.keyFile = keyFile;
this.perChatLimit = perChatLimit;
this.overlapRows = overlapRows;
}
public async Task<RemoteDataCollection> CollectAsync(
string accountId,
IReadOnlyList<RemoteReportingScope> scopes,
RemoteDataSyncCheckpoint checkpoint,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(accountId))
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "A database sync account id is required.");
var accounts = await DatabaseKeyStore.LoadAsync(keyFile, cancellationToken).ConfigureAwait(false);
var account = accounts.SingleOrDefault(item => string.Equals(item.AccountRootFingerprint, accountId, StringComparison.OrdinalIgnoreCase));
if (account is null)
throw new WxAgentException(WxAgentErrorCode.DatabaseKeyNotFound, "The verified database account does not match the requested sync account.");
if (account.Databases.Any(database => database.Confidence != KeyBindingConfidence.PageHmacVerified))
throw new WxAgentException(WxAgentErrorCode.DatabaseOpenFailed, "The account contains a database key without page-1 HMAC verification.");
// The verified account fingerprint is the stable source generation for v1;
// a future key-cache version can deliberately rotate it for a rebind.
var sourceGeneration = account.AccountRootFingerprint;
var cursors = string.Equals(checkpoint.SourceGeneration, sourceGeneration, StringComparison.Ordinal)
? checkpoint.ConfirmedCursors
: new Dictionary<string, long>(StringComparer.Ordinal);
var chatScopes = scopes.DistinctBy(scope => $"{scope.ChatType}:{scope.ChatId}").ToArray();
var chatIds = chatScopes.Select(scope => scope.ChatId).Distinct(StringComparer.Ordinal).ToArray();
var page = await WechatMessageDbReader.ReadIncrementalAsync(
account.AccountRootPath, account.Databases, chatIds, cursors, perChatLimit, cancellationToken, overlapRows).ConfigureAwait(false);
var scopeByChat = chatScopes
.GroupBy(scope => scope.ChatId, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => group.First(), StringComparer.Ordinal);
var observedAt = DateTimeOffset.UtcNow;
var conversations = chatScopes.Select(scope => new RemoteSyncConversation(
scope.ChatId,
scope.ChatType,
scope.ChatId,
null,
"authorized-db-scope",
observedAt,
"observed")).ToArray();
var messages = page.Messages.Select(message =>
{
var scope = scopeByChat.GetValueOrDefault(message.ChatId)
?? new RemoteReportingScope(message.ChatId, ReportingChatType.Private);
var sourceMessageId = message.LocalId.ToString(CultureInfo.InvariantCulture);
var messageId = message.ServerId > 0
? $"{message.ChatId}:server:{message.ServerId.ToString(CultureInfo.InvariantCulture)}"
: $"{message.ChatId}:local:{sourceMessageId}";
var text = message.Content ?? string.Empty;
return new RemoteSyncMessage(
messageId,
message.ChatId,
scope.ChatType,
sourceMessageId,
message.IsSelf == true ? "outgoing" : "incoming",
message.Type.ToString(CultureInfo.InvariantCulture),
text,
message.Timestamp,
observedAt,
account.WechatVersion ?? "unknown",
HashMessage(messageId, text, message.Timestamp));
}).ToArray();
return new RemoteDataCollection(sourceGeneration, page.NextCursors, conversations, messages, page.IsComplete, page.NewMessageCount > 0, page.UnavailableDatabases);
}
private static string HashMessage(string messageId, string text, DateTimeOffset sourceTime)
{
var canonical = JsonSerializer.Serialize(new { messageId, text, sourceTime = sourceTime.ToUniversalTime().ToString("O") });
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant();
}
}
node-agent/WxAgent.Windows/WechatMessageDbReader.cs
+121 -2
View File
@@ -3,6 +3,13 @@ using WxAgent.Core;
namespace WxAgent.Windows;
public sealed record DbMessageIncrementalPage(
IReadOnlyList<DbMessage> Messages,
IReadOnlyDictionary<string, long> NextCursors,
IReadOnlyList<string> UnavailableDatabases,
bool IsComplete,
int NewMessageCount);
/// <summary>Reads chat messages from the account's SQLCipher message databases using cached verified keys.</summary>
public static class WechatMessageDbReader
{
@@ -14,7 +21,7 @@ public static class WechatMessageDbReader
if (limit is < 1 or > 500) throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "Message limit must be between 1 and 500.");
// AccountRootPath points at the account's db_storage folder; the directory above it is <wxid>_<random>.
var accountRootDirectoryName = Path.GetFileName(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(accountRootPath))));
var accountRootDirectoryName = Path.GetFileName(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(accountRootPath)))) ?? string.Empty;
var tableName = WechatDbMessage.TableNameFor(chatId);
if (!tableName.StartsWith("Msg_", StringComparison.Ordinal) || tableName.Length != 36 || !tableName[4..].All(Uri.IsHexDigit))
@@ -39,6 +46,119 @@ public static class WechatMessageDbReader
.Where(sender => !string.IsNullOrEmpty(sender)).Cast<string>().Distinct(StringComparer.Ordinal).ToArray();
var contacts = await ReadContactsAsync(accountRootPath, databases, senders, cancellationToken).ConfigureAwait(false);
return BuildMessages(rows, chatId, accountRootDirectoryName, contacts);
}
public static async Task<DbMessageIncrementalPage> ReadIncrementalAsync(
string accountRootPath,
IReadOnlyList<DatabaseKeyEvidence> databases,
IReadOnlyList<string> chatIds,
IReadOnlyDictionary<string, long> cursors,
int perChatLimit,
CancellationToken cancellationToken,
int overlapRows = 1)
{
ArgumentException.ThrowIfNullOrWhiteSpace(accountRootPath);
if (chatIds.Count == 0) return new DbMessageIncrementalPage([], cursors, [], true, 0);
if (perChatLimit is < 1 or > 500) throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "Incremental message limit must be between 1 and 500.");
var accountRootDirectoryName = Path.GetFileName(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(accountRootPath)))) ?? string.Empty;
var candidates = databases
.Where(database => database.RelativePath.StartsWith("message/", StringComparison.OrdinalIgnoreCase)
&& database.RelativePath.EndsWith(".db", StringComparison.OrdinalIgnoreCase))
.OrderBy(database => database.RelativePath, StringComparer.OrdinalIgnoreCase)
.ToArray();
if (candidates.Length == 0)
return new DbMessageIncrementalPage([], cursors, ["message/*.db"], false, 0);
var rowsByChat = new List<(string ChatId, string RelativePath, IReadOnlyList<IReadOnlyDictionary<string, object?>> Rows)>();
var unavailable = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var chatId in chatIds.Distinct(StringComparer.Ordinal))
{
cancellationToken.ThrowIfCancellationRequested();
var tableName = WechatDbMessage.TableNameFor(chatId);
if (!tableName.StartsWith("Msg_", StringComparison.Ordinal) || tableName.Length != 36 || !tableName[4..].All(Uri.IsHexDigit))
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "Message table name could not be derived from the chat id.");
foreach (var database in candidates)
{
cancellationToken.ThrowIfCancellationRequested();
var key = CursorKey(chatId, database.RelativePath);
var cursor = cursors.GetValueOrDefault(key);
var queryCursor = cursor > 0 ? Math.Max(0, cursor - Math.Max(0, overlapRows)) : 0;
var path = DatabasePath(accountRootPath, database.RelativePath);
try
{
var tableRows = await SqlCipherDatabaseReader.QueryRowsAsync(path, database.EncKey,
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = $name;",
[new KeyValuePair<string, object?>("$name", tableName)], cancellationToken).ConfigureAwait(false);
if (tableRows.Count == 0) continue;
var sql = "SELECT m.local_id, m.server_id, m.local_type, m.create_time, " +
"hex(m.message_content) AS hex_content, m.WCDB_CT_message_content AS is_compressed, n.user_name AS sender_wxid " +
$"FROM \"{tableName}\" m LEFT JOIN Name2Id n ON m.real_sender_id = n.rowid " +
$"WHERE m.local_id {(cursor > 0 ? ">=" : ">")} $afterLocalId ORDER BY m.local_id ASC LIMIT $limit;";
var rows = await SqlCipherDatabaseReader.QueryRowsAsync(path, database.EncKey, sql,
[new KeyValuePair<string, object?>("$afterLocalId", queryCursor), new KeyValuePair<string, object?>("$limit", (long)perChatLimit)], cancellationToken).ConfigureAwait(false);
rowsByChat.Add((chatId, database.RelativePath, rows));
}
catch (OperationCanceledException) { throw; }
catch (WxAgentException)
{
unavailable.Add(database.RelativePath);
}
}
}
var allSenders = rowsByChat.SelectMany(item => item.Rows)
.Select(row => row.GetValueOrDefault("sender_wxid") as string)
.Where(sender => !string.IsNullOrWhiteSpace(sender)).Cast<string>()
.Distinct(StringComparer.Ordinal).ToArray();
IReadOnlyDictionary<string, WechatContactName> contacts;
try
{
contacts = await ReadContactsAsync(accountRootPath, databases, allSenders, cancellationToken).ConfigureAwait(false);
}
catch (WxAgentException)
{
contacts = new Dictionary<string, WechatContactName>(StringComparer.Ordinal);
unavailable.Add("contact/contact.db");
}
var nextCursors = new Dictionary<string, long>(cursors, StringComparer.Ordinal);
var messages = new List<DbMessage>();
var newMessageCount = 0;
foreach (var group in rowsByChat)
{
messages.AddRange(BuildMessages(group.Rows, group.ChatId, accountRootDirectoryName, contacts));
var cursorKey = CursorKey(group.ChatId, group.RelativePath);
var previousCursor = cursors.GetValueOrDefault(cursorKey);
newMessageCount += group.Rows.Count(row => Convert.ToInt64(row.GetValueOrDefault("local_id") ?? 0L, CultureInfo.InvariantCulture) > previousCursor);
if (group.Rows.Count > 0)
{
var next = group.Rows.Max(row => Convert.ToInt64(row.GetValueOrDefault("local_id") ?? 0L, CultureInfo.InvariantCulture));
nextCursors[CursorKey(group.ChatId, group.RelativePath)] = Math.Max(nextCursors.GetValueOrDefault(CursorKey(group.ChatId, group.RelativePath)), next);
}
}
return new DbMessageIncrementalPage(messages.OrderBy(message => message.Timestamp).ThenBy(message => message.LocalId).ToArray(), nextCursors, unavailable.OrderBy(item => item, StringComparer.OrdinalIgnoreCase).ToArray(), unavailable.Count == 0, newMessageCount);
}
public static string CursorKey(string chatId, string databaseRelativePath) => $"{chatId}\u001f{databaseRelativePath.Replace('\\', '/')}";
private static string DatabasePath(string accountRootPath, string relativePath)
{
var root = Path.GetFullPath(accountRootPath).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var relative = relativePath.Replace('/', Path.DirectorySeparatorChar);
var path = Path.GetFullPath(Path.Combine(root, relative));
if (Path.IsPathRooted(relative) || !path.StartsWith(root, StringComparison.OrdinalIgnoreCase))
throw new WxAgentException(WxAgentErrorCode.InvalidArgument, "The database path must remain inside the selected account root.");
return path;
}
private static IReadOnlyList<DbMessage> BuildMessages(
IReadOnlyList<IReadOnlyDictionary<string, object?>> rows,
string chatId,
string accountRootDirectoryName,
IReadOnlyDictionary<string, WechatContactName> contacts)
{
var messages = new List<DbMessage>(rows.Count);
foreach (var row in rows)
{
@@ -59,7 +179,6 @@ public static class WechatMessageDbReader
DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(row.GetValueOrDefault("create_time") ?? 0L, CultureInfo.InvariantCulture)),
WechatDbMessage.IsSelf(senderWxId, accountRootDirectoryName)));
}
return messages;
}