Files
wx-win-agent/node-agent/WxAgent.Service/RemoteAgentHostedService.cs
T
rogee 13c31fc902
Build web service image / build (push) Successful in 1m53s
feat: add remote control plane and whitelist reads
2026-09-12 09:46:05 +08:00

493 lines
25 KiB
C#

using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using WxAgent.Core;
namespace WxAgent.Service;
public sealed class RemoteAgentHostedService(
ServiceOptions options,
IAgentBackend backend,
ILogger<RemoteAgentHostedService> logger,
RemoteEventQueue? remoteQueue = null) : BackgroundService
{
private static readonly TimeSpan HeartbeatInterval = TimeSpan.FromSeconds(10);
private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(2);
private readonly RemoteAccountContext accountContext = new();
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
RemoteAgentOptions? configuredRemote = options.Remote;
if (!string.IsNullOrWhiteSpace(options.RemoteConfigurationFile))
{
try
{
configuredRemote = (await RemoteNodeConfigurationStore.LoadAsync(options.RemoteConfigurationFile, stoppingToken)).Remote;
}
catch (Exception exception) when (exception is IOException or JsonException or WxAgentException)
{
logger.LogWarning("Remote agent is disabled because the external configuration could not be loaded; type={ExceptionType}.", exception.GetType().Name);
return;
}
}
if (configuredRemote is null || !configuredRemote.IsConfigured)
{
logger.LogInformation("Remote agent is disabled because authAddress, token or nodeId is not configured.");
return;
}
var remote = configuredRemote;
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
using var client = new RemoteControlClient(remote, http);
var ledger = new RemoteTaskLedger(Path.Combine(options.DataDirectory, "remote-task-ledger.json"));
var eventQueue = remoteQueue ?? new RemoteEventQueue(Path.Combine(options.DataDirectory, "remote-event-queue.json"));
var retry = RetryDelay;
while (!stoppingToken.IsCancellationRequested)
{
try
{
var runtime = await LoadRuntimeConfigurationAsync(options, stoppingToken);
if (!runtime.Remote.IsConfigured || !runtime.Remote.Equals(remote))
{
logger.LogWarning("Remote configuration changed or is not valid; remote work is paused until the service is restarted.");
await DelayAsync(RetryDelay, stoppingToken);
continue;
}
var reporting = runtime.Reporting;
var snapshot = await ReadSnapshotAsync(remote, stoppingToken);
if (client.AuthState != RemoteAuthState.Authenticated)
{
await client.RegisterAsync(CreateRegistration(remote, reporting, snapshot), stoppingToken);
retry = RetryDelay;
}
await client.HeartbeatAsync(CreateHeartbeat(remote, reporting, snapshot, null), stoppingToken);
await ReplayUnreportedResultsAsync(client, ledger, reporting, stoppingToken);
await client.FlushEventsAsync(eventQueue, reporting, stoppingToken);
foreach (var account in reporting.Accounts.Where(account => account.Enabled))
{
var tasks = await client.PollTasksAsync(account.AccountId, stoppingToken, waitSeconds: 5);
foreach (var task in tasks)
{
await ProcessTaskAsync(client, remote, reporting, ledger, task, stoppingToken);
}
}
await Task.Delay(HeartbeatInterval, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (RemoteClientException exception)
{
logger.LogWarning("Remote control-plane request failed; code={Code}; status={StatusCode}; correlationId={CorrelationId}.",
exception.Code, exception.StatusCode, exception.CorrelationId);
await DelayAsync(retry, stoppingToken);
retry = TimeSpan.FromSeconds(Math.Min(retry.TotalSeconds * 2, 30));
}
catch (Exception exception)
{
logger.LogWarning("Remote agent cycle failed; type={ExceptionType}.", exception.GetType().Name);
await DelayAsync(retry, stoppingToken);
retry = TimeSpan.FromSeconds(Math.Min(retry.TotalSeconds * 2, 30));
}
}
}
private async Task ProcessTaskAsync(
RemoteControlClient client,
RemoteAgentOptions remote,
ReportingConfig reporting,
RemoteTaskLedger ledger,
RemoteTaskEnvelope task,
CancellationToken cancellationToken)
{
if (task.Status is RemoteTaskStatus.Accepted or RemoteTaskStatus.Running)
{
if (ledger.TryGet(task.TaskId, out var existing) && existing?.Result is not null)
{
await ReportResultAsync(client, ledger, existing.Result, reporting, cancellationToken);
}
else
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, task.LeaseGeneration,
RemoteTaskStatus.ResultUnconfirmed, "AgentRestartedWithIncompleteTask", "Execution was not replayed.", true, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
}
return;
}
if (task.Status != RemoteTaskStatus.Pending)
return;
if (!ledger.Accept(task))
{
if (ledger.TryGet(task.TaskId, out var existing) && existing?.Result is not null)
await ReportResultAsync(client, ledger, existing.Result, reporting, cancellationToken);
return;
}
var accepted = await client.AcknowledgeTaskAsync(task, cancellationToken);
ledger.MarkAccepted(accepted);
if (accepted.Status == RemoteTaskStatus.Cancelled || accepted.CancelRequestedAt is not null)
{
var result = new RemoteTaskResult(accepted.TaskId, accepted.AccountId, accepted.LeaseGeneration,
RemoteTaskStatus.Cancelled, "CancelRequestedBeforeExecution", "Execution was not started.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
if (accepted.Status != RemoteTaskStatus.Cancelled)
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
else
ledger.MarkReported(result.TaskId);
return;
}
var currentSnapshot = await ReadSnapshotAsync(remote, cancellationToken);
if (!string.Equals(remote.ActiveAccountId, task.AccountId, StringComparison.Ordinal) || !currentSnapshot.ActiveAccountVerified)
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, task.LeaseGeneration,
RemoteTaskStatus.Failed, "AccountContextUnconfirmed", "The active account binding could not be confirmed.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
return;
}
// Persist the local running barrier before asking the center to mark the lease Running.
ledger.MarkRunning(accepted);
var started = await client.StartTaskAsync(accepted, cancellationToken);
if (started.Status != RemoteTaskStatus.Running)
{
var terminalStatus = started.Status is RemoteTaskStatus.Cancelled or RemoteTaskStatus.Expired
? started.Status
: RemoteTaskStatus.ResultUnconfirmed;
var result = new RemoteTaskResult(task.TaskId, task.AccountId, task.LeaseGeneration,
terminalStatus, "ExecutionStartUnconfirmed", "Execution was not started after lease validation.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
ledger.MarkReported(result.TaskId);
return;
}
var renewed = await client.RenewTaskAsync(started, cancellationToken);
if (task.Kind == "send-text")
{
if (!TryReadSendTextPayload(task.Payload, out var targetId, out var text))
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Failed, "InvalidTaskPayload", "The task payload did not pass the node command allowlist.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
return;
}
try
{
await backend.SendTextAsync(task.AccountId, targetId, text, cancellationToken);
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Succeeded, null, null, true, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (ServiceException exception)
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Failed, exception.Code, "The node operation failed.", true, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
}
catch (WxAgentException exception)
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Failed, exception.Code.ToString(), "The node operation failed.", true, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
}
return;
}
if (!IsReadTaskKind(task.Kind))
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Failed, "UnsupportedTask", "The task kind is not enabled on this node.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
return;
}
try
{
var read = await ExecuteReadTaskAsync(task, reporting, cancellationToken);
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Succeeded, null, null, false, read.Content, Guid.NewGuid().ToString("N"));
ledger.Complete(result, read.ChatScopes);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken, chatScopes: read.ChatScopes);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (ServiceException exception)
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Failed, exception.Code, "The node read operation failed.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
}
catch (WxAgentException exception)
{
var result = new RemoteTaskResult(task.TaskId, task.AccountId, renewed.LeaseGeneration,
RemoteTaskStatus.Failed, exception.Code.ToString(), "The node read operation failed.", false, null, Guid.NewGuid().ToString("N"));
ledger.Complete(result);
await ReportResultAsync(client, ledger, result, reporting, cancellationToken);
}
}
private async Task<RemoteReadExecution> ExecuteReadTaskAsync(
RemoteTaskEnvelope task, ReportingConfig reporting, CancellationToken cancellationToken)
{
switch (task.Kind)
{
case "read-sessions":
{
var payload = ReadPayload<ReadSessionsPayload>(task.Payload);
ValidatePage(payload.Limit, payload.Offset);
var scopes = AuthorizedChatScopes(reporting, task.AccountId);
RequireScopes(scopes);
var sessions = (await backend.SessionsAsync(task.AccountId, cancellationToken))
.Where(session => scopes.Any(scope => scope.ChatId == session.AutomationId))
.ToArray();
var page = sessions.ToPage(payload.Limit, payload.Offset);
return new RemoteReadExecution(JsonSerializer.SerializeToElement(page, RemoteJson.Options), scopes);
}
case "read-contacts":
{
var payload = ReadPayload<ReadContactsPayload>(task.Payload);
ValidatePage(payload.Limit, payload.Offset);
var chatType = payload.GroupsOnly ? ReportingChatType.Group : ReportingChatType.Private;
var scopes = AuthorizedChatScopes(reporting, task.AccountId, chatType);
RequireScopes(scopes);
var all = new List<ContactInfo>();
for (var offset = 0; ;)
{
var page = await backend.ContactsAsync(task.AccountId, payload.Contains, payload.GroupsOnly, 200, offset, cancellationToken);
all.AddRange(page.Items);
if (!page.HasMore) break;
var next = page.NextOffset ?? offset + page.Items.Count;
if (next <= offset)
throw new ServiceException("InvalidPage", 500, "The node returned a non-advancing contact page.");
offset = next;
}
var allowed = scopes.Select(scope => scope.ChatId).ToHashSet(StringComparer.Ordinal);
var items = all.Where(contact => allowed.Contains(contact.Id)).ToArray();
var resultPage = items.ToPage(payload.Limit, payload.Offset);
return new RemoteReadExecution(JsonSerializer.SerializeToElement(resultPage, RemoteJson.Options), scopes);
}
case "read-messages":
{
var payload = ReadPayload<ReadMessagesPayload>(task.Payload);
ValidatePage(payload.Limit, payload.Offset);
var scopes = AuthorizedChatScopes(reporting, task.AccountId)
.Where(scope => string.Equals(scope.ChatId, payload.ChatId, StringComparison.Ordinal))
.ToArray();
if (scopes.Length != 1)
throw new ServiceException("ChatNotAuthorized", 403, "The requested chat is not enabled in the local whitelist.");
var sessions = (await backend.SessionsAsync(task.AccountId, cancellationToken))
.Where(session => string.Equals(session.AutomationId, payload.ChatId, StringComparison.Ordinal))
.ToArray();
if (sessions.Length != 1)
throw new ServiceException("ChatIdentityUnconfirmed", 409, "The requested chat identity is not uniquely visible.");
var messages = await backend.MessagesAsync(task.AccountId, sessions[0].Name, payload.IncludeContent, cancellationToken);
var page = messages.ToPage(payload.Limit, payload.Offset);
return new RemoteReadExecution(JsonSerializer.SerializeToElement(page, RemoteJson.Options), scopes);
}
default:
throw new ServiceException("UnsupportedTask", 400, "The task kind is not a read operation.");
}
}
private static IReadOnlyList<RemoteReportingScope> AuthorizedChatScopes(
ReportingConfig reporting, string accountId, ReportingChatType? chatType = null) =>
(reporting.FindAccount(accountId)?.AllowedChats ?? [])
.Where(chat => chat.Enabled && chat.IdentityVerified && (chatType is null || chat.Type == chatType))
.Select(chat => new RemoteReportingScope(chat.ChatId, chat.Type))
.Where(scope => ReportingAuthorization.IsAllowed(reporting, accountId, scope.ChatId, scope.ChatType, ReportingDataType.TaskResult))
.Distinct()
.ToArray();
private static void RequireScopes(IReadOnlyList<RemoteReportingScope> scopes)
{
if (scopes.Count == 0)
throw new ServiceException("ChatNotAuthorized", 403, "The requested data scope is not enabled in the local whitelist.");
}
private static T ReadPayload<T>(JsonElement payload)
{
try
{
return payload.Deserialize<T>(RemoteJson.Options)
?? throw new ServiceException("InvalidTaskPayload", 400, "The read task payload is empty.");
}
catch (JsonException)
{
throw new ServiceException("InvalidTaskPayload", 400, "The read task payload is invalid.");
}
}
private static void ValidatePage(int limit, int offset)
{
if (limit is < 1 or > 200 || offset < 0)
throw new ServiceException("InvalidPagination", 400, "limit must be 1..200 and offset must be non-negative.");
}
private static bool IsReadTaskKind(string kind) => kind is "read-sessions" or "read-contacts" or "read-messages";
private sealed record ReadSessionsPayload(
[property: JsonPropertyName("limit")] int Limit,
[property: JsonPropertyName("offset")] int Offset);
private sealed record ReadContactsPayload(
[property: JsonPropertyName("limit")] int Limit,
[property: JsonPropertyName("offset")] int Offset,
[property: JsonPropertyName("contains")] string? Contains,
[property: JsonPropertyName("groups_only")] bool GroupsOnly);
private sealed record ReadMessagesPayload(
[property: JsonPropertyName("limit")] int Limit,
[property: JsonPropertyName("offset")] int Offset,
[property: JsonPropertyName("chat_id")] string ChatId,
[property: JsonPropertyName("include_content")] bool IncludeContent);
private sealed record RemoteReadExecution(
JsonElement Content,
IReadOnlyList<RemoteReportingScope> ChatScopes);
private static async Task<RemoteNodeConfiguration> LoadRuntimeConfigurationAsync(ServiceOptions options, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(options.RemoteConfigurationFile))
return new RemoteNodeConfiguration { Remote = options.Remote ?? new RemoteAgentOptions(), Reporting = options.Reporting };
return await RemoteNodeConfigurationStore.LoadAsync(options.RemoteConfigurationFile, cancellationToken);
}
private static async Task ReplayUnreportedResultsAsync(
RemoteControlClient client,
RemoteTaskLedger ledger,
ReportingConfig reporting,
CancellationToken cancellationToken)
{
foreach (var pending in ledger.UnreportedResultsWithScopes())
await ReportResultAsync(client, ledger, pending.Result, reporting, cancellationToken, pending.ReportingScopes);
}
private static async Task ReportResultAsync(
RemoteControlClient client,
RemoteTaskLedger ledger,
RemoteTaskResult result,
ReportingConfig reporting,
CancellationToken cancellationToken,
IReadOnlyList<RemoteReportingScope>? chatScopes = null)
{
await client.SendTaskResultAsync(result, reporting, chatScopes: chatScopes, cancellationToken: cancellationToken);
ledger.MarkReported(result.TaskId);
}
private async Task<BackendSnapshot> ReadSnapshotAsync(RemoteAgentOptions remote, CancellationToken cancellationToken)
{
try
{
var status = await backend.StatusAsync(cancellationToken);
var element = JsonSerializer.SerializeToElement(status, RemoteJson.Options);
var wechatRunning = GetBoolean(element, "wechatAvailable");
var sessionAvailable = GetBoolean(element, "sessionAvailable");
var sessionLocked = GetBoolean(element, "sessionLocked");
var accounts = await backend.AccountsAsync(cancellationToken);
var identities = accounts.Select(account => new RemoteAccountIdentity(
account.AccountId,
account.IsUiBindingKnown && account.Binding is not null && string.Equals(account.BindingStatus, "Bound", StringComparison.Ordinal))).ToArray();
var activeAccountVerified = false;
if (remote.ActiveAccountId is { Length: > 0 })
{
try
{
accountContext.SwitchTo(remote.ActiveAccountId, identities);
activeAccountVerified = accountContext.IsConfirmedFor(remote.ActiveAccountId);
}
catch (WxAgentException)
{
accountContext.Invalidate();
}
}
else
{
accountContext.Invalidate();
}
var nodeStatus = sessionLocked ? RemoteNodeStatus.SessionLocked
: !wechatRunning ? RemoteNodeStatus.WechatNotRunning
: !sessionAvailable ? RemoteNodeStatus.WechatNotLoggedIn
: !activeAccountVerified ? RemoteNodeStatus.Degraded
: RemoteNodeStatus.Online;
return new BackendSnapshot(nodeStatus, wechatRunning, sessionAvailable, sessionLocked, remote.ActiveAccountId, 0, activeAccountVerified, identities);
}
catch
{
return new BackendSnapshot(RemoteNodeStatus.Degraded, false, false, false, remote.ActiveAccountId, 0, false, []);
}
}
private static RemoteNodeRegistration CreateRegistration(RemoteAgentOptions remote, ReportingConfig reporting, BackendSnapshot snapshot) =>
new(remote.NodeId!, typeof(RemoteAgentHostedService).Assembly.GetName().Version?.ToString() ?? "dev",
RemoteProtocol.Version, ["heartbeat", "poll-tasks", "send-text", "read-sessions", "read-contacts", "read-messages", "report-message"], reporting.ConfigVersion,
snapshot.Accounts
.Where(identity => reporting.Accounts.Any(account => account.Enabled && string.Equals(account.AccountId, identity.AccountId, StringComparison.Ordinal)))
.Select(identity =>
{
var account = reporting.FindAccount(identity.AccountId)!;
return new RemoteAccountSummary(
identity.AccountId,
string.Equals(identity.AccountId, remote.ActiveAccountId, StringComparison.Ordinal),
identity.Verified,
account.AllowedChats.Count(chat => chat.Type == ReportingChatType.Group && chat.Enabled && chat.IdentityVerified),
account.AllowedChats.Count(chat => chat.Type == ReportingChatType.Private && chat.Enabled && chat.IdentityVerified));
}).ToArray());
private static RemoteHeartbeat CreateHeartbeat(RemoteAgentOptions remote, ReportingConfig reporting, BackendSnapshot snapshot, string? errorCode) =>
new(remote.NodeId!, typeof(RemoteAgentHostedService).Assembly.GetName().Version?.ToString() ?? "dev",
RemoteProtocol.Version, snapshot.Status, snapshot.WechatRunning, snapshot.WechatLoggedIn,
snapshot.SessionLocked, snapshot.ActiveAccountVerified ? snapshot.ActiveAccountId : null, snapshot.QueueLength, reporting.ConfigVersion,
Guid.NewGuid().ToString("N"), errorCode);
private static bool TryReadSendTextPayload(JsonElement payload, out string targetId, out string text)
{
targetId = "";
text = "";
if (payload.ValueKind != JsonValueKind.Object
|| !payload.TryGetProperty("target_id", out var target)
|| !payload.TryGetProperty("text", out var content)
|| !payload.TryGetProperty("confirmed", out var confirmed)
|| target.ValueKind != JsonValueKind.String || content.ValueKind != JsonValueKind.String
|| confirmed.ValueKind != JsonValueKind.True)
return false;
targetId = target.GetString() ?? "";
text = content.GetString() ?? "";
return targetId.Length is > 0 and <= 512 && text.Length is > 0 and <= 4000;
}
private static bool GetBoolean(JsonElement value, string propertyName) =>
value.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.True;
private static async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) =>
await Task.Delay(delay, cancellationToken);
private sealed record BackendSnapshot(
RemoteNodeStatus Status,
bool WechatRunning,
bool WechatLoggedIn,
bool SessionLocked,
string? ActiveAccountId,
int QueueLength,
bool ActiveAccountVerified,
IReadOnlyList<RemoteAccountIdentity> Accounts);
}