admin/VolcanicAgenticCore
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(a2a-client): watches no longer hang on a silently dead task stream

Browse Source 
Hour-long remote tasks were never delivered: a subscribe watch held one SSE stream for up to a day, the remote
sent nothing while its agent worked without text, and once the quiet connection died on the way (no FIN/RST) the
watch waited on it with no error and no log line.

- A subscription lasts at most 5 minutes (SubscriptionSlice), then GetTask and a new subscription: a pause, not a
  failure. A slice ended mid-read counts as ours even when it surfaces as an I/O error.
- The shared handler connects with TCP keepalive (60 s, every 15 s, 4 probes) and HTTP/2 pings.
- Log lines when a watch starts or is restored, when a slice ends on a running task, when a result is delivered or
  kept; the Telegram host's console log carries a UTC timestamp.
- action=status on a settled task ends its watch, so the result isn't delivered a second time.
- A "context busy" refusal of send says nothing was sent, names the watched task in that context and that its
  result will arrive by itself; refusals never touch a watch.
This commit is contained in:
EugeneTes
2026-10-16 13:48:32 +00:00
parent 644f72068c
commit f42b00cb1c
9 changed files with 238 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
extensions/Volcanic.AgenticCore.Extensions.A2A/A2ATool.cs
View File

@@ -97,7 +97,9 @@ internal sealed class A2ATool(A2AClientSettings settings, string target, A2AWatc
return await CallAsync(entry, ct, async agent =>
{
SendMessageResponse response;
using (var bounded = Bounded(ct))
try
{
using var bounded = Bounded(ct);
response = await agent.Client.SendMessageAsync(new SendMessageRequest
{
Message = new A2AMessage
@@ -111,6 +113,11 @@ internal sealed class A2ATool(A2AClientSettings settings, string target, A2AWatc
Configuration = new SendMessageConfiguration { ReturnImmediately = true },
Metadata = new Dictionary<string, JsonElement> { ["volcanic.hops"] = JsonSerializer.SerializeToElement(inboundHops() + 1) },
}, bounded.Token).ConfigureAwait(false);
}
catch (A2AException ex) when (IsContextBusy(ex) && !string.IsNullOrWhiteSpace(args.ContextId))
{
throw new InvalidOperationException(BusyContext(entry, args.ContextId!), ex);
}
if (response.Task is not { } task)
{
@@ -150,12 +157,29 @@ internal sealed class A2ATool(A2AClientSettings settings, string target, A2AWatc
return await CallAsync(entry, ct, async agent =>
{
var task = await GetTaskAsync(agent, taskId, ct).ConfigureAwait(false);
return A2AResults.IsSettled(task.Status.State)
? await DescribeSettledAsync(workspace, entry.Name, agent, task, ct).ConfigureAwait(false)
: A2AResults.Describe(entry.Name, task, []);
if (!A2AResults.IsSettled(task.Status.State)) return A2AResults.Describe(entry.Name, task, []);
var described = await DescribeSettledAsync(workspace, entry.Name, agent, task, ct).ConfigureAwait(false);
// The outcome is in this conversation now: a watch of the task would only deliver it a second time.
await registry.StopWatchingAsync(target, workspace, taskId).ConfigureAwait(false);
return described;
}).ConfigureAwait(false);
}
/// <summary>The remote refuses a message to a context whose task still runs (our server and VolcanicAgents say "Context busy").</summary>
private static bool IsContextBusy(A2AException ex) =>
ex.ErrorCode == A2AErrorCode.UnsupportedOperation && ex.Message.Contains("busy", StringComparison.OrdinalIgnoreCase);
private string BusyContext(RemoteAgentEntry entry, string contextId)
{
var running = registry.List(target).FirstOrDefault(r => r.Agent == entry.Name && r.ContextId == contextId);
return running is null
? $"Agent \"{entry.Name}\" is still working on an earlier task in context {contextId} and refused this message. " +
"Nothing was sent. Send it again once that task has ended, or start a new conversation without contextId."
: $"Agent \"{entry.Name}\" is still working on task {running.TaskId} in context {contextId} and refused this message. " +
"Nothing was sent, and that task is still watched: its result will arrive here by itself. " +
"Send this message again after that result, or start a new conversation without contextId.";
}
private async Task<string> CancelAsync(string workspace, A2AArgs args, CancellationToken ct)
{
var entry = Entry(workspace, args.Agent);

66
extensions/Volcanic.AgenticCore.Extensions.A2A/A2AWatchRegistry.cs
View File

@@ -1,4 +1,6 @@
using System.Collections.Concurrent;
using System.Net;
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
@@ -25,15 +27,43 @@ namespace Volcanic.AgenticCore.Extensions.A2A;
/// </remarks>
public sealed class A2AWatchRegistry : IAsyncDisposable
{
public static A2AWatchRegistry Shared { get; } = new(TimeProvider.System,
new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(5) });
public static A2AWatchRegistry Shared { get; } = new(TimeProvider.System, CreateSharedHandler());
/// <summary>
/// The shared handler: pooled connections renewed every 5 minutes, and TCP keepalive (plus HTTP/2 pings) so a
/// connection that died silently on the way (a proxy or NAT dropping an idle stream) fails instead of hanging.
/// </summary>
internal static SocketsHttpHandler CreateSharedHandler() => new()
{
PooledConnectionLifetime = TimeSpan.FromMinutes(5),
KeepAlivePingDelay = TimeSpan.FromSeconds(60),
KeepAlivePingTimeout = TimeSpan.FromSeconds(20),
ConnectCallback = (context, ct) => ConnectWithKeepAliveAsync(context.DnsEndPoint, ct),
};
/// <summary>A TCP connection probed after 60 s of silence, every 15 s, given up after 4 unanswered probes.</summary>
internal static async ValueTask<Stream> ConnectWithKeepAliveAsync(DnsEndPoint endPoint, CancellationToken cancellationToken)
{
var socket = new Socket(SocketType.Stream, ProtocolType.Tcp) { NoDelay = true };
try
{
socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 60);
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, 15);
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveRetryCount, 4);
await socket.ConnectAsync(endPoint, cancellationToken).ConfigureAwait(false);
return new NetworkStream(socket, ownsSocket: true);
}
catch
{
socket.Dispose();
throw;
}
}
public static readonly TimeSpan[] DefaultPollDelays =
[TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(2), TimeSpan.FromMinutes(5)];
/// <summary>Longest single subscription; the stream is then reopened (a pause, not a failure).</summary>
private static readonly TimeSpan MaxSubscription = TimeSpan.FromDays(1);
private static readonly TimeSpan DeleteTimeout = TimeSpan.FromSeconds(10);
/// <summary>After this many failed checks in a row, a subscribe watch polls instead.</summary>
@@ -79,6 +109,13 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
internal TimeSpan[] ReconnectDelays { get; init; } = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)];
internal TimeSpan SafetyPollInterval { get; init; } = TimeSpan.FromMinutes(10);
/// <summary>
/// Longest single subscription: the watch then reads the task with <c>GetTask</c> and subscribes again (a pause, not
/// a failure). Remotes send nothing while their agent works without text, and a stream that went quiet may have died
/// on the way without either side noticing, so no stream is trusted for longer than this.
/// </summary>
internal TimeSpan SubscriptionSlice { get; init; } = TimeSpan.FromMinutes(5);
/// <summary>Bound of one remote call (get, create/delete push config), never past the watch deadline.</summary>
internal TimeSpan CallTimeout { get; init; } = TimeSpan.FromSeconds(60);
@@ -157,6 +194,8 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
watch.DisposeStop();
return watch.Record;
}
Logger.LogInformation("A2A: watching task {TaskId} of {Agent} for {Target} by {Source} (watch {WatchId})",
task.Id, agent.Entry.Name, target, watch.Record.Source, watchId);
watch.Loop = Task.Run(() => RunAsync(watch, settings));
return watch.Record;
}
@@ -199,6 +238,8 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
if (!ReferenceEquals(record, stored)) PendingFile.Upsert(workspace, record);
// Checked after the watch is in the map: a StopAsync that set the flag later sees (and stops) it.
if (_stopped) { _watches.TryRemove(new(record.WatchId, watch)); watch.DisposeStop(); return count; }
Logger.LogInformation("A2A: restored the watch of task {TaskId} of {Agent} for {Target} by {Source} (watch {WatchId})",
record.TaskId, record.Agent, target, record.Source, record.WatchId);
watch.Loop = Task.Run(() => RunAsync(watch, settings));
count++;
}
@@ -230,6 +271,7 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
{
if (_stopped) break; // shutting down: the rest stay in pending.json for the next start
if (!await TryDeliverAsync(target, record.Undelivered!, cancellationToken).ConfigureAwait(false)) { failed = true; continue; }
Logger.LogInformation("A2A: delivered the kept result of task {TaskId} of {Agent} to {Target}", record.TaskId, record.Agent, target);
using (await LockAsync(workspace).ConfigureAwait(false))
PendingFile.Remove(workspace, record.WatchId);
}
@@ -343,6 +385,9 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
(task, pause) = await SubscribeAsync(agent, record.TaskId, deadline, ct).ConfigureAwait(false);
if ((task is null || !A2AResults.IsSettled(task.Status.State)) && !pause)
throw new IOException("the task stream ended before the task settled");
if (pause && task is not null && !A2AResults.IsSettled(task.Status.State))
Logger.LogInformation("A2A: task {TaskId} of {Agent} is still {State}; subscribing again", record.TaskId, record.Agent,
A2AResults.StateName(task.Status.State));
}
else
{
@@ -415,7 +460,7 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
/// <summary>
/// Follows the task's stream until it settles or ends, then reads the task. <c>Paused</c>: the stream was ended by
/// us (the deadline or <see cref="MaxSubscription"/>), which is not a failure.
/// us (the deadline or <see cref="SubscriptionSlice"/>), which is not a failure.
/// </summary>
private async Task<(AgentTask? Task, bool Paused)> SubscribeAsync(RemoteAgent agent, string taskId, DateTimeOffset deadline, CancellationToken ct)
{
@@ -424,7 +469,7 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
using (var until = CancellationTokenSource.CreateLinkedTokenSource(ct))
{
var remaining = deadline - Time.GetUtcNow();
until.CancelAfter(remaining <= TimeSpan.Zero ? TimeSpan.Zero : Min(remaining, MaxSubscription));
until.CancelAfter(remaining <= TimeSpan.Zero ? TimeSpan.Zero : Min(remaining, SubscriptionSlice));
try
{
await foreach (var ev in agent.Client.SubscribeToTaskAsync(new SubscribeToTaskRequest { Id = taskId }, until.Token).ConfigureAwait(false))
@@ -436,7 +481,8 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
catch (A2AException) // e.g. the task is already terminal: read it instead
{
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested && until.IsCancellationRequested)
// Ending a stream mid-read may surface as an I/O error rather than a cancel: ours either way.
catch (Exception ex) when (ex is not OutOfMemoryException && !ct.IsCancellationRequested && until.IsCancellationRequested)
{
paused = true;
}
@@ -488,6 +534,10 @@ public sealed class A2AWatchRegistry : IAsyncDisposable
{
// A failed delivery writes nothing: the text is stored, and a cancel meanwhile has removed it for good.
delivered = await TryDeliverAsync(record.Target, text, CancellationToken.None).ConfigureAwait(false);
Logger.LogInformation(delivered
? "A2A: delivered the result of task {TaskId} of {Agent} to {Target}"
: "A2A: kept the result of task {TaskId} of {Agent} for {Target} until it can be delivered",
record.TaskId, record.Agent, record.Target);
if (delivered)
using (await LockAsync(watch.Workspace).ConfigureAwait(false))
PendingFile.Remove(watch.Workspace, record.WatchId);