cb877f697a
- A2AWatchRegistry.Shared was initialised before DefaultPollDelays (static fields run in source order), so the bot's registry had null poll delays and every poll watch crashed on its first wait, then sat in the registry doing nothing until a restart. The delays now come first; a test reads them through Shared. - TaskNotFound from the remote ends the watch at once with a "(lost)" notice instead of retrying for a day. - A crashed watch loop restarts by polling after a pause; the third crash in a row ends it with a "(stopped)" notice.
726 lines
41 KiB
C#
726 lines
41 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Text.Json;
|
|
using A2A;
|
|
using Microsoft.Extensions.Logging;
|
|
using Volcanic.AgenticCore.Extensions.A2A;
|
|
using Volcanic.AgenticCore.Extensions.Tests.A2A.Support;
|
|
using Volcanic.AgenticCore.Testing;
|
|
using A2AMessage = global::A2A.Message;
|
|
|
|
namespace Volcanic.AgenticCore.Extensions.Tests.A2A;
|
|
|
|
public sealed class A2AWatchTests : IDisposable
|
|
{
|
|
private readonly string _ws = Path.Combine(Path.GetTempPath(), "a2a-watch", Guid.NewGuid().ToString("N"));
|
|
|
|
private sealed class RecordingDelivery(string workspace) : IA2AResultDelivery
|
|
{
|
|
public ConcurrentQueue<(string Target, string Text)> Delivered { get; } = new();
|
|
public volatile bool Accept = true;
|
|
public Task<bool> DeliverAsync(string target, string text, CancellationToken cancellationToken)
|
|
{
|
|
if (Accept) Delivered.Enqueue((target, text));
|
|
return Task.FromResult(Accept);
|
|
}
|
|
public string? WorkspaceFor(string target) => workspace;
|
|
public IEnumerable<string> KnownTargets() => ["1"];
|
|
}
|
|
|
|
/// <summary>Delivery that blocks until <see cref="Release"/> and then answers <see cref="Accept"/>.</summary>
|
|
private sealed class GatedDelivery(string workspace) : IA2AResultDelivery
|
|
{
|
|
private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private int _entered;
|
|
public ConcurrentQueue<string> Delivered { get; } = new();
|
|
public bool Accept { get; init; } = true;
|
|
public int Entered => Volatile.Read(ref _entered);
|
|
public void Release() => _gate.TrySetResult();
|
|
public async Task<bool> DeliverAsync(string target, string text, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _entered);
|
|
await _gate.Task;
|
|
if (Accept) Delivered.Enqueue(text);
|
|
return Accept;
|
|
}
|
|
public string? WorkspaceFor(string target) => workspace;
|
|
public IEnumerable<string> KnownTargets() => ["1"];
|
|
}
|
|
|
|
/// <summary>First delivery blocks until <see cref="Release"/> and fails (no live session yet); later ones succeed at once.</summary>
|
|
private sealed class FailFirstDelivery(string workspace) : IA2AResultDelivery
|
|
{
|
|
private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private int _entered;
|
|
public ConcurrentQueue<string> Delivered { get; } = new();
|
|
public int Entered => Volatile.Read(ref _entered);
|
|
public void Release() => _gate.TrySetResult();
|
|
public async Task<bool> DeliverAsync(string target, string text, CancellationToken cancellationToken)
|
|
{
|
|
if (Interlocked.Increment(ref _entered) == 1)
|
|
{
|
|
await _gate.Task;
|
|
return false;
|
|
}
|
|
Delivered.Enqueue(text);
|
|
return true;
|
|
}
|
|
public string? WorkspaceFor(string target) => workspace;
|
|
public IEnumerable<string> KnownTargets() => ["1"];
|
|
}
|
|
|
|
/// <summary>Delivery that runs <see cref="OnDeliver"/> first, like a whole agent turn calling back into the registry.</summary>
|
|
private sealed class ReentrantDelivery(string workspace) : IA2AResultDelivery
|
|
{
|
|
public ConcurrentQueue<string> Delivered { get; } = new();
|
|
public Func<Task>? OnDeliver { get; set; }
|
|
public async Task<bool> DeliverAsync(string target, string text, CancellationToken cancellationToken)
|
|
{
|
|
if (Interlocked.Exchange(ref _onDeliverTaken, 1) == 0 && OnDeliver is { } turn) await turn();
|
|
Delivered.Enqueue(text);
|
|
return true;
|
|
}
|
|
private int _onDeliverTaken;
|
|
public string? WorkspaceFor(string target) => workspace;
|
|
public IEnumerable<string> KnownTargets() => ["1"];
|
|
}
|
|
|
|
private sealed class ListLogger : ILogger
|
|
{
|
|
public ConcurrentQueue<string> Lines { get; } = new();
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) =>
|
|
Lines.Enqueue(formatter(state, exception));
|
|
}
|
|
|
|
/// <summary>Over the remote: counts card fetches and, when <see cref="EndSubscribes"/>, ends every subscribe at once with no events.</summary>
|
|
private sealed class InterceptingHandler(HttpMessageHandler inner) : DelegatingHandler(inner)
|
|
{
|
|
private int _cards, _subscribes;
|
|
public bool EndSubscribes { get; init; }
|
|
public int Cards => Volatile.Read(ref _cards);
|
|
public int Subscribes => Volatile.Read(ref _subscribes);
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
if (request.Method == HttpMethod.Get && request.RequestUri!.AbsolutePath.EndsWith("agent-card.json", StringComparison.Ordinal))
|
|
Interlocked.Increment(ref _cards);
|
|
if (request.Content is not null)
|
|
{
|
|
var body = await request.Content.ReadAsStringAsync(cancellationToken);
|
|
if (body.Contains("\"SubscribeToTask\"", StringComparison.Ordinal))
|
|
{
|
|
Interlocked.Increment(ref _subscribes);
|
|
if (EndSubscribes)
|
|
return new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
|
{
|
|
Content = new StringContent("", System.Text.Encoding.UTF8, "text/event-stream"),
|
|
};
|
|
}
|
|
}
|
|
return await base.SendAsync(request, cancellationToken);
|
|
}
|
|
}
|
|
|
|
private void WriteAgents(string key = RemoteAgentServer.Key)
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(AgentsFile.PathFor(_ws))!);
|
|
File.WriteAllText(AgentsFile.PathFor(_ws), $$"""[ { "name": "remote", "url": "{{RemoteAgentServer.BaseUrl}}", "key": "{{key}}" } ]""");
|
|
}
|
|
|
|
private static async Task<AgentTask> StartRemoteTaskAsync(RemoteAgent agent, string text = "go") =>
|
|
(await agent.Client.SendMessageAsync(new SendMessageRequest
|
|
{
|
|
Message = new A2AMessage { Role = Role.User, MessageId = Guid.NewGuid().ToString("N"), Parts = [Part.FromText(text)] },
|
|
Configuration = new SendMessageConfiguration { ReturnImmediately = true },
|
|
})).Task!;
|
|
|
|
private static async Task WaitUntilAsync(Func<bool> condition, int timeoutMs = 15_000)
|
|
{
|
|
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
|
while (!condition())
|
|
{
|
|
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not met");
|
|
await Task.Delay(20);
|
|
}
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(WatchSources.Subscribe)]
|
|
[InlineData(WatchSources.Poll)]
|
|
public async Task Watch_DeliversResult(string source)
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Trip booked").WithDelay(TimeSpan.FromMilliseconds(500)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = source, PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
var record = await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
Assert.Equal(source, record.Source);
|
|
Assert.Contains(PendingFile.Read(_ws), r => r.WatchId == record.WatchId); // persisted before the watch can finish
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
var (target, text) = Assert.Single(delivery.Delivered);
|
|
Assert.Equal("1", target);
|
|
Assert.StartsWith($"[A2A result from \"remote\", task {task.Id} (completed)]", text);
|
|
Assert.Contains("Trip booked", text);
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Count == 0);
|
|
Assert.Empty(registry.List("1"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_Push_DeliversFollowUp()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Pushed result").WithDelay(TimeSpan.FromMilliseconds(500)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SafetyPollInterval = TimeSpan.FromMinutes(10) };
|
|
registry.UseDelivery(delivery);
|
|
var push = new PushToRegistryHandler(registry);
|
|
remote.PushHandler = push;
|
|
var settings = new A2AClientSettings { PushBaseUrl = "http://local.test" };
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
var record = await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
Assert.Equal(WatchSources.Push, record.Source);
|
|
Assert.NotNull(record.PushToken);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.True(push.Received > 0);
|
|
Assert.Contains("Pushed result", Assert.Single(delivery.Delivered).Text);
|
|
// Removed best-effort right after the result was delivered.
|
|
var deadline = DateTime.UtcNow.AddSeconds(15);
|
|
while ((await agent.Client.ListTaskPushNotificationConfigAsync(new ListTaskPushNotificationConfigRequest { TaskId = task.Id })).Configs?
|
|
.Any(c => c.Id == record.WatchId) == true)
|
|
{
|
|
Assert.True(DateTime.UtcNow < deadline, "the push config was not removed");
|
|
await Task.Delay(20);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_PushBadToken_Rejected()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("slow").WithDelay(TimeSpan.FromSeconds(5)));
|
|
WriteAgents();
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler);
|
|
registry.UseDelivery(new RecordingDelivery(_ws));
|
|
var settings = new A2AClientSettings { PushBaseUrl = "http://local.test" };
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var record = await registry.StartAsync("1", _ws, agent, await StartRemoteTaskAsync(agent), settings, CancellationToken.None);
|
|
var body = JsonDocument.Parse("{}").RootElement;
|
|
Assert.False(await registry.OnPushAsync(record.WatchId, "wrong", body, CancellationToken.None));
|
|
Assert.False(await registry.OnPushAsync(record.WatchId, null, body, CancellationToken.None));
|
|
Assert.False(await registry.OnPushAsync("unknown", record.PushToken, body, CancellationToken.None));
|
|
Assert.True(await registry.OnPushAsync(record.WatchId, record.PushToken, body, CancellationToken.None));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(WatchSources.Poll)]
|
|
[InlineData(WatchSources.Subscribe)]
|
|
public async Task Watch_Timeout_DeliversNotice(string source)
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("never").WithDelay(TimeSpan.FromSeconds(10)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = source, PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings { WatchTimeout = TimeSpan.FromMilliseconds(400) };
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
await registry.StartAsync("1", _ws, agent, await StartRemoteTaskAsync(agent), settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty, timeoutMs: 5_000); // well before the remote's 10 s answer
|
|
Assert.Contains("gave no result", Assert.Single(delivery.Delivered).Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_AgentRemoved_EndsWatch()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("never").WithDelay(TimeSpan.FromSeconds(10)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
await registry.StartAsync("1", _ws, agent, await StartRemoteTaskAsync(agent), settings, CancellationToken.None);
|
|
File.WriteAllText(AgentsFile.PathFor(_ws), "[]");
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.Contains("no longer in agents.json", Assert.Single(delivery.Delivered).Text);
|
|
Assert.Empty(registry.List("1"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_Undelivered_KeptThenFlushed()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Later"));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws) { Accept = false };
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
await registry.StartAsync("1", _ws, agent, await StartRemoteTaskAsync(agent), settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Any(r => r.Undelivered is not null));
|
|
delivery.Accept = true;
|
|
// The finishing watch holds its claim until its failed delivery returns: flush until the result goes out.
|
|
await WaitUntilAsync(() =>
|
|
{
|
|
registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None).GetAwaiter().GetResult();
|
|
return !delivery.Delivered.IsEmpty;
|
|
});
|
|
Assert.Contains("Later", Assert.Single(delivery.Delivered).Text);
|
|
Assert.Empty(PendingFile.Read(_ws));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_Restore_FromPendingFile()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Done before restart"));
|
|
WriteAgents();
|
|
var settings = new A2AClientSettings();
|
|
AgentTask task;
|
|
await using (var first = new A2AWatchRegistry(TimeProvider.System, remote.Handler))
|
|
{
|
|
using var agent = await first.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
task = await StartRemoteTaskAsync(agent);
|
|
}
|
|
PendingFile.Upsert(_ws, new WatchRecord("w1", "1", "remote", task.Id, task.ContextId, null, DateTimeOffset.UtcNow, WatchSources.Poll));
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(delivery);
|
|
Assert.Equal(1, await registry.RestoreAsync(settings, CancellationToken.None));
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.Contains("Done before restart", Assert.Single(delivery.Delivered).Text);
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Count == 0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StopWatching_EndsTheWatchAndForgetsIt()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("never").WithDelay(TimeSpan.FromSeconds(10)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
Assert.Single(registry.List("1"));
|
|
Assert.NotNull(await registry.StopWatchingAsync("1", _ws, task.Id));
|
|
Assert.Empty(registry.List("1"));
|
|
Assert.Empty(PendingFile.Read(_ws));
|
|
Assert.Null(await registry.StopWatchingAsync("1", _ws, task.Id));
|
|
Assert.Empty(delivery.Delivered);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_MalformedAgentsFile_KeepsWatching()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Survived").WithDelay(TimeSpan.FromMilliseconds(300)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
var log = new ListLogger();
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler, log) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(50)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
File.WriteAllText(AgentsFile.PathFor(_ws), "{ not json");
|
|
await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => log.Lines.Count(l => l.Contains("agents.json", StringComparison.Ordinal)) >= 3);
|
|
Assert.Empty(delivery.Delivered);
|
|
Assert.Single(registry.List("1"));
|
|
WriteAgents();
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.Contains("Survived", Assert.Single(delivery.Delivered).Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_SubscribeEndingUnsettled_FallsBackToPoll()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Polled at last").WithDelay(TimeSpan.FromMilliseconds(500)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
var http = new InterceptingHandler(remote.Handler) { EndSubscribes = true };
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, http)
|
|
{
|
|
SourceOverride = WatchSources.Subscribe, PollDelays = [TimeSpan.FromMilliseconds(50)], ReconnectDelays = [TimeSpan.FromMilliseconds(10)],
|
|
};
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var record = await registry.StartAsync("1", _ws, agent, await StartRemoteTaskAsync(agent), settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Any(r => r.WatchId == record.WatchId && r.Source == WatchSources.Poll) || !delivery.Delivered.IsEmpty);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.Contains("Polled at last", Assert.Single(delivery.Delivered).Text);
|
|
Assert.Equal(3, http.Subscribes); // three failed subscribes, then polling only
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_SilentStream_SubscribesAgainEachSlice_AndDelivers()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Finished in silence").WithDelay(TimeSpan.FromMilliseconds(700)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
var logger = new ListLogger();
|
|
var http = new RemoteRewriter(remote.Handler) { SilentSubscribes = true };
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, http, logger)
|
|
{
|
|
SourceOverride = WatchSources.Subscribe, SubscriptionSlice = TimeSpan.FromMilliseconds(200),
|
|
ReconnectDelays = [TimeSpan.FromSeconds(30)], // a failure would stall the watch past the test's wait
|
|
};
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
var record = await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty, timeoutMs: 10_000);
|
|
Assert.Contains("Finished in silence", Assert.Single(delivery.Delivered).Text);
|
|
Assert.True(http.Subscribes >= 2, $"subscribes: {http.Subscribes}"); // each slice ended, the task read, then a new stream
|
|
Assert.Equal(WatchSources.Subscribe, record.Source); // pauses aren't failures: it never fell back to polling
|
|
Assert.DoesNotContain(logger.Lines, l => l.Contains("failed", StringComparison.Ordinal));
|
|
Assert.Contains(logger.Lines, l => l.Contains($"watching task {task.Id} of remote for 1 by subscribe", StringComparison.Ordinal));
|
|
Assert.Contains(logger.Lines, l => l.Contains($"task {task.Id} of remote is still", StringComparison.Ordinal) && l.Contains("subscribing again", StringComparison.Ordinal));
|
|
await WaitUntilAsync(() => logger.Lines.Any(l => l.Contains($"delivered the result of task {task.Id} of remote to 1", StringComparison.Ordinal)));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SharedHandler_ConnectsWithTcpKeepAlive()
|
|
{
|
|
using var listener = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
|
|
listener.Start();
|
|
var port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port;
|
|
var accepted = listener.AcceptSocketAsync();
|
|
await using var stream = Assert.IsType<System.Net.Sockets.NetworkStream>(
|
|
await A2AWatchRegistry.ConnectWithKeepAliveAsync(new System.Net.DnsEndPoint("localhost", port), CancellationToken.None));
|
|
using var server = await accepted;
|
|
var socket = stream.Socket;
|
|
Assert.True(socket.Connected);
|
|
Assert.NotEqual(0, (int)socket.GetSocketOption(System.Net.Sockets.SocketOptionLevel.Socket, System.Net.Sockets.SocketOptionName.KeepAlive)!);
|
|
Assert.Equal(60, (int)socket.GetSocketOption(System.Net.Sockets.SocketOptionLevel.Tcp, System.Net.Sockets.SocketOptionName.TcpKeepAliveTime)!);
|
|
Assert.Equal(15, (int)socket.GetSocketOption(System.Net.Sockets.SocketOptionLevel.Tcp, System.Net.Sockets.SocketOptionName.TcpKeepAliveInterval)!);
|
|
using var handler = A2AWatchRegistry.CreateSharedHandler();
|
|
Assert.NotNull(handler.ConnectCallback);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_ReusesTheConnectionBetweenChecks()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Done").WithDelay(TimeSpan.FromMilliseconds(600)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
var http = new InterceptingHandler(remote.Handler);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, http) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(30)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
var before = http.Cards;
|
|
await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.Equal(1, http.Cards - before); // one connect for the whole watch, not one per poll
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Flush_Concurrent_DeliversOnce()
|
|
{
|
|
PendingFile.Upsert(_ws, new WatchRecord("w1", "1", "remote", "t1", "c1", null, DateTimeOffset.UtcNow, WatchSources.Poll, Undelivered: "the result"));
|
|
var delivery = new GatedDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, new HttpClientHandler());
|
|
registry.UseDelivery(delivery);
|
|
var first = registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None);
|
|
var second = registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None);
|
|
var restore = registry.RestoreAsync(new A2AClientSettings(), CancellationToken.None);
|
|
await WaitUntilAsync(() => delivery.Entered >= 1);
|
|
delivery.Release();
|
|
await Task.WhenAll(first, second, restore);
|
|
Assert.Single(delivery.Delivered);
|
|
Assert.Equal(1, delivery.Entered);
|
|
Assert.Empty(PendingFile.Read(_ws));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Flush_SkippingAClaimedRecord_RetriedWhenThatDeliveryFails()
|
|
{
|
|
PendingFile.Upsert(_ws, new WatchRecord("w1", "1", "remote", "t1", "c1", null, DateTimeOffset.UtcNow, WatchSources.Poll, Undelivered: "the result"));
|
|
var delivery = new FailFirstDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, new HttpClientHandler());
|
|
registry.UseDelivery(delivery);
|
|
var first = registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None);
|
|
await WaitUntilAsync(() => delivery.Entered >= 1);
|
|
// A session start meanwhile: its flush finds the record claimed and skips it.
|
|
await registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None);
|
|
Assert.Empty(delivery.Delivered);
|
|
delivery.Release(); // the first delivery fails: the skipped flush is run again
|
|
await first;
|
|
Assert.Equal("the result", Assert.Single(delivery.Delivered));
|
|
Assert.Empty(PendingFile.Read(_ws));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Finish_FailedDelivery_FlushesWhenAFlushSkippedIt()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Stranded?"));
|
|
WriteAgents();
|
|
var delivery = new FailFirstDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(50)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
await registry.StartAsync("1", _ws, agent, await StartRemoteTaskAsync(agent), settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => delivery.Entered >= 1); // the watch is finishing and its delivery is in flight
|
|
await registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None); // skipped: claimed
|
|
Assert.Empty(delivery.Delivered);
|
|
delivery.Release();
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
Assert.Contains("Stranded?", Assert.Single(delivery.Delivered));
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Count == 0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Start_SameTaskTwice_KeepsOneWatch()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("slow").WithDelay(TimeSpan.FromSeconds(5)));
|
|
WriteAgents();
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(100)] };
|
|
registry.UseDelivery(new RecordingDelivery(_ws));
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
var first = await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
var second = await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
Assert.Equal(first.WatchId, second.WatchId);
|
|
Assert.Single(registry.List("1"));
|
|
Assert.Single(PendingFile.Read(_ws));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Start_AfterStop_KeepsTheRecordButStartsNoLoop()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Later"));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(50)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
await registry.StopAsync(); // shutting down while a tool call hands a task over
|
|
var record = await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
await registry.RearmAsync(_ws, record with { WatchId = "w2", TaskId = "other" }, settings);
|
|
Assert.Empty(registry.List("1"));
|
|
Assert.Equal(2, PendingFile.Read(_ws).Count); // restored on the next start
|
|
Assert.Equal(0, await registry.RestoreAsync(settings, CancellationToken.None));
|
|
Assert.Empty(registry.List("1"));
|
|
await Task.Delay(500);
|
|
Assert.Empty(delivery.Delivered);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Flush_AfterStop_KeepsTheResults()
|
|
{
|
|
PendingFile.Upsert(_ws, new WatchRecord("w1", "1", "remote", "t1", "c1", null, DateTimeOffset.UtcNow, WatchSources.Poll, Undelivered: "kept"));
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, new HttpClientHandler());
|
|
registry.UseDelivery(delivery);
|
|
await registry.StopAsync();
|
|
await registry.FlushUndeliveredAsync("1", _ws, CancellationToken.None);
|
|
Assert.Empty(delivery.Delivered);
|
|
Assert.Equal("kept", Assert.Single(PendingFile.Read(_ws)).Undelivered);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Restore_RearmsWatchesBeforeFlushingKeptResults()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("slow").WithDelay(TimeSpan.FromSeconds(5)));
|
|
WriteAgents();
|
|
var settings = new A2AClientSettings();
|
|
AgentTask task;
|
|
await using (var first = new A2AWatchRegistry(TimeProvider.System, remote.Handler))
|
|
{
|
|
using var agent = await first.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
task = await StartRemoteTaskAsync(agent);
|
|
}
|
|
PendingFile.Upsert(_ws, new WatchRecord("w1", "1", "remote", task.Id, task.ContextId, null, DateTimeOffset.UtcNow, WatchSources.Poll));
|
|
PendingFile.Upsert(_ws, new WatchRecord("w2", "1", "remote", "done", "c2", null, DateTimeOffset.UtcNow, WatchSources.Poll, Undelivered: "kept"));
|
|
var delivery = new GatedDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { PollDelays = [TimeSpan.FromSeconds(10)] };
|
|
registry.UseDelivery(delivery);
|
|
var restore = registry.RestoreAsync(settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => delivery.Entered >= 1); // a slow chat turn delivering the kept result
|
|
Assert.Equal("w1", Assert.Single(registry.List("1")).WatchId); // already re-armed
|
|
delivery.Release();
|
|
Assert.Equal(1, await restore);
|
|
Assert.Equal("kept", Assert.Single(delivery.Delivered));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task StopWatching_RacingFinish_DoesNotResurrectTheRecord()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Too late"));
|
|
WriteAgents();
|
|
var delivery = new GatedDelivery(_ws) { Accept = false };
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(50)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => delivery.Entered >= 1); // the watch is finishing: delivering (and about to keep it undelivered)
|
|
var stop = registry.StopWatchingAsync("1", _ws, task.Id);
|
|
delivery.Release();
|
|
await stop;
|
|
Assert.Empty(PendingFile.Read(_ws));
|
|
Assert.Empty(registry.List("1"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Restore_PushWatchWithoutPushUrl_WatchesBySubscribe()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("Followed").WithDelay(TimeSpan.FromMilliseconds(800)));
|
|
WriteAgents();
|
|
var settings = new A2AClientSettings();
|
|
AgentTask task;
|
|
await using (var first = new A2AWatchRegistry(TimeProvider.System, remote.Handler))
|
|
{
|
|
using var agent = await first.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
task = await StartRemoteTaskAsync(agent);
|
|
}
|
|
PendingFile.Upsert(_ws, new WatchRecord("w1", "1", "remote", task.Id, task.ContextId, "tok", DateTimeOffset.UtcNow, WatchSources.Push));
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SafetyPollInterval = TimeSpan.FromMinutes(10) };
|
|
registry.UseDelivery(delivery);
|
|
Assert.Equal(1, await registry.RestoreAsync(settings, CancellationToken.None));
|
|
Assert.Equal(WatchSources.Subscribe, Assert.Single(registry.List("1")).Source);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty); // a 10-minute safety poll would never get here in time
|
|
Assert.Contains("Followed", Assert.Single(delivery.Delivered).Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Delivery_ReenteringTheRegistry_DoesNotDeadlock()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("never").WithDelay(TimeSpan.FromSeconds(10)));
|
|
WriteAgents();
|
|
var delivery = new ReentrantDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler) { SourceOverride = WatchSources.Poll, PollDelays = [TimeSpan.FromMilliseconds(50)] };
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var slow = await StartRemoteTaskAsync(agent, "slow");
|
|
await registry.StartAsync("1", _ws, agent, slow, settings, CancellationToken.None);
|
|
var turnDone = false;
|
|
// The delivered result's "turn" cancels the slow task's watch and starts a new watch in the same workspace.
|
|
delivery.OnDeliver = async () =>
|
|
{
|
|
Assert.NotNull(await registry.StopWatchingAsync("1", _ws, slow.Id));
|
|
await registry.StartAsync("1", _ws, agent, slow, settings, CancellationToken.None);
|
|
turnDone = true;
|
|
};
|
|
// The remote runs both tasks at once: script the quick answer only once the slow task has taken its turn.
|
|
await WaitUntilAsync(() => remote.Faux.Remaining == 0);
|
|
remote.Faux.Enqueue(FauxTurn.Say("Quick"));
|
|
var quick = await StartRemoteTaskAsync(agent, "quick");
|
|
await registry.StartAsync("1", _ws, agent, quick, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => Volatile.Read(ref turnDone) && !delivery.Delivered.IsEmpty, timeoutMs: 5_000);
|
|
Assert.Contains("Quick", Assert.Single(delivery.Delivered));
|
|
var watched = Assert.Single(registry.List("1"));
|
|
Assert.Equal(slow.Id, watched.TaskId);
|
|
Assert.Contains(PendingFile.Read(_ws), r => r.WatchId == watched.WatchId);
|
|
}
|
|
|
|
[Fact]
|
|
public void Shared_HasItsDelaysAndSlice()
|
|
{
|
|
// Static fields initialise in source order: Shared once read DefaultPollDelays before it was set (null).
|
|
var shared = A2AWatchRegistry.Shared;
|
|
Assert.Same(A2AWatchRegistry.DefaultPollDelays, shared.PollDelays);
|
|
Assert.NotEmpty(shared.ReconnectDelays);
|
|
Assert.Equal(TimeSpan.FromSeconds(15), A2AWatchRegistry.DelayAt(shared.PollDelays, 0));
|
|
Assert.Equal(TimeSpan.FromMinutes(5), shared.SubscriptionSlice);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(WatchSources.Subscribe)]
|
|
[InlineData(WatchSources.Poll)]
|
|
public async Task Watch_TaskUnknownToTheRemote_EndsAtOnceAsLost(string source)
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler)
|
|
{
|
|
SourceOverride = source, PollDelays = [TimeSpan.FromSeconds(30)], ReconnectDelays = [TimeSpan.FromSeconds(30)], // no retry fits the wait
|
|
};
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var forgotten = new AgentTask { Id = "forgotten-task", ContextId = "ctx-1", Status = new global::A2A.TaskStatus { State = TaskState.Working } };
|
|
await registry.StartAsync("1", _ws, agent, forgotten, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty, timeoutMs: 10_000);
|
|
var text = Assert.Single(delivery.Delivered).Text;
|
|
Assert.StartsWith("[A2A result from \"remote\", task forgotten-task (lost)]", text);
|
|
Assert.Contains("no longer knows this task", text);
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Count == 0);
|
|
Assert.Empty(registry.List("1"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Watch_CrashingLoop_RestartsThenEndsAsStopped_NeverLeftHanging()
|
|
{
|
|
await using var remote = await RemoteAgentServer.StartAsync();
|
|
remote.Faux.Enqueue(FauxTurn.Say("never seen").WithDelay(TimeSpan.FromSeconds(10)));
|
|
WriteAgents();
|
|
var delivery = new RecordingDelivery(_ws);
|
|
var logger = new ListLogger();
|
|
// No poll delays at all: the first wait of every loop throws, like the null delays of the shared registry did.
|
|
await using var registry = new A2AWatchRegistry(TimeProvider.System, remote.Handler, logger)
|
|
{
|
|
SourceOverride = WatchSources.Poll, PollDelays = [], ReconnectDelays = [TimeSpan.FromMilliseconds(20)],
|
|
};
|
|
registry.UseDelivery(delivery);
|
|
var settings = new A2AClientSettings();
|
|
using var agent = await registry.Connector(settings).ConnectAsync(AgentsFile.Find(_ws, "remote")!, CancellationToken.None);
|
|
var task = await StartRemoteTaskAsync(agent);
|
|
await registry.StartAsync("1", _ws, agent, task, settings, CancellationToken.None);
|
|
await WaitUntilAsync(() => !delivery.Delivered.IsEmpty);
|
|
var text = Assert.Single(delivery.Delivered).Text;
|
|
Assert.StartsWith($"[A2A result from \"remote\", task {task.Id} (stopped)]", text);
|
|
Assert.Contains("action=status", text);
|
|
Assert.Equal(3, logger.Lines.Count(l => l.Contains("crashed (", StringComparison.Ordinal)));
|
|
await WaitUntilAsync(() => PendingFile.Read(_ws).Count == 0);
|
|
Assert.Empty(registry.List("1"));
|
|
}
|
|
|
|
[Fact]
|
|
public void PollDelays_BackOffToFiveMinutes()
|
|
{
|
|
var delays = A2AWatchRegistry.DefaultPollDelays;
|
|
Assert.Equal(TimeSpan.FromSeconds(15), delays[0]);
|
|
Assert.Equal(TimeSpan.FromMinutes(5), delays[^1]);
|
|
Assert.Equal(TimeSpan.FromMinutes(5), A2AWatchRegistry.DelayAt(delays, 99));
|
|
}
|
|
|
|
public void Dispose() { try { Directory.Delete(_ws, true); } catch (IOException) { } }
|
|
}
|