forked from admin/VolcanicAgenticCore
fd6b9027b6
- JobScheduler: a stored job whose zone lookup throws no longer fails StartAsync; timer re-armed under the lock; a delivery cut off by shutdown isn't recorded (one-offs fire again after restart); CancelJobAsync in one step - Schedules.ParseAt: ISO 8601 with a date only (no machine-clock date for a bare 09:00) - core: a FollowUp prompt that loses the race to a starting run joins it as a follow-up, not steering - Telegram: TimeZoneFor tolerates non-numeric targets; RunPromptAsync waits only when queued; one agent dir - ScriptRunner: bad workspace paths are Failed, not exceptions; zone id helpers share TimeZoneIds.ToIana's rule - tests: no wall-clock sleeps in scheduler tests; transcript-entry, disabled-scheduler and failing-prompt coverage - docs: spec notes for the above; README row for the scheduler
327 lines
17 KiB
C#
327 lines
17 KiB
C#
using Volcanic.AgenticCore.Abstractions.Events;
|
|
using Volcanic.AgenticCore.Abstractions.Extensions;
|
|
using Volcanic.AgenticCore.Abstractions.Extensions.Events;
|
|
using Volcanic.AgenticCore.Abstractions.Messages;
|
|
using Volcanic.AgenticCore.Abstractions.Tools;
|
|
using Volcanic.AgenticCore.Features.Extensions;
|
|
using Volcanic.AgenticCore.Testing;
|
|
using Volcanic.AgenticCore.Tests.Support;
|
|
|
|
namespace Volcanic.AgenticCore.Tests.Features;
|
|
|
|
public class AgentLoopTests
|
|
{
|
|
private static ITool Echo(string name, Func<string, Task<string>> run, ToolExecutionMode mode = ToolExecutionMode.Parallel) =>
|
|
new DelegateTool<EchoArgs>(name, "echo", async (a, _, _) => ToolResult.Text(await run(a.Value)), executionMode: mode);
|
|
|
|
public sealed record EchoArgs(string Value);
|
|
|
|
private static IExtension WithTools(params ITool[] tools) => new LambdaExtension("tools", api =>
|
|
{
|
|
foreach (var t in tools) api.RegisterTool(t);
|
|
return ValueTask.CompletedTask;
|
|
});
|
|
|
|
[Fact]
|
|
public async Task Single_text_turn_emits_the_expected_event_sequence()
|
|
{
|
|
await using var host = new TestHost();
|
|
host.Faux.Enqueue(FauxTurn.Say("Hello there friend"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
|
|
await session.PromptAsync("hi");
|
|
|
|
var names = host.EventNames.ToList();
|
|
Assert.Equal("AgentStart", names[0]);
|
|
Assert.Equal("TurnStart", names[1]);
|
|
Assert.Equal(["MessageStart", "MessageEnd"], names.Skip(2).Take(2)); // user message
|
|
Assert.Contains("MessageUpdate", names);
|
|
Assert.Equal(["TurnEnd", "AgentEnd", "AgentSettled"], names.TakeLast(3));
|
|
Assert.Equal("Hello there friend", session.Messages.OfType<AssistantMessage>().Single().Text);
|
|
Assert.Equal(2, session.Messages.Count);
|
|
Assert.False(session.IsStreaming);
|
|
|
|
var request = Assert.Single(host.Faux.Requests);
|
|
Assert.Contains("Available tools", request.SystemPrompt);
|
|
Assert.Equal("hi", request.Messages.Single().Text);
|
|
Assert.Contains(request.Tools, t => t.Name == "bash");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tool_round_trip_feeds_the_result_into_the_next_call()
|
|
{
|
|
await using var host = new TestHost(null, WithTools(Echo("shout", v => Task.FromResult(v.ToUpperInvariant()))));
|
|
host.Faux.Enqueue(FauxTurn.Say("calling").WithTool("shout", new { value = "abc" }), FauxTurn.Say("done"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
|
|
await session.PromptAsync("go");
|
|
|
|
Assert.Equal(2, host.Faux.Requests.Count);
|
|
var second = host.Faux.Requests[1].Messages;
|
|
var result = Assert.IsType<ToolResultMessage>(second[^1]);
|
|
Assert.Equal("ABC", result.Text);
|
|
Assert.False(result.IsError);
|
|
Assert.Equal(["TurnStart", "TurnStart"], host.EventNames.Where(n => n == "TurnStart"));
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.ToolExecutionStart { ToolName: "shout" });
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.ToolExecutionEnd { IsError: false });
|
|
Assert.Equal(4, session.Messages.Count);
|
|
Assert.Equal(StopReason.ToolUse, session.Messages.OfType<AssistantMessage>().First().StopReason);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Unknown_tool_and_invalid_arguments_become_error_results()
|
|
{
|
|
await using var host = new TestHost(null, WithTools(Echo("shout", v => Task.FromResult(v))));
|
|
host.Faux.Enqueue(FauxTurn.Tool("nope", new { }).WithTool("shout", new { value = 5 }), FauxTurn.Say("ok"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
|
|
var results = session.Messages.OfType<ToolResultMessage>().ToList();
|
|
Assert.Equal(2, results.Count);
|
|
Assert.True(results[0].IsError);
|
|
Assert.Contains("not found", results[0].Text);
|
|
Assert.True(results[1].IsError);
|
|
Assert.Contains("should be string", results[1].Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Parallel_tools_end_in_completion_order_but_results_follow_source_order()
|
|
{
|
|
var slowGate = new TaskCompletionSource();
|
|
await using var host = new TestHost(null, WithTools(
|
|
Echo("slow", async v => { await slowGate.Task; return "slow:" + v; }),
|
|
Echo("fast", v => Task.FromResult("fast:" + v))));
|
|
host.Faux.Enqueue(FauxTurn.Tool("slow", new { value = "1" }).WithTool("fast", new { value = "2" }), FauxTurn.Say("ok"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
|
|
var run = session.PromptAsync("go");
|
|
// let "fast" finish before releasing "slow"
|
|
await WaitUntil(() => host.Events.OfType<AgentSessionEvent.ToolExecutionEnd>().Any(e => e.ToolName == "fast"));
|
|
slowGate.SetResult();
|
|
await run;
|
|
|
|
var ends = host.Events.OfType<AgentSessionEvent.ToolExecutionEnd>().Select(e => e.ToolName).ToList();
|
|
Assert.Equal(["fast", "slow"], ends);
|
|
var results = session.Messages.OfType<ToolResultMessage>().Select(r => r.ToolName).ToList();
|
|
Assert.Equal(["slow", "fast"], results);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Sequential_tool_forces_ordered_execution()
|
|
{
|
|
var order = new List<string>();
|
|
await using var host = new TestHost(null, WithTools(
|
|
Echo("a", async v => { await Task.Delay(30); lock (order) order.Add("a"); return v; }, ToolExecutionMode.Sequential),
|
|
Echo("b", v => { lock (order) order.Add("b"); return Task.FromResult(v); })));
|
|
host.Faux.Enqueue(FauxTurn.Tool("a", new { value = "1" }).WithTool("b", new { value = "2" }), FauxTurn.Say("ok"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
Assert.Equal(["a", "b"], order);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Steering_is_delivered_before_the_next_model_call_and_follow_up_after()
|
|
{
|
|
var gate = new TaskCompletionSource();
|
|
await using var host = new TestHost(null, WithTools(Echo("wait", async v => { await gate.Task; return v; })));
|
|
host.Faux.Enqueue(FauxTurn.Tool("wait", new { value = "x" }), FauxTurn.Say("after steer"), FauxTurn.Say("after follow-up"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
|
|
var run = session.PromptAsync("start");
|
|
await WaitUntil(() => session.IsStreaming && host.Events.OfType<AgentSessionEvent.ToolExecutionStart>().Any());
|
|
await session.PromptAsync("steer me", behavior: StreamingBehavior.Steer);
|
|
await session.PromptAsync("later", behavior: StreamingBehavior.FollowUp);
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.QueueUpdate { Steering.Count: 1, FollowUp.Count: 1 });
|
|
gate.SetResult();
|
|
await run;
|
|
|
|
Assert.Equal(3, host.Faux.Requests.Count);
|
|
Assert.Equal("steer me", host.Faux.Requests[1].Messages[^1].Text);
|
|
Assert.Equal("later", host.Faux.Requests[2].Messages[^1].Text);
|
|
Assert.Single(host.Events.OfType<AgentSessionEvent.AgentEnd>());
|
|
Assert.Equal(["start", "steer me", "later"], session.Messages.OfType<UserMessage>().Select(m => m.Text));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_follow_up_prompt_that_loses_the_race_to_a_run_is_queued_as_a_follow_up_not_steering()
|
|
{
|
|
var gate = new TaskCompletionSource();
|
|
await using var host = new TestHost(null, WithTools(Echo("wait", async v => { await gate.Task; return v; })));
|
|
host.Faux.Enqueue(FauxTurn.Tool("wait", new { value = "x" }), FauxTurn.Say("after tool"), FauxTurn.Say("after scheduled"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
|
|
var run = session.PromptAsync("user");
|
|
await WaitUntil(() => session.IsStreaming && host.Events.OfType<AgentSessionEvent.ToolExecutionStart>().Any());
|
|
// What PromptAsync(FollowUp) does when it saw an idle session just before the user's run took the run gate.
|
|
var scheduled = session.StartRunAsync(UserMessage.From("scheduled", null), null, CancellationToken.None, StreamingBehavior.FollowUp);
|
|
await WaitUntil(() => host.Events.OfType<AgentSessionEvent.QueueUpdate>().Any());
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.QueueUpdate { Steering.Count: 0, FollowUp.Count: 1 });
|
|
gate.SetResult();
|
|
await run;
|
|
await scheduled;
|
|
|
|
Assert.Equal(3, host.Faux.Requests.Count);
|
|
Assert.IsType<ToolResultMessage>(host.Faux.Requests[1].Messages[^1]); // not steered into the user's turn
|
|
Assert.Equal("scheduled", host.Faux.Requests[2].Messages[^1].Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Prompt_while_streaming_without_behavior_throws()
|
|
{
|
|
await using var host = new TestHost();
|
|
host.Faux.Enqueue(FauxTurn.Say("slow").WithDelay(TimeSpan.FromMilliseconds(300)));
|
|
await using var session = await host.CreateSessionAsync();
|
|
var run = session.PromptAsync("start");
|
|
await WaitUntil(() => session.IsStreaming);
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => session.PromptAsync("again"));
|
|
await run;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Abort_produces_an_aborted_message_and_settles()
|
|
{
|
|
await using var host = new TestHost();
|
|
host.Faux.Enqueue(FauxTurn.Say("never finishes").WithDelay(TimeSpan.FromSeconds(10)));
|
|
await using var session = await host.CreateSessionAsync();
|
|
var run = session.PromptAsync("start");
|
|
await WaitUntil(() => session.IsStreaming);
|
|
await session.AbortAsync();
|
|
await run;
|
|
|
|
var assistant = session.Messages.OfType<AssistantMessage>().Single();
|
|
Assert.Equal(StopReason.Aborted, assistant.StopReason);
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.AgentSettled);
|
|
Assert.False(session.IsStreaming);
|
|
|
|
host.Faux.Enqueue(FauxTurn.Say("recovered"));
|
|
await session.PromptAsync("again");
|
|
Assert.Equal("recovered", session.Messages.OfType<AssistantMessage>().Last().Text);
|
|
Assert.Equal(2, host.Faux.Requests[1].Messages.Count); // the aborted attempt is dropped from context
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Transient_errors_are_retried_with_events()
|
|
{
|
|
await using var host = new TestHost();
|
|
host.Faux.Enqueue(FauxTurn.Error("HTTP 529 overloaded"), FauxTurn.Error("HTTP 503"), FauxTurn.Say("third time lucky"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
|
|
Assert.Equal(3, host.Faux.Requests.Count);
|
|
var starts = host.Events.OfType<AgentSessionEvent.AutoRetryStart>().ToList();
|
|
Assert.Equal([1, 2], starts.Select(s => s.Attempt));
|
|
var ends = host.Events.OfType<AgentSessionEvent.AutoRetryEnd>().ToList();
|
|
Assert.Equal([false, true], ends.Select(e => e.Success));
|
|
Assert.Equal("third time lucky", session.Messages.OfType<AssistantMessage>().Last().Text);
|
|
Assert.Equal(2, session.Messages.OfType<AssistantMessage>().Count(a => a.StopReason == StopReason.Error));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Non_retryable_error_ends_the_run()
|
|
{
|
|
await using var host = new TestHost();
|
|
host.Faux.Enqueue(FauxTurn.Error("invalid api key"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
Assert.Single(host.Faux.Requests);
|
|
Assert.Equal("invalid api key", session.Messages.OfType<AssistantMessage>().Single().ErrorMessage);
|
|
Assert.Equal(["TurnEnd", "AgentEnd", "AgentSettled"], host.EventNames.TakeLast(3));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Length_stop_fails_all_tool_calls()
|
|
{
|
|
await using var host = new TestHost(null, WithTools(Echo("shout", v => Task.FromResult(v))));
|
|
host.Faux.Enqueue(FauxTurn.Length("cut").WithTool("shout", new { value = "x" }), FauxTurn.Say("ok"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
var result = session.Messages.OfType<ToolResultMessage>().Single();
|
|
Assert.True(result.IsError);
|
|
Assert.Contains("output token limit", result.Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Overflow_error_compacts_and_retries()
|
|
{
|
|
await using var host = new TestHost(o => o.Compaction = new Volcanic.AgenticCore.Features.Compaction.CompactionSettings(KeepRecentTokens: 50));
|
|
await using var session = await host.CreateSessionAsync();
|
|
host.Faux.Enqueue(FauxTurn.Say("first answer " + new string('a', 400)));
|
|
await session.PromptAsync("first question " + new string('q', 400));
|
|
host.Faux.Enqueue(FauxTurn.Say("second answer"));
|
|
await session.PromptAsync("second question");
|
|
host.Faux.Enqueue(FauxTurn.Error("prompt is too long: context_length_exceeded"), FauxTurn.Say("## Goal\nsummary text"), FauxTurn.Say("after compaction"));
|
|
|
|
await session.PromptAsync("third question");
|
|
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.CompactionStart { Reason: CompactionReason.Overflow });
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.CompactionEnd { Result: not null, WillRetry: true });
|
|
Assert.Equal("after compaction", session.Messages.OfType<AssistantMessage>().Last().Text);
|
|
var lastContext = host.Faux.Requests[^1].Messages;
|
|
Assert.Contains(lastContext, m => m.Text.Contains("summary text"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Threshold_compaction_runs_between_turns()
|
|
{
|
|
await using var host = new TestHost(o => o.Compaction = new Volcanic.AgenticCore.Features.Compaction.CompactionSettings(ReserveTokens: 31_000, KeepRecentTokens: 10));
|
|
await using var session = await host.CreateSessionAsync();
|
|
// The threshold is crossed by the first answer's usage, so compaction runs right after the first run.
|
|
host.Faux.Enqueue(FauxTurn.Say("one").WithUsage(2_000, 10), FauxTurn.Say("## Goal\nsummarised"));
|
|
await session.PromptAsync("q1 " + new string('x', 400));
|
|
host.Faux.Enqueue(FauxTurn.Say("two"));
|
|
await session.PromptAsync("q2");
|
|
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.CompactionStart { Reason: CompactionReason.Threshold });
|
|
Assert.Contains(host.Faux.Requests[^1].Messages, m => m.Text.Contains("summarised"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Extension_hooks_shape_the_run()
|
|
{
|
|
var ext = new LambdaExtension("hooks", api =>
|
|
{
|
|
api.On<BeforeAgentStartEvent>((e, _, _) => { e.SystemPromptOptions.Sections["extra"] = "be terse"; e.InjectedMessages.Add(CustomMessage.FromText("ctx", "injected")); return ValueTask.CompletedTask; });
|
|
api.On<ContextEvent>((e, _, _) => { e.Messages.Insert(0, UserMessage.FromText("from context hook")); return ValueTask.CompletedTask; });
|
|
api.On<ToolCallEvent>((e, _, _) => { if (e.ToolName == "bash") e.Block("no shell"); return ValueTask.CompletedTask; });
|
|
api.On<ToolResultEvent>((e, _, _) => { e.Content.Add(new TextContent(" [audited]")); return ValueTask.CompletedTask; });
|
|
api.On<MessageEndEvent>((e, _, _) => { if (e.Message is AssistantMessage a && a.Text == "final") e.Message = a with { Content = [new TextContent("FINAL")] }; return ValueTask.CompletedTask; });
|
|
return ValueTask.CompletedTask;
|
|
});
|
|
await using var host = new TestHost(null, ext);
|
|
host.Faux.Enqueue(FauxTurn.Tool("bash", new { command = "rm -rf /" }), FauxTurn.Say("final"));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
|
|
var first = host.Faux.Requests[0];
|
|
Assert.Contains("<extra>\nbe terse\n</extra>", first.SystemPrompt);
|
|
Assert.Equal("from context hook", first.Messages[0].Text);
|
|
Assert.Contains(first.Messages, m => m.Text == "injected");
|
|
var result = session.Messages.OfType<ToolResultMessage>().Single();
|
|
Assert.True(result.IsError);
|
|
Assert.Equal("no shell [audited]", result.Text);
|
|
Assert.Equal("FINAL", session.Messages.OfType<AssistantMessage>().Last().Text);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Terminate_result_stops_the_loop_without_another_model_call()
|
|
{
|
|
var tool = new DelegateTool<EchoArgs>("finish", "ends", (_, _, _) => Task.FromResult(ToolResult.Text("bye") with { Terminate = true }));
|
|
await using var host = new TestHost(null, WithTools(tool));
|
|
host.Faux.Enqueue(FauxTurn.Tool("finish", new { value = "x" }));
|
|
await using var session = await host.CreateSessionAsync();
|
|
await session.PromptAsync("go");
|
|
Assert.Single(host.Faux.Requests);
|
|
Assert.Contains(host.Events, e => e is AgentSessionEvent.AgentSettled);
|
|
}
|
|
|
|
internal static async Task WaitUntil(Func<bool> condition, int timeoutMs = 5000)
|
|
{
|
|
var start = Environment.TickCount64;
|
|
while (!condition())
|
|
{
|
|
if (Environment.TickCount64 - start > timeoutMs) throw new TimeoutException("Condition not met.");
|
|
await Task.Delay(10);
|
|
}
|
|
}
|
|
}
|