Files
Playa/tests/Volcanic.AgenticCore.Extensions.Tests/Scheduler/JobSchedulerTests.cs
EugeneTes fd6b9027b6 fix(scheduler): address review findings
- JobScheduler: a stored job whose zone lookup throws no longer fails StartAsync; timer re-armed under the lock;
  a delivery cut off by shutdown isn't recorded (one-offs fire again after restart); CancelJobAsync in one step
- Schedules.ParseAt: ISO 8601 with a date only (no machine-clock date for a bare 09:00)
- core: a FollowUp prompt that loses the race to a starting run joins it as a follow-up, not steering
- Telegram: TimeZoneFor tolerates non-numeric targets; RunPromptAsync waits only when queued; one agent dir
- ScriptRunner: bad workspace paths are Failed, not exceptions; zone id helpers share TimeZoneIds.ToIana's rule
- tests: no wall-clock sleeps in scheduler tests; transcript-entry, disabled-scheduler and failing-prompt coverage
- docs: spec notes for the above; README row for the scheduler
2026-09-27 10:56:45 +02:00

432 lines
20 KiB
C#

using Volcanic.AgenticCore.Extensions.Scheduler;
using Volcanic.AgenticCore.Extensions.Tests.GoogleCalendar;
using Volcanic.AgenticCore.Extensions.Tests.Scheduler.Support;
namespace Volcanic.AgenticCore.Extensions.Tests.Scheduler;
public sealed class JobSchedulerTests
{
private static readonly TimeSpan Minute = TimeSpan.FromMinutes(1);
private static readonly DateTimeOffset Start = SchedulerHarness.Start; // 12:03Z = Sat 14:03 Berlin
private static ScheduledJob Stored(string id, JobKind kind, JobSchedule schedule, string? text = "Call the dentist", string target = "111") => new()
{
Id = id, Target = target, Name = text ?? "Watch", Kind = kind, Schedule = schedule, CreatedAt = Start.AddDays(-1),
Text = kind == JobKind.Watcher ? null : text, Command = kind == JobKind.Watcher ? "scripts/check.sh" : null,
};
private static NewJob Watcher(string cron = "* * * * *") => new("Price check", JobKind.Watcher, Cron: cron, Command: "scripts/price.sh");
[Fact]
public async Task An_overdue_one_off_fires_on_start_with_the_late_note_and_is_removed_after_delivery()
{
await using var h = new SchedulerHarness();
await h.Store.SaveAsync([Stored("k7m2q", JobKind.Reminder, new JobSchedule(Start.AddMinutes(-5), null))], CancellationToken.None);
h.Delivery.HoldTexts = new TaskCompletionSource();
await h.StartAsync();
await Eventually.UntilAsync(() => h.Delivery.Texts.Count == 1, "the reminder send");
Assert.Single(await h.Store.LoadAsync(CancellationToken.None)); // not removed while the delivery is in progress
h.Delivery.HoldTexts.SetResult();
await h.Scheduler.IdleAsync();
Assert.Equal(("111", "⏰ Call the dentist (due 26 Sep 13:58, delivered late)"), Assert.Single(h.Delivery.Texts));
Assert.Empty(await h.Store.LoadAsync(CancellationToken.None));
Assert.Empty(h.Scheduler.List("111"));
}
[Fact]
public async Task An_overdue_one_off_within_60_s_has_no_late_note()
{
await using var h = new SchedulerHarness();
await h.Store.SaveAsync([Stored("k7m2q", JobKind.Reminder, new JobSchedule(Start.AddSeconds(-30), null))], CancellationToken.None);
await h.StartAsync();
await h.Scheduler.IdleAsync();
Assert.Equal(("111", "⏰ Call the dentist"), Assert.Single(h.Delivery.Texts));
}
[Fact]
public async Task An_overdue_prompt_gets_the_offline_note()
{
await using var h = new SchedulerHarness();
await h.Store.SaveAsync([Stored("p3xz9", JobKind.Prompt, new JobSchedule(Start.AddHours(-1), null), text: "Summarise inbox/")], CancellationToken.None);
await h.StartAsync();
await h.Scheduler.IdleAsync();
Assert.Equal("[Scheduled job \"Summarise inbox/\" (p3xz9)]\nThis was due at 26 Sep 13:03; the bot was offline.\nSummarise inbox/",
Assert.Single(h.Delivery.Prompts).Prompt);
}
[Fact]
public async Task Cron_does_not_catch_up_after_downtime()
{
await using var h = new SchedulerHarness();
var job = Stored("c1c1c", JobKind.Reminder, new JobSchedule(null, "*/10 * * * *")) with { LastRunAt = Start.AddHours(-3) };
await h.Store.SaveAsync([job], CancellationToken.None);
await h.StartAsync();
await h.Scheduler.IdleAsync();
Assert.Empty(h.Delivery.Texts);
Assert.Equal(new DateTimeOffset(2026, 9, 26, 12, 10, 0, TimeSpan.Zero), Assert.Single(h.Scheduler.List("111")).NextRunAt);
}
[Fact]
public async Task Advancing_the_clock_fires_and_the_1_h_cap_rearms_without_firing()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(new NewJob("Later", JobKind.Reminder, At: "2026-09-26T17:03", Text: "Later")); // 15:03Z, 3 h away
Assert.Equal(Start.AddHours(1), h.Scheduler.NextWakeAt);
await h.AdvanceAsync(TimeSpan.FromHours(1));
Assert.Empty(h.Delivery.Texts);
Assert.Equal(Start.AddHours(2), h.Scheduler.NextWakeAt);
await h.AdvanceAsync(TimeSpan.FromHours(1));
Assert.Empty(h.Delivery.Texts);
Assert.Equal(Start.AddHours(3), h.Scheduler.NextWakeAt);
await h.AdvanceAsync(TimeSpan.FromHours(1));
Assert.Equal(("111", "⏰ Later"), Assert.Single(h.Delivery.Texts));
Assert.Empty(h.Scheduler.List("111"));
}
[Fact]
public async Task A_reminder_is_sent_as_text_and_a_prompt_as_a_prompt()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(new NewJob("Stretch", JobKind.Reminder, At: "2026-09-26T14:04", Text: "Stretch"));
var prompt = await h.CreateAsync(new NewJob("Summary", JobKind.Prompt, At: "2026-09-26T14:04", Text: "Summarise inbox/"));
await h.AdvanceAsync(Minute);
Assert.Equal(("111", "⏰ Stretch"), Assert.Single(h.Delivery.Texts));
Assert.Equal(("111", $"[Scheduled job \"Summary\" ({prompt.Job.Id})]\nSummarise inbox/"), Assert.Single(h.Delivery.Prompts));
}
[Fact]
public async Task A_silent_watcher_makes_no_delivery_calls_and_only_updates_its_run_fields()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
var created = (await h.CreateAsync(Watcher())).Job;
h.Runner.Then(Enumerable.Repeat<ScriptOutcome>(new ScriptOutcome.Silent(), 10).Append(new ScriptOutcome.Output("price 48")).ToArray());
for (var i = 0; i < 10; i++) await h.AdvanceAsync(Minute);
Assert.Equal(10, h.Runner.Requests.Count);
Assert.Empty(h.Delivery.Prompts);
Assert.Empty(h.Delivery.Texts);
Assert.DoesNotContain(h.Delivery.Calls, c => c.StartsWith("RunPrompt", StringComparison.Ordinal) || c.StartsWith("SendText", StringComparison.Ordinal));
var stored = Assert.Single(await h.Store.LoadAsync(CancellationToken.None));
Assert.Equal(created, stored with { LastRunAt = null, LastResult = null }); // nothing but the run fields changed
Assert.Equal(Start.AddMinutes(10), stored.LastRunAt);
Assert.True(stored.LastResult!.Ok);
await h.AdvanceAsync(Minute);
Assert.Equal(("111", $"[Scheduled job \"Price check\" ({created.Id})] The watcher script printed:\nprice 48"), Assert.Single(h.Delivery.Prompts));
Assert.Empty(h.Delivery.Texts);
}
[Fact]
public async Task A_failure_is_reported_once_per_streak()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
var id = (await h.CreateAsync(Watcher())).Job.Id;
h.Runner.Then(
new ScriptOutcome.Failed(1, "boom", false),
new ScriptOutcome.Failed(1, "boom again", false),
new ScriptOutcome.Failed(2, "other", false),
new ScriptOutcome.Failed(null, "slow", true),
new ScriptOutcome.Silent(),
new ScriptOutcome.Failed(2, "back", false));
var counts = new List<int>();
for (var i = 0; i < 6; i++)
{
await h.AdvanceAsync(Minute);
counts.Add(h.Delivery.Texts.Count);
}
Assert.Equal([1, 1, 2, 3, 3, 4], counts);
Assert.Equal($"⚠️ Job \"Price check\" ({id}) failed (exit 1):\n```\nboom\n```", h.Delivery.Texts[0].Text);
Assert.Equal($"⚠️ Job \"Price check\" ({id}) failed (timed out):\n```\nslow\n```", h.Delivery.Texts[2].Text);
Assert.Empty(h.Delivery.Prompts); // failures never go through the model
}
[Fact]
public async Task Overlapping_occurrences_are_skipped_while_a_watcher_runs()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(Watcher());
h.Runner.Hold = new TaskCompletionSource();
h.Time.Advance(Minute);
await Eventually.UntilAsync(() => h.Runner.Requests.Count == 1, "the first run");
h.Time.Advance(Minute);
await h.Scheduler.PendingTick;
h.Time.Advance(Minute);
await h.Scheduler.PendingTick;
Assert.Equal(1, h.Scheduler.FiresStarted); // both occurrences were skipped by their ticks
Assert.Single(h.Runner.Requests);
h.Runner.Hold.SetResult();
h.Runner.Hold = null;
await h.Scheduler.IdleAsync();
await h.AdvanceAsync(Minute);
Assert.Equal(2, h.Runner.Requests.Count);
}
[Fact]
public async Task Overlapping_occurrences_are_skipped_while_a_prompt_run_is_held_open()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(new NewJob("Every minute", JobKind.Prompt, Cron: "* * * * *", Text: "tick"));
h.Delivery.HoldPrompts = new TaskCompletionSource();
h.Time.Advance(Minute);
await Eventually.UntilAsync(() => h.Delivery.Prompts.Count == 1, "the first prompt");
h.Time.Advance(Minute);
await h.Scheduler.PendingTick;
Assert.Equal(1, h.Scheduler.FiresStarted);
Assert.Single(h.Delivery.Prompts);
h.Delivery.HoldPrompts.SetResult();
h.Delivery.HoldPrompts = null;
await h.Scheduler.IdleAsync();
await h.AdvanceAsync(Minute);
Assert.Equal(2, h.Delivery.Prompts.Count);
}
[Fact]
public async Task Targets_are_isolated_in_List_and_CancelAsync()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
var mine = (await h.CreateAsync(new NewJob("Mine", JobKind.Reminder, Cron: "@daily", Text: "a"), "111")).Job;
var theirs = (await h.CreateAsync(new NewJob("Theirs", JobKind.Reminder, Cron: "@daily", Text: "b"), "222")).Job;
Assert.Equal([mine.Id], h.Scheduler.List("111").Select(v => v.Job.Id));
Assert.False(await h.Scheduler.CancelAsync("222", mine.Id, CancellationToken.None));
Assert.True(await h.Scheduler.CancelAsync("111", mine.Id, CancellationToken.None));
Assert.False(await h.Scheduler.CancelAsync("111", mine.Id, CancellationToken.None));
Assert.Equal([theirs.Id], (await h.Store.LoadAsync(CancellationToken.None)).Select(j => j.Id));
}
[Fact]
public async Task Cancel_kills_a_running_watcher_and_drops_its_output()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
var id = (await h.CreateAsync(Watcher())).Job.Id;
h.Runner.Fallback = new ScriptOutcome.Output("late output");
h.Runner.IgnoreCancellation = true; // the outcome still arrives after the cancel; it must be thrown away
h.Runner.Hold = new TaskCompletionSource();
h.Time.Advance(Minute);
await Eventually.UntilAsync(() => h.Runner.Requests.Count == 1, "the run");
Assert.True(await h.Scheduler.CancelAsync("111", id, CancellationToken.None));
Assert.Equal(1, h.Runner.CancellationsSeen);
h.Runner.Hold.SetResult();
await h.Scheduler.IdleAsync();
Assert.Empty(h.Delivery.Prompts);
Assert.Empty(await h.Store.LoadAsync(CancellationToken.None));
}
[Fact]
public async Task An_undeliverable_target_is_skipped_and_its_job_kept()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(new NewJob("Hourly", JobKind.Reminder, Cron: "0 * * * *", Text: "ping"));
await h.CreateAsync(new NewJob("Once", JobKind.Reminder, At: "2026-09-26T14:30", Text: "once"));
h.Delivery.Undeliverable.Add("111");
await h.AdvanceAsync(TimeSpan.FromMinutes(57)); // past 14:30 and 15:00 Berlin
Assert.Empty(h.Delivery.Texts);
var jobs = h.Scheduler.List("111");
Assert.Equal(2, jobs.Count);
Assert.Equal(new DateTimeOffset(2026, 9, 26, 14, 0, 0, TimeSpan.Zero), jobs.Single(j => j.Job.Name == "Hourly").NextRunAt); // advanced
Assert.Equal(2, (await h.Store.LoadAsync(CancellationToken.None)).Count);
}
[Fact]
public async Task StopAsync_kills_scripts_and_returns()
{
var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(Watcher());
h.Runner.Hold = new TaskCompletionSource();
h.Time.Advance(Minute);
await Eventually.UntilAsync(() => h.Runner.Requests.Count == 1, "the run");
await h.Scheduler.StopAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal(1, h.Runner.CancellationsSeen);
await h.DisposeAsync();
}
[Fact]
public async Task StopAsync_gives_up_after_the_grace_period()
{
var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(Watcher());
h.Runner.IgnoreCancellation = true;
h.Runner.Hold = new TaskCompletionSource();
h.Time.Advance(Minute);
await Eventually.UntilAsync(() => h.Runner.Requests.Count == 1, "the run");
var stoppedAt = h.Time.GetUtcNow();
var stop = h.Scheduler.StopAsync(CancellationToken.None);
Assert.False(stop.IsCompleted); // waiting on the fake clock's grace period
// Second by second: the grace timer may be created after the first advances.
await Eventually.UntilAsync(() => { h.Time.Advance(TimeSpan.FromSeconds(1)); return stop.IsCompleted; }, "the stop after the grace period");
Assert.True(h.Time.GetUtcNow() - stoppedAt >= JobScheduler.StopGrace);
await stop;
h.Runner.Hold.SetResult();
await h.DisposeAsync();
}
[Fact]
public async Task A_stored_job_whose_zone_lookup_throws_does_not_stop_the_start()
{
await using var h = new SchedulerHarness();
await h.Store.SaveAsync([
Stored("bad11", JobKind.Reminder, new JobSchedule(null, "*/10 * * * *"), target: "not-a-chat"),
Stored("good1", JobKind.Reminder, new JobSchedule(null, "*/10 * * * *")),
], CancellationToken.None);
h.Delivery.BadTargets.Add("not-a-chat");
await h.StartAsync();
Assert.True(h.Scheduler.IsStarted);
Assert.Null(Assert.Single(h.Scheduler.List("not-a-chat")).NextRunAt); // kept, unarmed
Assert.Equal(new DateTimeOffset(2026, 9, 26, 12, 10, 0, TimeSpan.Zero), Assert.Single(h.Scheduler.List("111")).NextRunAt);
await h.AdvanceAsync(TimeSpan.FromMinutes(7));
Assert.Equal(("111", "⏰ Call the dentist"), Assert.Single(h.Delivery.Texts));
}
[Fact]
public async Task A_one_off_whose_delivery_is_cut_off_by_shutdown_is_kept_and_fires_after_the_next_start()
{
var h = new SchedulerHarness();
await h.Store.SaveAsync([Stored("k7m2q", JobKind.Reminder, new JobSchedule(Start.AddSeconds(-30), null))], CancellationToken.None);
h.Delivery.HoldTexts = new TaskCompletionSource(); // the send is in flight when the bot stops
await h.StartAsync();
await Eventually.UntilAsync(() => h.Delivery.Texts.Count == 1, "the reminder send");
await h.Scheduler.StopAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal("k7m2q", Assert.Single(await h.Store.LoadAsync(CancellationToken.None)).Id);
var delivery = new FakeJobDelivery();
await using (var restarted = new JobScheduler(h.Options, h.Runner, h.Redactor, h.Time, Microsoft.Extensions.Logging.Abstractions.NullLogger<JobScheduler>.Instance))
{
await restarted.StartAsync(delivery, CancellationToken.None);
await restarted.IdleAsync();
}
Assert.Equal(("111", "⏰ Call the dentist"), Assert.Single(delivery.Texts));
Assert.Empty(await h.Store.LoadAsync(CancellationToken.None));
await h.DisposeAsync();
}
[Fact]
public async Task A_zone_change_recomputes_cron_and_leaves_one_offs()
{
var start = new DateTimeOffset(2026, 9, 26, 23, 30, 0, TimeSpan.Zero); // Sun 01:30 Berlin, Sun 08:30 Tokyo
await using var h = new SchedulerHarness(start);
await h.StartAsync();
var cron = (await h.CreateAsync(new NewJob("Morning", JobKind.Reminder, Cron: "0 9 * * *", Text: "Good morning"))).Job;
var once = (await h.CreateAsync(new NewJob("Dentist", JobKind.Reminder, At: "2026-09-27T10:00", Text: "Dentist"))).Job;
var other = (await h.CreateAsync(new NewJob("Other", JobKind.Reminder, Cron: "0 9 * * *", Text: "x"), "222")).Job;
var berlinNine = new DateTimeOffset(2026, 9, 27, 7, 0, 0, TimeSpan.Zero);
Assert.Equal(start.AddHours(1), h.Scheduler.NextWakeAt); // capped
h.Delivery.Zones["111"] = SchedulerHarness.Tokyo;
var change = await h.Scheduler.TimeZoneChangedAsync("111", CancellationToken.None);
Assert.Equal(new ZoneChange(1, 1), change);
var views = h.Scheduler.List("111").ToDictionary(v => v.Job.Id);
Assert.Equal(new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero), views[cron.Id].NextRunAt); // 09:00 Tokyo
Assert.Equal(new DateTimeOffset(2026, 9, 27, 8, 0, 0, TimeSpan.Zero), views[once.Id].NextRunAt); // unchanged instant
Assert.Equal(berlinNine, Assert.Single(h.Scheduler.List("222")).NextRunAt);
Assert.Equal(new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero), h.Scheduler.NextWakeAt); // re-armed
Assert.Equal(other.Id, h.Scheduler.List("222")[0].Job.Id);
}
[Fact]
public async Task Watcher_output_and_failure_tails_are_redacted()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(Watcher());
h.Runner.Then(new ScriptOutcome.Output("token=hunter2-secret"), new ScriptOutcome.Failed(1, "auth hunter2-secret rejected", false));
await h.AdvanceAsync(Minute);
await h.AdvanceAsync(Minute);
var prompt = Assert.Single(h.Delivery.Prompts).Prompt;
Assert.EndsWith("token=[REDACTED]", prompt);
var alert = Assert.Single(h.Delivery.Texts).Text;
Assert.Contains("auth [REDACTED] rejected", alert);
Assert.DoesNotContain("hunter2", prompt + alert);
Assert.Equal("auth [REDACTED] rejected", Assert.Single(await h.Store.LoadAsync(CancellationToken.None)).LastResult!.StderrTail);
}
[Fact]
public async Task Delivery_failures_are_logged_and_the_job_still_advances()
{
await using var h = new SchedulerHarness();
await h.StartAsync();
await h.CreateAsync(new NewJob("Tick", JobKind.Reminder, Cron: "* * * * *", Text: "tick"));
h.Delivery.FailSends = new InvalidOperationException("Telegram is down");
await h.AdvanceAsync(Minute);
var view = Assert.Single(h.Scheduler.List("111"));
Assert.Equal(Start.AddMinutes(1), view.Job.LastRunAt);
Assert.Equal(Start.AddMinutes(2), view.NextRunAt);
}
[Fact]
public async Task CreateAsync_validates_and_caps_jobs_per_target()
{
await using var h = new SchedulerHarness(configure: o => o.MaxJobsPerTarget = 2);
await h.StartAsync();
await h.CreateAsync(new NewJob("a", JobKind.Reminder, Cron: "@daily", Text: "a"));
await h.CreateAsync(new NewJob("b", JobKind.Reminder, Cron: "@daily", Text: "b"));
var full = await Assert.ThrowsAsync<JobValidationException>(() => h.CreateAsync(new NewJob("c", JobKind.Reminder, Cron: "@daily", Text: "c")));
Assert.Equal("This chat already has 2 scheduled jobs, the maximum. Cancel one first.", full.Message);
await h.CreateAsync(new NewJob("c", JobKind.Reminder, Cron: "@daily", Text: "c"), "222"); // other targets have their own cap
await Assert.ThrowsAsync<JobValidationException>(() => h.CreateAsync(new NewJob("bad", JobKind.Reminder, Cron: "0 0 9 * * *", Text: "x"), "333"));
var ids = h.Scheduler.List("111").Select(v => v.Job.Id).ToList();
Assert.All(ids, id => Assert.Matches("^[abcdefghjkmnpqrstuvwxyz23456789]{5}$", id));
}
[Fact]
public async Task Watcher_runs_with_the_target_workspace_zone_and_timeout()
{
await using var h = new SchedulerHarness();
h.Delivery.Zones["111"] = SchedulerHarness.Tokyo;
await h.StartAsync();
var job = (await h.CreateAsync(Watcher())).Job;
await h.CreateAsync(new NewJob("Quick", JobKind.Watcher, Cron: "* * * * *", Command: "true", TimeoutSeconds: 5), "222");
await h.AdvanceAsync(Minute);
var requests = h.Runner.Requests.OrderBy(r => r.Target).ToList();
Assert.Equal(new ScriptRequest("scripts/price.sh", Path.Combine(h.Delivery.WorkspaceRoot, "111"), TimeSpan.FromSeconds(300), "111", job.Id, "Asia/Tokyo"), requests[0]);
Assert.Equal(TimeSpan.FromSeconds(5), requests[1].Timeout);
Assert.Equal("Europe/Berlin", requests[1].TimeZoneId);
}
}