using Microsoft.Extensions.Logging; using Volcanic.AgenticCore.Abstractions.Processes; namespace Volcanic.AgenticCore.Extensions.Scheduler; /// /// The process-wide scheduler: loads jobs.json, arms one timer for the soonest job /// (capped at 1 h, re-armed on every tick, create, cancel, fire and zone change), and fires jobs through the host's /// . A fire is tracked in flight and never awaited by the tick; an occurrence that comes due /// while the job is still in flight is skipped. A watcher's script run touches only the runner, IsDeliverable, /// WorkspaceFor and TimeZoneFor: a silent run makes no delivery call at all. /// public sealed class JobScheduler : IAsyncDisposable { public static readonly TimeSpan MaxTimerDelay = TimeSpan.FromHours(1); public static readonly TimeSpan LateThreshold = TimeSpan.FromSeconds(60); public static readonly TimeSpan StopGrace = TimeSpan.FromSeconds(10); private const string IdAlphabet = "abcdefghjkmnpqrstuvwxyz23456789"; private readonly SchedulerOptions _options; private readonly IScriptRunner _runner; private readonly ISecretRedactor _redactor; private readonly TimeProvider _time; private readonly ILogger _logger; private readonly JobStore _store; private readonly SemaphoreSlim _saveGate = new(1, 1); private readonly Lock _lock = new(); private readonly Dictionary _jobs = new(StringComparer.Ordinal); private readonly Dictionary _inFlight = new(StringComparer.Ordinal); private readonly CancellationTokenSource _stopping = new(); private IJobDelivery? _delivery; private ITimer? _timer; private DateTimeOffset? _wakeAt; private bool _stopped; private Task _pendingTick = Task.CompletedTask; private int _firesStarted; public JobScheduler(SchedulerOptions options, IScriptRunner runner, ISecretRedactor redactor, TimeProvider time, ILogger logger) { ArgumentNullException.ThrowIfNull(options); if (string.IsNullOrWhiteSpace(options.StorePath)) throw new ArgumentException("SchedulerOptions.StorePath is required.", nameof(options)); _options = options; _runner = runner; _redactor = redactor; _time = time; _logger = logger; _store = new JobStore(options.StorePath, time, logger); } public bool IsStarted { get { lock (_lock) return _delivery is not null && !_stopped; } } public DateTimeOffset Now => _time.GetUtcNow(); public string StorePath => _store.Path; /// When the timer fires next (tests). internal DateTimeOffset? NextWakeAt { get { lock (_lock) return _wakeAt; } } /// How many fires have been started (tests: a skipped occurrence doesn't count). internal int FiresStarted { get { lock (_lock) return _firesStarted; } } /// The latest queued tick, without the fires it started (tests; unlike it doesn't wait for held fires). internal Task PendingTick { get { lock (_lock) return _pendingTick; } } /// Load the store, fire overdue one-offs (late), compute cron occurrences strictly after now (no catch-up), arm the timer. public async Task StartAsync(IJobDelivery delivery, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(delivery); var jobs = await _store.LoadAsync(cancellationToken).ConfigureAwait(false); var now = _time.GetUtcNow(); lock (_lock) { if (_delivery is not null || _stopped) throw new InvalidOperationException("The scheduler was already started."); _delivery = delivery; foreach (var job in jobs) { var entry = new Entry(job); if (job.Schedule.At is { } at) entry.Next = at; else { try { entry.Next = NextCron(job, now); var missed = Schedules.NextCron(job.Schedule.Cron!, job.LastRunAt ?? job.CreatedAt, delivery.TimeZoneFor(job.Target)); if (missed is { } m && m < now) _logger.LogInformation("Job {Id} ({Name}): occurrences missed while the bot was down are skipped; next {Next}", job.Id, job.Name, entry.Next); } catch (Exception ex) { // One bad entry never keeps the host down: it stays listed but unarmed until a zone change recomputes it. entry.Next = null; _logger.LogError(ex, "Job {Id} ({Name}): computing the next occurrence on start failed; the job isn't armed", job.Id, job.Name); } } _jobs[job.Id] = entry; } _timer = _time.CreateTimer(_ => OnTimer(), null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); } Tick(); // overdue one-offs fire now } /// Disarm, kill running scripts, and wait at most for in-flight fires. public async Task StopAsync(CancellationToken cancellationToken) { Fire[] fires; ITimer? timer; lock (_lock) { if (_stopped) return; _stopped = true; fires = [.. _inFlight.Values]; timer = _timer; } timer?.Dispose(); await _stopping.CancelAsync().ConfigureAwait(false); // every fire's token is linked to this one try { await Task.WhenAll(fires.Select(f => f.Task)).WaitAsync(StopGrace, _time, cancellationToken).ConfigureAwait(false); } catch (TimeoutException) { _logger.LogWarning("Scheduler stop: {Count} job(s) still running after {Grace}", fires.Count(f => !f.Task.IsCompleted), StopGrace); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } } public ValueTask DisposeAsync() => new(StopAsync(CancellationToken.None)); public IReadOnlyList List(string target) { lock (_lock) return _jobs.Values.Where(e => e.Job.Target == target) .Select(e => new JobView(e.Job, e.Next)) .OrderBy(v => v.NextRunAt ?? DateTimeOffset.MaxValue).ThenBy(v => v.Job.CreatedAt) .ToList(); } public TimeZoneInfo TimeZoneFor(string target) => Volatile.Read(ref _delivery)?.TimeZoneFor(target) ?? TimeZoneInfo.Local; /// Validate (an offset-less at is read in the target's zone), persist, arm. Throws . public async Task CreateAsync(string target, NewJob job, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(job); if (!IsStarted) throw new InvalidOperationException("The scheduler isn't running."); var zone = TimeZoneFor(target); var now = _time.GetUtcNow(); var schedule = JobValidation.ValidateNew(job, now, zone); ScheduledJob created; DateTimeOffset? next; lock (_lock) { if (_jobs.Values.Count(e => e.Job.Target == target) >= _options.MaxJobsPerTarget) throw new JobValidationException($"This chat already has {_options.MaxJobsPerTarget} scheduled jobs, the maximum. Cancel one first."); created = new ScheduledJob { Id = NewId(), Target = target, Name = job.Name.Trim(), Kind = job.Kind, Schedule = schedule, Text = job.Kind == JobKind.Watcher ? null : job.Text, Command = job.Kind == JobKind.Watcher ? job.Command : null, TimeoutSeconds = job.TimeoutSeconds, CreatedAt = now, }; next = schedule.At ?? Schedules.NextCron(schedule.Cron!, now, zone); _jobs[created.Id] = new Entry(created) { Next = next }; } await PersistAsync(cancellationToken).ConfigureAwait(false); Arm(); return new JobView(created, next); } /// Remove a job of . A running watcher is killed and its outcome dropped; a prompt run in progress isn't aborted. False: unknown, or another target's. public async Task CancelAsync(string target, string id, CancellationToken cancellationToken) => await CancelJobAsync(target, id, cancellationToken).ConfigureAwait(false) is not null; /// returning the removed job (null: unknown, or another target's); looked up and removed in one step. public async Task CancelJobAsync(string target, string id, CancellationToken cancellationToken) { Fire? running; ScheduledJob removed; lock (_lock) { if (!_jobs.TryGetValue(id, out var entry) || entry.Job.Target != target) return null; removed = entry.Job; _jobs.Remove(id); _inFlight.TryGetValue(id, out running); } if (running is not null) await TryCancelAsync(running.Cancel).ConfigureAwait(false); await PersistAsync(cancellationToken).ConfigureAwait(false); Arm(); return removed; } /// The target's zone changed: cron jobs get their next occurrence in the new zone (strictly after now), one-offs keep their instant. public Task TimeZoneChangedAsync(string target, CancellationToken cancellationToken) { var now = _time.GetUtcNow(); var zone = TimeZoneFor(target); int cron = 0, oneOffs = 0; lock (_lock) { foreach (var entry in _jobs.Values.Where(e => e.Job.Target == target)) { if (entry.Job.Schedule.Cron is { } expression) { cron++; entry.Next = Schedules.NextCron(expression, now, zone); } else oneOffs++; } } Arm(); return Task.FromResult(new ZoneChange(cron, oneOffs)); } /// Wait for the pending tick and every fire in flight (tests). internal async Task IdleAsync() { while (true) { Task tick; Task[] fires; lock (_lock) { tick = _pendingTick; fires = _inFlight.Values.Select(f => f.Task).ToArray(); } if (tick.IsCompleted && fires.All(f => f.IsCompleted)) return; await Task.WhenAll(fires.Append(tick)).WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); } } // ---- Timer and ticks ---- /// The timer callback: queues a tick behind the previous one (it may run synchronously inside ITimer.Change). private void OnTimer() { lock (_lock) { if (_stopped) return; _pendingTick = _pendingTick.ContinueWith(_ => Tick(), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default); } } /// Starts every due job that isn't in flight, advances each due job's next occurrence, re-arms. Never awaits a fire. private void Tick() { try { var now = _time.GetUtcNow(); lock (_lock) { if (_stopped || _delivery is null) return; foreach (var entry in _jobs.Values) { if (entry.Next is not { } next || next > now) continue; if (_inFlight.ContainsKey(entry.Job.Id)) _logger.LogInformation("Job {Id} ({Name}) is still running; skipped the occurrence due {Due}", entry.Job.Id, entry.Job.Name, next); else StartFire(entry.Job, next, now); // Cron: the next occurrence strictly after now. One-off: none (it's removed once its fire completes). entry.Next = null; if (entry.Job.Schedule.Cron is null) continue; try { entry.Next = NextCron(entry.Job, now); } catch (Exception ex) { // Left unarmed rather than due forever (a zero-delay timer loop); a restart or zone change recomputes it. _logger.LogError(ex, "Job {Id} ({Name}): computing the next occurrence failed", entry.Job.Id, entry.Job.Name); } } } } finally { Arm(); } } /// /// Computes the wake time and applies it to the timer under the lock, so two concurrent arms can't apply an older /// wake after a newer one. A synchronous callback from Change (zero delay) is safe: is /// re-entrant and only queues a tick. /// private void Arm() { lock (_lock) { if (_stopped || _timer is null) return; var now = _time.GetUtcNow(); var soonest = _jobs.Values.Select(e => e.Next).Where(n => n is not null).DefaultIfEmpty(null).Min(); var wake = soonest is { } s && s - now < MaxTimerDelay ? s : now + MaxTimerDelay; if (wake < now) wake = now; _wakeAt = wake; try { _timer.Change(wake - now, Timeout.InfiniteTimeSpan); } catch (ObjectDisposedException) { // Stopped meanwhile. } } } /// Under the lock. The next cron occurrence strictly after in the target's current zone. private DateTimeOffset? NextCron(ScheduledJob job, DateTimeOffset after) => Schedules.NextCron(job.Schedule.Cron!, after, _delivery!.TimeZoneFor(job.Target)); // ---- Firing ---- /// Called under the lock: registers the fire in flight before it can complete. private void StartFire(ScheduledJob job, DateTimeOffset dueAt, DateTimeOffset now) { var cancel = CancellationTokenSource.CreateLinkedTokenSource(_stopping.Token); var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var task = RunFireAsync(gate.Task, job, dueAt, now, cancel); _inFlight[job.Id] = new Fire(task, cancel); _firesStarted++; gate.SetResult(); } private async Task RunFireAsync(Task gate, ScheduledJob job, DateTimeOffset dueAt, DateTimeOffset now, CancellationTokenSource cancel) { await gate.ConfigureAwait(false); try { await FireAsync(job, dueAt, now, cancel.Token).ConfigureAwait(false); } catch (Exception ex) { _logger.LogError(ex, "Job {Id} ({Name}) failed to fire", job.Id, job.Name); } finally { lock (_lock) _inFlight.Remove(job.Id); cancel.Dispose(); } } private async Task FireAsync(ScheduledJob job, DateTimeOffset dueAt, DateTimeOffset now, CancellationToken scriptToken) { var delivery = _delivery!; if (!delivery.IsDeliverable(job.Target)) { // Kept: a cron job has already advanced; a one-off fires again after the next start. _logger.LogWarning("Job {Id} ({Name}): target {Target} can't be served any more; skipped", job.Id, job.Name, job.Target); return; } var zone = delivery.TimeZoneFor(job.Target); DateTimeOffset? lateDue = now - dueAt > LateThreshold ? dueAt : null; JobResult? result; switch (job.Kind) { case JobKind.Reminder: result = await DeliverAsync(job, ct => delivery.SendTextAsync(job.Target, JobMessages.Reminder(job, lateDue, zone), ct)).ConfigureAwait(false) ? new JobResult(true, null, false, null, _time.GetUtcNow()) : null; break; case JobKind.Prompt: result = await DeliverAsync(job, ct => delivery.RunPromptAsync(job.Target, JobMessages.Prompt(job, lateDue, zone), ct)).ConfigureAwait(false) ? new JobResult(true, null, false, null, _time.GetUtcNow()) : null; break; default: result = await RunWatcherAsync(job, delivery, zone, scriptToken).ConfigureAwait(false); break; } if (result is not null) await RecordAsync(job.Id, result).ConfigureAwait(false); } /// Null when the run or its delivery was cancelled (job cancelled or scheduler stopping): the outcome is dropped. private async Task RunWatcherAsync(ScheduledJob job, IJobDelivery delivery, TimeZoneInfo zone, CancellationToken scriptToken) { var timeout = job.TimeoutSeconds is { } seconds ? TimeSpan.FromSeconds(seconds) : _options.DefaultScriptTimeout; var request = new ScriptRequest(job.Command!, delivery.WorkspaceFor(job.Target), timeout, job.Target, job.Id, ZoneNames.Iana(zone)); ScriptOutcome outcome; try { outcome = await _runner.RunAsync(request, scriptToken).ConfigureAwait(false); } catch (OperationCanceledException) when (scriptToken.IsCancellationRequested) { return null; } if (scriptToken.IsCancellationRequested || !Exists(job.Id)) return null; var at = _time.GetUtcNow(); switch (outcome) { case ScriptOutcome.Output output: // The condition is met: exactly one agent prompt. Redacted: watcher stdout isn't a tool result. return await DeliverAsync(job, ct => delivery.RunPromptAsync(job.Target, JobMessages.WatcherMatch(job, _redactor.Redact(output.Text)), ct)).ConfigureAwait(false) ? new JobResult(true, 0, false, null, at) : null; case ScriptOutcome.Failed failed: var result = new JobResult(false, failed.ExitCode, failed.TimedOut, _redactor.Redact(failed.StderrTail), at); if (!IsRepeat(LastResult(job.Id), result) && !await DeliverAsync(job, ct => delivery.SendTextAsync(job.Target, JobMessages.Failure(job, result), ct)).ConfigureAwait(false)) return null; // the alert was cut off by shutdown: not recorded, so the next failure alerts again return result; default: return new JobResult(true, 0, false, null, at); // silent: nothing but LastRunAt/LastResult } } /// The previous run failed the same way (exit code and timeout): the alert isn't sent again. private static bool IsRepeat(JobResult? previous, JobResult current) => previous is { Ok: false } p && p.ExitCode == current.ExitCode && p.TimedOut == current.TimedOut; /// /// A delivery exception is logged and swallowed; the job's state still advances (no redelivery queue). False only /// when shutdown interrupted the delivery: the fire then isn't recorded, so a one-off stays stored and fires again /// after the next start (at-least-once). /// private async Task DeliverAsync(ScheduledJob job, Func deliver) { try { await deliver(_stopping.Token).ConfigureAwait(false); // a prompt run isn't aborted by a cancel, only by shutdown return true; } catch (OperationCanceledException) when (_stopping.IsCancellationRequested) { _logger.LogInformation("Job {Id} ({Name}): delivery interrupted by shutdown; kept for the next start", job.Id, job.Name); return false; } catch (Exception ex) { _logger.LogError(ex, "Job {Id} ({Name}): delivery to {Target} failed", job.Id, job.Name, job.Target); return true; } } /// Update and persist the run fields; a one-off is removed now that its fire has completed. private async Task RecordAsync(string id, JobResult result) { lock (_lock) { if (!_jobs.TryGetValue(id, out var entry)) return; // cancelled meanwhile if (entry.Job.Schedule.At is not null) _jobs.Remove(id); else entry.Job = entry.Job with { LastRunAt = result.At, LastResult = result }; } try { await PersistAsync(CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { _logger.LogError(ex, "Saving the job store failed"); } Arm(); } /// Saves the current jobs. The snapshot is taken inside the gate, so a later save never writes older state. private async Task PersistAsync(CancellationToken cancellationToken) { await _saveGate.WaitAsync(cancellationToken).ConfigureAwait(false); try { List snapshot; lock (_lock) snapshot = [.. _jobs.Values.Select(e => e.Job).OrderBy(j => j.CreatedAt).ThenBy(j => j.Id, StringComparer.Ordinal)]; await _store.SaveAsync(snapshot, cancellationToken).ConfigureAwait(false); } finally { _saveGate.Release(); } } private static async Task TryCancelAsync(CancellationTokenSource cancel) { try { await cancel.CancelAsync().ConfigureAwait(false); } catch (ObjectDisposedException) { // The fire completed meanwhile. } } private bool Exists(string id) { lock (_lock) return _jobs.ContainsKey(id); } private JobResult? LastResult(string id) { lock (_lock) return _jobs.TryGetValue(id, out var e) ? e.Job.LastResult : null; } /// Under the lock. Not reused while a cancelled job with that id is still in flight. private string NewId() { string id; do id = new string(Random.Shared.GetItems(IdAlphabet.AsSpan(), 5)); while (_jobs.ContainsKey(id) || _inFlight.ContainsKey(id)); return id; } private sealed class Entry(ScheduledJob job) { public ScheduledJob Job { get; set; } = job; public DateTimeOffset? Next { get; set; } } private sealed record Fire(Task Task, CancellationTokenSource Cancel); }