Playa/extensions/Volcanic.AgenticCore.Extensions.Scheduler/JobScheduler.SkillJobs.cs

using Microsoft.Extensions.Logging;
using Volcanic.AgenticCore.Abstractions.Skills;

namespace Volcanic.AgenticCore.Extensions.Scheduler;

/// <summary>Skill jobs: files are the source of truth, the scheduler mirrors them (see the class summary).</summary>
public sealed partial class JobScheduler
{
    /// <summary>
    /// The key of a job in the scheduler's tables. A skill job's id (<c>&lt;skill&gt;/&lt;job&gt;</c>) is only unique within
    /// its target, so its key carries the target; a <c>schedule</c> job's random id is unique across targets.
    /// </summary>
    private static string KeyOf(ScheduledJob job) => job.Skill is null ? job.Id : $"{job.Target}\n{job.Id}";

    /// <summary>The target's skill job files as of the last sync, in path order, with next runs and last results.</summary>
    public IReadOnlyList<SkillJobView> SkillJobs(string target)
    {
        lock (_lock)
        {
            if (!_skillViews.TryGetValue(target, out var views)) return [];
            var states = _skillStates.GetValueOrDefault(target);
            return views.Select(v => v with
            {
                NextRunAt = v.Status == SkillJobStatus.Active && _jobs.TryGetValue($"{target}\n{v.Id}", out var entry) ? entry.Next : null,
                LastResult = states?.GetValueOrDefault(v.Id)?.LastResult,
            }).ToList();
        }
    }

    /// <summary>The target's skill job with this id (case-insensitive), or null.</summary>
    public SkillJobView? FindSkillJob(string target, string id) =>
        SkillJobs(target).FirstOrDefault(v => string.Equals(v.Id, id.Trim(), StringComparison.OrdinalIgnoreCase));

    /// <summary>
    /// Re-read the target's <c>jobs/</c> files and bring the schedule in line: a new file is scheduled, a changed one
    /// re-planned (an unchanged one keeps its next run), a removed, disabled or broken one taken off (a running watcher is
    /// killed). Returns the target's views. Does nothing before the scheduler has started.
    /// </summary>
    public async Task<IReadOnlyList<SkillJobView>> SyncSkillJobsAsync(string target, CancellationToken cancellationToken)
    {
        var delivery = Volatile.Read(ref _delivery);
        if (delivery is null || !IsStarted) return SkillJobs(target);
        await _syncGate.WaitAsync(cancellationToken).ConfigureAwait(false);
        try
        {
            await SyncLockedAsync(delivery, target, cancellationToken).ConfigureAwait(false);
        }
        finally
        {
            _syncGate.Release();
        }
        Arm();
        return SkillJobs(target);
    }

    private async Task SyncLockedAsync(IJobDelivery delivery, string target, CancellationToken cancellationToken)
    {
        var workspace = delivery.WorkspaceFor(target);
        var zone = delivery.TimeZoneFor(target);
        var now = _time.GetUtcNow();
        var files = SkillFiles.ReadJobs(workspace);
        bool known;
        lock (_lock) known = _skillStates.ContainsKey(target);
        var loaded = known ? null : new SkillJobStateStore(SkillJobStateStore.PathIn(workspace), _time, _logger).Load();

        var stateChanged = false;
        var cancels = new List<CancellationTokenSource>();
        lock (_lock)
        {
            if (_stopped) return;
            if (loaded is not null) _skillStates.TryAdd(target, loaded);
            var states = _skillStates[target];
            var views = new List<SkillJobView>();
            var active = new List<ScheduledJob>();
            foreach (var file in files)
            {
                var parsed = SkillJobParser.Parse(file, workspace, target, zone, now);
                var view = parsed.View;
                if (parsed.Job is { } job)
                {
                    var state = states.GetValueOrDefault(job.Id);
                    var status = SkillJobStatus.Active;
                    if (job.Skill!.AtText is { } atText)
                    {
                        if (state?.DoneAt == atText) status = SkillJobStatus.Done;
                        else
                        {
                            if (state?.AtText != atText || state.AtSeenAt is null)
                            {
                                states[job.Id] = state = (state ?? new SkillJobState()) with { AtText = atText, AtSeenAt = now };
                                stateChanged = true;
                            }
                            if (job.Schedule.At < state.AtSeenAt - JobValidation.PastGrace) status = SkillJobStatus.Past;
                        }
                    }
                    if (status == SkillJobStatus.Active && active.Count >= _options.MaxSkillJobsPerTarget) status = SkillJobStatus.OverLimit;
                    view = view with
                    {
                        Status = status,
                        Problem = status == SkillJobStatus.OverLimit ? $"over the limit of {_options.MaxSkillJobsPerTarget} skill jobs in this workspace; not scheduled" : null,
                    };
                    if (status == SkillJobStatus.Active) active.Add(job with { LastRunAt = state?.LastRunAt, LastResult = state?.LastResult });
                }
                views.Add(view);
            }
            if (active.Count >= _options.MaxSkillJobsPerTarget && views.Any(v => v.Status == SkillJobStatus.OverLimit))
                _logger.LogWarning("Target {Target}: more than {Max} skill jobs; the rest aren't scheduled", target, _options.MaxSkillJobsPerTarget);

            // Reconcile the scheduler's entries with the files.
            var keys = new HashSet<string>(active.Select(KeyOf), StringComparer.Ordinal);
            foreach (var (key, entry) in _jobs.Where(e => e.Value.Job.Skill is not null && e.Value.Job.Target == target && !keys.Contains(e.Key)).ToList())
            {
                _jobs.Remove(key);
                if (_inFlight.TryGetValue(key, out var fire)) cancels.Add(fire.Cancel);
                _logger.LogInformation("Skill job {Id} of target {Target} was removed, disabled or broken; unscheduled", entry.Job.Id, target);
            }
            foreach (var job in active)
            {
                var key = KeyOf(job);
                if (_jobs.TryGetValue(key, out var existing) && Same(existing.Job, job)) continue;
                DateTimeOffset? next;
                try
                {
                    next = job.Schedule.At ?? Schedules.NextCron(job.Schedule.Cron!, now, zone);
                }
                catch (Exception ex)
                {
                    next = null;
                    _logger.LogError(ex, "Skill job {Id} of target {Target}: computing the next occurrence failed", job.Id, target);
                }
                _jobs[key] = new Entry(existing is null ? job : job with { CreatedAt = existing.Job.CreatedAt }) { Next = next };
                _logger.LogInformation("Skill job {Id} of target {Target} {What}; next {Next}", job.Id, target, existing is null ? "scheduled" : "re-planned", next);
            }

            // Forget the state of files that are gone.
            var ids = new HashSet<string>(files.Select(f => f.Id), StringComparer.Ordinal);
            foreach (var id in states.Keys.Where(id => !ids.Contains(id)).ToList())
            {
                states.Remove(id);
                stateChanged = true;
            }
            _skillViews[target] = views;
        }
        foreach (var cancel in cancels) await TryCancelAsync(cancel).ConfigureAwait(false);
        if (stateChanged) await SaveSkillStateAsync(delivery, target, cancellationToken).ConfigureAwait(false);
    }

    /// <summary>The same definition, ignoring run fields.</summary>
    private static bool Same(ScheduledJob a, ScheduledJob b) => Definition(a) == Definition(b);

    private static ScheduledJob Definition(ScheduledJob job) => job with { CreatedAt = default, LastRunAt = null, LastResult = null };

    /// <summary>A skill job fired: run fields to its state; a one-off is done (until its <c>at</c> changes) and unscheduled.</summary>
    private async Task RecordSkillJobAsync(ScheduledJob job, JobResult result)
    {
        var key = KeyOf(job);
        var delivery = _delivery!;
        lock (_lock)
        {
            var atText = job.Skill!.AtText;
            if (_jobs.TryGetValue(key, out var entry))
            {
                if (atText is null) entry.Job = entry.Job with { LastRunAt = result.At, LastResult = result };
                else if (entry.Job.Skill?.AtText == atText) _jobs.Remove(key);
            }
            // A file deleted meanwhile keeps no state.
            if (!_skillViews.TryGetValue(job.Target, out var views) || views.FindIndex(v => v.Id == job.Id) is var index && index < 0) return;
            var states = _skillStates[job.Target];
            var state = (states.GetValueOrDefault(job.Id) ?? new SkillJobState()) with { LastRunAt = result.At, LastResult = result };
            if (atText is not null)
            {
                state = state with { DoneAt = atText };
                if (views[index].Status == SkillJobStatus.Active) views[index] = views[index] with { Status = SkillJobStatus.Done };
            }
            states[job.Id] = state;
        }
        try
        {
            await SaveSkillStateAsync(delivery, job.Target, CancellationToken.None).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
        {
            _logger.LogError(ex, "Saving the skill job state failed");
        }
        Arm();
    }

    private async Task SaveSkillStateAsync(IJobDelivery delivery, string target, CancellationToken cancellationToken)
    {
        await _saveGate.WaitAsync(cancellationToken).ConfigureAwait(false);
        try
        {
            Dictionary<string, SkillJobState> snapshot;
            lock (_lock) snapshot = new(_skillStates[target], StringComparer.Ordinal);
            await new SkillJobStateStore(SkillJobStateStore.PathIn(delivery.WorkspaceFor(target)), _time, _logger).SaveAsync(snapshot, cancellationToken).ConfigureAwait(false);
        }
        finally
        {
            _saveGate.Release();
        }
    }

    /// <summary>The periodic sync: queued unless the previous one is still running.</summary>
    private void OnSyncTimer()
    {
        lock (_lock)
        {
            if (_stopped || !_pendingSync.IsCompleted) return;
            _pendingSync = Task.Run(SyncAllSkillJobsAsync);
        }
    }

    /// <summary>Every deliverable known target, plus every target that already has skill jobs (its files may be gone). Never throws.</summary>
    private async Task SyncAllSkillJobsAsync()
    {
        if (Volatile.Read(ref _delivery) is not { } delivery) return;
        List<string> targets;
        try
        {
            targets = delivery.KnownTargets().Where(delivery.IsDeliverable).ToList();
            lock (_lock) targets.AddRange(_skillViews.Keys);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Listing the targets for the skill job sync failed");
            return;
        }
        foreach (var target in targets.Distinct(StringComparer.Ordinal))
        {
            try
            {
                await SyncSkillJobsAsync(target, _stopping.Token).ConfigureAwait(false);
            }
            catch (OperationCanceledException) when (_stopping.IsCancellationRequested)
            {
                return;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Syncing the skill jobs of target {Target} failed", target);
            }
        }
    }
}