using System.Diagnostics; using System.Text; using Volcanic.AgenticCore.Abstractions.Messages; using Volcanic.AgenticCore.Abstractions.Skills; using Volcanic.AgenticCore.Features.Agent; using Volcanic.AgenticCore.Testing; using Volcanic.AgenticCore.Tests.Support; namespace Volcanic.AgenticCore.Tests.Features; public class SkillHookFilesTests : IDisposable { private readonly string _cwd = Path.Combine(Path.GetTempPath(), "volcanic-skill-hooks", Guid.NewGuid().ToString("N")); public SkillHookFilesTests() => Directory.CreateDirectory(_cwd); public void Dispose() { try { Directory.Delete(_cwd, true); } catch (IOException) { } } private SkillHookSet? Read(string frontmatter, string dir = "gtd") { var skillDir = Path.Combine(SkillFiles.Root(_cwd), dir); Directory.CreateDirectory(skillDir); File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), $"---\n{frontmatter}\n---\nBody\n"); return SkillHookFiles.Read(_cwd, skillDir); } [Fact] public void Reads_every_field_with_defaults_and_comments() { var set = Read(""" name: gtd description: GTD session-start: bin/gtd brief # start turn-end: "bin/gtd check --quiet" turn-end-timeout: 20 protected-paths: Входящие/, Сегодня , ./Проекты// protected-hint: use gtd move """)!; Assert.Equal("gtd", set.Skill); Assert.Equal(new SkillHookCommand("bin/gtd brief", TimeSpan.FromSeconds(30)), set.SessionStart); Assert.Equal(new SkillHookCommand("bin/gtd check --quiet", TimeSpan.FromSeconds(20)), set.TurnEnd); Assert.Equal(["Входящие", "Сегодня", "Проекты"], set.ProtectedPaths); Assert.Equal("use gtd move", set.ProtectedHint); Assert.Empty(set.Errors); Assert.Equal("hooks: session-start, turn-end; 3 protected paths", set.Summary()); } [Fact] public void A_flow_list_with_quotes_works_and_NFD_becomes_NFC() { var nfd = "Мой".Normalize(NormalizationForm.FormD); var set = Read($"description: x\nprotected-paths: [\"Входящие/\", '{nfd}/']")!; Assert.Equal(["Входящие", "Мой".Normalize(NormalizationForm.FormC)], set.ProtectedPaths); } [Fact] public void A_skill_without_the_fields_declares_nothing() { var set = Read("name: plain\ndescription: nothing else")!; Assert.True(set.IsEmpty); Assert.Empty(SkillHookFiles.ReadAll(_cwd)); } [Theory] [InlineData("session-start-timeout: 0\nsession-start: x", "session-start-timeout")] [InlineData("turn-end-timeout: 121\nturn-end: x", "turn-end-timeout")] [InlineData("turn-end-timeout: soon\nturn-end: x", "turn-end-timeout")] [InlineData("turn-end-timeout: 10", "turn-end-timeout without turn-end")] [InlineData("session-start: ", "the command is empty")] [InlineData("protected-paths: /etc", "must be relative")] [InlineData("protected-paths: ../other", "inside the workspace")] [InlineData("protected-paths: .", "inside the workspace")] [InlineData("protected-paths: .volcanic", "the skills themselves")] [InlineData("protected-paths: .volcanic/skills/gtd", "the skills themselves")] public void Broken_fields_are_reported_and_the_rest_still_works(string fields, string error) { var set = Read("description: x\nprotected-paths: Сегодня\n" + fields)!; Assert.Contains(set.Errors, e => e.Contains(error, StringComparison.Ordinal)); Assert.False(set.IsEmpty); if (!fields.StartsWith("protected-paths", StringComparison.Ordinal)) Assert.Equal(["Сегодня"], set.ProtectedPaths); } [Fact] public void The_name_falls_back_to_the_directory() { Assert.Equal("dir-name", Read("turn-end: x", "dir-name")!.Skill); } } public class SkillHooksSessionTests : IAsyncLifetime { private static readonly TimeZoneInfo Warsaw = TimeZoneInfo.FindSystemTimeZoneById("Europe/Warsaw"); private TestHost _host = null!; public Task InitializeAsync() { _host = new TestHost(); return Task.CompletedTask; } public async Task DisposeAsync() => await _host.DisposeAsync(); private string SkillDir(string name = "gtd") => Path.Combine(SkillFiles.Root(_host.Cwd), name); private void WriteSkill(string fields, string name = "gtd") { Directory.CreateDirectory(SkillDir(name)); File.WriteAllText(Path.Combine(SkillDir(name), "SKILL.md"), $"---\nname: {name}\ndescription: Test skill\n{fields}\n---\nBody\n"); } /// Lets the background turn-end hook of the last run finish before the test changes what it reads. private static Task Settle() => Task.Delay(TimeSpan.FromSeconds(1)); private Task Session() => _host.CreateSessionAsync(b => b.TimeZone = Warsaw); private string SystemPrompt(int request) => _host.Faux.Requests[request].SystemPrompt ?? ""; /// The turn-end notices the model got after the prompt of request (custom messages reach it as user messages). private List Notices(int request) { static bool IsNotice(Message m) => m is UserMessage && m.Text.StartsWith("[Skill hook", StringComparison.Ordinal); var messages = _host.Faux.Requests[request].Messages.ToList(); var prompt = messages.FindLastIndex(m => m is UserMessage && !IsNotice(m)); return messages.Skip(prompt + 1).Where(IsNotice).ToList(); } [Fact] public async Task Session_start_output_is_in_the_system_prompt_with_the_chat_zone_and_skill_environment() { if (OperatingSystem.IsWindows()) return; WriteSkill("session-start: echo \"dir=$(pwd) skill=$VOLCANIC_SKILL_DIR ws=$VOLCANIC_WORKSPACE tz=$TZ\""); _host.Faux.Enqueue(FauxTurn.Say("ok")); await using var session = await Session(); await session.PromptAsync("hi"); var prompt = SystemPrompt(0); Assert.Contains("", prompt); Assert.Contains("", prompt); var dir = Path.GetFullPath(SkillDir()); Assert.Contains($"dir={dir} skill={dir} ws={Path.GetFullPath(_host.Cwd)} tz=Europe/Warsaw", prompt); Assert.True(prompt.IndexOf("", StringComparison.Ordinal) > prompt.IndexOf("", StringComparison.Ordinal)); } [Fact] public async Task Session_start_runs_once_per_session_and_again_after_new() { if (OperatingSystem.IsWindows()) return; WriteSkill("session-start: echo run >> count.txt; wc -l < count.txt"); _host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3")); await using var session = await Session(); await session.PromptAsync("a"); await session.PromptAsync("b"); Assert.Contains(">\n1\n", SystemPrompt(1)); await session.NewSessionAsync(); await session.PromptAsync("c"); Assert.Contains(">\n2\n", SystemPrompt(2)); } [Fact] public async Task A_failing_session_start_is_reported_and_the_session_answers() { if (OperatingSystem.IsWindows()) return; WriteSkill("session-start: echo partial; echo broken >&2; exit 3"); _host.Faux.Enqueue(FauxTurn.Say("still here")); await using var session = await Session(); await session.PromptAsync("hi"); var prompt = SystemPrompt(0); Assert.Contains("partial\n[session-start hook failed: exit 3]\nbroken", prompt); Assert.Equal("still here", session.Messages.OfType().Last().Text); } [Fact] public async Task A_slow_session_start_holds_the_first_run_only_until_its_timeout() { if (OperatingSystem.IsWindows()) return; WriteSkill("session-start: sleep 20\nsession-start-timeout: 1"); _host.Faux.Enqueue(FauxTurn.Say("ok")); await using var session = await Session(); var watch = Stopwatch.StartNew(); await session.PromptAsync("hi"); Assert.True(watch.Elapsed < TimeSpan.FromSeconds(5), watch.Elapsed.ToString()); Assert.Matches("session-start hook (still running after 1 s|failed: timed out after 1 s)", SystemPrompt(0)); } [Fact] public async Task Turn_end_output_reaches_the_next_run_once_and_again_after_a_clean_run() { if (OperatingSystem.IsWindows()) return; // The hook reports whatever problem.txt says; an empty file means all is well. WriteSkill("turn-end: cat \"$VOLCANIC_WORKSPACE/problem.txt\""); var problem = Path.Combine(_host.Cwd, "problem.txt"); File.WriteAllText(problem, "Сегодня/x.md: no date"); _host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3"), FauxTurn.Say("4"), FauxTurn.Say("5"), FauxTurn.Say("6")); await using var session = await Session(); await session.PromptAsync("first"); Assert.Empty(Notices(0)); await session.PromptAsync("second"); var notice = Assert.Single(Notices(1)); Assert.Equal("[Skill hook \"gtd\" turn-end (`cat \"$VOLCANIC_WORKSPACE/problem.txt\"`, exit 0) after your previous turn:]\nСегодня/x.md: no date", notice.Text); var stored = Assert.Single(session.Messages.OfType()); Assert.Equal("skill_hook", stored.CustomType); Assert.False(stored.Display); await session.PromptAsync("third"); // the same problem again: quiet Assert.Empty(Notices(2)); await Settle(); File.WriteAllText(problem, ""); await session.PromptAsync("fourth"); // gets the third run's (repeated) check: still quiet Assert.Empty(Notices(3)); await Settle(); File.WriteAllText(problem, "Сегодня/x.md: no date"); await session.PromptAsync("fifth"); // gets the fourth run's clean check Assert.Empty(Notices(4)); await session.PromptAsync("sixth"); // the problem came back after a clean check: news again Assert.Single(Notices(5)); } [Fact] public async Task A_clean_turn_end_adds_nothing_and_a_failing_one_shows_its_exit_and_stderr() { if (OperatingSystem.IsWindows()) return; WriteSkill("turn-end: test -f fail || exit 0; echo oops >&2; exit 2"); _host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3")); await using var session = await Session(); await session.PromptAsync("a"); await session.PromptAsync("b"); Assert.Empty(Notices(1)); await Settle(); File.WriteAllText(Path.Combine(SkillDir(), "fail"), ""); await session.PromptAsync("c"); // b's check ran before the file existed await session.PromptAsync("d"); Assert.Contains("exit 2) after your previous turn:]\nstderr:\noops", Assert.Single(Notices(3)).Text); } [Fact] public async Task Turn_end_never_delays_the_run_and_the_next_run_waits_at_most_its_timeout() { if (OperatingSystem.IsWindows()) return; WriteSkill("turn-end: sleep 20\nturn-end-timeout: 2"); _host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3")); await using var session = await Session(); var watch = Stopwatch.StartNew(); await session.PromptAsync("a"); Assert.True(watch.Elapsed < TimeSpan.FromSeconds(1.5), $"first run took {watch.Elapsed}"); watch.Restart(); await session.PromptAsync("b"); Assert.True(watch.Elapsed < TimeSpan.FromSeconds(3.5), $"second run took {watch.Elapsed}"); Assert.True(watch.Elapsed >= TimeSpan.FromSeconds(1), $"second run took {watch.Elapsed}"); } [Fact] public async Task A_turn_end_timeout_is_reported() { if (OperatingSystem.IsWindows()) return; WriteSkill("turn-end: sleep 20\nturn-end-timeout: 1"); _host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2")); await using var session = await Session(); await session.PromptAsync("a"); await Task.Delay(TimeSpan.FromSeconds(2.5)); // the hook was killed after 1 s await session.PromptAsync("b"); Assert.Contains("timed out after 1 s) after your previous turn:]", Assert.Single(Notices(1)).Text); } [Fact] public async Task Write_and_edit_are_refused_inside_protected_paths_and_read_works() { WriteSkill("protected-paths: Сегодня/, Входящие\nprotected-hint: use gtd move"); var today = Path.Combine(_host.Cwd, "Сегодня"); Directory.CreateDirectory(today); File.WriteAllText(Path.Combine(today, "a.md"), "old"); _host.Faux.Enqueue( FauxTurn.Tool("write", new { path = "Сегодня/new.md", content = "x" }), FauxTurn.Tool("edit", new { path = "Сегодня/a.md", edits = new[] { new { oldText = "old", newText = "new" } } }), FauxTurn.Tool("write", new { path = Path.Combine(_host.Cwd, "Входящие", "deep", "b.md"), content = "x" }), FauxTurn.Tool("write", new { path = "notes/../Сегодня/c.md", content = "x" }), FauxTurn.Tool("read", new { path = "Сегодня/a.md" }), FauxTurn.Tool("write", new { path = "Сегодня-архив/ok.md", content = "x" }), FauxTurn.Say("done")); await using var session = await Session(); await session.PromptAsync("go"); var results = session.Messages.OfType().ToList(); Assert.Equal(6, results.Count); foreach (var refused in results.Take(4)) { Assert.True(refused.IsError); Assert.Contains("which the skill \"gtd\" manages: change it through the skill's CLI (see .volcanic/skills/gtd/SKILL.md), not with write or edit. Reading is allowed. Hint: use gtd move", refused.Text); } Assert.Contains("\"Сегодня/new.md\" is inside \"Сегодня/\"", results[0].Text); Assert.Equal("old", File.ReadAllText(Path.Combine(today, "a.md"))); Assert.False(File.Exists(Path.Combine(today, "new.md"))); Assert.False(Directory.Exists(Path.Combine(_host.Cwd, "Входящие"))); Assert.False(results[4].IsError); Assert.Contains("old", results[4].Text); Assert.False(results[5].IsError); Assert.Contains("Сегодня/, Входящие/", SystemPrompt(0)); } [Fact] public async Task A_symlink_or_NFD_spelling_does_not_get_around_a_protected_path() { if (OperatingSystem.IsWindows()) return; WriteSkill("protected-paths: Мой"); Directory.CreateDirectory(Path.Combine(_host.Cwd, "Мой")); Directory.CreateSymbolicLink(Path.Combine(_host.Cwd, "link"), Path.Combine(_host.Cwd, "Мой")); _host.Faux.Enqueue( FauxTurn.Tool("write", new { path = "link/a.md", content = "x" }), FauxTurn.Tool("write", new { path = "Мой".Normalize(NormalizationForm.FormD) + "/b.md", content = "x" }), FauxTurn.Say("done")); await using var session = await Session(); await session.PromptAsync("go"); Assert.All(session.Messages.OfType(), r => Assert.True(r.IsError, r.Text)); Assert.Empty(Directory.EnumerateFileSystemEntries(Path.Combine(_host.Cwd, "Мой"))); } [Fact] public async Task Skills_without_hook_fields_change_nothing() { WriteSkill(""); _host.Faux.Enqueue(FauxTurn.Tool("write", new { path = "Сегодня/a.md", content = "x" }), FauxTurn.Say("1"), FauxTurn.Say("2")); await using var session = await Session(); await session.PromptAsync("a"); await session.PromptAsync("b"); Assert.DoesNotContain("", SystemPrompt(0)); Assert.DoesNotContain("", SystemPrompt(0)); Assert.Empty(Notices(2)); Assert.False(session.Messages.OfType().Single().IsError); } [Fact] public async Task Skill_hooks_can_be_turned_off() { await using var host = new TestHost(o => o.SkillHooks = false); var dir = Path.Combine(SkillFiles.Root(host.Cwd), "gtd"); Directory.CreateDirectory(dir); File.WriteAllText(Path.Combine(dir, "SKILL.md"), "---\ndescription: x\nsession-start: echo hello\nprotected-paths: Сегодня\n---\n"); host.Faux.Enqueue(FauxTurn.Tool("write", new { path = "Сегодня/a.md", content = "x" }), FauxTurn.Say("1")); await using var session = await host.CreateSessionAsync(); await session.PromptAsync("a"); Assert.DoesNotContain("", host.Faux.Requests[0].SystemPrompt); Assert.False(session.Messages.OfType().Single().IsError); } }