644f72068c
A skill's SKILL.md frontmatter may declare session-start (output into <skill_session_context>, at session start, /new, resume and after compaction), turn-end (after every run, in the background; output or a failure reaches the next run as a hidden notice, repeats suppressed until a clean run) and protected-paths (write/edit refused inside them). Hooks run from the skill directory with VOLCANIC_WORKSPACE, VOLCANIC_SKILL_DIR and the chat's TZ; waits are bounded by each hook's timeout and failures never break a run. /skills shows declared hooks and broken fields.
368 lines
17 KiB
C#
368 lines
17 KiB
C#
using System.Diagnostics;
|
||
using System.Text;
|
||
using Volcanic.AgenticCore.Abstractions.Messages;
|
||
using Volcanic.AgenticCore.Abstractions.Skills;
|
||
using Volcanic.AgenticCore.Features.Agent;
|
||
using Volcanic.AgenticCore.Testing;
|
||
using Volcanic.AgenticCore.Tests.Support;
|
||
|
||
namespace Volcanic.AgenticCore.Tests.Features;
|
||
|
||
public class SkillHookFilesTests : IDisposable
|
||
{
|
||
private readonly string _cwd = Path.Combine(Path.GetTempPath(), "volcanic-skill-hooks", Guid.NewGuid().ToString("N"));
|
||
|
||
public SkillHookFilesTests() => Directory.CreateDirectory(_cwd);
|
||
public void Dispose() { try { Directory.Delete(_cwd, true); } catch (IOException) { } }
|
||
|
||
private SkillHookSet? Read(string frontmatter, string dir = "gtd")
|
||
{
|
||
var skillDir = Path.Combine(SkillFiles.Root(_cwd), dir);
|
||
Directory.CreateDirectory(skillDir);
|
||
File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), $"---\n{frontmatter}\n---\nBody\n");
|
||
return SkillHookFiles.Read(_cwd, skillDir);
|
||
}
|
||
|
||
[Fact]
|
||
public void Reads_every_field_with_defaults_and_comments()
|
||
{
|
||
var set = Read("""
|
||
name: gtd
|
||
description: GTD
|
||
session-start: bin/gtd brief # start
|
||
turn-end: "bin/gtd check --quiet"
|
||
turn-end-timeout: 20
|
||
protected-paths: Входящие/, Сегодня , ./Проекты//
|
||
protected-hint: use gtd move
|
||
""")!;
|
||
Assert.Equal("gtd", set.Skill);
|
||
Assert.Equal(new SkillHookCommand("bin/gtd brief", TimeSpan.FromSeconds(30)), set.SessionStart);
|
||
Assert.Equal(new SkillHookCommand("bin/gtd check --quiet", TimeSpan.FromSeconds(20)), set.TurnEnd);
|
||
Assert.Equal(["Входящие", "Сегодня", "Проекты"], set.ProtectedPaths);
|
||
Assert.Equal("use gtd move", set.ProtectedHint);
|
||
Assert.Empty(set.Errors);
|
||
Assert.Equal("hooks: session-start, turn-end; 3 protected paths", set.Summary());
|
||
}
|
||
|
||
[Fact]
|
||
public void A_flow_list_with_quotes_works_and_NFD_becomes_NFC()
|
||
{
|
||
var nfd = "Мой".Normalize(NormalizationForm.FormD);
|
||
var set = Read($"description: x\nprotected-paths: [\"Входящие/\", '{nfd}/']")!;
|
||
Assert.Equal(["Входящие", "Мой".Normalize(NormalizationForm.FormC)], set.ProtectedPaths);
|
||
}
|
||
|
||
[Fact]
|
||
public void A_skill_without_the_fields_declares_nothing()
|
||
{
|
||
var set = Read("name: plain\ndescription: nothing else")!;
|
||
Assert.True(set.IsEmpty);
|
||
Assert.Empty(SkillHookFiles.ReadAll(_cwd));
|
||
}
|
||
|
||
[Theory]
|
||
[InlineData("session-start-timeout: 0\nsession-start: x", "session-start-timeout")]
|
||
[InlineData("turn-end-timeout: 121\nturn-end: x", "turn-end-timeout")]
|
||
[InlineData("turn-end-timeout: soon\nturn-end: x", "turn-end-timeout")]
|
||
[InlineData("turn-end-timeout: 10", "turn-end-timeout without turn-end")]
|
||
[InlineData("session-start: ", "the command is empty")]
|
||
[InlineData("protected-paths: /etc", "must be relative")]
|
||
[InlineData("protected-paths: ../other", "inside the workspace")]
|
||
[InlineData("protected-paths: .", "inside the workspace")]
|
||
[InlineData("protected-paths: .volcanic", "the skills themselves")]
|
||
[InlineData("protected-paths: .volcanic/skills/gtd", "the skills themselves")]
|
||
public void Broken_fields_are_reported_and_the_rest_still_works(string fields, string error)
|
||
{
|
||
var set = Read("description: x\nprotected-paths: Сегодня\n" + fields)!;
|
||
Assert.Contains(set.Errors, e => e.Contains(error, StringComparison.Ordinal));
|
||
Assert.False(set.IsEmpty);
|
||
if (!fields.StartsWith("protected-paths", StringComparison.Ordinal)) Assert.Equal(["Сегодня"], set.ProtectedPaths);
|
||
}
|
||
|
||
[Fact]
|
||
public void The_name_falls_back_to_the_directory()
|
||
{
|
||
Assert.Equal("dir-name", Read("turn-end: x", "dir-name")!.Skill);
|
||
}
|
||
}
|
||
|
||
public class SkillHooksSessionTests : IAsyncLifetime
|
||
{
|
||
private static readonly TimeZoneInfo Warsaw = TimeZoneInfo.FindSystemTimeZoneById("Europe/Warsaw");
|
||
private TestHost _host = null!;
|
||
|
||
public Task InitializeAsync()
|
||
{
|
||
_host = new TestHost();
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
public async Task DisposeAsync() => await _host.DisposeAsync();
|
||
|
||
private string SkillDir(string name = "gtd") => Path.Combine(SkillFiles.Root(_host.Cwd), name);
|
||
|
||
private void WriteSkill(string fields, string name = "gtd")
|
||
{
|
||
Directory.CreateDirectory(SkillDir(name));
|
||
File.WriteAllText(Path.Combine(SkillDir(name), "SKILL.md"), $"---\nname: {name}\ndescription: Test skill\n{fields}\n---\nBody\n");
|
||
}
|
||
|
||
/// <summary>Lets the background turn-end hook of the last run finish before the test changes what it reads.</summary>
|
||
private static Task Settle() => Task.Delay(TimeSpan.FromSeconds(1));
|
||
|
||
private Task<AgentSession> Session() => _host.CreateSessionAsync(b => b.TimeZone = Warsaw);
|
||
|
||
private string SystemPrompt(int request) => _host.Faux.Requests[request].SystemPrompt ?? "";
|
||
|
||
/// <summary>The turn-end notices the model got after the prompt of request <paramref name="request"/> (custom messages reach it as user messages).</summary>
|
||
private List<Message> Notices(int request)
|
||
{
|
||
static bool IsNotice(Message m) => m is UserMessage && m.Text.StartsWith("[Skill hook", StringComparison.Ordinal);
|
||
var messages = _host.Faux.Requests[request].Messages.ToList();
|
||
var prompt = messages.FindLastIndex(m => m is UserMessage && !IsNotice(m));
|
||
return messages.Skip(prompt + 1).Where(IsNotice).ToList();
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Session_start_output_is_in_the_system_prompt_with_the_chat_zone_and_skill_environment()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("session-start: echo \"dir=$(pwd) skill=$VOLCANIC_SKILL_DIR ws=$VOLCANIC_WORKSPACE tz=$TZ\"");
|
||
_host.Faux.Enqueue(FauxTurn.Say("ok"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("hi");
|
||
|
||
var prompt = SystemPrompt(0);
|
||
Assert.Contains("<skill_session_context>", prompt);
|
||
Assert.Contains("<context skill=\"gtd\" command=\"echo "dir=", prompt);
|
||
Assert.Contains("(Europe/Warsaw)\">", prompt);
|
||
var dir = Path.GetFullPath(SkillDir());
|
||
Assert.Contains($"dir={dir} skill={dir} ws={Path.GetFullPath(_host.Cwd)} tz=Europe/Warsaw", prompt);
|
||
Assert.True(prompt.IndexOf("<skill_session_context>", StringComparison.Ordinal) > prompt.IndexOf("<skills>", StringComparison.Ordinal));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Session_start_runs_once_per_session_and_again_after_new()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("session-start: echo run >> count.txt; wc -l < count.txt");
|
||
_host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("a");
|
||
await session.PromptAsync("b");
|
||
Assert.Contains(">\n1\n</context>", SystemPrompt(1));
|
||
|
||
await session.NewSessionAsync();
|
||
await session.PromptAsync("c");
|
||
Assert.Contains(">\n2\n</context>", SystemPrompt(2));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_failing_session_start_is_reported_and_the_session_answers()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("session-start: echo partial; echo broken >&2; exit 3");
|
||
_host.Faux.Enqueue(FauxTurn.Say("still here"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("hi");
|
||
|
||
var prompt = SystemPrompt(0);
|
||
Assert.Contains("partial\n[session-start hook failed: exit 3]\nbroken", prompt);
|
||
Assert.Equal("still here", session.Messages.OfType<AssistantMessage>().Last().Text);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_slow_session_start_holds_the_first_run_only_until_its_timeout()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("session-start: sleep 20\nsession-start-timeout: 1");
|
||
_host.Faux.Enqueue(FauxTurn.Say("ok"));
|
||
await using var session = await Session();
|
||
|
||
var watch = Stopwatch.StartNew();
|
||
await session.PromptAsync("hi");
|
||
|
||
Assert.True(watch.Elapsed < TimeSpan.FromSeconds(5), watch.Elapsed.ToString());
|
||
Assert.Matches("session-start hook (still running after 1 s|failed: timed out after 1 s)", SystemPrompt(0));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Turn_end_output_reaches_the_next_run_once_and_again_after_a_clean_run()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
// The hook reports whatever problem.txt says; an empty file means all is well.
|
||
WriteSkill("turn-end: cat \"$VOLCANIC_WORKSPACE/problem.txt\"");
|
||
var problem = Path.Combine(_host.Cwd, "problem.txt");
|
||
File.WriteAllText(problem, "Сегодня/x.md: no date");
|
||
_host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3"), FauxTurn.Say("4"), FauxTurn.Say("5"), FauxTurn.Say("6"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("first");
|
||
Assert.Empty(Notices(0));
|
||
|
||
await session.PromptAsync("second");
|
||
var notice = Assert.Single(Notices(1));
|
||
Assert.Equal("[Skill hook \"gtd\" turn-end (`cat \"$VOLCANIC_WORKSPACE/problem.txt\"`, exit 0) after your previous turn:]\nСегодня/x.md: no date", notice.Text);
|
||
var stored = Assert.Single(session.Messages.OfType<CustomMessage>());
|
||
Assert.Equal("skill_hook", stored.CustomType);
|
||
Assert.False(stored.Display);
|
||
|
||
await session.PromptAsync("third"); // the same problem again: quiet
|
||
Assert.Empty(Notices(2));
|
||
|
||
await Settle();
|
||
File.WriteAllText(problem, "");
|
||
await session.PromptAsync("fourth"); // gets the third run's (repeated) check: still quiet
|
||
Assert.Empty(Notices(3));
|
||
|
||
await Settle();
|
||
File.WriteAllText(problem, "Сегодня/x.md: no date");
|
||
await session.PromptAsync("fifth"); // gets the fourth run's clean check
|
||
Assert.Empty(Notices(4));
|
||
await session.PromptAsync("sixth"); // the problem came back after a clean check: news again
|
||
Assert.Single(Notices(5));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_clean_turn_end_adds_nothing_and_a_failing_one_shows_its_exit_and_stderr()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("turn-end: test -f fail || exit 0; echo oops >&2; exit 2");
|
||
_host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("a");
|
||
await session.PromptAsync("b");
|
||
Assert.Empty(Notices(1));
|
||
|
||
await Settle();
|
||
File.WriteAllText(Path.Combine(SkillDir(), "fail"), "");
|
||
await session.PromptAsync("c"); // b's check ran before the file existed
|
||
await session.PromptAsync("d");
|
||
Assert.Contains("exit 2) after your previous turn:]\nstderr:\noops", Assert.Single(Notices(3)).Text);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Turn_end_never_delays_the_run_and_the_next_run_waits_at_most_its_timeout()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("turn-end: sleep 20\nturn-end-timeout: 2");
|
||
_host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"), FauxTurn.Say("3"));
|
||
await using var session = await Session();
|
||
|
||
var watch = Stopwatch.StartNew();
|
||
await session.PromptAsync("a");
|
||
Assert.True(watch.Elapsed < TimeSpan.FromSeconds(1.5), $"first run took {watch.Elapsed}");
|
||
|
||
watch.Restart();
|
||
await session.PromptAsync("b");
|
||
Assert.True(watch.Elapsed < TimeSpan.FromSeconds(3.5), $"second run took {watch.Elapsed}");
|
||
Assert.True(watch.Elapsed >= TimeSpan.FromSeconds(1), $"second run took {watch.Elapsed}");
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_turn_end_timeout_is_reported()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("turn-end: sleep 20\nturn-end-timeout: 1");
|
||
_host.Faux.Enqueue(FauxTurn.Say("1"), FauxTurn.Say("2"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("a");
|
||
await Task.Delay(TimeSpan.FromSeconds(2.5)); // the hook was killed after 1 s
|
||
await session.PromptAsync("b");
|
||
|
||
Assert.Contains("timed out after 1 s) after your previous turn:]", Assert.Single(Notices(1)).Text);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Write_and_edit_are_refused_inside_protected_paths_and_read_works()
|
||
{
|
||
WriteSkill("protected-paths: Сегодня/, Входящие\nprotected-hint: use gtd move");
|
||
var today = Path.Combine(_host.Cwd, "Сегодня");
|
||
Directory.CreateDirectory(today);
|
||
File.WriteAllText(Path.Combine(today, "a.md"), "old");
|
||
_host.Faux.Enqueue(
|
||
FauxTurn.Tool("write", new { path = "Сегодня/new.md", content = "x" }),
|
||
FauxTurn.Tool("edit", new { path = "Сегодня/a.md", edits = new[] { new { oldText = "old", newText = "new" } } }),
|
||
FauxTurn.Tool("write", new { path = Path.Combine(_host.Cwd, "Входящие", "deep", "b.md"), content = "x" }),
|
||
FauxTurn.Tool("write", new { path = "notes/../Сегодня/c.md", content = "x" }),
|
||
FauxTurn.Tool("read", new { path = "Сегодня/a.md" }),
|
||
FauxTurn.Tool("write", new { path = "Сегодня-архив/ok.md", content = "x" }),
|
||
FauxTurn.Say("done"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("go");
|
||
|
||
var results = session.Messages.OfType<ToolResultMessage>().ToList();
|
||
Assert.Equal(6, results.Count);
|
||
foreach (var refused in results.Take(4))
|
||
{
|
||
Assert.True(refused.IsError);
|
||
Assert.Contains("which the skill \"gtd\" manages: change it through the skill's CLI (see .volcanic/skills/gtd/SKILL.md), not with write or edit. Reading is allowed. Hint: use gtd move", refused.Text);
|
||
}
|
||
Assert.Contains("\"Сегодня/new.md\" is inside \"Сегодня/\"", results[0].Text);
|
||
Assert.Equal("old", File.ReadAllText(Path.Combine(today, "a.md")));
|
||
Assert.False(File.Exists(Path.Combine(today, "new.md")));
|
||
Assert.False(Directory.Exists(Path.Combine(_host.Cwd, "Входящие")));
|
||
Assert.False(results[4].IsError);
|
||
Assert.Contains("old", results[4].Text);
|
||
Assert.False(results[5].IsError);
|
||
Assert.Contains("<protected_paths>Сегодня/, Входящие/</protected_paths>", SystemPrompt(0));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_symlink_or_NFD_spelling_does_not_get_around_a_protected_path()
|
||
{
|
||
if (OperatingSystem.IsWindows()) return;
|
||
WriteSkill("protected-paths: Мой");
|
||
Directory.CreateDirectory(Path.Combine(_host.Cwd, "Мой"));
|
||
Directory.CreateSymbolicLink(Path.Combine(_host.Cwd, "link"), Path.Combine(_host.Cwd, "Мой"));
|
||
_host.Faux.Enqueue(
|
||
FauxTurn.Tool("write", new { path = "link/a.md", content = "x" }),
|
||
FauxTurn.Tool("write", new { path = "Мой".Normalize(NormalizationForm.FormD) + "/b.md", content = "x" }),
|
||
FauxTurn.Say("done"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("go");
|
||
|
||
Assert.All(session.Messages.OfType<ToolResultMessage>(), r => Assert.True(r.IsError, r.Text));
|
||
Assert.Empty(Directory.EnumerateFileSystemEntries(Path.Combine(_host.Cwd, "Мой")));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Skills_without_hook_fields_change_nothing()
|
||
{
|
||
WriteSkill("");
|
||
_host.Faux.Enqueue(FauxTurn.Tool("write", new { path = "Сегодня/a.md", content = "x" }), FauxTurn.Say("1"), FauxTurn.Say("2"));
|
||
await using var session = await Session();
|
||
|
||
await session.PromptAsync("a");
|
||
await session.PromptAsync("b");
|
||
|
||
Assert.DoesNotContain("<skill_session_context>", SystemPrompt(0));
|
||
Assert.DoesNotContain("</protected_paths>", SystemPrompt(0));
|
||
Assert.Empty(Notices(2));
|
||
Assert.False(session.Messages.OfType<ToolResultMessage>().Single().IsError);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Skill_hooks_can_be_turned_off()
|
||
{
|
||
await using var host = new TestHost(o => o.SkillHooks = false);
|
||
var dir = Path.Combine(SkillFiles.Root(host.Cwd), "gtd");
|
||
Directory.CreateDirectory(dir);
|
||
File.WriteAllText(Path.Combine(dir, "SKILL.md"), "---\ndescription: x\nsession-start: echo hello\nprotected-paths: Сегодня\n---\n");
|
||
host.Faux.Enqueue(FauxTurn.Tool("write", new { path = "Сегодня/a.md", content = "x" }), FauxTurn.Say("1"));
|
||
await using var session = await host.CreateSessionAsync();
|
||
|
||
await session.PromptAsync("a");
|
||
|
||
Assert.DoesNotContain("<skill_session_context>", host.Faux.Requests[0].SystemPrompt);
|
||
Assert.False(session.Messages.OfType<ToolResultMessage>().Single().IsError);
|
||
}
|
||
}
|