bb1c534a7a
A skill directory may hold INSTRUCTIONS.md, with or without SKILL.md. Its body (frontmatter cut) goes into every system prompt in <skill_instructions>, re-read on every run: 16 KB per file, 64 KB in total, truncation marked with the file's path. /skills marks skills with instructions and lists instruction-only dirs.
320 lines
18 KiB
C#
320 lines
18 KiB
C#
using Microsoft.Extensions.Time.Testing;
|
|
using Volcanic.AgenticCore.Abstractions.Extensions.Events;
|
|
using Volcanic.AgenticCore.Abstractions.Json;
|
|
using Volcanic.AgenticCore.Abstractions.Messages;
|
|
using Volcanic.AgenticCore.Abstractions.Sessions;
|
|
using Volcanic.AgenticCore.Abstractions.Tools;
|
|
using Volcanic.AgenticCore.Features.Compaction;
|
|
using Volcanic.AgenticCore.Features.Models;
|
|
using Volcanic.AgenticCore.Features.Sessions;
|
|
using Volcanic.AgenticCore.Features.SystemPrompt;
|
|
using Volcanic.AgenticCore.Testing;
|
|
|
|
namespace Volcanic.AgenticCore.Tests.Features;
|
|
|
|
public class SystemPromptBuilderTests
|
|
{
|
|
[Fact]
|
|
public void Renders_tools_guidelines_sections_project_context_and_environment()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
var sub = Path.Combine(dir, "sub");
|
|
Directory.CreateDirectory(sub);
|
|
File.WriteAllText(Path.Combine(dir, "AGENTS.md"), "Root rules");
|
|
File.WriteAllText(Path.Combine(sub, "CLAUDE.md"), "Sub rules");
|
|
try
|
|
{
|
|
var tools = new ITool[]
|
|
{
|
|
new DelegateTool<object>("alpha", "d", (_, _, _) => Task.FromResult(ToolResult.Text("")), promptSnippet: "alpha: does a", promptGuidelines: ["Use alpha wisely"]),
|
|
new DelegateTool<object>("silent", "d", (_, _, _) => Task.FromResult(ToolResult.Text(""))),
|
|
};
|
|
var builder = new SystemPromptBuilder(sub, () => tools, new FakeTimeProvider(new DateTimeOffset(2026, 9, 25, 0, 0, 0, TimeSpan.Zero)), () => TimeZoneInfo.Utc);
|
|
var options = builder.DefaultOptions("BASE");
|
|
options.Sections["extra"] = "more";
|
|
var prompt = builder.Render(options);
|
|
|
|
Assert.StartsWith("BASE", prompt);
|
|
Assert.Contains("<tools>\nAvailable tools:\n- alpha: does a\n</tools>", prompt);
|
|
Assert.DoesNotContain("silent", prompt);
|
|
Assert.Contains("- Use alpha wisely", prompt);
|
|
Assert.Contains("<extra>\nmore\n</extra>", prompt);
|
|
var ctxStart = prompt.IndexOf("<project_context>", StringComparison.Ordinal);
|
|
Assert.True(prompt.IndexOf("Sub rules", StringComparison.Ordinal) < prompt.IndexOf("Root rules", StringComparison.Ordinal));
|
|
Assert.True(ctxStart > 0);
|
|
Assert.Contains("Current date: 2026-09-25 (Friday)\nTime zone: ", prompt);
|
|
Assert.Contains($"Working directory: {sub}", prompt);
|
|
|
|
options.SelectedTools.Clear();
|
|
options.SelectedTools.Add("silent");
|
|
options.IncludeProjectContext = false;
|
|
var narrow = builder.Render(options);
|
|
Assert.DoesNotContain("<tools>", narrow);
|
|
Assert.DoesNotContain("project_context", narrow);
|
|
Assert.DoesNotContain("<memory>", narrow);
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Loads_AGENTS_md_in_preference_to_CLAUDE_md_then_persona_and_memory()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
Directory.CreateDirectory(dir);
|
|
File.WriteAllText(Path.Combine(dir, "agents.md"), "Agent rules");
|
|
File.WriteAllText(Path.Combine(dir, "CLAUDE.md"), "Claude rules");
|
|
File.WriteAllText(Path.Combine(dir, "persona.md"), "Dry wit.");
|
|
File.WriteAllText(Path.Combine(dir, "memory.md"), "- User prefers tabs");
|
|
try
|
|
{
|
|
var builder = new SystemPromptBuilder(dir, () => [], new FakeTimeProvider(DateTimeOffset.UnixEpoch), () => TimeZoneInfo.Utc);
|
|
var prompt = builder.Render(builder.DefaultOptions("BASE"));
|
|
|
|
Assert.Contains("Agent rules", prompt);
|
|
Assert.DoesNotContain("Claude rules", prompt);
|
|
Assert.Contains("<persona>\nDry wit.\n</persona>", prompt);
|
|
Assert.Contains($"from {Path.Combine(dir, "memory.md")}", prompt);
|
|
Assert.Contains("- User prefers tabs\n</memory>", prompt);
|
|
var order = new[] { "<project_context>", "<persona>", "<memory>", "<environment>" }.Select(t => prompt.IndexOf(t, StringComparison.Ordinal)).ToList();
|
|
Assert.Equal(order.Order(), order);
|
|
|
|
File.Delete(Path.Combine(dir, "agents.md"));
|
|
File.WriteAllText(Path.Combine(dir, "memory.md"), "");
|
|
prompt = builder.Render(builder.DefaultOptions("BASE"));
|
|
Assert.Contains("Claude rules", prompt);
|
|
Assert.Contains("<memory>", prompt);
|
|
|
|
File.Delete(Path.Combine(dir, "persona.md"));
|
|
File.Delete(Path.Combine(dir, "memory.md"));
|
|
prompt = builder.Render(builder.DefaultOptions("BASE"));
|
|
Assert.DoesNotContain("<persona>", prompt);
|
|
Assert.DoesNotContain("<memory>", prompt);
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
}
|
|
|
|
public class SkillCatalogTests
|
|
{
|
|
[Fact]
|
|
public void Lists_workspace_skills_and_picks_up_new_ones_on_the_next_render()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
var skills = Path.Combine(dir, ".volcanic", "skills");
|
|
Directory.CreateDirectory(Path.Combine(skills, "pdf"));
|
|
Directory.CreateDirectory(Path.Combine(skills, "no-description"));
|
|
Directory.CreateDirectory(Path.Combine(skills, "folded"));
|
|
File.WriteAllText(Path.Combine(skills, "pdf", "SKILL.md"), "---\nname: pdf-tools\ndescription: \"Fill <PDF> forms\"\n---\n# Body stays out");
|
|
File.WriteAllText(Path.Combine(skills, "no-description", "SKILL.md"), "---\nname: nope\n---\nbody");
|
|
File.WriteAllText(Path.Combine(skills, "folded", "skill.md"), "---\ndescription: >\n Plans trips\n and books hotels\n---\n");
|
|
try
|
|
{
|
|
var builder = new SystemPromptBuilder(dir, () => [], new FakeTimeProvider(DateTimeOffset.UnixEpoch), () => TimeZoneInfo.Utc);
|
|
var prompt = builder.Render(builder.DefaultOptions("BASE"));
|
|
|
|
Assert.Contains($"{skills}/<skill-name>/SKILL.md", prompt);
|
|
Assert.Contains("<name>pdf-tools</name>\n <description>Fill <PDF> forms</description>\n <location>" + Path.Combine(skills, "pdf", "SKILL.md"), prompt);
|
|
Assert.Contains("<name>folded</name>\n <description>Plans trips and books hotels</description>", prompt);
|
|
Assert.DoesNotContain("nope", prompt);
|
|
Assert.DoesNotContain("Body stays out", prompt);
|
|
var order = new[] { "<skills>", "<environment>" }.Select(t => prompt.IndexOf(t, StringComparison.Ordinal)).ToList();
|
|
Assert.Equal(order.Order(), order);
|
|
|
|
Directory.CreateDirectory(Path.Combine(skills, "zeta"));
|
|
File.WriteAllText(Path.Combine(skills, "zeta", "SKILL.md"), "---\nname: zeta\ndescription: Added mid-session\n---\n");
|
|
Assert.Contains("<name>zeta</name>", builder.Render(builder.DefaultOptions("BASE")));
|
|
Assert.Equal(["folded", "pdf-tools", "zeta"], SkillCatalog.Load(dir).Select(s => s.Name));
|
|
|
|
var narrow = builder.DefaultOptions("BASE");
|
|
narrow.IncludeProjectContext = false;
|
|
Assert.DoesNotContain("<skills>", builder.Render(narrow));
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Without_skills_the_section_still_says_where_to_write_them()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
Directory.CreateDirectory(dir);
|
|
try
|
|
{
|
|
Assert.Empty(SkillCatalog.Load(dir));
|
|
var section = SkillCatalog.Render(dir, []);
|
|
Assert.Contains(Path.Combine(dir, ".volcanic", "skills"), section);
|
|
Assert.EndsWith("No skills yet.", section);
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Instructions_go_into_every_prompt_with_or_without_a_skill_md()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
var skills = Path.Combine(dir, ".volcanic", "skills");
|
|
foreach (var name in new[] { "b-trips", "a-rules", "c-empty", "d-nameless" }) Directory.CreateDirectory(Path.Combine(skills, name));
|
|
File.WriteAllText(Path.Combine(skills, "b-trips", "SKILL.md"), "---\nname: trip-planner\ndescription: Plans trips\n---\nLazy body");
|
|
File.WriteAllText(Path.Combine(skills, "b-trips", "instructions.md"), "---\ntitle: ignored\n---\nAlways quote prices in EUR.\n");
|
|
File.WriteAllText(Path.Combine(skills, "a-rules", "INSTRUCTIONS.md"), "Never book before 9:00.");
|
|
File.WriteAllText(Path.Combine(skills, "c-empty", "INSTRUCTIONS.md"), "---\nx: y\n---\n \n");
|
|
File.WriteAllText(Path.Combine(skills, "d-nameless", "SKILL.md"), "---\nname: \"named \\\"rules\\\"\"\n---\n");
|
|
File.WriteAllText(Path.Combine(skills, "d-nameless", "INSTRUCTIONS.md"), "Use <b>tags</b> as is.");
|
|
try
|
|
{
|
|
var builder = new SystemPromptBuilder(dir, () => [], new FakeTimeProvider(DateTimeOffset.UnixEpoch), () => TimeZoneInfo.Utc);
|
|
var prompt = builder.Render(builder.DefaultOptions("BASE"));
|
|
|
|
Assert.Equal(["a-rules", "trip-planner", "named \\\"rules\\\""], SkillCatalog.LoadInstructions(dir).Select(i => i.Name));
|
|
Assert.Contains($"<instructions skill=\"a-rules\" location=\"{Path.Combine(skills, "a-rules", "INSTRUCTIONS.md")}\">\nNever book before 9:00.\n</instructions>", prompt);
|
|
Assert.Contains($"<instructions skill=\"trip-planner\" location=\"{Path.Combine(skills, "b-trips", "instructions.md")}\">\nAlways quote prices in EUR.\n</instructions>", prompt);
|
|
Assert.Contains("skill=\"named \\"rules\\"\"", prompt);
|
|
Assert.Contains("Use <b>tags</b> as is.", prompt);
|
|
Assert.DoesNotContain("ignored", prompt);
|
|
Assert.DoesNotContain("c-empty", prompt);
|
|
Assert.DoesNotContain("Lazy body", prompt);
|
|
Assert.Equal(["trip-planner"], SkillCatalog.Load(dir).Select(s => s.Name));
|
|
var order = new[] { "\n<skills>\n", "\n<skill_instructions>\n", "\n<environment>\n" }.Select(t => prompt.IndexOf(t, StringComparison.Ordinal)).ToList();
|
|
Assert.Equal(order.Order(), order);
|
|
Assert.True(order[0] >= 0);
|
|
|
|
File.WriteAllText(Path.Combine(skills, "a-rules", "INSTRUCTIONS.md"), "Never book before 10:00.");
|
|
Assert.Contains("Never book before 10:00.", builder.Render(builder.DefaultOptions("BASE")));
|
|
|
|
var narrow = builder.DefaultOptions("BASE");
|
|
narrow.IncludeProjectContext = false;
|
|
Assert.DoesNotContain("<skill_instructions>", builder.Render(narrow));
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Without_instructions_there_is_no_instructions_section()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
Directory.CreateDirectory(dir);
|
|
try
|
|
{
|
|
var builder = new SystemPromptBuilder(dir, () => [], new FakeTimeProvider(DateTimeOffset.UnixEpoch), () => TimeZoneInfo.Utc);
|
|
Assert.DoesNotContain("\n<skill_instructions>\n", builder.Render(builder.DefaultOptions("BASE")));
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Instructions_are_capped_per_file_and_in_total_with_a_pointer_to_the_file()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "volcanic-prompt", Guid.NewGuid().ToString("N"));
|
|
var skills = Path.Combine(dir, ".volcanic", "skills");
|
|
var per = SkillCatalog.MaxInstructionsLength;
|
|
// 3 full files use 48 KB, the fourth is cut by its own cap, the fifth gets nothing.
|
|
for (var i = 0; i < 5; i++)
|
|
{
|
|
Directory.CreateDirectory(Path.Combine(skills, $"s{i}"));
|
|
File.WriteAllText(Path.Combine(skills, $"s{i}", "INSTRUCTIONS.md"), new string((char)('a' + i), i == 3 ? per + 500 : per));
|
|
}
|
|
Directory.CreateDirectory(Path.Combine(skills, "s5"));
|
|
File.WriteAllText(Path.Combine(skills, "s5", "INSTRUCTIONS.md"), "late");
|
|
try
|
|
{
|
|
var loaded = SkillCatalog.LoadInstructions(dir);
|
|
Assert.Equal([per, per, per, per, 0, 0], loaded.Select(i => i.Content.Length));
|
|
Assert.Equal([false, false, false, true, true, true], loaded.Select(i => i.Truncated));
|
|
Assert.Equal(SkillCatalog.MaxTotalInstructionsLength, loaded.Sum(i => i.Content.Length));
|
|
|
|
var section = SkillCatalog.RenderInstructions(loaded);
|
|
Assert.Contains($"[…truncated, read the full file at {Path.Combine(skills, "s3", "INSTRUCTIONS.md")}]", section);
|
|
Assert.Contains($"location=\"{Path.Combine(skills, "s5", "INSTRUCTIONS.md")}\">\n[…truncated, read the full file at {Path.Combine(skills, "s5", "INSTRUCTIONS.md")}]\n</instructions>", section);
|
|
Assert.DoesNotContain("late", section);
|
|
}
|
|
finally { Directory.Delete(dir, true); }
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("---\na: b\n---\nbody", "body")]
|
|
[InlineData("\uFEFF---\r\na: b\r\n---\r\nbody", "body")]
|
|
[InlineData("---\nnever closed\nbody", "---\nnever closed\nbody")]
|
|
[InlineData("plain\n---\nrule", "plain\n---\nrule")]
|
|
public void StripFrontmatter_cuts_only_a_leading_closed_block(string text, string expected) =>
|
|
Assert.Equal(expected, SkillCatalog.StripFrontmatter(text));
|
|
}
|
|
|
|
public class CompactorTests
|
|
{
|
|
private static AssistantMessage Assistant(string text, int? usageTokens = null) => new()
|
|
{
|
|
Provider = "p", Api = "a", Model = "m", StopReason = StopReason.Stop, Content = [new TextContent(text)],
|
|
Usage = usageTokens is { } u ? new Usage(u, 0, 0, 0) : Usage.Zero,
|
|
};
|
|
|
|
[Fact]
|
|
public void ShouldCompact_and_ContextTokens()
|
|
{
|
|
var compactor = new Compactor(new CompactionSettings(ReserveTokens: 1_000), new ModelRegistry(new ApiKeyResolver(null, _ => null)));
|
|
var model = FauxProvider.EchoModel with { ContextWindow = 5_000 };
|
|
Assert.False(compactor.ShouldCompact(3_999, model));
|
|
Assert.True(compactor.ShouldCompact(4_001, model));
|
|
Assert.False(compactor.ShouldCompact(null, model));
|
|
Assert.False(new Compactor(new CompactionSettings(Enabled: false), new ModelRegistry(new ApiKeyResolver(null, _ => null))).ShouldCompact(999_999, model));
|
|
|
|
Assert.Null(Compactor.ContextTokens([]));
|
|
Assert.Equal(3, Compactor.ContextTokens([UserMessage.FromText("twelve chars")]));
|
|
var withUsage = Compactor.ContextTokens([UserMessage.FromText("q"), Assistant("a", 500), UserMessage.FromText("eight ch")]);
|
|
Assert.Equal(502, withUsage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Prepare_keeps_recent_messages_and_never_cuts_before_a_tool_result()
|
|
{
|
|
var session = await SessionManager.CreateAsync(new InMemorySessionStore(), "s", "/tmp");
|
|
await session.AppendMessageAsync(UserMessage.FromText(new string('a', 400)));
|
|
await session.AppendMessageAsync(Assistant(new string('b', 400)));
|
|
var call = new AssistantMessage { Provider = "p", Api = "a", Model = "m", StopReason = StopReason.ToolUse, Content = [new ToolCallContent("c1", "bash", VolcanicJson.EmptyObject)] };
|
|
var callEntry = await session.AppendMessageAsync(call);
|
|
await session.AppendMessageAsync(new ToolResultMessage { ToolCallId = "c1", ToolName = "bash", Content = [new TextContent(new string('r', 400))] });
|
|
await session.AppendMessageAsync(Assistant("short"));
|
|
|
|
var compactor = new Compactor(new CompactionSettings(KeepRecentTokens: 120), new ModelRegistry(new ApiKeyResolver(null, _ => null)));
|
|
var prep = compactor.Prepare(session);
|
|
Assert.Equal(callEntry.Id, prep.FirstKeptEntryId);
|
|
Assert.Equal(2, prep.EntriesToSummarize.Count);
|
|
Assert.Equal(3, prep.EntriesToKeep.Count);
|
|
Assert.Null(prep.PreviousSummary);
|
|
|
|
var text = Compactor.Serialize(prep.EntriesToSummarize, maxBlockChars: 10);
|
|
Assert.Contains("[User]: aaaaaaaaaa… [390 more chars]", text);
|
|
Assert.Contains("[Assistant]: bbbbbbbbbb", text);
|
|
var full = Compactor.Serialize(session.GetBranch());
|
|
Assert.Contains("[Assistant tool calls]: bash({})", full);
|
|
Assert.Contains("[Tool result]: rrr", full);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Summarize_uses_the_model_and_extracts_file_lists()
|
|
{
|
|
var faux = new FauxProvider().Enqueue(FauxTurn.Say("## Goal\nDo X\n<read-files>\na.cs\nb.cs\n</read-files>\n<modified-files>\nc.cs\n</modified-files>"));
|
|
var registry = new ModelRegistry(new ApiKeyResolver(null, _ => null));
|
|
registry.Register(faux.Registration);
|
|
var model = registry.Find(FauxProvider.ModelKey)!;
|
|
var session = await SessionManager.CreateAsync(new InMemorySessionStore(), "s", "/tmp");
|
|
await session.AppendMessageAsync(UserMessage.FromText("old question"));
|
|
await session.AppendMessageAsync(Assistant("old answer"));
|
|
await session.AppendMessageAsync(UserMessage.FromText("new"));
|
|
|
|
var compactor = new Compactor(new CompactionSettings(KeepRecentTokens: 1), registry);
|
|
var prep = compactor.Prepare(session);
|
|
var result = await compactor.SummarizeAsync(prep, model, "keep it short", CancellationToken.None);
|
|
|
|
Assert.StartsWith("## Goal", result.Summary);
|
|
Assert.Equal(prep.FirstKeptEntryId, result.FirstKeptEntryId);
|
|
Assert.Equal(["a.cs", "b.cs"], result.Details!.Value.GetProperty("readFiles").EnumerateArray().Select(e => e.GetString()));
|
|
var request = faux.Requests.Single();
|
|
Assert.Equal(CompactionPrompts.SystemPrompt, request.SystemPrompt);
|
|
Assert.Contains("[User]: old question", request.Messages[0].Text);
|
|
Assert.Contains("keep it short", request.Messages[0].Text);
|
|
Assert.Equal(Volcanic.AgenticCore.Abstractions.Providers.CacheRetention.None, faux.Options.Single().CacheRetention);
|
|
|
|
faux.Enqueue(FauxTurn.Error("nope"));
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => compactor.SummarizeAsync(prep, model, null, CancellationToken.None));
|
|
}
|
|
}
|