forked from admin/VolcanicAgenticCore
feat(processes): redact known secret values from tool output
ISecretRedactor (Abstractions) and SecretRedactor (ordinal, longest first, values under 8 characters ignored) registered as a singleton over ProcessEnvironment.SecretValues. AgentSession.RunOneAsync redacts progress snapshots, every result and error path before ToolResultEvent, and again after the handlers run.
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
namespace Volcanic.AgenticCore.Abstractions.Processes;
|
||||
|
||||
/// <summary>Replaces the host's known secret values in text the model, the transcript or a host would see.</summary>
|
||||
public interface ISecretRedactor
|
||||
{
|
||||
/// <summary>Replaces every known secret value in <paramref name="text"/> with <c>[REDACTED]</c>.</summary>
|
||||
string Redact(string text);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using Volcanic.AgenticCore.Abstractions.Extensions.Events;
|
||||
using Volcanic.AgenticCore.Abstractions.Json;
|
||||
using Volcanic.AgenticCore.Abstractions.Messages;
|
||||
using Volcanic.AgenticCore.Abstractions.Tools;
|
||||
using Volcanic.AgenticCore.Features.Processes;
|
||||
using Volcanic.AgenticCore.Features.Tools;
|
||||
using Volcanic.AgenticCore.Features.Tools.BuiltIn;
|
||||
|
||||
@@ -123,6 +124,7 @@ public sealed partial class AgentSession
|
||||
{
|
||||
var progress = new ToolProgress(partial =>
|
||||
{
|
||||
partial = _redactor.Redact(partial);
|
||||
Emit(new AgentSessionEvent.ToolExecutionUpdate(p.Call.Id, p.Call.Name, p.Args, partial));
|
||||
_ = _runner.EmitAsync(new ToolExecutionUpdateEvent(p.Call.Id, p.Call.Name, p.Args, partial), CancellationToken.None);
|
||||
});
|
||||
@@ -149,9 +151,10 @@ public sealed partial class AgentSession
|
||||
isError = true;
|
||||
}
|
||||
}
|
||||
result = _redactor.Redact(result);
|
||||
|
||||
var hook = await _runner.EmitAsync(new ToolResultEvent(p.Call.Id, p.Call.Name, p.Args, result.Content.ToList(), result.Details, isError, result.Usage), CancellationToken.None).ConfigureAwait(false);
|
||||
result = result with { Content = hook.Content, Details = hook.Details, Usage = hook.Usage };
|
||||
result = _redactor.Redact(result with { Content = hook.Content, Details = hook.Details, Usage = hook.Usage });
|
||||
isError = hook.IsError;
|
||||
|
||||
Emit(new AgentSessionEvent.ToolExecutionEnd(p.Call.Id, p.Call.Name, result, isError));
|
||||
|
||||
@@ -3,6 +3,7 @@ using Volcanic.AgenticCore.Abstractions.Events;
|
||||
using Volcanic.AgenticCore.Abstractions.Extensions;
|
||||
using Volcanic.AgenticCore.Abstractions.Extensions.Events;
|
||||
using Volcanic.AgenticCore.Abstractions.Messages;
|
||||
using Volcanic.AgenticCore.Abstractions.Processes;
|
||||
using Volcanic.AgenticCore.Abstractions.Providers;
|
||||
using Volcanic.AgenticCore.Abstractions.Sessions;
|
||||
using Volcanic.AgenticCore.Abstractions.Ui;
|
||||
@@ -48,6 +49,7 @@ public sealed partial class AgentSession : IAsyncDisposable
|
||||
private readonly AgenticCoreOptions _coreOptions;
|
||||
private readonly RetrySettings _retry;
|
||||
private readonly ILogger _logger;
|
||||
private readonly ISecretRedactor _redactor;
|
||||
private readonly IUserInterface _ui;
|
||||
private readonly PendingMessageQueue _steering;
|
||||
private readonly PendingMessageQueue _followUp;
|
||||
@@ -70,7 +72,7 @@ public sealed partial class AgentSession : IAsyncDisposable
|
||||
internal AgentSession(
|
||||
SessionManager session, ISessionStore store, ExtensionRunner runner, ToolRegistry tools, CommandRegistry commands,
|
||||
ModelRegistry models, Compactor compactor, SystemPromptBuilder promptBuilder, AgentSessionOptions options,
|
||||
AgenticCoreOptions coreOptions, Model? model, ThinkingLevel thinkingLevel, ILogger logger)
|
||||
AgenticCoreOptions coreOptions, Model? model, ThinkingLevel thinkingLevel, ILogger logger, ISecretRedactor redactor)
|
||||
{
|
||||
_session = session;
|
||||
_store = store;
|
||||
@@ -84,6 +86,7 @@ public sealed partial class AgentSession : IAsyncDisposable
|
||||
_coreOptions = coreOptions;
|
||||
_retry = options.Retry ?? coreOptions.Retry;
|
||||
_logger = logger;
|
||||
_redactor = redactor;
|
||||
_ui = options.Ui ?? NullUserInterface.Instance;
|
||||
_steering = new PendingMessageQueue(options.SteeringMode);
|
||||
_followUp = new PendingMessageQueue(options.FollowUpMode);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Volcanic.AgenticCore.Abstractions.Extensions.Events;
|
||||
using Volcanic.AgenticCore.Abstractions.Processes;
|
||||
using Volcanic.AgenticCore.Abstractions.Providers;
|
||||
using Volcanic.AgenticCore.Abstractions.Sessions;
|
||||
using Volcanic.AgenticCore.Features.Commands;
|
||||
@@ -31,12 +32,14 @@ public sealed class AgentSessionFactory : IAgentSessionFactory
|
||||
private readonly ILoggerFactory _loggerFactory;
|
||||
private readonly ISessionStore _defaultStore;
|
||||
private readonly VolcanicPaths _paths;
|
||||
private readonly ISecretRedactor _redactor;
|
||||
|
||||
public AgentSessionFactory(IServiceProvider services, IOptions<AgenticCoreOptions> options, ILoggerFactory loggerFactory)
|
||||
public AgentSessionFactory(IServiceProvider services, IOptions<AgenticCoreOptions> options, ILoggerFactory loggerFactory, ISecretRedactor redactor)
|
||||
{
|
||||
_services = services;
|
||||
_options = options.Value;
|
||||
_loggerFactory = loggerFactory;
|
||||
_redactor = redactor;
|
||||
_paths = new VolcanicPaths(_options.AgentDir);
|
||||
_defaultStore = _options.SessionStoreFactory?.Invoke(services) ?? new FileSessionStore(_paths.SessionsDir);
|
||||
}
|
||||
@@ -101,7 +104,7 @@ public sealed class AgentSessionFactory : IAgentSessionFactory
|
||||
}
|
||||
var thinking = options.ThinkingLevel ?? _options.DefaultThinkingLevel;
|
||||
|
||||
var agentSession = new AgentSession(session, store, runner, tools, commands, models, compactor, promptBuilder, options, _options, model, thinking, logger);
|
||||
var agentSession = new AgentSession(session, store, runner, tools, commands, models, compactor, promptBuilder, options, _options, model, thinking, logger, _redactor);
|
||||
runner.Bind(agentSession);
|
||||
if (resumed) agentSession.RestoreBranchSettings();
|
||||
else if (model is not null) await session.AppendModelChangeAsync(model, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using System.Text.Json;
|
||||
using Volcanic.AgenticCore.Abstractions.Messages;
|
||||
using Volcanic.AgenticCore.Abstractions.Processes;
|
||||
using Volcanic.AgenticCore.Abstractions.Tools;
|
||||
|
||||
namespace Volcanic.AgenticCore.Features.Processes;
|
||||
|
||||
/// <summary>Ordinal substring replacement of <see cref="ProcessEnvironmentOptions.SecretValues"/>, longest first (spec §9).</summary>
|
||||
public sealed class SecretRedactor : ISecretRedactor
|
||||
{
|
||||
/// <summary>The token that replaces every secret value.</summary>
|
||||
public const string Replacement = "[REDACTED]";
|
||||
private readonly string[] _secrets;
|
||||
|
||||
/// <summary>Keeps the distinct values of at least <see cref="ProcessEnvironmentOptions.MinimumSecretLength"/> characters.</summary>
|
||||
public SecretRedactor(IEnumerable<string> secretValues)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(secretValues);
|
||||
_secrets = [.. secretValues
|
||||
.Where(s => s is { Length: >= ProcessEnvironmentOptions.MinimumSecretLength })
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderByDescending(s => s.Length)];
|
||||
}
|
||||
|
||||
/// <summary>A redactor with no secrets: <see cref="Redact"/> returns its input.</summary>
|
||||
public static SecretRedactor Empty { get; } = new([]);
|
||||
|
||||
/// <summary>True when no value is long enough to redact.</summary>
|
||||
public bool IsEmpty => _secrets.Length == 0;
|
||||
|
||||
public string Redact(string text)
|
||||
{
|
||||
if (_secrets.Length == 0 || string.IsNullOrEmpty(text)) return text;
|
||||
var result = text;
|
||||
foreach (var secret in _secrets)
|
||||
if (result.Contains(secret, StringComparison.Ordinal))
|
||||
result = result.Replace(secret, Replacement, StringComparison.Ordinal);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Applies an <see cref="ISecretRedactor"/> to tool results: every text block and every string in Details.</summary>
|
||||
internal static class ToolResultRedaction
|
||||
{
|
||||
public static ToolResult Redact(this ISecretRedactor redactor, ToolResult result)
|
||||
{
|
||||
if (redactor is SecretRedactor { IsEmpty: true }) return result;
|
||||
return result with { Content = redactor.RedactContent(result.Content), Details = redactor.RedactDetails(result.Details) };
|
||||
}
|
||||
|
||||
public static List<ContentBlock> RedactContent(this ISecretRedactor redactor, IEnumerable<ContentBlock> content) =>
|
||||
[.. content.Select(b => b is TextContent t ? t with { Text = redactor.Redact(t.Text) } : b)];
|
||||
|
||||
public static JsonElement? RedactDetails(this ISecretRedactor redactor, JsonElement? details)
|
||||
{
|
||||
if (details is not { } element || redactor is SecretRedactor { IsEmpty: true }) return details;
|
||||
var buffer = new System.Buffers.ArrayBufferWriter<byte>();
|
||||
using (var writer = new Utf8JsonWriter(buffer)) Write(writer, element, redactor);
|
||||
using var document = JsonDocument.Parse(buffer.WrittenMemory);
|
||||
return document.RootElement.Clone();
|
||||
}
|
||||
|
||||
private static void Write(Utf8JsonWriter writer, JsonElement element, ISecretRedactor redactor)
|
||||
{
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Object:
|
||||
writer.WriteStartObject();
|
||||
foreach (var property in element.EnumerateObject())
|
||||
{
|
||||
writer.WritePropertyName(property.Name);
|
||||
Write(writer, property.Value, redactor);
|
||||
}
|
||||
writer.WriteEndObject();
|
||||
break;
|
||||
case JsonValueKind.Array:
|
||||
writer.WriteStartArray();
|
||||
foreach (var item in element.EnumerateArray()) Write(writer, item, redactor);
|
||||
writer.WriteEndArray();
|
||||
break;
|
||||
case JsonValueKind.String:
|
||||
writer.WriteStringValue(redactor.Redact(element.GetString()!));
|
||||
break;
|
||||
default:
|
||||
element.WriteTo(writer);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,8 @@ public static class ServiceCollectionExtensions
|
||||
services.TryAddSingleton<IProcessSpawnPolicy>(sp => new ProcessSpawnPolicy(
|
||||
sp.GetRequiredService<IOptions<AgenticCoreOptions>>().Value.ProcessEnvironment,
|
||||
sp.GetRequiredService<ILoggerFactory>().CreateLogger<ProcessSpawnPolicy>()));
|
||||
services.TryAddSingleton<ISecretRedactor>(sp => new SecretRedactor(
|
||||
sp.GetRequiredService<IOptions<AgenticCoreOptions>>().Value.ProcessEnvironment.SecretValues));
|
||||
services.TryAddSingleton<IAgentSessionFactory, AgentSessionFactory>();
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
using Volcanic.AgenticCore.Abstractions.Json;
|
||||
using Volcanic.AgenticCore.Abstractions.Messages;
|
||||
using Volcanic.AgenticCore.Abstractions.Tools;
|
||||
using Volcanic.AgenticCore.Features.Processes;
|
||||
|
||||
namespace Volcanic.AgenticCore.Tests.Processes;
|
||||
|
||||
public class SecretRedactorTests
|
||||
{
|
||||
[Fact]
|
||||
public void A_secret_inside_text_becomes_REDACTED()
|
||||
{
|
||||
var redactor = new SecretRedactor(["123456:bot-token-value"]);
|
||||
Assert.Equal("token=[REDACTED]; again [REDACTED]", redactor.Redact("token=123456:bot-token-value; again 123456:bot-token-value"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Values_shorter_than_8_characters_are_ignored()
|
||||
{
|
||||
var redactor = new SecretRedactor(["admin", "true"]);
|
||||
Assert.True(redactor.IsEmpty);
|
||||
Assert.Equal("admin is true", redactor.Redact("admin is true"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_longer_overlapping_secret_wins()
|
||||
{
|
||||
var redactor = new SecretRedactor(["abcdefgh", "abcdefgh-ijklmnop"]);
|
||||
Assert.Equal("x [REDACTED] y", redactor.Redact("x abcdefgh-ijklmnop y"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_empty_set_returns_the_same_string_instance()
|
||||
{
|
||||
var text = new string('a', 20);
|
||||
Assert.Same(text, SecretRedactor.Empty.Redact(text));
|
||||
Assert.Same(text, new SecretRedactor(["zzzzzzzzzz"]).Redact(text));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tool_results_have_text_and_every_details_string_redacted_and_images_untouched()
|
||||
{
|
||||
const string secret = "super-secret-value";
|
||||
var redactor = new SecretRedactor([secret]);
|
||||
var result = new ToolResult(
|
||||
[new TextContent($"a {secret}"), new ImageContent("aW1n", "image/png")],
|
||||
VolcanicJson.ToElement(new { note = $"b {secret}", nested = new { list = new[] { secret, "ok" } }, count = 3 }));
|
||||
|
||||
var redacted = redactor.Redact(result);
|
||||
|
||||
Assert.Equal("a [REDACTED]", Assert.IsType<TextContent>(redacted.Content[0]).Text);
|
||||
Assert.IsType<ImageContent>(redacted.Content[1]);
|
||||
var json = redacted.Details!.Value.GetRawText();
|
||||
Assert.DoesNotContain(secret, json);
|
||||
Assert.Contains("[REDACTED]", json);
|
||||
Assert.Equal(3, redacted.Details.Value.GetProperty("count").GetInt32());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
using System.Text.Json;
|
||||
using Volcanic.AgenticCore.Abstractions.Events;
|
||||
using Volcanic.AgenticCore.Abstractions.Extensions;
|
||||
using Volcanic.AgenticCore.Abstractions.Extensions.Events;
|
||||
using Volcanic.AgenticCore.Abstractions.Json;
|
||||
using Volcanic.AgenticCore.Abstractions.Messages;
|
||||
using Volcanic.AgenticCore.Abstractions.Tools;
|
||||
using Volcanic.AgenticCore.Features.Tools.BuiltIn;
|
||||
using Volcanic.AgenticCore.Testing;
|
||||
using Volcanic.AgenticCore.Tests.Features;
|
||||
using Volcanic.AgenticCore.Tests.Support;
|
||||
|
||||
namespace Volcanic.AgenticCore.Tests.Processes;
|
||||
|
||||
public class ToolOutputRedactionTests
|
||||
{
|
||||
private const string Secret = "123456789:AAH-secret-bot-token";
|
||||
public sealed record Args(string Value);
|
||||
|
||||
private static ITool Leaky() => new DelegateTool<Args>("leaky", "leaks", (_, inv, _) =>
|
||||
{
|
||||
inv.Report(ToolResult.Text($"progress {Secret}"));
|
||||
return Task.FromResult(ToolResult.Text($"result {Secret}", new { note = $"details {Secret}" }));
|
||||
});
|
||||
|
||||
private static ITool Thrower() => new DelegateTool<Args>("thrower", "throws", (_, _, _) => throw new InvalidOperationException($"failed with {Secret}"));
|
||||
|
||||
private static ITool ToolError() => new DelegateTool<Args>("toolerror", "tool error", (_, _, _) =>
|
||||
throw new ToolExecutionException(ToolResult.Text($"exit 1: {Secret}")));
|
||||
|
||||
private static string Json(object value) => JsonSerializer.Serialize(value, VolcanicJson.Options);
|
||||
|
||||
[Fact]
|
||||
public async Task Results_progress_details_and_errors_are_redacted_everywhere()
|
||||
{
|
||||
var handlerSaw = new List<string>();
|
||||
var extension = new LambdaExtension("tools", api =>
|
||||
{
|
||||
api.RegisterTool(Leaky());
|
||||
api.RegisterTool(Thrower());
|
||||
api.RegisterTool(ToolError());
|
||||
api.On<ToolResultEvent>((e, _, _) => { handlerSaw.Add(Json(e.Content) + (e.Details?.GetRawText() ?? "")); return ValueTask.CompletedTask; });
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
await using var host = new TestHost(o => o.ProcessEnvironment.SecretValues.Add(Secret), extension);
|
||||
host.Faux.Enqueue(
|
||||
FauxTurn.Tool("leaky", new { value = "x" }).WithTool("thrower", new { value = "x" }).WithTool("toolerror", new { value = "x" }),
|
||||
FauxTurn.Say("done"));
|
||||
await using var session = await host.CreateSessionAsync(b => b.ToolExecution = ToolExecutionMode.Sequential);
|
||||
|
||||
await session.PromptAsync("go");
|
||||
|
||||
var updates = host.Events.OfType<AgentSessionEvent.ToolExecutionUpdate>().ToList();
|
||||
Assert.NotEmpty(updates);
|
||||
var ends = host.Events.OfType<AgentSessionEvent.ToolExecutionEnd>().ToList();
|
||||
Assert.Equal(3, ends.Count);
|
||||
var persisted = session.Messages.OfType<ToolResultMessage>().ToList();
|
||||
var nextRequest = host.Faux.Requests[1].Messages;
|
||||
|
||||
foreach (var text in new[] { Json(updates), Json(ends), string.Join("", handlerSaw), Json(persisted), Json(nextRequest) })
|
||||
{
|
||||
Assert.DoesNotContain(Secret, text);
|
||||
Assert.Contains("[REDACTED]", text);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Blocked_calls_are_redacted()
|
||||
{
|
||||
var extension = new LambdaExtension("blocker", api =>
|
||||
{
|
||||
api.RegisterTool(Leaky());
|
||||
api.On<ToolCallEvent>((e, _, _) => { e.Block($"blocked because {Secret}"); return ValueTask.CompletedTask; });
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
await using var host = new TestHost(o => o.ProcessEnvironment.SecretValues.Add(Secret), extension);
|
||||
host.Faux.Enqueue(FauxTurn.Tool("leaky", new { value = "x" }), FauxTurn.Say("done"));
|
||||
await using var session = await host.CreateSessionAsync();
|
||||
|
||||
await session.PromptAsync("go");
|
||||
|
||||
var result = session.Messages.OfType<ToolResultMessage>().Single();
|
||||
Assert.Equal("blocked because [REDACTED]", result.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tool_result_handler_cannot_reintroduce_a_secret()
|
||||
{
|
||||
var extension = new LambdaExtension("appender", api =>
|
||||
{
|
||||
api.RegisterTool(Leaky());
|
||||
api.On<ToolResultEvent>((e, _, _) => { e.Content.Add(new TextContent($" appended {Secret}")); return ValueTask.CompletedTask; });
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
await using var host = new TestHost(o => o.ProcessEnvironment.SecretValues.Add(Secret), extension);
|
||||
host.Faux.Enqueue(FauxTurn.Tool("leaky", new { value = "x" }), FauxTurn.Say("done"));
|
||||
await using var session = await host.CreateSessionAsync();
|
||||
|
||||
await session.PromptAsync("go");
|
||||
|
||||
var sent = Json(host.Faux.Requests[1].Messages);
|
||||
Assert.DoesNotContain(Secret, sent);
|
||||
Assert.Contains("appended [REDACTED]", sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bash_reading_a_secret_file_returns_REDACTED()
|
||||
{
|
||||
await using var host = new TestHost(o => o.ProcessEnvironment.SecretValues.Add(Secret));
|
||||
await File.WriteAllTextAsync(Path.Combine(host.Cwd, "secret.txt"), Secret);
|
||||
host.Faux.Enqueue(FauxTurn.Tool("bash", new { command = OperatingSystem.IsWindows() ? "type secret.txt" : "cat secret.txt" }), FauxTurn.Say("done"));
|
||||
await using var session = await host.CreateSessionAsync();
|
||||
|
||||
await session.PromptAsync("show the file");
|
||||
|
||||
var result = Assert.IsType<ToolResultMessage>(host.Faux.Requests[1].Messages[^1]);
|
||||
Assert.Contains("[REDACTED]", result.Text);
|
||||
Assert.DoesNotContain(Secret, result.Text);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user