forked from admin/VolcanicAgenticCore
094d518827
Claude Code 2.1.287 loads cc-plugin-* built-ins regardless of --setting-sources, which tripped the fail-closed init check. Disable every known built-in in --settings, and when init lists only built-ins that slipped through, disable them process-wide and replay the request once. Other plugins still fail closed.
643 lines
34 KiB
C#
643 lines
34 KiB
C#
using System.Diagnostics;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Volcanic.AgenticCore.Abstractions.Messages;
|
|
using Volcanic.AgenticCore.Abstractions.Providers;
|
|
using Volcanic.AgenticCore.Abstractions.Ui;
|
|
using Volcanic.AgenticCore.Extensions.ClaudeCode;
|
|
|
|
namespace Volcanic.AgenticCore.Extensions.Tests.ClaudeCode;
|
|
|
|
/// <summary>Drives <see cref="ClaudeCodeApi"/> end to end against a fake <c>claude</c> shell script. No test ever spawns the real CLI.</summary>
|
|
[Collection(FakeScripts.Collection)]
|
|
public class ClaudeCodeApiTests : IDisposable
|
|
{
|
|
private readonly string _root = Directory.CreateTempSubdirectory("claude-api-test-").FullName;
|
|
private readonly string _cwd;
|
|
private readonly SessionRegistry _registry = new();
|
|
|
|
public ClaudeCodeApiTests()
|
|
{
|
|
_cwd = Path.Combine(_root, "project");
|
|
Directory.CreateDirectory(_cwd);
|
|
_registry.Register("session-1", _cwd);
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
_registry.Remove("session-1");
|
|
try { Directory.Delete(_root, recursive: true); } catch (IOException) { }
|
|
}
|
|
|
|
private static string BridgeDll => Path.Combine(AppContext.BaseDirectory, BridgeLocator.BridgeFileName);
|
|
|
|
/// <summary>A fake claude: touches the ready marker (found through the --system-prompt-file argument), prints the body, then runs the tail.</summary>
|
|
private string FakeClaude(string body, string tail = "exec sleep 30")
|
|
{
|
|
var script = Path.Combine(_root, $"fake-claude-{Guid.NewGuid():N}.sh");
|
|
File.WriteAllText(script, "#!/bin/sh\n" +
|
|
"prev=\"\"\ndir=\"\"\n" +
|
|
"for a in \"$@\"; do\n" +
|
|
" if [ \"$prev\" = \"--system-prompt-file\" ]; then dir=$(dirname \"$a\"); fi\n" +
|
|
" prev=\"$a\"\n" +
|
|
"done\n" +
|
|
"if [ -n \"$dir\" ]; then : > \"$dir/mcp-ready\"; fi\n" +
|
|
body + "\n" + tail + "\n");
|
|
if (!OperatingSystem.IsWindows()) File.SetUnixFileMode(script, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute);
|
|
return script;
|
|
}
|
|
|
|
private readonly BuiltinPlugins _builtins = new();
|
|
|
|
private ClaudeCodeApi Api(string claudePath, Func<CancellationToken, Task<IUserInterface?>>? loginUi = null, ClaudeLoginCoordinator? coordinator = null) => new(_registry, new ClaudeCodeApiOptions
|
|
{
|
|
BuiltinPlugins = _builtins,
|
|
LoginUi = loginUi,
|
|
Coordinator = coordinator ?? new ClaudeLoginCoordinator(),
|
|
ClaudePath = claudePath,
|
|
BridgeDllPath = BridgeDll,
|
|
DotnetPath = Environment.ProcessPath ?? "dotnet",
|
|
GetEnv = _ => null,
|
|
ParentEnvironment = new Dictionary<string, string?> { ["PATH"] = Environment.GetEnvironmentVariable("PATH"), ["HOME"] = _root },
|
|
ServerName = Fixtures.ServerName,
|
|
});
|
|
|
|
private static async Task<List<AssistantMessageEvent>> Collect(IAsyncEnumerable<AssistantMessageEvent> stream)
|
|
{
|
|
var events = new List<AssistantMessageEvent>();
|
|
await foreach (var e in stream) events.Add(e);
|
|
return events;
|
|
}
|
|
|
|
private static LlmContext ToolContext() => new("You are a test.", [UserMessage.FromText("run the probe")], [Fixtures.Probe]);
|
|
|
|
[Fact]
|
|
public async Task Tool_proposal_is_captured_and_the_process_is_terminated_at_the_latch()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var script = FakeClaude($"cat \"{Fixtures.FixturePath("claude-2.1.281-stream-tool-ok.jsonl")}\"");
|
|
var api = Api(script);
|
|
var stopwatch = Stopwatch.StartNew();
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1", Reasoning = ThinkingLevel.Medium }, CancellationToken.None));
|
|
stopwatch.Stop();
|
|
|
|
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(8), $"the sleeping fake claude must be terminated promptly; took {stopwatch.Elapsed}");
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "ToolCallStart", "ToolCallDelta", "ToolCallEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
var done = Assert.IsType<AssistantMessageEvent.Done>(events[^1]);
|
|
Assert.Equal(StopReason.ToolUse, done.Reason);
|
|
var call = Assert.Single(done.Message.ToolCalls);
|
|
Assert.Equal("probe", call.Name);
|
|
Assert.Equal("probe", call.Arguments.GetProperty("path").GetString());
|
|
Assert.Equal("calling", done.Message.Text);
|
|
Assert.Equal(20, done.Message.Usage.Input);
|
|
Assert.Equal("claude-code", done.Message.Provider);
|
|
Assert.Equal("sonnet", done.Message.Model);
|
|
// The private request directory is removed before the terminal event is published.
|
|
Assert.DoesNotContain(Directory.GetDirectories(Path.GetTempPath(), "volcanic-claude-*"),
|
|
d => File.Exists(Path.Combine(d, "system-prompt.txt")) && File.ReadAllText(Path.Combine(d, "system-prompt.txt")) == "You are a test.");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Exit_before_a_terminal_event_reports_the_stderr_excerpt()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var script = FakeClaude("echo \"boom: fake failure\" >&2", tail: "exit 1");
|
|
var api = Api(script);
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("sys", "hi"), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.Equal(StopReason.Error, error.Reason);
|
|
Assert.StartsWith("[process_exit]", error.Message.ErrorMessage);
|
|
Assert.Contains("code 1", error.Message.ErrorMessage);
|
|
Assert.Contains("boom: fake failure", error.Message.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Text_only_turn_without_tools_completes_and_runs_the_hooks()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var lines = new[]
|
|
{
|
|
Fixtures.Init().GetRawText(),
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_start\",\"message\":{\"id\":\"msg_t\",\"model\":\"claude-sonnet-5\",\"usage\":{\"input_tokens\":7}}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hello there\"}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_stop\",\"index\":0}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":3}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_stop\"}}",
|
|
"{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"hello there\",\"stop_reason\":\"end_turn\",\"usage\":{\"input_tokens\":7,\"output_tokens\":3}}",
|
|
};
|
|
var fixture = Path.Combine(_root, "text.jsonl");
|
|
File.WriteAllLines(fixture, lines);
|
|
// Echo stdin back into a file so the request payload can be checked, then print the response.
|
|
var script = FakeClaude($"cat > \"{Path.Combine(_root, "stdin.json")}\"\ncat \"{fixture}\"", tail: "exit 0");
|
|
var api = Api(script);
|
|
JsonNode? payload = null;
|
|
var responses = new List<ProviderResponseInfo>();
|
|
var records = 0;
|
|
var options = new StreamOptions
|
|
{
|
|
SessionId = "session-1",
|
|
OnPayload = p => { payload = p; return ValueTask.FromResult<JsonNode?>(null); },
|
|
OnResponse = r => { responses.Add(r); return ValueTask.CompletedTask; },
|
|
OnProviderStreamEvent = _ => { records++; return ValueTask.CompletedTask; },
|
|
};
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("system prompt", "hi @there"), options, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
var done = Assert.IsType<AssistantMessageEvent.Done>(events[^1]);
|
|
Assert.Equal(StopReason.Stop, done.Reason);
|
|
Assert.Equal("hello there", done.Message.Text);
|
|
Assert.Equal(7, done.Message.Usage.Input);
|
|
Assert.Equal(3, done.Message.Usage.Output);
|
|
Assert.Equal(lines.Length, records);
|
|
Assert.Equal(200, Assert.Single(responses).Status);
|
|
|
|
Assert.NotNull(payload);
|
|
var argv = payload["argv"]!.AsArray().Select(a => a!.GetValue<string>()).ToList();
|
|
Assert.Contains("--system-prompt-file", argv);
|
|
Assert.Equal("sonnet", argv[argv.IndexOf("--model") + 1]);
|
|
Assert.DoesNotContain("--effort", argv);
|
|
Assert.Equal("{\"mcpServers\":{}}", argv[argv.IndexOf("--mcp-config") + 1]);
|
|
|
|
var stdin = JsonSerializer.Deserialize<JsonElement>(File.ReadAllText(Path.Combine(_root, "stdin.json")));
|
|
Assert.Equal("user", stdin.GetProperty("type").GetString());
|
|
var blocks = stdin.GetProperty("message").GetProperty("content");
|
|
Assert.Equal(2, blocks.GetArrayLength());
|
|
Assert.DoesNotContain("@", blocks[1].GetProperty("text").GetString());
|
|
Assert.Contains("hi \\u0040there", blocks[1].GetProperty("text").GetString());
|
|
Assert.True(blocks[1].TryGetProperty("cache_control", out _));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Cancellation_terminates_the_process_and_reports_aborted()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var script = FakeClaude($"echo '{Fixtures.Init().GetRawText()}'");
|
|
var api = Api(script);
|
|
using var cts = new CancellationTokenSource();
|
|
var events = new List<AssistantMessageEvent>();
|
|
var stopwatch = Stopwatch.StartNew();
|
|
|
|
await foreach (var e in api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple(null, "hi"), new StreamOptions { SessionId = "session-1" }, cts.Token))
|
|
{
|
|
events.Add(e);
|
|
if (e is AssistantMessageEvent.Start) cts.CancelAfter(100);
|
|
}
|
|
stopwatch.Stop();
|
|
|
|
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(8), $"took {stopwatch.Elapsed}");
|
|
Assert.Equal(["Start", "Error"], events.Select(e => e.GetType().Name).ToArray());
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(events[^1]);
|
|
Assert.Equal(StopReason.Aborted, error.Reason);
|
|
Assert.Equal("aborted", error.Message.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Missing_executable_and_unresolvable_session_fail_without_throwing()
|
|
{
|
|
var missing = new ClaudeCodeApi(_registry, new ClaudeCodeApiOptions { ClaudePath = null, GetEnv = _ => null });
|
|
var events = await Collect(missing.StreamAsync(Fixtures.Sonnet, LlmContext.Simple(null, "hi"), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.StartsWith("[executable_missing]", error.Message.ErrorMessage);
|
|
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(FakeClaude("", tail: "exit 0"));
|
|
var unresolved = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "unknown-session" }, CancellationToken.None));
|
|
Assert.StartsWith("[working_directory]", Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(unresolved)).Message.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Api_key_in_the_parent_environment_is_refused()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = new ClaudeCodeApi(_registry, new ClaudeCodeApiOptions
|
|
{
|
|
ClaudePath = FakeClaude("", tail: "exit 0"),
|
|
GetEnv = _ => null,
|
|
ParentEnvironment = new Dictionary<string, string?> { ["ANTHROPIC_API_KEY"] = "sk-x" },
|
|
});
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple(null, "hi"), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
Assert.StartsWith("[auth]", Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events)).Message.ErrorMessage);
|
|
}
|
|
|
|
private string StatePath(string name) => Path.Combine(_root, "state", name);
|
|
|
|
private int Runs() => File.Exists(StatePath("runs")) ? File.ReadAllLines(StatePath("runs")).Length : 0;
|
|
|
|
/// <summary>
|
|
/// A fake claude whose subscription session expires: <c>-p</c> runs report the expiry until <c>auth login</c> has
|
|
/// accepted a code (and forever when <c>expire-always</c> exists); <c>auth status</c> reflects the login.
|
|
/// </summary>
|
|
private string ExpiringClaude()
|
|
{
|
|
var state = Path.Combine(_root, "state");
|
|
Directory.CreateDirectory(state);
|
|
return FakeClaude(
|
|
$"STATE=\"{state}\"\n" +
|
|
"if [ \"$1\" = \"auth\" ] && [ \"$2\" = \"login\" ]; then\n" +
|
|
" echo \"If the browser didn't open, visit: https://claude.com/cai/oauth/authorize?state=x\"\n" +
|
|
" printf \"Paste code here if prompted > \"; read code; : > \"$STATE/logged-in\"; exit 0\n" +
|
|
"fi\n" +
|
|
"if [ \"$1\" = \"auth\" ] && [ \"$2\" = \"status\" ]; then\n" +
|
|
" if [ -f \"$STATE/logged-in\" ]; then echo '{\"loggedIn\":true,\"authMethod\":\"claude.ai\"}'; else echo '{\"loggedIn\":false}'; fi; exit 0\n" +
|
|
"fi\n" +
|
|
"echo run >> \"$STATE/runs\"\n" +
|
|
$"if [ -f \"$STATE/logged-in\" ] && [ ! -f \"$STATE/expire-always\" ]; then cat \"{Fixtures.FixturePath("claude-2.1.281-stream-tool-ok.jsonl")}\"; " +
|
|
$"else cat \"{Fixtures.FixturePath("claude-2.1.281-stream-auth-expired.jsonl")}\"; fi",
|
|
tail: "exit 0");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Expired_session_logs_in_through_the_login_ui_and_replays()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var notes = new List<string>();
|
|
_registry.Register("session-1", _cwd, (m, _) => { lock (notes) notes.Add(m); });
|
|
var owner = new RecordingUi("the-code");
|
|
var api = Api(ExpiringClaude(), loginUi: _ => Task.FromResult<IUserInterface?>(owner));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
// Only the replay is published: one Start, the proposal, and Done.
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "ToolCallStart", "ToolCallDelta", "ToolCallEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
Assert.Equal(StopReason.ToolUse, Assert.IsType<AssistantMessageEvent.Done>(events[^1]).Reason);
|
|
Assert.Equal(2, Runs());
|
|
Assert.Contains(notes, n => n.Contains("waiting for the bot owner"));
|
|
Assert.Contains(owner.Notes, n => n.Message.Contains("login complete"));
|
|
var question = Assert.Single(owner.Questions);
|
|
Assert.Contains("oauth/authorize", question);
|
|
Assert.Contains(UiLoginPrompt.CodeQuestion, question);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Expired_again_after_login_is_reported_without_a_loop()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var script = ExpiringClaude();
|
|
File.WriteAllText(StatePath("expire-always"), "");
|
|
var owner = new RecordingUi("code");
|
|
var api = Api(script, loginUi: _ => Task.FromResult<IUserInterface?>(owner));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "Error"], events.Select(e => e.GetType().Name).ToArray());
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(events[^1]);
|
|
Assert.StartsWith("[auth] Claude Code session expired", error.Message.ErrorMessage);
|
|
Assert.Equal(2, Runs());
|
|
Assert.Single(owner.Questions);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Without_a_login_ui_the_expiry_is_reported_with_instructions()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(ExpiringClaude());
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(events[^1]);
|
|
Assert.StartsWith("[auth]", error.Message.ErrorMessage);
|
|
Assert.Contains("claude auth login", error.Message.ErrorMessage);
|
|
Assert.Equal(1, Runs());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Unanswered_login_fails_the_request()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var owner = new RecordingUi();
|
|
var api = Api(ExpiringClaude(), loginUi: _ => Task.FromResult<IUserInterface?>(owner));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.Equal(StopReason.Error, error.Reason);
|
|
Assert.Equal("[auth] Claude Code login failed: no code was entered", error.Message.ErrorMessage);
|
|
Assert.Equal(1, Runs());
|
|
Assert.Contains(owner.Notes, n => n.Kind == NotifyKind.Error && n.Message.Contains("login failed"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Unreachable_owner_fails_the_request()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(ExpiringClaude(), loginUi: _ => Task.FromResult<IUserInterface?>(null));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.StartsWith("[auth] Claude Code login failed: the owner could not be reached", error.Message.ErrorMessage);
|
|
Assert.Equal(1, Runs());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Cancelling_while_waiting_for_the_login_reports_aborted()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
using var cts = new CancellationTokenSource();
|
|
var asked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var never = new TaskCompletionSource<IUserInterface?>();
|
|
var api = Api(ExpiringClaude(), loginUi: _ => { asked.TrySetResult(); return never.Task; });
|
|
|
|
var collecting = Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, cts.Token));
|
|
await asked.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
cts.Cancel();
|
|
var events = await collecting.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.Equal(StopReason.Aborted, error.Reason);
|
|
Assert.Equal("aborted", error.Message.ErrorMessage);
|
|
Assert.Equal(1, Runs());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Authentication_failure_without_a_result_is_login_required()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var fixture = Path.Combine(_root, "auth-retry.jsonl");
|
|
File.WriteAllLines(fixture, [
|
|
Fixtures.Init().GetRawText(),
|
|
"{\"type\":\"system\",\"subtype\":\"api_retry\",\"error\":\"authentication_failed\",\"error_status\":401}",
|
|
]);
|
|
var api = Api(FakeClaude($"cat \"{fixture}\"", tail: "exit 1"));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("sys", "hi"), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(events[^1]);
|
|
Assert.StartsWith("[auth]", error.Message.ErrorMessage);
|
|
Assert.Contains("claude auth login", error.Message.ErrorMessage);
|
|
}
|
|
|
|
// ---- deferring mode: every request with a login UI holds its init Start until something else is published ----
|
|
|
|
private int _loginRequests;
|
|
|
|
/// <summary>A login UI that counts how often a login flow asked for the owner.</summary>
|
|
private Func<CancellationToken, Task<IUserInterface?>> CountingLoginUi() => _ =>
|
|
{
|
|
Interlocked.Increment(ref _loginRequests);
|
|
return Task.FromResult<IUserInterface?>(new RecordingUi());
|
|
};
|
|
|
|
/// <summary>Collects the stream, noting for each OnResponse call how many Starts the consumer had already seen.</summary>
|
|
private static async Task<(List<AssistantMessageEvent> Events, List<int> StartsBeforeResponse)> CollectWithResponses(
|
|
Func<StreamOptions, IAsyncEnumerable<AssistantMessageEvent>> stream, Action<StreamOptions>? configure = null)
|
|
{
|
|
var events = new List<AssistantMessageEvent>();
|
|
var startsBeforeResponse = new List<int>();
|
|
var options = new StreamOptions
|
|
{
|
|
SessionId = "session-1",
|
|
OnResponse = _ =>
|
|
{
|
|
lock (events) startsBeforeResponse.Add(events.Count(e => e is AssistantMessageEvent.Start));
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
};
|
|
await foreach (var e in stream(options)) lock (events) events.Add(e);
|
|
return (events, startsBeforeResponse);
|
|
}
|
|
|
|
private string TextFixture()
|
|
{
|
|
var fixture = Path.Combine(_root, "text.jsonl");
|
|
File.WriteAllLines(fixture, [
|
|
Fixtures.Init().GetRawText(),
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_start\",\"message\":{\"id\":\"msg_t\",\"model\":\"claude-sonnet-5\",\"usage\":{\"input_tokens\":7}}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hello there\"}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_stop\",\"index\":0}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":3}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_stop\"}}",
|
|
"{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"hello there\",\"stop_reason\":\"end_turn\",\"usage\":{\"input_tokens\":7,\"output_tokens\":3}}",
|
|
]);
|
|
return fixture;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deferring_text_turn_publishes_one_start_after_one_response()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(FakeClaude($"cat \"{TextFixture()}\"", tail: "exit 0"), loginUi: CountingLoginUi());
|
|
|
|
var (events, responses) = await CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("sys", "hi"), o, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
Assert.Equal("hello there", Assert.IsType<AssistantMessageEvent.Done>(events[^1]).Message.Text);
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(0, _loginRequests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deferring_tool_handoff_is_unchanged()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(FakeClaude($"cat \"{Fixtures.FixturePath("claude-2.1.281-stream-tool-ok.jsonl")}\""), loginUi: CountingLoginUi());
|
|
|
|
var (events, responses) = await CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, ToolContext(), o, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "ToolCallStart", "ToolCallDelta", "ToolCallEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
Assert.Equal(StopReason.ToolUse, Assert.IsType<AssistantMessageEvent.Done>(events[^1]).Reason);
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(0, _loginRequests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deferring_non_login_failure_after_init_publishes_the_held_start()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(FakeClaude($"echo '{Fixtures.Init().GetRawText()}'\necho \"boom: fake failure\" >&2", tail: "exit 1"), loginUi: CountingLoginUi());
|
|
|
|
var (events, responses) = await CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("sys", "hi"), o, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "Error"], events.Select(e => e.GetType().Name).ToArray());
|
|
Assert.StartsWith("[process_exit]", Assert.IsType<AssistantMessageEvent.Error>(events[^1]).Message.ErrorMessage);
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(0, _loginRequests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deferring_cancellation_after_init_reports_aborted_after_the_held_start()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
using var cts = new CancellationTokenSource();
|
|
var api = Api(FakeClaude($"echo '{Fixtures.Init().GetRawText()}'"), loginUi: CountingLoginUi());
|
|
var stopwatch = Stopwatch.StartNew();
|
|
|
|
// The Start is held, so cancel on the init record itself.
|
|
var (events, responses) = await CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("sys", "hi"),
|
|
o with { OnProviderStreamEvent = _ => { cts.CancelAfter(100); return ValueTask.CompletedTask; } }, cts.Token));
|
|
stopwatch.Stop();
|
|
|
|
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(8), $"took {stopwatch.Elapsed}");
|
|
Assert.Equal(["Start", "Error"], events.Select(e => e.GetType().Name).ToArray());
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(events[^1]);
|
|
Assert.Equal(StopReason.Aborted, error.Reason);
|
|
Assert.Equal("aborted", error.Message.ErrorMessage);
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(0, _loginRequests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Login_required_after_content_is_published_without_a_login()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var fixture = Path.Combine(_root, "expired-mid.jsonl");
|
|
File.WriteAllLines(fixture, [
|
|
Fixtures.Init().GetRawText(),
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"message_start\",\"message\":{\"id\":\"msg_t\",\"model\":\"claude-sonnet-5\",\"usage\":{\"input_tokens\":7}}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}}",
|
|
"{\"type\":\"stream_event\",\"event\":{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}}",
|
|
"{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":true,\"api_error_status\":401,\"result\":\"Failed to authenticate. API Error: 401 OAuth token has expired.\",\"usage\":{\"input_tokens\":7,\"output_tokens\":1}}",
|
|
]);
|
|
var runs = Path.Combine(_root, "runs");
|
|
var api = Api(FakeClaude($"echo run >> \"{runs}\"\ncat \"{fixture}\"", tail: "exit 1"), loginUi: CountingLoginUi());
|
|
|
|
var (events, responses) = await CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, LlmContext.Simple("sys", "hi"), o, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "Error"], events.Select(e => e.GetType().Name).ToArray());
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(events[^1]);
|
|
Assert.StartsWith("[auth] Claude Code session expired", error.Message.ErrorMessage);
|
|
Assert.Equal("partial", error.Message.Text);
|
|
Assert.Single(File.ReadAllLines(runs));
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(0, _loginRequests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Replay_after_login_announces_one_response()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(ExpiringClaude(), loginUi: _ => Task.FromResult<IUserInterface?>(new RecordingUi("the-code")));
|
|
|
|
var (events, responses) = await CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, ToolContext(), o, CancellationToken.None));
|
|
|
|
Assert.IsType<AssistantMessageEvent.Done>(events[^1]);
|
|
Assert.Single(events, e => e is AssistantMessageEvent.Start);
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(2, Runs());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Owner_lookup_that_never_answers_is_unreachable()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = new ClaudeCodeApi(_registry, new ClaudeCodeApiOptions
|
|
{
|
|
ClaudePath = ExpiringClaude(),
|
|
GetEnv = _ => null,
|
|
LoginOptions = new ClaudeLoginOptions { OwnerTimeout = TimeSpan.FromMilliseconds(200) },
|
|
});
|
|
|
|
var outcome = await api.RunLoginFlowAsync(_ => new TaskCompletionSource<IUserInterface?>().Task, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
Assert.False(outcome.Success);
|
|
Assert.StartsWith("the owner could not be reached", outcome.Error);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Login_required_after_a_login_completed_mid_attempt_replays_without_a_second_login()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var state = Path.Combine(_root, "state");
|
|
Directory.CreateDirectory(state);
|
|
// The first run reports the expiry only once released (after the test's own login); later runs succeed.
|
|
var script = FakeClaude(
|
|
$"STATE=\"{state}\"\n" +
|
|
"echo run >> \"$STATE/runs\"\n" +
|
|
"if [ \"$(wc -l < \"$STATE/runs\")\" -eq 1 ]; then\n" +
|
|
" : > \"$STATE/started\"\n" +
|
|
" while [ ! -f \"$STATE/release\" ]; do sleep 0.05; done\n" +
|
|
$" cat \"{Fixtures.FixturePath("claude-2.1.281-stream-auth-expired.jsonl")}\"\n" +
|
|
$"else cat \"{Fixtures.FixturePath("claude-2.1.281-stream-tool-ok.jsonl")}\"; fi",
|
|
tail: "exit 0");
|
|
var coordinator = new ClaudeLoginCoordinator();
|
|
var api = Api(script, loginUi: CountingLoginUi(), coordinator: coordinator);
|
|
|
|
var collecting = CollectWithResponses(o => api.StreamAsync(Fixtures.Sonnet, ToolContext(), o, CancellationToken.None));
|
|
var deadline = DateTime.UtcNow.AddSeconds(10);
|
|
while (!File.Exists(StatePath("started")) && DateTime.UtcNow < deadline) await Task.Delay(20);
|
|
Assert.True(File.Exists(StatePath("started")));
|
|
// A login elsewhere completes while this attempt is still running.
|
|
Assert.True((await coordinator.EnsureLoggedInAsync(_ => Task.FromResult(LoginOutcome.Ok), CancellationToken.None)).Success);
|
|
File.WriteAllText(StatePath("release"), "");
|
|
var (events, responses) = await collecting.WaitAsync(TimeSpan.FromSeconds(20));
|
|
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "ToolCallStart", "ToolCallDelta", "ToolCallEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
Assert.Equal(2, Runs());
|
|
Assert.Equal([0], responses);
|
|
Assert.Equal(0, _loginRequests);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A fake claude that loads <paramref name="plugin"/> unless <c>--settings</c> disables it (or always, with
|
|
/// <paramref name="ignoreSettings"/>); without the plugin it proposes the probe tool.
|
|
/// </summary>
|
|
private string PluginClaude(string plugin, bool ignoreSettings = false)
|
|
{
|
|
var state = Path.Combine(_root, "state");
|
|
Directory.CreateDirectory(state);
|
|
var init = Fixtures.ProbeInit().GetRawText().Replace("\"plugins\":[]", $"\"plugins\":[{plugin}]", StringComparison.Ordinal);
|
|
return FakeClaude(
|
|
$"echo run >> \"{state}/runs\"\n" +
|
|
"settings=\"\"\nprev=\"\"\n" +
|
|
"for a in \"$@\"; do if [ \"$prev\" = \"--settings\" ]; then settings=\"$a\"; fi; prev=\"$a\"; done\n" +
|
|
(ignoreSettings ? "" : "case \"$settings\" in *'\"cc-plugin-new@builtin\":false'*) " +
|
|
$"cat \"{Fixtures.FixturePath("claude-2.1.281-stream-tool-ok.jsonl")}\"; exit 0;; esac\n") +
|
|
$"echo '{init}'",
|
|
tail: "exit 0");
|
|
}
|
|
|
|
private const string NewBuiltin = "{\"name\":\"cc-plugin-new\",\"path\":\"builtin\",\"source\":\"cc-plugin-new@builtin\"}";
|
|
|
|
[Fact]
|
|
public async Task An_unexpected_builtin_plugin_is_disabled_and_the_request_replayed_once()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(PluginClaude(NewBuiltin));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
Assert.Equal(["Start", "TextStart", "TextDelta", "TextEnd", "ToolCallStart", "ToolCallDelta", "ToolCallEnd", "Done"], events.Select(e => e.GetType().Name).ToArray());
|
|
Assert.Equal(2, Runs());
|
|
Assert.Contains("cc-plugin-new@builtin", _builtins.Disabled);
|
|
|
|
// Learned for the process: the next request disables it from the start.
|
|
await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
Assert.Equal(3, Runs());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_builtin_plugin_that_stays_loaded_fails_after_one_replay()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(PluginClaude(NewBuiltin, ignoreSettings: true));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.Contains("[protocol_init] Claude Code loaded unexpected customizations (plugins: cc-plugin-new)", error.Message.ErrorMessage);
|
|
Assert.Equal(2, Runs());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_non_builtin_plugin_fails_closed_without_a_replay()
|
|
{
|
|
if (!Fixtures.IsPosix) return;
|
|
var api = Api(PluginClaude("{\"name\":\"cc-plugin-new\",\"path\":\"/home/u/.claude/plugins/x\",\"source\":\"cc-plugin-new@market\"}"));
|
|
|
|
var events = await Collect(api.StreamAsync(Fixtures.Sonnet, ToolContext(), new StreamOptions { SessionId = "session-1" }, CancellationToken.None));
|
|
|
|
var error = Assert.IsType<AssistantMessageEvent.Error>(Assert.Single(events));
|
|
Assert.Contains("unexpected customizations", error.Message.ErrorMessage);
|
|
Assert.Equal(1, Runs());
|
|
Assert.DoesNotContain("cc-plugin-new@market", _builtins.Disabled);
|
|
}
|
|
}
|