admin/VolcanicAgenticCore
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(telegram): ElevenLabs text to speech for the voice tool

Browse Source 
ElevenLabs speaks for the voice tool when ELEVENLABS_API_KEY and ELEVENLABS_VOICE_ID
are set (TTS_PROVIDER=elevenlabs|openai picks, the other is the fallback). Opus output
(opus_48000_64) goes to Telegram as is; voice names resolve through /v2/voices;
instructions become audio tags on eleven_v3 and are ignored on other models. The key
is redacted from tool output and never reaches agent processes.
This commit is contained in:
EugeneTes
2026-10-14 13:07:36 +00:00
parent 6e3241fed9
commit 7fef3ce94e
17 changed files with 554 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
tests/Volcanic.AgenticCore.Telegram.Tests/ElevenLabsTextToSpeechTests.cs Normal file
View File

@@ -0,0 +1,138 @@
using System.Net;
using System.Text;
using System.Text.Json;
using Volcanic.AgenticCore.Telegram.Speech;
namespace Volcanic.AgenticCore.Telegram.Tests;
public sealed class ElevenLabsTextToSpeechTests
{
private const string Key = "sk_eleven-test-secret-123";
private const string VoiceId = "21m00Tcm4TlvDq8ikWAM";
private static readonly byte[] Ogg = [0x4F, 0x67, 0x67, 0x53, 9];
private sealed class StubHandler(params (HttpStatusCode Status, byte[] Body)[] answers) : HttpMessageHandler
{
public List<(HttpRequestMessage Request, JsonElement? Body)> Requests { get; } = [];
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
JsonElement? body = request.Content is null
? null
: JsonDocument.Parse(await request.Content.ReadAsStringAsync(cancellationToken)).RootElement.Clone();
Requests.Add((request, body));
var (status, bytes) = answers[Math.Min(Requests.Count, answers.Length) - 1];
return new HttpResponseMessage(status) { Content = new ByteArrayContent(bytes) };
}
}
private static (ElevenLabsTextToSpeech Tts, StubHandler Handler) Create(string? model = null, params (HttpStatusCode, byte[])[] answers)
{
var handler = new StubHandler(answers.Length > 0 ? answers : [(HttpStatusCode.OK, Ogg)]);
var options = new SpeechOptions { RetryDelay = TimeSpan.Zero };
options.ElevenLabs = new ElevenLabsOptions { ApiKey = Key, VoiceId = VoiceId, Model = model ?? "", BaseUrl = "https://el.example/" };
return (new ElevenLabsTextToSpeech(new HttpClient(handler), options), handler);
}
private static byte[] Utf8(string text) => Encoding.UTF8.GetBytes(text);
[Fact]
public async Task Posts_text_and_model_to_the_voice_with_an_opus_format_and_the_key_header()
{
var (tts, handler) = Create();
Assert.Equal(Ogg, await tts.SynthesizeAsync("Привіт", null, "cheerful", CancellationToken.None));
var (request, body) = Assert.Single(handler.Requests);
Assert.Equal(HttpMethod.Post, request.Method);
Assert.Equal($"https://el.example/v1/text-to-speech/{VoiceId}?output_format=opus_48000_64", request.RequestUri!.ToString());
Assert.Equal(Key, Assert.Single(request.Headers.GetValues("xi-api-key")));
Assert.Null(request.Headers.Authorization);
Assert.Equal("Привіт", body!.Value.GetProperty("text").GetString()); // multilingual v2 ignores instructions
Assert.Equal("eleven_multilingual_v2", body.Value.GetProperty("model_id").GetString());
}
[Fact]
public async Task V3_turns_instructions_into_audio_tags()
{
var (tts, handler) = Create("eleven_v3");
await tts.SynthesizeAsync("Привет", null, "warm, calm", CancellationToken.None);
await tts.SynthesizeAsync("Привет", null, "[whispers] [sighs]", CancellationToken.None);
await tts.SynthesizeAsync("Привет", null, null, CancellationToken.None);
Assert.Equal(["[warm, calm] Привет", "[whispers] [sighs] Привет", "Привет"],
handler.Requests.Select(r => r.Body!.Value.GetProperty("text").GetString()));
Assert.Contains("audio tags", tts.ArgumentGuide, StringComparison.Ordinal);
Assert.Contains("ignored", Create().Tts.ArgumentGuide, StringComparison.Ordinal);
}
[Fact]
public async Task A_voice_id_argument_is_used_as_is()
{
var (tts, handler) = Create();
await tts.SynthesizeAsync("hi", "AZnzlk1XvdvUeBnXmlld", null, CancellationToken.None);
Assert.EndsWith("/v1/text-to-speech/AZnzlk1XvdvUeBnXmlld?output_format=opus_48000_64", handler.Requests[0].Request.RequestUri!.ToString());
}
[Fact]
public async Task A_voice_name_is_looked_up_once_and_cached()
{
var voices = Utf8("""{"voices":[{"voice_id":"aaaaaaaaaaaaaaaaaaa1","name":"Anna Pro"},{"voice_id":"bbbbbbbbbbbbbbbbbbb2","name":"Anna - calm narrator"}]}""");
var (tts, handler) = Create(null, (HttpStatusCode.OK, voices), (HttpStatusCode.OK, Ogg));
await tts.SynthesizeAsync("hi", "anna", null, CancellationToken.None);
await tts.SynthesizeAsync("hi", "Anna", null, CancellationToken.None);
Assert.Equal(3, handler.Requests.Count);
Assert.Equal(HttpMethod.Get, handler.Requests[0].Request.Method);
Assert.Equal("https://el.example/v2/voices?page_size=100&search=anna", handler.Requests[0].Request.RequestUri!.ToString());
Assert.All(handler.Requests.Skip(1), r => Assert.Contains("/v1/text-to-speech/bbbbbbbbbbbbbbbbbbb2?", r.Request.RequestUri!.ToString(), StringComparison.Ordinal));
}
[Fact]
public async Task An_unknown_voice_name_is_an_error()
{
var (tts, handler) = Create(null, (HttpStatusCode.OK, Utf8("""{"voices":[]}""")));
var ex = await Assert.ThrowsAsync<TextToSpeechException>(() => tts.SynthesizeAsync("hi", "Nobody", null, CancellationToken.None));
Assert.Contains("no voice named \"Nobody\"", ex.Message, StringComparison.Ordinal);
Assert.Single(handler.Requests);
}
[Fact]
public async Task Quota_errors_show_status_and_message_with_the_key_hidden_and_are_not_retried()
{
var error = Utf8($$$"""{"detail":{"status":"quota_exceeded","message":"This request exceeds your quota of 10000. Key {{{Key}}}"}}""");
var (tts, handler) = Create(null, (HttpStatusCode.Unauthorized, error));
var ex = await Assert.ThrowsAsync<TextToSpeechException>(() => tts.SynthesizeAsync("hi", null, null, CancellationToken.None));
Assert.Single(handler.Requests);
Assert.StartsWith("ElevenLabs speech synthesis failed: HTTP 401 quota_exceeded: This request exceeds your quota", ex.Message, StringComparison.Ordinal);
Assert.DoesNotContain(Key, ex.Message, StringComparison.Ordinal);
Assert.Contains("[REDACTED]", ex.Message, StringComparison.Ordinal);
}
[Fact]
public async Task Rate_limits_are_retried()
{
var busy = Utf8("""{"detail":{"status":"too_many_concurrent_requests","message":"busy"}}""");
var (tts, handler) = Create(null, (HttpStatusCode.TooManyRequests, busy), (HttpStatusCode.OK, Ogg));
Assert.Equal(Ogg, await tts.SynthesizeAsync("hi", null, null, CancellationToken.None));
Assert.Equal(2, handler.Requests.Count);
}
[Fact]
public async Task Rate_limits_that_persist_are_reported()
{
var busy = Utf8("""{"detail":{"status":"too_many_concurrent_requests","message":"busy"}}""");
var (tts, handler) = Create(null, (HttpStatusCode.TooManyRequests, busy));
var ex = await Assert.ThrowsAsync<TextToSpeechException>(() => tts.SynthesizeAsync("hi", null, null, CancellationToken.None));
Assert.Equal(SpeechOptions.MaxAttempts, handler.Requests.Count);
Assert.Equal("ElevenLabs speech synthesis failed: HTTP 429 too_many_concurrent_requests: busy", ex.Message);
}
[Fact]
public async Task Audio_that_is_not_ogg_is_refused()
{
var (tts, _) = Create(null, (HttpStatusCode.OK, Utf8("ID3 mp3 bytes")));
var ex = await Assert.ThrowsAsync<TextToSpeechException>(() => tts.SynthesizeAsync("hi", null, null, CancellationToken.None));
Assert.Contains("not OGG/Opus", ex.Message, StringComparison.Ordinal);
}
}

51
tests/Volcanic.AgenticCore.Telegram.Tests/FormattingAndOptionsTests.cs
View File

@@ -121,7 +121,7 @@ public sealed class FormattingAndOptionsTests
options.ApplyEnvironment(name => name is "OPENAI_API_KEY" or "STT_MODEL" ? "" : null);
Assert.False(options.Speech.IsEnabled);
Assert.Equal(SpeechOptions.DefaultModel, options.Speech.ResolvedModel);
Assert.Equal("Speech to text and text to speech: off (no OPENAI_API_KEY)", options.Speech.Describe());
Assert.Equal("Speech to text: off (no OPENAI_API_KEY); text to speech: off (no OPENAI_API_KEY or ELEVENLABS_API_KEY)", options.Speech.Describe());
}
[Fact]
@@ -143,6 +143,55 @@ public sealed class FormattingAndOptionsTests
Assert.Equal(("gpt-4o-mini-tts", "nova"), (new SpeechOptions().ResolvedTtsModel, new SpeechOptions().ResolvedTtsVoice));
}
[Fact]
public void ElevenLabs_reads_its_variables_and_becomes_the_voice_provider()
{
var options = Valid();
options.ApplyEnvironment(name => name switch
{
"OPENAI_API_KEY" => "sk-env", "ELEVENLABS_API_KEY" => " sk_el ", "ELEVENLABS_VOICE_ID" => "21m00Tcm4TlvDq8ikWAM",
"ELEVENLABS_MODEL" => "eleven_v3", _ => null,
});
var eleven = options.Speech.ElevenLabs;
Assert.Equal(("sk_el", "21m00Tcm4TlvDq8ikWAM", "eleven_v3"), (eleven.ApiKey, eleven.VoiceId, eleven.ResolvedModel));
Assert.True(eleven.SupportsAudioTags);
Assert.Equal(TextToSpeechProvider.ElevenLabs, options.Speech.ResolvedTtsProvider);
Assert.Equal("Speech to text: gpt-4o-transcribe; text to speech: ElevenLabs eleven_v3, voice 21m00Tcm4TlvDq8ikWAM", options.Speech.Describe());
Assert.Equal(("eleven_multilingual_v2", false), (new ElevenLabsOptions().ResolvedModel, new ElevenLabsOptions().SupportsAudioTags));
}
[Theory]
[InlineData(null, true, true, TextToSpeechProvider.ElevenLabs)]
[InlineData("elevenlabs", true, true, TextToSpeechProvider.ElevenLabs)]
[InlineData("openai", true, true, TextToSpeechProvider.OpenAi)]
[InlineData("OpenAI", false, true, TextToSpeechProvider.ElevenLabs)]
[InlineData("elevenlabs", true, false, TextToSpeechProvider.OpenAi)]
[InlineData(null, true, false, TextToSpeechProvider.OpenAi)]
[InlineData(null, false, false, TextToSpeechProvider.None)]
public void The_voice_provider_is_the_preferred_one_when_configured_else_the_other(string? provider, bool openAi, bool elevenLabs, TextToSpeechProvider expected)
{
var speech = new SpeechOptions { TtsProvider = provider ?? "", ApiKey = openAi ? "sk" : "" };
if (elevenLabs) (speech.ElevenLabs.ApiKey, speech.ElevenLabs.VoiceId) = ("el", "voice");
Assert.Equal(expected, speech.ResolvedTtsProvider);
}
[Fact]
public void ElevenLabs_without_a_voice_id_stays_off_and_says_why()
{
var options = Valid();
options.ApplyEnvironment(name => name switch { "OPENAI_API_KEY" => "sk-env", "ELEVENLABS_API_KEY" => "sk_el", "TTS_PROVIDER" => "elevenlabs", _ => null });
Assert.Equal(TextToSpeechProvider.OpenAi, options.Speech.ResolvedTtsProvider);
Assert.EndsWith("(ElevenLabs off: ELEVENLABS_VOICE_ID is not set)", options.Speech.Describe(), StringComparison.Ordinal);
}
[Fact]
public void An_unknown_TTS_PROVIDER_stops_the_host()
{
var options = Valid();
options.ApplyEnvironment(name => name == "TTS_PROVIDER" ? "google" : null);
Assert.Contains("TTS_PROVIDER (Telegram:Speech:TtsProvider) must be 'elevenlabs' or 'openai'.", options.Validate());
}
[Fact]
public void Speech_defaults_to_gpt_4o_transcribe_on_the_openai_endpoint()
{

42
tests/Volcanic.AgenticCore.Telegram.Tests/ProcessEnvironmentTests.cs
View File

@@ -65,6 +65,48 @@ public sealed class ProcessEnvironmentTests
Assert.Contains("sk-speech-secret", target.SecretValues);
}
[Fact]
public void Configure_adds_the_elevenlabs_api_key_to_the_secret_values()
{
var target = new ProcessEnvironmentOptions();
var telegram = new TelegramOptions { BotToken = Token };
telegram.Speech.ElevenLabs.ApiKey = "sk_eleven-secret";
TelegramProcessEnvironment.Configure(target, telegram, new ProcessEnvironmentSection(), _ => null);
Assert.Contains("sk_eleven-secret", target.SecretValues);
}
[Fact]
public async Task Bash_never_sees_ELEVENLABS_API_KEY_and_its_value_shows_as_REDACTED()
{
const string ElevenKey = "sk_eleven-test-key-0123456789";
var environment = new Dictionary<string, string?>(OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
foreach (var name in new[] { "PATH", "HOME", "SystemRoot", "ComSpec", "TEMP", "TMP" })
if (Environment.GetEnvironmentVariable(name) is { } value) environment[name] = value;
environment["ELEVENLABS_API_KEY"] = ElevenKey;
await using var host = new ChatHost((h, services) =>
{
h.Options.BotToken = Token;
h.Options.Speech.ElevenLabs.ApiKey = ElevenKey;
services.Configure<AgenticCoreOptions>(o =>
TelegramProcessEnvironment.Configure(o.ProcessEnvironment, h.Options, new ProcessEnvironmentSection { HardenHostProcess = false }, _ => null));
services.AddSingleton<IProcessSpawnPolicy>(sp =>
new ProcessSpawnPolicy(sp.GetRequiredService<IOptions<AgenticCoreOptions>>().Value.ProcessEnvironment, null, () => environment));
});
var chat = await host.ChatAsync();
await File.WriteAllTextAsync(Path.Combine(chat.Session.Cwd, "key.txt"), ElevenKey);
var command = OperatingSystem.IsWindows() ? "set & type key.txt" : "env; cat key.txt";
host.Faux.Enqueue(FauxTurn.Tool("bash", new { command }), FauxTurn.Say("done"));
await host.SendAsync("show the environment");
await host.SettleAsync();
var result = LastToolResult(host);
Assert.Contains("PATH=", result, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("ELEVENLABS_API_KEY", result);
Assert.DoesNotContain(ElevenKey, result);
Assert.Contains("[REDACTED]", result);
}
[Fact]
public void The_section_binds_from_configuration()
{

2
tests/Volcanic.AgenticCore.Telegram.Tests/Support/FakeTextToSpeech.cs
View File

@@ -6,6 +6,8 @@ namespace Volcanic.AgenticCore.Telegram.Tests.Support;
public sealed class FakeTextToSpeech : ITextToSpeech
{
public Exception? Failure { get; set; }
public string ArgumentGuide { get; set; } = "voice: fake voices. instructions: fake style.";
public List<(string Text, string? Voice, string? Instructions)> Calls { get; } = [];
public Task<byte[]> SynthesizeAsync(string text, string? voice, string? instructions, CancellationToken cancellationToken)