admin/Playa
1
0
Fork 0
forked from admin/VolcanicAgenticCore
Code Pull Requests Activity

feat(telegram): OpenAI speech to text service and speech options

Browse Source 
Spec 5: the HttpClient registration is named as AddHttpClient<ISpeechToText, OpenAiSpeechToText>() (package Microsoft.Extensions.Http), which is how Program.cs registers it.
This commit is contained in:
EugeneTes
2026-09-26 22:04:27 +02:00
parent ee3049e47a
commit 8758f4b3d1
11 changed files with 282 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/superpowers/specs/2026-09-26-telegram-attachments-design.md
View File

@@ -256,7 +256,7 @@ public interface ISpeechToText
as given), `model`, `response_format=json`, and `language` when configured.
- Non-2xx → `SpeechToTextException("OpenAI transcription failed: HTTP <status> <first 200 chars of body>")`.
A non-JSON body or a missing `text` gives its own message. The key never appears in messages or logs.
- `HttpClient` comes from `AddHttpClient<OpenAiSpeechToText>()`, with a 2-minute timeout.
- `HttpClient` comes from `AddHttpClient<ISpeechToText, OpenAiSpeechToText>()` (package `Microsoft.Extensions.Http`), with a 2-minute timeout.
`SpeechOptions`, the `Telegram:Speech` section:

5
hosts/Volcanic.AgenticCore.Telegram/Program.cs
View File

@@ -17,6 +17,7 @@ using Volcanic.AgenticCore.Telegram.Chats;
using Volcanic.AgenticCore.Telegram.Inbound;
using Volcanic.AgenticCore.Telegram.Polling;
using Volcanic.AgenticCore.Telegram.Rendering;
using Volcanic.AgenticCore.Telegram.Speech;
var builder = Host.CreateApplicationBuilder(args);
@@ -39,6 +40,9 @@ if (errors.Count > 0)
builder.Services.AddSingleton(options);
builder.Services.AddSingleton(mcp);
builder.Services.AddSingleton(options.Speech);
if (options.Speech.IsEnabled) // no key, no ISpeechToText: voice notes get the setup notice
builder.Services.AddHttpClient<ISpeechToText, OpenAiSpeechToText>(client => client.Timeout = TimeSpan.FromMinutes(2));
if (calendar is not null) builder.Services.AddSingleton(calendar); // ChatRegistry and BotSetupService take it optionally
builder.Services.AddSingleton<OwnerLoginChannel>();
// The extension is constructed per session through ActivatorUtilities, which injects this. LoginUi resolves
@@ -74,6 +78,7 @@ var logger = host.Services.GetRequiredService<ILoggerFactory>().CreateLogger("Vo
HostProcessHardening.Apply(processEnvironment, logger);
logger.LogInformation("{ProcessEnvironment}", TelegramProcessEnvironment.Describe(processEnvironment));
logger.LogInformation("{Mcp}", mcp.DescribePlaywright());
logger.LogInformation("{Speech}", options.Speech.Describe());
if (ClaudeInstallation.Locate(Environment.GetEnvironmentVariable) is { } claude)
logger.LogInformation("Using Claude Code at {Path}, model {Model}", claude, options.Model);
else

11
hosts/Volcanic.AgenticCore.Telegram/Speech/ISpeechToText.cs Normal file
View File

@@ -0,0 +1,11 @@
namespace Volcanic.AgenticCore.Telegram.Speech;
/// <summary>Turns a voice note or audio file into text before a prompt exists. The core only ever sees the text.</summary>
public interface ISpeechToText
{
/// <summary>The trimmed transcript; throws <see cref="SpeechToTextException"/> when the service fails.</summary>
Task<string> TranscribeAsync(ReadOnlyMemory<byte> audio, string fileName, string mimeType, CancellationToken cancellationToken);
}
/// <summary>A transcription failed; the message is fit for the user and never contains the API key.</summary>
public sealed class SpeechToTextException(string message, Exception? inner = null) : Exception(message, inner);

49
hosts/Volcanic.AgenticCore.Telegram/Speech/OpenAiSpeechToText.cs Normal file
View File

@@ -0,0 +1,49 @@
using System.Net.Http.Headers;
using System.Text.Json;
namespace Volcanic.AgenticCore.Telegram.Speech;
/// <summary>OpenAI's <c>/audio/transcriptions</c> (port of VolcanicSecretary's <c>stt/openai.ts</c>).</summary>
public sealed class OpenAiSpeechToText(HttpClient http, SpeechOptions options) : ISpeechToText
{
private const string Prefix = "OpenAI transcription failed: ";
private const int MaxExcerpt = 200;
public async Task<string> TranscribeAsync(ReadOnlyMemory<byte> audio, string fileName, string mimeType, CancellationToken cancellationToken)
{
using var form = new MultipartFormDataContent();
var file = new ReadOnlyMemoryContent(audio);
file.Headers.ContentType = new MediaTypeHeaderValue(mimeType);
form.Add(file, "file", fileName);
form.Add(new StringContent(options.ResolvedModel), "model");
form.Add(new StringContent("json"), "response_format");
if (!string.IsNullOrWhiteSpace(options.Language)) form.Add(new StringContent(options.Language.Trim()), "language");
using var request = new HttpRequestMessage(HttpMethod.Post, options.BaseUrl.TrimEnd('/') + "/audio/transcriptions") { Content = form };
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiKey);
using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var hidden = Hide(body); // hide the key before cutting, so half a key can't leak
var excerpt = hidden.Length > MaxExcerpt ? hidden[..MaxExcerpt] : hidden;
throw new SpeechToTextException($"{Prefix}HTTP {(int)response.StatusCode} {excerpt}".TrimEnd());
}
try
{
using var json = JsonDocument.Parse(body);
if (json.RootElement.ValueKind == JsonValueKind.Object
&& json.RootElement.TryGetProperty("text", out var text) && text.ValueKind == JsonValueKind.String)
return text.GetString()!.Trim();
}
catch (JsonException ex)
{
throw new SpeechToTextException(Prefix + "the response is not JSON.", ex);
}
throw new SpeechToTextException(Prefix + "the response has no text.");
}
private string Hide(string text) =>
string.IsNullOrEmpty(options.ApiKey) ? text : text.Replace(options.ApiKey, "[REDACTED]", StringComparison.Ordinal);
}

27
hosts/Volcanic.AgenticCore.Telegram/Speech/SpeechOptions.cs Normal file
View File

@@ -0,0 +1,27 @@
namespace Volcanic.AgenticCore.Telegram.Speech;
/// <summary>The <c>Telegram:Speech</c> section. Speech to text is on exactly when <see cref="ApiKey"/> is set.</summary>
public sealed class SpeechOptions
{
public const string DefaultModel = "gpt-4o-transcribe";
public const string DefaultBaseUrl = "https://api.openai.com/v1";
/// <summary>Falls back to <c>OPENAI_API_KEY</c>. Empty turns speech to text off.</summary>
public string ApiKey { get; set; } = "";
/// <summary>Falls back to <c>STT_MODEL</c>, then <see cref="DefaultModel"/>.</summary>
public string Model { get; set; } = "";
/// <summary>Any OpenAI-compatible transcription endpoint.</summary>
public string BaseUrl { get; set; } = DefaultBaseUrl;
/// <summary>Optional ISO-639-1 hint.</summary>
public string? Language { get; set; }
public bool IsEnabled => !string.IsNullOrWhiteSpace(ApiKey);
public string ResolvedModel => string.IsNullOrWhiteSpace(Model) ? DefaultModel : Model.Trim();
/// <summary>The startup log line (never the key).</summary>
public string Describe() => IsEnabled ? $"Speech to text: {ResolvedModel}" : "Speech to text: off (no OPENAI_API_KEY)";
}

19
hosts/Volcanic.AgenticCore.Telegram/TelegramOptions.cs
View File

@@ -1,3 +1,5 @@
using Volcanic.AgenticCore.Telegram.Speech;
namespace Volcanic.AgenticCore.Telegram;
/// <summary>The <c>Telegram</c> configuration section.</summary>
@@ -36,6 +38,12 @@ public sealed class TelegramOptions
public TimeSpan ReconnectMaxDelay { get; set; } = TimeSpan.FromSeconds(60);
/// <summary>Image documents up to this size are also shown to the model inline (base64 stays under providers' 5 MB cap).</summary>
public long InlineImageMaxBytes { get; set; } = 3_750_000;
/// <summary>Speech to text for voice notes and audio (<c>Telegram:Speech</c>).</summary>
public SpeechOptions Speech { get; set; } = new();
/// <summary>Reads the environment-variable shortcuts the README documents, when the section leaves them empty.</summary>
public void ApplyEnvironment(Func<string, string?> getEnv)
{
@@ -45,6 +53,13 @@ public sealed class TelegramOptions
foreach (var part in ids.Split([',', ';', ' '], StringSplitOptions.RemoveEmptyEntries))
if (long.TryParse(part, out var id)) AllowedUserIds.Add(id);
}
// Speech: the environment fills only empty settings, and an empty variable counts as unset.
Speech ??= new SpeechOptions();
if (string.IsNullOrWhiteSpace(Speech.ApiKey) && getEnv("OPENAI_API_KEY") is { } key && !string.IsNullOrWhiteSpace(key))
Speech.ApiKey = key.Trim();
if (string.IsNullOrWhiteSpace(Speech.Model) && getEnv("STT_MODEL") is { } model && !string.IsNullOrWhiteSpace(model))
Speech.Model = model.Trim();
}
/// <summary>Configuration problems that must stop the host, empty when valid.</summary>
@@ -62,6 +77,10 @@ public sealed class TelegramOptions
if (ReconnectInitialDelay <= TimeSpan.Zero || ReconnectMaxDelay < ReconnectInitialDelay)
errors.Add("Telegram reconnect delays must be positive with max >= initial.");
if (InputDebounce < TimeSpan.Zero) errors.Add("Telegram:InputDebounce must not be negative.");
if (InlineImageMaxBytes <= 0) errors.Add("Telegram:InlineImageMaxBytes must be positive.");
// On Linux "/v1" parses as an absolute file:// URI; the scheme check rejects it.
if (!Uri.TryCreate(Speech?.BaseUrl, UriKind.Absolute, out var sttUri) || sttUri.Scheme is not ("http" or "https"))
errors.Add("Telegram:Speech:BaseUrl must be an absolute http(s) URL.");
return errors;
}

3
hosts/Volcanic.AgenticCore.Telegram/TelegramProcessEnvironment.cs
View File

@@ -8,7 +8,7 @@ public static class TelegramProcessEnvironment
{
public const string AllowVariable = "PROCESS_ENV_ALLOW";
/// <summary>Adds the section's lists, <c>PROCESS_ENV_ALLOW</c>, the bot token and the Google client secret to <paramref name="target"/>.</summary>
/// <summary>Adds the section's lists, <c>PROCESS_ENV_ALLOW</c>, the bot token, the speech API key and the Google client secret to <paramref name="target"/>.</summary>
public static void Configure(ProcessEnvironmentOptions target, TelegramOptions telegram, ProcessEnvironmentSection section, Func<string, string?> getEnv,
GoogleCalendarOptions? calendar = null)
{
@@ -22,6 +22,7 @@ public static class TelegramProcessEnvironment
target.Allow.AddRange(allow.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
target.HardenHostProcess = section.HardenHostProcess;
if (!string.IsNullOrWhiteSpace(telegram.BotToken)) target.SecretValues.Add(telegram.BotToken);
if (!string.IsNullOrWhiteSpace(telegram.Speech?.ApiKey)) target.SecretValues.Add(telegram.Speech.ApiKey);
if (!string.IsNullOrWhiteSpace(calendar?.ClientSecret)) target.SecretValues.Add(calendar.ClientSecret);
}

1
hosts/Volcanic.AgenticCore.Telegram/Volcanic.AgenticCore.Telegram.csproj
View File

@@ -9,6 +9,7 @@
<ProjectReference Include="../../extensions/Volcanic.AgenticCore.Extensions.GoogleCalendar/Volcanic.AgenticCore.Extensions.GoogleCalendar.csproj" />
<ProjectReference Include="../../extensions/Volcanic.AgenticCore.Extensions.Mcp/Volcanic.AgenticCore.Extensions.Mcp.csproj" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0" />
<PackageReference Include="Telegram.Bot" Version="22.10.3.1" />
</ItemGroup>
<ItemGroup>

62
tests/Volcanic.AgenticCore.Telegram.Tests/FormattingAndOptionsTests.cs
View File

@@ -1,5 +1,6 @@
using Volcanic.AgenticCore.Telegram.Chats;
using Volcanic.AgenticCore.Telegram.Rendering;
using Volcanic.AgenticCore.Telegram.Speech;
namespace Volcanic.AgenticCore.Telegram.Tests;
@@ -98,4 +99,65 @@ public sealed class FormattingAndOptionsTests
Assert.False(options.IsAllowed(4));
Assert.False(options.IsAllowed(null));
}
private static TelegramOptions Valid() => new() { BotToken = "t", AllowedUserIds = [1] };
[Fact]
public void Speech_falls_back_to_OPENAI_API_KEY_and_STT_MODEL()
{
var options = Valid();
options.ApplyEnvironment(name => name switch { "OPENAI_API_KEY" => "sk-env", "STT_MODEL" => "whisper-1", _ => null });
Assert.Equal("sk-env", options.Speech.ApiKey);
Assert.Equal("whisper-1", options.Speech.ResolvedModel);
Assert.True(options.Speech.IsEnabled);
Assert.Equal("Speech to text: whisper-1", options.Speech.Describe());
}
[Fact]
public void Empty_speech_variables_count_as_unset()
{
var options = Valid();
options.ApplyEnvironment(name => name is "OPENAI_API_KEY" or "STT_MODEL" ? "" : null);
Assert.False(options.Speech.IsEnabled);
Assert.Equal(SpeechOptions.DefaultModel, options.Speech.ResolvedModel);
Assert.Equal("Speech to text: off (no OPENAI_API_KEY)", options.Speech.Describe());
}
[Fact]
public void Configured_speech_settings_win_over_the_environment()
{
var options = Valid();
options.Speech.ApiKey = "sk-config";
options.Speech.Model = "gpt-4o-mini-transcribe";
options.ApplyEnvironment(name => name switch { "OPENAI_API_KEY" => "sk-env", "STT_MODEL" => "whisper-1", _ => null });
Assert.Equal(("sk-config", "gpt-4o-mini-transcribe"), (options.Speech.ApiKey, options.Speech.ResolvedModel));
}
[Fact]
public void Speech_defaults_to_gpt_4o_transcribe_on_the_openai_endpoint()
{
var speech = new SpeechOptions();
Assert.Equal(("gpt-4o-transcribe", "https://api.openai.com/v1"), (speech.ResolvedModel, speech.BaseUrl));
Assert.False(speech.IsEnabled);
}
[Fact]
public void Inline_image_limit_defaults_to_3_75_MB_and_must_be_positive()
{
Assert.Equal(3_750_000, Valid().InlineImageMaxBytes);
var options = Valid();
options.InlineImageMaxBytes = 0;
Assert.Contains("Telegram:InlineImageMaxBytes must be positive.", options.Validate());
}
[Theory]
[InlineData("ftp://stt.example/v1")]
[InlineData("/v1")]
[InlineData("")]
public void Speech_base_url_must_be_an_absolute_http_url(string url)
{
var options = Valid();
options.Speech.BaseUrl = url;
Assert.Contains("Telegram:Speech:BaseUrl must be an absolute http(s) URL.", options.Validate());
}
}

95
tests/Volcanic.AgenticCore.Telegram.Tests/OpenAiSpeechToTextTests.cs Normal file
View File

@@ -0,0 +1,95 @@
using System.Net;
using System.Text;
using Volcanic.AgenticCore.Telegram.Speech;
namespace Volcanic.AgenticCore.Telegram.Tests;
public sealed class OpenAiSpeechToTextTests
{
private const string Key = "sk-test-secret-key-123";
private sealed record Part(string? FileName, string? ContentType, string Text);
private sealed class StubHandler(HttpStatusCode status, string body, string mediaType = "application/json") : HttpMessageHandler
{
public HttpRequestMessage? Request { get; private set; }
public Dictionary<string, Part> Parts { get; } = [];
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
Request = request;
foreach (var part in (MultipartFormDataContent)request.Content!)
{
var disposition = part.Headers.ContentDisposition!;
Parts[disposition.Name!.Trim('"')] = new Part(disposition.FileName?.Trim('"'), part.Headers.ContentType?.MediaType,
await part.ReadAsStringAsync(cancellationToken));
}
return new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) };
}
}
private static (OpenAiSpeechToText Stt, StubHandler Handler) Create(HttpStatusCode status, string body, string? language = null)
{
var handler = new StubHandler(status, body);
var options = new SpeechOptions { ApiKey = Key, Language = language, BaseUrl = "https://stt.example/v1/" };
return (new OpenAiSpeechToText(new HttpClient(handler), options), handler);
}
[Fact]
public async Task Posts_a_multipart_form_with_file_model_and_response_format()
{
var (stt, handler) = Create(HttpStatusCode.OK, "{\"text\":\"hello\"}");
await stt.TranscribeAsync(new byte[] { 1, 2, 3 }, "voice.ogg", "audio/ogg", CancellationToken.None);
Assert.Equal(HttpMethod.Post, handler.Request!.Method);
Assert.Equal("https://stt.example/v1/audio/transcriptions", handler.Request.RequestUri!.ToString());
Assert.Equal("Bearer", handler.Request.Headers.Authorization!.Scheme);
Assert.Equal(Key, handler.Request.Headers.Authorization.Parameter);
Assert.Equal(("voice.ogg", "audio/ogg"), (handler.Parts["file"].FileName, handler.Parts["file"].ContentType));
Assert.Equal("gpt-4o-transcribe", handler.Parts["model"].Text);
Assert.Equal("json", handler.Parts["response_format"].Text);
Assert.False(handler.Parts.ContainsKey("language"));
}
[Fact]
public async Task Language_is_sent_only_when_configured()
{
var (stt, handler) = Create(HttpStatusCode.OK, "{\"text\":\"hallo\"}", language: "de");
await stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None);
Assert.Equal("de", handler.Parts["language"].Text);
}
[Fact]
public async Task The_transcript_is_trimmed()
{
var (stt, _) = Create(HttpStatusCode.OK, "{\"text\":\" hello there \\n\"}");
Assert.Equal("hello there", await stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None));
}
[Fact]
public async Task A_non_success_status_names_the_status_and_hides_the_key()
{
var body = "{\"error\":{\"message\":\"Incorrect API key provided: " + Key + "\"}}" + new string('x', 300);
var (stt, _) = Create(HttpStatusCode.Unauthorized, body);
var ex = await Assert.ThrowsAsync<SpeechToTextException>(() => stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None));
Assert.StartsWith("OpenAI transcription failed: HTTP 401 ", ex.Message, StringComparison.Ordinal);
Assert.DoesNotContain(Key, ex.Message, StringComparison.Ordinal);
Assert.True(ex.Message.Length <= "OpenAI transcription failed: HTTP 401 ".Length + 200);
}
[Fact]
public async Task A_non_json_body_has_its_own_message()
{
var (stt, _) = Create(HttpStatusCode.OK, "<html>gateway</html>");
var ex = await Assert.ThrowsAsync<SpeechToTextException>(() => stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None));
Assert.Equal("OpenAI transcription failed: the response is not JSON.", ex.Message);
}
[Fact]
public async Task A_missing_text_has_its_own_message()
{
var (stt, _) = Create(HttpStatusCode.OK, "{\"segments\":[]}");
var ex = await Assert.ThrowsAsync<SpeechToTextException>(() => stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None));
Assert.Equal("OpenAI transcription failed: the response has no text.", ex.Message);
}
}

10
tests/Volcanic.AgenticCore.Telegram.Tests/ProcessEnvironmentTests.cs
View File

@@ -32,6 +32,16 @@ public sealed class ProcessEnvironmentTests
Assert.Empty(target.Validate());
}
[Fact]
public void Configure_adds_the_speech_api_key_to_the_secret_values()
{
var target = new ProcessEnvironmentOptions();
var telegram = new TelegramOptions { BotToken = Token };
telegram.Speech.ApiKey = "sk-speech-secret";
TelegramProcessEnvironment.Configure(target, telegram, new ProcessEnvironmentSection(), _ => null);
Assert.Contains("sk-speech-secret", target.SecretValues);
}
[Fact]
public void The_section_binds_from_configuration()
{