From 8758f4b3d18e2d7bb2c929558fabb511fcb868b6 Mon Sep 17 00:00:00 2001 From: EugeneTes Date: Sat, 26 Sep 2026 22:04:27 +0200 Subject: [PATCH] feat(telegram): OpenAI speech to text service and speech options Spec 5: the HttpClient registration is named as AddHttpClient() (package Microsoft.Extensions.Http), which is how Program.cs registers it. --- .../2026-09-26-telegram-attachments-design.md | 2 +- .../Volcanic.AgenticCore.Telegram/Program.cs | 5 + .../Speech/ISpeechToText.cs | 11 +++ .../Speech/OpenAiSpeechToText.cs | 49 ++++++++++ .../Speech/SpeechOptions.cs | 27 ++++++ .../TelegramOptions.cs | 19 ++++ .../TelegramProcessEnvironment.cs | 3 +- .../Volcanic.AgenticCore.Telegram.csproj | 1 + .../FormattingAndOptionsTests.cs | 62 ++++++++++++ .../OpenAiSpeechToTextTests.cs | 95 +++++++++++++++++++ .../ProcessEnvironmentTests.cs | 10 ++ 11 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 hosts/Volcanic.AgenticCore.Telegram/Speech/ISpeechToText.cs create mode 100644 hosts/Volcanic.AgenticCore.Telegram/Speech/OpenAiSpeechToText.cs create mode 100644 hosts/Volcanic.AgenticCore.Telegram/Speech/SpeechOptions.cs create mode 100644 tests/Volcanic.AgenticCore.Telegram.Tests/OpenAiSpeechToTextTests.cs diff --git a/docs/superpowers/specs/2026-09-26-telegram-attachments-design.md b/docs/superpowers/specs/2026-09-26-telegram-attachments-design.md index 246528b..21ed7b9 100644 --- a/docs/superpowers/specs/2026-09-26-telegram-attachments-design.md +++ b/docs/superpowers/specs/2026-09-26-telegram-attachments-design.md @@ -256,7 +256,7 @@ public interface ISpeechToText as given), `model`, `response_format=json`, and `language` when configured. - Non-2xx → `SpeechToTextException("OpenAI transcription failed: HTTP ")`. A non-JSON body or a missing `text` gives its own message. The key never appears in messages or logs. -- `HttpClient` comes from `AddHttpClient()`, with a 2-minute timeout. +- `HttpClient` comes from `AddHttpClient()` (package `Microsoft.Extensions.Http`), with a 2-minute timeout. `SpeechOptions`, the `Telegram:Speech` section: diff --git a/hosts/Volcanic.AgenticCore.Telegram/Program.cs b/hosts/Volcanic.AgenticCore.Telegram/Program.cs index d6217c4..35726c2 100644 --- a/hosts/Volcanic.AgenticCore.Telegram/Program.cs +++ b/hosts/Volcanic.AgenticCore.Telegram/Program.cs @@ -17,6 +17,7 @@ using Volcanic.AgenticCore.Telegram.Chats; using Volcanic.AgenticCore.Telegram.Inbound; using Volcanic.AgenticCore.Telegram.Polling; using Volcanic.AgenticCore.Telegram.Rendering; +using Volcanic.AgenticCore.Telegram.Speech; var builder = Host.CreateApplicationBuilder(args); @@ -39,6 +40,9 @@ if (errors.Count > 0) builder.Services.AddSingleton(options); builder.Services.AddSingleton(mcp); +builder.Services.AddSingleton(options.Speech); +if (options.Speech.IsEnabled) // no key, no ISpeechToText: voice notes get the setup notice + builder.Services.AddHttpClient(client => client.Timeout = TimeSpan.FromMinutes(2)); if (calendar is not null) builder.Services.AddSingleton(calendar); // ChatRegistry and BotSetupService take it optionally builder.Services.AddSingleton(); // The extension is constructed per session through ActivatorUtilities, which injects this. LoginUi resolves @@ -74,6 +78,7 @@ var logger = host.Services.GetRequiredService().CreateLogger("Vo HostProcessHardening.Apply(processEnvironment, logger); logger.LogInformation("{ProcessEnvironment}", TelegramProcessEnvironment.Describe(processEnvironment)); logger.LogInformation("{Mcp}", mcp.DescribePlaywright()); +logger.LogInformation("{Speech}", options.Speech.Describe()); if (ClaudeInstallation.Locate(Environment.GetEnvironmentVariable) is { } claude) logger.LogInformation("Using Claude Code at {Path}, model {Model}", claude, options.Model); else diff --git a/hosts/Volcanic.AgenticCore.Telegram/Speech/ISpeechToText.cs b/hosts/Volcanic.AgenticCore.Telegram/Speech/ISpeechToText.cs new file mode 100644 index 0000000..8439e70 --- /dev/null +++ b/hosts/Volcanic.AgenticCore.Telegram/Speech/ISpeechToText.cs @@ -0,0 +1,11 @@ +namespace Volcanic.AgenticCore.Telegram.Speech; + +/// Turns a voice note or audio file into text before a prompt exists. The core only ever sees the text. +public interface ISpeechToText +{ + /// The trimmed transcript; throws when the service fails. + Task TranscribeAsync(ReadOnlyMemory audio, string fileName, string mimeType, CancellationToken cancellationToken); +} + +/// A transcription failed; the message is fit for the user and never contains the API key. +public sealed class SpeechToTextException(string message, Exception? inner = null) : Exception(message, inner); diff --git a/hosts/Volcanic.AgenticCore.Telegram/Speech/OpenAiSpeechToText.cs b/hosts/Volcanic.AgenticCore.Telegram/Speech/OpenAiSpeechToText.cs new file mode 100644 index 0000000..9883310 --- /dev/null +++ b/hosts/Volcanic.AgenticCore.Telegram/Speech/OpenAiSpeechToText.cs @@ -0,0 +1,49 @@ +using System.Net.Http.Headers; +using System.Text.Json; + +namespace Volcanic.AgenticCore.Telegram.Speech; + +/// OpenAI's /audio/transcriptions (port of VolcanicSecretary's stt/openai.ts). +public sealed class OpenAiSpeechToText(HttpClient http, SpeechOptions options) : ISpeechToText +{ + private const string Prefix = "OpenAI transcription failed: "; + private const int MaxExcerpt = 200; + + public async Task TranscribeAsync(ReadOnlyMemory audio, string fileName, string mimeType, CancellationToken cancellationToken) + { + using var form = new MultipartFormDataContent(); + var file = new ReadOnlyMemoryContent(audio); + file.Headers.ContentType = new MediaTypeHeaderValue(mimeType); + form.Add(file, "file", fileName); + form.Add(new StringContent(options.ResolvedModel), "model"); + form.Add(new StringContent("json"), "response_format"); + if (!string.IsNullOrWhiteSpace(options.Language)) form.Add(new StringContent(options.Language.Trim()), "language"); + + using var request = new HttpRequestMessage(HttpMethod.Post, options.BaseUrl.TrimEnd('/') + "/audio/transcriptions") { Content = form }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiKey); + using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false); + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var hidden = Hide(body); // hide the key before cutting, so half a key can't leak + var excerpt = hidden.Length > MaxExcerpt ? hidden[..MaxExcerpt] : hidden; + throw new SpeechToTextException($"{Prefix}HTTP {(int)response.StatusCode} {excerpt}".TrimEnd()); + } + + try + { + using var json = JsonDocument.Parse(body); + if (json.RootElement.ValueKind == JsonValueKind.Object + && json.RootElement.TryGetProperty("text", out var text) && text.ValueKind == JsonValueKind.String) + return text.GetString()!.Trim(); + } + catch (JsonException ex) + { + throw new SpeechToTextException(Prefix + "the response is not JSON.", ex); + } + throw new SpeechToTextException(Prefix + "the response has no text."); + } + + private string Hide(string text) => + string.IsNullOrEmpty(options.ApiKey) ? text : text.Replace(options.ApiKey, "[REDACTED]", StringComparison.Ordinal); +} diff --git a/hosts/Volcanic.AgenticCore.Telegram/Speech/SpeechOptions.cs b/hosts/Volcanic.AgenticCore.Telegram/Speech/SpeechOptions.cs new file mode 100644 index 0000000..7edd194 --- /dev/null +++ b/hosts/Volcanic.AgenticCore.Telegram/Speech/SpeechOptions.cs @@ -0,0 +1,27 @@ +namespace Volcanic.AgenticCore.Telegram.Speech; + +/// The Telegram:Speech section. Speech to text is on exactly when is set. +public sealed class SpeechOptions +{ + public const string DefaultModel = "gpt-4o-transcribe"; + public const string DefaultBaseUrl = "https://api.openai.com/v1"; + + /// Falls back to OPENAI_API_KEY. Empty turns speech to text off. + public string ApiKey { get; set; } = ""; + + /// Falls back to STT_MODEL, then . + public string Model { get; set; } = ""; + + /// Any OpenAI-compatible transcription endpoint. + public string BaseUrl { get; set; } = DefaultBaseUrl; + + /// Optional ISO-639-1 hint. + public string? Language { get; set; } + + public bool IsEnabled => !string.IsNullOrWhiteSpace(ApiKey); + + public string ResolvedModel => string.IsNullOrWhiteSpace(Model) ? DefaultModel : Model.Trim(); + + /// The startup log line (never the key). + public string Describe() => IsEnabled ? $"Speech to text: {ResolvedModel}" : "Speech to text: off (no OPENAI_API_KEY)"; +} diff --git a/hosts/Volcanic.AgenticCore.Telegram/TelegramOptions.cs b/hosts/Volcanic.AgenticCore.Telegram/TelegramOptions.cs index ad7b6e9..5b39b6d 100644 --- a/hosts/Volcanic.AgenticCore.Telegram/TelegramOptions.cs +++ b/hosts/Volcanic.AgenticCore.Telegram/TelegramOptions.cs @@ -1,3 +1,5 @@ +using Volcanic.AgenticCore.Telegram.Speech; + namespace Volcanic.AgenticCore.Telegram; /// The Telegram configuration section. @@ -36,6 +38,12 @@ public sealed class TelegramOptions public TimeSpan ReconnectMaxDelay { get; set; } = TimeSpan.FromSeconds(60); + /// Image documents up to this size are also shown to the model inline (base64 stays under providers' 5 MB cap). + public long InlineImageMaxBytes { get; set; } = 3_750_000; + + /// Speech to text for voice notes and audio (Telegram:Speech). + public SpeechOptions Speech { get; set; } = new(); + /// Reads the environment-variable shortcuts the README documents, when the section leaves them empty. public void ApplyEnvironment(Func getEnv) { @@ -45,6 +53,13 @@ public sealed class TelegramOptions foreach (var part in ids.Split([',', ';', ' '], StringSplitOptions.RemoveEmptyEntries)) if (long.TryParse(part, out var id)) AllowedUserIds.Add(id); } + + // Speech: the environment fills only empty settings, and an empty variable counts as unset. + Speech ??= new SpeechOptions(); + if (string.IsNullOrWhiteSpace(Speech.ApiKey) && getEnv("OPENAI_API_KEY") is { } key && !string.IsNullOrWhiteSpace(key)) + Speech.ApiKey = key.Trim(); + if (string.IsNullOrWhiteSpace(Speech.Model) && getEnv("STT_MODEL") is { } model && !string.IsNullOrWhiteSpace(model)) + Speech.Model = model.Trim(); } /// Configuration problems that must stop the host, empty when valid. @@ -62,6 +77,10 @@ public sealed class TelegramOptions if (ReconnectInitialDelay <= TimeSpan.Zero || ReconnectMaxDelay < ReconnectInitialDelay) errors.Add("Telegram reconnect delays must be positive with max >= initial."); if (InputDebounce < TimeSpan.Zero) errors.Add("Telegram:InputDebounce must not be negative."); + if (InlineImageMaxBytes <= 0) errors.Add("Telegram:InlineImageMaxBytes must be positive."); + // On Linux "/v1" parses as an absolute file:// URI; the scheme check rejects it. + if (!Uri.TryCreate(Speech?.BaseUrl, UriKind.Absolute, out var sttUri) || sttUri.Scheme is not ("http" or "https")) + errors.Add("Telegram:Speech:BaseUrl must be an absolute http(s) URL."); return errors; } diff --git a/hosts/Volcanic.AgenticCore.Telegram/TelegramProcessEnvironment.cs b/hosts/Volcanic.AgenticCore.Telegram/TelegramProcessEnvironment.cs index 5f2f0c1..39cad52 100644 --- a/hosts/Volcanic.AgenticCore.Telegram/TelegramProcessEnvironment.cs +++ b/hosts/Volcanic.AgenticCore.Telegram/TelegramProcessEnvironment.cs @@ -8,7 +8,7 @@ public static class TelegramProcessEnvironment { public const string AllowVariable = "PROCESS_ENV_ALLOW"; - /// Adds the section's lists, PROCESS_ENV_ALLOW, the bot token and the Google client secret to . + /// Adds the section's lists, PROCESS_ENV_ALLOW, the bot token, the speech API key and the Google client secret to . public static void Configure(ProcessEnvironmentOptions target, TelegramOptions telegram, ProcessEnvironmentSection section, Func getEnv, GoogleCalendarOptions? calendar = null) { @@ -22,6 +22,7 @@ public static class TelegramProcessEnvironment target.Allow.AddRange(allow.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); target.HardenHostProcess = section.HardenHostProcess; if (!string.IsNullOrWhiteSpace(telegram.BotToken)) target.SecretValues.Add(telegram.BotToken); + if (!string.IsNullOrWhiteSpace(telegram.Speech?.ApiKey)) target.SecretValues.Add(telegram.Speech.ApiKey); if (!string.IsNullOrWhiteSpace(calendar?.ClientSecret)) target.SecretValues.Add(calendar.ClientSecret); } diff --git a/hosts/Volcanic.AgenticCore.Telegram/Volcanic.AgenticCore.Telegram.csproj b/hosts/Volcanic.AgenticCore.Telegram/Volcanic.AgenticCore.Telegram.csproj index 5bf1fae..ff3d67d 100644 --- a/hosts/Volcanic.AgenticCore.Telegram/Volcanic.AgenticCore.Telegram.csproj +++ b/hosts/Volcanic.AgenticCore.Telegram/Volcanic.AgenticCore.Telegram.csproj @@ -9,6 +9,7 @@ + diff --git a/tests/Volcanic.AgenticCore.Telegram.Tests/FormattingAndOptionsTests.cs b/tests/Volcanic.AgenticCore.Telegram.Tests/FormattingAndOptionsTests.cs index 14dcd1d..6c538c7 100644 --- a/tests/Volcanic.AgenticCore.Telegram.Tests/FormattingAndOptionsTests.cs +++ b/tests/Volcanic.AgenticCore.Telegram.Tests/FormattingAndOptionsTests.cs @@ -1,5 +1,6 @@ using Volcanic.AgenticCore.Telegram.Chats; using Volcanic.AgenticCore.Telegram.Rendering; +using Volcanic.AgenticCore.Telegram.Speech; namespace Volcanic.AgenticCore.Telegram.Tests; @@ -98,4 +99,65 @@ public sealed class FormattingAndOptionsTests Assert.False(options.IsAllowed(4)); Assert.False(options.IsAllowed(null)); } + + private static TelegramOptions Valid() => new() { BotToken = "t", AllowedUserIds = [1] }; + + [Fact] + public void Speech_falls_back_to_OPENAI_API_KEY_and_STT_MODEL() + { + var options = Valid(); + options.ApplyEnvironment(name => name switch { "OPENAI_API_KEY" => "sk-env", "STT_MODEL" => "whisper-1", _ => null }); + Assert.Equal("sk-env", options.Speech.ApiKey); + Assert.Equal("whisper-1", options.Speech.ResolvedModel); + Assert.True(options.Speech.IsEnabled); + Assert.Equal("Speech to text: whisper-1", options.Speech.Describe()); + } + + [Fact] + public void Empty_speech_variables_count_as_unset() + { + var options = Valid(); + options.ApplyEnvironment(name => name is "OPENAI_API_KEY" or "STT_MODEL" ? "" : null); + Assert.False(options.Speech.IsEnabled); + Assert.Equal(SpeechOptions.DefaultModel, options.Speech.ResolvedModel); + Assert.Equal("Speech to text: off (no OPENAI_API_KEY)", options.Speech.Describe()); + } + + [Fact] + public void Configured_speech_settings_win_over_the_environment() + { + var options = Valid(); + options.Speech.ApiKey = "sk-config"; + options.Speech.Model = "gpt-4o-mini-transcribe"; + options.ApplyEnvironment(name => name switch { "OPENAI_API_KEY" => "sk-env", "STT_MODEL" => "whisper-1", _ => null }); + Assert.Equal(("sk-config", "gpt-4o-mini-transcribe"), (options.Speech.ApiKey, options.Speech.ResolvedModel)); + } + + [Fact] + public void Speech_defaults_to_gpt_4o_transcribe_on_the_openai_endpoint() + { + var speech = new SpeechOptions(); + Assert.Equal(("gpt-4o-transcribe", "https://api.openai.com/v1"), (speech.ResolvedModel, speech.BaseUrl)); + Assert.False(speech.IsEnabled); + } + + [Fact] + public void Inline_image_limit_defaults_to_3_75_MB_and_must_be_positive() + { + Assert.Equal(3_750_000, Valid().InlineImageMaxBytes); + var options = Valid(); + options.InlineImageMaxBytes = 0; + Assert.Contains("Telegram:InlineImageMaxBytes must be positive.", options.Validate()); + } + + [Theory] + [InlineData("ftp://stt.example/v1")] + [InlineData("/v1")] + [InlineData("")] + public void Speech_base_url_must_be_an_absolute_http_url(string url) + { + var options = Valid(); + options.Speech.BaseUrl = url; + Assert.Contains("Telegram:Speech:BaseUrl must be an absolute http(s) URL.", options.Validate()); + } } diff --git a/tests/Volcanic.AgenticCore.Telegram.Tests/OpenAiSpeechToTextTests.cs b/tests/Volcanic.AgenticCore.Telegram.Tests/OpenAiSpeechToTextTests.cs new file mode 100644 index 0000000..9102634 --- /dev/null +++ b/tests/Volcanic.AgenticCore.Telegram.Tests/OpenAiSpeechToTextTests.cs @@ -0,0 +1,95 @@ +using System.Net; +using System.Text; +using Volcanic.AgenticCore.Telegram.Speech; + +namespace Volcanic.AgenticCore.Telegram.Tests; + +public sealed class OpenAiSpeechToTextTests +{ + private const string Key = "sk-test-secret-key-123"; + + private sealed record Part(string? FileName, string? ContentType, string Text); + + private sealed class StubHandler(HttpStatusCode status, string body, string mediaType = "application/json") : HttpMessageHandler + { + public HttpRequestMessage? Request { get; private set; } + public Dictionary Parts { get; } = []; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Request = request; + foreach (var part in (MultipartFormDataContent)request.Content!) + { + var disposition = part.Headers.ContentDisposition!; + Parts[disposition.Name!.Trim('"')] = new Part(disposition.FileName?.Trim('"'), part.Headers.ContentType?.MediaType, + await part.ReadAsStringAsync(cancellationToken)); + } + return new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) }; + } + } + + private static (OpenAiSpeechToText Stt, StubHandler Handler) Create(HttpStatusCode status, string body, string? language = null) + { + var handler = new StubHandler(status, body); + var options = new SpeechOptions { ApiKey = Key, Language = language, BaseUrl = "https://stt.example/v1/" }; + return (new OpenAiSpeechToText(new HttpClient(handler), options), handler); + } + + [Fact] + public async Task Posts_a_multipart_form_with_file_model_and_response_format() + { + var (stt, handler) = Create(HttpStatusCode.OK, "{\"text\":\"hello\"}"); + await stt.TranscribeAsync(new byte[] { 1, 2, 3 }, "voice.ogg", "audio/ogg", CancellationToken.None); + + Assert.Equal(HttpMethod.Post, handler.Request!.Method); + Assert.Equal("https://stt.example/v1/audio/transcriptions", handler.Request.RequestUri!.ToString()); + Assert.Equal("Bearer", handler.Request.Headers.Authorization!.Scheme); + Assert.Equal(Key, handler.Request.Headers.Authorization.Parameter); + Assert.Equal(("voice.ogg", "audio/ogg"), (handler.Parts["file"].FileName, handler.Parts["file"].ContentType)); + Assert.Equal("gpt-4o-transcribe", handler.Parts["model"].Text); + Assert.Equal("json", handler.Parts["response_format"].Text); + Assert.False(handler.Parts.ContainsKey("language")); + } + + [Fact] + public async Task Language_is_sent_only_when_configured() + { + var (stt, handler) = Create(HttpStatusCode.OK, "{\"text\":\"hallo\"}", language: "de"); + await stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None); + Assert.Equal("de", handler.Parts["language"].Text); + } + + [Fact] + public async Task The_transcript_is_trimmed() + { + var (stt, _) = Create(HttpStatusCode.OK, "{\"text\":\" hello there \\n\"}"); + Assert.Equal("hello there", await stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None)); + } + + [Fact] + public async Task A_non_success_status_names_the_status_and_hides_the_key() + { + var body = "{\"error\":{\"message\":\"Incorrect API key provided: " + Key + "\"}}" + new string('x', 300); + var (stt, _) = Create(HttpStatusCode.Unauthorized, body); + var ex = await Assert.ThrowsAsync(() => stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None)); + Assert.StartsWith("OpenAI transcription failed: HTTP 401 ", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(Key, ex.Message, StringComparison.Ordinal); + Assert.True(ex.Message.Length <= "OpenAI transcription failed: HTTP 401 ".Length + 200); + } + + [Fact] + public async Task A_non_json_body_has_its_own_message() + { + var (stt, _) = Create(HttpStatusCode.OK, "gateway"); + var ex = await Assert.ThrowsAsync(() => stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None)); + Assert.Equal("OpenAI transcription failed: the response is not JSON.", ex.Message); + } + + [Fact] + public async Task A_missing_text_has_its_own_message() + { + var (stt, _) = Create(HttpStatusCode.OK, "{\"segments\":[]}"); + var ex = await Assert.ThrowsAsync(() => stt.TranscribeAsync(new byte[] { 1 }, "voice.ogg", "audio/ogg", CancellationToken.None)); + Assert.Equal("OpenAI transcription failed: the response has no text.", ex.Message); + } +} diff --git a/tests/Volcanic.AgenticCore.Telegram.Tests/ProcessEnvironmentTests.cs b/tests/Volcanic.AgenticCore.Telegram.Tests/ProcessEnvironmentTests.cs index d3d66a3..11bdc52 100644 --- a/tests/Volcanic.AgenticCore.Telegram.Tests/ProcessEnvironmentTests.cs +++ b/tests/Volcanic.AgenticCore.Telegram.Tests/ProcessEnvironmentTests.cs @@ -32,6 +32,16 @@ public sealed class ProcessEnvironmentTests Assert.Empty(target.Validate()); } + [Fact] + public void Configure_adds_the_speech_api_key_to_the_secret_values() + { + var target = new ProcessEnvironmentOptions(); + var telegram = new TelegramOptions { BotToken = Token }; + telegram.Speech.ApiKey = "sk-speech-secret"; + TelegramProcessEnvironment.Configure(target, telegram, new ProcessEnvironmentSection(), _ => null); + Assert.Contains("sk-speech-secret", target.SecretValues); + } + [Fact] public void The_section_binds_from_configuration() {