admin/Playa
1
0
Fork 0
forked from admin/VolcanicAgenticCore
Code Pull Requests Activity

feat(telegram): transcribe voice notes and audio with an echo

Browse Source 
Content step 3 (InboundMedia.AddSpeechAsync): voice and audio go through ISpeechToText, the transcript is echoed as '🎤 …' and prompted as '[Voice message] …'. Spec §4 edited: audio is transcribed from the file step 2 saved (one download); an oversize or undownloadable voice note sends its notice and isn't prompted.
This commit is contained in:
EugeneTes
2026-09-26 22:13:10 +02:00
parent a0cd2975b8
commit f90731baf0
6 changed files with 334 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs/superpowers/specs/2026-09-26-telegram-attachments-design.md
View File

@@ -227,9 +227,12 @@ Router changes (`UpdateRouter.OnMessageAsync`), on top of the message-context sp
1. No `ISpeechToText` configured → notice `Voice messages need speech-to-text: set OPENAI_API_KEY (or
Telegram:Speech:ApiKey) on the bot host.` `Audio` is still saved to the inbox as a file (below); `Voice` isn't
prompted unless it has a caption.
2. Over 20 MB → the file-size notice, as for files.
3. Download into memory. Voice arrives as `.oga`, which OpenAI rejects, so it's renamed to `voice.ogg`
(`audio/ogg`). Audio keeps `FileName ?? "audio"` and its `MimeType`.
2. Over 20 MB → the file-size notice, as for files, and the message isn't prompted (for `Audio`, step 2 has already
refused it).
3. Voice is downloaded into memory; a failed download sends `I could not download that file.` and the message isn't
prompted. Audio is read back from the file step 2 saved, so it's downloaded only once. Voice arrives as `.oga`,
which OpenAI rejects, so it's renamed to `voice.ogg` (`audio/ogg`). Audio keeps `FileName ?? "audio"` and its
`MimeType`.
4. `Transcript = await stt.TranscribeAsync(bytes, fileName, mime, ct)`, trimmed. Empty → notice `I couldn't make
out any words in that voice message.` An exception → notice `Could not transcribe the voice message: <reason>`.
5. Echo `🎤 <transcript>` as plain text, cut to 4096 characters with `…`.

2
hosts/Volcanic.AgenticCore.Telegram/Inbound/InboundComposer.cs
View File

@@ -56,7 +56,7 @@ public sealed class InboundComposer(TelegramSender sender, InboundMedia media, I
// Content steps, in order. Each may add body lines and images, send a notice, or refuse the message.
await AddTextAndPhotoAsync(chat, message, draft, cancellationToken).ConfigureAwait(false); // step 1
if (!draft.Refused) await media.AddFilesAsync(chat, message, draft, cancellationToken).ConfigureAwait(false); // step 2
// Step 3 (voice and audio transcription) goes here.
if (!draft.Refused) await media.AddSpeechAsync(chat, message, draft, cancellationToken).ConfigureAwait(false); // step 3
if (draft.Refused || (draft.Body.Count == 0 && draft.Images.Count == 0)) return null;
return new InboundPart(

81
hosts/Volcanic.AgenticCore.Telegram/Inbound/InboundMedia.cs
View File

@@ -4,16 +4,25 @@ using Telegram.Bot.Exceptions;
using Telegram.Bot.Types;
using Volcanic.AgenticCore.Telegram.Chats;
using Volcanic.AgenticCore.Telegram.Rendering;
using Volcanic.AgenticCore.Telegram.Speech;
using ImageContent = Volcanic.AgenticCore.Abstractions.Messages.ImageContent;
namespace Volcanic.AgenticCore.Telegram.Inbound;
/// <summary>
/// Content steps for attachments (spec: telegram-attachments §4): step 2 saves documents, videos and audio to the
/// workspace inbox. Notices go through the chat renderer (<see cref="ChatRenderer.SayAsync"/>) so they stay in order.
/// workspace inbox, step 3 transcribes voice notes and audio through <see cref="ISpeechToText"/> (null: speech is off).
/// Notices and the echo go through the chat renderer (<see cref="ChatRenderer.SayAsync"/>) so they stay in order.
/// </summary>
public sealed class InboundMedia(TelegramSender sender, TelegramOptions options, TimeProvider time, ILogger<InboundMedia> logger)
public sealed class InboundMedia(TelegramSender sender, TelegramOptions options, TimeProvider time, ILogger<InboundMedia> logger,
ISpeechToText? speech = null)
{
/// <summary>Sent for a voice note or audio file when no speech-to-text service is configured.</summary>
public const string SpeechNotConfiguredNotice = "Voice messages need speech-to-text: set OPENAI_API_KEY (or Telegram:Speech:ApiKey) on the bot host.";
/// <summary>Sent when the transcript is empty.</summary>
public const string NoWordsNotice = "I couldn't make out any words in that voice message.";
/// <summary>Sent when a file's download fails (the partial file is deleted and the message isn't prompted).</summary>
public const string DownloadFailedNotice = "I could not download that file.";
@@ -90,6 +99,74 @@ public sealed class InboundMedia(TelegramSender sender, TelegramOptions options,
draft.Body.Add($"[File received: {InboxFiles.Relative(workspace, path)} ({mime}, {TelegramLimits.FormatSize(bytes)}){note}]");
}
/// <summary>
/// Step 3: a voice note or audio file is transcribed, echoed as <c>🎤 …</c> and added as <c>[Voice message] …</c>.
/// Audio is read back from the file step 2 saved (one download). Without a transcript nothing is added (the caption,
/// if any, is still prompted); an oversize or undownloadable voice note refuses the message.
/// </summary>
internal async Task AddSpeechAsync(ChatContext chat, Message message, InboundDraft draft, CancellationToken cancellationToken)
{
var (fileId, size, name, mime, isVoice) = message switch
{
{ Voice: { } v } => (v.FileId, v.FileSize, "voice.ogg", "audio/ogg", true), // Telegram's .oga is rejected by OpenAI
{ Audio: { } a } => (a.FileId, a.FileSize, a.FileName ?? "audio", a.MimeType ?? "audio/mpeg", false),
_ => ((string?)null, (long?)null, "", "", false),
};
if (fileId is null) return;
if (speech is null)
{
await chat.Renderer.SayAsync(SpeechNotConfiguredNotice).ConfigureAwait(false);
return;
}
if (isVoice && size > TelegramLimits.MaxDownloadBytes)
{
await chat.Renderer.SayAsync(TooBigNotice(size)).ConfigureAwait(false);
draft.Refused = true;
return;
}
byte[] audio;
if (draft.SavedPath is { } saved) // audio: step 2 saved it
{
audio = await File.ReadAllBytesAsync(saved, cancellationToken).ConfigureAwait(false);
}
else
{
using var buffer = new MemoryStream();
try
{
await DownloadAsync(fileId, buffer, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Downloading a voice message in chat {ChatId} failed", chat.ChatId);
await chat.Renderer.SayAsync(ex is ApiRequestException api && IsTooBig(api) ? TooBigNotice(size) : DownloadFailedNotice).ConfigureAwait(false);
draft.Refused = true;
return;
}
audio = buffer.ToArray();
}
string transcript;
try
{
transcript = (await speech.TranscribeAsync(audio, name, mime, cancellationToken).ConfigureAwait(false)).Trim();
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Transcribing a voice message in chat {ChatId} failed", chat.ChatId);
await chat.Renderer.SayAsync($"Could not transcribe the voice message: {ex.Message}").ConfigureAwait(false);
return;
}
if (transcript.Length == 0)
{
await chat.Renderer.SayAsync(NoWordsNotice).ConfigureAwait(false);
return;
}
await chat.Renderer.SayAsync("🎤 " + transcript).ConfigureAwait(false); // plain text, cut to 4096 by the sender
draft.Body.Add($"[Voice message] {transcript}");
}
private void TryDelete(string path)
{
try { File.Delete(path); }

5
tests/Volcanic.AgenticCore.Telegram.Tests/Support/ChatHost.cs
View File

@@ -29,7 +29,8 @@ public sealed class ChatHost : IAsyncDisposable
private readonly string _root;
public ChatHost(Action<ChatHost, IServiceCollection>? configure = null, Action<AgenticCoreOptions>? core = null, TimeProvider? time = null,
Volcanic.AgenticCore.Extensions.GoogleCalendar.GoogleCalendarOptions? calendar = null)
Volcanic.AgenticCore.Extensions.GoogleCalendar.GoogleCalendarOptions? calendar = null,
Volcanic.AgenticCore.Telegram.Speech.ISpeechToText? speech = null)
{
_root = Path.Combine(Path.GetTempPath(), "volcanic-telegram", Guid.NewGuid().ToString("N"));
Options = new TelegramOptions
@@ -59,7 +60,7 @@ public sealed class ChatHost : IAsyncDisposable
ILoggerFactory loggers = NullLoggerFactory.Instance;
var sender = new TelegramSender(Bot, NullLogger<TelegramSender>.Instance);
Chats = new ChatRegistry(_services.GetRequiredService<IAgentSessionFactory>(), sender, Options, loggers, time, calendar);
var media = new InboundMedia(sender, Options, time ?? TimeProvider.System, NullLogger<InboundMedia>.Instance);
var media = new InboundMedia(sender, Options, time ?? TimeProvider.System, NullLogger<InboundMedia>.Instance, speech);
Composer = new InboundComposer(sender, media, NullLogger<InboundComposer>.Instance);
Router = new UpdateRouter(Options, Chats, new ChatCommands(sender), sender, Composer, NullLogger<UpdateRouter>.Instance);
}

22
tests/Volcanic.AgenticCore.Telegram.Tests/Support/FakeSpeechToText.cs Normal file
View File

@@ -0,0 +1,22 @@
using Volcanic.AgenticCore.Telegram.Speech;
namespace Volcanic.AgenticCore.Telegram.Tests.Support;
/// <summary>Scripted transcripts. <see cref="Gate"/> holds a call until released; <see cref="Failure"/> makes it throw.</summary>
public sealed class FakeSpeechToText(string transcript = "hello there") : ISpeechToText
{
public string Transcript { get; set; } = transcript;
public Exception? Failure { get; set; }
public TimeSpan Delay { get; set; }
public TaskCompletionSource? Gate { get; set; }
public List<(string FileName, string MimeType, byte[] Audio)> Calls { get; } = [];
public async Task<string> TranscribeAsync(ReadOnlyMemory<byte> audio, string fileName, string mimeType, CancellationToken cancellationToken)
{
lock (Calls) Calls.Add((fileName, mimeType, audio.ToArray()));
if (Gate is not null) await Gate.Task.WaitAsync(cancellationToken);
if (Delay > TimeSpan.Zero) await Task.Delay(Delay, cancellationToken);
if (Failure is not null) throw Failure;
return Transcript;
}
}

223
tests/Volcanic.AgenticCore.Telegram.Tests/VoiceChatTests.cs Normal file
View File

@@ -0,0 +1,223 @@
using Microsoft.Extensions.Time.Testing;
using Telegram.Bot.Requests;
using Volcanic.AgenticCore.Telegram.Speech;
using Volcanic.AgenticCore.Telegram.Tests.Support;
using Volcanic.AgenticCore.Testing;
namespace Volcanic.AgenticCore.Telegram.Tests;
public sealed class VoiceChatTests
{
private const string NotConfigured = "Voice messages need speech-to-text: set OPENAI_API_KEY (or Telegram:Speech:ApiKey) on the bot host.";
private static string Prompt(ChatHost host) => host.Faux.Requests.Single().Messages.Last().Text;
[Fact]
public async Task A_voice_note_is_echoed_before_the_answer_and_prompted()
{
var stt = new FakeSpeechToText("remind me to call Ada");
await using var host = new ChatHost(speech: stt);
host.Bot.Files["voice"] = [1, 2, 3];
host.Faux.Enqueue(FauxTurn.Say("Will do."));
await host.RouteAsync(Updates.VoiceMessage());
await host.SettleAsync();
var texts = host.Texts.ToList();
Assert.InRange(texts.IndexOf("🎤 remind me to call Ada"), 0, texts.IndexOf("Will do.") - 1);
Assert.EndsWith("\n[Voice message] remind me to call Ada", Prompt(host), StringComparison.Ordinal);
var call = Assert.Single(stt.Calls);
Assert.Equal(("voice.ogg", "audio/ogg"), (call.FileName, call.MimeType));
Assert.Equal(new byte[] { 1, 2, 3 }, call.Audio);
}
[Fact]
public async Task A_long_transcript_echo_is_cut_to_4096_characters()
{
var stt = new FakeSpeechToText(new string('a', 5000));
await using var host = new ChatHost(speech: stt);
host.Bot.Files["voice"] = [1];
host.Faux.Enqueue(FauxTurn.Say("ok"));
await host.RouteAsync(Updates.VoiceMessage());
await host.SettleAsync();
var echo = host.Bot.Sent.Single(s => s.Text.StartsWith("🎤", StringComparison.Ordinal)).Text;
Assert.Equal(4096, echo.Length);
Assert.EndsWith("…", echo, StringComparison.Ordinal);
Assert.Contains(new string('a', 5000), Prompt(host), StringComparison.Ordinal); // the model gets all of it
}
[Fact]
public async Task Without_speech_to_text_a_voice_note_gets_the_setup_notice_and_no_run()
{
await using var host = new ChatHost();
host.Bot.Files["voice"] = [1];
await host.RouteAsync(Updates.VoiceMessage());
await host.SettleAsync();
Assert.Contains(NotConfigured, host.Texts);
Assert.Empty(host.Faux.Requests);
Assert.Empty(host.Bot.Requests.OfType<GetFileRequest>());
}
[Fact]
public async Task Without_speech_to_text_a_captioned_voice_note_still_prompts_the_caption()
{
await using var host = new ChatHost();
host.Faux.Enqueue(FauxTurn.Say("ok"));
await host.RouteAsync(Updates.VoiceMessage(caption: "listen to this"));
await host.SettleAsync();
Assert.Contains(NotConfigured, host.Texts);
Assert.EndsWith("\nlisten to this", Prompt(host), StringComparison.Ordinal);
}
[Fact]
public async Task A_transcription_failure_gets_a_notice_and_no_run()
{
var stt = new FakeSpeechToText { Failure = new SpeechToTextException("OpenAI transcription failed: HTTP 500 oops") };
await using var host = new ChatHost(speech: stt);
host.Bot.Files["voice"] = [1];
await host.RouteAsync(Updates.VoiceMessage());
await host.SettleAsync();
Assert.Contains("Could not transcribe the voice message: OpenAI transcription failed: HTTP 500 oops", host.Texts);
Assert.Empty(host.Faux.Requests);
}
[Fact]
public async Task An_empty_transcript_gets_a_notice_and_no_run()
{
await using var host = new ChatHost(speech: new FakeSpeechToText(" "));
host.Bot.Files["voice"] = [1];
await host.RouteAsync(Updates.VoiceMessage());
await host.SettleAsync();
Assert.Contains("I couldn't make out any words in that voice message.", host.Texts);
Assert.Empty(host.Faux.Requests);
}
[Fact]
public async Task A_voice_note_over_20_MB_gets_the_size_notice_and_no_download()
{
var stt = new FakeSpeechToText();
await using var host = new ChatHost(speech: stt);
await host.RouteAsync(Updates.VoiceMessage(size: 21L * 1024 * 1024, caption: "about this"));
await host.SettleAsync();
Assert.Contains("That file is 21.0 MB; Telegram lets bots download at most 20 MB. Share a link or a smaller file.", host.Texts);
Assert.Empty(host.Faux.Requests);
Assert.Empty(host.Bot.Requests.OfType<GetFileRequest>());
Assert.Empty(stt.Calls);
}
[Fact]
public async Task A_failed_voice_download_gets_a_notice_and_no_run()
{
var stt = new FakeSpeechToText();
await using var host = new ChatHost(speech: stt);
host.Bot.Files["voice"] = [1];
host.Bot.FailDownloads = true;
await host.RouteAsync(Updates.VoiceMessage(caption: "about this"));
await host.SettleAsync();
Assert.Contains("I could not download that file.", host.Texts);
Assert.Empty(host.Faux.Requests);
Assert.Empty(stt.Calls);
}
[Fact]
public async Task An_audio_file_is_saved_and_transcribed()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 26, 10, 0, 0, TimeSpan.Zero));
var stt = new FakeSpeechToText("welcome to the show");
await using var host = new ChatHost(time: time, speech: stt);
host.Bot.Files["audio"] = [4, 5, 6];
host.Faux.Enqueue(FauxTurn.Say("ok"));
await host.RouteAsync(Updates.AudioMessage("talk.mp3", "audio/mpeg", 3));
await MessageContextChatTests.FlushAsync(host, time, 1);
await host.SettleAsync();
Assert.EndsWith("\n[File received: inbox/2026-09-26-talk.mp3 (audio/mpeg, 3 bytes)]\n\n[Voice message] welcome to the show", Prompt(host), StringComparison.Ordinal);
Assert.Equal(("talk.mp3", "audio/mpeg"), (stt.Calls.Single().FileName, stt.Calls.Single().MimeType));
Assert.Equal(new byte[] { 4, 5, 6 }, stt.Calls.Single().Audio);
Assert.True(File.Exists(Path.Combine(host.WorkspaceOf(), "inbox", "2026-09-26-talk.mp3")));
Assert.Single(host.Bot.Requests.OfType<GetFileRequest>()); // downloaded once, transcribed from the saved file
}
[Fact]
public async Task Without_speech_to_text_an_audio_file_is_still_saved_and_prompted()
{
var time = new FakeTimeProvider(new DateTimeOffset(2026, 9, 26, 10, 0, 0, TimeSpan.Zero));
await using var host = new ChatHost(time: time);
host.Bot.Files["audio"] = [4, 5, 6];
host.Faux.Enqueue(FauxTurn.Say("ok"));
await host.RouteAsync(Updates.AudioMessage("talk.mp3", "audio/mpeg", 3));
await MessageContextChatTests.FlushAsync(host, time, 1);
await host.SettleAsync();
Assert.Contains(NotConfigured, host.Texts);
Assert.EndsWith("[File received: inbox/2026-09-26-talk.mp3 (audio/mpeg, 3 bytes)]", Prompt(host), StringComparison.Ordinal);
}
[Fact]
public async Task A_voice_note_and_a_text_right_after_it_are_one_prompt_in_order()
{
var stt = new FakeSpeechToText("first the voice") { Delay = TimeSpan.FromMilliseconds(300) }; // well past the 20 ms window
await using var host = new ChatHost(speech: stt);
host.Bot.Files["voice"] = [1];
host.Faux.Enqueue(FauxTurn.Say("ok"));
await host.RouteAsync(Updates.VoiceMessage());
await host.SendAsync("then the text");
await host.SettleAsync();
var prompt = Prompt(host);
Assert.InRange(prompt.IndexOf("[Voice message] first the voice", StringComparison.Ordinal), 0, prompt.IndexOf("then the text", StringComparison.Ordinal) - 1);
}
[Fact]
public async Task Stop_is_handled_during_a_slow_transcription()
{
var stt = new FakeSpeechToText("late") { Gate = new TaskCompletionSource() };
await using var host = new ChatHost(speech: stt);
host.Bot.Files["voice"] = [1];
host.Faux.Enqueue(FauxTurn.Say("too late").WithDelay(TimeSpan.FromSeconds(30)), FauxTurn.Say("Heard you."));
await host.SendAsync("take your time");
var chat = await host.ChatAsync();
await Wait.UntilAsync(() => chat.Session.IsStreaming, "the run to start");
await host.RouteAsync(Updates.VoiceMessage()).WaitAsync(TimeSpan.FromSeconds(2)); // returns while transcription hangs
await Wait.UntilAsync(() => stt.Calls.Count == 1, "the transcription to start");
await host.SendAsync("/stop").WaitAsync(TimeSpan.FromSeconds(2));
await Wait.UntilAsync(() => !chat.Session.IsStreaming, "the run to stop");
Assert.False(stt.Gate.Task.IsCompleted);
stt.Gate.SetResult();
await host.SettleAsync();
Assert.Contains("⏹ Stopped.", host.Texts);
}
[Fact]
public async Task A_voice_note_does_not_answer_a_pending_question()
{
await using var host = new ChatHost(speech: new FakeSpeechToText("Ada Lovelace"));
host.Bot.Files["voice"] = [1];
host.Faux.Enqueue(FauxTurn.Say("ok"));
var chat = await host.ChatAsync();
var answer = chat.Ui.InputAsync("Your name?", "name", null, CancellationToken.None);
await Wait.UntilAsync(() => chat.Ui.IsWaitingForInput, "the question");
await host.RouteAsync(Updates.VoiceMessage());
await host.SettleAsync();
Assert.True(chat.Ui.IsWaitingForInput);
Assert.False(answer.IsCompleted);
Assert.EndsWith("[Voice message] Ada Lovelace", Prompt(host), StringComparison.Ordinal);
await host.SendAsync("Ada");
Assert.Equal("Ada", await answer.WaitAsync(TimeSpan.FromSeconds(5)));
}
}