feat(telegram): voice tool sends spoken replies as Telegram voice notes
OpenAI gpt-4o-mini-tts (female voice nova by default; TTS_MODEL/TTS_VOICE) renders OGG/Opus, sent with sendVoice in order with rendered output. Up to 4500 characters per call, split into 1500-character parts at paragraph/sentence/word boundaries; all parts are synthesized before any is sent, and API errors come back to the agent as text with the key hidden. On whenever OPENAI_API_KEY is set.
This commit is contained in:
@@ -116,7 +116,8 @@ limited by `AllowedUserIds`. Incoming messages go through `Inbound/`: the router
|
|||||||
message in update order, `InboundComposer` builds its part (reply/forward headers from `MessageContext`, text and
|
message in update order, `InboundComposer` builds its part (reply/forward headers from `MessageContext`, text and
|
||||||
photo) in the background, and the inbox flushes a batch `InputDebounce` (600 ms) after the last message as one prompt
|
photo) in the background, and the inbox flushes a batch `InputDebounce` (600 ms) after the last message as one prompt
|
||||||
with a local-time header. Files, audio and voice are saved to `<workspace>/inbox/` (voice transcribed by
|
with a local-time header. Files, audio and voice are saved to `<workspace>/inbox/` (voice transcribed by
|
||||||
`Speech/OpenAiSpeechToText` when `OPENAI_API_KEY` is set) through `InboundComposer`'s content steps
|
`Speech/OpenAiSpeechToText` when `OPENAI_API_KEY` is set; the same key gives each chat the `voice` tool,
|
||||||
|
`Chats/VoiceExtension` over `Speech/OpenAiTextToSpeech`, which sends OGG/Opus voice notes) through `InboundComposer`'s content steps
|
||||||
(`Inbound/InboundMedia`), and `Chats/TelegramChatChannel` implements `IChatChannel` for the core `send` tool through
|
(`Inbound/InboundMedia`), and `Chats/TelegramChatChannel` implements `IChatChannel` for the core `send` tool through
|
||||||
`ChatRenderer.RunInOrderAsync`. The first allowed id is the "owner": `OwnerLoginChannel` sends Claude Code re-login
|
`ChatRenderer.RunInOrderAsync`. The first allowed id is the "owner": `OwnerLoginChannel` sends Claude Code re-login
|
||||||
prompts to that chat. It resolves lazily to break the DI cycle ChatRegistry → session factory → extension → ChatRegistry.
|
prompts to that chat. It resolves lazily to break the DI cycle ChatRegistry → session factory → extension → ChatRegistry.
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ services:
|
|||||||
# Optional features; empty means off / default.
|
# Optional features; empty means off / default.
|
||||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-} # speech to text (attachments spec)
|
- OPENAI_API_KEY=${OPENAI_API_KEY:-} # speech to text (attachments spec)
|
||||||
- STT_MODEL=${STT_MODEL:-}
|
- STT_MODEL=${STT_MODEL:-}
|
||||||
|
- TTS_MODEL=${TTS_MODEL:-}
|
||||||
|
- TTS_VOICE=${TTS_VOICE:-}
|
||||||
- OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} # group topic triggers via Jev (group triggers spec)
|
- OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} # group topic triggers via Jev (group triggers spec)
|
||||||
- TRIGGER_MODEL=${TRIGGER_MODEL:-}
|
- TRIGGER_MODEL=${TRIGGER_MODEL:-}
|
||||||
- GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-} # Google Calendar (calendar spec)
|
- GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-} # Google Calendar (calendar spec)
|
||||||
|
|||||||
@@ -74,8 +74,10 @@ The compose file maps them into the container.
|
|||||||
| `TELEGRAM_MODEL` | `claude-code/claude-opus-5-5` | Model every chat uses; compose passes it as `Telegram__Model` | Yes, as `Telegram__Model` |
|
| `TELEGRAM_MODEL` | `claude-code/claude-opus-5-5` | Model every chat uses; compose passes it as `Telegram__Model` | Yes, as `Telegram__Model` |
|
||||||
| `TZ` | `UTC` | The container's time zone | The agent's `bash` gets the chat's zone as `TZ` instead |
|
| `TZ` | `UTC` | The container's time zone | The agent's `bash` gets the chat's zone as `TZ` instead |
|
||||||
| `LOG_LEVEL` | `Information` | Passed as `Logging__LogLevel__Default` | Yes |
|
| `LOG_LEVEL` | `Information` | Passed as `Logging__LogLevel__Default` | Yes |
|
||||||
| `OPENAI_API_KEY` | — | Turns on speech to text for voice notes | No, and redacted |
|
| `OPENAI_API_KEY` | — | Turns on speech to text for voice notes and the `voice` tool | No, and redacted |
|
||||||
| `STT_MODEL` | `gpt-4o-transcribe` | Transcription model | Yes |
|
| `STT_MODEL` | `gpt-4o-transcribe` | Transcription model | Yes |
|
||||||
|
| `TTS_MODEL` | `gpt-4o-mini-tts` | Speech model of the `voice` tool | Yes |
|
||||||
|
| `TTS_VOICE` | `nova` | Default voice of the `voice` tool | Yes |
|
||||||
| `OPENROUTER_API_KEY` | — | Turns on group watching with topic triggers, classified by Jev through OpenRouter ([telegram-bot.md](telegram-bot.md#groups-and-triggers)) | No, and redacted |
|
| `OPENROUTER_API_KEY` | — | Turns on group watching with topic triggers, classified by Jev through OpenRouter ([telegram-bot.md](telegram-bot.md#groups-and-triggers)) | No, and redacted |
|
||||||
| `TRIGGER_MODEL` | `typesafe/jev-1.13` | The Jev model triggers use | Yes |
|
| `TRIGGER_MODEL` | `typesafe/jev-1.13` | The Jev model triggers use | Yes |
|
||||||
| `GOOGLE_CLIENT_ID` | — | With the secret, turns on Google Calendar ([google-calendar.md](google-calendar.md)) | Yes |
|
| `GOOGLE_CLIENT_ID` | — | With the secret, turns on Google Calendar ([google-calendar.md](google-calendar.md)) | Yes |
|
||||||
|
|||||||
@@ -42,9 +42,12 @@ settings use the `GoogleCalendar` section, the scheduler the `Scheduler` section
|
|||||||
| `Scheduler:DefaultScriptTimeout` | `00:05:00` | Watcher timeout when a job sets none (1 s to 1 h) |
|
| `Scheduler:DefaultScriptTimeout` | `00:05:00` | Watcher timeout when a job sets none (1 s to 1 h) |
|
||||||
| `Scheduler:MaxJobsPerTarget` | `50` | Jobs per chat |
|
| `Scheduler:MaxJobsPerTarget` | `50` | Jobs per chat |
|
||||||
| `Scheduler:Shell` | `/bin/sh -c` (`cmd.exe /d /s /c` on Windows) | The watcher shell as a list, command appended; same as `bash` by default |
|
| `Scheduler:Shell` | `/bin/sh -c` (`cmd.exe /d /s /c` on Windows) | The watcher shell as a list, command appended; same as `bash` by default |
|
||||||
| `OPENAI_API_KEY` / `Telegram:Speech:ApiKey` | — | Turns voice transcription on. Hidden from `bash` and redacted from tool output |
|
| `OPENAI_API_KEY` / `Telegram:Speech:ApiKey` | — | Turns voice transcription and the `voice` tool on. Hidden from `bash` and redacted from tool output |
|
||||||
| `STT_MODEL` / `Telegram:Speech:Model` | `gpt-4o-transcribe` | `gpt-4o-mini-transcribe` and `whisper-1` also work |
|
| `STT_MODEL` / `Telegram:Speech:Model` | `gpt-4o-transcribe` | `gpt-4o-mini-transcribe` and `whisper-1` also work |
|
||||||
| `Telegram:Speech:BaseUrl` | `https://api.openai.com/v1` | Any OpenAI-compatible transcription endpoint |
|
| `TTS_MODEL` / `Telegram:Speech:TtsModel` | `gpt-4o-mini-tts` | Speech model of the `voice` tool; `tts-1` and `tts-1-hd` also work (they ignore `instructions`) |
|
||||||
|
| `TTS_VOICE` / `Telegram:Speech:TtsVoice` | `nova` | Default voice of the `voice` tool |
|
||||||
|
| `Telegram:Speech:TtsInstructions` | — | Default delivery instructions (tone, pace) for the `voice` tool |
|
||||||
|
| `Telegram:Speech:BaseUrl` | `https://api.openai.com/v1` | Any OpenAI-compatible audio endpoint (transcription and speech) |
|
||||||
| `Telegram:Speech:Language` | — | Optional ISO-639-1 hint, e.g. `de` |
|
| `Telegram:Speech:Language` | — | Optional ISO-639-1 hint, e.g. `de` |
|
||||||
| `OPENROUTER_API_KEY` / `Telegram:Triggers:ApiKey` | — | Turns on [groups and triggers](#groups-and-triggers). Hidden from `bash` and redacted from tool output |
|
| `OPENROUTER_API_KEY` / `Telegram:Triggers:ApiKey` | — | Turns on [groups and triggers](#groups-and-triggers). Hidden from `bash` and redacted from tool output |
|
||||||
| `TRIGGER_MODEL` / `Telegram:Triggers:Model` | `typesafe/jev-1.13` | The Jev model that classifies group messages |
|
| `TRIGGER_MODEL` / `Telegram:Triggers:Model` | `typesafe/jev-1.13` | The Jev model that classifies group messages |
|
||||||
@@ -154,6 +157,12 @@ question in that chat (for example from an extension) replaces the pending promp
|
|||||||
line). Without an API key the bot replies with how to set one up (`OPENAI_API_KEY`). A transcription is tried up
|
line). Without an API key the bot replies with how to set one up (`OPENAI_API_KEY`). A transcription is tried up
|
||||||
to 3 times (1 s, then 2 s apart) when the request fails on the network, times out, or gets HTTP 408, 429 or 5xx;
|
to 3 times (1 s, then 2 s apart) when the request fails on the network, times out, or gets HTTP 408, 429 or 5xx;
|
||||||
other errors (a bad key, a rejected file) fail at once.
|
other errors (a bad key, a rejected file) fail at once.
|
||||||
|
- **Voice replies.** With an OpenAI key every chat's agent has a `voice` tool (`text`, optional `voice` and
|
||||||
|
`instructions`): it speaks the text with `gpt-4o-mini-tts` (voice `nova` by default) as OGG/Opus and sends it with
|
||||||
|
`sendVoice`, so it shows as a voice message, in order with the other replies. One call takes up to 4500 characters;
|
||||||
|
text over 1500 characters is split at paragraph, sentence or word boundaries into several voice messages. Longer
|
||||||
|
text is refused. Every part is synthesized before the first is sent, so an API error (returned to the agent as text,
|
||||||
|
key hidden) never leaves half a message; synthesis is retried like transcription.
|
||||||
- Telegram lets bots download at most 20 MB. A bigger file is refused with a note, and nothing reaches the agent.
|
- Telegram lets bots download at most 20 MB. A bigger file is refused with a note, and nothing reaches the agent.
|
||||||
- A message with an attachment is never a command and never answers a pending question: its caption is prompt text.
|
- A message with an attachment is never a command and never answers a pending question: its caption is prompt text.
|
||||||
- Anything else (stickers, video notes, locations, polls) gets `I can read text, photos, files and voice messages.`
|
- Anything else (stickers, video notes, locations, polls) gets `I can read text, photos, files and voice messages.`
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ using Volcanic.AgenticCore.Features.Extensions;
|
|||||||
using Volcanic.AgenticCore.Hosting;
|
using Volcanic.AgenticCore.Hosting;
|
||||||
using Volcanic.AgenticCore.Telegram.Inbound;
|
using Volcanic.AgenticCore.Telegram.Inbound;
|
||||||
using Volcanic.AgenticCore.Telegram.Rendering;
|
using Volcanic.AgenticCore.Telegram.Rendering;
|
||||||
|
using Volcanic.AgenticCore.Telegram.Speech;
|
||||||
using Volcanic.AgenticCore.Telegram.Triggers;
|
using Volcanic.AgenticCore.Telegram.Triggers;
|
||||||
|
|
||||||
namespace Volcanic.AgenticCore.Telegram.Chats;
|
namespace Volcanic.AgenticCore.Telegram.Chats;
|
||||||
@@ -83,7 +84,8 @@ public sealed class ChatRegistry(
|
|||||||
Impersonation? impersonation = null,
|
Impersonation? impersonation = null,
|
||||||
TriggerServices? triggers = null,
|
TriggerServices? triggers = null,
|
||||||
A2AClientSettings? a2aClient = null,
|
A2AClientSettings? a2aClient = null,
|
||||||
A2AWatchRegistry? a2aWatches = null) : IAsyncDisposable
|
A2AWatchRegistry? a2aWatches = null,
|
||||||
|
ITextToSpeech? textToSpeech = null) : IAsyncDisposable
|
||||||
{
|
{
|
||||||
private readonly ConcurrentDictionary<long, Lazy<Task<ChatContext>>> _chats = new();
|
private readonly ConcurrentDictionary<long, Lazy<Task<ChatContext>>> _chats = new();
|
||||||
private readonly ILogger _logger = loggerFactory.CreateLogger<ChatRegistry>();
|
private readonly ILogger _logger = loggerFactory.CreateLogger<ChatRegistry>();
|
||||||
@@ -132,6 +134,7 @@ public sealed class ChatRegistry(
|
|||||||
if (zones is not null) extras.Add(ExtensionRegistration.FromInstance(new ChatSettingsExtension(chatId, zones)));
|
if (zones is not null) extras.Add(ExtensionRegistration.FromInstance(new ChatSettingsExtension(chatId, zones)));
|
||||||
if (scheduler is not null)
|
if (scheduler is not null)
|
||||||
extras.Add(ExtensionRegistration.FromInstance(new SchedulerExtension(scheduler, Scheduling.TelegramJobDelivery.TargetOf(chatId))));
|
extras.Add(ExtensionRegistration.FromInstance(new SchedulerExtension(scheduler, Scheduling.TelegramJobDelivery.TargetOf(chatId))));
|
||||||
|
if (textToSpeech is not null) extras.Add(ExtensionRegistration.FromInstance(new VoiceExtension(textToSpeech, channel)));
|
||||||
if (triggers is not null) extras.Add(ExtensionRegistration.FromInstance(new TriggerExtension(chatId, triggers, () => TimeZoneFor(chatId))));
|
if (triggers is not null) extras.Add(ExtensionRegistration.FromInstance(new TriggerExtension(chatId, triggers, () => TimeZoneFor(chatId))));
|
||||||
// Chat sessions only: A2A server sessions are built by the server's own factory call and never get the client.
|
// Chat sessions only: A2A server sessions are built by the server's own factory call and never get the client.
|
||||||
if (a2aClient is { Enabled: true })
|
if (a2aClient is { Enabled: true })
|
||||||
|
|||||||
@@ -40,6 +40,17 @@ public sealed class TelegramChatChannel(TelegramSender sender, ChatRenderer rend
|
|||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Send OGG/Opus audio as a voice note, in order with rendered output (the <c>voice</c> tool). Returns the message id.</summary>
|
||||||
|
public Task<int> SendVoiceAsync(byte[] ogg, string fileName, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(ogg);
|
||||||
|
return renderer.RunInOrderAsync(async ct =>
|
||||||
|
{
|
||||||
|
var result = await sender.SendVoiceAsync(renderer.DeliveryChatId, ogg, fileName, ct).ConfigureAwait(false);
|
||||||
|
return result.MessageId ?? throw new ChatDeliveryException($"Telegram refused the voice message: {result.Error}");
|
||||||
|
}, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
private bool LooksLikePhoto(string path) =>
|
private bool LooksLikePhoto(string path) =>
|
||||||
PhotoExtensions.Contains(Path.GetExtension(path)) && new FileInfo(path) is { Exists: true } info && info.Length <= Limits.MaxPhotoBytes;
|
PhotoExtensions.Contains(Path.GetExtension(path)) && new FileInfo(path) is { Exists: true } info && info.Length <= Limits.MaxPhotoBytes;
|
||||||
|
|
||||||
|
|||||||
118
hosts/Volcanic.AgenticCore.Telegram/Chats/VoiceExtension.cs
Normal file
118
hosts/Volcanic.AgenticCore.Telegram/Chats/VoiceExtension.cs
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
using System.ComponentModel;
|
||||||
|
using System.Globalization;
|
||||||
|
using Volcanic.AgenticCore.Abstractions.Extensions;
|
||||||
|
using Volcanic.AgenticCore.Abstractions.Tools;
|
||||||
|
using Volcanic.AgenticCore.Abstractions.Ui;
|
||||||
|
using Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
|
||||||
|
namespace Volcanic.AgenticCore.Telegram.Chats;
|
||||||
|
|
||||||
|
/// <summary>Arguments of <c>voice</c>.</summary>
|
||||||
|
public sealed record VoiceArgs(
|
||||||
|
[property: Description("What to say, as plain spoken text in any language (no Markdown, links or emoji)")] string Text,
|
||||||
|
[property: Description("Optional OpenAI voice: nova (default, female), marin, coral, shimmer, cedar, alloy, ash, ballad, echo, fable, onyx, sage, verse")] string? Voice = null,
|
||||||
|
[property: Description("Optional delivery instructions in English, e.g. \"warm and calm, unhurried\" or \"cheerful, upbeat\"")] string? Instructions = null);
|
||||||
|
|
||||||
|
/// <summary>Host extension per chat: the <c>voice</c> tool, which speaks text with OpenAI and sends it as Telegram voice notes.</summary>
|
||||||
|
public sealed class VoiceExtension(ITextToSpeech speech, TelegramChatChannel channel) : IExtension
|
||||||
|
{
|
||||||
|
public const string ExtensionName = "telegram-voice";
|
||||||
|
public const string ToolName = "voice";
|
||||||
|
|
||||||
|
/// <summary>The most text one call may speak; more is refused, not cut.</summary>
|
||||||
|
public const int MaxTextLength = 4500;
|
||||||
|
|
||||||
|
/// <summary>The longest text per voice note (the speech API takes 4096 characters and about 2000 tokens).</summary>
|
||||||
|
public const int MaxPartLength = 1500;
|
||||||
|
|
||||||
|
public string Name => ExtensionName;
|
||||||
|
|
||||||
|
public ValueTask ActivateAsync(IExtensionApi api, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(api);
|
||||||
|
api.RegisterTool(new DelegateTool<VoiceArgs>(
|
||||||
|
ToolName,
|
||||||
|
$"Speak text aloud and send it to the user in this chat as a Telegram voice message. Up to {MaxTextLength} characters per call; " +
|
||||||
|
$"text over {MaxPartLength} characters is split at sentence boundaries into several voice messages, sent in order. " +
|
||||||
|
"Longer text is refused: shorten it, or send it as text. Errors come back as text, so you can tell the user.",
|
||||||
|
(args, _, ct) => SpeakAsync(args, ct),
|
||||||
|
promptSnippet: "voice: send text to the user as a spoken voice message",
|
||||||
|
promptGuidelines:
|
||||||
|
[
|
||||||
|
"Use voice only when the user asks for a voice message or audio; otherwise answer in text.",
|
||||||
|
"Write the voice text as natural speech: no Markdown, lists, links or emoji. Don't repeat the voice text in your reply.",
|
||||||
|
],
|
||||||
|
executionMode: ToolExecutionMode.Sequential));
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<ToolResult> SpeakAsync(VoiceArgs args, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var text = args.Text?.Trim() ?? "";
|
||||||
|
if (text.Length == 0) throw new ToolExecutionException(ToolResult.Text("Nothing to say: text is empty."));
|
||||||
|
if (text.Length > MaxTextLength)
|
||||||
|
throw new ToolExecutionException(ToolResult.Text(
|
||||||
|
$"The text is {text.Length} characters; one voice call takes at most {MaxTextLength}. Shorten it, or send it as text."));
|
||||||
|
|
||||||
|
var parts = SplitForSpeech(text, MaxPartLength);
|
||||||
|
// Synthesize every part before sending any, so a speech failure never leaves half a message in the chat.
|
||||||
|
var audio = new List<byte[]>(parts.Count);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
foreach (var part in parts) audio.Add(await speech.SynthesizeAsync(part, args.Voice, args.Instructions, cancellationToken).ConfigureAwait(false));
|
||||||
|
}
|
||||||
|
catch (TextToSpeechException ex)
|
||||||
|
{
|
||||||
|
throw new ToolExecutionException(ToolResult.Text($"{ex.Message} Nothing was sent."));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var i = 0; i < audio.Count; i++)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await channel.SendVoiceAsync(audio[i], $"voice-{i + 1}.ogg", cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (ChatDeliveryException ex)
|
||||||
|
{
|
||||||
|
var sent = i == 0 ? "Nothing was sent." : $"{i} of {audio.Count} voice messages were sent before it.";
|
||||||
|
throw new ToolExecutionException(ToolResult.Text($"{ex.Message} {sent}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ToolResult.Text(audio.Count == 1
|
||||||
|
? "Sent 1 voice message."
|
||||||
|
: string.Create(CultureInfo.InvariantCulture, $"Sent {audio.Count} voice messages."));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Cuts <paramref name="text"/> into parts of at most <paramref name="max"/> characters, preferring paragraph breaks,
|
||||||
|
/// then sentence ends, then spaces; a run without any of them is cut hard.
|
||||||
|
/// </summary>
|
||||||
|
public static IReadOnlyList<string> SplitForSpeech(string text, int max)
|
||||||
|
{
|
||||||
|
ArgumentOutOfRangeException.ThrowIfLessThan(max, 1);
|
||||||
|
var parts = new List<string>();
|
||||||
|
var rest = text.Trim();
|
||||||
|
while (rest.Length > max)
|
||||||
|
{
|
||||||
|
var cut = LastBreak(rest, max);
|
||||||
|
parts.Add(rest[..cut].Trim());
|
||||||
|
rest = rest[cut..].TrimStart();
|
||||||
|
}
|
||||||
|
if (rest.Length > 0) parts.Add(rest);
|
||||||
|
return parts;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The length of the best first part within max: after the last paragraph break past half of max, else the last sentence
|
||||||
|
// end, else the last whitespace, else max.
|
||||||
|
private static int LastBreak(string text, int max)
|
||||||
|
{
|
||||||
|
var window = text[..(max + 1)]; // includes the character just past max, so a break right at max counts
|
||||||
|
var paragraph = window.LastIndexOf("\n\n", StringComparison.Ordinal);
|
||||||
|
if (paragraph >= max / 2) return paragraph;
|
||||||
|
for (var i = max - 1; i >= max / 2; i--)
|
||||||
|
if (text[i] is '.' or '!' or '?' or '…' or ';' && char.IsWhiteSpace(window[i + 1])) return i + 1;
|
||||||
|
for (var i = max; i > 0; i--)
|
||||||
|
if (char.IsWhiteSpace(window[i])) return i;
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -65,6 +65,10 @@ if (options.Speech.IsEnabled) // no key, no ISpeechToText: voice notes get the s
|
|||||||
builder.Services.AddHttpClient<ISpeechToText, OpenAiSpeechToText>(client => client.Timeout = TimeSpan.FromMinutes(2))
|
builder.Services.AddHttpClient<ISpeechToText, OpenAiSpeechToText>(client => client.Timeout = TimeSpan.FromMinutes(2))
|
||||||
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(5) })
|
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(5) })
|
||||||
.SetHandlerLifetime(Timeout.InfiniteTimeSpan);
|
.SetHandlerLifetime(Timeout.InfiniteTimeSpan);
|
||||||
|
if (options.Speech.IsEnabled) // the voice tool; ChatRegistry (a singleton) keeps this client too, so the same handler setup
|
||||||
|
builder.Services.AddHttpClient<ITextToSpeech, OpenAiTextToSpeech>(client => client.Timeout = TimeSpan.FromMinutes(2))
|
||||||
|
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(5) })
|
||||||
|
.SetHandlerLifetime(Timeout.InfiniteTimeSpan);
|
||||||
if (calendar is not null) builder.Services.AddSingleton(calendar); // ChatRegistry and BotSetupService take it optionally
|
if (calendar is not null) builder.Services.AddSingleton(calendar); // ChatRegistry and BotSetupService take it optionally
|
||||||
builder.Services.AddSingleton<OwnerLoginChannel>();
|
builder.Services.AddSingleton<OwnerLoginChannel>();
|
||||||
// The extension is constructed per session through ActivatorUtilities, which injects this. LoginUi resolves
|
// The extension is constructed per session through ActivatorUtilities, which injects this. LoginUi resolves
|
||||||
|
|||||||
@@ -143,6 +143,38 @@ public sealed class TelegramSender(ITelegramBotClient bot, ILogger<TelegramSende
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Upload OGG/Opus audio as a voice note. Never throws except on cancellation: a network failure is retried like
|
||||||
|
/// <see cref="SendFileAsync"/>, an API error is returned.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<FileSendResult> SendVoiceAsync(long chatId, byte[] ogg, string fileName, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
for (var attempt = 0; ; attempt++)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var stream = new MemoryStream(ogg, writable: false);
|
||||||
|
var message = await bot.SendVoice(chatId, InputFile.FromStream(stream, fileName), cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||||
|
return new FileSendResult(message.Id, null, false);
|
||||||
|
}
|
||||||
|
catch (ApiRequestException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Sending a voice note to chat {ChatId} failed", chatId);
|
||||||
|
return new FileSendResult(null, ex.Message, false);
|
||||||
|
}
|
||||||
|
catch (RequestException ex) when (attempt < FileRetryDelays.Count)
|
||||||
|
{
|
||||||
|
logger.LogDebug(ex, "Sending a voice note to chat {ChatId} failed; retrying", chatId);
|
||||||
|
await Task.Delay(FileRetryDelays[attempt], cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Sending a voice note to chat {ChatId} failed", chatId);
|
||||||
|
return new FileSendResult(null, ex.Message, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async Task SendTypingAsync(long chatId, CancellationToken cancellationToken = default)
|
public async Task SendTypingAsync(long chatId, CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
|
|||||||
18
hosts/Volcanic.AgenticCore.Telegram/Speech/ITextToSpeech.cs
Normal file
18
hosts/Volcanic.AgenticCore.Telegram/Speech/ITextToSpeech.cs
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
namespace Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
|
||||||
|
/// <summary>Turns text into a Telegram voice note (OGG/Opus) for the <c>voice</c> tool.</summary>
|
||||||
|
public interface ITextToSpeech
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// OGG/Opus audio of <paramref name="text"/>; <paramref name="voice"/> and <paramref name="instructions"/> override the
|
||||||
|
/// configured defaults when set. Throws <see cref="TextToSpeechException"/> when the service fails.
|
||||||
|
/// </summary>
|
||||||
|
Task<byte[]> SynthesizeAsync(string text, string? voice, string? instructions, CancellationToken cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Speech synthesis failed; the message is fit for the user and never contains the API key.</summary>
|
||||||
|
public sealed class TextToSpeechException(string message, Exception? inner = null) : Exception(message, inner)
|
||||||
|
{
|
||||||
|
/// <summary>Another attempt may succeed (network error, timeout, 408, 429, 5xx).</summary>
|
||||||
|
internal bool IsTransient { get; init; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
using System.Net.Http.Headers;
|
||||||
|
using System.Net.Http.Json;
|
||||||
|
|
||||||
|
namespace Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
|
||||||
|
/// <summary>OpenAI's <c>/audio/speech</c> with <c>response_format=opus</c>, which Telegram shows as a voice note.</summary>
|
||||||
|
public sealed class OpenAiTextToSpeech(HttpClient http, SpeechOptions options) : ITextToSpeech
|
||||||
|
{
|
||||||
|
private const string Prefix = "OpenAI speech synthesis failed: ";
|
||||||
|
private const int MaxExcerpt = 200;
|
||||||
|
|
||||||
|
public async Task<byte[]> SynthesizeAsync(string text, string? voice, string? instructions, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
for (var attempt = 1; ; attempt++)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return await SynthesizeOnceAsync(text, voice, instructions, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (TextToSpeechException ex) when (ex.IsTransient && attempt < SpeechOptions.MaxAttempts)
|
||||||
|
{
|
||||||
|
await Task.Delay(options.RetryDelay * (1 << (attempt - 1)), cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<byte[]> SynthesizeOnceAsync(string text, string? voice, string? instructions, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var body = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
["model"] = options.ResolvedTtsModel,
|
||||||
|
["input"] = text,
|
||||||
|
["voice"] = string.IsNullOrWhiteSpace(voice) ? options.ResolvedTtsVoice : voice.Trim(),
|
||||||
|
["response_format"] = "opus",
|
||||||
|
};
|
||||||
|
var tone = string.IsNullOrWhiteSpace(instructions) ? options.TtsInstructions : instructions.Trim();
|
||||||
|
if (!string.IsNullOrWhiteSpace(tone)) body["instructions"] = tone.Trim();
|
||||||
|
|
||||||
|
using var request = new HttpRequestMessage(HttpMethod.Post, options.BaseUrl.TrimEnd('/') + "/audio/speech")
|
||||||
|
{
|
||||||
|
Content = JsonContent.Create(body),
|
||||||
|
};
|
||||||
|
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiKey);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||||
|
if (response.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
var audio = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
return audio.Length > 0 ? audio : throw new TextToSpeechException(Prefix + "the response is empty.");
|
||||||
|
}
|
||||||
|
var error = Hide(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); // hide before cutting
|
||||||
|
var excerpt = error.Length > MaxExcerpt ? error[..MaxExcerpt] : error;
|
||||||
|
var status = (int)response.StatusCode;
|
||||||
|
throw new TextToSpeechException($"{Prefix}HTTP {status} {excerpt}".TrimEnd()) { IsTransient = status is 408 or 429 or >= 500 };
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) // HttpClient.Timeout
|
||||||
|
{
|
||||||
|
throw new TextToSpeechException(Prefix + "the request timed out.", ex) { IsTransient = true };
|
||||||
|
}
|
||||||
|
catch (HttpRequestException ex)
|
||||||
|
{
|
||||||
|
throw new TextToSpeechException(Prefix + Hide(ex.Message), ex) { IsTransient = true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private string Hide(string text) =>
|
||||||
|
string.IsNullOrEmpty(options.ApiKey) ? text : text.Replace(options.ApiKey, "[REDACTED]", StringComparison.Ordinal);
|
||||||
|
}
|
||||||
@@ -1,10 +1,15 @@
|
|||||||
namespace Volcanic.AgenticCore.Telegram.Speech;
|
namespace Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
|
||||||
/// <summary>The <c>Telegram:Speech</c> section. Speech to text is on exactly when <see cref="ApiKey"/> is set.</summary>
|
/// <summary>
|
||||||
|
/// The <c>Telegram:Speech</c> section. Speech to text (voice notes in) and text to speech (the <c>voice</c> tool) are on
|
||||||
|
/// exactly when <see cref="ApiKey"/> is set.
|
||||||
|
/// </summary>
|
||||||
public sealed class SpeechOptions
|
public sealed class SpeechOptions
|
||||||
{
|
{
|
||||||
public const string DefaultModel = "gpt-4o-transcribe";
|
public const string DefaultModel = "gpt-4o-transcribe";
|
||||||
public const string DefaultBaseUrl = "https://api.openai.com/v1";
|
public const string DefaultBaseUrl = "https://api.openai.com/v1";
|
||||||
|
public const string DefaultTtsModel = "gpt-4o-mini-tts";
|
||||||
|
public const string DefaultTtsVoice = "nova";
|
||||||
|
|
||||||
/// <summary>Falls back to <c>OPENAI_API_KEY</c>. Empty turns speech to text off.</summary>
|
/// <summary>Falls back to <c>OPENAI_API_KEY</c>. Empty turns speech to text off.</summary>
|
||||||
public string ApiKey { get; set; } = "";
|
public string ApiKey { get; set; } = "";
|
||||||
@@ -12,7 +17,16 @@ public sealed class SpeechOptions
|
|||||||
/// <summary>Falls back to <c>STT_MODEL</c>, then <see cref="DefaultModel"/>.</summary>
|
/// <summary>Falls back to <c>STT_MODEL</c>, then <see cref="DefaultModel"/>.</summary>
|
||||||
public string Model { get; set; } = "";
|
public string Model { get; set; } = "";
|
||||||
|
|
||||||
/// <summary>Any OpenAI-compatible transcription endpoint.</summary>
|
/// <summary>Falls back to <c>TTS_MODEL</c>, then <see cref="DefaultTtsModel"/>.</summary>
|
||||||
|
public string TtsModel { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>Falls back to <c>TTS_VOICE</c>, then <see cref="DefaultTtsVoice"/>. The tool's <c>voice</c> argument overrides it.</summary>
|
||||||
|
public string TtsVoice { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>Default delivery instructions (tone, pace); the tool's <c>instructions</c> argument overrides them. Ignored by tts-1.</summary>
|
||||||
|
public string? TtsInstructions { get; set; }
|
||||||
|
|
||||||
|
/// <summary>Any OpenAI-compatible audio endpoint (transcriptions and speech).</summary>
|
||||||
public string BaseUrl { get; set; } = DefaultBaseUrl;
|
public string BaseUrl { get; set; } = DefaultBaseUrl;
|
||||||
|
|
||||||
/// <summary>Optional ISO-639-1 hint.</summary>
|
/// <summary>Optional ISO-639-1 hint.</summary>
|
||||||
@@ -28,6 +42,12 @@ public sealed class SpeechOptions
|
|||||||
|
|
||||||
public string ResolvedModel => string.IsNullOrWhiteSpace(Model) ? DefaultModel : Model.Trim();
|
public string ResolvedModel => string.IsNullOrWhiteSpace(Model) ? DefaultModel : Model.Trim();
|
||||||
|
|
||||||
|
public string ResolvedTtsModel => string.IsNullOrWhiteSpace(TtsModel) ? DefaultTtsModel : TtsModel.Trim();
|
||||||
|
|
||||||
|
public string ResolvedTtsVoice => string.IsNullOrWhiteSpace(TtsVoice) ? DefaultTtsVoice : TtsVoice.Trim();
|
||||||
|
|
||||||
/// <summary>The startup log line (never the key).</summary>
|
/// <summary>The startup log line (never the key).</summary>
|
||||||
public string Describe() => IsEnabled ? $"Speech to text: {ResolvedModel}" : "Speech to text: off (no OPENAI_API_KEY)";
|
public string Describe() => IsEnabled
|
||||||
|
? $"Speech to text: {ResolvedModel}; text to speech: {ResolvedTtsModel}, voice {ResolvedTtsVoice}"
|
||||||
|
: "Speech to text and text to speech: off (no OPENAI_API_KEY)";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,6 +79,10 @@ public sealed class TelegramOptions
|
|||||||
Speech.ApiKey = key.Trim();
|
Speech.ApiKey = key.Trim();
|
||||||
if (string.IsNullOrWhiteSpace(Speech.Model) && getEnv("STT_MODEL") is { } model && !string.IsNullOrWhiteSpace(model))
|
if (string.IsNullOrWhiteSpace(Speech.Model) && getEnv("STT_MODEL") is { } model && !string.IsNullOrWhiteSpace(model))
|
||||||
Speech.Model = model.Trim();
|
Speech.Model = model.Trim();
|
||||||
|
if (string.IsNullOrWhiteSpace(Speech.TtsModel) && getEnv("TTS_MODEL") is { } ttsModel && !string.IsNullOrWhiteSpace(ttsModel))
|
||||||
|
Speech.TtsModel = ttsModel.Trim();
|
||||||
|
if (string.IsNullOrWhiteSpace(Speech.TtsVoice) && getEnv("TTS_VOICE") is { } ttsVoice && !string.IsNullOrWhiteSpace(ttsVoice))
|
||||||
|
Speech.TtsVoice = ttsVoice.Trim();
|
||||||
Triggers ??= new TriggerOptions();
|
Triggers ??= new TriggerOptions();
|
||||||
Triggers.ApplyEnvironment(getEnv);
|
Triggers.ApplyEnvironment(getEnv);
|
||||||
if (string.IsNullOrWhiteSpace(DefaultTimeZone) && getEnv("DEFAULT_TIMEZONE") is { } tz && !string.IsNullOrWhiteSpace(tz))
|
if (string.IsNullOrWhiteSpace(DefaultTimeZone) && getEnv("DEFAULT_TIMEZONE") is { } tz && !string.IsNullOrWhiteSpace(tz))
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ public sealed class FormattingAndOptionsTests
|
|||||||
Assert.Equal("sk-env", options.Speech.ApiKey);
|
Assert.Equal("sk-env", options.Speech.ApiKey);
|
||||||
Assert.Equal("whisper-1", options.Speech.ResolvedModel);
|
Assert.Equal("whisper-1", options.Speech.ResolvedModel);
|
||||||
Assert.True(options.Speech.IsEnabled);
|
Assert.True(options.Speech.IsEnabled);
|
||||||
Assert.Equal("Speech to text: whisper-1", options.Speech.Describe());
|
Assert.Equal("Speech to text: whisper-1; text to speech: gpt-4o-mini-tts, voice nova", options.Speech.Describe());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -121,7 +121,7 @@ public sealed class FormattingAndOptionsTests
|
|||||||
options.ApplyEnvironment(name => name is "OPENAI_API_KEY" or "STT_MODEL" ? "" : null);
|
options.ApplyEnvironment(name => name is "OPENAI_API_KEY" or "STT_MODEL" ? "" : null);
|
||||||
Assert.False(options.Speech.IsEnabled);
|
Assert.False(options.Speech.IsEnabled);
|
||||||
Assert.Equal(SpeechOptions.DefaultModel, options.Speech.ResolvedModel);
|
Assert.Equal(SpeechOptions.DefaultModel, options.Speech.ResolvedModel);
|
||||||
Assert.Equal("Speech to text: off (no OPENAI_API_KEY)", options.Speech.Describe());
|
Assert.Equal("Speech to text and text to speech: off (no OPENAI_API_KEY)", options.Speech.Describe());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -134,6 +134,15 @@ public sealed class FormattingAndOptionsTests
|
|||||||
Assert.Equal(("sk-config", "gpt-4o-mini-transcribe"), (options.Speech.ApiKey, options.Speech.ResolvedModel));
|
Assert.Equal(("sk-config", "gpt-4o-mini-transcribe"), (options.Speech.ApiKey, options.Speech.ResolvedModel));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Text_to_speech_falls_back_to_TTS_MODEL_and_TTS_VOICE()
|
||||||
|
{
|
||||||
|
var options = Valid();
|
||||||
|
options.ApplyEnvironment(name => name switch { "TTS_MODEL" => "tts-1-hd", "TTS_VOICE" => " shimmer ", _ => null });
|
||||||
|
Assert.Equal(("tts-1-hd", "shimmer"), (options.Speech.ResolvedTtsModel, options.Speech.ResolvedTtsVoice));
|
||||||
|
Assert.Equal(("gpt-4o-mini-tts", "nova"), (new SpeechOptions().ResolvedTtsModel, new SpeechOptions().ResolvedTtsVoice));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Speech_defaults_to_gpt_4o_transcribe_on_the_openai_endpoint()
|
public void Speech_defaults_to_gpt_4o_transcribe_on_the_openai_endpoint()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
using System.Net;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
|
||||||
|
namespace Volcanic.AgenticCore.Telegram.Tests;
|
||||||
|
|
||||||
|
public sealed class OpenAiTextToSpeechTests
|
||||||
|
{
|
||||||
|
private const string Key = "sk-test-secret-key-123";
|
||||||
|
|
||||||
|
private sealed class StubHandler(params (HttpStatusCode Status, byte[] Body)[] answers) : HttpMessageHandler
|
||||||
|
{
|
||||||
|
public List<(HttpRequestMessage Request, JsonElement Body)> Requests { get; } = [];
|
||||||
|
|
||||||
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
Requests.Add((request, JsonDocument.Parse(await request.Content!.ReadAsStringAsync(cancellationToken)).RootElement.Clone()));
|
||||||
|
var (status, body) = answers[Math.Min(Requests.Count, answers.Length) - 1];
|
||||||
|
return new HttpResponseMessage(status) { Content = new ByteArrayContent(body) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (OpenAiTextToSpeech Tts, StubHandler Handler) Create(SpeechOptions? options = null, params (HttpStatusCode, byte[])[] answers)
|
||||||
|
{
|
||||||
|
var handler = new StubHandler(answers.Length > 0 ? answers : [(HttpStatusCode.OK, [1, 2, 3])]);
|
||||||
|
options ??= new SpeechOptions();
|
||||||
|
options.ApiKey = Key;
|
||||||
|
options.BaseUrl = "https://tts.example/v1/";
|
||||||
|
options.RetryDelay = TimeSpan.Zero;
|
||||||
|
return (new OpenAiTextToSpeech(new HttpClient(handler), options), handler);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] Utf8(string text) => Encoding.UTF8.GetBytes(text);
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Posts_json_with_model_voice_input_and_opus_format()
|
||||||
|
{
|
||||||
|
var (tts, handler) = Create();
|
||||||
|
var audio = await tts.SynthesizeAsync("Привет", null, null, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(new byte[] { 1, 2, 3 }, audio);
|
||||||
|
var (request, body) = Assert.Single(handler.Requests);
|
||||||
|
Assert.Equal("https://tts.example/v1/audio/speech", request.RequestUri!.ToString());
|
||||||
|
Assert.Equal(("Bearer", Key), (request.Headers.Authorization!.Scheme, request.Headers.Authorization.Parameter));
|
||||||
|
Assert.Equal("gpt-4o-mini-tts", body.GetProperty("model").GetString());
|
||||||
|
Assert.Equal("nova", body.GetProperty("voice").GetString());
|
||||||
|
Assert.Equal("Привет", body.GetProperty("input").GetString());
|
||||||
|
Assert.Equal("opus", body.GetProperty("response_format").GetString());
|
||||||
|
Assert.False(body.TryGetProperty("instructions", out _));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Arguments_override_the_configured_voice_and_instructions()
|
||||||
|
{
|
||||||
|
var (tts, handler) = Create(new SpeechOptions { TtsVoice = "cedar", TtsInstructions = "calm" });
|
||||||
|
await tts.SynthesizeAsync("hi", "shimmer", "cheerful", CancellationToken.None);
|
||||||
|
var body = handler.Requests[0].Body;
|
||||||
|
Assert.Equal(("shimmer", "cheerful"), (body.GetProperty("voice").GetString(), body.GetProperty("instructions").GetString()));
|
||||||
|
|
||||||
|
await tts.SynthesizeAsync("hi", null, null, CancellationToken.None);
|
||||||
|
body = handler.Requests[1].Body;
|
||||||
|
Assert.Equal(("cedar", "calm"), (body.GetProperty("voice").GetString(), body.GetProperty("instructions").GetString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task An_api_error_is_reported_with_the_key_hidden_and_not_retried()
|
||||||
|
{
|
||||||
|
var (tts, handler) = Create(null, (HttpStatusCode.BadRequest, Utf8($"{{\"error\":\"bad voice for {Key}\"}}")));
|
||||||
|
var ex = await Assert.ThrowsAsync<TextToSpeechException>(() => tts.SynthesizeAsync("hi", "nobody", null, CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Single(handler.Requests);
|
||||||
|
Assert.StartsWith("OpenAI speech synthesis failed: HTTP 400", ex.Message, StringComparison.Ordinal);
|
||||||
|
Assert.DoesNotContain(Key, ex.Message, StringComparison.Ordinal);
|
||||||
|
Assert.Contains("[REDACTED]", ex.Message, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Server_errors_are_retried()
|
||||||
|
{
|
||||||
|
var (tts, handler) = Create(null, (HttpStatusCode.InternalServerError, Utf8("oops")), (HttpStatusCode.OK, [7]));
|
||||||
|
Assert.Equal(new byte[] { 7 }, await tts.SynthesizeAsync("hi", null, null, CancellationToken.None));
|
||||||
|
Assert.Equal(2, handler.Requests.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task An_empty_response_is_an_error()
|
||||||
|
{
|
||||||
|
var (tts, _) = Create(null, (HttpStatusCode.OK, []));
|
||||||
|
var ex = await Assert.ThrowsAsync<TextToSpeechException>(() => tts.SynthesizeAsync("hi", null, null, CancellationToken.None));
|
||||||
|
Assert.Equal("OpenAI speech synthesis failed: the response is empty.", ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -59,7 +59,8 @@ public sealed class ChatHost : IAsyncDisposable
|
|||||||
Volcanic.AgenticCore.Extensions.GoogleCalendar.GoogleCalendarOptions? calendar = null,
|
Volcanic.AgenticCore.Extensions.GoogleCalendar.GoogleCalendarOptions? calendar = null,
|
||||||
Volcanic.AgenticCore.Telegram.Speech.ISpeechToText? speech = null, string? root = null, SchedulerOptions? scheduler = null,
|
Volcanic.AgenticCore.Telegram.Speech.ISpeechToText? speech = null, string? root = null, SchedulerOptions? scheduler = null,
|
||||||
IScriptRunner? runner = null, TriggerOptions? triggers = null, IJevClient? jev = null,
|
IScriptRunner? runner = null, TriggerOptions? triggers = null, IJevClient? jev = null,
|
||||||
Volcanic.AgenticCore.A2A.Server.A2AHostOptions? a2a = null, Volcanic.AgenticCore.Extensions.A2A.A2AClientSettings? a2aClient = null)
|
Volcanic.AgenticCore.A2A.Server.A2AHostOptions? a2a = null, Volcanic.AgenticCore.Extensions.A2A.A2AClientSettings? a2aClient = null,
|
||||||
|
Volcanic.AgenticCore.Telegram.Speech.ITextToSpeech? tts = null)
|
||||||
{
|
{
|
||||||
_ownsRoot = root is null;
|
_ownsRoot = root is null;
|
||||||
_root = root ?? Path.Combine(Path.GetTempPath(), "volcanic-telegram", Guid.NewGuid().ToString("N"));
|
_root = root ?? Path.Combine(Path.GetTempPath(), "volcanic-telegram", Guid.NewGuid().ToString("N"));
|
||||||
@@ -117,7 +118,7 @@ public sealed class ChatHost : IAsyncDisposable
|
|||||||
}
|
}
|
||||||
if (a2aClient is not null) A2AWatchRegistry = new Volcanic.AgenticCore.Extensions.A2A.A2AWatchRegistry(clock, new HttpClientHandler());
|
if (a2aClient is not null) A2AWatchRegistry = new Volcanic.AgenticCore.Extensions.A2A.A2AWatchRegistry(clock, new HttpClientHandler());
|
||||||
Chats = new ChatRegistry(_services.GetRequiredService<IAgentSessionFactory>(), sender, Options, loggers, time, calendar, Settings, Zones, Scheduler, Impersonation, Triggers,
|
Chats = new ChatRegistry(_services.GetRequiredService<IAgentSessionFactory>(), sender, Options, loggers, time, calendar, Settings, Zones, Scheduler, Impersonation, Triggers,
|
||||||
a2aClient, A2AWatchRegistry);
|
a2aClient, A2AWatchRegistry, tts);
|
||||||
if (Scheduler is not null) Delivery = new TelegramJobDelivery(Chats, Settings, Options, clock, NullLogger<TelegramJobDelivery>.Instance);
|
if (Scheduler is not null) Delivery = new TelegramJobDelivery(Chats, Settings, Options, clock, NullLogger<TelegramJobDelivery>.Instance);
|
||||||
var media = new InboundMedia(sender, Options, time ?? TimeProvider.System, NullLogger<InboundMedia>.Instance, speech);
|
var media = new InboundMedia(sender, Options, time ?? TimeProvider.System, NullLogger<InboundMedia>.Instance, speech);
|
||||||
Composer = new InboundComposer(sender, media, NullLogger<InboundComposer>.Instance);
|
Composer = new InboundComposer(sender, media, NullLogger<InboundComposer>.Instance);
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ public sealed class FakeBotClient : ITelegramBotClient
|
|||||||
|
|
||||||
public sealed record Upload(string Method, string FileName, byte[] Bytes, string? Caption, ParseMode ParseMode, int MessageId);
|
public sealed record Upload(string Method, string FileName, byte[] Bytes, string? Caption, ParseMode ParseMode, int MessageId);
|
||||||
private readonly List<Upload> _uploads = [];
|
private readonly List<Upload> _uploads = [];
|
||||||
/// <summary>Photos and documents sent, with the uploaded bytes (copied inside the request).</summary>
|
/// <summary>Photos, documents and voice notes sent, with the uploaded bytes (copied inside the request).</summary>
|
||||||
public IReadOnlyList<Upload> Uploads { get { lock (_uploads) return _uploads.ToList(); } }
|
public IReadOnlyList<Upload> Uploads { get { lock (_uploads) return _uploads.ToList(); } }
|
||||||
|
|
||||||
private Message RecordUpload(string method, ChatId chatId, InputFile file, string? caption, ParseMode parseMode)
|
private Message RecordUpload(string method, ChatId chatId, InputFile file, string? caption, ParseMode parseMode)
|
||||||
@@ -140,6 +140,9 @@ public sealed class FakeBotClient : ITelegramBotClient
|
|||||||
case SendDocumentRequest document:
|
case SendDocumentRequest document:
|
||||||
result = RecordUpload("sendDocument", document.ChatId, document.Document, document.Caption, document.ParseMode);
|
result = RecordUpload("sendDocument", document.ChatId, document.Document, document.Caption, document.ParseMode);
|
||||||
break;
|
break;
|
||||||
|
case SendVoiceRequest voice:
|
||||||
|
result = RecordUpload("sendVoice", voice.ChatId, voice.Voice, voice.Caption, voice.ParseMode);
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
result = request is GetMeRequest ? new User { Id = BotId, IsBot = true, FirstName = "Fake", Username = "fake_bot" } : true;
|
result = request is GetMeRequest ? new User { Id = BotId, IsBot = true, FirstName = "Fake", Username = "fake_bot" } : true;
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
using Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
|
||||||
|
namespace Volcanic.AgenticCore.Telegram.Tests.Support;
|
||||||
|
|
||||||
|
/// <summary>Answers every text with fake OGG bytes; <see cref="Failure"/> makes a call throw.</summary>
|
||||||
|
public sealed class FakeTextToSpeech : ITextToSpeech
|
||||||
|
{
|
||||||
|
public Exception? Failure { get; set; }
|
||||||
|
public List<(string Text, string? Voice, string? Instructions)> Calls { get; } = [];
|
||||||
|
|
||||||
|
public Task<byte[]> SynthesizeAsync(string text, string? voice, string? instructions, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
lock (Calls) Calls.Add((text, voice, instructions));
|
||||||
|
if (Failure is not null) throw Failure;
|
||||||
|
return Task.FromResult(new byte[] { 0x4F, 0x67, 0x67, 0x53, (byte)Calls.Count }); // "OggS" + call number
|
||||||
|
}
|
||||||
|
}
|
||||||
122
tests/Volcanic.AgenticCore.Telegram.Tests/VoiceToolChatTests.cs
Normal file
122
tests/Volcanic.AgenticCore.Telegram.Tests/VoiceToolChatTests.cs
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
using Telegram.Bot.Exceptions;
|
||||||
|
using Telegram.Bot.Requests;
|
||||||
|
using Volcanic.AgenticCore.Abstractions.Messages;
|
||||||
|
using Volcanic.AgenticCore.Telegram.Chats;
|
||||||
|
using Volcanic.AgenticCore.Telegram.Speech;
|
||||||
|
using Volcanic.AgenticCore.Telegram.Tests.Support;
|
||||||
|
using Volcanic.AgenticCore.Testing;
|
||||||
|
|
||||||
|
namespace Volcanic.AgenticCore.Telegram.Tests;
|
||||||
|
|
||||||
|
public sealed class VoiceToolChatTests
|
||||||
|
{
|
||||||
|
private static async Task<ToolResultMessage> ResultAsync(ChatHost host) =>
|
||||||
|
(await host.ChatAsync()).Session.Messages.OfType<ToolResultMessage>().Single();
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Voice_text_is_spoken_and_sent_as_a_voice_note_before_the_answer()
|
||||||
|
{
|
||||||
|
var tts = new FakeTextToSpeech();
|
||||||
|
await using var host = new ChatHost(tts: tts);
|
||||||
|
host.Faux.Enqueue(
|
||||||
|
FauxTurn.Tool("voice", new { text = "Привет! Встреча в три.", voice = "coral", instructions = "warm" }),
|
||||||
|
FauxTurn.Say("Sent."));
|
||||||
|
|
||||||
|
await host.SendAsync("скажи голосом");
|
||||||
|
await host.SettleAsync();
|
||||||
|
|
||||||
|
Assert.Equal(("Привет! Встреча в три.", "coral", "warm"), Assert.Single(tts.Calls));
|
||||||
|
var upload = Assert.Single(host.Bot.Uploads);
|
||||||
|
Assert.Equal(("sendVoice", "voice-1.ogg"), (upload.Method, upload.FileName));
|
||||||
|
Assert.Equal(new byte[] { 0x4F, 0x67, 0x67, 0x53, 1 }, upload.Bytes);
|
||||||
|
var requests = host.Bot.Requests.ToList();
|
||||||
|
Assert.InRange(requests.FindIndex(r => r is SendVoiceRequest), 0,
|
||||||
|
requests.FindIndex(r => r is SendMessageRequest m && m.Text.Contains("Sent.", StringComparison.Ordinal)) - 1);
|
||||||
|
var result = await ResultAsync(host);
|
||||||
|
Assert.False(result.IsError);
|
||||||
|
Assert.Equal("Sent 1 voice message.", result.Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Long_text_goes_out_as_several_voice_notes_in_order()
|
||||||
|
{
|
||||||
|
var tts = new FakeTextToSpeech();
|
||||||
|
await using var host = new ChatHost(tts: tts);
|
||||||
|
var sentence = new string('а', 99) + ". ";
|
||||||
|
var text = string.Concat(Enumerable.Repeat(sentence, 20)).Trim(); // about 2000 characters
|
||||||
|
host.Faux.Enqueue(FauxTurn.Tool("voice", new { text }), FauxTurn.Say("Done."));
|
||||||
|
|
||||||
|
await host.SendAsync("long voice");
|
||||||
|
await host.SettleAsync();
|
||||||
|
|
||||||
|
Assert.Equal(2, tts.Calls.Count);
|
||||||
|
Assert.All(tts.Calls, c => Assert.True(c.Text.Length <= VoiceExtension.MaxPartLength));
|
||||||
|
Assert.Equal(["voice-1.ogg", "voice-2.ogg"], host.Bot.Uploads.Select(u => u.FileName));
|
||||||
|
Assert.Equal("Sent 2 voice messages.", (await ResultAsync(host)).Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Too_long_text_is_refused_without_calling_the_api()
|
||||||
|
{
|
||||||
|
var tts = new FakeTextToSpeech();
|
||||||
|
await using var host = new ChatHost(tts: tts);
|
||||||
|
host.Faux.Enqueue(FauxTurn.Tool("voice", new { text = new string('x', VoiceExtension.MaxTextLength + 1) }), FauxTurn.Say("Too long."));
|
||||||
|
|
||||||
|
await host.SendAsync("very long voice");
|
||||||
|
await host.SettleAsync();
|
||||||
|
|
||||||
|
Assert.Empty(tts.Calls);
|
||||||
|
Assert.Empty(host.Bot.Uploads);
|
||||||
|
var result = await ResultAsync(host);
|
||||||
|
Assert.True(result.IsError);
|
||||||
|
Assert.Contains("at most 4500", result.Text, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Api_errors_come_back_as_text_and_nothing_is_sent()
|
||||||
|
{
|
||||||
|
var tts = new FakeTextToSpeech { Failure = new TextToSpeechException("OpenAI speech synthesis failed: HTTP 401 bad key") };
|
||||||
|
await using var host = new ChatHost(tts: tts);
|
||||||
|
host.Faux.Enqueue(FauxTurn.Tool("voice", new { text = "hello" }), FauxTurn.Say("It failed."));
|
||||||
|
|
||||||
|
await host.SendAsync("voice");
|
||||||
|
await host.SettleAsync();
|
||||||
|
|
||||||
|
Assert.Empty(host.Bot.Uploads);
|
||||||
|
var result = await ResultAsync(host);
|
||||||
|
Assert.True(result.IsError);
|
||||||
|
Assert.Equal("OpenAI speech synthesis failed: HTTP 401 bad key Nothing was sent.", result.Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task A_refused_voice_note_is_reported()
|
||||||
|
{
|
||||||
|
await using var host = new ChatHost(tts: new FakeTextToSpeech());
|
||||||
|
host.Bot.OnSend = r => { if (r is SendVoiceRequest) throw new ApiRequestException("Forbidden: VOICE_MESSAGES_FORBIDDEN", 400); };
|
||||||
|
host.Faux.Enqueue(FauxTurn.Tool("voice", new { text = "hello" }), FauxTurn.Say("It failed."));
|
||||||
|
|
||||||
|
await host.SendAsync("voice");
|
||||||
|
await host.SettleAsync();
|
||||||
|
|
||||||
|
var result = await ResultAsync(host);
|
||||||
|
Assert.True(result.IsError);
|
||||||
|
Assert.Contains("VOICE_MESSAGES_FORBIDDEN", result.Text, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Without_text_to_speech_there_is_no_voice_tool()
|
||||||
|
{
|
||||||
|
await using var host = new ChatHost();
|
||||||
|
var chat = await host.ChatAsync();
|
||||||
|
Assert.DoesNotContain(chat.Session.Tools, t => t.Name == "voice");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("One. Two. Three.", 100, new[] { "One. Two. Three." })]
|
||||||
|
[InlineData("Aaaa bbbb. Cccc dddd.", 12, new[] { "Aaaa bbbb.", "Cccc dddd." })]
|
||||||
|
[InlineData("aaaa bbbb cccc", 9, new[] { "aaaa bbbb", "cccc" })]
|
||||||
|
[InlineData("abcdefghij", 4, new[] { "abcd", "efgh", "ij" })]
|
||||||
|
[InlineData("First para\n\nSecond para", 15, new[] { "First para", "Second para" })]
|
||||||
|
public void Splits_at_paragraphs_sentences_spaces_then_hard(string text, int max, string[] expected) =>
|
||||||
|
Assert.Equal(expected, VoiceExtension.SplitForSpeech(text, max));
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user