Playa/hosts/Volcanic.AgenticCore.Telegram/Inbound/InboundMedia.cs

using Microsoft.Extensions.Logging;
using Telegram.Bot;
using Telegram.Bot.Exceptions;
using Telegram.Bot.Types;
using Volcanic.AgenticCore.Telegram.Chats;
using Volcanic.AgenticCore.Telegram.Rendering;
using Volcanic.AgenticCore.Telegram.Speech;
using ImageContent = Volcanic.AgenticCore.Abstractions.Messages.ImageContent;

namespace Volcanic.AgenticCore.Telegram.Inbound;

/// <summary>
/// Content steps for attachments (spec: telegram-attachments §4): step 2 saves documents, videos and audio to the
/// workspace inbox, step 3 transcribes voice notes and audio through <see cref="ISpeechToText"/> (null: speech is off).
/// Notices and the echo go through the chat renderer (<see cref="ChatRenderer.SayAsync"/>) so they stay in order.
/// </summary>
public sealed class InboundMedia(TelegramSender sender, TelegramOptions options, TimeProvider time, ILogger<InboundMedia> logger,
    ISpeechToText? speech = null)
{
    /// <summary>Sent for a voice note or audio file when no speech-to-text service is configured.</summary>
    public const string SpeechNotConfiguredNotice = "Voice messages need speech-to-text: set OPENAI_API_KEY (or Telegram:Speech:ApiKey) on the bot host.";

    /// <summary>The longest <c>🎤</c> echo, in UTF-16 code units; a long transcript is cut so it doesn't flood the chat.</summary>
    public const int MaxEchoLength = 1024;

    /// <summary>Sent when the transcript is empty.</summary>
    public const string NoWordsNotice = "I couldn't make out any words in that voice message.";

    /// <summary>Sent when a file's download fails (the partial file is deleted and the message isn't prompted).</summary>
    public const string DownloadFailedNotice = "I could not download that file.";

    /// <summary>Suffix of a <c>[File received: …]</c> line for an image too large to attach inline.</summary>
    public const string TooLargeInlineNote = " (too large to show inline; downscale it with bash if you need to see it)";

    /// <summary>Suffix of a <c>[File received: …]</c> line for an image document whose bytes aren't a known image.</summary>
    public const string NotAnImageNote = " (not shown inline: the contents are not a JPEG, PNG, WebP or GIF image)";

    private static readonly HashSet<string> InlineImageTypes = new(StringComparer.OrdinalIgnoreCase) { "image/jpeg", "image/png", "image/webp", "image/gif" };

    /// <summary>The notice for a file over the 20 MB download limit; <c>over 20 MB</c> when the size is unknown.</summary>
    public static string TooBigNotice(long? bytes) =>
        $"That file is {(bytes is { } b ? TelegramLimits.Megabytes(b) + " MB" : "over 20 MB")}; Telegram lets bots download at most 20 MB. Share a link or a smaller file.";

    /// <summary>Download a file by id into <paramref name="destination"/> (shared with step 1's photos).</summary>
    internal Task DownloadAsync(string fileId, Stream destination, CancellationToken cancellationToken) =>
        sender.Bot.GetInfoAndDownloadFile(fileId, destination, cancellationToken);

    /// <summary>Telegram's <c>getFile</c> refusal for a file over the download limit.</summary>
    internal static bool IsTooBig(ApiRequestException ex) => ex.Message.Contains("file is too big", StringComparison.OrdinalIgnoreCase);

    /// <summary>Step 2: a document, video or audio file goes to the inbox and gets a <c>[File received: …]</c> line.</summary>
    internal async Task AddFilesAsync(ChatContext chat, Message message, InboundDraft draft, CancellationToken cancellationToken)
    {
        var (fileId, size, name, mime) = message switch
        {
            { Document: { } d } => (d.FileId, d.FileSize, d.FileName ?? "file", d.MimeType ?? "application/octet-stream"),
            { Video: { } v } => (v.FileId, v.FileSize, v.FileName ?? "video.mp4", v.MimeType ?? "video/mp4"),
            { Audio: { } a } => (a.FileId, a.FileSize, a.FileName ?? "audio", a.MimeType ?? "audio/mpeg"),
            _ => ((string?)null, (long?)null, "", ""),
        };
        if (fileId is null) return;
        if (size > TelegramLimits.MaxDownloadBytes)
        {
            await chat.Renderer.SayAsync(TooBigNotice(size)).ConfigureAwait(false);
            draft.Refused = true;
            return;
        }

        var workspace = chat.Session.Cwd;
        var (path, stream) = InboxFiles.Create(workspace, time.GetLocalNow(), name);
        try
        {
            await using (stream.ConfigureAwait(false))
                await DownloadAsync(fileId, stream, cancellationToken).ConfigureAwait(false);
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            TryDelete(path);
            throw;
        }
        catch (Exception ex)
        {
            TryDelete(path);
            logger.LogWarning(ex, "Downloading {File} in chat {ChatId} failed", name, chat.ChatId);
            await chat.Renderer.SayAsync(ex is ApiRequestException api && IsTooBig(api) ? TooBigNotice(size) : DownloadFailedNotice).ConfigureAwait(false);
            draft.Refused = true;
            return;
        }

        var bytes = new FileInfo(path).Length;
        draft.SavedPath = path;
        var note = "";
        if (InlineImageTypes.Contains(mime))
        {
            if (bytes <= options.InlineImageMaxBytes)
            {
                // Documents aren't re-encoded by Telegram, so the declared type can be wrong (HEIC named .png, a
                // truncated file). A provider rejects such an image, and the persisted prompt would break every later
                // turn: attach only bytes that really are one of the four types, under the type they show.
                var data = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
                if (SniffImageType(data) is { } actual) draft.Images.Add(new ImageContent(Convert.ToBase64String(data), actual));
                else note = NotAnImageNote;
            }
            else
            {
                note = TooLargeInlineNote;
            }
        }
        draft.Body.Add($"[File received: {InboxFiles.Relative(workspace, path)} ({mime}, {TelegramLimits.FormatSize(bytes)}){note}]");
    }

    /// <summary>
    /// Step 3: a voice note or audio file is transcribed, echoed as <c>🎤 …</c> and added as <c>[Voice message] …</c>.
    /// Audio is read back from the file step 2 saved (one download). Without a transcript nothing is added (the caption,
    /// if any, is still prompted); an oversize or undownloadable voice note refuses the message.
    /// </summary>
    internal async Task AddSpeechAsync(ChatContext chat, Message message, InboundDraft draft, CancellationToken cancellationToken)
    {
        var (fileId, size, name, mime, isVoice) = message switch
        {
            { Voice: { } v } => (v.FileId, v.FileSize, "voice.ogg", "audio/ogg", true), // Telegram's .oga is rejected by OpenAI
            { Audio: { } a } => (a.FileId, a.FileSize, SpeechFileName(a.FileName, a.MimeType), a.MimeType ?? "audio/mpeg", false),
            _ => ((string?)null, (long?)null, "", "", false),
        };
        if (fileId is null) return;
        if (speech is null)
        {
            await chat.Renderer.SayAsync(SpeechNotConfiguredNotice).ConfigureAwait(false);
            return;
        }
        if (isVoice && size > TelegramLimits.MaxDownloadBytes)
        {
            await chat.Renderer.SayAsync(TooBigNotice(size)).ConfigureAwait(false);
            draft.Refused = true;
            return;
        }

        byte[] audio;
        if (draft.SavedPath is { } saved) // audio: step 2 saved it
        {
            audio = await File.ReadAllBytesAsync(saved, cancellationToken).ConfigureAwait(false);
        }
        else
        {
            using var buffer = new MemoryStream();
            try
            {
                await DownloadAsync(fileId, buffer, cancellationToken).ConfigureAwait(false);
            }
            catch (Exception ex) when (!cancellationToken.IsCancellationRequested) // a client timeout is a failure, not a shutdown
            {
                logger.LogWarning(ex, "Downloading a voice message in chat {ChatId} failed", chat.ChatId);
                await chat.Renderer.SayAsync(ex is ApiRequestException api && IsTooBig(api) ? TooBigNotice(size) : DownloadFailedNotice).ConfigureAwait(false);
                draft.Refused = true;
                return;
            }
            audio = buffer.ToArray();
        }

        string transcript;
        try
        {
            transcript = (await speech.TranscribeAsync(audio, name, mime, cancellationToken).ConfigureAwait(false)).Trim();
        }
        catch (Exception ex) when (!cancellationToken.IsCancellationRequested) // a client timeout is a failure, not a shutdown
        {
            logger.LogWarning(ex, "Transcribing a voice message in chat {ChatId} failed", chat.ChatId);
            await chat.Renderer.SayAsync($"Could not transcribe the voice message: {ex.Message}").ConfigureAwait(false);
            return;
        }
        if (transcript.Length == 0)
        {
            await chat.Renderer.SayAsync(NoWordsNotice).ConfigureAwait(false);
            return;
        }
        await chat.Renderer.SayAsync(Echo(transcript)).ConfigureAwait(false); // plain text
        draft.Body.Add($"[Voice message] {transcript}");
    }

    /// <summary>The <c>🎤 …</c> echo, cut to <see cref="MaxEchoLength"/> characters with <c>…</c> (the model gets all of it).</summary>
    internal static string Echo(string transcript)
    {
        var echo = "🎤 " + transcript;
        if (echo.Length <= MaxEchoLength) return echo;
        var cut = MaxEchoLength - 1;
        if (char.IsHighSurrogate(echo[cut - 1])) cut--; // don't split an emoji
        return echo[..cut] + "…";
    }

    /// <summary>
    /// The name an audio file is sent to speech to text under: OpenAI picks the format from the extension, so it's an
    /// ASCII <c>audio.&lt;ext&gt;</c> from the original extension when OpenAI knows it, else from the MIME type.
    /// </summary>
    internal static string SpeechFileName(string? fileName, string? mimeType)
    {
        var ext = Path.GetExtension(fileName ?? "").ToLowerInvariant();
        if (ext is ".flac" or ".m4a" or ".mp3" or ".mp4" or ".mpeg" or ".mpga" or ".ogg" or ".wav" or ".webm") return "audio" + ext;
        if (ext is ".oga" or ".opus") return "audio.ogg";
        return "audio" + (mimeType?.ToLowerInvariant() switch
        {
            "audio/mp4" or "audio/m4a" or "audio/x-m4a" => ".m4a",
            "audio/ogg" or "audio/opus" => ".ogg",
            "audio/wav" or "audio/x-wav" or "audio/wave" => ".wav",
            "audio/flac" or "audio/x-flac" => ".flac",
            "audio/webm" => ".webm",
            _ => ".mp3", // audio/mpeg, Telegram's default for audio
        });
    }

    /// <summary>The image type the leading bytes show (JPEG, PNG, GIF or WebP), or null.</summary>
    internal static string? SniffImageType(ReadOnlySpan<byte> data) => data switch
    {
        [0xFF, 0xD8, 0xFF, ..] => "image/jpeg",
        [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, ..] => "image/png",
        [0x47, 0x49, 0x46, 0x38, ..] => "image/gif",
        [0x52, 0x49, 0x46, 0x46, _, _, _, _, 0x57, 0x45, 0x42, 0x50, ..] => "image/webp",
        _ => null,
    };

    private void TryDelete(string path)
    {
        try { File.Delete(path); }
        catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
        {
            logger.LogDebug(ex, "Could not delete the partial file {Path}", path);
        }
    }
}