plugin system docs

custom buttons for plugins
calibration functions for Leeform

Plugins/LindtLeerformPlugin/LeerformModule.cs

@@ -0,0 +1,61 @@
+using Avalonia;
+using Avalonia.Controls;
+using Avalonia.Controls.ApplicationLifetimes;
+using LindtLeerformPlugin.ViewModels;
+using LindtLeerformPlugin.Views;
+using Serilog;
+using VisionBuilder.UI.Common;
+using VisionBuilder.UI.Common.Processing;
+using VisionBuilder.UI.Common.ViewModel.Classes;
+
+namespace LindtLeerformPlugin;
+
+public class LeerformModule : IVisionBuilderModule
+{
+    private readonly SingleCameraVM _singleCameraVm;
+    private readonly LeerformSettings _settings;
+    private readonly IImageSource _imageSource;
+
+    public LeerformModule(SingleCameraVM singleCameraVm, LeerformSettings settings, IImageSource imageSource)
+    {
+        _singleCameraVm = singleCameraVm;
+        _settings = settings;
+        _imageSource = imageSource;
+    }
+
+    public void InitializeModule()
+    {
+        var calibrateButton = new ButtonDefinition("leerform_calibrate", "Calibrate", OnCalibrateClicked);
+        _singleCameraVm.AddButton(calibrateButton);
+    }
+
+    private void OnCalibrateClicked()
+    {
+        _ = ShowCalibrationWindowAsync();
+    }
+
+    private async Task ShowCalibrationWindowAsync()
+    {
+        try
+        {
+            if (Application.Current?.ApplicationLifetime is not IClassicDesktopStyleApplicationLifetime desktop)
+            {
+                Log.Warning("Avalonia desktop lifetime not available, cannot show calibration window.");
+                return;
+            }
+
+            var parent = desktop.MainWindow;
+            var vm = new CalibrationWindowViewModel(_imageSource, _settings);
+            var window = new CalibrationWindow { DataContext = vm };
+
+            if (parent != null)
+                await window.ShowDialog(parent);
+            else
+                window.Show();
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, "Error showing calibration window");
+        }
+    }
+}

Plugins/LindtLeerformPlugin/LeerformSettings.cs

@@ -0,0 +1,65 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Inspectron.Settings;
+using LindtLeerformPlugin.Models;
+using VisionBuilder.UI.Common;
+
+namespace LindtLeerformPlugin;
+
+public class LeerformSettings : ISettings
+{
+    public string CameraName { get; }
+
+    public LeerformSettings(string cameraName)
+    {
+        CameraName = cameraName;
+    }
+
+    public CalibrationData? CalibrationData { get; set; }
+    public int CheckerboardRows { get; set; } = 6;
+    public int CheckerboardCols { get; set; } = 9;
+    public string CalibrationImageDirectory { get; set; } = "CalibrationImages";
+
+    public void RegisterSettings(InspectronSettings settings)
+    {
+        settings.RegisterSimple(this, () => CalibrationData!, $"{CameraName}/Leerform", nameof(CalibrationData));
+        settings.RegisterSimple(this, () => CheckerboardRows, $"{CameraName}/Leerform", nameof(CheckerboardRows));
+        settings.RegisterSimple(this, () => CheckerboardCols, $"{CameraName}/Leerform", nameof(CheckerboardCols));
+        settings.RegisterSimple(this, () => CalibrationImageDirectory, $"{CameraName}/Leerform", nameof(CalibrationImageDirectory));
+    }
+}
+
+public class CalibrationDataConverter : ITypeConverter
+{
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        PropertyNameCaseInsensitive = true,
+        Converters = { new JsonStringEnumConverter() }
+    };
+
+    public object ConvertFrom(object value)
+    {
+        if (value is string json)
+        {
+            if (string.IsNullOrWhiteSpace(json) || json == "null")
+                return null!;
+
+            return JsonSerializer.Deserialize<CalibrationData>(json, JsonOptions)
+                   ?? throw new InvalidOperationException("Failed to deserialize CalibrationData from JSON.");
+        }
+        throw new InvalidOperationException("Value must be a JSON string.");
+    }
+
+    public object ConvertTo(object value, Type destinationType)
+    {
+        if (value is CalibrationData data)
+        {
+            return JsonSerializer.Serialize(data, JsonOptions);
+        }
+        if (value == null)
+        {
+            return "null";
+        }
+        throw new InvalidOperationException("Value must be a CalibrationData.");
+    }
+}

Plugins/LindtLeerformPlugin/LindtLeerformPlugin.csproj

@@ -0,0 +1,45 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <EnableDynamicLoading>true</EnableDynamicLoading>
+    <OutDir>..\..\VisionBuilder.UI.Avalonia.Uno\bin\Debug\Data\Plugins\LindtLeerformPlugin</OutDir>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\VisionBuilder.UI.Common\VisionBuilder.UI.Common.csproj">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </ProjectReference>
+    <ProjectReference Include="..\..\framework\Inspectron.Settings\Inspectron.Settings.csproj">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </ProjectReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Avalonia" Version="11.2.3">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Avalonia.Desktop" Version="11.2.3">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Avalonia.Themes.Fluent" Version="11.2.3">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="CommunityToolkit.Mvvm" Version="8.4.0">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="OpenCvSharp4" Version="4.10.0.20241108">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+</Project>

Plugins/LindtLeerformPlugin/Models/CalibrationData.cs

@@ -0,0 +1,10 @@
+namespace LindtLeerformPlugin.Models;
+
+public class CalibrationData
+{
+    public double[] CameraMatrix { get; set; } = [];
+    public double[] DistCoeffs { get; set; } = [];
+    public double RmsError { get; set; }
+    public int ImageWidth { get; set; }
+    public int ImageHeight { get; set; }
+}

Plugins/LindtLeerformPlugin/Plugin.cs

@@ -0,0 +1,21 @@
+using Inspectron.Settings;
+using LindtLeerformPlugin.Models;
+using Ninject;
+using VisionBuilder.UI.Common;
+using VisionBuilder.UI.Common.Plugins;
+
+namespace LindtLeerformPlugin;
+
+public class Plugin : IPlugin
+{
+    public void RegisterGlobalModules(IKernel kernel)
+    {
+        TypeConverterRegistry.Register<CalibrationData>(new CalibrationDataConverter());
+    }
+
+    public void RegisterCameraModules(IKernel kernel, string cameraName)
+    {
+        kernel.Bind<LeerformSettings, ISettings>().ToConstant(new LeerformSettings(cameraName));
+        kernel.RegisterModule<LeerformModule>();
+    }
+}

Plugins/LindtLeerformPlugin/Services/CameraCalibrationService.cs

@@ -0,0 +1,101 @@
+using LindtLeerformPlugin.Models;
+using OpenCvSharp;
+
+namespace LindtLeerformPlugin.Services;
+
+public static class CameraCalibrationService
+{
+    public static CalibrationData Calibrate(string imageDirectory, Size patternSize)
+    {
+        var imageFiles = Directory.GetFiles(imageDirectory, "*.png");
+        if (imageFiles.Length == 0)
+            throw new InvalidOperationException($"No PNG images found in {imageDirectory}");
+
+        var objectPointsList = new List<Mat>();
+        var imagePointsList = new List<Mat>();
+        Size imageSize = default;
+
+        int cornerCount = patternSize.Width * patternSize.Height;
+        var objPts = new Point3f[cornerCount];
+        for (int row = 0; row < patternSize.Height; row++)
+            for (int col = 0; col < patternSize.Width; col++)
+                objPts[row * patternSize.Width + col] = new Point3f(col, row, 0);
+
+        int found = 0;
+        foreach (var file in imageFiles)
+        {
+            using var image = Cv2.ImRead(file);
+            using var gray = new Mat();
+            Cv2.CvtColor(image, gray, ColorConversionCodes.BGR2GRAY);
+            imageSize = new Size(image.Width, image.Height);
+
+            if (Cv2.FindChessboardCorners(gray, patternSize, out var corners,
+                    ChessboardFlags.AdaptiveThresh | ChessboardFlags.FastCheck))
+            {
+                Cv2.CornerSubPix(gray, corners,
+                    new Size(11, 11), new Size(-1, -1),
+                    new TermCriteria(CriteriaTypes.Eps | CriteriaTypes.MaxIter, 30, 0.001));
+
+                var objMat = new Mat(cornerCount, 1, MatType.CV_32FC3);
+                for (int i = 0; i < cornerCount; i++)
+                    objMat.Set(i, 0, objPts[i]);
+                objectPointsList.Add(objMat);
+
+                var imgMat = new Mat(corners.Length, 1, MatType.CV_32FC2);
+                for (int i = 0; i < corners.Length; i++)
+                    imgMat.Set(i, 0, corners[i]);
+                imagePointsList.Add(imgMat);
+
+                found++;
+            }
+        }
+
+        if (found == 0)
+            throw new InvalidOperationException("Checkerboard corners not found in any image");
+
+        var cameraMatrix = new Mat();
+        var distCoeffs = new Mat();
+        double rms = Cv2.CalibrateCamera(
+            objectPointsList, imagePointsList, imageSize,
+            cameraMatrix, distCoeffs,
+            out _, out _);
+
+        var data = new CalibrationData
+        {
+            CameraMatrix = MatToArray(cameraMatrix),
+            DistCoeffs = MatToArray(distCoeffs),
+            RmsError = rms,
+            ImageWidth = imageSize.Width,
+            ImageHeight = imageSize.Height
+        };
+
+        cameraMatrix.Dispose();
+        distCoeffs.Dispose();
+        foreach (var m in objectPointsList) m.Dispose();
+        foreach (var m in imagePointsList) m.Dispose();
+
+        return data;
+    }
+
+    public static (Mat cameraMatrix, Mat distCoeffs) LoadCalibrationMats(CalibrationData data)
+    {
+        var cameraMatrix = new Mat(3, 3, MatType.CV_64F);
+        for (int i = 0; i < 9; i++)
+            cameraMatrix.Set(i / 3, i % 3, data.CameraMatrix[i]);
+
+        var distCoeffs = new Mat(1, data.DistCoeffs.Length, MatType.CV_64F);
+        for (int i = 0; i < data.DistCoeffs.Length; i++)
+            distCoeffs.Set(0, i, data.DistCoeffs[i]);
+
+        return (cameraMatrix, distCoeffs);
+    }
+
+    private static double[] MatToArray(Mat mat)
+    {
+        var total = mat.Rows * mat.Cols;
+        var arr = new double[total];
+        for (int i = 0; i < total; i++)
+            arr[i] = mat.At<double>(i / mat.Cols, i % mat.Cols);
+        return arr;
+    }
+}

Plugins/LindtLeerformPlugin/Services/ImageConverter.cs

@@ -0,0 +1,13 @@
+using OpenCvSharp;
+
+namespace LindtLeerformPlugin.Services;
+
+public static class ImageConverter
+{
+    public static Avalonia.Media.Imaging.Bitmap MatToAvaloniaBitmap(Mat mat)
+    {
+        var encoded = mat.ImEncode(".bmp");
+        using var ms = new MemoryStream(encoded);
+        return new Avalonia.Media.Imaging.Bitmap(ms);
+    }
+}

Plugins/LindtLeerformPlugin/ViewModels/CalibrationWindowViewModel.cs

@@ -0,0 +1,151 @@
+using Avalonia.Threading;
+using CommunityToolkit.Mvvm.ComponentModel;
+using CommunityToolkit.Mvvm.Input;
+using LindtLeerformPlugin.Models;
+using LindtLeerformPlugin.Services;
+using OpenCvSharp;
+using VisionBuilder.UI.Common.Processing;
+
+namespace LindtLeerformPlugin.ViewModels;
+
+public partial class CalibrationWindowViewModel : ObservableObject, IDisposable
+{
+    private readonly IImageSource _imageSource;
+    private readonly LeerformSettings _settings;
+    private CancellationTokenSource? _previewCts;
+    private Mat? _lastFrame;
+
+    [ObservableProperty] private Avalonia.Media.Imaging.Bitmap? _previewImage;
+    [ObservableProperty] private string _statusText = "Ready";
+    [ObservableProperty] private int _capturedImageCount;
+    [ObservableProperty] private bool _isPreviewRunning;
+    [ObservableProperty] private bool _isCalibrated;
+    [ObservableProperty] private string _calibrationImageDirectory;
+    [ObservableProperty] private double _rmsError;
+
+    public CalibrationWindowViewModel(IImageSource imageSource, LeerformSettings settings)
+    {
+        _imageSource = imageSource;
+        _settings = settings;
+
+        _calibrationImageDirectory = settings.CalibrationImageDirectory;
+        _isCalibrated = settings.CalibrationData is { CameraMatrix.Length: > 0 };
+        _rmsError = settings.CalibrationData?.RmsError ?? 0;
+
+        UpdateCapturedImageCount();
+    }
+
+    private void UpdateCapturedImageCount()
+    {
+        if (Directory.Exists(CalibrationImageDirectory))
+            CapturedImageCount = Directory.GetFiles(CalibrationImageDirectory, "*.png").Length;
+        else
+            CapturedImageCount = 0;
+    }
+
+    [RelayCommand]
+    private void StartPreview()
+    {
+        if (IsPreviewRunning) return;
+
+        IsPreviewRunning = true;
+        _previewCts = new CancellationTokenSource();
+        _ = PreviewLoopAsync(_previewCts.Token);
+        StatusText = "Preview running";
+    }
+
+    [RelayCommand]
+    private void StopPreview()
+    {
+        if (!IsPreviewRunning) return;
+
+        _previewCts?.Cancel();
+        IsPreviewRunning = false;
+        StatusText = "Preview stopped";
+    }
+
+    [RelayCommand]
+    private void CaptureImage()
+    {
+        if (_lastFrame == null || _lastFrame.Empty()) return;
+
+        Directory.CreateDirectory(CalibrationImageDirectory);
+        var filename = $"calib_{DateTime.Now:yyyyMMdd_HHmmss_fff}.png";
+        var path = Path.Combine(CalibrationImageDirectory, filename);
+        Cv2.ImWrite(path, _lastFrame);
+        UpdateCapturedImageCount();
+        StatusText = $"Captured: {filename}";
+    }
+
+    [RelayCommand]
+    private void RunCalibration()
+    {
+        try
+        {
+            StatusText = "Calibrating...";
+            var patternSize = new Size(_settings.CheckerboardCols, _settings.CheckerboardRows);
+            var data = CameraCalibrationService.Calibrate(CalibrationImageDirectory, patternSize);
+
+            _settings.CalibrationData = data;
+            _settings.CalibrationImageDirectory = CalibrationImageDirectory;
+
+            IsCalibrated = true;
+            RmsError = data.RmsError;
+            StatusText = $"Calibrated (RMS: {data.RmsError:F4})";
+        }
+        catch (Exception ex)
+        {
+            StatusText = $"Calibration error: {ex.Message}";
+        }
+    }
+
+    [RelayCommand]
+    private void ClearCalibrationImages()
+    {
+        if (!Directory.Exists(CalibrationImageDirectory)) return;
+
+        foreach (var file in Directory.GetFiles(CalibrationImageDirectory, "*.png"))
+            File.Delete(file);
+
+        UpdateCapturedImageCount();
+        StatusText = "Calibration images cleared";
+    }
+
+    private async Task PreviewLoopAsync(CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                var frame = await _imageSource.GetImage(ct);
+
+                _lastFrame?.Dispose();
+                _lastFrame = frame;
+
+                var bitmap = ImageConverter.MatToAvaloniaBitmap(frame);
+                await Dispatcher.UIThread.InvokeAsync(() =>
+                {
+                    var old = PreviewImage;
+                    PreviewImage = bitmap;
+                    old?.Dispose();
+                });
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                await Dispatcher.UIThread.InvokeAsync(() =>
+                    StatusText = $"Preview error: {ex.Message}");
+                break;
+            }
+        }
+    }
+
+    public void Dispose()
+    {
+        StopPreview();
+        _lastFrame?.Dispose();
+    }
+}

Plugins/LindtLeerformPlugin/Views/CalibrationWindow.axaml

@@ -0,0 +1,67 @@
+<Window xmlns="https://github.com/avaloniaui"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:vm="using:LindtLeerformPlugin.ViewModels"
+        x:Class="LindtLeerformPlugin.Views.CalibrationWindow"
+        x:DataType="vm:CalibrationWindowViewModel"
+        Title="Leerform Calibration"
+        Width="1024" Height="800">
+
+    <DockPanel>
+        <!-- Toolbar -->
+        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Spacing="8" Margin="8">
+            <Button Content="Start Preview" Command="{Binding StartPreviewCommand}" IsEnabled="{Binding !IsPreviewRunning}" />
+            <Button Content="Stop Preview" Command="{Binding StopPreviewCommand}" IsEnabled="{Binding IsPreviewRunning}" />
+            <Button Content="Capture Image" Command="{Binding CaptureImageCommand}" />
+        </StackPanel>
+
+        <!-- Status Bar -->
+        <Border DockPanel.Dock="Bottom" Background="#1E1E1E" Padding="8,4">
+            <StackPanel Orientation="Horizontal" Spacing="16">
+                <TextBlock Text="{Binding StatusText}" Foreground="LightGray" />
+                <TextBlock Foreground="LightGray">
+                    <TextBlock.Text>
+                        <MultiBinding StringFormat="Captured: {0}">
+                            <Binding Path="CapturedImageCount" />
+                        </MultiBinding>
+                    </TextBlock.Text>
+                </TextBlock>
+            </StackPanel>
+        </Border>
+
+        <!-- Settings Panel -->
+        <Border DockPanel.Dock="Right" Width="220" Padding="12" Background="#2D2D2D">
+            <ScrollViewer>
+                <StackPanel Spacing="8">
+                    <TextBlock Text="Calibration Settings" FontWeight="Bold" FontSize="14" Foreground="White" />
+
+                    <TextBlock Text="Image Directory" Foreground="LightGray" />
+                    <TextBox Text="{Binding CalibrationImageDirectory}" />
+
+                    <Separator Margin="0,8" />
+
+                    <Button Content="Calibrate" Command="{Binding RunCalibrationCommand}" Margin="0,4"
+                            Background="#00C853" Foreground="White" FontWeight="Bold" />
+                    <Button Content="Clear Images" Command="{Binding ClearCalibrationImagesCommand}" Margin="0,4"
+                            Background="#FF6D00" Foreground="White" FontWeight="Bold" />
+
+                    <Separator Margin="0,8" />
+
+                    <TextBlock Text="Calibrated" Foreground="LimeGreen" IsVisible="{Binding IsCalibrated}" />
+                    <TextBlock Foreground="LightGray" IsVisible="{Binding IsCalibrated}">
+                        <TextBlock.Text>
+                            <MultiBinding StringFormat="RMS Error: {0:F4}">
+                                <Binding Path="RmsError" />
+                            </MultiBinding>
+                        </TextBlock.Text>
+                    </TextBlock>
+                </StackPanel>
+            </ScrollViewer>
+        </Border>
+
+        <!-- Live Preview -->
+        <Border Background="#1A1A1A" Margin="4">
+            <Image Source="{Binding PreviewImage}" Stretch="Uniform" />
+        </Border>
+    </DockPanel>
+
+</Window>

Plugins/LindtLeerformPlugin/Views/CalibrationWindow.axaml.cs

@@ -0,0 +1,18 @@
+using Avalonia.Controls;
+using LindtLeerformPlugin.ViewModels;
+
+namespace LindtLeerformPlugin.Views;
+
+public partial class CalibrationWindow : Window
+{
+    public CalibrationWindow()
+    {
+        InitializeComponent();
+    }
+
+    protected override void OnClosing(WindowClosingEventArgs e)
+    {
+        (DataContext as CalibrationWindowViewModel)?.Dispose();
+        base.OnClosing(e);
+    }
+}