plugin system docs

custom buttons for plugins
calibration functions for Leeform
This commit is contained in:
EugeneTes
2026-03-30 16:15:47 +02:00
parent a1838cafeb
commit 4bd117af47
76 changed files with 4521 additions and 59 deletions

View File

@@ -0,0 +1,7 @@
<Application xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
x:Class="VisionBuilder.UI.RaspberryAcquisition.App">
<Application.Styles>
<FluentTheme />
</Application.Styles>
</Application>

View File

@@ -0,0 +1,28 @@
using Avalonia;
using Avalonia.Controls.ApplicationLifetimes;
using Avalonia.Markup.Xaml;
using VisionBuilder.UI.RaspberryAcquisition.ViewModels;
using VisionBuilder.UI.RaspberryAcquisition.Views;
namespace VisionBuilder.UI.RaspberryAcquisition;
public class App : Application
{
public override void Initialize()
{
AvaloniaXamlLoader.Load(this);
}
public override void OnFrameworkInitializationCompleted()
{
if (ApplicationLifetime is IClassicDesktopStyleApplicationLifetime desktop)
{
desktop.MainWindow = new MainWindow
{
DataContext = new MainWindowViewModel()
};
}
base.OnFrameworkInitializationCompleted();
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.9 MiB

View File

@@ -0,0 +1,10 @@
namespace VisionBuilder.UI.RaspberryAcquisition.Models;
public class CalibrationData
{
public double[] CameraMatrix { get; set; } = [];
public double[] DistCoeffs { get; set; } = [];
public double RmsError { get; set; }
public int ImageWidth { get; set; }
public int ImageHeight { get; set; }
}

View File

@@ -0,0 +1,18 @@
namespace VisionBuilder.UI.RaspberryAcquisition.Models;
public class CaptureSettings
{
public int Width { get; set; } = 2028;
public int Height { get; set; } = 1520;
public int ShutterSpeed { get; set; } = 10000;
public int Framerate { get; set; } = 2;
public double AwbGainRed { get; set; } = 3.86;
public double AwbGainBlue { get; set; } = 1.46;
public string BuildArguments()
{
return $"--codec mjpeg -t0 --width {Width} --height {Height} " +
$"--shutter {ShutterSpeed} --framerate {Framerate} " +
$"--awbgains {AwbGainRed:F2},{AwbGainBlue:F2} --nopreview -o -";
}
}

View File

@@ -0,0 +1,21 @@
using Avalonia;
using Avalonia.LinuxFramebuffer;
namespace VisionBuilder.UI.RaspberryAcquisition;
public class Program
{
[STAThread]
public static void Main(string[] args)
{
if (args.Contains("--drm"))
BuildAvaloniaApp().StartLinuxDrm(args, card: null, scaling: 1.0);
else
BuildAvaloniaApp().StartWithClassicDesktopLifetime(args);
}
public static AppBuilder BuildAvaloniaApp()
=> AppBuilder.Configure<App>()
.UsePlatformDetect()
.LogToTrace();
}

View File

@@ -0,0 +1,116 @@
using System.Text.Json;
using OpenCvSharp;
using VisionBuilder.UI.RaspberryAcquisition.Models;
namespace VisionBuilder.UI.RaspberryAcquisition.Services;
public static class CameraCalibrationService
{
private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true };
public static CalibrationData Calibrate(string imageDirectory, Size patternSize)
{
var imageFiles = Directory.GetFiles(imageDirectory, "*.png");
if (imageFiles.Length == 0)
throw new InvalidOperationException($"No PNG images found in {imageDirectory}");
var objectPointsList = new List<Mat>();
var imagePointsList = new List<Mat>();
Size imageSize = default;
// Build the 3D object points for the checkerboard (z=0 plane)
int cornerCount = patternSize.Width * patternSize.Height;
var objPts = new Point3f[cornerCount];
for (int row = 0; row < patternSize.Height; row++)
for (int col = 0; col < patternSize.Width; col++)
objPts[row * patternSize.Width + col] = new Point3f(col, row, 0);
int found = 0;
foreach (var file in imageFiles)
{
using var image = Cv2.ImRead(file);
using var gray = new Mat();
Cv2.CvtColor(image, gray, ColorConversionCodes.BGR2GRAY);
imageSize = new Size(image.Width, image.Height);
if (Cv2.FindChessboardCorners(gray, patternSize, out var corners,
ChessboardFlags.AdaptiveThresh | ChessboardFlags.FastCheck))
{
Cv2.CornerSubPix(gray, corners,
new Size(11, 11), new Size(-1, -1),
new TermCriteria(CriteriaTypes.Eps | CriteriaTypes.MaxIter, 30, 0.001));
// Convert to Mat for CalibrateCamera
var objMat = new Mat(cornerCount, 1, MatType.CV_32FC3);
for (int i = 0; i < cornerCount; i++)
objMat.Set(i, 0, objPts[i]);
objectPointsList.Add(objMat);
var imgMat = new Mat(corners.Length, 1, MatType.CV_32FC2);
for (int i = 0; i < corners.Length; i++)
imgMat.Set(i, 0, corners[i]);
imagePointsList.Add(imgMat);
found++;
}
}
if (found == 0)
throw new InvalidOperationException("Checkerboard corners not found in any image");
var cameraMatrix = new Mat();
var distCoeffs = new Mat();
double rms = Cv2.CalibrateCamera(
objectPointsList, imagePointsList, imageSize,
cameraMatrix, distCoeffs,
out _, out _);
var data = new CalibrationData
{
CameraMatrix = MatToArray(cameraMatrix),
DistCoeffs = MatToArray(distCoeffs),
RmsError = rms,
ImageWidth = imageSize.Width,
ImageHeight = imageSize.Height
};
cameraMatrix.Dispose();
distCoeffs.Dispose();
foreach (var m in objectPointsList) m.Dispose();
foreach (var m in imagePointsList) m.Dispose();
return data;
}
public static void SaveCalibration(CalibrationData data, string path)
{
var json = JsonSerializer.Serialize(data, JsonOptions);
File.WriteAllText(path, json);
}
public static (Mat cameraMatrix, Mat distCoeffs) LoadCalibration(string path)
{
var json = File.ReadAllText(path);
var data = JsonSerializer.Deserialize<CalibrationData>(json)
?? throw new InvalidOperationException("Failed to deserialize calibration data");
var cameraMatrix = new Mat(3, 3, MatType.CV_64F);
for (int i = 0; i < 9; i++)
cameraMatrix.Set(i / 3, i % 3, data.CameraMatrix[i]);
var distCoeffs = new Mat(1, data.DistCoeffs.Length, MatType.CV_64F);
for (int i = 0; i < data.DistCoeffs.Length; i++)
distCoeffs.Set(0, i, data.DistCoeffs[i]);
return (cameraMatrix, distCoeffs);
}
private static double[] MatToArray(Mat mat)
{
var total = mat.Rows * mat.Cols;
var arr = new double[total];
for (int i = 0; i < total; i++)
arr[i] = mat.At<double>(i / mat.Cols, i % mat.Cols);
return arr;
}
}

View File

@@ -0,0 +1,163 @@
using System.Diagnostics;
using System.Threading.Channels;
using OpenCvSharp;
using VisionBuilder.UI.Common.Processing;
using VisionBuilder.UI.RaspberryAcquisition.Models;
namespace VisionBuilder.UI.RaspberryAcquisition.Services;
public class ConfigurableLibcameraSource : IImageSource, IDisposable
{
private static readonly byte[] JpegHeader = [0xff, 0xd8];
private static readonly byte[] JpegFooter = [0xff, 0xd9];
private const int ChunkSize = 1024;
private readonly Channel<Mat> _imageChannel = Channel.CreateBounded<Mat>(
new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropOldest });
private CaptureSettings _settings;
private CancellationTokenSource? _cts;
private Thread? _captureThread;
public int FrameCount { get; private set; }
public event Action<string>? StatusChanged;
public ConfigurableLibcameraSource(CaptureSettings settings)
{
_settings = settings;
}
public Task<Mat> GetImage(CancellationToken token)
{
return _imageChannel.Reader.ReadAsync(token).AsTask();
}
public void Start()
{
Stop();
FrameCount = 0;
_cts = new CancellationTokenSource();
_captureThread = new Thread(CaptureLoop) { IsBackground = true };
_captureThread.Start();
StatusChanged?.Invoke("Running");
}
public void Stop()
{
_cts?.Cancel();
_captureThread?.Join(2000);
_cts?.Dispose();
_cts = null;
_captureThread = null;
StatusChanged?.Invoke("Stopped");
}
public void UpdateSettings(CaptureSettings newSettings)
{
_settings = newSettings;
Start();
}
private void CaptureLoop()
{
var psi = new ProcessStartInfo
{
FileName = "rpicam-vid",
Arguments = _settings.BuildArguments(),
RedirectStandardOutput = true,
UseShellExecute = false
};
Process? process = null;
try
{
process = Process.Start(psi);
if (process == null)
{
StatusChanged?.Invoke("Error: failed to start rpicam-vid");
return;
}
using var br = new BinaryReader(process.StandardOutput.BaseStream);
var imageBuffer = new byte[1024 * 1024];
var buff = br.ReadBytes(ChunkSize);
while (!_cts!.Token.IsCancellationRequested)
{
var imageStart = Find(buff, JpegHeader);
if (imageStart == -1)
{
StatusChanged?.Invoke("Error: JPEG header not found");
break;
}
var size = buff.Length - imageStart;
Array.Copy(buff, imageStart, imageBuffer, 0, size);
while (!_cts.Token.IsCancellationRequested)
{
buff = br.ReadBytes(ChunkSize);
var imageEnd = Find(buff, JpegFooter);
if (imageEnd != -1)
{
imageEnd += JpegFooter.Length;
var frame = new byte[size + imageEnd];
Array.Copy(imageBuffer, frame, size);
Array.Copy(buff, 0, frame, size, imageEnd);
var decoded = Mat.ImDecode(frame);
FrameCount++;
_imageChannel.Writer.TryWrite(decoded);
var remaining = buff.Length - imageEnd;
var newBuff = new byte[ChunkSize];
Array.Copy(buff, imageEnd, newBuff, 0, remaining);
var temp = br.ReadBytes(imageEnd);
Array.Copy(temp, 0, newBuff, remaining, temp.Length);
buff = newBuff;
break;
}
Array.Copy(buff, 0, imageBuffer, size, buff.Length);
size += buff.Length;
}
}
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
StatusChanged?.Invoke($"Error: {ex.Message}");
}
finally
{
if (process is { HasExited: false })
{
process.Kill();
process.Dispose();
}
}
}
private static int Find(byte[] buff, byte[] search)
{
for (int start = 0; start <= buff.Length - search.Length; start++)
{
if (buff[start] == search[0])
{
int next;
for (next = 1; next < search.Length; next++)
{
if (buff[start + next] != search[next])
break;
}
if (next == search.Length)
return start;
}
}
return -1;
}
public void Dispose()
{
Stop();
}
}

View File

@@ -0,0 +1,13 @@
using OpenCvSharp;
namespace VisionBuilder.UI.RaspberryAcquisition.Services;
public static class ImageConverter
{
public static Avalonia.Media.Imaging.Bitmap MatToAvaloniaBitmap(Mat mat)
{
var encoded = mat.ImEncode(".bmp");
using var ms = new MemoryStream(encoded);
return new Avalonia.Media.Imaging.Bitmap(ms);
}
}

View File

@@ -0,0 +1,206 @@
using System.Globalization;
using Avalonia.Threading;
using CommunityToolkit.Mvvm.ComponentModel;
using CommunityToolkit.Mvvm.Input;
using OpenCvSharp;
using VisionBuilder.UI.RaspberryAcquisition.Models;
using VisionBuilder.UI.RaspberryAcquisition.Services;
namespace VisionBuilder.UI.RaspberryAcquisition.ViewModels;
public partial class MainWindowViewModel : ObservableObject, IDisposable
{
private const string CalibrationFile = "calibration.json";
private ConfigurableLibcameraSource? _source;
private CancellationTokenSource? _previewCts;
private Mat? _lastFrame;
private Mat? _cameraMatrix;
private Mat? _distCoeffs;
[ObservableProperty] private Avalonia.Media.Imaging.Bitmap? _previewImage;
[ObservableProperty] private string _statusText = "Stopped";
[ObservableProperty] private int _frameCount;
[ObservableProperty] private bool _isRunning;
[ObservableProperty] private bool _isCalibrated;
[ObservableProperty] private string _width = "2028";
[ObservableProperty] private string _height = "1520";
[ObservableProperty] private string _shutterSpeed = "10000";
[ObservableProperty] private string _framerate = "2";
[ObservableProperty] private string _awbGainRed = "3.86";
[ObservableProperty] private string _awbGainBlue = "1.46";
[ObservableProperty] private string _saveDirectory = "/home/oem/captures";
[ObservableProperty] private string _calibrationDirectory = "/home/oem/captures";
public MainWindowViewModel()
{
TryLoadCalibration();
}
private void TryLoadCalibration()
{
if (!File.Exists(CalibrationFile)) return;
try
{
(_cameraMatrix, _distCoeffs) = CameraCalibrationService.LoadCalibration(CalibrationFile);
IsCalibrated = true;
StatusText = "Calibration loaded";
}
catch (Exception ex)
{
StatusText = $"Calibration load error: {ex.Message}";
}
}
[RelayCommand]
private void StartCapture()
{
if (IsRunning) return;
var settings = BuildSettings();
if (settings == null) return;
_source = new ConfigurableLibcameraSource(settings);
_source.StatusChanged += status =>
Dispatcher.UIThread.Post(() => StatusText = status);
_source.Start();
IsRunning = true;
_previewCts = new CancellationTokenSource();
_ = PreviewLoopAsync(_previewCts.Token);
}
[RelayCommand]
private void StopCapture()
{
if (!IsRunning) return;
_previewCts?.Cancel();
_source?.Stop();
_source?.Dispose();
_source = null;
IsRunning = false;
StatusText = "Stopped";
}
[RelayCommand]
private void ApplySettings()
{
if (!IsRunning || _source == null) return;
var settings = BuildSettings();
if (settings == null) return;
_source.UpdateSettings(settings);
}
[RelayCommand]
private void SaveImage()
{
if (_lastFrame == null || _lastFrame.Empty()) return;
Directory.CreateDirectory(SaveDirectory);
var filename = $"capture_{DateTime.Now:yyyyMMdd_HHmmss}.png";
var path = Path.Combine(SaveDirectory, filename);
Cv2.ImWrite(path, _lastFrame);
StatusText = $"Saved: {filename}";
}
[RelayCommand]
private void CalibrateCamera()
{
try
{
StatusText = "Calibrating...";
var data = CameraCalibrationService.Calibrate(CalibrationDirectory, new Size(9, 6));
CameraCalibrationService.SaveCalibration(data, CalibrationFile);
_cameraMatrix?.Dispose();
_distCoeffs?.Dispose();
(_cameraMatrix, _distCoeffs) = CameraCalibrationService.LoadCalibration(CalibrationFile);
IsCalibrated = true;
StatusText = $"Calibrated (RMS: {data.RmsError:F4})";
}
catch (Exception ex)
{
StatusText = $"Calibration error: {ex.Message}";
}
}
private async Task PreviewLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested && _source != null)
{
try
{
var frame = await _source.GetImage(ct);
if (_cameraMatrix != null && _distCoeffs != null)
{
var corrected = new Mat();
Cv2.Undistort(frame, corrected, _cameraMatrix, _distCoeffs);
frame.Dispose();
frame = corrected;
}
_lastFrame?.Dispose();
_lastFrame = frame;
var bitmap = ImageConverter.MatToAvaloniaBitmap(frame);
await Dispatcher.UIThread.InvokeAsync(() =>
{
var old = PreviewImage;
PreviewImage = bitmap;
FrameCount = _source?.FrameCount ?? 0;
old?.Dispose();
});
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
await Dispatcher.UIThread.InvokeAsync(() =>
StatusText = $"Preview error: {ex.Message}");
break;
}
}
}
private CaptureSettings? BuildSettings()
{
if (!int.TryParse(Width, out var w) ||
!int.TryParse(Height, out var h) ||
!int.TryParse(ShutterSpeed, out var s) ||
!int.TryParse(Framerate, out var f) ||
!double.TryParse(AwbGainRed, CultureInfo.InvariantCulture, out var r) ||
!double.TryParse(AwbGainBlue, CultureInfo.InvariantCulture, out var b))
{
StatusText = "Error: invalid settings values";
return null;
}
return new CaptureSettings
{
Width = w,
Height = h,
ShutterSpeed = s,
Framerate = f,
AwbGainRed = r,
AwbGainBlue = b
};
}
public void Dispose()
{
StopCapture();
_lastFrame?.Dispose();
_cameraMatrix?.Dispose();
_distCoeffs?.Dispose();
}
}

View File

@@ -0,0 +1,78 @@
<Window xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:vm="using:VisionBuilder.UI.RaspberryAcquisition.ViewModels"
x:Class="VisionBuilder.UI.RaspberryAcquisition.Views.MainWindow"
x:DataType="vm:MainWindowViewModel"
Title="RPi Image Acquisition"
Width="1024" Height="800">
<DockPanel>
<!-- Toolbar -->
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Spacing="8" Margin="8">
<Button Content="Start" Command="{Binding StartCaptureCommand}" IsEnabled="{Binding !IsRunning}" />
<Button Content="Stop" Command="{Binding StopCaptureCommand}" IsEnabled="{Binding IsRunning}" />
<Button Content="Save Image" Command="{Binding SaveImageCommand}" />
<Button Content="Apply Settings" Command="{Binding ApplySettingsCommand}" IsEnabled="{Binding IsRunning}" />
</StackPanel>
<!-- Status Bar -->
<Border DockPanel.Dock="Bottom" Background="#1E1E1E" Padding="8,4">
<StackPanel Orientation="Horizontal" Spacing="16">
<TextBlock Text="{Binding StatusText}" Foreground="LightGray" />
<TextBlock Foreground="LightGray">
<TextBlock.Text>
<MultiBinding StringFormat="Frames: {0}">
<Binding Path="FrameCount" />
</MultiBinding>
</TextBlock.Text>
</TextBlock>
</StackPanel>
</Border>
<!-- Settings Panel -->
<Border DockPanel.Dock="Right" Width="220" Padding="12" Background="#2D2D2D">
<ScrollViewer>
<StackPanel Spacing="8">
<TextBlock Text="Camera Settings" FontWeight="Bold" FontSize="14" Foreground="White" />
<TextBlock Text="Width" Foreground="LightGray" />
<TextBox Text="{Binding Width}" />
<TextBlock Text="Height" Foreground="LightGray" />
<TextBox Text="{Binding Height}" />
<TextBlock Text="Shutter (µs)" Foreground="LightGray" />
<TextBox Text="{Binding ShutterSpeed}" />
<TextBlock Text="Framerate" Foreground="LightGray" />
<TextBox Text="{Binding Framerate}" />
<TextBlock Text="AWB Gain Red" Foreground="LightGray" />
<TextBox Text="{Binding AwbGainRed}" />
<TextBlock Text="AWB Gain Blue" Foreground="LightGray" />
<TextBox Text="{Binding AwbGainBlue}" />
<Separator Margin="0,8" />
<TextBlock Text="Save Directory" Foreground="LightGray" />
<TextBox Text="{Binding SaveDirectory}" />
<Separator Margin="0,8" />
<TextBlock Text="Calibration" FontWeight="Bold" FontSize="14" Foreground="White" />
<TextBlock Text="Calibration Dir" Foreground="LightGray" />
<TextBox Text="{Binding CalibrationDirectory}" />
<Button Content="Calibrate" Command="{Binding CalibrateCameraCommand}" Margin="0,4" />
<TextBlock Text="Calibrated" Foreground="LimeGreen" IsVisible="{Binding IsCalibrated}" />
</StackPanel>
</ScrollViewer>
</Border>
<!-- Live Preview -->
<Border Background="#1A1A1A" Margin="4">
<Image Source="{Binding PreviewImage}" Stretch="Uniform" />
</Border>
</DockPanel>
</Window>

View File

@@ -0,0 +1,11 @@
using Avalonia.Controls;
namespace VisionBuilder.UI.RaspberryAcquisition.Views;
public partial class MainWindow : Window
{
public MainWindow()
{
InitializeComponent();
}
}

View File

@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<RuntimeIdentifiers>linux-arm64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Avalonia" Version="11.2.3" />
<PackageReference Include="Avalonia.Themes.Fluent" Version="11.2.3" />
<PackageReference Include="Avalonia.LinuxFramebuffer" Version="11.2.3" />
<PackageReference Include="Avalonia.Desktop" Version="11.2.3" />
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.4.0" />
<PackageReference Include="OpenCvSharp4" Version="4.10.0.20241108" />
<PackageReference Include="swety.OpenCvSharp4.runtime.linux-arm64" Version="4.10.0.20241108" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VisionBuilder.UI.Common\VisionBuilder.UI.Common.csproj" />
</ItemGroup>
</Project>