diff --git a/CLAUDE.md b/CLAUDE.md index 37d1bc9..8be0db2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,6 +68,8 @@ Plugins implement `IPlugin` with two registration points: Modules implement `IVisionBuilderModule.InitializeModule()` for deferred initialization. +For detailed plugin architecture documentation and step-by-step implementation guide, see [`docs/PLUGIN_SYSTEM.md`](docs/PLUGIN_SYSTEM.md). + ### DI Container Ninject `StandardKernel` is the root container. Child kernels (`Ninject.Extensions.ChildKernel`) scope per-camera services. All service resolution flows through the kernel — avoid `new` for services. diff --git a/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/CaptureSettings.cs b/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/CaptureSettings.cs new file mode 100644 index 0000000..01be2d0 --- /dev/null +++ b/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/CaptureSettings.cs @@ -0,0 +1,38 @@ +using Inspectron.Settings; +using VisionBuilder.UI.Common; + +namespace Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera; + +public class CaptureSettings : ISettings +{ + public string CameraName { get; } + + public CaptureSettings(string cameraName) + { + CameraName = cameraName; + } + + public int Width { get; set; } = 2028; + public int Height { get; set; } = 1520; + public int ShutterSpeed { get; set; } = 10000; + public int Framerate { get; set; } = 2; + public double AwbGainRed { get; set; } = 3.86; + public double AwbGainBlue { get; set; } = 1.46; + + public string BuildArguments() + { + return $"--codec mjpeg -t0 --width {Width} --height {Height} " + + $"--shutter {ShutterSpeed} --framerate {Framerate} " + + $"--awbgains {AwbGainRed:F2},{AwbGainBlue:F2} --nopreview -o -"; + } + + public void RegisterSettings(InspectronSettings settings) + { + settings.RegisterSimple(this, () => Width, CameraName + "/Sources/Libcamera", nameof(Width)); + settings.RegisterSimple(this, () => Height, CameraName + "/Sources/Libcamera", nameof(Height)); + settings.RegisterSimple(this, () => ShutterSpeed, CameraName + "/Sources/Libcamera", nameof(ShutterSpeed)); + settings.RegisterSimple(this, () => Framerate, CameraName + "/Sources/Libcamera", nameof(Framerate)); + settings.RegisterSimple(this, () => AwbGainRed, CameraName + "/Sources/Libcamera", nameof(AwbGainRed)); + settings.RegisterSimple(this, () => AwbGainBlue, CameraName + "/Sources/Libcamera", nameof(AwbGainBlue)); + } +} diff --git a/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/LibcameraImageSource.cs b/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/LibcameraImageSource.cs index e046a8d..f1df722 100644 --- a/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/LibcameraImageSource.cs +++ b/Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera/LibcameraImageSource.cs @@ -8,8 +8,12 @@ namespace Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera { public class LibcameraImageSource : IImageSource, IVisionBuilderModule { + private CaptureSettings _settings; - + public LibcameraImageSource(CaptureSettings settings) + { + _settings = settings; + } public async Task GetImage(CancellationToken token) { @@ -25,15 +29,6 @@ namespace Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera byte[] JpegFooter = new byte[] { 0xff, 0xd9 }; int ChunkSize = 1024; - ProcessStartInfo psi = new ProcessStartInfo - { - FileName = "libcamera-vid", - Arguments = "--codec mjpeg -t0 --width 2028 --height 1520 --shutter 10000 --framerate 16 --awbgains 3.86,1.46 --nopreview -o -", - - RedirectStandardOutput = true, - UseShellExecute = false - }; - Channel _imageChannel = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropOldest }); public void Start() @@ -46,6 +41,14 @@ namespace Hawkeye.VisionBuilder.UI.Sources.Raspberry.Libcamera private void StartLoop() { + var psi = new ProcessStartInfo + { + FileName = "rpicam-vid", + Arguments = _settings.BuildArguments(), + RedirectStandardOutput = true, + UseShellExecute = false + }; + using (Process process = Process.Start(psi)) using (BinaryReader br = new BinaryReader(process.StandardOutput.BaseStream)) { diff --git a/Plugins/LindtLeerformPlugin/LeerformModule.cs b/Plugins/LindtLeerformPlugin/LeerformModule.cs new file mode 100644 index 0000000..dfb9f83 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/LeerformModule.cs @@ -0,0 +1,61 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Controls.ApplicationLifetimes; +using LindtLeerformPlugin.ViewModels; +using LindtLeerformPlugin.Views; +using Serilog; +using VisionBuilder.UI.Common; +using VisionBuilder.UI.Common.Processing; +using VisionBuilder.UI.Common.ViewModel.Classes; + +namespace LindtLeerformPlugin; + +public class LeerformModule : IVisionBuilderModule +{ + private readonly SingleCameraVM _singleCameraVm; + private readonly LeerformSettings _settings; + private readonly IImageSource _imageSource; + + public LeerformModule(SingleCameraVM singleCameraVm, LeerformSettings settings, IImageSource imageSource) + { + _singleCameraVm = singleCameraVm; + _settings = settings; + _imageSource = imageSource; + } + + public void InitializeModule() + { + var calibrateButton = new ButtonDefinition("leerform_calibrate", "Calibrate", OnCalibrateClicked); + _singleCameraVm.AddButton(calibrateButton); + } + + private void OnCalibrateClicked() + { + _ = ShowCalibrationWindowAsync(); + } + + private async Task ShowCalibrationWindowAsync() + { + try + { + if (Application.Current?.ApplicationLifetime is not IClassicDesktopStyleApplicationLifetime desktop) + { + Log.Warning("Avalonia desktop lifetime not available, cannot show calibration window."); + return; + } + + var parent = desktop.MainWindow; + var vm = new CalibrationWindowViewModel(_imageSource, _settings); + var window = new CalibrationWindow { DataContext = vm }; + + if (parent != null) + await window.ShowDialog(parent); + else + window.Show(); + } + catch (Exception ex) + { + Log.Error(ex, "Error showing calibration window"); + } + } +} diff --git a/Plugins/LindtLeerformPlugin/LeerformSettings.cs b/Plugins/LindtLeerformPlugin/LeerformSettings.cs new file mode 100644 index 0000000..c073c9c --- /dev/null +++ b/Plugins/LindtLeerformPlugin/LeerformSettings.cs @@ -0,0 +1,65 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Inspectron.Settings; +using LindtLeerformPlugin.Models; +using VisionBuilder.UI.Common; + +namespace LindtLeerformPlugin; + +public class LeerformSettings : ISettings +{ + public string CameraName { get; } + + public LeerformSettings(string cameraName) + { + CameraName = cameraName; + } + + public CalibrationData? CalibrationData { get; set; } + public int CheckerboardRows { get; set; } = 6; + public int CheckerboardCols { get; set; } = 9; + public string CalibrationImageDirectory { get; set; } = "CalibrationImages"; + + public void RegisterSettings(InspectronSettings settings) + { + settings.RegisterSimple(this, () => CalibrationData!, $"{CameraName}/Leerform", nameof(CalibrationData)); + settings.RegisterSimple(this, () => CheckerboardRows, $"{CameraName}/Leerform", nameof(CheckerboardRows)); + settings.RegisterSimple(this, () => CheckerboardCols, $"{CameraName}/Leerform", nameof(CheckerboardCols)); + settings.RegisterSimple(this, () => CalibrationImageDirectory, $"{CameraName}/Leerform", nameof(CalibrationImageDirectory)); + } +} + +public class CalibrationDataConverter : ITypeConverter +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter() } + }; + + public object ConvertFrom(object value) + { + if (value is string json) + { + if (string.IsNullOrWhiteSpace(json) || json == "null") + return null!; + + return JsonSerializer.Deserialize(json, JsonOptions) + ?? throw new InvalidOperationException("Failed to deserialize CalibrationData from JSON."); + } + throw new InvalidOperationException("Value must be a JSON string."); + } + + public object ConvertTo(object value, Type destinationType) + { + if (value is CalibrationData data) + { + return JsonSerializer.Serialize(data, JsonOptions); + } + if (value == null) + { + return "null"; + } + throw new InvalidOperationException("Value must be a CalibrationData."); + } +} diff --git a/Plugins/LindtLeerformPlugin/LindtLeerformPlugin.csproj b/Plugins/LindtLeerformPlugin/LindtLeerformPlugin.csproj new file mode 100644 index 0000000..0df8aa0 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/LindtLeerformPlugin.csproj @@ -0,0 +1,45 @@ + + + + net8.0 + enable + enable + true + ..\..\VisionBuilder.UI.Avalonia.Uno\bin\Debug\Data\Plugins\LindtLeerformPlugin + + + + + false + runtime + + + false + runtime + + + + + + false + runtime + + + false + runtime + + + false + runtime + + + false + runtime + + + false + runtime + + + + diff --git a/Plugins/LindtLeerformPlugin/Models/CalibrationData.cs b/Plugins/LindtLeerformPlugin/Models/CalibrationData.cs new file mode 100644 index 0000000..0867a38 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/Models/CalibrationData.cs @@ -0,0 +1,10 @@ +namespace LindtLeerformPlugin.Models; + +public class CalibrationData +{ + public double[] CameraMatrix { get; set; } = []; + public double[] DistCoeffs { get; set; } = []; + public double RmsError { get; set; } + public int ImageWidth { get; set; } + public int ImageHeight { get; set; } +} diff --git a/Plugins/LindtLeerformPlugin/Plugin.cs b/Plugins/LindtLeerformPlugin/Plugin.cs new file mode 100644 index 0000000..5b180d3 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/Plugin.cs @@ -0,0 +1,21 @@ +using Inspectron.Settings; +using LindtLeerformPlugin.Models; +using Ninject; +using VisionBuilder.UI.Common; +using VisionBuilder.UI.Common.Plugins; + +namespace LindtLeerformPlugin; + +public class Plugin : IPlugin +{ + public void RegisterGlobalModules(IKernel kernel) + { + TypeConverterRegistry.Register(new CalibrationDataConverter()); + } + + public void RegisterCameraModules(IKernel kernel, string cameraName) + { + kernel.Bind().ToConstant(new LeerformSettings(cameraName)); + kernel.RegisterModule(); + } +} diff --git a/Plugins/LindtLeerformPlugin/Services/CameraCalibrationService.cs b/Plugins/LindtLeerformPlugin/Services/CameraCalibrationService.cs new file mode 100644 index 0000000..4459ad6 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/Services/CameraCalibrationService.cs @@ -0,0 +1,101 @@ +using LindtLeerformPlugin.Models; +using OpenCvSharp; + +namespace LindtLeerformPlugin.Services; + +public static class CameraCalibrationService +{ + public static CalibrationData Calibrate(string imageDirectory, Size patternSize) + { + var imageFiles = Directory.GetFiles(imageDirectory, "*.png"); + if (imageFiles.Length == 0) + throw new InvalidOperationException($"No PNG images found in {imageDirectory}"); + + var objectPointsList = new List(); + var imagePointsList = new List(); + Size imageSize = default; + + int cornerCount = patternSize.Width * patternSize.Height; + var objPts = new Point3f[cornerCount]; + for (int row = 0; row < patternSize.Height; row++) + for (int col = 0; col < patternSize.Width; col++) + objPts[row * patternSize.Width + col] = new Point3f(col, row, 0); + + int found = 0; + foreach (var file in imageFiles) + { + using var image = Cv2.ImRead(file); + using var gray = new Mat(); + Cv2.CvtColor(image, gray, ColorConversionCodes.BGR2GRAY); + imageSize = new Size(image.Width, image.Height); + + if (Cv2.FindChessboardCorners(gray, patternSize, out var corners, + ChessboardFlags.AdaptiveThresh | ChessboardFlags.FastCheck)) + { + Cv2.CornerSubPix(gray, corners, + new Size(11, 11), new Size(-1, -1), + new TermCriteria(CriteriaTypes.Eps | CriteriaTypes.MaxIter, 30, 0.001)); + + var objMat = new Mat(cornerCount, 1, MatType.CV_32FC3); + for (int i = 0; i < cornerCount; i++) + objMat.Set(i, 0, objPts[i]); + objectPointsList.Add(objMat); + + var imgMat = new Mat(corners.Length, 1, MatType.CV_32FC2); + for (int i = 0; i < corners.Length; i++) + imgMat.Set(i, 0, corners[i]); + imagePointsList.Add(imgMat); + + found++; + } + } + + if (found == 0) + throw new InvalidOperationException("Checkerboard corners not found in any image"); + + var cameraMatrix = new Mat(); + var distCoeffs = new Mat(); + double rms = Cv2.CalibrateCamera( + objectPointsList, imagePointsList, imageSize, + cameraMatrix, distCoeffs, + out _, out _); + + var data = new CalibrationData + { + CameraMatrix = MatToArray(cameraMatrix), + DistCoeffs = MatToArray(distCoeffs), + RmsError = rms, + ImageWidth = imageSize.Width, + ImageHeight = imageSize.Height + }; + + cameraMatrix.Dispose(); + distCoeffs.Dispose(); + foreach (var m in objectPointsList) m.Dispose(); + foreach (var m in imagePointsList) m.Dispose(); + + return data; + } + + public static (Mat cameraMatrix, Mat distCoeffs) LoadCalibrationMats(CalibrationData data) + { + var cameraMatrix = new Mat(3, 3, MatType.CV_64F); + for (int i = 0; i < 9; i++) + cameraMatrix.Set(i / 3, i % 3, data.CameraMatrix[i]); + + var distCoeffs = new Mat(1, data.DistCoeffs.Length, MatType.CV_64F); + for (int i = 0; i < data.DistCoeffs.Length; i++) + distCoeffs.Set(0, i, data.DistCoeffs[i]); + + return (cameraMatrix, distCoeffs); + } + + private static double[] MatToArray(Mat mat) + { + var total = mat.Rows * mat.Cols; + var arr = new double[total]; + for (int i = 0; i < total; i++) + arr[i] = mat.At(i / mat.Cols, i % mat.Cols); + return arr; + } +} diff --git a/Plugins/LindtLeerformPlugin/Services/ImageConverter.cs b/Plugins/LindtLeerformPlugin/Services/ImageConverter.cs new file mode 100644 index 0000000..c3c2583 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/Services/ImageConverter.cs @@ -0,0 +1,13 @@ +using OpenCvSharp; + +namespace LindtLeerformPlugin.Services; + +public static class ImageConverter +{ + public static Avalonia.Media.Imaging.Bitmap MatToAvaloniaBitmap(Mat mat) + { + var encoded = mat.ImEncode(".bmp"); + using var ms = new MemoryStream(encoded); + return new Avalonia.Media.Imaging.Bitmap(ms); + } +} diff --git a/Plugins/LindtLeerformPlugin/ViewModels/CalibrationWindowViewModel.cs b/Plugins/LindtLeerformPlugin/ViewModels/CalibrationWindowViewModel.cs new file mode 100644 index 0000000..e763171 --- /dev/null +++ b/Plugins/LindtLeerformPlugin/ViewModels/CalibrationWindowViewModel.cs @@ -0,0 +1,151 @@ +using Avalonia.Threading; +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using LindtLeerformPlugin.Models; +using LindtLeerformPlugin.Services; +using OpenCvSharp; +using VisionBuilder.UI.Common.Processing; + +namespace LindtLeerformPlugin.ViewModels; + +public partial class CalibrationWindowViewModel : ObservableObject, IDisposable +{ + private readonly IImageSource _imageSource; + private readonly LeerformSettings _settings; + private CancellationTokenSource? _previewCts; + private Mat? _lastFrame; + + [ObservableProperty] private Avalonia.Media.Imaging.Bitmap? _previewImage; + [ObservableProperty] private string _statusText = "Ready"; + [ObservableProperty] private int _capturedImageCount; + [ObservableProperty] private bool _isPreviewRunning; + [ObservableProperty] private bool _isCalibrated; + [ObservableProperty] private string _calibrationImageDirectory; + [ObservableProperty] private double _rmsError; + + public CalibrationWindowViewModel(IImageSource imageSource, LeerformSettings settings) + { + _imageSource = imageSource; + _settings = settings; + + _calibrationImageDirectory = settings.CalibrationImageDirectory; + _isCalibrated = settings.CalibrationData is { CameraMatrix.Length: > 0 }; + _rmsError = settings.CalibrationData?.RmsError ?? 0; + + UpdateCapturedImageCount(); + } + + private void UpdateCapturedImageCount() + { + if (Directory.Exists(CalibrationImageDirectory)) + CapturedImageCount = Directory.GetFiles(CalibrationImageDirectory, "*.png").Length; + else + CapturedImageCount = 0; + } + + [RelayCommand] + private void StartPreview() + { + if (IsPreviewRunning) return; + + IsPreviewRunning = true; + _previewCts = new CancellationTokenSource(); + _ = PreviewLoopAsync(_previewCts.Token); + StatusText = "Preview running"; + } + + [RelayCommand] + private void StopPreview() + { + if (!IsPreviewRunning) return; + + _previewCts?.Cancel(); + IsPreviewRunning = false; + StatusText = "Preview stopped"; + } + + [RelayCommand] + private void CaptureImage() + { + if (_lastFrame == null || _lastFrame.Empty()) return; + + Directory.CreateDirectory(CalibrationImageDirectory); + var filename = $"calib_{DateTime.Now:yyyyMMdd_HHmmss_fff}.png"; + var path = Path.Combine(CalibrationImageDirectory, filename); + Cv2.ImWrite(path, _lastFrame); + UpdateCapturedImageCount(); + StatusText = $"Captured: {filename}"; + } + + [RelayCommand] + private void RunCalibration() + { + try + { + StatusText = "Calibrating..."; + var patternSize = new Size(_settings.CheckerboardCols, _settings.CheckerboardRows); + var data = CameraCalibrationService.Calibrate(CalibrationImageDirectory, patternSize); + + _settings.CalibrationData = data; + _settings.CalibrationImageDirectory = CalibrationImageDirectory; + + IsCalibrated = true; + RmsError = data.RmsError; + StatusText = $"Calibrated (RMS: {data.RmsError:F4})"; + } + catch (Exception ex) + { + StatusText = $"Calibration error: {ex.Message}"; + } + } + + [RelayCommand] + private void ClearCalibrationImages() + { + if (!Directory.Exists(CalibrationImageDirectory)) return; + + foreach (var file in Directory.GetFiles(CalibrationImageDirectory, "*.png")) + File.Delete(file); + + UpdateCapturedImageCount(); + StatusText = "Calibration images cleared"; + } + + private async Task PreviewLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var frame = await _imageSource.GetImage(ct); + + _lastFrame?.Dispose(); + _lastFrame = frame; + + var bitmap = ImageConverter.MatToAvaloniaBitmap(frame); + await Dispatcher.UIThread.InvokeAsync(() => + { + var old = PreviewImage; + PreviewImage = bitmap; + old?.Dispose(); + }); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + await Dispatcher.UIThread.InvokeAsync(() => + StatusText = $"Preview error: {ex.Message}"); + break; + } + } + } + + public void Dispose() + { + StopPreview(); + _lastFrame?.Dispose(); + } +} diff --git a/Plugins/LindtLeerformPlugin/Views/CalibrationWindow.axaml b/Plugins/LindtLeerformPlugin/Views/CalibrationWindow.axaml new file mode 100644 index 0000000..2055c1f --- /dev/null +++ b/Plugins/LindtLeerformPlugin/Views/CalibrationWindow.axaml @@ -0,0 +1,67 @@ + + + + + +