admin/HawkeyeVision
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

leerform recipe subsystem created

Browse Source
This commit is contained in:
EugeneTes
2026-03-31 13:26:26 +02:00
parent 4bd117af47
commit 1c7a4abd5c
20 changed files with 823 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CLAUDE.md
View File

@@ -70,6 +70,9 @@ Modules implement `IVisionBuilderModule.InitializeModule()` for deferred initial
For detailed plugin architecture documentation and step-by-step implementation guide, see [`docs/PLUGIN_SYSTEM.md`](docs/PLUGIN_SYSTEM.md).
### Recognition Control
`IRecognitionControl` manages the per-camera image processing lifecycle (start/stop/pause, image loop, events). `BaseRecognitionControl` provides the threading, loop, and event infrastructure — subclasses implement `Initialize`, `WarmUp`, `ProcessImage`, and `GetRecipesData`. Each camera gets its own singleton instance via the child kernel. For full implementation guide, settings reference, and existing implementations, see [`docs/RECOGNITION_CONTROL.md`](docs/RECOGNITION_CONTROL.md).
### DI Container
Ninject `StandardKernel` is the root container. Child kernels (`Ninject.Extensions.ChildKernel`) scope per-camera services. All service resolution flows through the kernel — avoid `new` for services.
@@ -89,7 +92,7 @@ Ninject `StandardKernel` is the root container. Child kernels (`Ninject.Extensio
## Conventions
- Target platform is **x64** Windows.
- Target platform is **x64** Windows, but new code should be cross-platform — use Avalonia for UI, avoid WinForms-only or Windows-specific APIs.
- Culture is forced to `en-US` at startup.
- Nullable reference types are enabled across most projects.
- Operation attributes: `[Category("name")]` for UI grouping, `[NotForTool]` to exclude properties from serialization, `[IgnoreOperation]` to hide from discovery.

5
Plugins/CandyboxPlugin/Modules/CandyboxRecipeCreationTool.cs
View File

@@ -6,8 +6,9 @@ namespace CandyboxPlugin.Module;
public class CandyboxRecipeCreationTool:IRecipeCreationTool
{
public bool Enabled { get; set; } = true;
public bool CreateRecipe(out string recipeName)
public Task<string?> CreateRecipeAsync()
{
return MaterialInputBox.Prompt("Recipe name","", out recipeName)==DialogResult.OK;
var result = MaterialInputBox.Prompt("Recipe name","", out var recipeName);
return Task.FromResult(result == DialogResult.OK ? recipeName : null);
}
}

26
Plugins/LindtLeerformPlugin/LeerformRecipeCreationTool.cs Normal file
View File

@@ -0,0 +1,26 @@
using Avalonia;
using Avalonia.Controls.ApplicationLifetimes;
using LindtLeerformPlugin.Views;
using VisionBuilder.UI.Common.ViewModel.Interfaces.UI;
namespace LindtLeerformPlugin;
public class LeerformRecipeCreationTool : IRecipeCreationTool
{
public bool Enabled { get; set; } = true;
public async Task<string?> CreateRecipeAsync()
{
if (Application.Current?.ApplicationLifetime is not IClassicDesktopStyleApplicationLifetime desktop)
return null;
var parent = desktop.MainWindow;
if (parent == null)
return null;
var dialog = new RecipeNameDialog();
var result = await dialog.ShowDialog<string?>(parent);
return string.IsNullOrWhiteSpace(result) ? null : result;
}
}

72
Plugins/LindtLeerformPlugin/LeerformRecognitionControl.cs Normal file
View File

@@ -0,0 +1,72 @@
using System.Diagnostics;
using LindtLeerformPlugin.Services;
using OpenCvSharp;
using Serilog;
using VisionBuilder.UI.Common.Processing;
using VisionBuilder.UI.Common.ViewModel.Classes;
namespace LindtLeerformPlugin;
public class LeerformRecognitionControl : BaseRecognitionControl
{
private readonly IImageSource _imageSource;
private Mat? _cameraMatrix;
private Mat? _distCoeffs;
public LeerformRecognitionControl(
LeerformRecognitionControlSettings settings,
IImageSource imageSource,
ILoadingService loadingService) : base(settings, loadingService)
{
_imageSource = imageSource;
}
public override List<RecipeData> GetRecipesData()
{
return [new RecipeData { RecipeName = "Default" }];
}
protected override void Initialize(RecipeData currentRecipe)
{
_cameraMatrix?.Dispose();
_distCoeffs?.Dispose();
_cameraMatrix = null;
_distCoeffs = null;
var calibrationData = CalibrationDataStore.Load();
if (calibrationData != null)
{
(_cameraMatrix, _distCoeffs) = CameraCalibrationService.LoadCalibrationMats(calibrationData);
Log.Information("Calibration data loaded (RMS error: {RmsError:F3})", calibrationData.RmsError);
}
}
protected override void WarmUp()
{
}
protected override (Mat originalImage, Mat analysisImage, TimeSpan processingTime, TimeSpan acquisitionTime, string[] errorNames)?
ProcessImage(CancellationToken token)
{
var sw = Stopwatch.StartNew();
var image = _imageSource.GetImage(token).Result;
if (image == null)
return null;
var acquisitionTime = sw.Elapsed;
if (_cameraMatrix != null && _distCoeffs != null)
{
var undistorted = new Mat();
Cv2.Undistort(image, undistorted, _cameraMatrix, _distCoeffs);
sw.Stop();
return (image, undistorted, sw.Elapsed, acquisitionTime, []);
}
sw.Stop();
return (image, image, TimeSpan.Zero, acquisitionTime, []);
}
}

12
Plugins/LindtLeerformPlugin/LeerformRecognitionControlSettings.cs Normal file
View File

@@ -0,0 +1,12 @@
using Inspectron.Settings;
using VisionBuilder.UI.Common.Processing;
namespace LindtLeerformPlugin;
public class LeerformRecognitionControlSettings(string cameraName) : BaseRecognitionControlSettings(cameraName)
{
public override void RegisterSettings(InspectronSettings settings)
{
base.RegisterSettings(settings);
}
}

42
Plugins/LindtLeerformPlugin/LeerformSettings.cs
View File

@@ -1,7 +1,4 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Inspectron.Settings;
using LindtLeerformPlugin.Models;
using VisionBuilder.UI.Common;
namespace LindtLeerformPlugin;
@@ -15,51 +12,14 @@ public class LeerformSettings : ISettings
CameraName = cameraName;
}
public CalibrationData? CalibrationData { get; set; }
public int CheckerboardRows { get; set; } = 6;
public int CheckerboardCols { get; set; } = 9;
public string CalibrationImageDirectory { get; set; } = "CalibrationImages";
public string CalibrationImageDirectory { get; set; } = Path.Combine("..", "Data", "CalibrationImages");
public void RegisterSettings(InspectronSettings settings)
{
settings.RegisterSimple(this, () => CalibrationData!, $"{CameraName}/Leerform", nameof(CalibrationData));
settings.RegisterSimple(this, () => CheckerboardRows, $"{CameraName}/Leerform", nameof(CheckerboardRows));
settings.RegisterSimple(this, () => CheckerboardCols, $"{CameraName}/Leerform", nameof(CheckerboardCols));
settings.RegisterSimple(this, () => CalibrationImageDirectory, $"{CameraName}/Leerform", nameof(CalibrationImageDirectory));
}
}
public class CalibrationDataConverter : ITypeConverter
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
Converters = { new JsonStringEnumConverter() }
};
public object ConvertFrom(object value)
{
if (value is string json)
{
if (string.IsNullOrWhiteSpace(json) || json == "null")
return null!;
return JsonSerializer.Deserialize<CalibrationData>(json, JsonOptions)
?? throw new InvalidOperationException("Failed to deserialize CalibrationData from JSON.");
}
throw new InvalidOperationException("Value must be a JSON string.");
}
public object ConvertTo(object value, Type destinationType)
{
if (value is CalibrationData data)
{
return JsonSerializer.Serialize(data, JsonOptions);
}
if (value == null)
{
return "null";
}
throw new InvalidOperationException("Value must be a CalibrationData.");
}
}

19
Plugins/LindtLeerformPlugin/Plugin.cs
View File

@@ -1,8 +1,8 @@
using Inspectron.Settings;
using LindtLeerformPlugin.Models;
using Ninject;
using VisionBuilder.UI.Common;
using VisionBuilder.UI.Common.Plugins;
using VisionBuilder.UI.Common.Processing;
using VisionBuilder.UI.Common.ViewModel.Interfaces.UI;
namespace LindtLeerformPlugin;
@@ -10,12 +10,25 @@ public class Plugin : IPlugin
{
public void RegisterGlobalModules(IKernel kernel)
{
TypeConverterRegistry.Register<CalibrationData>(new CalibrationDataConverter());
kernel.Rebind<IRecipeCreationTool>().To<LeerformRecipeCreationTool>().InSingletonScope();
}
public void RegisterCameraModules(IKernel kernel, string cameraName)
{
kernel.Bind<LeerformSettings, ISettings>().ToConstant(new LeerformSettings(cameraName));
// Remove existing recognition settings ISettings binding before rebinding
var existingRecognitionControlSettings = kernel.GetBindings(typeof(BaseRecognitionControlSettings)).First();
var toRemove = kernel.GetBindings(typeof(ISettings))
.First(x => x.ProviderCallback.Target == existingRecognitionControlSettings.ProviderCallback.Target);
kernel.RemoveBinding(toRemove);
kernel.Bind<LeerformRecognitionControlSettings, BaseRecognitionControlSettings, ISettings>()
.ToConstant(new LeerformRecognitionControlSettings(cameraName));
kernel.Rebind<IRecognitionControl>()
.To<LeerformRecognitionControl>()
.InSingletonScope();
kernel.RegisterModule<LeerformModule>();
}
}

36
Plugins/LindtLeerformPlugin/Services/CalibrationDataStore.cs Normal file
View File

@@ -0,0 +1,36 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using LindtLeerformPlugin.Models;
namespace LindtLeerformPlugin.Services;
public static class CalibrationDataStore
{
private static readonly string FilePath = Path.Combine("..", "Data", "Config", "calibration.json");
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
WriteIndented = true,
Converters = { new JsonStringEnumConverter() }
};
public static CalibrationData? Load()
{
if (!File.Exists(FilePath))
return null;
var json = File.ReadAllText(FilePath);
return JsonSerializer.Deserialize<CalibrationData>(json, JsonOptions);
}
public static void Save(CalibrationData data)
{
var directory = Path.GetDirectoryName(FilePath);
if (directory != null)
Directory.CreateDirectory(directory);
var json = JsonSerializer.Serialize(data, JsonOptions);
File.WriteAllText(FilePath, json);
}
}

38
Plugins/LindtLeerformPlugin/ViewModels/CalibrationWindowViewModel.cs
View File

@@ -1,7 +1,6 @@
using Avalonia.Threading;
using CommunityToolkit.Mvvm.ComponentModel;
using CommunityToolkit.Mvvm.Input;
using LindtLeerformPlugin.Models;
using LindtLeerformPlugin.Services;
using OpenCvSharp;
using VisionBuilder.UI.Common.Processing;
@@ -14,12 +13,15 @@ public partial class CalibrationWindowViewModel : ObservableObject, IDisposable
private readonly LeerformSettings _settings;
private CancellationTokenSource? _previewCts;
private Mat? _lastFrame;
private Mat? _calibCameraMatrix;
private Mat? _calibDistCoeffs;
[ObservableProperty] private Avalonia.Media.Imaging.Bitmap? _previewImage;
[ObservableProperty] private string _statusText = "Ready";
[ObservableProperty] private int _capturedImageCount;
[ObservableProperty] private bool _isPreviewRunning;
[ObservableProperty] private bool _isCalibrated;
[ObservableProperty] private bool _applyCalibration;
[ObservableProperty] private string _calibrationImageDirectory;
[ObservableProperty] private double _rmsError;
@@ -29,8 +31,13 @@ public partial class CalibrationWindowViewModel : ObservableObject, IDisposable
_settings = settings;
_calibrationImageDirectory = settings.CalibrationImageDirectory;
_isCalibrated = settings.CalibrationData is { CameraMatrix.Length: > 0 };
_rmsError = settings.CalibrationData?.RmsError ?? 0;
var calibrationData = CalibrationDataStore.Load();
_isCalibrated = calibrationData is { CameraMatrix.Length: > 0 };
_rmsError = calibrationData?.RmsError ?? 0;
if (_isCalibrated)
LoadCalibrationMats(calibrationData!);
UpdateCapturedImageCount();
}
@@ -86,9 +93,10 @@ public partial class CalibrationWindowViewModel : ObservableObject, IDisposable
var patternSize = new Size(_settings.CheckerboardCols, _settings.CheckerboardRows);
var data = CameraCalibrationService.Calibrate(CalibrationImageDirectory, patternSize);
_settings.CalibrationData = data;
CalibrationDataStore.Save(data);
_settings.CalibrationImageDirectory = CalibrationImageDirectory;
LoadCalibrationMats(data);
IsCalibrated = true;
RmsError = data.RmsError;
StatusText = $"Calibrated (RMS: {data.RmsError:F4})";
@@ -122,7 +130,18 @@ public partial class CalibrationWindowViewModel : ObservableObject, IDisposable
_lastFrame?.Dispose();
_lastFrame = frame;
var bitmap = ImageConverter.MatToAvaloniaBitmap(frame);
var displayFrame = frame;
if (ApplyCalibration && _calibCameraMatrix != null && _calibDistCoeffs != null)
{
displayFrame = new Mat();
Cv2.Undistort(frame, displayFrame, _calibCameraMatrix, _calibDistCoeffs);
}
var bitmap = ImageConverter.MatToAvaloniaBitmap(displayFrame);
if (displayFrame != frame)
displayFrame.Dispose();
await Dispatcher.UIThread.InvokeAsync(() =>
{
var old = PreviewImage;
@@ -143,9 +162,18 @@ public partial class CalibrationWindowViewModel : ObservableObject, IDisposable
}
}
private void LoadCalibrationMats(Models.CalibrationData data)
{
_calibCameraMatrix?.Dispose();
_calibDistCoeffs?.Dispose();
(_calibCameraMatrix, _calibDistCoeffs) = CameraCalibrationService.LoadCalibrationMats(data);
}
public void Dispose()
{
StopPreview();
_lastFrame?.Dispose();
_calibCameraMatrix?.Dispose();
_calibDistCoeffs?.Dispose();
}
}

19
Plugins/LindtLeerformPlugin/Views/CalibrationWindow.axaml
View File

@@ -29,31 +29,30 @@
</Border>
<!-- Settings Panel -->
<Border DockPanel.Dock="Right" Width="220" Padding="12" Background="#2D2D2D">
<Border DockPanel.Dock="Right" Width="220" Padding="12" Background="White">
<ScrollViewer>
<StackPanel Spacing="8">
<TextBlock Text="Calibration Settings" FontWeight="Bold" FontSize="14" Foreground="White" />
<TextBlock Text="Image Directory" Foreground="LightGray" />
<TextBox Text="{Binding CalibrationImageDirectory}" />
<Separator Margin="0,8" />
<TextBlock Text="Calibration Settings" FontWeight="Bold" FontSize="14" Foreground="Black" />
<Button Content="Calibrate" Command="{Binding RunCalibrationCommand}" Margin="0,4"
Background="#00C853" Foreground="White" FontWeight="Bold" />
Background="Red" Foreground="White" FontWeight="Bold" />
<Button Content="Clear Images" Command="{Binding ClearCalibrationImagesCommand}" Margin="0,4"
Background="#FF6D00" Foreground="White" FontWeight="Bold" />
Background="White" Foreground="Red" FontWeight="Bold"
BorderBrush="Red" BorderThickness="1" />
<Separator Margin="0,8" />
<TextBlock Text="Calibrated" Foreground="LimeGreen" IsVisible="{Binding IsCalibrated}" />
<TextBlock Foreground="LightGray" IsVisible="{Binding IsCalibrated}">
<TextBlock Foreground="Gray" IsVisible="{Binding IsCalibrated}">
<TextBlock.Text>
<MultiBinding StringFormat="RMS Error: {0:F4}">
<Binding Path="RmsError" />
</MultiBinding>
</TextBlock.Text>
</TextBlock>
<CheckBox Content="Apply Calibration" IsChecked="{Binding ApplyCalibration}"
IsEnabled="{Binding IsCalibrated}" Foreground="Black" Margin="0,4" />
</StackPanel>
</ScrollViewer>
</Border>

33
Plugins/LindtLeerformPlugin/Views/RecipeNameDialog.axaml Normal file
View File

@@ -0,0 +1,33 @@
<Window xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
x:Class="LindtLeerformPlugin.Views.RecipeNameDialog"
Title="New Recipe"
Width="400" Height="180"
CanResize="False"
WindowStartupLocation="CenterOwner">
<Grid RowDefinitions="Auto,Auto,Auto" Margin="16">
<TextBlock Grid.Row="0"
Text="Recipe name:"
FontSize="14"
Margin="0,0,0,8" />
<TextBox Grid.Row="1"
x:Name="TxtRecipeName"
Margin="0,0,0,16" />
<StackPanel Grid.Row="2"
Orientation="Horizontal"
HorizontalAlignment="Right"
Spacing="8">
<Button x:Name="BtnOk"
Content="OK"
Width="80" Height="36"
HorizontalContentAlignment="Center" />
<Button x:Name="BtnCancel"
Content="Cancel"
Width="80" Height="36"
HorizontalContentAlignment="Center" />
</StackPanel>
</Grid>
</Window>

23
Plugins/LindtLeerformPlugin/Views/RecipeNameDialog.axaml.cs Normal file
View File

@@ -0,0 +1,23 @@
using Avalonia.Controls;
using Avalonia.Input;
namespace LindtLeerformPlugin.Views;
public partial class RecipeNameDialog : Window
{
public RecipeNameDialog()
{
InitializeComponent();
BtnOk.Click += (_, _) => Close(TxtRecipeName.Text);
BtnCancel.Click += (_, _) => Close(null);
TxtRecipeName.KeyDown += (_, e) =>
{
if (e.Key == Key.Enter)
Close(TxtRecipeName.Text);
else if (e.Key == Key.Escape)
Close(null);
};
}
}

5
VisionBuilder.UI.Avalonia.Uno/Views/RecipeSelectionDialog.axaml.cs
View File

@@ -115,9 +115,10 @@ public partial class RecipeSelectionDialog : global::Avalonia.Controls.Window
}
}
private void BtnCreateNew_Click(object? sender, global::Avalonia.Interactivity.RoutedEventArgs e)
private async void BtnCreateNew_Click(object? sender, global::Avalonia.Interactivity.RoutedEventArgs e)
{
if (_recipeCreationTool.CreateRecipe(out var recipeName))
var recipeName = await _recipeCreationTool.CreateRecipeAsync();
if (recipeName != null)
{
_recipeSelectionVm.SelectedRecipe = new RecipeData { RecipeName = recipeName };
_tcs.TrySetResult(true);

2
VisionBuilder.UI.Common/NullClasses/NoRecipeCreation.cs
View File

@@ -5,7 +5,7 @@ namespace VisionBuilder.UI.Common.NullClasses;
public class NoRecipeCreation: IRecipeCreationTool
{
public bool Enabled { get; set; } = false;
public bool CreateRecipe(out string recipeName)
public Task<string?> CreateRecipeAsync()
{
throw new NotImplementedException();
}

2
VisionBuilder.UI.Common/ViewModel/Interfaces/UI/IRecipeCreationTool.cs
View File

@@ -3,5 +3,5 @@
public interface IRecipeCreationTool
{
public bool Enabled { get; set; }
public bool CreateRecipe(out string recipeName);
public Task<string?> CreateRecipeAsync();
}

BIN
VisionBuilder.UI.RaspberryAcquisition/Calibration/capture_20260331_105918.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.8 MiB

6
VisionBuilder.UI.Windows/Dialogs/RecipeSelectionDialog.cs
View File

@@ -83,14 +83,14 @@ namespace VisionBuilder.UI.Windows.Dialogs
DialogResult = DialogResult.Cancel;
}
private void btnCreateNewRecipe_Click(object sender, EventArgs e)
private async void btnCreateNewRecipe_Click(object sender, EventArgs e)
{
if (_recipeCreationTool.CreateRecipe(out var recipeName))
var recipeName = await _recipeCreationTool.CreateRecipeAsync();
if (recipeName != null)
{
_recipeSelectionVm.SelectedRecipe = new RecipeData() { RecipeName = recipeName };
DialogResult = DialogResult.OK;
}
}
}
}

3
VisionBuilder.UI.sln
View File

@@ -111,6 +111,9 @@ EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VisionBuilder.UI.RaspberryAcquisition", "VisionBuilder.UI.RaspberryAcquisition\VisionBuilder.UI.RaspberryAcquisition.csproj", "{149E2AFC-A714-4645-883A-2EE685E63FB9}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VisionBuilder.UI.Avalonia.Uno", "VisionBuilder.UI.Avalonia.Uno\VisionBuilder.UI.Avalonia.Uno.csproj", "{F70920C6-EF98-42B7-9F21-6E94781E9900}"
ProjectSection(ProjectDependencies) = postProject
{5A1D9E37-353C-4E30-A302-48487DC3A9E8} = {5A1D9E37-353C-4E30-A302-48487DC3A9E8}
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LindtLeerformPlugin", "Plugins\LindtLeerformPlugin\LindtLeerformPlugin.csproj", "{5A1D9E37-353C-4E30-A302-48487DC3A9E8}"
EndProject

2
docs/PLUGIN_SYSTEM.md
View File

@@ -363,7 +363,7 @@ UI tool for creating new recipes. Default implementation (`NoRecipeCreation`) is
public interface IRecipeCreationTool
{
bool Enabled { get; set; }
bool CreateRecipe(out string recipeName);
Task<string?> CreateRecipeAsync();
}
```

543
docs/RECOGNITION_CONTROL.md Normal file
View File

@@ -0,0 +1,543 @@
# Recognition Control System
The recognition control system manages the image processing lifecycle: acquiring images, running them through a processing pipeline, and emitting results as events. Each camera gets its own `IRecognitionControl` instance, registered in the per-camera Ninject child kernel.
## Architecture Overview
```
IRecognitionControl (interface)
^
|
BaseRecognitionControl (abstract base - lifecycle, threading, events)
^
|--- HawkeyeRecognitionControl (workflow-based pipeline processing)
|--- CandyboxImageProcessingControl (histogram-based processing)
|--- TestImageProcessingControl (minimal test stub)
|--- [Your custom implementation]
```
```
BaseRecognitionControlSettings (abstract base - shared config)
^
|--- HawkeyeRecognitionSettings
|--- CandyboxRecognitionControlSettings
|--- TestRecognitionControlSettings
|--- [Your custom settings]
```
## IRecognitionControl Interface
Defined in `VisionBuilder.UI.Common/Processing/IRecognitionControl.cs`.
```csharp
public interface IRecognitionControl
{
bool IsRunning { get; }
List<RecipeData> GetRecipesData();
void SetRecipe(RecipeData recipe);
void Start();
void Stop();
void Pause();
void Resume();
event Action<ImageProcessedEvent> ImageProcessed;
event Action<SessionStartedEvent> SessionStarted;
event Action<SessionEndedEvent> SessionEnded;
event Action<ErrorsInSequenceEvent> ErrorsInSequenceAlarm;
void HotReload();
event Action RecipeFileChanged;
}
```
| Member | Description |
|---|---|
| `IsRunning` | Whether the processing loop is active. |
| `GetRecipesData()` | Returns available recipes (name + optional preview image). |
| `SetRecipe(recipe)` | Selects a recipe. Throws if `IsRunning` is true. |
| `Start()` | Initializes the recipe, warms up, then starts the processing loop on a background task. |
| `Stop()` | Cancels the processing loop, waits for it to finish, runs cleanup. |
| `Pause()` / `Resume()` | Temporarily suspends/resumes image processing without stopping the session. |
| `HotReload()` | Reloads recipe parameters without stopping the session. |
| `ImageProcessed` | Fired after every processed image with full result data. |
| `SessionStarted` / `SessionEnded` | Fired when a session starts/stops. |
| `ErrorsInSequenceAlarm` | Fired when consecutive errors reach the configured threshold. |
| `RecipeFileChanged` | Fired when the recipe file on disk changes (for hot-reload UI). |
## BaseRecognitionControl
Defined in `VisionBuilder.UI.Common/Processing/BaseRecognitionControl.cs`. This abstract class implements all of `IRecognitionControl` and provides the processing loop, threading, pause logic, error-in-sequence tracking, and event plumbing. Subclasses only need to implement four methods.
### Abstract Methods to Implement
```csharp
// Load/prepare resources for the given recipe
protected abstract void Initialize(RecipeData currentRecipe);
// Run a throwaway processing pass to warm up caches, JIT, models, etc.
protected abstract void WarmUp();
// Acquire and process one image. Return null to stop the loop.
protected abstract (
Mat originalImage,
Mat analysisImage,
TimeSpan processingTime,
TimeSpan acquisitionTime,
string[] errorNames
)? ProcessImage(CancellationToken token);
// Return the list of available recipes
public abstract List<RecipeData> GetRecipesData();
```
### Virtual Methods (Optional Overrides)
```csharp
// Release resources when the session stops (called from Stop())
protected virtual void Cleanup() { }
// Reload recipe parameters without restarting the session
public virtual void HotReload() { }
```
### Protected Helper
```csharp
// Call this to raise the RecipeFileChanged event (e.g. from a file watcher)
protected void RaiseRecipeFileChanged();
```
### Processing Loop Behavior
The `Start()` method:
1. Validates that a recipe is set and the control is not already running.
2. Shows a loading indicator via `ILoadingService`.
3. Calls `Initialize(currentRecipe)`.
4. Calls `WarmUp()` (errors are logged, not thrown).
5. Launches `Loop()` on `Task.Run` with a `CancellationToken`.
6. Fires `SessionStarted`.
7. Hides the loading indicator.
The `Loop()` runs continuously until cancellation:
1. If paused, sleeps 100ms and continues.
2. Calls `ProcessImage(token)`. If it returns `null`, the loop exits.
3. If processing time is below `MinimumProcessingTime`, sleeps the difference.
4. Tracks consecutive errors. If they reach `ErrorsInSequenceAlarm`, fires the alarm event.
5. Builds an `ImageProcessedEvent` and fires it.
The `Stop()` method:
1. Cancels the token, waits for the loop task to finish.
2. Calls `Cleanup()`.
3. Fires `SessionEnded`.
### Thread Safety
All public lifecycle methods (`Start`, `Stop`, `Pause`, `Resume`, `SetRecipe`) are protected by a shared `lock`. The processing loop runs on a background `Task`. `ProcessImage` receives a `CancellationToken` and should respect it for timely shutdown.
## BaseRecognitionControlSettings
Defined in `VisionBuilder.UI.Common/Processing/BaseRecognitionControlSettings.cs`. Provides the shared configuration for all recognition controls.
```csharp
public abstract class BaseRecognitionControlSettings(string cameraName) : ISettings
{
public string CameraName { get; } // Injected camera identifier
public int MinimumProcessingTime { get; set; } // Floor for loop iteration (ms)
public string RecipeNameFilter { get; set; } = "*"; // Glob filter for recipe discovery
public int ErrorsInSequenceAlarm { get; set; } = 5; // Consecutive errors before alarm
}
```
All properties are registered in the `Inspectron.Settings` system under `{CameraName}/Recognition` by the `RegisterSettings` method. Override it to add custom settings (always call `base.RegisterSettings` first).
## Events
All event classes are in `VisionBuilder.UI.Common/Commands/`.
### ImageProcessedEvent
Fired after every image. Contains everything needed to update the UI and record statistics.
```csharp
public class ImageProcessedEvent : IEvent
{
public DateTime SessionStart { get; set; }
public string ImageSource { get; set; } // Camera name
public string RecipeName { get; set; }
public Mat ImageOriginal { get; set; } // Raw camera image
public Mat ImageAnalysis { get; set; } // Annotated/processed image
public bool HasError { get; set; }
public List<string> ErrorNames { get; set; } // Names of failed operations
public TimeSpan AnalysisTime { get; set; } // Processing minus acquisition
public TimeSpan AcquisitionTime { get; set; } // Camera capture time
public TimeSpan SleepTime { get; set; } // MinimumProcessingTime padding
}
```
### SessionStartedEvent / SessionEndedEvent
```csharp
public class SessionStartedEvent : IEvent
{
public string RecipeName { get; set; }
public DateTime SessionStarted { get; set; }
}
public class SessionEndedEvent : IEvent
{
public DateTime SessionEnded { get; set; }
}
```
### ErrorsInSequenceEvent
```csharp
public class ErrorsInSequenceEvent
{
public int Errors { get; set; } // Current consecutive error count
}
```
## RecipeData
Defined in `VisionBuilder.UI.Common/ViewModel/Classes/RecipeData.cs`.
```csharp
public class RecipeData
{
public Mat? Image { get; set; } // Optional preview thumbnail
public string RecipeName { get; set; } // Recipe identifier (typically filename without extension)
}
```
## ILoadingService
Defined in `VisionBuilder.UI.Common/Processing/ILoadingService.cs`. Called by `BaseRecognitionControl` during `Start()` and `Stop()` to show/hide a loading overlay in the UI.
```csharp
public interface ILoadingService
{
void StartLoading(string title);
void StopLoading(string title);
}
```
The loading service is injected into `BaseRecognitionControl` via the constructor and exposed as the `LoadingService` protected property, so subclasses can use it in `GetRecipesData()` or elsewhere.
## How to Implement a Custom Recognition Control
### Step 1: Create Settings
Extend `BaseRecognitionControlSettings` with any additional configuration your processing needs.
```csharp
using Inspectron.Settings;
using VisionBuilder.UI.Common.Processing;
public class MyRecognitionSettings(string cameraName) : BaseRecognitionControlSettings(cameraName)
{
public double Threshold { get; set; } = 0.5;
public override void RegisterSettings(InspectronSettings settings)
{
base.RegisterSettings(settings);
settings.RegisterSimple(this, () => Threshold,
CameraName + "/Recognition", nameof(Threshold));
}
}
```
The `RegisterSettings` call makes the property editable in the application's settings UI. Always call `base.RegisterSettings(settings)` first to register the base properties (`MinimumProcessingTime`, `RecipeNameFilter`, `ErrorsInSequenceAlarm`).
### Step 2: Create the Recognition Control
Extend `BaseRecognitionControl` and implement the four abstract members.
```csharp
using System.Diagnostics;
using OpenCvSharp;
using VisionBuilder.UI.Common.Processing;
using VisionBuilder.UI.Common.ViewModel.Classes;
public class MyRecognitionControl : BaseRecognitionControl
{
private readonly MyRecognitionSettings _settings;
private readonly IImageSource _imageSource;
private MyProcessor _processor; // your processing logic
public MyRecognitionControl(
MyRecognitionSettings settings,
IImageSource imageSource,
ILoadingService loadingService)
: base(settings, loadingService)
{
_settings = settings;
_imageSource = imageSource;
}
public override List<RecipeData> GetRecipesData()
{
// Return available recipes. Use _settings.RecipeNameFilter if applicable.
var recipes = new List<RecipeData>();
foreach (var file in Directory.GetFiles(@"..\Data\Recipes", "*.myrecipe"))
{
recipes.Add(new RecipeData
{
RecipeName = Path.GetFileNameWithoutExtension(file),
Image = null // or load a preview thumbnail
});
}
return recipes;
}
protected override void Initialize(RecipeData currentRecipe)
{
// Load the recipe and prepare your processing pipeline.
// Called once when Start() is invoked.
var path = Path.Combine(@"..\Data\Recipes", currentRecipe.RecipeName + ".myrecipe");
_processor = MyProcessor.Load(path, _settings.Threshold);
}
protected override void WarmUp()
{
// Optional: run a throwaway pass to warm up JIT, GPU, ONNX models, etc.
// Errors here are caught and logged, they won't prevent startup.
var dummy = new Mat(100, 100, MatType.CV_8UC3, Scalar.All(0));
_processor.Process(dummy);
dummy.Dispose();
}
protected override (Mat originalImage, Mat analysisImage,
TimeSpan processingTime, TimeSpan acquisitionTime,
string[] errorNames)?
ProcessImage(CancellationToken token)
{
// 1. Acquire image
var swTotal = Stopwatch.StartNew();
var swAcquire = Stopwatch.StartNew();
var image = _imageSource.GetImage(token).Result;
swAcquire.Stop();
if (image == null)
return null; // null signals "stop the loop"
// 2. Process
var result = _processor.Process(image);
swTotal.Stop();
// 3. Build annotated image
var annotated = image.Clone();
// ... draw overlays on annotated ...
// 4. Collect error names (empty array = pass)
string[] errors = result.Defects
.Select(d => d.Name)
.ToArray();
return (image, annotated, swTotal.Elapsed, swAcquire.Elapsed, errors);
}
protected override void Cleanup()
{
// Release resources when the session stops.
_processor?.Dispose();
_processor = null;
}
}
```
### Key implementation notes for ProcessImage
- **Return `null`** to signal the loop should exit (e.g. when the image source is exhausted or cancelled).
- **Respect the `CancellationToken`** — pass it to blocking calls like `GetImage()`. Catch `OperationCanceledException` if you need cleanup before returning null.
- **`errorNames`** drives the pass/fail logic. An empty array means the image passed. Each string becomes a named error in statistics and UI.
- **`originalImage`** is the raw camera image; **`analysisImage`** is the annotated version shown in the preview UI.
- **`processingTime`** is the total wall-clock time including acquisition; **`acquisitionTime`** is just the camera capture. The base class computes `AnalysisTime = processingTime - acquisitionTime` for the event.
### Step 3: Register via DI
Register your settings and control in the per-camera Ninject child kernel. This is typically done in a plugin's `RegisterCameraModules` or in an extension method.
**Option A: In a plugin**
```csharp
public class MyPlugin : IPlugin
{
public void RegisterGlobalModules(IKernel kernel) { }
public void RegisterCameraModules(IKernel kernel, string cameraName)
{
kernel.Bind<MyRecognitionSettings, BaseRecognitionControlSettings, ISettings>()
.ToConstant(new MyRecognitionSettings(cameraName));
kernel.Rebind<IRecognitionControl>()
.To<MyRecognitionControl>()
.InSingletonScope();
}
}
```
**Option B: As an extension method (like HawkeyeRecipe)**
```csharp
public static class ModuleExtensions
{
public static IChildKernel UseMyRecipes(this IChildKernel self, string cameraName)
{
self.Bind<MyRecognitionSettings, BaseRecognitionControlSettings, ISettings>()
.ToConstant(new MyRecognitionSettings(cameraName));
self.Bind<IRecognitionControl>()
.To<MyRecognitionControl>()
.InSingletonScope();
return self;
}
}
```
**Important DI details:**
- Bind your settings class to **three types**: itself, `BaseRecognitionControlSettings`, and `ISettings`. The `ISettings` binding makes `RegisterSettings` get called by the settings system. The `BaseRecognitionControlSettings` binding lets the base class constructor receive it.
- Use `Rebind<IRecognitionControl>` (not `Bind`) if replacing an existing binding from another recipe provider.
- Always use `InSingletonScope()` — there should be exactly one recognition control per camera.
### Step 4: Hot Reload (Optional)
To support live recipe parameter updates without stopping:
1. Set up a `FileSystemWatcher` in `Initialize()` to monitor the recipe file.
2. When the file changes, call `RaiseRecipeFileChanged()` to notify the UI.
3. Override `HotReload()` to apply the new parameters on the next loop iteration.
Example from `HawkeyeRecognitionControl`:
```csharp
private FileSystemWatcher? _fileWatcher;
private System.Timers.Timer? _debounceTimer;
private string? _pendingHotReloadFile;
private string? _currentRecipeFilePath;
protected override void Initialize(RecipeData currentRecipe)
{
_currentRecipeFilePath = Path.GetFullPath(/* recipe path */);
_workflow = LoadRecipe(_currentRecipeFilePath);
StartFileWatcher(_currentRecipeFilePath);
}
private void StartFileWatcher(string filePath)
{
var directory = Path.GetDirectoryName(filePath)!;
var fileName = Path.GetFileName(filePath);
_debounceTimer = new System.Timers.Timer(500) { AutoReset = false };
_debounceTimer.Elapsed += (_, _) => RaiseRecipeFileChanged();
_fileWatcher = new FileSystemWatcher(directory, fileName)
{
NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size,
EnableRaisingEvents = true
};
_fileWatcher.Changed += (_, _) =>
{
_debounceTimer.Stop();
_debounceTimer.Start(); // debounce to 500ms
};
}
public override void HotReload()
{
// Store path; ProcessImage picks it up on the next iteration
Interlocked.Exchange(ref _pendingHotReloadFile, _currentRecipeFilePath);
}
protected override void Cleanup()
{
_fileWatcher?.Dispose();
_debounceTimer?.Dispose();
}
```
The UI flow: `RecipeFileChanged` event -> `SingleCameraVM` sets `IsHotReloadAvailable = true` -> user clicks hot-reload button -> calls `HotReload()` -> next `ProcessImage` iteration applies new parameters.
## How the UI Consumes IRecognitionControl
`SingleCameraVM` (in `VisionBuilder.UI.Common/ViewModel/SingleCameraVM.cs`) is the primary consumer:
**Commands:**
```csharp
// Start/Stop run on background tasks to avoid blocking the UI
await Task.Run(() => _recognitionControl.Start());
await Task.Run(() => _recognitionControl.Stop());
// Pause/Resume are synchronous (just flip a flag)
_recognitionControl.Pause();
_recognitionControl.Resume();
// Recipe selection
_recognitionControl.SetRecipe(selectedRecipe);
// Hot reload
_recognitionControl.HotReload();
```
**Event subscriptions (in constructor):**
```csharp
recognitionControl.ImageProcessed += Handle; // -> PreviewVm, StatisticsVm, ErrorsVm
recognitionControl.SessionStarted += StatisticsVm.Handle;
recognitionControl.SessionStarted += ErrorsVm.Handle;
recognitionControl.RecipeFileChanged += () =>
SynchronizationContext?.Post(_ => IsHotReloadAvailable = true, null);
```
**Other modules subscribing to events:**
- `VisionBuilderStatistics` — writes CSV logs per session, tracks pass/fail counts.
- `VisionBuilderRingbuffer` — saves images to ringbuffers (separate good/bad).
- `IOCommanderCameraModule` — emits GPIO signals on session start/end, image result, and error alarms.
- Plugin modules (e.g. `PralinenModule`) — custom PLC integration based on results.
## Existing Implementations Reference
### HawkeyeRecognitionControl
**Project:** `VisionBuilder.UI.Recipes.HawkeyeRecipe`
The primary implementation for workflow-based image processing.
- **GetRecipesData**: Loads `.jhrcp` files from `Data/Recipes/`, deserializes each to extract recipe name and preview image.
- **Initialize**: Loads the workflow JSON file, starts a file watcher for hot-reload.
- **WarmUp**: Executes the workflow once with an `EmptyImageSource`.
- **ProcessImage**: Runs the full workflow pipeline, collects errors from operations where `Result == false`, draws graphics overlays on the analysis image using `OpenCVCanvas`.
- **HotReload**: Uses `Interlocked.Exchange` to pass the file path; `ProcessImage` checks for pending reload via `_workflow.HotReloadParameters()`.
- **Cleanup**: Disposes the file watcher and debounce timer.
- **Settings**: `HawkeyeRecognitionSettings` adds `AlwaysError` (bool) for testing.
### CandyboxImageProcessingControl
**Project:** `Plugins/CandyboxPlugin`
Histogram-based image processing for candy box quality control.
- **GetRecipesData**: Loads `.json` files from `Data/Recipes/`, finds matching sample `.bmp` images for previews.
- **Initialize**: Creates a `HistoRecipe`, applies camera width settings to HawkEye hardware.
- **WarmUp**: No-op.
- **ProcessImage**: Acquires image, processes via `HistoRecipe`, collects errors from `ErrorPoints` and `ErrorReason`.
- **Settings**: `CandyboxRecognitionControlSettings` adds `AlwaysError` with `[SettingDescription]` attribute.
- **DI**: Plugin replaces the existing `BaseRecognitionControlSettings` binding before adding its own.
### TestImageProcessingControl
**Project:** `Plugins/TestPlugin`
Minimal stub for testing and development.
- **GetRecipesData**: Returns two hardcoded recipes ("Hello", "World") with red 100x100 Mat images.
- **Initialize / WarmUp**: No-op.
- **ProcessImage**: Returns a fixed red image with `"TestError"`.
- **Settings**: `TestRecognitionControlSettings` with no extra properties.
## Checklist for New Implementations
1. Create a settings class extending `BaseRecognitionControlSettings`.
2. Override `RegisterSettings` and call `base.RegisterSettings(settings)` first.
3. Create a recognition control class extending `BaseRecognitionControl`.
4. Implement `GetRecipesData()`, `Initialize()`, `WarmUp()`, `ProcessImage()`.
5. Override `Cleanup()` to release resources.
6. (Optional) Override `HotReload()` and set up file watching with `RaiseRecipeFileChanged()`.
7. Register both classes in the DI container (bind settings to 3 types, bind control as singleton).
8. Ensure `ProcessImage` respects `CancellationToken` and returns `null` when cancelled/exhausted.