check point

This commit is contained in:
meelstorm
2025-07-14 12:03:59 +02:00
commit d3cb790bd9
431 changed files with 44078 additions and 0 deletions

484
.gitignore vendored Normal file
View File

@@ -0,0 +1,484 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from `dotnet new gitignore`
# dotenv files
.env
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Mono auto generated files
mono_crash.*
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Ww][Ii][Nn]32/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUnit
*.VisualState.xml
TestResult.xml
nunit-*.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET
project.lock.json
project.fragment.lock.json
artifacts/
# Tye
.tye/
# ASP.NET Scaffolding
ScaffoldingReadMe.txt
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.tlog
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Coverlet is a free, cross platform Code Coverage Tool
coverage*.json
coverage*.xml
coverage*.info
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# NuGet Symbol Packages
*.snupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
*.appxbundle
*.appxupload
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
*- [Bb]ackup.rdl
*- [Bb]ackup ([0-9]).rdl
*- [Bb]ackup ([0-9][0-9]).rdl
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
*.vbp
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
*.dsw
*.dsp
# Visual Studio 6 technical files
*.ncb
*.aps
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# Visual Studio History (VSHistory) files
.vshistory/
# BeatPulse healthcheck temp database
healthchecksdb
# Backup folder for Package Reference Convert tool in Visual Studio 2017
MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd
# VS Code files for those working on multiple tools
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
# Local History for Visual Studio Code
.history/
# Windows Installer files from build outputs
*.cab
*.msi
*.msix
*.msm
*.msp
# JetBrains Rider
*.sln.iml
.idea/
##
## Visual studio for Mac
##
# globs
Makefile.in
*.userprefs
*.usertasks
config.make
config.status
aclocal.m4
install-sh
autom4te.cache/
*.tar.gz
tarballs/
test-results/
# Mac bundle stuff
*.dmg
*.app
# content below from: https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
# content below from: https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
# Windows thumbnail cache files
Thumbs.db
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# Vim temporary swap files
*.swp

View File

@@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.6.0.20220608" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Hawkeye.VisionBuilder.Workflow\Hawkeye.VisionBuilder.Workflow.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="gold_b97.hrcp">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,11 @@
using System.Xml;
using Hawkeye.VisionBuilder.Workflow;
WorkflowList workflowList = new WorkflowList();
var file = File.OpenRead("gold_b97.hrcp");
using (var br = new BinaryReader(file))
{
workflowList.Load(br,new OperationDiscoveryService(workflowList));
}

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,100 @@
using OpenCvSharp;
using System.Drawing;
using Ninject;
using VisionBuilder.UI.Common.RecipeProcessing;
namespace Hawkeye.VisionBuilder.UI.Sources.Emulation
{
public class EmulationCameraImageSource:IImageSource, IInitializable
{
private readonly EmulationSettings _settings;
private readonly string _name;
private string _currentFolder;
private string[] _selectedFiles;
public EmulationCameraImageSource(EmulationSettings settings)
{
_settings = settings;
}
public string GetName()
{
return _name;
}
public async Task<Mat> GetImage(CancellationToken token)
{
if (_selectedFiles.Length == 0) return null;
var id = (_index) % _selectedFiles.Length;
if (id == 0)
{
if (_index > 0)
OnLoopOver();
if (_settings.SingleRun && _index > 0)
{
// infinite sleep
await Task.Delay(Timeout.Infinite, token);
}
Console.WriteLine("starting new emulation cycle");
await Task.Delay(_settings.CycleDelay, token);
}
else
{
await Task.Delay(_settings.PerImageDelay, token);
}
var path = _selectedFiles[id];
_index++;
Console.WriteLine($"Getting image {path}");
return Cv2.ImRead(path);
}
public void SetFolder(string imagesFolder)
{
try
{
_currentFolder = imagesFolder;
var path = Path.Combine(@"..\Data\Emulation", imagesFolder);
_selectedFiles = Directory.GetFiles(path, "*.bmp")
.Concat(Directory.GetFiles(path, "*.png"))
.Concat(Directory.GetFiles(path, "*.jpg"))
.Where(x => !Path.GetFileNameWithoutExtension(x).Contains("_analysis"))
.ToArray();
}
catch
{
}
}
public void SetFolderAbsolute(string imagesFolder)
{
try
{
_currentFolder = imagesFolder;
var path = imagesFolder;
_selectedFiles = Directory.GetFiles(path, "*.bmp")
.Concat(Directory.GetFiles(path, "*.png"))
.Concat(Directory.GetFiles(path, "*.jpg"))
.Where(x => !Path.GetFileNameWithoutExtension(x).Contains("_analysis"))
.ToArray();
}
catch
{
}
}
public event Action OnLoopOver = delegate { };
private int _index = 0;
public void Initialize()
{
if(string.IsNullOrEmpty(_settings.EmulationPath)) return;
SetFolderAbsolute(_settings.EmulationPath);
}
}
}

View File

@@ -0,0 +1,27 @@
using Inspectron.Settings;
using VisionBuilder.UI.Common;
namespace Hawkeye.VisionBuilder.UI.Sources.Emulation;
public class EmulationSettings : ISettings
{
public string CameraName { get; }
public EmulationSettings(string cameraName)
{
CameraName = cameraName;
}
public int CycleDelay { get; set; } = 1;
public int PerImageDelay { get; set; } = 5000;
public bool SingleRun { get; set; } = false;
public string EmulationPath { get; set; }
public void RegisterSettings(InspectronSettings settings)
{
settings.RegisterSimple(this, () => CycleDelay, CameraName+"/Emulation", nameof(CycleDelay));
settings.RegisterSimple(this, () => PerImageDelay, CameraName + "/Emulation", nameof(PerImageDelay));
settings.RegisterSimple(this, () => SingleRun, CameraName + "/Emulation", nameof(SingleRun));
settings.RegisterSimple(this, () => EmulationPath, CameraName + "/Emulation", nameof(EmulationPath));
}
}

View File

@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\VisionBuilder.UI.Common\VisionBuilder.UI.Common.csproj" />
</ItemGroup>
</Project>

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,284 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Links;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using Microsoft.Scripting.Hosting;
using Ninject.Infrastructure.Language;
using OpenCvSharp;
using System.Diagnostics;
using System.Reflection;
using System.Runtime;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Linq;
using RectangleElement = Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle.RectangleElement;
namespace Hawkeye.VisionBuilder.Workflow
{
public abstract class BaseOperation
{
private static readonly ScriptScope _scope;
private static readonly ScriptEngine _engine;
static BaseOperation()
{
_engine = IronPython.Hosting.Python.CreateEngine();
_scope = _engine.CreateScope();
}
protected BaseOperation()
{
TypeName = MakeOperationName(this.GetType().Name);
Label = MakeOperationName(this.GetType().Name,false);
}
public static void SetScriptValue<T>(string name, T value)
{
_scope.SetVariable(name,value);
}
public static T GetScriptValue<T>(ScriptValue scriptValue)
{
return _engine.Execute<T>(scriptValue.Script,_scope);
}
[NotForTool]
public bool CanHaveProcessingError { get; set; } = true;
[NotForTool]
public string Status { get; set; }
protected bool CheckImageExists(Context context)
{
if (context.ActiveImage == null)
{
Status = "No image available";
Result = false;
return false;
}
return true;
}
protected bool CheckColorful(Context context)
{
if (context.ActiveImage.ImageData.Type() != MatType.CV_8UC3)
{
Status="Image is not colorful";
Result = false;
return false;
}
return true;
}
protected bool CheckGrayscale(Context context)
{
if (context.ActiveImage.ImageData.Type() != MatType.CV_8UC1)
{
Status="Image is not grayscale";
Result = false;
return false;
}
return true;
}
[NotForTool]
public bool NeedsUpdate { get; set; } = true;
[NotForTool]
public Guid Id { get; protected set; }= Guid.NewGuid();
public static string MakeOperationName(string typeName,bool useSpaces=true)
{
return Regex.Replace(typeName.Replace("Operation", ""), "(\\B[A-Z]+?(?=[A-Z][^A-Z])|\\B[A-Z]+?(?=[^A-Z]))", (useSpaces?" ":"")+"$1");
}
public bool Result { get; set; } = true;
public string ResultString
{
get
{
return this.ToString();
}
}
[NotForTool]
public string TypeName { get; private set; }
[NotForTool]
public string Label { get; set; }
public TimeSpan ExecutionTime { get; set; }
public string ExecutionTimeString
{
get
{
return ((int)ExecutionTime.TotalMilliseconds) + " ms";
}
}
public void Interpret(Context context)
{
Result = true;
Status = "";
Stopwatch sw = Stopwatch.StartNew();
InterpretInternal(context);
sw.Stop();
ExecutionTime = sw.Elapsed;
_scope.SetVariable(Label,this);
}
protected abstract void InterpretInternal(Context context);
public Dictionary<string, object> GetParameters()
{
var props = this.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly)
.Where(x=>!x.HasAttribute(typeof(NotForToolAttribute)));
return props.ToDictionary(x => x.Name, x => x.GetValue(this)!);
}
public virtual void SetParameters(Dictionary<string, object> parameters)
{
foreach (KeyValuePair<string, object> pair in parameters)
{
this.GetType().GetProperty(pair.Key,
BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly)
?.SetValue(this,pair.Value);
}
}
public virtual void SaveXML(XmlWriter writer)
{
writer.WriteAttributeString("Id", Id.ToString());
writer.WriteAttributeString("Label", Label);
// Save all public properties that are not marked with NotForTool
var properties = GetParameters();
foreach (var property in properties)
{
if (property.Value != null)
{
writer.WriteStartElement("Property");
writer.WriteAttributeString("Name", property.Key);
writer.WriteAttributeString("Type", property.Value.GetType().FullName);
// Handle different types appropriately
if (property.Value is string || property.Value.GetType().IsPrimitive || property.Value.GetType().IsEnum)
{
writer.WriteString(property.Value.ToString());
}
else if (property.Value is Guid guid)
{
writer.WriteString(guid.ToString());
}
else
{
// For complex types, convert to string representation
writer.WriteString(property.Value.ToString());
}
writer.WriteEndElement();
}
}
}
public virtual void LoadXML(XmlReader reader)
{
if (reader.GetAttribute("Id") != null)
Id = new Guid(reader.GetAttribute("Id"));
if (reader.GetAttribute("Label") != null)
Label = reader.GetAttribute("Label");
var properties = new Dictionary<string, object>();
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && reader.Name == "Property")
{
string propertyName = reader.GetAttribute("Name");
string propertyType = reader.GetAttribute("Type");
string propertyValue = reader.ReadElementContentAsString();
if (!string.IsNullOrEmpty(propertyName) && !string.IsNullOrEmpty(propertyValue))
{
// Convert string value back to appropriate type
object convertedValue = ConvertFromString(propertyValue, propertyType);
if (convertedValue != null)
{
properties[propertyName] = convertedValue;
}
}
}
else if (reader.NodeType == XmlNodeType.EndElement && reader.Name == GetOperationElementName())
{
break;
}
}
if (properties.Count > 0)
{
SetParameters(properties);
}
}
private string GetOperationElementName()
{
return this.GetType().Name;
}
private object ConvertFromString(string value, string typeName)
{
try
{
var type = Type.GetType(typeName);
if (type == null) return value;
if (type == typeof(string))
return value;
else if (type == typeof(int))
return int.Parse(value);
else if (type == typeof(double))
return double.Parse(value);
else if (type == typeof(float))
return float.Parse(value);
else if (type == typeof(bool))
return bool.Parse(value);
else if (type == typeof(Guid))
return new Guid(value);
else if (type.IsEnum)
return Enum.Parse(type, value);
else
return value; // Fallback to string
}
catch
{
return value; // Fallback to string if conversion fails
}
}
public override string ToString()
{
return Status;
}
public virtual void Save(BinaryWriter bw)
{
bw.Write(Id.ToString());
bw.Write(Label);
}
public virtual void Load(BinaryReader br)
{
Id = new Guid(br.ReadString());
Label = br.ReadString();
}
public void SetError(string eMessage)
{
Status=eMessage;
Result = false;
}
}
}

View File

@@ -0,0 +1,24 @@
namespace Hawkeye.VisionBuilder.Workflow;
public class BaseOperationDecorator
{
private readonly BaseOperation _baseOperation;
public BaseOperationDecorator(BaseOperation baseOperation)
{
_baseOperation = baseOperation;
}
public bool Result => Operation.Result;
public string ResultString => Operation.ToString();
public string TypeName => Operation.TypeName;
public string Label
{
get => Operation.Label;
set => Operation.Label = value;
}
public string ExecutionTimeString => Operation.ExecutionTimeString;
public BaseOperation Operation => _baseOperation;
}

View File

@@ -0,0 +1,157 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
Hawkeye.VisionBuilder.Workflow is a computer vision workflow system built on .NET 8.0 that processes images through configurable operation pipelines. The system supports both CPU and GPU configurations with ONNX runtime for AI models.
## Build Configuration
The project uses conditional package references based on configuration:
- **CPU Configuration**: Uses `YoloV8` (4.1.5) and `Microsoft.ML.OnnxRuntime` (1.18.0)
- **GPU Configuration**: Uses `YoloV8.Gpu` (4.1.7) and `Microsoft.ML.OnnxRuntime.Gpu` (1.18.0)
Build the project with:
```bash
dotnet build -c Debug # For GPU configuration
dotnet build -c CPU # For CPU-only configuration
```
## Core Architecture
### Operation System
- **BaseOperation**: Abstract base class for all image processing operations. All operations inherit from this and implement `InterpretInternal(Context context)`
- **Context**: Holds the current processing state including `ActiveImage`, `LastCameraImage`, `GraphicsElements`, and image `Memory`
- **OperationDiscoveryService**: Dynamically discovers and instantiates operations using reflection and Ninject DI
### Operation Categories
Operations are organized in folders by category:
- `AI/`: AI model operations (YOLO, ONNX models, color classification)
- `Simple/`: Basic operations (blob detection, edge finding)
- `Filters/`: Image filtering operations
- `Morphology/`: Morphological operations (dilation, erosion, etc.)
- `Image/`: Image manipulation (convert, channels, etc.)
- `Basic/`: Fundamental operations (threshold, blur, etc.)
### Key Components
- **HawkeyeImage**: Wrapper around OpenCV Mat with filename metadata
- **WorkflowConfiguration**: Manages camera types, Python paths, outputs, and recipe selection
- **IImageSource**: Interface for image input sources with event-driven architecture
- **Graphics Elements**: Drawable elements (rectangles, lines, polygons) for ROI definition
### Python Integration
- Uses IronPython for scripting capabilities
- Python models executed via `PythonModelProxy` and `PythonModelProxyRPC`
- Default Python path: `~/miniconda3/envs/ai3_cpu/python.exe`
### Operation Attributes
- `[Category("name")]`: Groups operations in UI
- `[NotForTool]`: Excludes properties from parameter serialization
- `[IgnoreOperation]`: Excludes operations from discovery
## Development Patterns
### Creating New Operations
1. Inherit from `BaseOperation`
2. Add `[Category("CategoryName")]` attribute
3. Implement `InterpretInternal(Context context)` method
4. Use helper methods: `CheckImageExists()`, `CheckColorful()`, `CheckGrayscale()`
5. Set `Result` and `Status` properties for operation feedback
### Image Processing Flow
1. Operations receive `Context` with current `ActiveImage`
2. Process image using OpenCV operations
3. Update context with results (new images, graphics elements)
4. Set operation status and result state
### Memory Management
Operations can store/retrieve images using `context.Memory` dictionary for intermediate results.
## Recipe Serialization System
The workflow system supports two serialization formats for saving and loading recipes:
### Binary Serialization (.hrcp files)
The original format using `BinaryWriter` and `BinaryReader`:
- **WorkflowList.Save(BinaryWriter)**: Saves configuration, operations, and recipe image
- **WorkflowList.Load(BinaryReader, OperationDiscoveryService)**: Loads complete workflow
- **BaseOperation.Save(BinaryWriter)**: Saves operation ID and label
- **BaseOperation.Load(BinaryReader)**: Loads operation ID and label
### XML Serialization (.xhrcp files)
The new human-readable format using `XmlWriter` and `XmlReader`:
- **WorkflowList.SaveXML(XmlWriter)**: Saves complete workflow as structured XML
- **WorkflowList.LoadXML(XmlReader, OperationDiscoveryService)**: Loads workflow from XML
- **BaseOperation.SaveXML(XmlWriter)**: Saves operation properties and metadata
- **BaseOperation.LoadXML(XmlReader)**: Loads operation properties with type conversion
### XML Structure
```xml
<Workflow>
<Configuration RuntimeCameraType="..." DevelopmentCameraType="..." ... />
<Operations Count="N">
<Operation Type="AssemblyQualifiedName" Id="..." Label="...">
<Property Name="PropertyName" Type="System.Int32">Value</Property>
...
</Operation>
</Operations>
<RecipeImage Length="N">Base64EncodedImageData</RecipeImage>
</Workflow>
```
### Implementing Custom Operation Serialization
For operations that need custom XML serialization beyond the default property serialization:
#### Override SaveXML for Custom Save Logic:
```csharp
public override void SaveXML(XmlWriter writer)
{
base.SaveXML(writer); // Save standard properties
// Custom serialization logic
writer.WriteStartElement("CustomData");
writer.WriteAttributeString("SpecialProperty", SpecialValue.ToString());
writer.WriteEndElement();
}
```
#### Override LoadXML for Custom Load Logic:
```csharp
public override void LoadXML(XmlReader reader)
{
base.LoadXML(reader); // Load standard properties
// Custom deserialization logic
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && reader.Name == "CustomData")
{
SpecialValue = reader.GetAttribute("SpecialProperty");
}
else if (reader.NodeType == XmlNodeType.EndElement && reader.Name == GetOperationElementName())
{
break;
}
}
}
```
#### Override GetOperationElementName for Custom Element Names:
```csharp
protected override string GetOperationElementName()
{
return "MyCustomOperation"; // Used for XML end element detection
}
```
### Supported Property Types in XML Serialization
The default XML serialization handles these types automatically:
- Primitive types (int, double, float, bool)
- String
- Guid
- Enums
- Complex types (converted to string representation)
For complex types requiring special serialization, override the SaveXML/LoadXML methods.

View File

@@ -0,0 +1,44 @@
using Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses;
public class CategoryItem:IFilterParent
{
public string Name { get; set; }
public List<IFilter> Filters { get; set; } = new List<IFilter>();
public void Save(BinaryWriter writer)
{
writer.Write(Name);
writer.Write(Filters.Count);
foreach (var filter in Filters)
{
var type = filter.GetType();
writer.Write(type.FullName);
filter.Save(writer);
}
}
public static CategoryItem Load(BinaryReader reader)
{
var category = new CategoryItem();
category.Name = reader.ReadString();
var count = reader.ReadInt32();
for (int i = 0; i < count; i++)
{
var typeName = reader.ReadString();
var type = Type.GetType(typeName);
var filter = (IFilter)Activator.CreateInstance(type);
filter.Load(reader);
filter.Parent = category;
category.Filters.Add(filter);
}
return category;
}
public override string ToString()
{
return Name;
}
}

View File

@@ -0,0 +1,33 @@
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses;
public class CategoryTree
{
public List<CategoryItem> CategoryItems { get; set; }=new List<CategoryItem>();
public void Save(BinaryWriter writer)
{
writer.Write(CategoryItems.Count);
foreach (var category in CategoryItems)
{
category.Save(writer);
}
}
public static CategoryTree Load(BinaryReader reader)
{
var tree = new CategoryTree();
var count = reader.ReadInt32();
for (int i = 0; i < count; i++)
{
tree.CategoryItems.Add(CategoryItem.Load(reader));
}
return tree;
}
public static CategoryTree LoadFromFile(string path)
{
using var file = File.OpenRead(path);
using var reader = new BinaryReader(file);
return Load(reader);
}
}

View File

@@ -0,0 +1,47 @@
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses
{
public class ColorConfig
{
private int _intensityMaxGrey;
public ColorConfig()
{
MaxBlobSize = 2000;
HueHighSaturationMinGrey = 20;
HueHighSaturationMaxGrey = 50;
HighSaturationMinGrey = 100;
HighSaturationMaxGrey = 255;
ClosingCircleRadius = 8.0;
IntensityMaxGrey = 255;
IntensityMinGrey = 0;
MedianRadius = 10;
MedianMargin = 1;
UseMedianFilter = true;
}
public int MaxBlobSize { get; set; }
public int HueHighSaturationMinGrey { get; set; }
public int HueHighSaturationMaxGrey { get; set; }
public int HighSaturationMinGrey { get; set; }
public int HighSaturationMaxGrey { get; set; }
public int IntensityMaxGrey
{
get { return _intensityMaxGrey; }
set { _intensityMaxGrey = value; }
}
public int IntensityMinGrey { get; set; }
public double ClosingCircleRadius { get; set; }
public int MedianRadius { get; set; }
public int MedianMargin { get; set; }
public bool UseMedianFilter { get; set; }
public bool GroupColorEnabled { get; set; }
public List<HSIValue> GroupColorValues { get; set; }=new List<HSIValue>();
public HSIValue HSI { get; set; }
}
}

View File

@@ -0,0 +1,52 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class AddFilterItem : IFilter, IFilterParent
{
public IFilterParent Parent { get; set; }
public List<IFilter> Filters { get; set; } = new List<IFilter>();
public void Transform(ref Mat image, ref Mat res)
{
using var tracker = new ResourcesTracker();
var temp = tracker.NewMat(res.Size(), MatType.CV_8UC1, Scalar.Black);
foreach (var filter in Filters)
{
filter.Transform(ref image, ref temp);
}
res = res + temp;
temp.Dispose();
}
public void Save(BinaryWriter bw)
{
bw.Write(Filters.Count);
foreach (var filter in Filters)
{
bw.Write(filter.GetType().FullName);
filter.Save(bw);
}
}
public void Load(BinaryReader reader)
{
int count = reader.ReadInt32();
for (int i = 0; i < count; i++)
{
var type = reader.ReadString();
var filter = (IFilter)Activator.CreateInstance(Type.GetType(type));
filter.Load(reader);
filter.Parent = this;
Filters.Add(filter);
}
}
public bool IsSelected { get; set; }
public override string ToString()
{
return "Add";
}
}

View File

@@ -0,0 +1,66 @@
using System.Text;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class BlobsFilterItem:IFilter
{
public int MinArea { get; set; } = 0;
public int MaxArea { get; set; } = 999999;
public int MinRadius { get; set; } = 0;
public int MaxRadius { get; set; } = 999999;
public IFilterParent Parent { get; set; }
public void Transform(ref Mat image, ref Mat res)
{
var contours = res
.FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple);
Mat result = new Mat(res.Rows, res.Cols, MatType.CV_8U, new Scalar(0));
foreach (Point[] contour in contours)
{
var area = Cv2.ContourArea(contour);
if (area < MinArea || area > MaxArea)
continue;
var radius = GetContourRadius(contour);
if (radius < MinRadius || radius > MaxRadius)
continue;
Cv2.DrawContours(result, new[] { contour }, -1, new Scalar(255), -1);
}
res = result;
}
int GetContourRadius(Point[] contour)
{
Cv2.MinEnclosingCircle(contour, out var center, out var radius);
return (int)radius;
}
public void Save(BinaryWriter writer)
{
writer.Write(MinArea);
writer.Write(MaxArea);
writer.Write(MinRadius);
writer.Write(MaxRadius);
}
public void Load(BinaryReader reader)
{
MinArea = reader.ReadInt32();
MaxArea = reader.ReadInt32();
MinRadius = reader.ReadInt32();
MaxRadius = reader.ReadInt32();
}
public bool IsSelected { get; set; }
public override string ToString()
{
StringBuilder sb = new StringBuilder();
sb.Append("Blobs: ");
sb.Append($"Area:{MinArea}-{MaxArea}, ");
sb.Append($"Radius:{MinRadius}-{MaxRadius}");
return sb.ToString();
}
}

View File

@@ -0,0 +1,30 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class BlurFilterItem:IFilter
{
public IFilterParent Parent { get; set; }
public int Radius { get; set; } = 15;
public void Transform(ref Mat image, ref Mat res)
{
image = image.Blur(new Size(Radius, Radius));
}
public void Save(BinaryWriter writer)
{
writer.Write(Radius);
}
public void Load(BinaryReader reader)
{
Radius = reader.ReadInt32();
}
public bool IsSelected { get; set; }
public override string ToString()
{
return "Blur "+Radius;
}
}

View File

@@ -0,0 +1,13 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public interface IFilter
{
public IFilterParent Parent { get; set; }
void Transform(ref Mat image, ref Mat res);
void Save(BinaryWriter writer);
void Load(BinaryReader reader);
public bool IsSelected { get; set; }
}

View File

@@ -0,0 +1,6 @@
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public interface IFilterParent
{
public List<IFilter> Filters { get; set; }
}

View File

@@ -0,0 +1,52 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class IntersectionFilterItem : IFilter, IFilterParent
{
public IFilterParent Parent { get; set; }
public List<IFilter> Filters { get; set; } = new List<IFilter>();
public void Transform(ref Mat image, ref Mat res)
{
using var tracker = new ResourcesTracker();
var temp = tracker.NewMat(res.Size(), MatType.CV_8UC1, Scalar.Black);
foreach (var filter in Filters)
{
filter.Transform(ref image, ref temp);
}
res = res & temp;
temp.Dispose();
}
public void Save(BinaryWriter bw)
{
bw.Write(Filters.Count);
foreach (var filter in Filters)
{
bw.Write(filter.GetType().FullName);
filter.Save(bw);
}
}
public void Load(BinaryReader reader)
{
int count = reader.ReadInt32();
for (int i = 0; i < count; i++)
{
var type = reader.ReadString();
var filter = (IFilter)Activator.CreateInstance(Type.GetType(type));
filter.Load(reader);
filter.Parent = this;
Filters.Add(filter);
}
}
public bool IsSelected { get; set; }
public override string ToString()
{
return "Intersection";
}
}

View File

@@ -0,0 +1,78 @@
using System.Diagnostics;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class MorphologyFilterItem:IFilter
{
public enum EMorphologyType
{
Erode,
Dilate,
Close,
Open
}
public int Size { get; set; } = 1;
public EMorphologyType MorphologyType { get; set; }=EMorphologyType.Erode;
public void Load(BinaryReader reader)
{
Size = reader.ReadInt32();
MorphologyType = (EMorphologyType)reader.ReadInt32();
}
public bool IsSelected { get; set; }
public IFilterParent Parent { get; set; }
public MorphologyFilterItem()
{
}
public MorphologyFilterItem(CategoryItem parent)
{
Parent = parent;
}
public void Transform(ref Mat image, ref Mat res)
{
if(Size==0)
return;
var sw=Stopwatch.StartNew();
var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse,new Size((int)(Size), (int)(Size)));
switch (MorphologyType)
{
case EMorphologyType.Dilate:
//Cv2.Dilate(res, res,kernel);
Cv2.MorphologyEx(res, res, MorphTypes.Dilate, kernel);
break;
case EMorphologyType.Erode:
Cv2.Erode(res, res,kernel);
break;
case EMorphologyType.Close:
Cv2.MorphologyEx(res, res, MorphTypes.Close, kernel);
break;
case EMorphologyType.Open:
Cv2.MorphologyEx(res, res, MorphTypes.Open, kernel);
break;
}
sw.Stop();
}
public void Save(BinaryWriter writer)
{
writer.Write(Size);
writer.Write((int)MorphologyType);
}
public override string ToString()
{
return $"{MorphologyType} {Size}";
}
}

View File

@@ -0,0 +1,101 @@
using System.Windows.Markup;
using OpenCvSharp;
using Size = OpenCvSharp.Size;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class RangeFilterItem : IFilter
{
public IFilterParent Parent { get; set; }
public RangeFilterItem(CategoryItem parent)
{
Parent = parent;
}
public RangeFilterItem()
{
}
public int HMin { get; set; } = 0;
public int HMax { get; set; } = 100;
public int SMin { get; set; } = 0;
public int SMax { get; set; } = 255;
public int VMin { get; set; } = 0;
public int VMax { get; set; } = 255;
public int Dilation { get; set; } = 0;
public int Closing { get; set; } = 0;
public override string ToString()
{
return $"H: {HMin}-{HMax}, S: {SMin}-{SMax}, V: {VMin}-{VMax}";
}
public void Transform(ref Mat hsv, ref Mat res)
{
var lower = new Scalar(HMin, SMin, VMin);
var upper = new Scalar(HMax, SMax, VMax);
var mask = hsv.InRange(lower, upper);
if (Dilation > 0)
{
// generate circular kernel
var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(Dilation, Dilation));
mask = mask.Dilate(kernel);
kernel.Dispose();
}
if (Closing > 0)
{
// generate circular kernel
var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(Closing, Closing));
mask = mask.MorphologyEx(MorphTypes.Close, kernel);
kernel.Dispose();
}
res+=mask;
mask.Dispose();
}
public void Save(BinaryWriter writer)
{
writer.Write(HMin);
writer.Write(HMax);
writer.Write(SMin);
writer.Write(SMax);
writer.Write(VMin);
writer.Write(VMax);
writer.Write(Dilation);
writer.Write(Closing);
}
public void Load(BinaryReader reader)
{
HMin = reader.ReadInt32();
HMax = reader.ReadInt32();
SMin = reader.ReadInt32();
SMax = reader.ReadInt32();
VMin = reader.ReadInt32();
VMax = reader.ReadInt32();
Dilation = reader.ReadInt32();
Closing = reader.ReadInt32();
}
public bool IsSelected { get; set; }
public static RangeFilterItem Load(BinaryReader reader, CategoryItem parent)
{
var filter = new RangeFilterItem(parent);
filter.HMin = reader.ReadInt32();
filter.HMax = reader.ReadInt32();
filter.SMin = reader.ReadInt32();
filter.SMax = reader.ReadInt32();
filter.VMin = reader.ReadInt32();
filter.VMax = reader.ReadInt32();
filter.Dilation = reader.ReadInt32();
filter.Closing = reader.ReadInt32();
return filter;
}
}

View File

@@ -0,0 +1,25 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class SobelFilterItem:IFilter
{
public IFilterParent Parent { get; set; }
public void Transform(ref Mat image, ref Mat res)
{
var filtered=image.Sobel(MatType.CV_32F, 1, 1, 3);
filtered.ConvertTo(image, MatType.CV_8UC3);
}
public void Save(BinaryWriter writer)
{
}
public void Load(BinaryReader reader)
{
}
public bool IsSelected { get; set; }
}

View File

@@ -0,0 +1,176 @@
using System.Diagnostics;
using OpenCvSharp;
using System.Drawing;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Hawkeye.VisionBuilder.Workflow.Datatypes;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class SpatialFilterItem : IFilter
{
private Mat? _mask=null;
public IFilterParent Parent { get; set; }
public bool Subtract { get; set; }
public string? FilePath { get; set; }
public void ReloadMask()
{
if (!File.Exists(FilePath))
{
CreateMask();
}else
if (!string.IsNullOrEmpty(FilePath))
{
_mask = new Mat(FilePath,ImreadModes.Grayscale);
}
}
public SpatialFilterItem()
{
}
void CreateMask()
{
_mask = new Mat(13005 + 256, 256, MatType.CV_8UC1, (Scalar)0);
}
public SpatialFilterItem(CategoryItem parent) : this()
{
Parent = parent;
}
public Mat Mask
{
get
{
if (_mask == null)
{
if (FilePath != null)
{
ReloadMask();
}
else
{
CreateMask();
}
}
return _mask;
}
set
{
_mask = value;
}
}
struct PixelStruct
{
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 65536*3)]
public byte[] Pixels;
}
struct PixelStructMask
{
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 3394816)]
public byte[] Pixels;
}
public static Mat lastInput;
public unsafe void Transform(ref Mat image, ref Mat res)
{
//var pixelStruct = Marshal.PtrToStructure<PixelStruct>(new IntPtr(image.DataPointer));
var candyBytes = GetPixels<byte>(image);
var candidates = new Mat(image.Size(), MatType.CV_8UC1, (Scalar)0);
Mat tmp=new Mat();
image.CopyTo(tmp);
lastInput = tmp;
//var pixelStructMask = Marshal.PtrToStructure<PixelStructMask>(new IntPtr(Mask.DataPointer));
var maskBytes = GetPixels<byte>(Mask);
var len = candyBytes.Length;
var w = image.Width;
var mw = Mask.Width;
var lenDiv3 = len / 3;
for (var q = 0; q < lenDiv3; q++)
{
var i = q * 3;
var p = i / 3;
var y = p / w;
var x = p % w;
var yp = (int)candyBytes[i + 0];
var xp = candyBytes[i + 1];
var vp = ((byte) (candyBytes[i + 2] / 5f)) * 255;
var hp = yp + vp;
var maskColor = maskBytes[hp * mw + xp];
if (maskColor > 0)
candidates.Set<byte>(y, x, 255);
}
if (Subtract && !IsSelected)
{
res -= candidates;
}
else
{
res += candidates;
}
}
string Hash(Span<byte> bytes) {
using var sha = SHA256.Create();
var hash = sha.ComputeHash(bytes.ToArray());
return Convert.ToBase64String(hash);
}
unsafe Span<T> GetPixels<T>(Mat image) where T : unmanaged
{
return new Span<T>(image.DataPointer, image.Width * image.Height * image.Channels());
}
public void Save(BinaryWriter writer)
{
writer.Write(FilePath??"");
writer.Write(Subtract);
}
public void Load(BinaryReader reader)
{
FilePath = reader.ReadString();
Console.WriteLine("Loaded path:"+FilePath);
if(FilePath.StartsWith("..\\../"))
FilePath = FilePath.Replace("..\\../", "..\\");
Subtract = reader.ReadBoolean();
}
public bool IsSelected { get; set; }
public override string ToString()
{
var totalColors = 0;
if (Mask != null)
{
totalColors = Mask.CountNonZero();
}
return $"{(Subtract ? "-" : "+")}Spatial Filter({totalColors} colors)";
}
}

View File

@@ -0,0 +1,177 @@
using System.Diagnostics;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using ILGPU;
using ILGPU.Runtime;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
[Obsolete]
public class SpatialFilterItemGPU:IFilter
{
private Mat? _mask = null;
private MemoryBuffer1D<byte, Stride1D.Dense> _maskGpu;
public IFilterParent Parent { get; set; }
public bool Subtract { get; set; }
public string? FilePath { get; set; }
public void ReloadMask()
{
if (!File.Exists(FilePath))
{
CreateMask();
}
else
if (!string.IsNullOrEmpty(FilePath))
{
_mask = new Mat(FilePath).CvtColor(ColorConversionCodes.BGR2GRAY);
}
LoadMaskToGpu();
}
private unsafe void LoadMaskToGpu()
{
var maskBytes = new Span<byte>(Mask.DataPointer, Mask.Width * Mask.Height).ToArray();
if (_maskGpu != null)
{
_maskGpu.Dispose();
}
_maskGpu = Kernels.Accelerator.Allocate1D<byte>(maskBytes);
}
private static MemoryBuffer1D<byte, Stride1D.Dense>? _imageMemory;
private static MemoryBuffer1D<byte, Stride1D.Dense>? _resMemory;
public SpatialFilterItemGPU()
{
}
void CreateMask()
{
_mask = new Mat(13005 + 256, 256, MatType.CV_8UC1, (Scalar)0);
}
public SpatialFilterItemGPU(CategoryItem parent) : this()
{
Parent = parent;
}
public Mat Mask
{
get
{
if (_mask == null)
{
if (FilePath != null)
{
ReloadMask();
}
else
{
CreateMask();
}
}
return _mask;
}
set
{
_mask = value;
}
}
private static Mat? _lastLoadedImage;
public unsafe void Transform(ref Mat image, ref Mat res)
{
var w = image.Width;
var mw = Mask.Width;
if (_lastLoadedImage != image)
{
var imageBytes = new ReadOnlySpan<byte>(image.DataPointer, image.Width * image.Height * 3);
_imageMemory ??= Kernels.Accelerator.Allocate1D<byte>(imageBytes.Length);
_imageMemory.View.BaseView.CopyFromCPU(imageBytes);
_lastLoadedImage = image;
}
var len = (int)_imageMemory.Length;
_resMemory ??= Kernels.Accelerator.Allocate1D<byte>(len / 3);
Kernels.SpatialKernel(len/3, w, mw, _imageMemory.View, _maskGpu.View, _resMemory.View);
var arr = _resMemory.GetAsArray1D();
var candidates=new Mat(new[]{image.Height,image.Width}, MatType.CV_8UC1);
candidates.SetArray(arr);
if (Subtract && !IsSelected)
{
res -= candidates;
}
else
{
res |= candidates;
}
candidates.Dispose();
}
public void Save(BinaryWriter writer)
{
writer.Write(FilePath ?? "");
writer.Write(Subtract);
}
public void Load(BinaryReader reader)
{
FilePath = reader.ReadString();
Console.WriteLine("Loaded path:" + FilePath);
if (FilePath.StartsWith("..\\../"))
FilePath = FilePath.Replace("..\\../", "..\\");
Subtract = reader.ReadBoolean();
}
public bool IsSelected { get; set; }
public override string ToString()
{
var totalColors = 0;
if (Mask != null)
{
totalColors = Mask.CountNonZero();
}
return $"{(Subtract ? "-" : "+")}Spatial Filter({totalColors} colors)";
}
}

View File

@@ -0,0 +1,52 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses.Filters;
public class SubtractFilterItem : IFilter,IFilterParent
{
public IFilterParent Parent { get; set; }
public List<IFilter> Filters { get; set; } = new List<IFilter>();
public void Transform(ref Mat image, ref Mat res)
{
using var tracker = new ResourcesTracker();
var temp = tracker.NewMat(res.Size(),MatType.CV_8UC1,Scalar.Black);
foreach (var filter in Filters)
{
filter.Transform(ref image, ref temp);
}
res = res - temp;
temp.Dispose();
}
public void Save(BinaryWriter bw)
{
bw.Write(Filters.Count);
foreach (var filter in Filters)
{
bw.Write(filter.GetType().FullName);
filter.Save(bw);
}
}
public void Load(BinaryReader reader)
{
int count = reader.ReadInt32();
for (int i = 0; i < count; i++)
{
var type = reader.ReadString();
var filter = (IFilter)Activator.CreateInstance(Type.GetType(type));
filter.Load(reader);
filter.Parent = this;
Filters.Add(filter);
}
}
public bool IsSelected { get; set; }
public override string ToString()
{
return "Subtract";
}
}

View File

@@ -0,0 +1,125 @@
using System.ComponentModel;
using System.Drawing;
using System.Runtime.CompilerServices;
using System.Xml.Serialization;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses
{
public class HSIValue
{
public HSIValue()
{
}
public int MinSize { get; set; } = 1;
public int MinComponentSize { get; set; } = 1;
public int PreFilterId { get; set; } = 0;
public int PreFilterSize { get; set; } = 10;
public int Closing { get; set; }
public int Circle { get; set; } = 1;
public string Tag { get; set; }
private Range<int> _hue = new Range<int>() {Minimum = 0,Maximum = 10};
private Range<int> _saturation = new Range<int>() { Minimum = 0, Maximum = 10 };
private Range<int> _intensity = new Range<int>() { Minimum = 0, Maximum = 10 };
public string Name { get; set; }
public Range<int> Hue
{
get { return _hue; }
set
{
if (Equals(value, _hue)) return;
_hue = value;
OnPropertyChanged();
}
}
public Range<int> Saturation
{
get { return _saturation; }
set
{
if (Equals(value, _saturation)) return;
_saturation = value;
OnPropertyChanged();
}
}
public Range<int> Intensity
{
get { return _intensity; }
set
{
if (Equals(value, _intensity)) return;
_intensity = value;
OnPropertyChanged();
}
}
public bool Active { get; set; }
public List<HSIValue> SubColors { get; set; }=new List<HSIValue>();
public Rectangle GetRectangle()
{
Rectangle th = new Rectangle((int) (Hue.Minimum), (int) (Saturation.Minimum),
(int) (Hue.Maximum) - (int) (Hue.Minimum), (int) (Saturation.Maximum) - (int) (Saturation.Minimum));
return th;
}
[XmlIgnore]
public bool ShowStar { get; set; }
public int Dilation { get; set; }
public override string ToString()
{
return $"{Name} {Hue}:{Saturation}:{Intensity}{(ShowStar?"*":"")}";
}
public static explicit operator ColorConfig(HSIValue value)
{
var config = new ColorConfig()
{
ClosingCircleRadius = value.Circle,
HueHighSaturationMaxGrey = value.Hue.Maximum,
HueHighSaturationMinGrey = value.Hue.Minimum,
HighSaturationMaxGrey = value.Saturation.Maximum,
HighSaturationMinGrey = value.Saturation.Minimum,
IntensityMaxGrey = value.Intensity.Maximum,
IntensityMinGrey = value.Intensity.Minimum,
MaxBlobSize = value.MinSize,
MedianMargin = 1,
MedianRadius = 10,
UseMedianFilter = true,
};
return config;
}
public static implicit operator HSIValue(ColorConfig config)
{
return new HSIValue()
{
Hue = new Range<int>(config.HueHighSaturationMinGrey, config.HueHighSaturationMaxGrey),
Saturation = new Range<int>(config.HighSaturationMinGrey, config.HighSaturationMaxGrey),
Intensity = new Range<int>(config.IntensityMinGrey, config.IntensityMaxGrey),
Circle = (int)config.ClosingCircleRadius,
MinSize = config.MaxBlobSize
};
}
public event PropertyChangedEventHandler PropertyChanged;
protected virtual void OnPropertyChanged([CallerMemberName] string propertyName = null)
{
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
}
}
}

View File

@@ -0,0 +1,157 @@
using ILGPU;
using ILGPU.Runtime;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses;
public static class Kernels
{
static Kernels()
{
Init();
AppDomain.CurrentDomain.ProcessExit +=
Destruct;
}
private static void Destruct(object? sender, EventArgs e)
{
_accelerator.Dispose();
_gpuContext.Dispose();
}
private static object _lock = new object();
private static ILGPU.Context _gpuContext;
private static Accelerator _accelerator;
private static Action<Index1D, int, int, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>> _spatialKernel;
private static Action<Index1D, int, int, ArrayView<byte>, ArrayView<byte>> _dilationKernel;
public static Action<Index1D, int, int, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>> SpatialKernel => _spatialKernel;
public static Action<Index1D, int, int, ArrayView<byte>, ArrayView<byte>> Dilation => _dilationKernel;
public static Accelerator Accelerator
{
get => _accelerator;
set => _accelerator = value;
}
static Dictionary<Mat, MemoryBuffer1D<byte,Stride1D.Dense>> _imageCache = new ();
public static unsafe MemoryBuffer1D<byte, Stride1D.Dense> TransformImageCached(Mat image)
{
if (_imageCache.ContainsKey(image))
return _imageCache[image];
var imageBytes = new Span<byte>(image.DataPointer, image.Width * image.Height * 3);
var res=Kernels.Accelerator.Allocate1D<byte>(imageBytes.ToArray());
_imageCache[image] = res;
return res;
}
static void Init()
{
lock (_lock)
{
if (_gpuContext != null) return;
_gpuContext = ILGPU.Context.Create(builder => builder.AllAccelerators());
_accelerator = _gpuContext.Devices.First(x => x.AcceleratorType == AcceleratorType.Cuda)
.CreateAccelerator(_gpuContext);
Console.WriteLine("GPU acceleration initialized");
_spatialKernel = _accelerator
.LoadAutoGroupedStreamKernel<Index1D, int, int, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>>(
Kernels.TransformSpatialKernel);
_dilationKernel = _accelerator
.LoadAutoGroupedStreamKernel<Index1D, int, int, ArrayView<byte>,ArrayView<byte>>(
Kernels.DilationKernel);
}
}
private static void TransformSpatialKernel(Index1D index, int w, int mw, ArrayView<byte> candyBytes, ArrayView<byte> maskBytes, ArrayView<byte> res)
{
var i = index;
var p = i * 3;
var y = p / w;
var x = p % w;
var yp = (int)candyBytes[p + 0];
var xp = candyBytes[p + 1];
var vp = (byte)(candyBytes[p + 2] / 5f) * 255;
var hp = yp + vp;
var maskColor = maskBytes[hp * mw + xp];
if (maskColor > 0)
res[i] = 255;
else
res[i] = 0;
}
private static void DilationKernel(Index1D index, int w, int size, ArrayView<byte> maskBytes,
ArrayView<byte> res)
{
var i = index;
var p = i ;
var y = p / w;
var x = p % w;
var halfSize = size-1 / 2;
if (maskBytes[p] > 0)
{
for (int j = -halfSize; j <= halfSize; j++)
{
for (int k = -halfSize; k <= halfSize; k++)
{
var yp = y + j;
var xp = x + k;
if (yp < 0 || yp >= w || xp < 0 || xp >= w)
continue;
res[yp * w + xp] = 255;
}
}
}
}
private static void ErosionKernel(Index1D index, int w, int size, ArrayView<byte> maskBytes,
ArrayView<byte> res)
{
var i = index;
var p = i ;
var y = p / w;
var x = p % w;
var halfSize = size-1 / 2;
if (maskBytes[p] <255)
{
for (int j = -halfSize; j <= halfSize; j++)
{
for (int k = -halfSize; k <= halfSize; k++)
{
var yp = y + j;
var xp = x + k;
if (yp < 0 || yp >= w || xp < 0 || xp >= w)
continue;
res[yp * w + xp] = 0;
}
}
}
}
}

View File

@@ -0,0 +1,64 @@
namespace Hawkeye.VisionBuilder.Workflow.ColorStudioClasses
{
/// <summary>The Range class.</summary>
/// <typeparam name="T">Generic parameter.</typeparam>
public class Range<T> where T : IComparable<T>
{
public Range()
{
}
public Range(T min,T max)
{
this.Maximum = max;
this.Minimum = min;
}
/// <summary>Minimum value of the range.</summary>
public T Minimum { get; set; }
/// <summary>Maximum value of the range.</summary>
public T Maximum { get; set; }
/// <summary>Presents the Range in readable format.</summary>
/// <returns>String representation of the Range</returns>
public override string ToString()
{
return string.Format("[{0} - {1}]", this.Minimum, this.Maximum);
}
/// <summary>Determines if the range is valid.</summary>
/// <returns>True if range is valid, else false</returns>
public bool IsValid()
{
return this.Minimum.CompareTo(this.Maximum) <= 0;
}
/// <summary>Determines if the provided value is inside the range.</summary>
/// <param name="value">The value to test</param>
/// <returns>True if the value is inside Range, else false</returns>
public bool ContainsValue(T value)
{
return (this.Minimum.CompareTo(value) <= 0) && (value.CompareTo(this.Maximum) <= 0);
}
/// <summary>Determines if this Range is inside the bounds of another range.</summary>
/// <param name="Range">The parent range to test on</param>
/// <returns>True if range is inclusive, else false</returns>
public bool IsInsideRange(Range<T> range)
{
return this.IsValid() && range.IsValid() && range.ContainsValue(this.Minimum) && range.ContainsValue(this.Maximum);
}
/// <summary>Determines if another range is inside the bounds of this range.</summary>
/// <param name="Range">The child range to test</param>
/// <returns>True if range is inside, else false</returns>
public bool ContainsRange(Range<T> range)
{
return this.IsValid() && range.IsValid() && this.ContainsValue(range.Minimum) && this.ContainsValue(range.Maximum);
}
}
}

View File

@@ -0,0 +1,20 @@
namespace Hawkeye.VisionBuilder.Workflow.Configuration;
public class BadTweakLearning
{
public bool Enabled { get; set; } = false;
public int ThresholdLevel { get; set; } = 80;
public void Save(BinaryWriter bw)
{
bw.Write(Enabled);
bw.Write(ThresholdLevel);
}
public void Load(BinaryReader br)
{
Enabled = br.ReadBoolean();
ThresholdLevel = br.ReadInt32();
}
}

View File

@@ -0,0 +1,8 @@
namespace Hawkeye.VisionBuilder.Workflow.Configuration;
public enum ECameraType
{
Emulation=0,
Hawkeye=1
}

View File

@@ -0,0 +1,8 @@
namespace Hawkeye.VisionBuilder.Workflow.Configuration;
public enum EOutputs
{
None,
Firmata,
NetServer
}

View File

@@ -0,0 +1,19 @@
namespace Hawkeye.VisionBuilder.Workflow.Configuration;
public class RecipeSelectionConfig
{
public bool SerialEnabled { get; set; } = false;
public string ComPort { get; set; } = "COM3";
public void Save(BinaryWriter bw)
{
bw.Write(SerialEnabled);
bw.Write(ComPort);
}
public void Load(BinaryReader br)
{
SerialEnabled = br.ReadBoolean();
ComPort = br.ReadString();
}
}

View File

@@ -0,0 +1,58 @@
using System.Reflection.Metadata.Ecma335;
using System.Security.AccessControl;
using Hawkeye.VisionBuilder.Workflow.Operations;
namespace Hawkeye.VisionBuilder.Workflow.Configuration;
public class WorkflowConfiguration
{
public WorkflowConfiguration()
{
var userFolder = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
PythonPath = Path.Combine(userFolder, @"miniconda3\envs\ai3_cpu\python.exe");
}
public ECameraType RuntimeCameraType { get; set; } = ECameraType.Emulation;
public ECameraType DevelopmentCameraType { get; set; } = ECameraType.Emulation;
public string EmulationPath { get; set; } = "..\\Data\\Emulation";
public string PythonPath { get; set; }
public EOutputs Outputs { get; set; }
public int ResultPin { get; set; }
public string SerialPort { get; set; } = "";
public int Delay { get; set; } = 0;
public BadTweakLearning BadTweakLearning { get; set; } = new BadTweakLearning();
public RecipeSelectionConfig RecipeSelectionConfig { get; set; }=new RecipeSelectionConfig();
public void Save(BinaryWriter bw)
{
bw.Write((int)RuntimeCameraType);
bw.Write((int)DevelopmentCameraType);
bw.Write((int)Outputs);
bw.Write(EmulationPath);
bw.Write(PythonPath);
bw.Write(ResultPin);
bw.Write(SerialPort);
bw.Write(Delay);
BadTweakLearning.Save(bw);
RecipeSelectionConfig.Save(bw);
}
public void Load(BinaryReader br)
{
RuntimeCameraType = (ECameraType) br.ReadInt32();
DevelopmentCameraType = (ECameraType) br.ReadInt32();
Outputs = (EOutputs) br.ReadInt32();
EmulationPath = br.ReadString();
PythonPath = br.ReadString();
ResultPin = br.ReadInt32();
SerialPort = br.ReadString();
Delay = br.ReadInt32();
BadTweakLearning.Load(br);
RecipeSelectionConfig.Load(br);
}
}

View File

@@ -0,0 +1,13 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements;
namespace Hawkeye.VisionBuilder.Workflow;
public class Context
{
public HawkeyeImage? ActiveImage { get; set; }
public HawkeyeImage? LastCameraImage { get; set; }
public List<IGraphicsElement> GraphicsElements { get; set; }=new List<IGraphicsElement>();
public Dictionary<string, HawkeyeImage> Memory { get; set; } = new Dictionary<string, HawkeyeImage>();
}

View File

@@ -0,0 +1,55 @@
using NetMQ;
using NetMQ.Sockets;
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
public class CSharpDataTransferMQ
{
private object _syncObject = new object();
private readonly ResponseSocket _server;
public CSharpDataTransferMQ()
{
this._server = new ResponseSocket();
_server.Bind("tcp://*:5555");
}
public byte[] TransferData(string command, byte mode, byte[] data)
{
lock (_syncObject)
{
return TransferDataInternal(command,mode,data);
}
}
private byte[] TransferDataInternal(string command,byte mode,byte[] data)
{
Console.WriteLine("Waiting for request");
_server.ReceiveFrameBytes();
MemoryStream ms = new MemoryStream();
BinaryWriter bw = new BinaryWriter(ms);
var functionBytes=System.Text.Encoding.ASCII.GetBytes(command);
bw.Write(functionBytes.Length);
bw.Write(functionBytes);
bw.Write(mode);
bw.Write(data);
Console.WriteLine("Sending response");
_server.SendFrame(ms.ToArray());
Console.WriteLine("Waiting for request");
var result= _server.ReceiveFrameBytes();
_server.SendFrame("OK");
return result;
}
}

View File

@@ -0,0 +1,130 @@
using MessagePack;
using NetMQ;
using NetMQ.Sockets;
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
[MessagePackObject]
public class MethodCall
{
[Key(0)]
public string Name { get; set; }
[Key(1)]
public List<object> Parameters { get; set; }
public MethodCall()
{
}
public MethodCall(string name, params object[] parameters)
{
Name=name;
Parameters=new List<object>(parameters);
}
}
[MessagePackObject]
public class TransferData
{
[Key(0)]
public ETransferType DataType { get; set; }
[Key(1)]
public int Int { get; set; }
[Key(2)]
public string String { get; set; }
[Key(3)]
public byte[] Bytes { get; set; }
[Key(4)]
public float[] Floats { get; set; }
public static implicit operator TransferData(int value)
{
return new TransferData()
{
DataType = ETransferType.Int,
Int = value
};
}
public static implicit operator TransferData(string value)
{
return new TransferData()
{
DataType = ETransferType.String,
String = value
};
}
public static implicit operator TransferData(byte[] value)
{
return new TransferData()
{
DataType = ETransferType.Bytes,
Bytes = value
};
}
public static implicit operator TransferData(float[] value)
{
return new TransferData()
{
DataType = ETransferType.Floats,
Floats = value
};
}
}
public enum ETransferType
{
Void=0,
Int=1,
String=2,
Bytes=3,
Floats=4
}
public class CSharpDataTransferMQRPC
{
private object _syncObject = new object();
private readonly ResponseSocket _server;
public CSharpDataTransferMQRPC()
{
this._server = new ResponseSocket();
_server.Bind("tcp://*:5555");
}
public T TransferData<T>(MethodCall call)
{
lock (_syncObject)
{
return TransferDataInternal<T>(call);
}
}
private T TransferDataInternal<T>(MethodCall call)
{
Console.WriteLine("Waiting for request");
_server.ReceiveFrameBytes();
Console.WriteLine("Sending response");
var serialized = MessagePack.MessagePackSerializer.Serialize(call);
_server.SendFrame(serialized);
Console.WriteLine("Waiting for request");
var result = _server.ReceiveFrameBytes();
var decoded=MessagePackSerializer.Deserialize<object>(result);
_server.SendFrame("OK");
return (T)Convert.ChangeType(decoded, typeof(T));
}
}

View File

@@ -0,0 +1,135 @@
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
public static class ChildProcessTracker
{
/// <summary>
/// Add the process to be tracked. If our current process is killed, the child processes
/// that we are tracking will be automatically killed, too. If the child process terminates
/// first, that's fine, too.</summary>
/// <param name="process"></param>
public static void AddProcess(Process process)
{
if (s_jobHandle != IntPtr.Zero)
{
bool success = AssignProcessToJobObject(s_jobHandle, process.Handle);
if (!success && !process.HasExited)
throw new Win32Exception();
}
}
static ChildProcessTracker()
{
// This feature requires Windows 8 or later. To support Windows 7 requires
// registry settings to be added if you are using Visual Studio plus an
// app.manifest change.
// https://stackoverflow.com/a/4232259/386091
// https://stackoverflow.com/a/9507862/386091
if (Environment.OSVersion.Version < new Version(6, 2))
return;
// The job name is optional (and can be null) but it helps with diagnostics.
// If it's not null, it has to be unique. Use SysInternals' Handle command-line
// utility: handle -a ChildProcessTracker
string jobName = "ChildProcessTracker" + Process.GetCurrentProcess().Id;
s_jobHandle = CreateJobObject(IntPtr.Zero, jobName);
var info = new JOBOBJECT_BASIC_LIMIT_INFORMATION();
// This is the key flag. When our process is killed, Windows will automatically
// close the job handle, and when that happens, we want the child processes to
// be killed, too.
info.LimitFlags = JOBOBJECTLIMIT.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
var extendedInfo = new JOBOBJECT_EXTENDED_LIMIT_INFORMATION();
extendedInfo.BasicLimitInformation = info;
int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length);
try
{
Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false);
if (!SetInformationJobObject(s_jobHandle, JobObjectInfoType.ExtendedLimitInformation,
extendedInfoPtr, (uint)length))
{
throw new Win32Exception();
}
}
finally
{
Marshal.FreeHGlobal(extendedInfoPtr);
}
}
[DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
static extern IntPtr CreateJobObject(IntPtr lpJobAttributes, string name);
[DllImport("kernel32.dll")]
static extern bool SetInformationJobObject(IntPtr job, JobObjectInfoType infoType,
IntPtr lpJobObjectInfo, uint cbJobObjectInfoLength);
[DllImport("kernel32.dll", SetLastError = true)]
static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process);
// Windows will automatically close any open job handles when our process terminates.
// This can be verified by using SysInternals' Handle utility. When the job handle
// is closed, the child processes will be killed.
private static readonly IntPtr s_jobHandle;
}
public enum JobObjectInfoType
{
AssociateCompletionPortInformation = 7,
BasicLimitInformation = 2,
BasicUIRestrictions = 4,
EndOfJobTimeInformation = 6,
ExtendedLimitInformation = 9,
SecurityLimitInformation = 5,
GroupInformation = 11
}
[StructLayout(LayoutKind.Sequential)]
public struct JOBOBJECT_BASIC_LIMIT_INFORMATION
{
public Int64 PerProcessUserTimeLimit;
public Int64 PerJobUserTimeLimit;
public JOBOBJECTLIMIT LimitFlags;
public UIntPtr MinimumWorkingSetSize;
public UIntPtr MaximumWorkingSetSize;
public UInt32 ActiveProcessLimit;
public Int64 Affinity;
public UInt32 PriorityClass;
public UInt32 SchedulingClass;
}
[Flags]
public enum JOBOBJECTLIMIT : uint
{
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
}
[StructLayout(LayoutKind.Sequential)]
public struct IO_COUNTERS
{
public UInt64 ReadOperationCount;
public UInt64 WriteOperationCount;
public UInt64 OtherOperationCount;
public UInt64 ReadTransferCount;
public UInt64 WriteTransferCount;
public UInt64 OtherTransferCount;
}
[StructLayout(LayoutKind.Sequential)]
public struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION
{
public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation;
public IO_COUNTERS IoInfo;
public UIntPtr ProcessMemoryLimit;
public UIntPtr JobMemoryLimit;
public UIntPtr PeakProcessMemoryUsed;
public UIntPtr PeakJobMemoryUsed;
}

View File

@@ -0,0 +1,31 @@
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
public static class PythonModelProxy
{
private static bool _isInitialized = false;
private static PythonScriptRunner _pythonScriptRunner;
private static CSharpDataTransferMQ _cSharpDataTransfer;
public static void Initialize(string pythonPath, string scriptFile= "Hawkeye.ModelLoader.py")
{
if (_isInitialized) return;
if (!File.Exists(pythonPath)) return;
_isInitialized = true;
var relativePath = @"..\Data\AI";
var absolutePath = Path.GetFullPath(relativePath);
_cSharpDataTransfer = new CSharpDataTransferMQ();
_pythonScriptRunner = new PythonScriptRunner(pythonPath, scriptFile, absolutePath);
_pythonScriptRunner.Start();
}
public static CSharpDataTransferMQ GetInterface()
{
if (!_isInitialized) return null;
return _cSharpDataTransfer;
}
}

View File

@@ -0,0 +1,38 @@
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
public static class PythonModelProxyRPC
{
private static bool _isInitialized = false;
private static PythonScriptRunner _pythonScriptRunner;
private static CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
public static void Initialize(string scriptFile = @"..\Data\models\Segment128Half.py")
{
if (_isInitialized) return;
var pythonPath = Environment.ExpandEnvironmentVariables(@"%userprofile%\miniconda3\envs\ai4\python.exe");
//var pythonPath = Environment.ExpandEnvironmentVariables(@"/root/ai3/bin/python3");
var scriptRelPath = scriptFile;
var scriptAbsPath = Path.GetFullPath(scriptRelPath);
var workDirectory = Path.GetDirectoryName(scriptAbsPath);
Directory.CreateDirectory(workDirectory);
if (!File.Exists(pythonPath)) return;
_isInitialized = true;
_cSharpDataTransferMQRPC = new CSharpDataTransferMQRPC();
_pythonScriptRunner = new PythonScriptRunner(pythonPath, scriptAbsPath, workDirectory);
_pythonScriptRunner.Start();
}
public static CSharpDataTransferMQRPC GetInterface()
{
if (!_isInitialized) Initialize();
return _cSharpDataTransferMQRPC;
}
}

View File

@@ -0,0 +1,53 @@
using System.Diagnostics;
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
public class PythonScriptRunner
{
private Process process;
private readonly string _pythonPath;
private string scriptPath;
private readonly string _workingDirectory;
public PythonScriptRunner(string pythonPath, string scriptPath,string workingDirectory)
{
_pythonPath = pythonPath;
this.scriptPath = scriptPath;
_workingDirectory = workingDirectory;
}
public void Start()
{
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.FileName = _pythonPath; // set the path to the Python executable if not in PATH
startInfo.Arguments = "\""+scriptPath+ "\"";
startInfo.WorkingDirectory = _workingDirectory;
// enable running in the background
startInfo.UseShellExecute = false;
//startInfo.CreateNoWindow = true;
// create and start the process
process = new Process();
process.StartInfo = startInfo;
process.Start();
// on windows only
if(Environment.OSVersion.Platform == PlatformID.Win32NT)
ChildProcessTracker.AddProcess(process);
}
public void Stop()
{
if (process != null && !process.HasExited)
{
process.Kill();
process.Dispose();
process = null;
}
}
}

View File

@@ -0,0 +1,66 @@
using System.Diagnostics;
namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
public class WslScriptRunner
{
private Process process;
private readonly string _pythonPath;
private string scriptPath;
private readonly string _workingDirectory;
public WslScriptRunner(string pythonPath, string scriptPath,string workingDirectory)
{
_pythonPath = pythonPath;
this.scriptPath = scriptPath;
_workingDirectory = workingDirectory;
}
public static string ConvertToWslPath(string windowsPath)
{
if (string.IsNullOrWhiteSpace(windowsPath))
throw new ArgumentException("Path cannot be null or empty.", nameof(windowsPath));
// Replace backslashes with forward slashes
string wslPath = windowsPath.Replace('\\', '/');
// Extract the drive letter and convert it to WSL format
if (wslPath.Length > 1 && wslPath[1] == ':')
{
char driveLetter = char.ToLower(wslPath[0]);
wslPath = $"/mnt/{driveLetter}{wslPath.Substring(2)}";
}
return wslPath;
}
public void Start()
{
//startInfo.CreateNoWindow = true;
// create and start the process
process = new Process();
var arg =
$@"--distribution ubuntu --user root --cd ""{ConvertToWslPath(_workingDirectory)}"" -- /root/ai3/bin/python3 ""{ConvertToWslPath(scriptPath)}""";
process.StartInfo = new ProcessStartInfo("wsl",arg)
{
UseShellExecute = false
};
process.Start();
// on windows only
if(Environment.OSVersion.Platform == PlatformID.Win32NT)
ChildProcessTracker.AddProcess(process);
}
public void Stop()
{
if (process != null && !process.HasExited)
{
process.Kill();
process.Dispose();
process = null;
}
}
}

View File

@@ -0,0 +1,35 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Align;
public class AlignElement:IGraphicsElement
{
public bool Editable { get; set; }
public bool IsGood { get; set; }
public Vector2 Location { get; set; }
public void Draw(ICanvas canvas)
{
canvas.DrawCross(Location);
}
public void FillMat(Mat mat)
{
throw new NotSupportedException();
}
public INode? GetNode(Vector2 position, float thDistance)
{
return null;
}
public void SetPivot(Vector2 pivot)
{
throw new NotSupportedException();
}
public void MovePivot(Vector2 pivot)
{
throw new NotSupportedException();
}
}

View File

@@ -0,0 +1,18 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
public class ArrayHorizontalAnchorNode:INode
{
private readonly ArrayHorizontalElement _parent;
public ArrayHorizontalAnchorNode(ArrayHorizontalElement parent)
{
_parent = parent;
}
public void SetPosition(Vector2 position)
{
_parent.Location = position - _parent.Offset;
}
}

View File

@@ -0,0 +1,86 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
public class ArrayHorizontalElement:IGraphicsElement
{
public bool Editable { get; set; }
public Dictionary<int,bool> IsGood=new Dictionary<int, bool>();
public Vector2 Location { get; set; }
public Vector2 Offset { get; set; }
public Vector2 BlockSize { get; set; }
public int Margin { get; set; } = 5;
public int BlockCount { get; set; } = 5;
public void Draw(ICanvas canvas)
{
GenerateLocations().Select((l, i) => (l, i)).ToList().ForEach(((Vector2 l, int i) x) => canvas.DrawRectangle(x.l, BlockSize, IsGood.GetValueOrDefault(x.i,true)));
canvas.DrawNode(Offset + Location);
}
public IEnumerable<Vector2> GenerateLocations()
{
for (int i = 0; i < BlockCount; i++)
yield return Offset + Location + (new Vector2(1, 0) * i * (BlockSize.X + Margin));
}
public void FillMat(Mat mat)
{
}
public INode? GetNode(Vector2 position, float thDistance)
{
if ((Location + Offset - position).Length() < thDistance)
{
return new ArrayHorizontalAnchorNode(this);
}
return null;
}
public void SetPivot(Vector2 pivot)
{
Location += Offset;
Offset = pivot;
Location -= Offset;
}
public void MovePivot(Vector2 pivot)
{
Offset = pivot;
}
public void Save(BinaryWriter bw)
{
bw.Write(Editable);
bw.Write(Location.X);
bw.Write(Location.Y);
bw.Write(Offset.X);
bw.Write(Offset.Y);
bw.Write(BlockSize.X);
bw.Write(BlockSize.Y);
bw.Write(BlockCount);
}
public void Load(BinaryReader br)
{
Editable = br.ReadBoolean();
Location = new Vector2(br.ReadSingle(), br.ReadSingle());
Offset = new Vector2(br.ReadSingle(), br.ReadSingle());
BlockSize = new Vector2(br.ReadSingle(), br.ReadSingle());
BlockCount = br.ReadInt32();
}
}

View File

@@ -0,0 +1,40 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Edge;
public class EdgeElement:IGraphicsElement
{
public bool Editable { get; set; }
public bool IsGood { get; set; }
public Vector2 Location { get; set; }
public float Rotation { get; set; }
public void Draw(ICanvas canvas)
{
float endX = Location.X + MathF.Cos(Rotation)*1000;
float endY = Location.Y + MathF.Sin(Rotation) * 1000;
float startX = Location.X - MathF.Cos(Rotation) * 1000;
float startY = Location.Y - MathF.Sin(Rotation) * 1000;
canvas.DrawLine(new Vector2(startX,startY), new Vector2(endX, endY), LineType.Construction);
}
public void FillMat(Mat mat)
{
throw new NotImplementedException();
}
public INode? GetNode(Vector2 position, float thDistance)
{
throw new NotImplementedException();
}
public void SetPivot(Vector2 pivot)
{
throw new NotImplementedException();
}
public void MovePivot(Vector2 pivot)
{
throw new NotImplementedException();
}
}

View File

@@ -0,0 +1,54 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Edge;
public struct EdgeOrigin
{
public Vector2 Location { get; set; }
public float Rotation { get; set; }
public bool Found { get; set; }
public float endX => Location.X + MathF.Cos(Rotation) * 1000;
public float endY => Location.Y + MathF.Sin(Rotation) * 1000;
public float startX => Location.X - MathF.Cos(Rotation) * 1000;
public float startY => Location.Y - MathF.Sin(Rotation) * 1000;
public float Slope=>(endY-startY)/(endX-startX);
public float YIntercept => startY - (Slope * startX);
public bool Intersection(EdgeOrigin other, out float x, out float y)
{
x = 0;
y = 0;
// Line 1
Vector2 line1Start = new Vector2(startX, startY);
// Line 2
Vector2 line2Start = new Vector2(other.startX, other.startY);
// Calculate the slopes of the lines
double line1Slope = Slope;
double line2Slope = other.Slope;
if (line1Slope == line2Slope) return false;
// Calculate the y-intercepts of the lines
double line1Intercept = line1Start.Y - line1Slope * line1Start.X;
double line2Intercept = line2Start.Y - line2Slope * line2Start.X;
// Calculate the intersection point
double intersectionX = (line2Intercept - line1Intercept) / (line1Slope - line2Slope);
double intersectionY = line1Slope * intersectionX + line1Intercept;
// Create the intersection point object
x = (float)intersectionX;
y = (float)intersectionY;
return true;
}
public override string ToString()
{
return $"{Location.ToString()} {Rotation} rad";
}
}

View File

@@ -0,0 +1,21 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle;
public class FixedRectangleAnchorNode:INode
{
private readonly FixedRectangleElement _owner;
public FixedRectangleAnchorNode(FixedRectangleElement owner)
{
_owner = owner;
}
public void SetPosition(Vector2 position)
{
_owner.Location=position-_owner.Offset;
}
}

View File

@@ -0,0 +1,58 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle;
public class FixedRectangleElement : IGraphicsElement
{
public bool Editable { get; set; }
public bool IsGood { get; set; }
public Vector2 Location { get; set; }
public Vector2 Offset { get; set; }
public Vector2 Size { get; set; }
public void Draw(ICanvas canvas)
{
canvas.DrawRectangle(Offset + Location, Size, IsGood);
canvas.DrawNode(Offset + Location);
}
public void FillMat(Mat mat)
{
var location = Offset + Location;
Cv2.Rectangle(mat,
new Rect((int)location.X, (int)location.Y, (int)Size.X,
(int)Size.Y),
Scalar.White, -1);
}
public INode? GetNode(Vector2 position, float thDistance)
{
if ((Location + Offset - position).Length() < thDistance)
{
return new FixedRectangleAnchorNode(this);
}
return null;
}
public void SetPivot(Vector2 pivot)
{
Location += Offset;
Offset = pivot;
Location -= Offset;
}
public void MovePivot(Vector2 pivot)
{
Offset = pivot;
}
public FixedRectangleElement()
{
Location = Vector2.Zero;
Size = Vector2.Zero;
Offset = Vector2.Zero;
}
}

View File

@@ -0,0 +1,13 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements;
public interface ICanvas
{
void DrawRectangle(Vector2 start, Vector2 size, bool isGood);
void DrawLine(Vector2 start, Vector2 end, LineType type);
void DrawPoly(Vector2[] points, bool isGood);
void DrawCross(Vector2 location);
void DrawNode(Vector2 location);
void DrawArrow(Vector2 location, Vector2 direction);
}

View File

@@ -0,0 +1,23 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Parent;
using Hawkeye.VisionBuilder.Workflow.Links;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements;
public interface IGraphicsElement
{
public bool Editable { get; set; }
public void Draw(ICanvas canvas);
public void FillMat(Mat mat);
public INode? GetNode(Vector2 position, float thDistance);
public void SetPivot(Vector2 pivot);
public void MovePivot(Vector2 pivot);
}

View File

@@ -0,0 +1,9 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements;
public interface INode
{
public void SetPosition(Vector2 position);
}

View File

@@ -0,0 +1,38 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Line;
public class LineElement:IGraphicsElement
{
public bool Editable { get; set; }
public bool IsGood { get; set; }
public Vector2 Start { get; set; }
public Vector2 End { get; set; }
public void Draw(ICanvas canvas)
{
canvas.DrawLine(Start, End,LineType.Construction);
}
public void FillMat(Mat mat)
{
throw new NotImplementedException();
}
public INode? GetNode(Vector2 position, float thDistance)
{
return null;
}
public void SetPivot(Vector2 pivot)
{
}
public void MovePivot(Vector2 pivot)
{
}
}

View File

@@ -0,0 +1,8 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements;
public enum LineType
{
Good,
Bad,
Construction
}

View File

@@ -0,0 +1,27 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Parent;
using Hawkeye.VisionBuilder.Workflow.Links;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
public class OriginElement
{
private static readonly OriginElement _default = new OriginElement();
private Vector2 _location;
public static OriginElement Default => _default;
public Vector2 Location
{
get => _location;
set => _location = value;
}
public OriginElement()
{
}
}

View File

@@ -0,0 +1,9 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Parent;
public interface IHaveParentCoordinates
{
public Vector2 Location { get; }
}

View File

@@ -0,0 +1,14 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Parent;
public class NoParentCoordinates:IHaveParentCoordinates
{
private NoParentCoordinates()
{
}
public static NoParentCoordinates Instance { get; }=new NoParentCoordinates();
public Vector2 Location { get; set; }=Vector2.Zero;
public float Rotation { get; set; } = 0;
}

View File

@@ -0,0 +1,42 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Parent;
using Hawkeye.VisionBuilder.Workflow.Links;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly;
public class PolyElement : IGraphicsElement
{
public IGraphicsElement Parent { get; set; }
public Vector2 Location { get; set; }
public bool Editable { get; set; }
public bool IsGood { get; set; }
public void Draw(ICanvas canvas)
{
canvas.DrawPoly(Points, IsGood);
}
public void FillMat(Mat mat)
{
throw new NotImplementedException();
}
public INode? GetNode(Vector2 position, float thDistance)
{
return null;
}
public void SetPivot(Vector2 pivot)
{
throw new NotImplementedException();
}
public void MovePivot(Vector2 pivot)
{
throw new NotImplementedException();
}
public Vector2[] Points { get; set; } = Array.Empty<Vector2>();
}

View File

@@ -0,0 +1,20 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
public class RectangleAnchorNode:INode
{
private readonly RectangleElement _owner;
public RectangleAnchorNode(RectangleElement owner)
{
_owner = owner;
}
public void SetPosition(Vector2 position)
{
_owner.Location=position-_owner.Offset;
}
}

View File

@@ -0,0 +1,177 @@
using System.Runtime.CompilerServices;
using System.Xml;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Parent;
using Hawkeye.VisionBuilder.Workflow.Links;
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
public class RectangleElement : IGraphicsElement
{
public bool Editable { get; set; }
public bool IsGood { get; set; }
public Vector2 Location { get; set; }
public Vector2 Offset { get; set; }
public Vector2 Size { get; set; }
public void Draw(ICanvas canvas)
{
canvas.DrawRectangle(Offset+Location, Size,IsGood);
canvas.DrawNode(Offset+Location);
canvas.DrawNode(Offset+Location + Size);
}
public void FillMat(Mat mat)
{
var location = Offset + Location;
Cv2.Rectangle(mat,
new Rect((int)location.X, (int)location.Y, (int)Size.X,
(int)Size.Y),
Scalar.White, -1);
}
public INode? GetNode(Vector2 position,float thDistance)
{
if ((Location + Offset - position).Length()< thDistance)
{
return new RectangleAnchorNode(this);
}
if((Size + Location+Offset - position).Length() < thDistance)
{
return new RectangleSizeNode(this);
}
return null;
}
public void SetPivot(Vector2 pivot)
{
Location += Offset;
Offset =pivot;
Location-=Offset;
}
public void MovePivot(Vector2 pivot)
{
Offset = pivot;
}
public RectangleElement()
{
}
public void Save(BinaryWriter bw)
{
bw.Write(Editable);
bw.Write(IsGood);
bw.Write(Location.X);
bw.Write(Location.Y);
bw.Write(Offset.X);
bw.Write(Offset.Y);
bw.Write(Size.X);
bw.Write(Size.Y);
}
public void Load(BinaryReader br)
{
Editable = br.ReadBoolean();
IsGood = br.ReadBoolean();
Location = new Vector2(br.ReadSingle(), br.ReadSingle());
Offset = new Vector2(br.ReadSingle(), br.ReadSingle());
Size = new Vector2(br.ReadSingle(), br.ReadSingle());
}
public void SaveXML(XmlWriter writer)
{
writer.WriteStartElement("Editable");
writer.WriteString(Editable.ToString());
writer.WriteEndElement();
writer.WriteStartElement("IsGood");
writer.WriteString(IsGood.ToString());
writer.WriteEndElement();
writer.WriteStartElement("LocationX");
writer.WriteString(Location.X.ToString());
writer.WriteEndElement();
writer.WriteStartElement("LocationY");
writer.WriteString(Location.Y.ToString());
writer.WriteEndElement();
writer.WriteStartElement("OffsetX");
writer.WriteString(Offset.X.ToString());
writer.WriteEndElement();
writer.WriteStartElement("OffsetY");
writer.WriteString(Offset.Y.ToString());
writer.WriteEndElement();
writer.WriteStartElement("SizeX");
writer.WriteString(Size.X.ToString());
writer.WriteEndElement();
writer.WriteStartElement("SizeY");
writer.WriteString(Size.Y.ToString());
writer.WriteEndElement();
}
public void LoadXML(XmlReader reader)
{
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && reader.Name == "Editable")
{
Editable = bool.Parse(reader.ReadElementContentAsString());
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "IsGood")
{
IsGood = bool.Parse(reader.ReadElementContentAsString());
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "LocationX")
{
var x = float.Parse(reader.ReadElementContentAsString());
Location = new Vector2(x, Location.Y);
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "LocationY")
{
var y = float.Parse(reader.ReadElementContentAsString());
Location = new Vector2(Location.X, y);
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "OffsetX")
{
var x = float.Parse(reader.ReadElementContentAsString());
Offset = new Vector2(x, Offset.Y);
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "OffsetY")
{
var y = float.Parse(reader.ReadElementContentAsString());
Offset = new Vector2(Offset.X, y);
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "SizeX")
{
var x = float.Parse(reader.ReadElementContentAsString());
Size = new Vector2(x, Size.Y);
}
else if (reader.NodeType == XmlNodeType.Element && reader.Name == "SizeY")
{
var y = float.Parse(reader.ReadElementContentAsString());
Size = new Vector2(Size.X, y);
}
else if (reader.NodeType == XmlNodeType.EndElement && reader.Name == "RectangleElement")
{
break;
}
}
}
}

View File

@@ -0,0 +1,18 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
public class RectangleSizeNode:INode
{
private readonly RectangleElement _owner;
public RectangleSizeNode(RectangleElement owner)
{
_owner = owner;
}
public void SetPosition(Vector2 position)
{
_owner.Size=position-_owner.Location - _owner.Offset;
}
}

View File

@@ -0,0 +1,18 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.RotatedRectangle;
public class RotatedRectangleAnchorNode:INode
{
private readonly RotatedRectangleElement _owner;
public RotatedRectangleAnchorNode(RotatedRectangleElement owner)
{
_owner = owner;
}
public void SetPosition(Vector2 position)
{
_owner.Location=position-_owner.Offset;
}
}

View File

@@ -0,0 +1,123 @@
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
using OpenCvSharp;
using Point = OpenCvSharp.Point;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.RotatedRectangle;
public class RotatedRectangleElement:IGraphicsElement
{
public bool Editable { get; set; }
public bool IsGood { get; set; }
public Vector2 Location { get; set; }
public Vector2 Offset { get; set; }
public Vector2 Height { get; set; }
public float HalfWidth { get; set; }
public void Draw(ICanvas canvas)
{
var type = IsGood ? LineType.Good : LineType.Bad;
var perpendicular = new Vector2(Height.Y, -Height.X);
perpendicular.Normalize();
canvas.DrawLine(Offset + Location - perpendicular * HalfWidth, Offset + Location+perpendicular*HalfWidth, type);
canvas.DrawLine(Offset + Location + Height - perpendicular * HalfWidth, Offset + Location + Height + perpendicular * HalfWidth, type);
canvas.DrawLine(
Offset + Location + perpendicular * HalfWidth
, Offset + Location + Height + perpendicular * HalfWidth
, type);
canvas.DrawLine(
Offset + Location - perpendicular * HalfWidth
, Offset + Location + Height - perpendicular * HalfWidth
, type);
canvas.DrawNode(Offset + Location);
canvas.DrawNode(Offset + Location + Height);
canvas.DrawNode(Offset + Location + Height + perpendicular * HalfWidth);
canvas.DrawArrow(Offset + Location, Height);
}
public void FillMat(Mat mat)
{
var perpendicular = new Vector2(Height.Y, -Height.X);
perpendicular.Normalize();
var p1 = Offset + Location - perpendicular * HalfWidth;
var p2 = Offset + Location + perpendicular * HalfWidth;
var p3 = Offset + Location + Height + perpendicular * HalfWidth;
var p4 = Offset + Location + Height - perpendicular * HalfWidth;
mat.FillPoly(new IEnumerable<Point>[]
{
new List<Point>()
{
new Point(p1.X,p1.Y),
new Point(p2.X,p2.Y),
new Point(p3.X,p3.Y),
new Point(p4.X,p4.Y),
}
},Scalar.White);
}
public INode? GetNode(Vector2 position, float thDistance)
{
if ((Location + Offset - position).Length() < thDistance)
{
return new RotatedRectangleAnchorNode(this);
}
if ((Height + Location + Offset - position).Length() < thDistance)
{
return new RotatedRectangleHeightNode(this);
}
var perpendicular = new Vector2(Height.Y, -Height.X);
perpendicular.Normalize();
if (((Offset + Location + Height + perpendicular * HalfWidth) - position).Length() < thDistance)
{
return new RotatedRectangleHalfWidthNode(this);
}
return null;
}
public void SetPivot(Vector2 pivot)
{
Location += Offset;
Offset = pivot;
Location -= Offset;
}
public void MovePivot(Vector2 pivot)
{
Offset = pivot;
}
public void Save(BinaryWriter bw)
{
bw.Write(Editable);
bw.Write(IsGood);
bw.Write(Location.X);
bw.Write(Location.Y);
bw.Write(Offset.X);
bw.Write(Offset.Y);
bw.Write(Height.X);
bw.Write(Height.Y);
bw.Write(HalfWidth);
}
public void Load(BinaryReader br)
{
Editable = br.ReadBoolean();
IsGood = br.ReadBoolean();
Location = new Vector2(br.ReadSingle(), br.ReadSingle());
Offset = new Vector2(br.ReadSingle(), br.ReadSingle());
Height = new Vector2(br.ReadSingle(), br.ReadSingle());
HalfWidth = br.ReadSingle();
}
}

View File

@@ -0,0 +1,18 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.RotatedRectangle;
public class RotatedRectangleHalfWidthNode:INode
{
private readonly RotatedRectangleElement _owner;
public RotatedRectangleHalfWidthNode(RotatedRectangleElement owner)
{
_owner = owner;
}
public void SetPosition(Vector2 position)
{
_owner.HalfWidth = (position - (_owner.Height + _owner.Location + _owner.Offset)).Length();
}
}

View File

@@ -0,0 +1,16 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.RotatedRectangle;
public class RotatedRectangleHeightNode:INode
{
private readonly RotatedRectangleElement _owner;
public RotatedRectangleHeightNode(RotatedRectangleElement owner)
{
_owner = owner;
}
public void SetPosition(Vector2 position)
{
_owner.Height=position-_owner.Location;
}
}

View File

@@ -0,0 +1,7 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes;
public class FilePath
{
public string Path { get; set; }
public string Format { get; set; } = "Model file(*.h5)|*.h5";
}

View File

@@ -0,0 +1,10 @@
using OpenCvSharp;
namespace Hawkeye.VisionBuilder.Workflow.Datatypes;
public class HawkeyeImage
{
public Mat ImageData { get; init; }
public string Filename { get; init; }
}

View File

@@ -0,0 +1,68 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes;
public class LutData
{
public LutData()
{
LUT = new byte[256 * 4];
Points = new Vector2[4][];
for (int i = 0; i < 4; i++)
{
Points[i] = new Vector2[2];
Points[i][0] = new Vector2(0, 0);
Points[i][1] = new Vector2(255, 255);
}
for (int i = 0; i < 256; i++)
{
LUT[i] = (byte)i;
LUT[i+256] = (byte)i;
LUT[i+256*2] = (byte)i;
LUT[i+256*3] = (byte)i;
}
}
public byte[] LUT { get; set; }
public Vector2[][] Points { get; set; }
public void Serialize(BinaryWriter bw)
{
bw.Write(LUT.Length);
bw.Write(LUT);
bw.Write(Points.Length);
for (int i = 0; i < Points.Length; i++)
{
bw.Write(Points[i].Length);
for (int j = 0; j < Points[i].Length; j++)
{
bw.Write(Points[i][j].X);
bw.Write(Points[i][j].Y);
}
}
}
public void Deserialize(BinaryReader br)
{
var length = br.ReadInt32();
LUT = br.ReadBytes(length);
length = br.ReadInt32();
Points = new Vector2[length][];
for (int i = 0; i < length; i++)
{
var length2 = br.ReadInt32();
Points[i] = new Vector2[length2];
for (int j = 0; j < length2; j++)
{
Points[i][j] = new Vector2(br.ReadSingle(), br.ReadSingle());
}
}
}
}

View File

@@ -0,0 +1,6 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes;
public class Pattern
{
}

View File

@@ -0,0 +1,7 @@
namespace Hawkeye.VisionBuilder.Workflow.Datatypes;
public class SelectFromList
{
public string Value { get; set; }
public Func<List<string>> GetList { get; set; }
}

View File

@@ -0,0 +1,46 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
<PlatformTarget>x64</PlatformTarget>
<Configurations>Debug;Release;CPU</Configurations>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ILGPU" Version="1.5.1" />
<PackageReference Include="IronPython" Version="3.4.0" />
<PackageReference Include="MessagePack" Version="2.5.108" />
<PackageReference Include="NetMQ" Version="4.0.1.11" />
<PackageReference Include="Ninject" Version="3.3.6" />
<PackageReference Include="OpenCvSharp4" Version="4.6.0.20220608" />
<PackageReference Include="OpenCvSharp4.Extensions" Version="4.6.0.20220608" />
<PackageReference Include="Serilog" Version="4.2.0" />
</ItemGroup>
<ItemGroup Condition="'$(Configuration)'=='CPU'">
<PackageReference Include="YoloV8" Version="4.1.5" />
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.18.0" />
</ItemGroup>
<ItemGroup Condition="'$(Configuration)'!='CPU'">
<PackageReference Include="YoloV8.Gpu" Version="4.1.7" />
<PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.18.0" />
</ItemGroup>
<ItemGroup>
<Folder Include="Datatypes\Elements\Edge\" />
<Folder Include="Datatypes\Elements\Line\" />
<Folder Include="Overrides\" />
<Folder Include="Operations\Edge\" />
<Folder Include="Operations\Transformations\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VisionBuilder.UI.Common\VisionBuilder.UI.Common.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,8 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Edge;
namespace Hawkeye.VisionBuilder.Workflow.Links;
public interface IHaveEdge
{
public EdgeOrigin EdgeOrigin { get; }
}

View File

@@ -0,0 +1,8 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
namespace Hawkeye.VisionBuilder.Workflow.Links;
public interface IHaveImage
{
public HawkeyeImage Image { get; }
}

View File

@@ -0,0 +1,12 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
namespace Hawkeye.VisionBuilder.Workflow.Links;
public interface IHaveOrigin
{
public Guid Id { get; }
public OriginElement Origin { get; }
}

View File

@@ -0,0 +1,8 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
namespace Hawkeye.VisionBuilder.Workflow.Links;
public interface IHaveSearchArea
{
public RectangleElement SearchArea { get; set; }
}

View File

@@ -0,0 +1,37 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Edge;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
namespace Hawkeye.VisionBuilder.Workflow.Links;
public class NoOrigin:BaseOperation,IHaveOrigin,IHaveEdge
{
private NoOrigin()
{
}
public static NoOrigin Instance { get; } = new NoOrigin();
public Guid Id { get; } = new Guid();
public OriginElement Origin { get; } = new OriginElement();
public string Label { get; set; } = "No origin";
protected override void InterpretInternal(Context context)
{
throw new NotImplementedException();
}
public override void Save(BinaryWriter bw)
{
throw new NotImplementedException();
}
public override void Load(BinaryReader br)
{
throw new NotImplementedException();
}
public IHaveOrigin Parent { get; set; }
public EdgeOrigin EdgeOrigin { get; }
}

View File

@@ -0,0 +1,314 @@
namespace Hawkeye.VisionBuilder.Workflow
{
/// <summary>
/// Contains commonly used precalculated values and mathematical operations.
/// </summary>
public static class MathHelper
{
/// <summary>
/// Represents the mathematical constant e(2.71828175).
/// </summary>
public const float E = (float)Math.E;
/// <summary>
/// Represents the log base ten of e(0.4342945).
/// </summary>
public const float Log10E = 0.4342945f;
/// <summary>
/// Represents the log base two of e(1.442695).
/// </summary>
public const float Log2E = 1.442695f;
/// <summary>
/// Represents the value of pi(3.14159274).
/// </summary>
public const float Pi = (float)Math.PI;
/// <summary>
/// Represents the value of pi divided by two(1.57079637).
/// </summary>
public const float PiOver2 = (float)(Math.PI / 2.0);
/// <summary>
/// Represents the value of pi divided by four(0.7853982).
/// </summary>
public const float PiOver4 = (float)(Math.PI / 4.0);
/// <summary>
/// Represents the value of pi times two(6.28318548).
/// </summary>
public const float TwoPi = (float)(Math.PI * 2.0);
/// <summary>
/// Returns the Cartesian coordinate for one axis of a point that is defined by a given triangle and two normalized barycentric (areal) coordinates.
/// </summary>
/// <param name="value1">The coordinate on one axis of vertex 1 of the defining triangle.</param>
/// <param name="value2">The coordinate on the same axis of vertex 2 of the defining triangle.</param>
/// <param name="value3">The coordinate on the same axis of vertex 3 of the defining triangle.</param>
/// <param name="amount1">The normalized barycentric (areal) coordinate b2, equal to the weighting factor for vertex 2, the coordinate of which is specified in value2.</param>
/// <param name="amount2">The normalized barycentric (areal) coordinate b3, equal to the weighting factor for vertex 3, the coordinate of which is specified in value3.</param>
/// <returns>Cartesian coordinate of the specified point with respect to the axis being used.</returns>
public static float Barycentric(float value1, float value2, float value3, float amount1, float amount2)
{
return value1 + (value2 - value1) * amount1 + (value3 - value1) * amount2;
}
/// <summary>
/// Performs a Catmull-Rom interpolation using the specified positions.
/// </summary>
/// <param name="value1">The first position in the interpolation.</param>
/// <param name="value2">The second position in the interpolation.</param>
/// <param name="value3">The third position in the interpolation.</param>
/// <param name="value4">The fourth position in the interpolation.</param>
/// <param name="amount">Weighting factor.</param>
/// <returns>A position that is the result of the Catmull-Rom interpolation.</returns>
public static float CatmullRom(float value1, float value2, float value3, float value4, float amount)
{
// Using formula from http://www.mvps.org/directx/articles/catmull/
// Internally using doubles not to lose precission
double amountSquared = amount * amount;
double amountCubed = amountSquared * amount;
return (float)(0.5 * (2.0 * value2 +
(value3 - value1) * amount +
(2.0 * value1 - 5.0 * value2 + 4.0 * value3 - value4) * amountSquared +
(3.0 * value2 - value1 - 3.0 * value3 + value4) * amountCubed));
}
/// <summary>
/// Restricts a value to be within a specified range.
/// </summary>
/// <param name="value">The value to clamp.</param>
/// <param name="min">The minimum value. If <c>value</c> is less than <c>min</c>, <c>min</c> will be returned.</param>
/// <param name="max">The maximum value. If <c>value</c> is greater than <c>max</c>, <c>max</c> will be returned.</param>
/// <returns>The clamped value.</returns>
public static float Clamp(float value, float min, float max)
{
// First we check to see if we're greater than the max
value = (value > max) ? max : value;
// Then we check to see if we're less than the min.
value = (value < min) ? min : value;
// There's no check to see if min > max.
return value;
}
/// <summary>
/// Restricts a value to be within a specified range.
/// </summary>
/// <param name="value">The value to clamp.</param>
/// <param name="min">The minimum value. If <c>value</c> is less than <c>min</c>, <c>min</c> will be returned.</param>
/// <param name="max">The maximum value. If <c>value</c> is greater than <c>max</c>, <c>max</c> will be returned.</param>
/// <returns>The clamped value.</returns>
public static int Clamp(int value, int min, int max)
{
value = (value > max) ? max : value;
value = (value < min) ? min : value;
return value;
}
/// <summary>
/// Calculates the absolute value of the difference of two values.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Source value.</param>
/// <returns>Distance between the two values.</returns>
public static float Distance(float value1, float value2)
{
return Math.Abs(value1 - value2);
}
/// <summary>
/// Performs a Hermite spline interpolation.
/// </summary>
/// <param name="value1">Source position.</param>
/// <param name="tangent1">Source tangent.</param>
/// <param name="value2">Source position.</param>
/// <param name="tangent2">Source tangent.</param>
/// <param name="amount">Weighting factor.</param>
/// <returns>The result of the Hermite spline interpolation.</returns>
public static float Hermite(float value1, float tangent1, float value2, float tangent2, float amount)
{
// All transformed to double not to lose precission
// Otherwise, for high numbers of param:amount the result is NaN instead of Infinity
double v1 = value1, v2 = value2, t1 = tangent1, t2 = tangent2, s = amount, result;
double sCubed = s * s * s;
double sSquared = s * s;
if (amount == 0f)
result = value1;
else if (amount == 1f)
result = value2;
else
result = (2 * v1 - 2 * v2 + t2 + t1) * sCubed +
(3 * v2 - 3 * v1 - 2 * t1 - t2) * sSquared +
t1 * s +
v1;
return (float)result;
}
/// <summary>
/// Linearly interpolates between two values.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Destination value.</param>
/// <param name="amount">Value between 0 and 1 indicating the weight of value2.</param>
/// <returns>Interpolated value.</returns>
/// <remarks>This method performs the linear interpolation based on the following formula:
/// <code>value1 + (value2 - value1) * amount</code>.
/// Passing amount a value of 0 will cause value1 to be returned, a value of 1 will cause value2 to be returned.
/// See <see cref="MathHelper.LerpPrecise"/> for a less efficient version with more precision around edge cases.
/// </remarks>
public static float Lerp(float value1, float value2, float amount)
{
return value1 + (value2 - value1) * amount;
}
/// <summary>
/// Linearly interpolates between two values.
/// This method is a less efficient, more precise version of <see cref="MathHelper.Lerp"/>.
/// See remarks for more info.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Destination value.</param>
/// <param name="amount">Value between 0 and 1 indicating the weight of value2.</param>
/// <returns>Interpolated value.</returns>
/// <remarks>This method performs the linear interpolation based on the following formula:
/// <code>((1 - amount) * value1) + (value2 * amount)</code>.
/// Passing amount a value of 0 will cause value1 to be returned, a value of 1 will cause value2 to be returned.
/// This method does not have the floating point precision issue that <see cref="MathHelper.Lerp"/> has.
/// i.e. If there is a big gap between value1 and value2 in magnitude (e.g. value1=10000000000000000, value2=1),
/// right at the edge of the interpolation range (amount=1), <see cref="MathHelper.Lerp"/> will return 0 (whereas it should return 1).
/// This also holds for value1=10^17, value2=10; value1=10^18,value2=10^2... so on.
/// For an in depth explanation of the issue, see below references:
/// Relevant Wikipedia Article: https://en.wikipedia.org/wiki/Linear_interpolation#Programming_language_support
/// Relevant StackOverflow Answer: http://stackoverflow.com/questions/4353525/floating-point-linear-interpolation#answer-23716956
/// </remarks>
public static float LerpPrecise(float value1, float value2, float amount)
{
return ((1 - amount) * value1) + (value2 * amount);
}
/// <summary>
/// Returns the greater of two values.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Source value.</param>
/// <returns>The greater value.</returns>
public static float Max(float value1, float value2)
{
return value1 > value2 ? value1 : value2;
}
/// <summary>
/// Returns the greater of two values.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Source value.</param>
/// <returns>The greater value.</returns>
public static int Max(int value1, int value2)
{
return value1 > value2 ? value1 : value2;
}
/// <summary>
/// Returns the lesser of two values.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Source value.</param>
/// <returns>The lesser value.</returns>
public static float Min(float value1, float value2)
{
return value1 < value2 ? value1 : value2;
}
/// <summary>
/// Returns the lesser of two values.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Source value.</param>
/// <returns>The lesser value.</returns>
public static int Min(int value1, int value2)
{
return value1 < value2 ? value1 : value2;
}
/// <summary>
/// Interpolates between two values using a cubic equation.
/// </summary>
/// <param name="value1">Source value.</param>
/// <param name="value2">Source value.</param>
/// <param name="amount">Weighting value.</param>
/// <returns>Interpolated value.</returns>
public static float SmoothStep(float value1, float value2, float amount)
{
// It is expected that 0 < amount < 1
// If amount < 0, return value1
// If amount > 1, return value2
float result = MathHelper.Clamp(amount, 0f, 1f);
result = MathHelper.Hermite(value1, 0f, value2, 0f, result);
return result;
}
/// <summary>
/// Converts radians to degrees.
/// </summary>
/// <param name="radians">The angle in radians.</param>
/// <returns>The angle in degrees.</returns>
/// <remarks>
/// This method uses double precission internally,
/// though it returns single float
/// Factor = 180 / pi
/// </remarks>
public static float ToDegrees(float radians)
{
return (float)(radians * 57.295779513082320876798154814105);
}
/// <summary>
/// Converts degrees to radians.
/// </summary>
/// <param name="degrees">The angle in degrees.</param>
/// <returns>The angle in radians.</returns>
/// <remarks>
/// This method uses double precission internally,
/// though it returns single float
/// Factor = pi / 180
/// </remarks>
public static float ToRadians(float degrees)
{
return (float)(degrees * 0.017453292519943295769236907684886);
}
/// <summary>
/// Reduces a given angle to a value between π and -π.
/// </summary>
/// <param name="angle">The angle to reduce, in radians.</param>
/// <returns>The new angle, in radians.</returns>
public static float WrapAngle(float angle)
{
if ((angle > -Pi) && (angle <= Pi))
return angle;
angle %= TwoPi;
if (angle <= -Pi)
return angle + TwoPi;
if (angle > Pi)
return angle - TwoPi;
return angle;
}
/// <summary>
/// Determines if value is powered by two.
/// </summary>
/// <param name="value">A value.</param>
/// <returns><c>true</c> if <c>value</c> is powered by two; otherwise <c>false</c>.</returns>
public static bool IsPowerOfTwo(int value)
{
return (value > 0) && ((value & (value - 1)) == 0);
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,8 @@
namespace Hawkeye.VisionBuilder.Workflow;
public class OperationDescription
{
public string Name { get; set; }
public string Category { get; set; }
public Type Type { get; set; }
}

View File

@@ -0,0 +1,58 @@
using System.Reflection;
using System.Text.RegularExpressions;
using Hawkeye.VisionBuilder.Workflow.Configuration;
using Hawkeye.VisionBuilder.Workflow.Links;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using Ninject;
using Ninject.Infrastructure.Language;
namespace Hawkeye.VisionBuilder.Workflow;
public class OperationDiscoveryService
{
private readonly WorkflowList _workflowList;
private readonly StandardKernel _kernel;
public OperationDiscoveryService( WorkflowList workflowList)
{
_workflowList = workflowList;
_kernel = new StandardKernel();
_kernel.Bind<WorkflowList>().ToConstant(workflowList);
}
public List<OperationDescription> DiscoverOperations()
{
return this.GetType().Assembly.GetTypes()
.Where(x => x.IsSubclassOf(typeof(BaseOperation))&&!x.HasAttribute(typeof(IgnoreOperationAttribute)))
.Where(x=>x.Name!=nameof(NoOrigin))
.Select(x=>new OperationDescription()
{
Name = BaseOperation.MakeOperationName(x.Name),
Category = GetCategory(x),
Type = x
}).ToList();
}
private string GetCategory(Type t)
{
var category=t.GetCustomAttribute(typeof(Category));
if (category!=null)
{
return (category as Category).Name;
}
return "General";
}
public BaseOperation CreateInstance(Type type)
{
return (BaseOperation)_kernel.Get(type);
}
}

View File

@@ -0,0 +1,125 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class AnomalyAI: BaseOperation
{
private readonly string _modelName;
public AnomalyAI()
{
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
CanHaveProcessingError = false;
}
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
private int _imageWidth;
private int _imageHeight;
private bool _initialized = false;
private FilePath _modelFilePath = new FilePath();
public string MemorySlotName { get; set; } = "Anomaly";
public FilePath ModelFilePath
{
get => _modelFilePath;
set
{
_modelFilePath = value;
_initialized = false;
}
}
protected override void InterpretInternal(Context context)
{
if (!CheckImageExists(context)) return;
if (!CheckColorful(context)) return;
if (!_initialized)
{
if (!File.Exists(ModelFilePath?.Path))
{
this.SetError("Model file not found");
return;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
_initialized = true;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
_imageWidth = size[2];
_imageHeight = size[1];
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
var currentImage = context.ActiveImage;
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
var resized = rightColor.Resize(new Size(_imageWidth,_imageHeight));
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
var byteArray = new byte[dataSize];
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall( "predict_anomaly", byteArray));
var result = resultEncoded.Select(Convert.ToByte).ToArray();
var defect = result.Take(1).ToArray()[0];
if (defect > 50)
{
byte[] resultColor = result.Skip(1).ToArray();
// black and white mask
var mask = new Mat(30, 48, MatType.CV_8UC1, resultColor);
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
context.ActiveImage = new HawkeyeImage() {ImageData = maskResized};
}
else
{
context.ActiveImage = new HawkeyeImage() { ImageData = new Mat(currentImage.ImageData.Size(), MatType.CV_8UC1, new Scalar(0)) };
}
Result = true;
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(_modelFilePath.Path);
bw.Write(_modelFilePath.Format);
}
public override void Load(BinaryReader br)
{
base.Load(br);
_modelFilePath.Path = br.ReadString();
_modelFilePath.Format = br.ReadString();
}
}

View File

@@ -0,0 +1,184 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
[IgnoreOperation]
public class ArrayModelMatchingOperation:BaseOperation
{
private readonly WorkflowList _workflowList;
private readonly CSharpDataTransferMQ _transfer;
public Guid ReferenceId { get; set; } = new Guid();
public ArrayHorizontalElement SearchArea { get; set; }
public string ModelName { get; set; } = "cls_candy5.h5";
public Action ExportImages { get; set; }
public int Margin { get; set; } = 5;
public int Amount { get; set; } = 4;
private int _channels = 3;
private List<Mat> _lastCutImages;
public ArrayModelMatchingOperation(WorkflowList workflowList)
{
_workflowList = workflowList;
_transfer = PythonModelProxy.GetInterface();
SearchArea = new ArrayHorizontalElement()
{
Editable = true,
Location = Vector2.One * 100,
BlockSize = new Vector2(100, 100),
};
ExportImages = () =>
{
if (!Directory.Exists(Path.Combine(@"..\Data\Export", ModelName)))
{
Directory.CreateDirectory(Path.Combine(@"..\Data\Export", ModelName));
}
// save last cut images
var i = 0;
foreach (var cutImage in _lastCutImages)
{
cutImage.SaveImage(Path.Combine(@"..\Data\Export", ModelName, $"{Guid.NewGuid().ToString()}.png"));
}
};
ReloadModelInfo();
}
private void ReloadModelInfo()
{
var responseBytes = _transfer.TransferData(ModelName, 2, Array.Empty<byte>());
int[] sizes = new int[3];
Buffer.BlockCopy(responseBytes, 0, sizes, 0, 12);
SearchArea.BlockSize = new Vector2(sizes[1], sizes[0]);
_channels = sizes[2];
}
byte[] MatToBytes(Mat mat)
{
if (mat.Channels() == 3)
{
mat = mat.CvtColor(_channels == 3 ? ColorConversionCodes.BGR2RGB : ColorConversionCodes.BGR2GRAY);
}
IntPtr dataPtr = mat.Data;
// Calculate the size of the image data
int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
// Copy the image data into a byte array
byte[] byteArray = new byte[dataSize];
Marshal.Copy(dataPtr, byteArray, 0, dataSize);
return byteArray;
}
protected override void InterpretInternal(Context context)
{
Result = true;
if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
{
Status = $"Model `{ModelName}` not found";
Result = false;
return;
}
if (!CheckImageExists(context)) return;
switch (_channels)
{
case 3 when !CheckColorful(context):
case 1 when !CheckGrayscale(context):
return;
}
var refLocation = _workflowList.GetOriginById(ReferenceId).Origin.Location;
SearchArea.MovePivot(refLocation);
SearchArea.Margin=Margin;
SearchArea.BlockCount=Amount;
var img = context.ActiveImage.ImageData;
_lastCutImages = SearchArea
.GenerateLocations()
.Select(x => img[(int) x.Y, (int) x.Y + (int) SearchArea.BlockSize.Y, (int) x.X,
(int) x.X + (int) SearchArea.BlockSize.X]).ToList();
var images= _lastCutImages.Select(MatToBytes).ToList();
var bytes= images.SelectMany(x => x).ToArray();
var len = images.Count;
var bytes2 = new byte[bytes.Length + 4];
bytes2[3] = (byte)(len >> 24);
bytes2[2] = (byte)(len >> 16);
bytes2[1] = (byte)(len >> 8);
bytes2[0] = (byte)(len >> 0);
Array.Copy(bytes, 0, bytes2, 4, bytes.Length);
var answer = _transfer.TransferData(ModelName, 0,bytes2);
float[] floatArray = new float[answer.Length / 4];
Buffer.BlockCopy(answer, 0, floatArray, 0, answer.Length);
// Print answer
for (int i = 0; i < Amount; i++)
{
Console.WriteLine(floatArray[i]);
}
for (int i = 0; i < Amount; i++)
{
SearchArea.IsGood[i] = floatArray[i] > 0.7;
}
context.GraphicsElements.Add(SearchArea);
Result = SearchArea.IsGood.All(x=>x.Value==true);
}
public override void SetParameters(Dictionary<string, object> parameters)
{
base.SetParameters(parameters);
ReloadModelInfo();
SearchArea.SetPivot(_workflowList.GetOriginById(ReferenceId).Origin.Location);
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(ReferenceId.ToString());
bw.Write(Margin);
bw.Write(Amount);
bw.Write(ModelName);
SearchArea.Save(bw);
}
public override void Load(BinaryReader br)
{
base.Load(br);
ReferenceId = new Guid(br.ReadString());
Margin = br.ReadInt32();
Amount = br.ReadInt32();
ModelName = br.ReadString();
SearchArea = new ArrayHorizontalElement();
SearchArea.Load(br);
ReloadModelInfo();
}
}

View File

@@ -0,0 +1,87 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
[IgnoreOperation]
public class ArrayModelSamplerOperation:BaseOperation
{
private Mat _lastImage;
public RectangleElement SearchArea { get; set; }
public string SampleStorage { get; set; } = "samples.storage";
public Action Sample { get; set; }
public ArrayModelSamplerOperation()
{
SearchArea = new RectangleElement()
{
Editable = true,
Location = Vector2.One * 100,
Size = new Vector2(100, 100),
};
Sample = SaveSample;
}
byte[] MatToBytes(Mat mat)
{
if (mat.Channels() == 3)
{
mat = mat.CvtColor(ColorConversionCodes.BGR2RGB);
}
IntPtr dataPtr = mat.Data;
int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
byte[] byteArray = new byte[dataSize];
Marshal.Copy(dataPtr, byteArray, 0, dataSize);
return byteArray;
}
private void SaveSample()
{
Mat mask = new Mat(_lastImage.Size(), MatType.CV_8UC1, Scalar.Black);
SearchArea.FillMat(mask);
var imageBytes = MatToBytes(_lastImage);
var maskBytes = MatToBytes(mask);
// Append to the end of storage file
var storagePath=Path.Combine("..\\Data\\AI", SampleStorage);
using (var file = new System.IO.FileStream(storagePath, System.IO.FileMode.Append))
{
// Write size
file.Write(BitConverter.GetBytes(_lastImage.Width), 0, 4);
file.Write(BitConverter.GetBytes(_lastImage.Height), 0, 4);
file.Write(imageBytes, 0, imageBytes.Length);
// Write mask
file.Write(maskBytes, 0, maskBytes.Length);
}
}
protected override void InterpretInternal(Context context)
{
if (!CheckImageExists(context)) return;
if(!CheckColorful(context)) return;
context.GraphicsElements.Add(SearchArea);
_lastImage=context.ActiveImage.ImageData;
Result = true;
}
}

View File

@@ -0,0 +1,124 @@
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Drawing;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly;
using Hawkeye.VisionBuilder.Workflow.Links;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
[IgnoreOperation]
public class BackgroundSeparationModelOperation:BaseOperation,IHaveOrigin
{
private readonly CSharpDataTransferMQ _transfer;
private int _channels;
public string ModelName { get; set; } = "background.h5";
public BackgroundSeparationModelOperation()
{
_transfer = PythonModelProxy.GetInterface();
}
byte[] MatToBytes(Mat mat)
{
if (mat.Channels() == 3)
{
mat = mat.CvtColor(ColorConversionCodes.BGR2RGB);
}
IntPtr dataPtr = mat.Data;
int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
byte[] byteArray = new byte[dataSize];
Marshal.Copy(dataPtr, byteArray, 0, dataSize);
return byteArray;
}
protected override void InterpretInternal(Context context)
{
Result = true;
if (!CheckImageExists(context)) return;
if (!CheckColorful(context)) return;
if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
{
Status= $"Model `{ModelName}` not found";
Result = false;
return;
}
var img = context.ActiveImage.ImageData;
var bytes= MatToBytes(img);
var responseBytes = _transfer.TransferData(ModelName, 3, bytes);
var response = new Mat(img.Rows, img.Cols, MatType.CV_8UC1, responseBytes);
var contours = response.Threshold(128, 255, ThresholdTypes.Binary).FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple);
if (contours.Length == 0)
{
Status = "No object found";
Result = false;
return;
}
// find biggest contour
var max = 0;
var maxIndex = 0;
for (var i = 0; i < contours.Length; i++)
{
var area = Cv2.ContourArea(contours[i]);
if (area > max)
{
max = (int)area;
maxIndex = i;
}
}
PolyElement poly = new PolyElement() { IsGood = true,Editable = false};
poly.Points = contours[maxIndex].Select(x => new Vector2(x.X, x.Y)).ToArray();
context.GraphicsElements.Add(poly);
var rect = Cv2.MinAreaRect(contours[maxIndex]);
var bound = rect.BoundingRect();
Origin=new OriginElement()
{
Location = new Vector2(bound.Left, bound.Top),
};
Status= $"Object found at {Origin.Location}";
Result = true;
}
public override void SetParameters(Dictionary<string, object> parameters)
{
base.SetParameters(parameters);
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(ModelName);
}
public override void Load(BinaryReader br)
{
base.Load(br);
ModelName = br.ReadString();
}
public OriginElement Origin { get; set; }
}

View File

@@ -0,0 +1,33 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
[IgnoreOperation]
public class Color128BinaryOperation:ColorAIOperation
{
public Color128BinaryOperation() : base("prob128binary")
{
IsRaw = true;
}
public override string FilterClasses { get; set; } = "1";
public override FilePath ModelFilePath { get; set; } = new FilePath() { Format = "Model file(*.h5)|*.h5" };
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(FilterClasses);
bw.Write(ModelFilePath.Path);
}
public override void Load(BinaryReader br)
{
base.Load(br);
FilterClasses = br.ReadString();
ModelFilePath.Path = br.ReadString();
}
}

View File

@@ -0,0 +1,113 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
[IgnoreOperation]
public class Color128HalfOperation:BaseOperation
{
public Color128HalfOperation()
{
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", "segment128half"));
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
_imageWidth = size[0];
_imageHeight = size[1];
CanHaveProcessingError = false;
}
public string FilterClasses { get; set; } = "0";
public FilePath ModelFilePath { get; set; } = new FilePath() { Format = "Model file(*.h5)|*.h5" };
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
private readonly int _imageWidth;
private readonly int _imageHeight;
private bool _initialized = false;
protected override void InterpretInternal(Context context)
{
if(!CheckImageExists(context))return;
if(!CheckColorful(context))return;
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", "segment128half"));
if (!_initialized)
{
if (!File.Exists(ModelFilePath.Path))
{
this.SetError("Model file not found");
return;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path));
_initialized = true;
}
var currentImage = context.ActiveImage;
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
var byteArray = new byte[dataSize];
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
var allowed = FilterClasses.Split(',',StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
var allowedFlags = new byte[128];
foreach (var i in allowed)
{
allowedFlags[i] = 1;
}
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("predict", byteArray));
var result = resultEncoded.Select(Convert.ToByte).ToArray();
var resultColor = result.Select(x =>
allowedFlags[x]==1
? (byte)255
: (byte)0)
.ToArray();
// black and white mask
var mask = new Mat(_imageHeight, _imageWidth, MatType.CV_8UC1, resultColor);
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
context.ActiveImage = new HawkeyeImage() {ImageData = maskResized};
Result = true;
}
public override void SetParameters(Dictionary<string, object> parameters)
{
base.SetParameters(parameters);
_initialized = false;
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(FilterClasses);
bw.Write(ModelFilePath.Path);
}
public override void Load(BinaryReader br)
{
base.Load(br);
FilterClasses = br.ReadString();
ModelFilePath.Path = br.ReadString();
}
}

View File

@@ -0,0 +1,36 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
[IgnoreOperation]
public class Color128SimpleOperation: ColorAIOperation
{
public Color128SimpleOperation():base("bg128simple")
{
}
public override string FilterClasses { get; set; } = "1";
public override FilePath ModelFilePath { get; set; } = new FilePath(){Format = "Model file(*.h5)|*.h5"};
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(FilterClasses);
bw.Write(ModelFilePath.Path);
}
public override void Load(BinaryReader br)
{
base.Load(br);
FilterClasses = br.ReadString();
ModelFilePath.Path = br.ReadString();
}
}

View File

@@ -0,0 +1,114 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[IgnoreOperation]
public abstract class ColorAIOperation:BaseOperation
{
private readonly string _modelName;
public ColorAIOperation(string model_name)
{
_modelName = model_name;
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
CanHaveProcessingError = false;
}
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
private int _imageWidth;
private int _imageHeight;
private bool _initialized = false;
private FilePath _modelFilePath;
public virtual string FilterClasses { get; set; }
public virtual FilePath ModelFilePath
{
get => _modelFilePath;
set
{
_modelFilePath = value;
_initialized = false;
}
}
public bool IsRaw { get; set; }
protected override void InterpretInternal(Context context)
{
if (!CheckImageExists(context)) return;
if (!CheckColorful(context)) return;
if (!_initialized)
{
if (!File.Exists(ModelFilePath.Path))
{
this.SetError("Model file not found");
return;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
_initialized = true;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
_imageWidth = size[1];
_imageHeight = size[2];
var currentImage = context.ActiveImage;
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
var byteArray = new byte[dataSize];
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
var allowedFlags = new byte[128];
foreach (var i in allowed)
{
allowedFlags[i] = 1;
}
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall(IsRaw?"predict_raw": "predict", byteArray));
var result = resultEncoded.Select(Convert.ToByte).ToArray();
byte[] resultColor;
if (IsRaw)
{
resultColor = result.Chunk(25).Select(x =>
{
return x.Select((x, i) => allowedFlags[i] == 1 ? (byte) (x) : (byte) 0).Max();
}).ToArray();
//resultColor = result.Select(x => (byte)(x)).ToArray();
}
else
{
resultColor = result.Select(x => allowedFlags[x]==1? (byte)(x * 255) : (byte)0).ToArray();
}
// black and white mask
var mask = new Mat(_imageHeight, _imageWidth, MatType.CV_8UC1, resultColor);
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };
Result = true;
}
}

View File

@@ -0,0 +1,42 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class ColorModelOperation : ColorAIOperation
{
public ColorModelOperation() : base("office2")
{
IsRaw = true;
}
public override string FilterClasses { get; set; } = "1";
public override FilePath ModelFilePath { get; set; } = new FilePath() { Format = "Model file(*.h5)|*.h5" };
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(FilterClasses);
// get current directory
var dir = Directory.GetCurrentDirectory();
// get relative path
var relPath = Path.GetRelativePath(dir, ModelFilePath.Path);
bw.Write(relPath);
}
public override void Load(BinaryReader br)
{
base.Load(br);
FilterClasses = br.ReadString();
ModelFilePath.Path = br.ReadString();
// get current directory
var dir = Directory.GetCurrentDirectory();
// get absolute path
var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
ModelFilePath.Path = absPath;
}
}

View File

@@ -0,0 +1,159 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class ModelAIOperation: BaseOperation
{
private readonly string _modelName;
public ModelAIOperation()
{
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
CanHaveProcessingError = false;
}
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
private int _imageWidth;
private int _imageHeight;
public string FilterClasses { get; set; } = "1";
private bool _initialized = false;
private FilePath _modelFilePath=new FilePath();
public FilePath ModelFilePath
{
get => _modelFilePath;
set
{
_modelFilePath = value;
_initialized = false;
}
}
public bool IsCategorical { get; set; }
protected override void InterpretInternal(Context context)
{
if (!CheckImageExists(context)) return;
if (!CheckColorful(context)) return;
if (!_initialized)
{
if (!File.Exists(ModelFilePath?.Path))
{
this.SetError("Model file not found");
return;
}
// is relative path
if (ModelFilePath.Path.StartsWith("."))
{
var dir = Directory.GetCurrentDirectory();
var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
ModelFilePath.Path = absPath;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
_initialized = true;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
_imageWidth = size[1];
_imageHeight = size[2];
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
int classes;
if (outputSizeArray.Length<3)
{
classes=1;
}
else
{
classes = outputSizeArray[2];
}
var currentImage = context.ActiveImage;
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
var byteArray = new byte[dataSize];
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
var allowedFlags = new byte[128];
foreach (var i in allowed)
{
allowedFlags[i] = 1;
}
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall(IsCategorical ? "predict_raw" : "predict", byteArray));
var result = resultEncoded.Select(Convert.ToByte).ToArray();
// todo: copy and name RawModelAIOperation
byte[] resultColor;
if (IsCategorical)
{
resultColor = result.Chunk(classes).Select(x =>
{
return x.Select((x, i) => allowedFlags[i] == 1 ? (byte)(x) : (byte)0).Max();
}).ToArray();
}
else
{
resultColor = result.Select(x => allowedFlags[x] == 1 ? (byte)(x * 255) : (byte)0).ToArray();
}
// black and white mask
var mask = new Mat(outputSizeArray[0], outputSizeArray[1], MatType.CV_8UC1, resultColor);
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };
Result = true;
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(_modelFilePath.Path);
bw.Write(_modelFilePath.Format);
bw.Write(FilterClasses);
bw.Write(IsCategorical);
}
public override void Load(BinaryReader br)
{
base.Load(br);
_modelFilePath.Path = br.ReadString();
_modelFilePath.Format = br.ReadString();
FilterClasses = br.ReadString();
IsCategorical = br.ReadBoolean();
}
}

View File

@@ -0,0 +1,148 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class MultichannelAI: BaseOperation
{
private readonly string _modelName;
public MultichannelAI()
{
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
CanHaveProcessingError = false;
}
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
private int _imageWidth;
private int _imageHeight;
private bool _initialized = false;
private FilePath _modelFilePath = new FilePath();
public FilePath ModelFilePath
{
get => _modelFilePath;
set
{
_modelFilePath = value;
_initialized = false;
}
}
protected unsafe override void InterpretInternal(Context context)
{
if (!CheckImageExists(context)) return;
if (!CheckColorful(context)) return;
if (!_initialized)
{
if (!File.Exists(ModelFilePath?.Path))
{
this.SetError("Model file not found");
return;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
_initialized = true;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
_imageWidth = size[1];
_imageHeight = size[2];
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
int classes;
if (outputSizeArray.Length < 3)
{
classes = 1;
}
else
{
classes = outputSizeArray[2];
}
var currentImage = context.ActiveImage;
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
var byteArray = new byte[dataSize];
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
Mat[] channels = new Mat[classes];
for (int i = 0; i < classes; i++)
{
channels[i] = new Mat(_imageHeight, _imageWidth, MatType.CV_8UC1);
}
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("predict_raw", byteArray));
var result = resultEncoded.Select(Convert.ToByte).ToArray();
var chunked = result.Chunk(classes).ToArray();
int pixel=0;
foreach (var chunk in chunked)
{
for (int i = 0; i < classes; i++)
{
var data = (byte*)channels[i].Data.ToPointer();
var coordX = (int)(pixel / _imageWidth);
var coordY = pixel % _imageWidth;
data[coordX * _imageWidth + coordY] = chunk[i];
}
pixel++;
}
//Cv2.ImShow("0", channels[0]);
//Cv2.ImShow("1", channels[1]);
using var res = new Mat();
Cv2.Merge(channels,res);
var resResized = res.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
context.ActiveImage = new HawkeyeImage() {ImageData = resResized };
Result = true;
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(_modelFilePath.Path);
bw.Write(_modelFilePath.Format);
}
public override void Load(BinaryReader br)
{
base.Load(br);
_modelFilePath.Path = br.ReadString();
_modelFilePath.Format = br.ReadString();
}
}

View File

@@ -0,0 +1,150 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class RawModelAIOperation: BaseOperation
{
private string _modelName;
public RawModelAIOperation()
{
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
CanHaveProcessingError = false;
}
private CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
private int _imageWidth;
private int _imageHeight;
public string FilterClasses { get; set; } = "1";
private bool _initialized = false;
private FilePath _modelFilePath=new FilePath();
public FilePath ModelFilePath
{
get => _modelFilePath;
set
{
_modelFilePath = value;
_initialized = false;
}
}
public bool IsCategorical { get; set; }
protected override void InterpretInternal(Context context)
{
if (!CheckImageExists(context)) return;
if (!CheckColorful(context)) return;
if (!_initialized)
{
if (!File.Exists(ModelFilePath?.Path))
{
this.SetError("Model file not found");
return;
}
_modelName = Path.GetFileNameWithoutExtension(ModelFilePath.Path);
// is relative path
if (ModelFilePath.Path.StartsWith("."))
{
var dir = Directory.GetCurrentDirectory();
var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
ModelFilePath.Path = absPath;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
_initialized = true;
}
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
_imageWidth = size[1];
_imageHeight = size[2];
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
int classes;
if (outputSizeArray.Length<3)
{
classes=1;
}
else
{
classes = outputSizeArray[2];
}
var currentImage = context.ActiveImage;
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
var byteArray = new byte[dataSize];
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
var allowedFlags = new byte[128];
foreach (var i in allowed)
{
allowedFlags[i] = 1;
}
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("predict_raw", byteArray));
var result = resultEncoded.Select(Convert.ToByte).ToArray();
byte[] resultColor;
resultColor = result.Select(x => (byte)(x) ).ToArray();
// black and white mask
var mask = new Mat(outputSizeArray[0], outputSizeArray[1], MatType.CV_8UC1, resultColor);
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };
Result = true;
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(_modelFilePath.Path);
bw.Write(_modelFilePath.Format);
bw.Write(FilterClasses);
bw.Write(IsCategorical);
}
public override void Load(BinaryReader br)
{
base.Load(br);
_modelFilePath.Path = br.ReadString();
_modelFilePath.Format = br.ReadString();
FilterClasses = br.ReadString();
IsCategorical = br.ReadBoolean();
}
}

View File

@@ -0,0 +1,142 @@
using System.Diagnostics;
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using static IronPython.Runtime.Profiler;
using System.Runtime.InteropServices;
using System.Text;
using Compunet.YoloV8;
using Compunet.YoloV8.Data;
using OpenCvSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Processing;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class YoloDetectionOperation:BaseOperation
{
private bool _initialized = false;
private FilePath _modelFilePath = new FilePath(){ Format = "Onnx format(*.onnx)|*.onnx" };
public FilePath ModelFilePath
{
get => _modelFilePath;
set
{
_modelFilePath = value;
_initialized = false;
}
}
bool EnsureInitialized()
{
if (!_initialized)
{
if (!File.Exists(ModelFilePath?.Path))
{
this.SetError("Model file not found");
return false;
}
_predictor = YoloV8Predictor.Create(ModelFilePath?.Path);
_predictor.Configuration.SuppressParallelInference=true;
_initialized = true;
}
return true;
}
byte[] _buffer=new byte[0];
private YoloV8Predictor _predictor;
void EnsureBuffer(int size)
{
if (_buffer.Length != size)
{
_buffer = new byte[size];
}
}
protected override void InterpretInternal(Context context)
{
CheckColorful(context);
try
{
if (!EnsureInitialized()) return;
var swImagePreparation = Stopwatch.StartNew();
var image = context.ActiveImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
EnsureBuffer(image.Cols * image.Rows * image.Channels());
Marshal.Copy(image.Data, _buffer, 0, _buffer.Length);
Image<Rgb24> img = SixLabors.ImageSharp.Image.LoadPixelData<Rgb24>(_buffer, image.Width, image.Height);
// convert to rgb24
swImagePreparation.Stop();
var swPrediction = Stopwatch.StartNew();
var result = _predictor.Detect(img);
swPrediction.Stop();
var swPostProcessing = Stopwatch.StartNew();
Mat res = new Mat(image.Size(), MatType.CV_8UC1, new Scalar(0));
StringBuilder sb = new StringBuilder();
Dictionary<int, string> classes = new Dictionary<int, string>();
foreach (BoundingBox box in result.Boxes.OrderBy(x=>x.Class.Id).ToList())
{
if (box.Confidence<0.7)
{
continue;
}
if (box.Class.Id + 1 == 10)
{
}
classes[box.Class.Id + 1] = box.Class.Name;
var rect = box.Bounds;
res.Rectangle(new OpenCvSharp.Rect(rect.Left, rect.Top, rect.Width, rect.Height),
new Scalar(box.Class.Id + 1), -1);
}
swPostProcessing.Stop();
foreach (var c in classes)
{
sb.AppendLine($"{c.Key}={c.Value};");
}
Status = sb.ToString();
context.ActiveImage = new HawkeyeImage() {ImageData = res};
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(ModelFilePath.Path);
}
public override void Load(BinaryReader br)
{
base.Load(br);
ModelFilePath.Path = br.ReadString();
}
}

View File

@@ -0,0 +1,115 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using static Hawkeye.VisionBuilder.Workflow.Operations.FindManyBlobsOperation;
using Point = OpenCvSharp.Point;
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
[Category("AI")]
public class YoloPickDetected:BaseOperation
{
public string SlotName { get; set; } = "Image";
public int TypeId { get; set; }
public int MinArea { get; set; } = 0;
public int MaxArea { get; set; } = 99999;
public int MinWidth { get; set; } = 0;
public int MaxWidth { get; set; } = 99999;
public int MinHeight { get; set; } = 0;
public int MaxHeight { get; set; } = 99999;
protected override void InterpretInternal(Context context)
{
Status = "No blobs found";
var image=context.Memory[SlotName];
var threshold= image.ImageData.InRange(TypeId, TypeId);
var contours = threshold
.FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple);
//case: no blobs found
if (contours.Length == 0)
{
return;
}
var validContours = new List<Point[]>();
foreach (var contour in contours)
{
var boundingRect = Cv2.BoundingRect(contour);
var area = Cv2.ContourArea(contour);
if (boundingRect.Width >= MinWidth && boundingRect.Width <= MaxWidth &&
boundingRect.Height >= MinHeight && boundingRect.Height <= MaxHeight &&
area >= MinArea && area <= MaxArea
)
{
validContours.Add(contour);
}
}
int badCount = 0;
foreach (var contourMatch in validContours)
{
PolyElement poly = new PolyElement() { IsGood = false };
poly.Points = contourMatch.Select(x => new Vector2(x.X, x.Y)).ToArray();
context.GraphicsElements.Add(poly);
badCount++;
}
if (badCount > 0)
{
// find smallest contour size
var smallest = validContours.OrderBy(x => Cv2.ContourArea(x)).First();
var smallestBoundingRect = Cv2.BoundingRect(smallest);
var biggest = validContours.OrderBy(x => Cv2.ContourArea(x)).Last();
var biggestBoundingRect = Cv2.BoundingRect(biggest);
var smallestArea = Cv2.ContourArea(smallest);
var biggestArea = Cv2.ContourArea(biggest);
Status = $"Found: {badCount} Min area:{smallestArea} Max area:{biggestArea}";
Result = false;
return;
}
Result = true;
}
public override void Save(BinaryWriter bw)
{
base.Save(bw);
bw.Write(SlotName);
bw.Write(TypeId);
bw.Write(MinWidth);
bw.Write(MaxWidth);
bw.Write(MinHeight);
bw.Write(MaxHeight);
bw.Write(MinArea);
bw.Write(MaxArea);
}
public override void Load(BinaryReader br)
{
base.Load(br);
SlotName = br.ReadString();
TypeId = br.ReadInt32();
MinWidth = br.ReadInt32();
MaxWidth = br.ReadInt32();
MinHeight = br.ReadInt32();
MaxHeight = br.ReadInt32();
MinArea = br.ReadInt32();
MaxArea = br.ReadInt32();
}
}

View File

@@ -0,0 +1,13 @@
using Hawkeye.VisionBuilder.Workflow.Datatypes;
namespace Hawkeye.VisionBuilder.Workflow.Operations;
public class ActivateImageOperation:BaseOperation
{
public ScriptValue ImageSource { get; set; } = new ScriptValue();
protected override void InterpretInternal(Context context)
{
context.ActiveImage = GetScriptValue<HawkeyeImage>(ImageSource);
}
}

View File

@@ -0,0 +1,12 @@
namespace Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
[AttributeUsage(AttributeTargets.Class)]
public class Category:Attribute
{
public string Name { get; }
public Category(string name)
{
Name = name;
}
}

View File

@@ -0,0 +1,7 @@
namespace Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
[AttributeUsage(AttributeTargets.Class,Inherited = false)]
public class IgnoreOperationAttribute : Attribute
{
}

View File

@@ -0,0 +1,7 @@
namespace Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
[AttributeUsage(AttributeTargets.Property)]
public class NotForToolAttribute:Attribute
{
}

View File

@@ -0,0 +1,12 @@
namespace Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
[AttributeUsage(AttributeTargets.Property)]
public class RestrictReferenceAttribute:Attribute
{
public RestrictReferenceAttribute(Type type)
{
Type = type;
}
public Type Type { get; set; }
}

Some files were not shown because too many files have changed in this diff Show More