check point
This commit is contained in:
meelstorm
125
Hawkeye.VisionBuilder.Workflow/Operations/AI/AnomalyAI.cs
Normal file
125
Hawkeye.VisionBuilder.Workflow/Operations/AI/AnomalyAI.cs
Normal file
@@ -0,0 +1,125 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
public class AnomalyAI: BaseOperation
|
||||
{
|
||||
|
||||
private readonly string _modelName;
|
||||
|
||||
public AnomalyAI()
|
||||
{
|
||||
|
||||
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
|
||||
CanHaveProcessingError = false;
|
||||
}
|
||||
|
||||
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
|
||||
private int _imageWidth;
|
||||
private int _imageHeight;
|
||||
|
||||
private bool _initialized = false;
|
||||
private FilePath _modelFilePath = new FilePath();
|
||||
|
||||
|
||||
public string MemorySlotName { get; set; } = "Anomaly";
|
||||
|
||||
public FilePath ModelFilePath
|
||||
{
|
||||
get => _modelFilePath;
|
||||
set
|
||||
{
|
||||
_modelFilePath = value;
|
||||
_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
if (!CheckImageExists(context)) return;
|
||||
if (!CheckColorful(context)) return;
|
||||
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath?.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return;
|
||||
}
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
|
||||
|
||||
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
|
||||
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
|
||||
_imageWidth = size[2];
|
||||
_imageHeight = size[1];
|
||||
|
||||
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
|
||||
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
|
||||
|
||||
|
||||
var currentImage = context.ActiveImage;
|
||||
|
||||
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
var resized = rightColor.Resize(new Size(_imageWidth,_imageHeight));
|
||||
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
|
||||
var byteArray = new byte[dataSize];
|
||||
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
|
||||
|
||||
|
||||
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall( "predict_anomaly", byteArray));
|
||||
|
||||
var result = resultEncoded.Select(Convert.ToByte).ToArray();
|
||||
|
||||
var defect = result.Take(1).ToArray()[0];
|
||||
|
||||
if (defect > 50)
|
||||
{
|
||||
|
||||
|
||||
byte[] resultColor = result.Skip(1).ToArray();
|
||||
|
||||
|
||||
|
||||
// black and white mask
|
||||
var mask = new Mat(30, 48, MatType.CV_8UC1, resultColor);
|
||||
|
||||
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
|
||||
|
||||
context.ActiveImage = new HawkeyeImage() {ImageData = maskResized};
|
||||
}
|
||||
else
|
||||
{
|
||||
context.ActiveImage = new HawkeyeImage() { ImageData = new Mat(currentImage.ImageData.Size(), MatType.CV_8UC1, new Scalar(0)) };
|
||||
}
|
||||
|
||||
Result = true;
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(_modelFilePath.Path);
|
||||
bw.Write(_modelFilePath.Format);
|
||||
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
_modelFilePath.Path = br.ReadString();
|
||||
_modelFilePath.Format = br.ReadString();
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
[IgnoreOperation]
|
||||
public class ArrayModelMatchingOperation:BaseOperation
|
||||
{
|
||||
private readonly WorkflowList _workflowList;
|
||||
private readonly CSharpDataTransferMQ _transfer;
|
||||
public Guid ReferenceId { get; set; } = new Guid();
|
||||
public ArrayHorizontalElement SearchArea { get; set; }
|
||||
public string ModelName { get; set; } = "cls_candy5.h5";
|
||||
public Action ExportImages { get; set; }
|
||||
public int Margin { get; set; } = 5;
|
||||
public int Amount { get; set; } = 4;
|
||||
|
||||
private int _channels = 3;
|
||||
private List<Mat> _lastCutImages;
|
||||
|
||||
public ArrayModelMatchingOperation(WorkflowList workflowList)
|
||||
{
|
||||
_workflowList = workflowList;
|
||||
_transfer = PythonModelProxy.GetInterface();
|
||||
|
||||
|
||||
SearchArea = new ArrayHorizontalElement()
|
||||
{
|
||||
Editable = true,
|
||||
Location = Vector2.One * 100,
|
||||
BlockSize = new Vector2(100, 100),
|
||||
|
||||
};
|
||||
ExportImages = () =>
|
||||
{
|
||||
if (!Directory.Exists(Path.Combine(@"..\Data\Export", ModelName)))
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(@"..\Data\Export", ModelName));
|
||||
}
|
||||
|
||||
// save last cut images
|
||||
var i = 0;
|
||||
foreach (var cutImage in _lastCutImages)
|
||||
{
|
||||
cutImage.SaveImage(Path.Combine(@"..\Data\Export", ModelName, $"{Guid.NewGuid().ToString()}.png"));
|
||||
}
|
||||
|
||||
};
|
||||
ReloadModelInfo();
|
||||
}
|
||||
|
||||
private void ReloadModelInfo()
|
||||
{
|
||||
var responseBytes = _transfer.TransferData(ModelName, 2, Array.Empty<byte>());
|
||||
int[] sizes = new int[3];
|
||||
Buffer.BlockCopy(responseBytes, 0, sizes, 0, 12);
|
||||
SearchArea.BlockSize = new Vector2(sizes[1], sizes[0]);
|
||||
_channels = sizes[2];
|
||||
}
|
||||
|
||||
byte[] MatToBytes(Mat mat)
|
||||
{
|
||||
if (mat.Channels() == 3)
|
||||
{
|
||||
mat = mat.CvtColor(_channels == 3 ? ColorConversionCodes.BGR2RGB : ColorConversionCodes.BGR2GRAY);
|
||||
}
|
||||
|
||||
|
||||
IntPtr dataPtr = mat.Data;
|
||||
|
||||
// Calculate the size of the image data
|
||||
int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
|
||||
|
||||
|
||||
|
||||
|
||||
// Copy the image data into a byte array
|
||||
byte[] byteArray = new byte[dataSize];
|
||||
|
||||
Marshal.Copy(dataPtr, byteArray, 0, dataSize);
|
||||
|
||||
return byteArray;
|
||||
}
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
Result = true;
|
||||
if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
|
||||
{
|
||||
Status = $"Model `{ModelName}` not found";
|
||||
Result = false;
|
||||
return;
|
||||
}
|
||||
if (!CheckImageExists(context)) return;
|
||||
switch (_channels)
|
||||
{
|
||||
case 3 when !CheckColorful(context):
|
||||
case 1 when !CheckGrayscale(context):
|
||||
return;
|
||||
}
|
||||
|
||||
var refLocation = _workflowList.GetOriginById(ReferenceId).Origin.Location;
|
||||
SearchArea.MovePivot(refLocation);
|
||||
|
||||
SearchArea.Margin=Margin;
|
||||
SearchArea.BlockCount=Amount;
|
||||
var img = context.ActiveImage.ImageData;
|
||||
|
||||
_lastCutImages = SearchArea
|
||||
.GenerateLocations()
|
||||
.Select(x => img[(int) x.Y, (int) x.Y + (int) SearchArea.BlockSize.Y, (int) x.X,
|
||||
(int) x.X + (int) SearchArea.BlockSize.X]).ToList();
|
||||
var images= _lastCutImages.Select(MatToBytes).ToList();
|
||||
|
||||
|
||||
var bytes= images.SelectMany(x => x).ToArray();
|
||||
|
||||
var len = images.Count;
|
||||
var bytes2 = new byte[bytes.Length + 4];
|
||||
bytes2[3] = (byte)(len >> 24);
|
||||
bytes2[2] = (byte)(len >> 16);
|
||||
bytes2[1] = (byte)(len >> 8);
|
||||
bytes2[0] = (byte)(len >> 0);
|
||||
|
||||
Array.Copy(bytes, 0, bytes2, 4, bytes.Length);
|
||||
|
||||
var answer = _transfer.TransferData(ModelName, 0,bytes2);
|
||||
float[] floatArray = new float[answer.Length / 4];
|
||||
Buffer.BlockCopy(answer, 0, floatArray, 0, answer.Length);
|
||||
|
||||
// Print answer
|
||||
for (int i = 0; i < Amount; i++)
|
||||
{
|
||||
Console.WriteLine(floatArray[i]);
|
||||
}
|
||||
|
||||
for (int i = 0; i < Amount; i++)
|
||||
{
|
||||
SearchArea.IsGood[i] = floatArray[i] > 0.7;
|
||||
}
|
||||
|
||||
|
||||
|
||||
context.GraphicsElements.Add(SearchArea);
|
||||
Result = SearchArea.IsGood.All(x=>x.Value==true);
|
||||
}
|
||||
|
||||
public override void SetParameters(Dictionary<string, object> parameters)
|
||||
{
|
||||
base.SetParameters(parameters);
|
||||
ReloadModelInfo();
|
||||
SearchArea.SetPivot(_workflowList.GetOriginById(ReferenceId).Origin.Location);
|
||||
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(ReferenceId.ToString());
|
||||
bw.Write(Margin);
|
||||
bw.Write(Amount);
|
||||
bw.Write(ModelName);
|
||||
SearchArea.Save(bw);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
ReferenceId = new Guid(br.ReadString());
|
||||
Margin = br.ReadInt32();
|
||||
Amount = br.ReadInt32();
|
||||
ModelName = br.ReadString();
|
||||
SearchArea = new ArrayHorizontalElement();
|
||||
SearchArea.Load(br);
|
||||
ReloadModelInfo();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
[IgnoreOperation]
|
||||
public class ArrayModelSamplerOperation:BaseOperation
|
||||
{
|
||||
private Mat _lastImage;
|
||||
|
||||
public RectangleElement SearchArea { get; set; }
|
||||
public string SampleStorage { get; set; } = "samples.storage";
|
||||
public Action Sample { get; set; }
|
||||
|
||||
public ArrayModelSamplerOperation()
|
||||
{
|
||||
SearchArea = new RectangleElement()
|
||||
{
|
||||
Editable = true,
|
||||
Location = Vector2.One * 100,
|
||||
Size = new Vector2(100, 100),
|
||||
|
||||
};
|
||||
Sample = SaveSample;
|
||||
}
|
||||
|
||||
byte[] MatToBytes(Mat mat)
|
||||
{
|
||||
if (mat.Channels() == 3)
|
||||
{
|
||||
mat = mat.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
}
|
||||
|
||||
|
||||
IntPtr dataPtr = mat.Data;
|
||||
int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
|
||||
byte[] byteArray = new byte[dataSize];
|
||||
|
||||
Marshal.Copy(dataPtr, byteArray, 0, dataSize);
|
||||
|
||||
return byteArray;
|
||||
}
|
||||
|
||||
private void SaveSample()
|
||||
{
|
||||
|
||||
|
||||
|
||||
Mat mask = new Mat(_lastImage.Size(), MatType.CV_8UC1, Scalar.Black);
|
||||
SearchArea.FillMat(mask);
|
||||
var imageBytes = MatToBytes(_lastImage);
|
||||
var maskBytes = MatToBytes(mask);
|
||||
// Append to the end of storage file
|
||||
|
||||
var storagePath=Path.Combine("..\\Data\\AI", SampleStorage);
|
||||
|
||||
|
||||
|
||||
using (var file = new System.IO.FileStream(storagePath, System.IO.FileMode.Append))
|
||||
{
|
||||
// Write size
|
||||
file.Write(BitConverter.GetBytes(_lastImage.Width), 0, 4);
|
||||
file.Write(BitConverter.GetBytes(_lastImage.Height), 0, 4);
|
||||
file.Write(imageBytes, 0, imageBytes.Length);
|
||||
// Write mask
|
||||
file.Write(maskBytes, 0, maskBytes.Length);
|
||||
}
|
||||
}
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
if (!CheckImageExists(context)) return;
|
||||
if(!CheckColorful(context)) return;
|
||||
|
||||
context.GraphicsElements.Add(SearchArea);
|
||||
|
||||
_lastImage=context.ActiveImage.ImageData;
|
||||
Result = true;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using OpenCvSharp;
|
||||
using System.Drawing;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly;
|
||||
using Hawkeye.VisionBuilder.Workflow.Links;
|
||||
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
[IgnoreOperation]
|
||||
public class BackgroundSeparationModelOperation:BaseOperation,IHaveOrigin
|
||||
{
|
||||
private readonly CSharpDataTransferMQ _transfer;
|
||||
private int _channels;
|
||||
public string ModelName { get; set; } = "background.h5";
|
||||
|
||||
|
||||
public BackgroundSeparationModelOperation()
|
||||
{
|
||||
_transfer = PythonModelProxy.GetInterface();
|
||||
|
||||
}
|
||||
|
||||
byte[] MatToBytes(Mat mat)
|
||||
{
|
||||
if (mat.Channels() == 3)
|
||||
{
|
||||
mat = mat.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
}
|
||||
|
||||
|
||||
IntPtr dataPtr = mat.Data;
|
||||
int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
|
||||
byte[] byteArray = new byte[dataSize];
|
||||
|
||||
Marshal.Copy(dataPtr, byteArray, 0, dataSize);
|
||||
|
||||
return byteArray;
|
||||
}
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
Result = true;
|
||||
if (!CheckImageExists(context)) return;
|
||||
if (!CheckColorful(context)) return;
|
||||
if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
|
||||
{
|
||||
Status= $"Model `{ModelName}` not found";
|
||||
Result = false;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
var img = context.ActiveImage.ImageData;
|
||||
|
||||
var bytes= MatToBytes(img);
|
||||
var responseBytes = _transfer.TransferData(ModelName, 3, bytes);
|
||||
var response = new Mat(img.Rows, img.Cols, MatType.CV_8UC1, responseBytes);
|
||||
|
||||
var contours = response.Threshold(128, 255, ThresholdTypes.Binary).FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple);
|
||||
|
||||
if (contours.Length == 0)
|
||||
{
|
||||
Status = "No object found";
|
||||
Result = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// find biggest contour
|
||||
var max = 0;
|
||||
var maxIndex = 0;
|
||||
for (var i = 0; i < contours.Length; i++)
|
||||
{
|
||||
var area = Cv2.ContourArea(contours[i]);
|
||||
if (area > max)
|
||||
{
|
||||
max = (int)area;
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
PolyElement poly = new PolyElement() { IsGood = true,Editable = false};
|
||||
|
||||
poly.Points = contours[maxIndex].Select(x => new Vector2(x.X, x.Y)).ToArray();
|
||||
|
||||
context.GraphicsElements.Add(poly);
|
||||
|
||||
|
||||
var rect = Cv2.MinAreaRect(contours[maxIndex]);
|
||||
var bound = rect.BoundingRect();
|
||||
Origin=new OriginElement()
|
||||
{
|
||||
Location = new Vector2(bound.Left, bound.Top),
|
||||
};
|
||||
|
||||
Status= $"Object found at {Origin.Location}";
|
||||
Result = true;
|
||||
|
||||
}
|
||||
|
||||
public override void SetParameters(Dictionary<string, object> parameters)
|
||||
{
|
||||
base.SetParameters(parameters);
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(ModelName);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
ModelName = br.ReadString();
|
||||
}
|
||||
|
||||
public OriginElement Origin { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
[IgnoreOperation]
|
||||
public class Color128BinaryOperation:ColorAIOperation
|
||||
{
|
||||
public Color128BinaryOperation() : base("prob128binary")
|
||||
{
|
||||
IsRaw = true;
|
||||
}
|
||||
|
||||
|
||||
public override string FilterClasses { get; set; } = "1";
|
||||
public override FilePath ModelFilePath { get; set; } = new FilePath() { Format = "Model file(*.h5)|*.h5" };
|
||||
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(FilterClasses);
|
||||
bw.Write(ModelFilePath.Path);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
FilterClasses = br.ReadString();
|
||||
ModelFilePath.Path = br.ReadString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using OpenCvSharp;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
[IgnoreOperation]
|
||||
public class Color128HalfOperation:BaseOperation
|
||||
{
|
||||
public Color128HalfOperation()
|
||||
{
|
||||
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", "segment128half"));
|
||||
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
|
||||
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
|
||||
_imageWidth = size[0];
|
||||
_imageHeight = size[1];
|
||||
CanHaveProcessingError = false;
|
||||
}
|
||||
|
||||
public string FilterClasses { get; set; } = "0";
|
||||
public FilePath ModelFilePath { get; set; } = new FilePath() { Format = "Model file(*.h5)|*.h5" };
|
||||
|
||||
|
||||
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
|
||||
private readonly int _imageWidth;
|
||||
private readonly int _imageHeight;
|
||||
|
||||
private bool _initialized = false;
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
if(!CheckImageExists(context))return;
|
||||
if(!CheckColorful(context))return;
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", "segment128half"));
|
||||
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return;
|
||||
}
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path));
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
var currentImage = context.ActiveImage;
|
||||
|
||||
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
|
||||
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
|
||||
var byteArray = new byte[dataSize];
|
||||
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
|
||||
|
||||
var allowed = FilterClasses.Split(',',StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
|
||||
|
||||
var allowedFlags = new byte[128];
|
||||
foreach (var i in allowed)
|
||||
{
|
||||
allowedFlags[i] = 1;
|
||||
}
|
||||
|
||||
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("predict", byteArray));
|
||||
|
||||
var result = resultEncoded.Select(Convert.ToByte).ToArray();
|
||||
|
||||
var resultColor = result.Select(x =>
|
||||
allowedFlags[x]==1
|
||||
? (byte)255
|
||||
: (byte)0)
|
||||
.ToArray();
|
||||
|
||||
// black and white mask
|
||||
var mask = new Mat(_imageHeight, _imageWidth, MatType.CV_8UC1, resultColor);
|
||||
|
||||
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
|
||||
|
||||
context.ActiveImage = new HawkeyeImage() {ImageData = maskResized};
|
||||
|
||||
|
||||
Result = true;
|
||||
|
||||
|
||||
}
|
||||
|
||||
public override void SetParameters(Dictionary<string, object> parameters)
|
||||
{
|
||||
base.SetParameters(parameters);
|
||||
_initialized = false;
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(FilterClasses);
|
||||
bw.Write(ModelFilePath.Path);
|
||||
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
FilterClasses = br.ReadString();
|
||||
ModelFilePath.Path = br.ReadString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
[IgnoreOperation]
|
||||
public class Color128SimpleOperation: ColorAIOperation
|
||||
{
|
||||
public Color128SimpleOperation():base("bg128simple")
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
|
||||
public override string FilterClasses { get; set; } = "1";
|
||||
public override FilePath ModelFilePath { get; set; } = new FilePath(){Format = "Model file(*.h5)|*.h5"};
|
||||
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(FilterClasses);
|
||||
bw.Write(ModelFilePath.Path);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
FilterClasses = br.ReadString();
|
||||
ModelFilePath.Path = br.ReadString();
|
||||
}
|
||||
}
|
||||
114
Hawkeye.VisionBuilder.Workflow/Operations/AI/ColorAIOperation.cs
Normal file
114
Hawkeye.VisionBuilder.Workflow/Operations/AI/ColorAIOperation.cs
Normal file
@@ -0,0 +1,114 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[IgnoreOperation]
|
||||
public abstract class ColorAIOperation:BaseOperation
|
||||
{
|
||||
private readonly string _modelName;
|
||||
|
||||
public ColorAIOperation(string model_name)
|
||||
{
|
||||
_modelName = model_name;
|
||||
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
|
||||
CanHaveProcessingError = false;
|
||||
}
|
||||
|
||||
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
|
||||
private int _imageWidth;
|
||||
private int _imageHeight;
|
||||
|
||||
private bool _initialized = false;
|
||||
private FilePath _modelFilePath;
|
||||
|
||||
|
||||
public virtual string FilterClasses { get; set; }
|
||||
|
||||
public virtual FilePath ModelFilePath
|
||||
{
|
||||
get => _modelFilePath;
|
||||
set
|
||||
{
|
||||
_modelFilePath = value;
|
||||
_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsRaw { get; set; }
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
if (!CheckImageExists(context)) return;
|
||||
if (!CheckColorful(context)) return;
|
||||
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return;
|
||||
}
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
|
||||
|
||||
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
|
||||
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
|
||||
_imageWidth = size[1];
|
||||
_imageHeight = size[2];
|
||||
|
||||
|
||||
var currentImage = context.ActiveImage;
|
||||
|
||||
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
|
||||
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
|
||||
var byteArray = new byte[dataSize];
|
||||
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
|
||||
|
||||
var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
|
||||
|
||||
var allowedFlags = new byte[128];
|
||||
foreach (var i in allowed)
|
||||
{
|
||||
allowedFlags[i] = 1;
|
||||
}
|
||||
|
||||
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall(IsRaw?"predict_raw": "predict", byteArray));
|
||||
|
||||
var result = resultEncoded.Select(Convert.ToByte).ToArray();
|
||||
|
||||
byte[] resultColor;
|
||||
if (IsRaw)
|
||||
{
|
||||
resultColor = result.Chunk(25).Select(x =>
|
||||
{
|
||||
return x.Select((x, i) => allowedFlags[i] == 1 ? (byte) (x) : (byte) 0).Max();
|
||||
}).ToArray();
|
||||
//resultColor = result.Select(x => (byte)(x)).ToArray();
|
||||
}
|
||||
else
|
||||
{
|
||||
resultColor = result.Select(x => allowedFlags[x]==1? (byte)(x * 255) : (byte)0).ToArray();
|
||||
}
|
||||
|
||||
|
||||
// black and white mask
|
||||
var mask = new Mat(_imageHeight, _imageWidth, MatType.CV_8UC1, resultColor);
|
||||
|
||||
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
|
||||
|
||||
context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };
|
||||
|
||||
|
||||
Result = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
public class ColorModelOperation : ColorAIOperation
|
||||
{
|
||||
public ColorModelOperation() : base("office2")
|
||||
{
|
||||
IsRaw = true;
|
||||
}
|
||||
|
||||
|
||||
public override string FilterClasses { get; set; } = "1";
|
||||
public override FilePath ModelFilePath { get; set; } = new FilePath() { Format = "Model file(*.h5)|*.h5" };
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(FilterClasses);
|
||||
// get current directory
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
// get relative path
|
||||
var relPath = Path.GetRelativePath(dir, ModelFilePath.Path);
|
||||
|
||||
|
||||
bw.Write(relPath);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
FilterClasses = br.ReadString();
|
||||
ModelFilePath.Path = br.ReadString();
|
||||
// get current directory
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
// get absolute path
|
||||
var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
|
||||
ModelFilePath.Path = absPath;
|
||||
}
|
||||
}
|
||||
159
Hawkeye.VisionBuilder.Workflow/Operations/AI/ModelAIOperation.cs
Normal file
159
Hawkeye.VisionBuilder.Workflow/Operations/AI/ModelAIOperation.cs
Normal file
@@ -0,0 +1,159 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
public class ModelAIOperation: BaseOperation
|
||||
{
|
||||
|
||||
|
||||
private readonly string _modelName;
|
||||
|
||||
public ModelAIOperation()
|
||||
{
|
||||
|
||||
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
|
||||
CanHaveProcessingError = false;
|
||||
}
|
||||
|
||||
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
|
||||
private int _imageWidth;
|
||||
private int _imageHeight;
|
||||
|
||||
|
||||
|
||||
public string FilterClasses { get; set; } = "1";
|
||||
|
||||
private bool _initialized = false;
|
||||
private FilePath _modelFilePath=new FilePath();
|
||||
|
||||
|
||||
public FilePath ModelFilePath
|
||||
{
|
||||
get => _modelFilePath;
|
||||
set
|
||||
{
|
||||
_modelFilePath = value;
|
||||
_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsCategorical { get; set; }
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
if (!CheckImageExists(context)) return;
|
||||
if (!CheckColorful(context)) return;
|
||||
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath?.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return;
|
||||
}
|
||||
|
||||
// is relative path
|
||||
if (ModelFilePath.Path.StartsWith("."))
|
||||
{
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
|
||||
ModelFilePath.Path = absPath;
|
||||
}
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
|
||||
|
||||
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
|
||||
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
|
||||
_imageWidth = size[1];
|
||||
_imageHeight = size[2];
|
||||
|
||||
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
|
||||
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
|
||||
|
||||
int classes;
|
||||
if (outputSizeArray.Length<3)
|
||||
{
|
||||
classes=1;
|
||||
}
|
||||
else
|
||||
{
|
||||
classes = outputSizeArray[2];
|
||||
}
|
||||
|
||||
|
||||
|
||||
var currentImage = context.ActiveImage;
|
||||
|
||||
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
|
||||
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
|
||||
var byteArray = new byte[dataSize];
|
||||
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
|
||||
|
||||
var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
|
||||
|
||||
var allowedFlags = new byte[128];
|
||||
foreach (var i in allowed)
|
||||
{
|
||||
allowedFlags[i] = 1;
|
||||
}
|
||||
|
||||
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall(IsCategorical ? "predict_raw" : "predict", byteArray));
|
||||
|
||||
var result = resultEncoded.Select(Convert.ToByte).ToArray();
|
||||
// todo: copy and name RawModelAIOperation
|
||||
byte[] resultColor;
|
||||
if (IsCategorical)
|
||||
{
|
||||
resultColor = result.Chunk(classes).Select(x =>
|
||||
{
|
||||
return x.Select((x, i) => allowedFlags[i] == 1 ? (byte)(x) : (byte)0).Max();
|
||||
}).ToArray();
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
resultColor = result.Select(x => allowedFlags[x] == 1 ? (byte)(x * 255) : (byte)0).ToArray();
|
||||
}
|
||||
|
||||
|
||||
// black and white mask
|
||||
var mask = new Mat(outputSizeArray[0], outputSizeArray[1], MatType.CV_8UC1, resultColor);
|
||||
|
||||
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
|
||||
|
||||
context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };
|
||||
|
||||
|
||||
Result = true;
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(_modelFilePath.Path);
|
||||
bw.Write(_modelFilePath.Format);
|
||||
bw.Write(FilterClasses);
|
||||
bw.Write(IsCategorical);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
_modelFilePath.Path = br.ReadString();
|
||||
_modelFilePath.Format = br.ReadString();
|
||||
FilterClasses = br.ReadString();
|
||||
IsCategorical = br.ReadBoolean();
|
||||
}
|
||||
}
|
||||
148
Hawkeye.VisionBuilder.Workflow/Operations/AI/MultichannelAI.cs
Normal file
148
Hawkeye.VisionBuilder.Workflow/Operations/AI/MultichannelAI.cs
Normal file
@@ -0,0 +1,148 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading.Tasks;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
public class MultichannelAI: BaseOperation
|
||||
{
|
||||
|
||||
private readonly string _modelName;
|
||||
|
||||
public MultichannelAI()
|
||||
{
|
||||
|
||||
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
|
||||
CanHaveProcessingError = false;
|
||||
}
|
||||
|
||||
private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
|
||||
private int _imageWidth;
|
||||
private int _imageHeight;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private bool _initialized = false;
|
||||
private FilePath _modelFilePath = new FilePath();
|
||||
|
||||
|
||||
public FilePath ModelFilePath
|
||||
{
|
||||
get => _modelFilePath;
|
||||
set
|
||||
{
|
||||
_modelFilePath = value;
|
||||
_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
protected unsafe override void InterpretInternal(Context context)
|
||||
{
|
||||
if (!CheckImageExists(context)) return;
|
||||
if (!CheckColorful(context)) return;
|
||||
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath?.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return;
|
||||
}
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
|
||||
|
||||
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
|
||||
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
|
||||
_imageWidth = size[1];
|
||||
_imageHeight = size[2];
|
||||
|
||||
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
|
||||
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
|
||||
|
||||
int classes;
|
||||
if (outputSizeArray.Length < 3)
|
||||
{
|
||||
classes = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
classes = outputSizeArray[2];
|
||||
}
|
||||
|
||||
|
||||
|
||||
var currentImage = context.ActiveImage;
|
||||
|
||||
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
|
||||
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
|
||||
var byteArray = new byte[dataSize];
|
||||
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
|
||||
|
||||
Mat[] channels = new Mat[classes];
|
||||
for (int i = 0; i < classes; i++)
|
||||
{
|
||||
channels[i] = new Mat(_imageHeight, _imageWidth, MatType.CV_8UC1);
|
||||
}
|
||||
|
||||
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("predict_raw", byteArray));
|
||||
|
||||
var result = resultEncoded.Select(Convert.ToByte).ToArray();
|
||||
|
||||
var chunked = result.Chunk(classes).ToArray();
|
||||
|
||||
int pixel=0;
|
||||
foreach (var chunk in chunked)
|
||||
{
|
||||
for (int i = 0; i < classes; i++)
|
||||
{
|
||||
var data = (byte*)channels[i].Data.ToPointer();
|
||||
var coordX = (int)(pixel / _imageWidth);
|
||||
var coordY = pixel % _imageWidth;
|
||||
data[coordX * _imageWidth + coordY] = chunk[i];
|
||||
}
|
||||
pixel++;
|
||||
}
|
||||
|
||||
//Cv2.ImShow("0", channels[0]);
|
||||
//Cv2.ImShow("1", channels[1]);
|
||||
|
||||
using var res = new Mat();
|
||||
Cv2.Merge(channels,res);
|
||||
|
||||
var resResized = res.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
|
||||
context.ActiveImage = new HawkeyeImage() {ImageData = resResized };
|
||||
Result = true;
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(_modelFilePath.Path);
|
||||
bw.Write(_modelFilePath.Format);
|
||||
|
||||
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
_modelFilePath.Path = br.ReadString();
|
||||
_modelFilePath.Format = br.ReadString();
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using OpenCvSharp;
|
||||
using System.Runtime.InteropServices;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
public class RawModelAIOperation: BaseOperation
|
||||
{
|
||||
|
||||
|
||||
private string _modelName;
|
||||
|
||||
public RawModelAIOperation()
|
||||
{
|
||||
|
||||
_cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
|
||||
CanHaveProcessingError = false;
|
||||
}
|
||||
|
||||
private CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
|
||||
private int _imageWidth;
|
||||
private int _imageHeight;
|
||||
|
||||
|
||||
|
||||
public string FilterClasses { get; set; } = "1";
|
||||
|
||||
private bool _initialized = false;
|
||||
private FilePath _modelFilePath=new FilePath();
|
||||
|
||||
|
||||
public FilePath ModelFilePath
|
||||
{
|
||||
get => _modelFilePath;
|
||||
set
|
||||
{
|
||||
_modelFilePath = value;
|
||||
_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsCategorical { get; set; }
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
if (!CheckImageExists(context)) return;
|
||||
if (!CheckColorful(context)) return;
|
||||
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath?.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return;
|
||||
}
|
||||
|
||||
_modelName = Path.GetFileNameWithoutExtension(ModelFilePath.Path);
|
||||
|
||||
// is relative path
|
||||
if (ModelFilePath.Path.StartsWith("."))
|
||||
{
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
|
||||
ModelFilePath.Path = absPath;
|
||||
}
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
_cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));
|
||||
|
||||
var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
|
||||
var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
|
||||
_imageWidth = size[1];
|
||||
_imageHeight = size[2];
|
||||
|
||||
var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
|
||||
var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();
|
||||
|
||||
int classes;
|
||||
if (outputSizeArray.Length<3)
|
||||
{
|
||||
classes=1;
|
||||
}
|
||||
else
|
||||
{
|
||||
classes = outputSizeArray[2];
|
||||
}
|
||||
|
||||
|
||||
|
||||
var currentImage = context.ActiveImage;
|
||||
|
||||
var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
|
||||
int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
|
||||
var byteArray = new byte[dataSize];
|
||||
Marshal.Copy(resized.Data, byteArray, 0, dataSize);
|
||||
|
||||
var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();
|
||||
|
||||
var allowedFlags = new byte[128];
|
||||
foreach (var i in allowed)
|
||||
{
|
||||
allowedFlags[i] = 1;
|
||||
}
|
||||
|
||||
var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("predict_raw", byteArray));
|
||||
|
||||
var result = resultEncoded.Select(Convert.ToByte).ToArray();
|
||||
|
||||
byte[] resultColor;
|
||||
resultColor = result.Select(x => (byte)(x) ).ToArray();
|
||||
|
||||
|
||||
// black and white mask
|
||||
var mask = new Mat(outputSizeArray[0], outputSizeArray[1], MatType.CV_8UC1, resultColor);
|
||||
|
||||
var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));
|
||||
|
||||
context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };
|
||||
|
||||
|
||||
Result = true;
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(_modelFilePath.Path);
|
||||
bw.Write(_modelFilePath.Format);
|
||||
bw.Write(FilterClasses);
|
||||
bw.Write(IsCategorical);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
_modelFilePath.Path = br.ReadString();
|
||||
_modelFilePath.Format = br.ReadString();
|
||||
FilterClasses = br.ReadString();
|
||||
IsCategorical = br.ReadBoolean();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using System.Diagnostics;
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
using static IronPython.Runtime.Profiler;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using Compunet.YoloV8;
|
||||
using Compunet.YoloV8.Data;
|
||||
using OpenCvSharp;
|
||||
using SixLabors.ImageSharp.PixelFormats;
|
||||
using SixLabors.ImageSharp;
|
||||
using SixLabors.ImageSharp.Processing;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
[Category("AI")]
|
||||
public class YoloDetectionOperation:BaseOperation
|
||||
{
|
||||
|
||||
private bool _initialized = false;
|
||||
private FilePath _modelFilePath = new FilePath(){ Format = "Onnx format(*.onnx)|*.onnx" };
|
||||
|
||||
|
||||
public FilePath ModelFilePath
|
||||
{
|
||||
get => _modelFilePath;
|
||||
set
|
||||
{
|
||||
_modelFilePath = value;
|
||||
_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
bool EnsureInitialized()
|
||||
{
|
||||
if (!_initialized)
|
||||
{
|
||||
if (!File.Exists(ModelFilePath?.Path))
|
||||
{
|
||||
this.SetError("Model file not found");
|
||||
return false;
|
||||
}
|
||||
_predictor = YoloV8Predictor.Create(ModelFilePath?.Path);
|
||||
|
||||
_predictor.Configuration.SuppressParallelInference=true;
|
||||
_initialized = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
byte[] _buffer=new byte[0];
|
||||
private YoloV8Predictor _predictor;
|
||||
|
||||
|
||||
void EnsureBuffer(int size)
|
||||
{
|
||||
if (_buffer.Length != size)
|
||||
{
|
||||
_buffer = new byte[size];
|
||||
}
|
||||
}
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
CheckColorful(context);
|
||||
try
|
||||
{
|
||||
if (!EnsureInitialized()) return;
|
||||
var swImagePreparation = Stopwatch.StartNew();
|
||||
|
||||
var image = context.ActiveImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
|
||||
EnsureBuffer(image.Cols * image.Rows * image.Channels());
|
||||
Marshal.Copy(image.Data, _buffer, 0, _buffer.Length);
|
||||
Image<Rgb24> img = SixLabors.ImageSharp.Image.LoadPixelData<Rgb24>(_buffer, image.Width, image.Height);
|
||||
// convert to rgb24
|
||||
|
||||
|
||||
swImagePreparation.Stop();
|
||||
|
||||
var swPrediction = Stopwatch.StartNew();
|
||||
|
||||
|
||||
|
||||
var result = _predictor.Detect(img);
|
||||
|
||||
|
||||
|
||||
swPrediction.Stop();
|
||||
|
||||
|
||||
var swPostProcessing = Stopwatch.StartNew();
|
||||
Mat res = new Mat(image.Size(), MatType.CV_8UC1, new Scalar(0));
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Dictionary<int, string> classes = new Dictionary<int, string>();
|
||||
|
||||
|
||||
|
||||
foreach (BoundingBox box in result.Boxes.OrderBy(x=>x.Class.Id).ToList())
|
||||
{
|
||||
if (box.Confidence<0.7)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (box.Class.Id + 1 == 10)
|
||||
{
|
||||
|
||||
}
|
||||
classes[box.Class.Id + 1] = box.Class.Name;
|
||||
var rect = box.Bounds;
|
||||
res.Rectangle(new OpenCvSharp.Rect(rect.Left, rect.Top, rect.Width, rect.Height),
|
||||
new Scalar(box.Class.Id + 1), -1);
|
||||
}
|
||||
swPostProcessing.Stop();
|
||||
|
||||
|
||||
foreach (var c in classes)
|
||||
{
|
||||
sb.AppendLine($"{c.Key}={c.Value};");
|
||||
}
|
||||
|
||||
Status = sb.ToString();
|
||||
|
||||
context.ActiveImage = new HawkeyeImage() {ImageData = res};
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(ModelFilePath.Path);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
ModelFilePath.Path = br.ReadString();
|
||||
}
|
||||
}
|
||||
115
Hawkeye.VisionBuilder.Workflow/Operations/AI/YoloPickDetected.cs
Normal file
115
Hawkeye.VisionBuilder.Workflow/Operations/AI/YoloPickDetected.cs
Normal file
@@ -0,0 +1,115 @@
|
||||
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly;
|
||||
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
|
||||
|
||||
using OpenCvSharp;
|
||||
using static Hawkeye.VisionBuilder.Workflow.Operations.FindManyBlobsOperation;
|
||||
using Point = OpenCvSharp.Point;
|
||||
|
||||
namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
|
||||
|
||||
[Category("AI")]
|
||||
public class YoloPickDetected:BaseOperation
|
||||
{
|
||||
public string SlotName { get; set; } = "Image";
|
||||
public int TypeId { get; set; }
|
||||
|
||||
public int MinArea { get; set; } = 0;
|
||||
public int MaxArea { get; set; } = 99999;
|
||||
public int MinWidth { get; set; } = 0;
|
||||
public int MaxWidth { get; set; } = 99999;
|
||||
public int MinHeight { get; set; } = 0;
|
||||
public int MaxHeight { get; set; } = 99999;
|
||||
|
||||
|
||||
protected override void InterpretInternal(Context context)
|
||||
{
|
||||
Status = "No blobs found";
|
||||
var image=context.Memory[SlotName];
|
||||
var threshold= image.ImageData.InRange(TypeId, TypeId);
|
||||
var contours = threshold
|
||||
.FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple);
|
||||
|
||||
//case: no blobs found
|
||||
if (contours.Length == 0)
|
||||
{
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var validContours = new List<Point[]>();
|
||||
|
||||
foreach (var contour in contours)
|
||||
{
|
||||
var boundingRect = Cv2.BoundingRect(contour);
|
||||
var area = Cv2.ContourArea(contour);
|
||||
if (boundingRect.Width >= MinWidth && boundingRect.Width <= MaxWidth &&
|
||||
boundingRect.Height >= MinHeight && boundingRect.Height <= MaxHeight &&
|
||||
area >= MinArea && area <= MaxArea
|
||||
)
|
||||
{
|
||||
validContours.Add(contour);
|
||||
}
|
||||
}
|
||||
|
||||
int badCount = 0;
|
||||
foreach (var contourMatch in validContours)
|
||||
{
|
||||
PolyElement poly = new PolyElement() { IsGood = false };
|
||||
|
||||
poly.Points = contourMatch.Select(x => new Vector2(x.X, x.Y)).ToArray();
|
||||
|
||||
context.GraphicsElements.Add(poly);
|
||||
badCount++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (badCount > 0)
|
||||
{
|
||||
// find smallest contour size
|
||||
var smallest = validContours.OrderBy(x => Cv2.ContourArea(x)).First();
|
||||
var smallestBoundingRect = Cv2.BoundingRect(smallest);
|
||||
var biggest = validContours.OrderBy(x => Cv2.ContourArea(x)).Last();
|
||||
var biggestBoundingRect = Cv2.BoundingRect(biggest);
|
||||
var smallestArea = Cv2.ContourArea(smallest);
|
||||
var biggestArea = Cv2.ContourArea(biggest);
|
||||
Status = $"Found: {badCount} Min area:{smallestArea} Max area:{biggestArea}";
|
||||
|
||||
|
||||
|
||||
Result = false;
|
||||
return;
|
||||
}
|
||||
|
||||
Result = true;
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
public override void Save(BinaryWriter bw)
|
||||
{
|
||||
base.Save(bw);
|
||||
bw.Write(SlotName);
|
||||
bw.Write(TypeId);
|
||||
bw.Write(MinWidth);
|
||||
bw.Write(MaxWidth);
|
||||
bw.Write(MinHeight);
|
||||
bw.Write(MaxHeight);
|
||||
bw.Write(MinArea);
|
||||
bw.Write(MaxArea);
|
||||
}
|
||||
|
||||
public override void Load(BinaryReader br)
|
||||
{
|
||||
base.Load(br);
|
||||
SlotName = br.ReadString();
|
||||
TypeId = br.ReadInt32();
|
||||
MinWidth = br.ReadInt32();
|
||||
MaxWidth = br.ReadInt32();
|
||||
MinHeight = br.ReadInt32();
|
||||
MaxHeight = br.ReadInt32();
|
||||
MinArea = br.ReadInt32();
|
||||
MaxArea = br.ReadInt32();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user