using Hawkeye.VisionBuilder.Workflow.Datatypes; using Hawkeye.VisionBuilder.Workflow.Operations.Attributes; using OpenCvSharp; using System.Runtime.InteropServices; using Hawkeye.VisionBuilder.Workflow.DataTransfer; namespace Hawkeye.VisionBuilder.Workflow.Operations.AI; [Category("AI")] public class AnomalyAI: BaseOperation { private readonly string _modelName; public AnomalyAI() { _cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface(); CanHaveProcessingError = false; } private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC; private int _imageWidth; private int _imageHeight; private bool _initialized = false; private FilePath _modelFilePath = new FilePath(); public string MemorySlotName { get; set; } = "Anomaly"; public FilePath ModelFilePath { get => _modelFilePath; set { _modelFilePath = value; _initialized = false; } } protected override void InterpretInternal(Context context) { if (!CheckImageExists(context)) return; if (!CheckColorful(context)) return; if (!_initialized) { if (!File.Exists(ModelFilePath?.Path)) { this.SetError("Model file not found"); return; } _cSharpDataTransferMQRPC.TransferData(new MethodCall("load_model", ModelFilePath.Path, _modelName)); _initialized = true; } _cSharpDataTransferMQRPC.TransferData(new MethodCall("activate_model", _modelName)); var sizeEncoded = _cSharpDataTransferMQRPC.TransferData(new MethodCall("get_accept_size")); var size = sizeEncoded.Select(Convert.ToInt32).ToArray(); _imageWidth = size[2]; _imageHeight = size[1]; var outputSize = _cSharpDataTransferMQRPC.TransferData(new MethodCall("get_output_size")); var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray(); var currentImage = context.ActiveImage; var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB); var resized = rightColor.Resize(new Size(_imageWidth,_imageHeight)); int dataSize = resized.Rows * resized.Cols * resized.ElemSize(); var byteArray = new byte[dataSize]; Marshal.Copy(resized.Data, byteArray, 0, dataSize); var resultEncoded = _cSharpDataTransferMQRPC.TransferData(new MethodCall( "predict_anomaly", byteArray)); var result = resultEncoded.Select(Convert.ToByte).ToArray(); var defect = result.Take(1).ToArray()[0]; if (defect > 50) { byte[] resultColor = result.Skip(1).ToArray(); // black and white mask var mask = new Mat(30, 48, MatType.CV_8UC1, resultColor); var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height)); context.ActiveImage = new HawkeyeImage() {ImageData = maskResized}; } else { context.ActiveImage = new HawkeyeImage() { ImageData = new Mat(currentImage.ImageData.Size(), MatType.CV_8UC1, new Scalar(0)) }; } Result = true; } public override void Save(BinaryWriter bw) { base.Save(bw); bw.Write(_modelFilePath.Path); bw.Write(_modelFilePath.Format); } public override void Load(BinaryReader br) { base.Load(br); _modelFilePath.Path = br.ReadString(); _modelFilePath.Format = br.ReadString(); } }