using Hawkeye.VisionBuilder.Workflow.Datatypes; using OpenCvSharp; using System.Runtime.InteropServices; using Hawkeye.VisionBuilder.Workflow.Operations.Attributes; using Hawkeye.VisionBuilder.Workflow.DataTransfer; namespace Hawkeye.VisionBuilder.Workflow.Operations.AI; [Category("AI")] public class RawModelAIOperation: BaseOperation { private string _modelName; public RawModelAIOperation() { _cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface(); CanHaveProcessingError = false; } private CSharpDataTransferMQRPC _cSharpDataTransferMQRPC; private int _imageWidth; private int _imageHeight; public string FilterClasses { get; set; } = "1"; private bool _initialized = false; private FilePath _modelFilePath=new FilePath(); public FilePath ModelFilePath { get => _modelFilePath; set { _modelFilePath = value; _initialized = false; } } public bool IsCategorical { get; set; } protected override void InterpretInternal(Context context) { if (!CheckImageExists(context)) return; if (!CheckColorful(context)) return; if (!_initialized) { if (!File.Exists(ModelFilePath?.Path)) { this.SetError("Model file not found"); return; } _modelName = Path.GetFileNameWithoutExtension(ModelFilePath.Path); // is relative path if (ModelFilePath.Path.StartsWith(".")) { var dir = Directory.GetCurrentDirectory(); var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path)); ModelFilePath.Path = absPath; } _cSharpDataTransferMQRPC.TransferData(new MethodCall("load_model", ModelFilePath.Path, _modelName)); _initialized = true; } _cSharpDataTransferMQRPC.TransferData(new MethodCall("activate_model", _modelName)); var sizeEncoded = _cSharpDataTransferMQRPC.TransferData(new MethodCall("get_accept_size")); var size = sizeEncoded.Select(Convert.ToInt32).ToArray(); _imageWidth = size[1]; _imageHeight = size[2]; var outputSize = _cSharpDataTransferMQRPC.TransferData(new MethodCall("get_output_size")); var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray(); int classes; if (outputSizeArray.Length<3) { classes=1; } else { classes = outputSizeArray[2]; } var currentImage = context.ActiveImage; var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB); var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth)); int dataSize = resized.Rows * resized.Cols * resized.ElemSize(); var byteArray = new byte[dataSize]; Marshal.Copy(resized.Data, byteArray, 0, dataSize); var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray(); var allowedFlags = new byte[128]; foreach (var i in allowed) { allowedFlags[i] = 1; } var resultEncoded = _cSharpDataTransferMQRPC.TransferData(new MethodCall("predict_raw", byteArray)); var result = resultEncoded.Select(Convert.ToByte).ToArray(); byte[] resultColor; resultColor = result.Select(x => (byte)(x) ).ToArray(); // black and white mask var mask = new Mat(outputSizeArray[0], outputSizeArray[1], MatType.CV_8UC1, resultColor); var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height)); context.ActiveImage = new HawkeyeImage() { ImageData = maskResized }; Result = true; } public override void Save(BinaryWriter bw) { base.Save(bw); bw.Write(_modelFilePath.Path); bw.Write(_modelFilePath.Format); bw.Write(FilterClasses); bw.Write(IsCategorical); } public override void Load(BinaryReader br) { base.Load(br); _modelFilePath.Path = br.ReadString(); _modelFilePath.Format = br.ReadString(); FilterClasses = br.ReadString(); IsCategorical = br.ReadBoolean(); } }