using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle; using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle; using Hawkeye.VisionBuilder.Workflow.Operations.Attributes; using OpenCvSharp; using System.Runtime.InteropServices; using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal; using Hawkeye.VisionBuilder.Workflow.DataTransfer; namespace Hawkeye.VisionBuilder.Workflow.Operations.AI; [Category("AI")] [IgnoreOperation] public class ArrayModelMatchingOperation:BaseOperation { private readonly WorkflowList _workflowList; private readonly CSharpDataTransferMQ _transfer; public Guid ReferenceId { get; set; } = new Guid(); public ArrayHorizontalElement SearchArea { get; set; } public string ModelName { get; set; } = "cls_candy5.h5"; public Action ExportImages { get; set; } public int Margin { get; set; } = 5; public int Amount { get; set; } = 4; private int _channels = 3; private List _lastCutImages; public ArrayModelMatchingOperation(WorkflowList workflowList) { _workflowList = workflowList; _transfer = PythonModelProxy.GetInterface(); SearchArea = new ArrayHorizontalElement() { Editable = true, Location = Vector2.One * 100, BlockSize = new Vector2(100, 100), }; ExportImages = () => { if (!Directory.Exists(Path.Combine(@"..\Data\Export", ModelName))) { Directory.CreateDirectory(Path.Combine(@"..\Data\Export", ModelName)); } // save last cut images var i = 0; foreach (var cutImage in _lastCutImages) { cutImage.SaveImage(Path.Combine(@"..\Data\Export", ModelName, $"{Guid.NewGuid().ToString()}.png")); } }; ReloadModelInfo(); } private void ReloadModelInfo() { var responseBytes = _transfer.TransferData(ModelName, 2, Array.Empty()); int[] sizes = new int[3]; Buffer.BlockCopy(responseBytes, 0, sizes, 0, 12); SearchArea.BlockSize = new Vector2(sizes[1], sizes[0]); _channels = sizes[2]; } byte[] MatToBytes(Mat mat) { if (mat.Channels() == 3) { mat = mat.CvtColor(_channels == 3 ? ColorConversionCodes.BGR2RGB : ColorConversionCodes.BGR2GRAY); } IntPtr dataPtr = mat.Data; // Calculate the size of the image data int dataSize = mat.Rows * mat.Cols * mat.ElemSize(); // Copy the image data into a byte array byte[] byteArray = new byte[dataSize]; Marshal.Copy(dataPtr, byteArray, 0, dataSize); return byteArray; } protected override void InterpretInternal(Context context) { Result = true; if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName))) { Status = $"Model `{ModelName}` not found"; Result = false; return; } if (!CheckImageExists(context)) return; switch (_channels) { case 3 when !CheckColorful(context): case 1 when !CheckGrayscale(context): return; } var refLocation = _workflowList.GetOriginById(ReferenceId).Origin.Location; SearchArea.MovePivot(refLocation); SearchArea.Margin=Margin; SearchArea.BlockCount=Amount; var img = context.ActiveImage.ImageData; _lastCutImages = SearchArea .GenerateLocations() .Select(x => img[(int) x.Y, (int) x.Y + (int) SearchArea.BlockSize.Y, (int) x.X, (int) x.X + (int) SearchArea.BlockSize.X]).ToList(); var images= _lastCutImages.Select(MatToBytes).ToList(); var bytes= images.SelectMany(x => x).ToArray(); var len = images.Count; var bytes2 = new byte[bytes.Length + 4]; bytes2[3] = (byte)(len >> 24); bytes2[2] = (byte)(len >> 16); bytes2[1] = (byte)(len >> 8); bytes2[0] = (byte)(len >> 0); Array.Copy(bytes, 0, bytes2, 4, bytes.Length); var answer = _transfer.TransferData(ModelName, 0,bytes2); float[] floatArray = new float[answer.Length / 4]; Buffer.BlockCopy(answer, 0, floatArray, 0, answer.Length); // Print answer for (int i = 0; i < Amount; i++) { Console.WriteLine(floatArray[i]); } for (int i = 0; i < Amount; i++) { SearchArea.IsGood[i] = floatArray[i] > 0.7; } context.GraphicsElements.Add(SearchArea); Result = SearchArea.IsGood.All(x=>x.Value==true); } public override void SetParameters(Dictionary parameters) { base.SetParameters(parameters); ReloadModelInfo(); SearchArea.SetPivot(_workflowList.GetOriginById(ReferenceId).Origin.Location); } public override void Save(BinaryWriter bw) { base.Save(bw); bw.Write(ReferenceId.ToString()); bw.Write(Margin); bw.Write(Amount); bw.Write(ModelName); SearchArea.Save(bw); } public override void Load(BinaryReader br) { base.Load(br); ReferenceId = new Guid(br.ReadString()); Margin = br.ReadInt32(); Amount = br.ReadInt32(); ModelName = br.ReadString(); SearchArea = new ArrayHorizontalElement(); SearchArea.Load(br); ReloadModelInfo(); } public override void Save(Dictionary dict) { base.Save(dict); dict[nameof(ReferenceId)] = ReferenceId.ToString(); dict[nameof(Margin)] = Margin; dict[nameof(Amount)] = Amount; dict[nameof(ModelName)] = ModelName; // Save ArrayHorizontalElement SearchArea properties manually dict["SearchArea_Editable"] = SearchArea.Editable; dict["SearchArea_LocationX"] = SearchArea.Location.X; dict["SearchArea_LocationY"] = SearchArea.Location.Y; dict["SearchArea_OffsetX"] = SearchArea.Offset.X; dict["SearchArea_OffsetY"] = SearchArea.Offset.Y; dict["SearchArea_BlockSizeX"] = SearchArea.BlockSize.X; dict["SearchArea_BlockSizeY"] = SearchArea.BlockSize.Y; dict["SearchArea_BlockCount"] = SearchArea.BlockCount; } public override void Load(Dictionary dict) { base.Load(dict); if (dict.ContainsKey(nameof(ReferenceId))) ReferenceId = new Guid(dict[nameof(ReferenceId)].ToString()); if (dict.ContainsKey(nameof(Margin))) Margin = Convert.ToInt32(dict[nameof(Margin)]); if (dict.ContainsKey(nameof(Amount))) Amount = Convert.ToInt32(dict[nameof(Amount)]); if (dict.ContainsKey(nameof(ModelName))) ModelName = dict[nameof(ModelName)].ToString(); // Load ArrayHorizontalElement SearchArea properties manually if (dict.ContainsKey("SearchArea_Editable") || dict.ContainsKey("SearchArea_LocationX") || dict.ContainsKey("SearchArea_BlockSizeX")) { SearchArea = new ArrayHorizontalElement(); if (dict.ContainsKey("SearchArea_Editable")) SearchArea.Editable = Convert.ToBoolean(dict["SearchArea_Editable"]); if (dict.ContainsKey("SearchArea_LocationX") && dict.ContainsKey("SearchArea_LocationY")) SearchArea.Location = new Vector2(Convert.ToSingle(dict["SearchArea_LocationX"]), Convert.ToSingle(dict["SearchArea_LocationY"])); if (dict.ContainsKey("SearchArea_OffsetX") && dict.ContainsKey("SearchArea_OffsetY")) SearchArea.Offset = new Vector2(Convert.ToSingle(dict["SearchArea_OffsetX"]), Convert.ToSingle(dict["SearchArea_OffsetY"])); if (dict.ContainsKey("SearchArea_BlockSizeX") && dict.ContainsKey("SearchArea_BlockSizeY")) SearchArea.BlockSize = new Vector2(Convert.ToSingle(dict["SearchArea_BlockSizeX"]), Convert.ToSingle(dict["SearchArea_BlockSizeY"])); if (dict.ContainsKey("SearchArea_BlockCount")) SearchArea.BlockCount = Convert.ToInt32(dict["SearchArea_BlockCount"]); } ReloadModelInfo(); } }