using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle; using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle; using Hawkeye.VisionBuilder.Workflow.Operations.Attributes; using OpenCvSharp; using System.Runtime.InteropServices; using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal; using Hawkeye.VisionBuilder.Workflow.DataTransfer; namespace Hawkeye.VisionBuilder.Workflow.Operations.AI; [Category("AI")] [IgnoreOperation] public class ArrayModelMatchingOperation:BaseOperation { private readonly WorkflowList _workflowList; private readonly CSharpDataTransferMQ _transfer; public Guid ReferenceId { get; set; } = new Guid(); public ArrayHorizontalElement SearchArea { get; set; } public string ModelName { get; set; } = "cls_candy5.h5"; public Action ExportImages { get; set; } public int Margin { get; set; } = 5; public int Amount { get; set; } = 4; private int _channels = 3; private List _lastCutImages; public ArrayModelMatchingOperation(WorkflowList workflowList) { _workflowList = workflowList; _transfer = PythonModelProxy.GetInterface(); SearchArea = new ArrayHorizontalElement() { Editable = true, Location = Vector2.One * 100, BlockSize = new Vector2(100, 100), }; ExportImages = () => { if (!Directory.Exists(Path.Combine(@"..\Data\Export", ModelName))) { Directory.CreateDirectory(Path.Combine(@"..\Data\Export", ModelName)); } // save last cut images var i = 0; foreach (var cutImage in _lastCutImages) { cutImage.SaveImage(Path.Combine(@"..\Data\Export", ModelName, $"{Guid.NewGuid().ToString()}.png")); } }; ReloadModelInfo(); } private void ReloadModelInfo() { var responseBytes = _transfer.TransferData(ModelName, 2, Array.Empty()); int[] sizes = new int[3]; Buffer.BlockCopy(responseBytes, 0, sizes, 0, 12); SearchArea.BlockSize = new Vector2(sizes[1], sizes[0]); _channels = sizes[2]; } byte[] MatToBytes(Mat mat) { if (mat.Channels() == 3) { mat = mat.CvtColor(_channels == 3 ? ColorConversionCodes.BGR2RGB : ColorConversionCodes.BGR2GRAY); } IntPtr dataPtr = mat.Data; // Calculate the size of the image data int dataSize = mat.Rows * mat.Cols * mat.ElemSize(); // Copy the image data into a byte array byte[] byteArray = new byte[dataSize]; Marshal.Copy(dataPtr, byteArray, 0, dataSize); return byteArray; } protected override void InterpretInternal(Context context) { Result = true; if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName))) { Status = $"Model `{ModelName}` not found"; Result = false; return; } if (!CheckImageExists(context)) return; switch (_channels) { case 3 when !CheckColorful(context): case 1 when !CheckGrayscale(context): return; } var refLocation = _workflowList.GetOriginById(ReferenceId).Origin.Location; SearchArea.MovePivot(refLocation); SearchArea.Margin=Margin; SearchArea.BlockCount=Amount; var img = context.ActiveImage.ImageData; _lastCutImages = SearchArea .GenerateLocations() .Select(x => img[(int) x.Y, (int) x.Y + (int) SearchArea.BlockSize.Y, (int) x.X, (int) x.X + (int) SearchArea.BlockSize.X]).ToList(); var images= _lastCutImages.Select(MatToBytes).ToList(); var bytes= images.SelectMany(x => x).ToArray(); var len = images.Count; var bytes2 = new byte[bytes.Length + 4]; bytes2[3] = (byte)(len >> 24); bytes2[2] = (byte)(len >> 16); bytes2[1] = (byte)(len >> 8); bytes2[0] = (byte)(len >> 0); Array.Copy(bytes, 0, bytes2, 4, bytes.Length); var answer = _transfer.TransferData(ModelName, 0,bytes2); float[] floatArray = new float[answer.Length / 4]; Buffer.BlockCopy(answer, 0, floatArray, 0, answer.Length); // Print answer for (int i = 0; i < Amount; i++) { Console.WriteLine(floatArray[i]); } for (int i = 0; i < Amount; i++) { SearchArea.IsGood[i] = floatArray[i] > 0.7; } context.GraphicsElements.Add(SearchArea); Result = SearchArea.IsGood.All(x=>x.Value==true); } public override void SetParameters(Dictionary parameters) { base.SetParameters(parameters); ReloadModelInfo(); SearchArea.SetPivot(_workflowList.GetOriginById(ReferenceId).Origin.Location); } public override void Save(BinaryWriter bw) { base.Save(bw); bw.Write(ReferenceId.ToString()); bw.Write(Margin); bw.Write(Amount); bw.Write(ModelName); SearchArea.Save(bw); } public override void Load(BinaryReader br) { base.Load(br); ReferenceId = new Guid(br.ReadString()); Margin = br.ReadInt32(); Amount = br.ReadInt32(); ModelName = br.ReadString(); SearchArea = new ArrayHorizontalElement(); SearchArea.Load(br); ReloadModelInfo(); } }