check point

Hawkeye.VisionBuilder.Workflow/Operations/AI/ArrayModelMatchingOperation.cs

@@ -0,0 +1,184 @@
+using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle;
+using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
+using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
+
+using OpenCvSharp;
+using System.Runtime.InteropServices;
+using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
+using Hawkeye.VisionBuilder.Workflow.DataTransfer;
+
+namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;
+
+[Category("AI")]
+[IgnoreOperation]
+public class ArrayModelMatchingOperation:BaseOperation
+{
+    private readonly WorkflowList _workflowList;
+    private readonly CSharpDataTransferMQ _transfer;
+    public Guid ReferenceId { get; set; } = new Guid();
+    public ArrayHorizontalElement SearchArea { get; set; }
+    public string ModelName { get; set; } = "cls_candy5.h5";
+    public Action ExportImages { get; set; }
+    public int Margin { get; set; } = 5;
+    public int Amount { get; set; } = 4;
+
+    private int _channels = 3;
+    private List<Mat> _lastCutImages;
+
+    public ArrayModelMatchingOperation(WorkflowList workflowList)
+    {
+        _workflowList = workflowList;
+        _transfer = PythonModelProxy.GetInterface();
+        
+
+        SearchArea = new ArrayHorizontalElement()
+        {
+            Editable = true,
+            Location = Vector2.One * 100,
+            BlockSize = new Vector2(100, 100),
+            
+        };
+        ExportImages = () =>
+        {
+            if (!Directory.Exists(Path.Combine(@"..\Data\Export", ModelName)))
+            {
+                Directory.CreateDirectory(Path.Combine(@"..\Data\Export", ModelName));
+            }
+
+            // save last cut images
+            var i = 0;
+            foreach (var cutImage in _lastCutImages)
+            {
+                cutImage.SaveImage(Path.Combine(@"..\Data\Export", ModelName, $"{Guid.NewGuid().ToString()}.png"));
+            }
+            
+        };
+        ReloadModelInfo();
+    }
+
+    private void ReloadModelInfo()
+    {
+        var responseBytes = _transfer.TransferData(ModelName, 2, Array.Empty<byte>());
+        int[] sizes = new int[3];
+        Buffer.BlockCopy(responseBytes, 0, sizes, 0, 12);
+        SearchArea.BlockSize = new Vector2(sizes[1], sizes[0]);
+        _channels = sizes[2];
+    }
+
+    byte[] MatToBytes(Mat mat)
+    {
+        if (mat.Channels() == 3)
+        {
+            mat = mat.CvtColor(_channels == 3 ? ColorConversionCodes.BGR2RGB : ColorConversionCodes.BGR2GRAY);
+        }
+        
+
+        IntPtr dataPtr = mat.Data;
+
+        // Calculate the size of the image data
+        int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
+
+
+
+
+        // Copy the image data into a byte array
+        byte[] byteArray = new byte[dataSize];
+
+        Marshal.Copy(dataPtr, byteArray, 0, dataSize);
+
+        return byteArray;
+    }
+
+    protected override void InterpretInternal(Context context)
+    {
+        Result = true;
+        if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
+        {
+            Status = $"Model `{ModelName}` not found";
+            Result = false;
+            return;
+        }
+        if (!CheckImageExists(context)) return;
+        switch (_channels)
+        {
+            case 3 when !CheckColorful(context):
+            case 1 when !CheckGrayscale(context):
+                return;
+        }
+
+        var refLocation = _workflowList.GetOriginById(ReferenceId).Origin.Location;
+        SearchArea.MovePivot(refLocation);
+
+        SearchArea.Margin=Margin;
+        SearchArea.BlockCount=Amount;
+        var img = context.ActiveImage.ImageData;
+
+        _lastCutImages = SearchArea
+            .GenerateLocations()
+            .Select(x => img[(int) x.Y, (int) x.Y + (int) SearchArea.BlockSize.Y, (int) x.X,
+                (int) x.X + (int) SearchArea.BlockSize.X]).ToList();
+        var images= _lastCutImages.Select(MatToBytes).ToList();
+
+
+        var bytes= images.SelectMany(x => x).ToArray();
+        
+        var len = images.Count;
+        var bytes2 = new byte[bytes.Length + 4];
+        bytes2[3] = (byte)(len >> 24);
+        bytes2[2] = (byte)(len >> 16);
+        bytes2[1] = (byte)(len >> 8);
+        bytes2[0] = (byte)(len >> 0);
+
+        Array.Copy(bytes, 0, bytes2, 4, bytes.Length);
+
+        var answer = _transfer.TransferData(ModelName, 0,bytes2);
+        float[] floatArray = new float[answer.Length / 4];
+        Buffer.BlockCopy(answer, 0, floatArray, 0, answer.Length);
+
+        // Print answer
+        for (int i = 0; i < Amount; i++)
+        {
+            Console.WriteLine(floatArray[i]);
+        }
+
+        for (int i = 0; i < Amount; i++)
+        {
+            SearchArea.IsGood[i] = floatArray[i] > 0.7;
+        }
+
+
+
+        context.GraphicsElements.Add(SearchArea);
+        Result = SearchArea.IsGood.All(x=>x.Value==true);
+    }
+
+    public override void SetParameters(Dictionary<string, object> parameters)
+    {
+        base.SetParameters(parameters);
+        ReloadModelInfo();
+        SearchArea.SetPivot(_workflowList.GetOriginById(ReferenceId).Origin.Location);
+        
+    }
+
+    public override void Save(BinaryWriter bw)
+    {
+        base.Save(bw);
+        bw.Write(ReferenceId.ToString());
+        bw.Write(Margin);
+        bw.Write(Amount);
+        bw.Write(ModelName);
+        SearchArea.Save(bw);
+    }
+
+    public override void Load(BinaryReader br)
+    {
+        base.Load(br);
+        ReferenceId = new Guid(br.ReadString());
+        Margin = br.ReadInt32();
+        Amount = br.ReadInt32();
+        ModelName = br.ReadString();
+        SearchArea = new ArrayHorizontalElement();
+        SearchArea.Load(br);
+        ReloadModelInfo();
+    }
+}