HawkeyeVision/Hawkeye.VisionBuilder.Workflow/Operations/AI/ArrayModelMatchingOperation.cs

using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.FixedRectangle;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Rectangle;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;

using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.ArrayHorizontal;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;

namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;

[Category("AI")]
[IgnoreOperation]
public class ArrayModelMatchingOperation:BaseOperation
{
    private readonly WorkflowList _workflowList;
    private readonly CSharpDataTransferMQ _transfer;
    public Guid ReferenceId { get; set; } = new Guid();
    public ArrayHorizontalElement SearchArea { get; set; }
    public string ModelName { get; set; } = "cls_candy5.h5";
    public Action ExportImages { get; set; }
    public int Margin { get; set; } = 5;
    public int Amount { get; set; } = 4;

    private int _channels = 3;
    private List<Mat> _lastCutImages;

    public ArrayModelMatchingOperation(WorkflowList workflowList)
    {
        _workflowList = workflowList;
        _transfer = PythonModelProxy.GetInterface();
        

        SearchArea = new ArrayHorizontalElement()
        {
            Editable = true,
            Location = Vector2.One * 100,
            BlockSize = new Vector2(100, 100),
            
        };
        ExportImages = () =>
        {
            if (!Directory.Exists(Path.Combine(@"..\Data\Export", ModelName)))
            {
                Directory.CreateDirectory(Path.Combine(@"..\Data\Export", ModelName));
            }

            // save last cut images
            var i = 0;
            foreach (var cutImage in _lastCutImages)
            {
                cutImage.SaveImage(Path.Combine(@"..\Data\Export", ModelName, $"{Guid.NewGuid().ToString()}.png"));
            }
            
        };
        ReloadModelInfo();
    }

    private void ReloadModelInfo()
    {
        var responseBytes = _transfer.TransferData(ModelName, 2, Array.Empty<byte>());
        int[] sizes = new int[3];
        Buffer.BlockCopy(responseBytes, 0, sizes, 0, 12);
        SearchArea.BlockSize = new Vector2(sizes[1], sizes[0]);
        _channels = sizes[2];
    }

    byte[] MatToBytes(Mat mat)
    {
        if (mat.Channels() == 3)
        {
            mat = mat.CvtColor(_channels == 3 ? ColorConversionCodes.BGR2RGB : ColorConversionCodes.BGR2GRAY);
        }
        

        IntPtr dataPtr = mat.Data;

        // Calculate the size of the image data
        int dataSize = mat.Rows * mat.Cols * mat.ElemSize();




        // Copy the image data into a byte array
        byte[] byteArray = new byte[dataSize];

        Marshal.Copy(dataPtr, byteArray, 0, dataSize);

        return byteArray;
    }

    protected override void InterpretInternal(Context context)
    {
        Result = true;
        if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
        {
            Status = $"Model `{ModelName}` not found";
            Result = false;
            return;
        }
        if (!CheckImageExists(context)) return;
        switch (_channels)
        {
            case 3 when !CheckColorful(context):
            case 1 when !CheckGrayscale(context):
                return;
        }

        var refLocation = _workflowList.GetOriginById(ReferenceId).Origin.Location;
        SearchArea.MovePivot(refLocation);

        SearchArea.Margin=Margin;
        SearchArea.BlockCount=Amount;
        var img = context.ActiveImage.ImageData;

        _lastCutImages = SearchArea
            .GenerateLocations()
            .Select(x => img[(int) x.Y, (int) x.Y + (int) SearchArea.BlockSize.Y, (int) x.X,
                (int) x.X + (int) SearchArea.BlockSize.X]).ToList();
        var images= _lastCutImages.Select(MatToBytes).ToList();


        var bytes= images.SelectMany(x => x).ToArray();
        
        var len = images.Count;
        var bytes2 = new byte[bytes.Length + 4];
        bytes2[3] = (byte)(len >> 24);
        bytes2[2] = (byte)(len >> 16);
        bytes2[1] = (byte)(len >> 8);
        bytes2[0] = (byte)(len >> 0);

        Array.Copy(bytes, 0, bytes2, 4, bytes.Length);

        var answer = _transfer.TransferData(ModelName, 0,bytes2);
        float[] floatArray = new float[answer.Length / 4];
        Buffer.BlockCopy(answer, 0, floatArray, 0, answer.Length);

        // Print answer
        for (int i = 0; i < Amount; i++)
        {
            Console.WriteLine(floatArray[i]);
        }

        for (int i = 0; i < Amount; i++)
        {
            SearchArea.IsGood[i] = floatArray[i] > 0.7;
        }



        context.GraphicsElements.Add(SearchArea);
        Result = SearchArea.IsGood.All(x=>x.Value==true);
    }

    public override void SetParameters(Dictionary<string, object> parameters)
    {
        base.SetParameters(parameters);
        ReloadModelInfo();
        SearchArea.SetPivot(_workflowList.GetOriginById(ReferenceId).Origin.Location);
        
    }

    public override void Save(BinaryWriter bw)
    {
        base.Save(bw);
        bw.Write(ReferenceId.ToString());
        bw.Write(Margin);
        bw.Write(Amount);
        bw.Write(ModelName);
        SearchArea.Save(bw);
    }

    public override void Load(BinaryReader br)
    {
        base.Load(br);
        ReferenceId = new Guid(br.ReadString());
        Margin = br.ReadInt32();
        Amount = br.ReadInt32();
        ModelName = br.ReadString();
        SearchArea = new ArrayHorizontalElement();
        SearchArea.Load(br);
        ReloadModelInfo();
    }

    public override void Save(Dictionary<string, object> dict)
    {
        base.Save(dict);
        dict[nameof(ReferenceId)] = ReferenceId.ToString();
        dict[nameof(Margin)] = Margin;
        dict[nameof(Amount)] = Amount;
        dict[nameof(ModelName)] = ModelName;
        
        // Save ArrayHorizontalElement SearchArea properties manually
        dict["SearchArea_Editable"] = SearchArea.Editable;
        dict["SearchArea_LocationX"] = SearchArea.Location.X;
        dict["SearchArea_LocationY"] = SearchArea.Location.Y;
        dict["SearchArea_OffsetX"] = SearchArea.Offset.X;
        dict["SearchArea_OffsetY"] = SearchArea.Offset.Y;
        dict["SearchArea_BlockSizeX"] = SearchArea.BlockSize.X;
        dict["SearchArea_BlockSizeY"] = SearchArea.BlockSize.Y;
        dict["SearchArea_BlockCount"] = SearchArea.BlockCount;
    }

    public override void Load(Dictionary<string, object> dict)
    {
        base.Load(dict);
        if (dict.ContainsKey(nameof(ReferenceId)))
            ReferenceId = new Guid(dict[nameof(ReferenceId)].ToString());
        if (dict.ContainsKey(nameof(Margin)))
            Margin = Convert.ToInt32(dict[nameof(Margin)]);
        if (dict.ContainsKey(nameof(Amount)))
            Amount = Convert.ToInt32(dict[nameof(Amount)]);
        if (dict.ContainsKey(nameof(ModelName)))
            ModelName = dict[nameof(ModelName)].ToString();
        
        // Load ArrayHorizontalElement SearchArea properties manually
        if (dict.ContainsKey("SearchArea_Editable") || dict.ContainsKey("SearchArea_LocationX") || dict.ContainsKey("SearchArea_BlockSizeX"))
        {
            SearchArea = new ArrayHorizontalElement();
            if (dict.ContainsKey("SearchArea_Editable"))
                SearchArea.Editable = Convert.ToBoolean(dict["SearchArea_Editable"]);
            if (dict.ContainsKey("SearchArea_LocationX") && dict.ContainsKey("SearchArea_LocationY"))
                SearchArea.Location = new Vector2(Convert.ToSingle(dict["SearchArea_LocationX"]), Convert.ToSingle(dict["SearchArea_LocationY"]));
            if (dict.ContainsKey("SearchArea_OffsetX") && dict.ContainsKey("SearchArea_OffsetY"))
                SearchArea.Offset = new Vector2(Convert.ToSingle(dict["SearchArea_OffsetX"]), Convert.ToSingle(dict["SearchArea_OffsetY"]));
            if (dict.ContainsKey("SearchArea_BlockSizeX") && dict.ContainsKey("SearchArea_BlockSizeY"))
                SearchArea.BlockSize = new Vector2(Convert.ToSingle(dict["SearchArea_BlockSizeX"]), Convert.ToSingle(dict["SearchArea_BlockSizeY"]));
            if (dict.ContainsKey("SearchArea_BlockCount"))
                SearchArea.BlockCount = Convert.ToInt32(dict["SearchArea_BlockCount"]);
        }
        
        ReloadModelInfo();
    }
}