HawkeyeVision/Hawkeye.VisionBuilder.Workflow/Operations/AI/BackgroundSeparationModelOperation.cs

using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;
using OpenCvSharp;
using System.Drawing;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.Datatypes;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly;
using Hawkeye.VisionBuilder.Workflow.Links;


namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;

[Category("AI")]
[IgnoreOperation]
public class BackgroundSeparationModelOperation:BaseOperation,IHaveOrigin
{
    private readonly CSharpDataTransferMQ _transfer;
    private int _channels;
    public string ModelName { get; set; } = "background.h5";


    public BackgroundSeparationModelOperation()
    {
        _transfer = PythonModelProxy.GetInterface();
        
    }

    byte[] MatToBytes(Mat mat)
    {
        if (mat.Channels() == 3)
        {
            mat = mat.CvtColor(ColorConversionCodes.BGR2RGB);
        }


        IntPtr dataPtr = mat.Data;
        int dataSize = mat.Rows * mat.Cols * mat.ElemSize();
        byte[] byteArray = new byte[dataSize];

        Marshal.Copy(dataPtr, byteArray, 0, dataSize);

        return byteArray;
    }

    protected override void InterpretInternal(Context context)
    {
        Result = true;
        if (!CheckImageExists(context)) return;
        if (!CheckColorful(context)) return;
        if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName)))
        {
            Status= $"Model `{ModelName}` not found";
            Result = false;
            return;
        }
            

        var img = context.ActiveImage.ImageData;

        var bytes= MatToBytes(img);
        var responseBytes = _transfer.TransferData(ModelName, 3, bytes);
        var response = new Mat(img.Rows, img.Cols, MatType.CV_8UC1);
        response.SetArray(responseBytes);

        var contours = response.Threshold(128, 255, ThresholdTypes.Binary).FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple);

        if (contours.Length == 0)
        {
            Status = "No object found";
            Result = false;
            return;
        }

        // find biggest contour
        var max = 0;
        var maxIndex = 0;
        for (var i = 0; i < contours.Length; i++)
        {
            var area = Cv2.ContourArea(contours[i]);
            if (area > max)
            {
                max = (int)area;
                maxIndex = i;
            }
        }

        PolyElement poly = new PolyElement() { IsGood = true,Editable = false};

        poly.Points = contours[maxIndex].Select(x => new Vector2(x.X, x.Y)).ToArray();

        context.GraphicsElements.Add(poly);


        var rect = Cv2.MinAreaRect(contours[maxIndex]);
        var bound = rect.BoundingRect();
        Origin=new OriginElement()
        {
           Location = new Vector2(bound.Left, bound.Top),
        };

        Status= $"Object found at {Origin.Location}";
        Result = true;

    }

    public override void SetParameters(Dictionary<string, object> parameters)
    {
        base.SetParameters(parameters);
    }

    public override void Save(BinaryWriter bw)
    {
        base.Save(bw);
        bw.Write(ModelName);
    }

    public override void Load(BinaryReader br)
    {
        base.Load(br);
        ModelName = br.ReadString();
    }

    public override void Save(Dictionary<string, object> dict)
    {
        base.Save(dict);
        dict[nameof(ModelName)] = ModelName;
    }

    public override void Load(Dictionary<string, object> dict)
    {
        base.Load(dict);
        if (dict.ContainsKey(nameof(ModelName)))
            ModelName = dict[nameof(ModelName)].ToString();
    }

    public OriginElement Origin { get; set; }
}