HawkeyeVision/Hawkeye.VisionBuilder.Workflow/Operations/AI/ModelAIOperation.cs

using Hawkeye.VisionBuilder.Workflow.Datatypes;
using OpenCvSharp;
using System.Runtime.InteropServices;
using Hawkeye.VisionBuilder.Workflow.DataTransfer;
using Hawkeye.VisionBuilder.Workflow.Operations.Attributes;

namespace Hawkeye.VisionBuilder.Workflow.Operations.AI;

[Category("AI")]
public class ModelAIOperation: BaseOperation
{


    private readonly string _modelName;

    public ModelAIOperation()
    {
        
        _cSharpDataTransferMQRPC = PythonModelProxyRPC.GetInterface();
        CanHaveProcessingError = false;
    }

    private readonly CSharpDataTransferMQRPC _cSharpDataTransferMQRPC;
    private int _imageWidth;
    private int _imageHeight;



    public string FilterClasses { get; set; } = "1";

    private bool _initialized = false;
    private FilePath _modelFilePath=new FilePath();


    public FilePath ModelFilePath
    {
        get => _modelFilePath;
        set
        {
            _modelFilePath = value;
            _initialized = false;
        }
    }

    public bool IsCategorical { get; set; }

    protected override void InterpretInternal(Context context)
    {
        if (!CheckImageExists(context)) return;
        if (!CheckColorful(context)) return;

        if (!_initialized)
        {
            if (!File.Exists(ModelFilePath?.Path))
            {
                this.SetError("Model file not found");
                return;
            }

            // is relative path
            if (ModelFilePath.Path.StartsWith("."))
            {
                var dir = Directory.GetCurrentDirectory();
                var absPath = Path.GetFullPath(Path.Combine(dir, ModelFilePath.Path));
                ModelFilePath.Path = absPath;
            }

            _cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("load_model", ModelFilePath.Path, _modelName));
            _initialized = true;
        }



        _cSharpDataTransferMQRPC.TransferData<object>(new MethodCall("activate_model", _modelName));

        var sizeEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_accept_size"));
        var size = sizeEncoded.Select(Convert.ToInt32).ToArray();
        _imageWidth = size[1];
        _imageHeight = size[2];

        var outputSize = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall("get_output_size"));
        var outputSizeArray = outputSize.Select(Convert.ToInt32).ToArray();

        int classes;
        if (outputSizeArray.Length<3)
        {
            classes=1;
        }
        else
        {
            classes = outputSizeArray[2];
        }
        


        var currentImage = context.ActiveImage;

        var rightColor = currentImage.ImageData.CvtColor(ColorConversionCodes.BGR2RGB);
        var resized = rightColor.Resize(new Size(_imageHeight, _imageWidth));
        int dataSize = resized.Rows * resized.Cols * resized.ElemSize();
        var byteArray = new byte[dataSize];
        Marshal.Copy(resized.Data, byteArray, 0, dataSize);

        var allowed = FilterClasses.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => Convert.ToInt32(x)).ToArray();

        var allowedFlags = new byte[128];
        foreach (var i in allowed)
        {
            allowedFlags[i] = 1;
        }

        var resultEncoded = _cSharpDataTransferMQRPC.TransferData<object[]>(new MethodCall(IsCategorical ? "predict_raw" : "predict", byteArray));

        var result = resultEncoded.Select(Convert.ToByte).ToArray();
        // todo: copy and name RawModelAIOperation
        byte[] resultColor;
        if (IsCategorical)
        {
            resultColor = result.Chunk(classes).Select(x =>
            {
                return x.Select((x, i) => allowedFlags[i] == 1 ? (byte)(x) : (byte)0).Max();
            }).ToArray();
            
        }
        else
        {
            resultColor = result.Select(x => allowedFlags[x] == 1 ? (byte)(x * 255) : (byte)0).ToArray();
        }


        // black and white mask
        var mask = new Mat(outputSizeArray[0], outputSizeArray[1], MatType.CV_8UC1);
        mask.SetArray(resultColor);

        var maskResized = mask.Resize(new Size(currentImage.ImageData.Width, currentImage.ImageData.Height));

        context.ActiveImage = new HawkeyeImage() { ImageData = maskResized };


        Result = true;
    }

    public override void Save(BinaryWriter bw)
    {
        base.Save(bw);
        bw.Write(_modelFilePath.Path);
        bw.Write(_modelFilePath.Format);
        bw.Write(FilterClasses);
        bw.Write(IsCategorical);
    }

    public override void Load(BinaryReader br)
    {
        base.Load(br);
        _modelFilePath.Path = br.ReadString();
        _modelFilePath.Format = br.ReadString();
        FilterClasses = br.ReadString();
        IsCategorical = br.ReadBoolean();
    }

    public override void Save(Dictionary<string, object> dict)
    {
        base.Save(dict);
        dict[nameof(ModelFilePath)] = ModelFilePath.Path;
        dict[nameof(FilterClasses)] = FilterClasses;
        dict[nameof(IsCategorical)] = IsCategorical;
    }

    public override void Load(Dictionary<string, object> dict)
    {
        base.Load(dict);
        if (dict.ContainsKey(nameof(ModelFilePath)))
            ModelFilePath.Path = dict[nameof(ModelFilePath)].ToString();
        if (dict.ContainsKey(nameof(FilterClasses)))
            FilterClasses = dict[nameof(FilterClasses)].ToString();
        if (dict.ContainsKey(nameof(IsCategorical)))
            IsCategorical = Convert.ToBoolean(dict[nameof(IsCategorical)]);
    }
}