using Hawkeye.VisionBuilder.Workflow.Operations.Attributes; using OpenCvSharp; using System.Drawing; using System.Runtime.InteropServices; using Hawkeye.VisionBuilder.Workflow.Datatypes; using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Origin; using Hawkeye.VisionBuilder.Workflow.DataTransfer; using Hawkeye.VisionBuilder.Workflow.Datatypes.Elements.Poly; using Hawkeye.VisionBuilder.Workflow.Links; namespace Hawkeye.VisionBuilder.Workflow.Operations.AI; [Category("AI")] [IgnoreOperation] public class BackgroundSeparationModelOperation:BaseOperation,IHaveOrigin { private readonly CSharpDataTransferMQ _transfer; private int _channels; public string ModelName { get; set; } = "background.h5"; public BackgroundSeparationModelOperation() { _transfer = PythonModelProxy.GetInterface(); } byte[] MatToBytes(Mat mat) { if (mat.Channels() == 3) { mat = mat.CvtColor(ColorConversionCodes.BGR2RGB); } IntPtr dataPtr = mat.Data; int dataSize = mat.Rows * mat.Cols * mat.ElemSize(); byte[] byteArray = new byte[dataSize]; Marshal.Copy(dataPtr, byteArray, 0, dataSize); return byteArray; } protected override void InterpretInternal(Context context) { Result = true; if (!CheckImageExists(context)) return; if (!CheckColorful(context)) return; if (!File.Exists(Path.Combine(@"..\Data\AI", ModelName))) { Status= $"Model `{ModelName}` not found"; Result = false; return; } var img = context.ActiveImage.ImageData; var bytes= MatToBytes(img); var responseBytes = _transfer.TransferData(ModelName, 3, bytes); var response = new Mat(img.Rows, img.Cols, MatType.CV_8UC1); response.SetArray(responseBytes); var contours = response.Threshold(128, 255, ThresholdTypes.Binary).FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxSimple); if (contours.Length == 0) { Status = "No object found"; Result = false; return; } // find biggest contour var max = 0; var maxIndex = 0; for (var i = 0; i < contours.Length; i++) { var area = Cv2.ContourArea(contours[i]); if (area > max) { max = (int)area; maxIndex = i; } } PolyElement poly = new PolyElement() { IsGood = true,Editable = false}; poly.Points = contours[maxIndex].Select(x => new Vector2(x.X, x.Y)).ToArray(); context.GraphicsElements.Add(poly); var rect = Cv2.MinAreaRect(contours[maxIndex]); var bound = rect.BoundingRect(); Origin=new OriginElement() { Location = new Vector2(bound.Left, bound.Top), }; Status= $"Object found at {Origin.Location}"; Result = true; } public override void SetParameters(Dictionary parameters) { base.SetParameters(parameters); } public override void Save(BinaryWriter bw) { base.Save(bw); bw.Write(ModelName); } public override void Load(BinaryReader br) { base.Load(br); ModelName = br.ReadString(); } public override void Save(Dictionary dict) { base.Save(dict); dict[nameof(ModelName)] = ModelName; } public override void Load(Dictionary dict) { base.Load(dict); if (dict.ContainsKey(nameof(ModelName))) ModelName = dict[nameof(ModelName)].ToString(); } public OriginElement Origin { get; set; } }