using CandyboxPlugin.Detectors; using CandyboxPlugin.Geometry; using CandyboxPlugin.Voronoi; using OpenCvSharp; using System.Diagnostics; using System.Drawing.Imaging; using System.Text.Json; using Lindt.Candybox.Demo; using VisionBuilder.UI.Common.NullClasses; using Point = System.Drawing.Point; using Size = OpenCvSharp.Size; namespace CandyboxPlugin.Recipe { //todo: update camera settings according to learned parameters public class HistoRecipe { private readonly string _recipeName; private CandyDetector _learner; private BlisterDetector _blister; private DefaultRectangle2Algorithm _rectangle2; private LearningParameters _learnedParameters; private ColorHeatMap _colorHeatMap; private Size AnalysisSize { get; }= new Size(512, 512); Size ThumbnailSize { get; } = new Size(256, 200); public const string RECIPES_DIRECTORY = "..\\Data\\Recipes"; public HistoRecipe(string recipeName) { _recipeName = recipeName; _configurationPath = Path.GetFullPath(Path.Combine(RECIPES_DIRECTORY, _recipeName + ".json")); _learner = new CandyDetector(); _blister = new BlisterDetector(); _rectangle2 = new DefaultRectangle2Algorithm(); _learnedParameters = new LearningParameters(); _colorHeatMap = new ColorHeatMap(255); Reload(); } public void Reload() { lock (_lockObject) { if (File.Exists(_configurationPath)) { _learnedParameters = JsonSerializer.Deserialize(File.ReadAllText(_configurationPath)); } } } public void LearnContours(OpenCvSharp.Point[][] contours, Mat image, List cuts, bool relearn) { lock (_lockObject) { _learnedParameters = relearn ? new LearningParameters() : _learnedParameters; // Convert cuts to be relative to blister center _learnedParameters.Cuts = cuts; if (_learnedParameters.CandyParameters.Count == 0)//initial learning { _learnedParameters.BlisterAngle = _blisterAngle; _learnedParameters.BlisterPosition = _blisterCenter; _learnedParameters.ImageWidth = (int)(_blisterEdge / 512f * 1800f); for (int i = 0; i < contours.Length; i++) { var hist = CalcHist(contours, i, _resizedMat, out var outerRadius, out var innerRadius); var moments = Cv2.Moments(contours[i]); var cX = (int)(moments.M10 / moments.M00); var cY = (int)(moments.M01 / moments.M00); hist.GetArray(out float[] histData); var p = new CandyParameter() { Position = new Point(cX, cY), Histogram = new Queue(), OuterRadius = (int)outerRadius, InnerRadius = (int)innerRadius }; p.Histogram.Enqueue(histData); _learnedParameters.CandyParameters.Add(p); } } else { foreach (CandyParameter parameter in _learnedParameters.CandyParameters) { parameter.ResetPoint(); parameter.RotatePoint(Point.Empty, _blisterAngle - _learnedParameters.BlisterAngle); } for (int i = 0; i < contours.Length; i++) { var hist = CalcHist(contours, i, _resizedMat, out var outerRadius, out var innerRadius); var moments = Cv2.Moments(contours[i]); var cX = (int)(moments.M10 / moments.M00); var cY = (int)(moments.M01 / moments.M00); hist.GetArray(out float[] histData); var point = ClosestPoint(_blisterCenter, cX, cY, (int)outerRadius, (int)innerRadius); point.Histogram.Enqueue(histData); if (point.Histogram.Count > _learnedParameters.Configuration.MemoryBuffer) { point.Histogram.Dequeue(); } } foreach (CandyParameter parameter in _learnedParameters.CandyParameters) { parameter.ResetPoint(); } } File.WriteAllText(_configurationPath, JsonSerializer.Serialize(_learnedParameters, new JsonSerializerOptions() { WriteIndented = true })); var fullSamplePath = Path.GetFullPath(Path.Combine("..\\Data\\Samples", _recipeName + ".bmp")); Directory.CreateDirectory(Path.GetDirectoryName(fullSamplePath)); image.Resize(ThumbnailSize).SaveImage(fullSamplePath); } } public HistoRecipe Clone() { var clone = new HistoRecipe(_recipeName); return clone; } public Point BlisterCenter => _blisterCenter; public ProcessingResult ProcessImage(Mat image) { lock (_lockObject) { var sw = Stopwatch.StartNew(); try { var framedImage = new Mat(new Size(1800, 1408),MatType.CV_8UC3,Scalar.FromRgb(36, 67, 182)); // draw the image in top left corner of framed image // this does not actually create new Mat. it just creates a header for the region of interest image.CopyTo(framedImage[0, image.Height, 0, image.Width]); // add alpha channel var rgb = framedImage; framedImage = framedImage.CvtColor(ColorConversionCodes.BGR2BGRA); var resized = rgb.Resize(AnalysisSize); _resizedMat = resized; var sw1 = Stopwatch.StartNew(); Mat res = _learner.Eval(resized); Mat resBlister = _blister.Eval(resized); var element = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(50, 50)); Mat blisterOpened = resBlister.Threshold(128,255,ThresholdTypes.Binary).MorphologyEx(MorphTypes.Open,element); resBlister.Dispose(); resBlister = blisterOpened; sw1.Stop(); Console.WriteLine("AI Time:" + sw1.ElapsedMilliseconds); Mat inverted = null; Mat overlayMat = new Mat(res.Width, res.Height, MatType.CV_8UC4, new Scalar(0, 0, 0, 0)); bool wasError = false; List errorPoints = new List(); OpenCvSharp.Point[] blisterRectangle; Mat numbersOverlay = new Mat(512, 512, MatType.CV_8UC4, new Scalar(0, 0, 0, 0)); List errorReason = new List(); using (var tracker = new ResourcesTracker()) { blisterRectangle = FindBlister(resBlister, out _blisterAngle, out _blisterCenter, out _blisterEdge); if (_learnedParameters.CandyParameters.Count > 0) { DelaunayTriangulator triangulator = new DelaunayTriangulator(); triangulator.GenerateBorder(512, 512); var points=_learnedParameters.CandyParameters .Select((x, i) => new VoronoiPoint( x.Position.X + _blisterCenter.X - _learnedParameters.BlisterPosition.X, x.Position.Y + _blisterCenter.Y - _learnedParameters.BlisterPosition.Y )) .ToList(); var triangles = triangulator.BowyerWatson(_learnedParameters.CandyParameters .Select((x,i) => new VoronoiPoint( x.Position.X + _blisterCenter.X - _learnedParameters.BlisterPosition.X, x.Position.Y + _blisterCenter.Y - _learnedParameters.BlisterPosition.Y )) .ToList()); var edges = Voronoi.Voronoi.GenerateEdgesFromDelaunay(triangles); foreach (Edge edge in edges) { Cv2.Line( res, new OpenCvSharp.Point(edge.Point1.X, edge.Point1.Y), new OpenCvSharp.Point(edge.Point2.X, edge.Point2.Y), new Scalar(0, 0, 0), // Black color in BGR 3 // Thickness ); } } foreach (VectorLine cut in _learnedParameters.Cuts) { // Convert from relative to absolute coordinates var absoluteStart = new OpenCvSharp.Point( cut.StartPoint.X + _blisterCenter.X, cut.StartPoint.Y + _blisterCenter.Y ); var absoluteEnd = new OpenCvSharp.Point( cut.EndPoint.X + _blisterCenter.X, cut.EndPoint.Y + _blisterCenter.Y ); Cv2.Line(res, absoluteStart, absoluteEnd, new Scalar(0, 0, 0), 3); } var mat = res; Cv2.Threshold(mat, mat, 128, 255, ThresholdTypes.Binary); OpenCvSharp.Point[][] contours; contours = mat.FindContoursAsArray(RetrievalModes.External, ContourApproximationModes.ApproxNone); contours = contours.Where(x => Cv2.ContourArea(x) > 500).ToArray(); var sw2 = Stopwatch.StartNew(); LastContours = contours; int c = 0; foreach (CandyParameter parameter in _learnedParameters.CandyParameters) { parameter.ResetPoint(); parameter.RotatePoint(Point.Empty, _blisterAngle - _learnedParameters.BlisterAngle); var coordX = parameter.Position.X + (_blisterCenter.X - _learnedParameters.BlisterPosition.X); var coordY = parameter.Position.Y + (_blisterCenter.Y - _learnedParameters.BlisterPosition.Y); // Draw filled white circle (ellipse) Cv2.Circle(numbersOverlay, new OpenCvSharp.Point(coordX, coordY), 10, new Scalar(255, 255, 255, 255), thickness: -1); // Draw black ellipse outline Cv2.Ellipse(numbersOverlay, new OpenCvSharp.Point(coordX, coordY), new Size(10, 10), 0, 0, 360, new Scalar(0, 0, 0, 255), thickness: 1); // Draw the number as text (centered, adjust offset as needed) Cv2.PutText( numbersOverlay, c.ToString(), new OpenCvSharp.Point(coordX - 7, coordY + 7), // Y offset for baseline alignment HersheyFonts.HersheySimplex, 0.4, // Font scale new Scalar(0,0,0,255), 1, LineTypes.AntiAlias ); bool found = false; for (int i = 0; i < contours.Length; i++) { if (Cv2.PointPolygonTest(contours[i], new Point2f(coordX, coordY), false) > 0) { var hist = CalcHist(contours, i, _resizedMat, out var outerRadius, out var innerRadius); var correl = 0; foreach (float[] h in parameter.Histogram) { Mat histC = new Mat(new Size(1, 125), MatType.CV_32FC1); histC.SetArray(h); var correl_0 = Cv2.CompareHist(histC, hist, HistCompMethods.Correl) * 100; if (correl_0 > correl) { correl = (int) correl_0; } } if (_learnedParameters.Configuration.IgnoreCorrelation.Contains(c)) correl = 100; if (correl > _learnedParameters.Configuration.SimilarityTolerance) { var min = _learnedParameters.Configuration.SimilarityTolerance; var max = 100; var value = correl; value = value < min ? min : value; var value2 = (int) Map(value, min, max, 0, 255); var color = _colorHeatMap.GetColorForValue(value2, 255); Cv2.DrawContours(overlayMat, contours, i, new Scalar(color.B, color.G, color.R, color.A)); found = true; } else { Cv2.DrawContours(overlayMat, contours, i, OpaqueScalar(Scalar.Red)); Console.WriteLine($"{c} wrong type. similarity is {correl}%"); errorReason.Add(EErrorReason.WrongType); ErrorTextHelper.DrawErrorText(overlayMat, new OpenCvSharp.Point(coordX, coordY), EErrorReason.WrongType); errorPoints.Add(c); found = true; wasError = true; break; } var outerDiff = parameter.OuterRadius - outerRadius; if (Math.Abs(outerDiff / parameter.OuterRadius) * 100 > _learnedParameters.Configuration.OuterTolerance && outerDiff < 0) { Cv2.DrawContours(overlayMat, contours, i, OpaqueScalar(Scalar.Red)); errorReason.Add(EErrorReason.WrongSize); ErrorTextHelper.DrawErrorText(overlayMat, new OpenCvSharp.Point(coordX, coordY), EErrorReason.WrongSize); Console.WriteLine( $"{c} wrong outer size. expected {parameter.OuterRadius}, got {outerRadius} ({Math.Abs(outerDiff / parameter.OuterRadius) * 100f})"); errorPoints.Add(c); wasError = true; break; } var innerDiff = parameter.InnerRadius - innerRadius; if (Math.Abs(innerDiff / parameter.InnerRadius) * 100 > _learnedParameters.Configuration.InnerTolerance && innerDiff < 0) { Cv2.DrawContours(overlayMat, contours, i, OpaqueScalar(Scalar.Red)); Console.WriteLine( $"{c} wrong inner size. expected {parameter.InnerRadius}, got {innerRadius} ({Math.Abs(innerDiff / parameter.InnerRadius) * 100f})"); errorReason.Add(EErrorReason.WrongSize); ErrorTextHelper.DrawErrorText(overlayMat, new OpenCvSharp.Point(coordX, coordY), EErrorReason.WrongSize); errorPoints.Add(c); wasError = true; break; } break; } } if (!found) { wasError = true; errorReason.Add(EErrorReason.Missing); errorPoints.Add(c); Cv2.Circle(overlayMat, coordX, coordY, 10, OpaqueScalar(Scalar.Red), Cv2.FILLED); ErrorTextHelper.DrawErrorText(overlayMat, new OpenCvSharp.Point(coordX, coordY), EErrorReason.Missing); } c++; } if (wasError) { Cv2.DrawContours(overlayMat, new[] {blisterRectangle}, 0, OpaqueScalar(Scalar.Red), thickness: 5); } else { Cv2.DrawContours(overlayMat, new[] {blisterRectangle}, 0, OpaqueScalar(Scalar.Pink), thickness: 1); } inverted = overlayMat; sw2.Stop(); Console.WriteLine("CVTime:" + sw2.ElapsedMilliseconds); } sw.Stop(); Console.WriteLine("Time:" + sw.ElapsedMilliseconds); inverted = Overlap(numbersOverlay, inverted); var final = inverted.Resize(framedImage.Size()); final = Overlap(final,framedImage); if (_learnedParameters.ImageWidth > 0) { float cut = _learnedParameters.ImageWidth; Cv2.Line(final, new OpenCvSharp.Point(cut, 0), new OpenCvSharp.Point(cut, 1408), Scalar.Red, 1); } if(_learnedParameters.CandyParameters.Count==0)errorReason.Add(EErrorReason.NotLearned); Thread.Sleep(50); return new ProcessingResult(framedImage, final, _learnedParameters.CandyParameters.Count == 0 || wasError, (int) sw.ElapsedMilliseconds, errorReason) { BlisterRectangle = blisterRectangle.Select(x => new Point(x.X, x.Y)).ToList(), ErrorPoints = errorPoints }; } catch(Exception ex) { Thread.Sleep(50); return new ProcessingResult(image, image, true, (int)sw.ElapsedMilliseconds, new List() {EErrorReason.Unknown}) { }; } } } public OpenCvSharp.Point[][] LastContours { get; set; } public LearningParameters LearnedParameters => _learnedParameters; private CandyParameter ClosestPoint(Point blisterCenter, int cX, int cY, int outerRadius, int innerRadius) { CandyParameter closest = null; double minDistance = double.MaxValue; foreach (CandyParameter parameter in _learnedParameters.CandyParameters) { var coordX = parameter.Position.X + (blisterCenter.X - _learnedParameters.BlisterPosition.X); var coordY = parameter.Position.Y + (blisterCenter.Y - _learnedParameters.BlisterPosition.Y); var distance = Distance(coordX - cX, coordY - cY); if ( distance< minDistance) { minDistance = distance; closest = parameter; } } return closest; } private static double Distance(int x, int y) { return Math.Sqrt(x * x + y * y); } static void ShowHistogram(Mat hist) { Mat render = new Mat(new Size(125, 125), MatType.CV_8UC3, Scalar.All(255)); double minVal, maxVal; Cv2.MinMaxLoc(hist, out minVal, out maxVal); Scalar color = Scalar.All(100); // Scales and draws histogram hist = hist * (maxVal != 0 ? 125 / maxVal : 0.0); hist.GetArray(out float[] histArr); for (int j = 0; j < 125; ++j) { render.Rectangle( new OpenCvSharp.Point(j , render.Rows - (int)histArr[j]), new OpenCvSharp.Point(j + 1 , render.Rows), color, -1); } Cv2.ImShow("hist",render); Cv2.WaitKey(1); } public static double Map (double value, double fromSource, double toSource, double fromTarget, double toTarget) { return (value - fromSource) / (toSource - fromSource) * (toTarget - fromTarget) + fromTarget; } private static object _lockObject = new object(); private float _blisterAngle; private Point _blisterCenter; private int _blisterEdge; private Mat _resizedMat; private readonly string _configurationPath; private static Scalar OpaqueScalar(Scalar scalar) { return new Scalar(scalar.Val0, scalar.Val1, scalar.Val2, 255); } private static Mat CalcHist(OpenCvSharp.Point[][] contours, int i, Mat resizedMat,out float outerRadius,out float innerRadius) { var c = new Mat(new Size(512, 512), MatType.CV_8UC1,Scalar.Black); Cv2.DrawContours(c, contours, i, Scalar.White, thickness: Cv2.FILLED); Cv2.MinEnclosingCircle(contours[i],out var center,out outerRadius); var distance = c.DistanceTransform(DistanceTypes.L2, DistanceTransformMasks.Mask5); distance.MinMaxIdx(out _,out var innerRadiusD); innerRadius = (float) innerRadiusD; var hist = new Mat(); Cv2.CalcHist(new[] {resizedMat}, new[] {0, 1, 2}, c, hist, 3, new[] {5, 5, 5}, new[] {new Rangef(0, 256), new Rangef(0, 256), new Rangef(0, 256)}); Cv2.Normalize(hist, hist); hist = hist.Reshape(1, 125, 1); //if (i == 22) //{ // ShowHistogram(hist); //} return hist; } private OpenCvSharp.Point[] FindBlister(Mat resBlister,out float blisterAngle,out Point center, out int blisterEdge) { var blisterMat = resBlister.Threshold(128, 255, ThresholdTypes.Binary); var blisterContours = blisterMat.FindContoursAsArray(RetrievalModes.List, ContourApproximationModes.ApproxNone); var cnt = blisterContours.Length; int? blisterArea = null; blisterEdge = 0; for (int i = 0; i < cnt; i++) { var area = Cv2.ContourArea(blisterContours[i]); if (area>5000) { var max= blisterContours[i].Max(x => x.X); if (max>blisterEdge) { blisterEdge = max; } var rect2 = Cv2.MinAreaRect(blisterContours[i]); var tmp = rect2.Angle; if (tmp > 15) { tmp = -(90 - tmp); } blisterAngle = tmp/180f*3.14f; center = new Point((int)rect2.Center.X, (int)rect2.Center.Y); Console.WriteLine("blister angle:"+blisterAngle); //var c = blisterMat.CvtColor(ColorConversionCodes.GRAY2BGR); //Cv2.Rectangle(c,rect2.BoundingRect(),Scalar.Red); //Cv2.ImShow("test",c); //Cv2.WaitKey(0); return rect2.Points().Select(x=>new OpenCvSharp.Point(x.X,x.Y)).ToArray(); } } throw new Exception("Blister not found"); } public static Bitmap SetImageOpacity(Bitmap image, float opacity) { try { //create a Bitmap the size of the image provided var bmp = new Bitmap(image.Width, image.Height); //create a graphics object from the image using (var gfx = Graphics.FromImage(bmp)) { //create a color matrix object var matrix = new ColorMatrix(); //set the opacity matrix.Matrix33 = opacity; //create image attributes var attributes = new ImageAttributes(); //set the color(opacity) of the image attributes.SetColorMatrix(matrix, ColorMatrixFlag.Default, ColorAdjustType.Bitmap); //now draw the image gfx.DrawImage(image, new Rectangle(0, 0, bmp.Width, bmp.Height), 0, 0, image.Width, image.Height, GraphicsUnit.Pixel, attributes); } return bmp; } catch (Exception ex) { //MessageBox.Show(ex.Message); throw ex; //return null; } } public static Mat Overlap(Mat source1, Mat source2) { var target = new Mat(source2.Size(),source2.Type()); Cv2.Split(source1, out Mat[] src2Channels); var alpha = src2Channels[3]; source2.CopyTo(target); source1.CopyTo(target,alpha); return target; } public int GetCameraWidth() { return _learnedParameters.ImageWidth; } } }