using MainShell.Common; using MainShell.Log; using MainShell.Models; using MaxwellFramework.Core.Attributes; using SemiconductorVisionAlgorithm.SemiParams; using SemiconductorVisionAlgorithm.SemiWaferRecip; using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; namespace MainShell.Vision { /// /// ????????????? /// [Singleton] public class FindTemplateService : IFindTemplateService { private readonly IImageCaptureService _imageCaptureService; public FindTemplateService(IImageCaptureService imageCaptureService) { _imageCaptureService = imageCaptureService ?? throw new ArgumentNullException(nameof(imageCaptureService)); } public async Task> ProcessAsync( FindTemplateRequest request, CancellationToken cancellationToken = default(CancellationToken)) { if (request == null) { throw new ArgumentNullException(nameof(request)); } FindTemplateImageRequest imageRequest = CreateImageRequest(request); return await ExecuteWithCaptureAsync( request.CameraSource, request.CaptureOptions, imageRequest, ProcessImageAsync, cancellationToken).ConfigureAwait(false); } public Task> ProcessImageAsync( MxImage image, FindTemplateImageRequest request, CancellationToken cancellationToken = default(CancellationToken)) { if (image == null) { throw new ArgumentNullException(nameof(image)); } if (request == null) { throw new ArgumentNullException(nameof(request)); } FindTemplateImageRequest normalizedRequest = NormalizeRequest(request); VisionProcessResult validationResult = ValidateRequest(normalizedRequest); if (validationResult != null) { return Task.FromResult(CreateFailureResult(validationResult)); } try { cancellationToken.ThrowIfCancellationRequested(); VisionProcessResult result = ExecuteSingleMatch(image, normalizedRequest); if (!result.Succeeded) { result.Message.LogSysError(); } return Task.FromResult(result); } catch (OperationCanceledException ex) { string message = $"FindTemplateService: process image was cancelled. Timeout={normalizedRequest.TimeoutMilliseconds} ms."; message.LogInfo(); return Task.FromResult(VisionProcessResult.Failure( VisionFailureCategory.Cancelled, VisionErrorCode.OperationCancelled, null, MessageKey.VisionOperationCancelled, message, ex)); } catch (Exception ex) { string message = $"FindTemplateService: process image failed. {ex.Message}"; message.LogSysError(); return Task.FromResult(VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateMatchFailed, VisionAlarmIds.TemplateMatchFailed, MessageKey.VisionTemplateMatchFailed, message, ex)); } } public async Task> ProcessMultipleAsync( FindTemplateRequest request, CancellationToken cancellationToken = default(CancellationToken)) { if (request == null) { throw new ArgumentNullException(nameof(request)); } FindTemplateImageRequest imageRequest = CreateImageRequest(request); return await ExecuteWithCaptureAsync( request.CameraSource, request.CaptureOptions, imageRequest, ProcessMultipleImageAsync, cancellationToken).ConfigureAwait(false); } public Task> ProcessMultipleImageAsync( MxImage image, FindTemplateImageRequest request, CancellationToken cancellationToken = default(CancellationToken)) { if (image == null) { throw new ArgumentNullException(nameof(image)); } if (request == null) { throw new ArgumentNullException(nameof(request)); } FindTemplateImageRequest normalizedRequest = NormalizeRequest(request); VisionProcessResult validationResult = ValidateRequest(normalizedRequest); if (validationResult != null) { return Task.FromResult(CreateFailureResult(validationResult)); } try { cancellationToken.ThrowIfCancellationRequested(); VisionProcessResult result = ExecuteMultipleMatch(image, normalizedRequest); if (!result.Succeeded) { result.Message.LogSysError(); } return Task.FromResult(result); } catch (OperationCanceledException ex) { string message = $"FindTemplateService: multiple template process image was cancelled. Timeout={normalizedRequest.TimeoutMilliseconds} ms."; message.LogInfo(); return Task.FromResult(VisionProcessResult.Failure( VisionFailureCategory.Cancelled, VisionErrorCode.OperationCancelled, null, MessageKey.VisionOperationCancelled, message, ex)); } catch (Exception ex) { string message = $"FindTemplateService: multiple template process image failed. {ex.Message}"; message.LogSysError(); return Task.FromResult(VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateMatchFailed, VisionAlarmIds.TemplateMatchFailed, MessageKey.VisionTemplateMatchFailed, message, ex)); } } private static FindTemplateImageRequest NormalizeRequest(FindTemplateImageRequest request) { FindTemplateImageRequest normalizedRequest = new FindTemplateImageRequest(); normalizedRequest.TimeoutMilliseconds = request.TimeoutMilliseconds > 0 ? request.TimeoutMilliseconds : 3000; normalizedRequest.Parameters = CloneParameters(request.Parameters); return normalizedRequest; } private static VisionProcessResult ValidateRequest(FindTemplateImageRequest request) { if (request.Parameters == null) { return VisionProcessResult.Failure( VisionFailureCategory.Validation, VisionErrorCode.RequestInvalid, VisionAlarmIds.RequestInvalid, MessageKey.VisionRequestInvalid, "FindTemplateService: parameters cannot be null."); } if (request.TimeoutMilliseconds <= 0 || request.TimeoutMilliseconds > 600000) { return VisionProcessResult.Failure( VisionFailureCategory.Validation, VisionErrorCode.TimeoutInvalid, VisionAlarmIds.RequestInvalid, MessageKey.VisionTimeoutInvalid, string.Format("FindTemplateService: TimeoutMilliseconds {0} is invalid. Expected range is [1, 60000].", request.TimeoutMilliseconds)); } if (request.Parameters.MinScore <= 0d || request.Parameters.MinScore > 1d) { return VisionProcessResult.Failure( VisionFailureCategory.Validation, VisionErrorCode.MinScoreInvalid, VisionAlarmIds.TemplateMinScoreInvalid, MessageKey.VisionTemplateMinScoreInvalid, string.Format("FindTemplateService: MinScore {0} is invalid. Expected range is (0, 1].", request.Parameters.MinScore)); } if (string.IsNullOrWhiteSpace(request.Parameters.TemplatePath)) { return VisionProcessResult.Failure( VisionFailureCategory.Validation, VisionErrorCode.TemplatePathEmpty, VisionAlarmIds.TemplatePathEmpty, MessageKey.VisionTemplatePathEmpty, "FindTemplateService: TemplatePath cannot be empty."); } if (request.Parameters.UseRoi && string.IsNullOrWhiteSpace(request.Parameters.RoiName)) { return VisionProcessResult.Failure( VisionFailureCategory.Validation, VisionErrorCode.RoiNameEmpty, VisionAlarmIds.TemplateRoiInvalid, MessageKey.VisionTemplateRoiInvalid, "FindTemplateService: RoiName cannot be empty when UseRoi is true."); } return null; } private static FindTemplateParameters CloneParameters(FindTemplateParameters parameters) { if (parameters == null) { return new FindTemplateParameters(); } FindTemplateParameters clone = new FindTemplateParameters(); clone.UseRoi = parameters.UseRoi; clone.RoiName = parameters.RoiName; clone.MinScore = parameters.MinScore; clone.TemplatePath = parameters.TemplatePath; return clone; } private static FindTemplatesResult CreateMultipleResult(IList matches) { FindTemplatesResult result = new FindTemplatesResult(); if (matches == null) { return result; } foreach (FindTemplateResult match in matches) { result.Items.Add(match); } return result; } private static FindTemplateImageRequest CreateImageRequest(FindTemplateRequest request) { FindTemplateImageRequest imageRequest = new FindTemplateImageRequest(); imageRequest.TimeoutMilliseconds = request.TimeoutMilliseconds; imageRequest.Parameters = CloneParameters(request.Parameters); return imageRequest; } private static VisionProcessResult ExecuteSingleMatch( MxImage image, FindTemplateImageRequest request) { Camera camera = image.Image; if (camera == null) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.ImageNull, VisionAlarmIds.ImageNull, MessageKey.VisionImageIsNull, "FindTemplateService: input image does not contain a valid camera object."); } Template template = null; double score = 0d; Rectangle1 contour = null; List circles = null; List lines = null; List rectangles = null; int matchCode = WaferRecips.Instance.match_image( camera, request.Parameters.TemplatePath, out template, out score, out contour, out circles, out lines, out rectangles); if (matchCode != 0) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateMatchFailed, VisionAlarmIds.TemplateMatchFailed, MessageKey.VisionTemplateMatchFailed, $"FindTemplateService: single template match failed. Code={matchCode}, TemplatePath={request.Parameters.TemplatePath}."); } if (template == null || double.IsNaN(score) || double.IsInfinity(score)) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateResultInvalid, VisionAlarmIds.TemplateResultInvalid, MessageKey.VisionTemplateMatchFailed, $"FindTemplateService: single template match returned an invalid result. TemplatePath={request.Parameters.TemplatePath}, Score={score}."); } if (score < request.Parameters.MinScore) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateMatchFailed, VisionAlarmIds.TemplateMatchFailed, MessageKey.VisionTemplateMatchFailed, $"FindTemplateService: single template match score {score} is lower than required {request.Parameters.MinScore}. TemplatePath={request.Parameters.TemplatePath}."); } FindTemplateResult result = CreateSingleResult(template, score, contour, circles, lines, rectangles); return VisionProcessResult.Success(result); } private static VisionProcessResult ExecuteMultipleMatch( MxImage image, FindTemplateImageRequest request) { Camera camera = image.Image; if (camera == null) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.ImageNull, VisionAlarmIds.ImageNull, MessageKey.VisionImageIsNull, "FindTemplateService: input image does not contain a valid camera object."); } double[] rows = null; double[] cols = null; double[] angles = null; double[] scores = null; List contours = null; bool succeeded = WaferRecips.Instance.match_multi_0bject( camera, request.Parameters.TemplatePath, out rows, out cols, out angles, out scores, out contours); if (!succeeded) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateMatchFailed, VisionAlarmIds.TemplateMatchFailed, MessageKey.VisionTemplateMatchFailed, $"FindTemplateService: multiple template match failed. TemplatePath={request.Parameters.TemplatePath}."); } if (!HasConsistentMultipleResult(rows, cols, angles, scores, contours)) { return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateResultInvalid, VisionAlarmIds.TemplateResultInvalid, MessageKey.VisionTemplateMatchFailed, $"FindTemplateService: multiple template match returned inconsistent result data. TemplatePath={request.Parameters.TemplatePath}."); } FindTemplatesResult result = CreateMultipleResult(rows, cols, angles, scores, contours); return VisionProcessResult.Success(result); } private static FindTemplateResult CreateSingleResult( Template template, double score, Rectangle1 contour, IList circles, IList lines, IList rectangles) { FindTemplateResult result = new FindTemplateResult(); result.CenterX = template.X; result.CenterY = template.Y; result.Score = score; result.Match = CreateMatchResult(template.Phi, score, contour, circles, lines, rectangles); result.Contour = CreateContourResult(contour); AppendCircleResults(result.Circles, circles); AppendLineResults(result.Lines, lines); AppendRectangleResults(result.Rectangles, rectangles); return result; } private static FindTemplatesResult CreateMultipleResult( double[] rows, double[] cols, double[] angles, double[] scores, IList contours) { FindTemplatesResult result = new FindTemplatesResult(); for (int i = 0; i < rows.Length; i++) { FindTemplateResult item = new FindTemplateResult(); item.CenterX = cols[i]; item.CenterY = rows[i]; item.Score = scores[i]; item.Match = CreateMatchResult(angles[i], scores[i], contours[i], null, null, null); item.Contour = CreateContourResult(contours[i]); result.Items.Add(item); } return result; } private static VisionMatchResult CreateMatchResult( double angle, double score, Rectangle1 contour, IList circles, IList lines, IList rectangles) { VisionMatchResult match = new VisionMatchResult(); match.Angle = angle; match.Score = score; match.Tag = CreateMatchTag(contour, circles, lines, rectangles); return match; } private static FindTemplateContourResult CreateContourResult(Rectangle1 contour) { if (contour == null) { return null; } FindTemplateContourResult result = new FindTemplateContourResult(); result.StartX = contour.Start_X; result.StartY = contour.Start_Y; result.EndX = contour.End_X; result.EndY = contour.End_Y; return result; } private static void AppendCircleResults(IList destination, IList circles) { if (destination == null || circles == null) { return; } foreach (Circle circle in circles) { if (circle == null) { continue; } FindTemplateCircleResult item = new FindTemplateCircleResult(); item.CenterX = circle.X; item.CenterY = circle.Y; item.Radius = circle.R; destination.Add(item); } } private static void AppendLineResults(IList destination, IList lines) { if (destination == null || lines == null) { return; } foreach (LineSeg line in lines) { if (line == null) { continue; } FindTemplateLineResult item = new FindTemplateLineResult(); item.StartX = line.Start_X; item.StartY = line.Start_Y; item.EndX = line.End_X; item.EndY = line.End_Y; destination.Add(item); } } private static void AppendRectangleResults(IList destination, IList rectangles) { if (destination == null || rectangles == null) { return; } foreach (Rectangle2 rectangle in rectangles) { if (rectangle == null) { continue; } FindTemplateRectangleResult item = new FindTemplateRectangleResult(); item.StartX = rectangle.XStart; item.StartY = rectangle.YStart; item.Width = rectangle.Width; item.Height = rectangle.Height; item.Angle = rectangle.Phi; destination.Add(item); } } private static string CreateMatchTag( Rectangle1 contour, IList circles, IList lines, IList rectangles) { if (contour == null && circles == null && lines == null && rectangles == null) { return null; } int circleCount = circles != null ? circles.Count : 0; int lineCount = lines != null ? lines.Count : 0; int rectangleCount = rectangles != null ? rectangles.Count : 0; if (contour == null) { return $"Circles={circleCount};Lines={lineCount};Rects={rectangleCount}"; } return $"Contour=({contour.Start_X},{contour.Start_Y})-({contour.End_X},{contour.End_Y});Circles={circleCount};Lines={lineCount};Rects={rectangleCount}"; } private static bool HasConsistentMultipleResult( double[] rows, double[] cols, double[] angles, double[] scores, IList contours) { if (rows == null || cols == null || angles == null || scores == null || contours == null) { return false; } if (rows.Length == 0) { return false; } if (rows.Length != cols.Length || rows.Length != angles.Length || rows.Length != scores.Length) { return false; } if (rows.Length != contours.Count) { return false; } for (int i = 0; i < rows.Length; i++) { if (double.IsNaN(rows[i]) || double.IsInfinity(rows[i]) || double.IsNaN(cols[i]) || double.IsInfinity(cols[i]) || double.IsNaN(angles[i]) || double.IsInfinity(angles[i]) || double.IsNaN(scores[i]) || double.IsInfinity(scores[i])) { return false; } } return true; } private async Task> ExecuteWithCaptureAsync( MainShell.Common.CameraType cameraSource, CameraCaptureOptions captureOptions, FindTemplateImageRequest imageRequest, Func>> imageProcessor, CancellationToken cancellationToken) { ImageCaptureResult captureResult = null; MxImage image = null; try { captureResult = await _imageCaptureService.CaptureAsync( cameraSource, captureOptions, cancellationToken).ConfigureAwait(false); if (captureResult == null) { return VisionProcessResult.Failure( VisionFailureCategory.Capture, VisionErrorCode.NoFrame, VisionAlarmIds.NoFrame, MessageKey.VisionNoFrameReturned, string.Format( "FindTemplateService: image capture returned null result for camera '{0}'.", cameraSource)); } if (!captureResult.Succeeded) { return CreateFailureResult(VisionResultMapper.CreateCaptureFailure(captureResult)); } image = captureResult.Image; if (image == null) { string message = string.Format( "FindTemplateService: image capture succeeded but returned null image for camera '{0}'.", cameraSource); message.LogSysError(); return VisionProcessResult.Failure( VisionFailureCategory.Capture, VisionErrorCode.ImageNull, VisionAlarmIds.ImageNull, MessageKey.VisionImageIsNull, message); } return await imageProcessor(image, imageRequest, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException ex) { string message = $"FindTemplateService: capture orchestration was cancelled for camera '{cameraSource}'."; message.LogInfo(); return VisionProcessResult.Failure( VisionFailureCategory.Cancelled, VisionErrorCode.OperationCancelled, null, MessageKey.VisionOperationCancelled, message, ex); } catch (Exception ex) { string message = $"FindTemplateService: capture orchestration failed for camera '{cameraSource}'. {ex.Message}"; message.LogSysError(); return VisionProcessResult.Failure( VisionFailureCategory.Algorithm, VisionErrorCode.TemplateMatchFailed, VisionAlarmIds.TemplateMatchFailed, MessageKey.VisionTemplateMatchFailed, message, ex); } finally { if (image != null) { image.Dispose(); } } } private static VisionProcessResult CreateFailureResult(VisionProcessResult result) { return VisionProcessResult.Failure( result.FailureCategory, result.ErrorCode, result.AlarmId, result.UserMessageKey, result.Message, result.Exception); } } }