Error while using VNDetectRectanglesRequest and CameraPreview CVPixelBuffer in Xamarin - xamarin

Question

I have been stuck on a problem for a few days.
Our CameraPreview works perfectly, along with the VNDetectRectanglesRequest.
Once we move over a rectangle our Handler is called, which makes me believe the CVPixelBuffer is passing a valid image into the processor.
When the handler is called we request GetResults and get the exception:
[0:] Rectangle Observations Exception - Objective-C exception thrown. Name: NSRangeException Reason: *** -[__NSSingleObjectArrayI objectAtIndex:]: index 4354863128 beyond bounds [0 .. 0]
I can obviously tell that this is related to an array however I have a working copy using: UIImagePickerController which is working perfectly within a simulator, but moving the code over to a live preview is having issues.
Any help would be highly appreciated.
Here is my code snippet:
public class VisionDocumentDecoder : DefaultDecoderBase
{
private TaskCompletionSource<DocumentScanResult> documentResult = null;
private VNDetectRectanglesRequest documentRequest;
private UIImageOrientation orientation = UIImageOrientation.Down;
public VisionDocumentDecoder() : base()
{
documentRequest = new VNDetectRectanglesRequest(OnImageRectangleHandled);
//documentRequest.MaximumObservations = 6;
//documentRequest.MinimumConfidence = 0.1f;
//documentRequest.MinimumAspectRatio = 0.3f;
//documentRequest.QuadratureTolerance = 75;
}
public UIImage Convert(CVPixelBuffer pixelBuffer)
{
CIImage ciImage = CIImage.FromImageBuffer(pixelBuffer);
CIContext temporaryContext = CIContext.FromOptions(null);
using (CGImage cgImage = temporaryContext.CreateCGImage(ciImage, new CGRect(0, 0, pixelBuffer.Width, pixelBuffer.Height)))
{
UIImage uiImage = UIImage.FromImage(cgImage);
return uiImage;
}
}
public override IScanResult Decode(CVPixelBuffer pixelBuffer)
{
Logger.Log("Decoding sample ...");
documentResult = new TaskCompletionSource<DocumentScanResult>();
CIImage ciImage = CIImage.FromImageBuffer(pixelBuffer);
UIImage image = Convert(pixelBuffer);
if (image == null)
{
documentResult.SetResult(null);
return documentResult.Task.Result;
}
var handler = new VNImageRequestHandler(image.CGImage,
ConvertToCGImagePropertyOrientation(image.Orientation), new NSDictionary());
DispatchQueue.DefaultGlobalQueue.DispatchAsync(() =>
{
handler.Perform(new VNRequest[]
{
documentRequest
}, out NSError error);
});
documentResult.Task.Wait();
return documentResult.Task.Result;
}
void OnImageRectangleHandled(VNRequest request, NSError error)
{
if (error != null)
{
documentResult.SetException(new NSErrorException(error));
return;
}
Logger.Log("Rectangle image handler");
var decoder = PerformanceCounter.Start();
var scanResults = new DocumentScanResult();
var documentResults = new List<DocumentPointResults>();
scanResults.Success = true;
try
{
Logger.Log($"Taking observations ...");
var observations = request.GetResults<VNRectangleObservation>();
//request.
//var observations = request.GetResults<VNRectangleObservation>();
Logger.Log($"Rectangles found {observations.Length}");
if (observations.Length > 0)
{
Logger.Log($"Rectangles found {observations.Length}");
// foreach (var obs in observations)
// {
// Logger.Log("Found observation!");
// var boundingBox = obs.BoundingBox;
// var topLeft = new Point((int)boundingBox.X, (int)boundingBox.Y);
// var topRight = new Point((int)boundingBox.X + (int)boundingBox.Width, (int)boundingBox.Y);
// var bottomLeft = new Point((int)boundingBox.X, (int)boundingBox.Y + (int)boundingBox.Height);
// var bottomRight = new Point((int)boundingBox.X + (int)boundingBox.Width, (int)boundingBox.Y + (int)boundingBox.Height);
// documentResults.Add(new DocumentPointResults(topLeft, topRight, bottomLeft, bottomRight));
// }
}
}
catch (Exception e)
{
Logger.Log($"Rectangle Observations Exception - {e.Message}");
}
PerformanceCounter.Stop(decoder, "Vision document rectangle handler take {0} ms.");
scanResults.Points = documentResults;
documentResult.SetResult(scanResults);
}
//public override void ScanningOptionsUpdate(ScanningOptionsBase options)
//{
//}
CGImagePropertyOrientation ConvertToCGImagePropertyOrientation(UIImageOrientation orientation)
{
switch (orientation)
{
case UIImageOrientation.Up:
return CGImagePropertyOrientation.Up;
case UIImageOrientation.UpMirrored:
return CGImagePropertyOrientation.UpMirrored;
case UIImageOrientation.Down:
return CGImagePropertyOrientation.Down;
case UIImageOrientation.DownMirrored:
return CGImagePropertyOrientation.DownMirrored;
case UIImageOrientation.Left:
return CGImagePropertyOrientation.Left;
case UIImageOrientation.LeftMirrored:
return CGImagePropertyOrientation.LeftMirrored;
case UIImageOrientation.Right:
return CGImagePropertyOrientation.Right;
case UIImageOrientation.RightMirrored:
return CGImagePropertyOrientation.RightMirrored;
default:
throw new Exception("unrecognized orientation");
}
}
}