Xmarin.forms iOS check whether captured photo contains face or not - xamarin

Question

In my xamarin.forms app, I created a custom camera by using Camera view and custom renders. Everything works fine. In android after the photo capture I can check whether the taken photo contains a face or not.It is done by using Camera.IFaceDetectionListener. My question is how can I achieve this in iOS? I know there is vision API. But I don't want the live face tracking. I just simply want to check whether the taken photo contains face. Any help is appreciated.
My iOS CameraPreview
public class UICameraPreview : UIView, IAVCaptureMetadataOutputObjectsDelegate
{
AVCaptureVideoPreviewLayer previewLayer;
public AVCaptureDevice[] videoDevices;
CameraOptions cameraOptions;
public AVCaptureStillImageOutput stillImageOutput;
public AVCaptureDeviceInput captureDeviceInput;
public AVCaptureDevice device;
public event EventHandler<EventArgs> Tapped;
public AVCaptureSession CaptureSession { get; set; }
public bool IsPreviewing { get; set; }
public AVCaptureStillImageOutput CaptureOutput { get; set; }
public UICameraPreview(CameraOptions options)
{
cameraOptions = options;
IsPreviewing = false;
Initialize();
}
public override void LayoutSubviews()
{
base.LayoutSubviews();
if (previewLayer != null)
previewLayer.Frame = Bounds;
}
public override void TouchesBegan(NSSet touches, UIEvent evt)
{
base.TouchesBegan(touches, evt);
OnTapped();
}
protected virtual void OnTapped()
{
var eventHandler = Tapped;
if (eventHandler != null)
{
eventHandler(this, new EventArgs());
}
}
void Initialize()
{
CaptureSession = new AVCaptureSession();
previewLayer = new AVCaptureVideoPreviewLayer(CaptureSession)
{
Frame = Bounds,
VideoGravity = AVLayerVideoGravity.ResizeAspectFill
};
videoDevices = AVCaptureDevice.DevicesWithMediaType(AVMediaType.Video);
var cameraPosition = (cameraOptions == CameraOptions.Front) ? AVCaptureDevicePosition.Front : AVCaptureDevicePosition.Back;
device = videoDevices.FirstOrDefault(d => d.Position == cameraPosition);
if (device == null)
{
return;
}
NSError error;
captureDeviceInput = new AVCaptureDeviceInput(device, out error);
CaptureSession.AddInput(captureDeviceInput);
var dictionary = new NSMutableDictionary();
dictionary[AVVideo.CodecKey] = new NSNumber((int)AVVideoCodec.JPEG);
stillImageOutput = new AVCaptureStillImageOutput()
{
OutputSettings = new NSDictionary()
};
CaptureSession.AddOutput(stillImageOutput);
Layer.AddSublayer(previewLayer);
CaptureSession.StartRunning();
IsPreviewing = true;
}
// Photo Capturing
public async Task CapturePhoto()
{
try
{
var videoConnection = stillImageOutput.ConnectionFromMediaType(AVMediaType.Video);
var sampleBuffer = await stillImageOutput.CaptureStillImageTaskAsync(videoConnection);
var jpegData = AVCaptureStillImageOutput.JpegStillToNSData(sampleBuffer);
var photo = new UIImage(jpegData);
var rotatedPhoto = RotateImage(photo, 180f);
var img = rotatedPhoto;
CALayer layer = new CALayer
{
ContentsScale = 1.0f,
Frame = Bounds,
Contents = rotatedPhoto.CGImage //Contents = photo.CGImage,
};
CaptureSession.StopRunning();
photo.SaveToPhotosAlbum((image, error) =>
{
if (!string.IsNullOrEmpty(error?.LocalizedDescription))
{
Console.Error.WriteLine($"\t\t\tError: {error.LocalizedDescription}");
}
});
}
catch (Exception ex)
{
}
//MainPage.UpdateSource(UIImageFromLayer(layer).AsJPEG().AsStream());
//MainPage.UpdateImage(UIImageFromLayer(layer).AsJPEG().AsStream());
}
}
My CameraPreviewRenderer
public class CameraPreviewRenderer : ViewRenderer<CameraPreview, UICameraPreview>, IAVCaptureMetadataOutputObjectsDelegate
{
UICameraPreview uiCameraPreview;
AVCaptureSession captureSession;
AVCaptureDeviceInput captureDeviceInput;
AVCaptureStillImageOutput stillImageOutput;
protected override void OnElementChanged(ElementChangedEventArgs<CameraPreview> e)
{
base.OnElementChanged(e);
if (e.OldElement != null)
{
// Unsubscribe
uiCameraPreview.Tapped -= OnCameraPreviewTapped;
}
if (e.NewElement != null)
{
if (Control == null)
{
uiCameraPreview = new UICameraPreview(e.NewElement.Camera);
SetNativeControl(uiCameraPreview);
MessagingCenter.Subscribe<Camera_Popup>(this, "CaptureClick", async (sender) =>
{
try
{
// Using messeging center to take photo when clicking button from shared code
var data = new AVCaptureMetadataOutputObjectsDelegate();
await uiCameraPreview.CapturePhoto();
}
catch (Exception ex)
{
return;
}
});
}
MessagingCenter.Subscribe<Camera_Popup>(this, "RetryClick", (sender) =>
{
Device.BeginInvokeOnMainThread(() =>
{
uiCameraPreview.CaptureSession.StartRunning();
uiCameraPreview.IsPreviewing = true;
});
});
MessagingCenter.Subscribe<Camera_Popup>(this, "FlipClick", (sender) =>
{
try
{
var devicePosition = uiCameraPreview.captureDeviceInput.Device.Position;
if (devicePosition == AVCaptureDevicePosition.Front)
{
devicePosition = AVCaptureDevicePosition.Back;
}
else
{
devicePosition = AVCaptureDevicePosition.Front;
}
uiCameraPreview.device = uiCameraPreview.videoDevices.FirstOrDefault(d => d.Position == devicePosition);
uiCameraPreview.CaptureSession.BeginConfiguration();
uiCameraPreview.CaptureSession.RemoveInput(uiCameraPreview.captureDeviceInput);
uiCameraPreview.captureDeviceInput = AVCaptureDeviceInput.FromDevice(uiCameraPreview.device);
uiCameraPreview.CaptureSession.AddInput(uiCameraPreview.captureDeviceInput);
uiCameraPreview.CaptureSession.CommitConfiguration();
}
catch (Exception ex)
{
var abc = ex.InnerException.Message;
}
});
uiCameraPreview.Tapped += OnCameraPreviewTapped;
}
}
void OnCameraPreviewTapped(object sender, EventArgs e)
{
if (uiCameraPreview.IsPreviewing)
{
uiCameraPreview.CaptureSession.StopRunning();
uiCameraPreview.IsPreviewing = false;
}
else
{
uiCameraPreview.CaptureSession.StartRunning();
uiCameraPreview.IsPreviewing = true;
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
Control.CaptureSession.Dispose();
Control.Dispose();
}
base.Dispose(disposing);
}
}

Answer 1

The CoreImage framework has CIDetector that provides image detectors for faces, QR codes, text, .... in which you pass in an image and you get a specific "feature set" back.
Example from Xamarin docs:
var imageFile = "photoFace2.jpg";
var image = new UIImage(imageFile);
var context = new CIContext ();
var detector = CIDetector.CreateFaceDetector (context, true);
var ciImage = CIImage.FromCGImage (image.CGImage);
var features = detector.GetFeatures (ciImage);
Console.WriteLine ("Found " + features.Length + " faces");
re: https://learn.microsoft.com/en-us/dotnet/api/CoreImage.CIDetector?view=xamarin-ios-sdk-12