Core Image: Photo effect react/change slowly in real iPhone when changing intensity from a Slider - performance

Question

The below code apply an image Picker and showing filtered image in contentView, in this case I used SepiaTone effect. When I do the movements in slider, to change the filter intensity. The photo react delay and the sliding is not fluent.
But when using some apps from App Store like "Afterlight", with their photo filter, the changing to photo effect is smoothly when a user moves a slider.
So what is the different, is those apps use other framework? How could we improve the performance on continuously changing filter intensity if we still use Core Image?
ImagePicker
import SwiftUI
struct ImagePicker: UIViewControllerRepresentable {
class Coordinator: NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate {
let parent: ImagePicker
init(_ parent: ImagePicker) {
self.parent = parent
}
func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
if let uiImage = info[.originalImage] as? UIImage {
parent.image = uiImage
}
picker.dismiss(animated: true, completion: nil)
}
}
#Binding var image: UIImage?
func makeCoordinator() -> Coordinator {
Coordinator(self)
}
func makeUIViewController(context: UIViewControllerRepresentableContext<ImagePicker>) -> UIImagePickerController {
let picker = UIImagePickerController()
picker.delegate = context.coordinator
return picker
}
func updateUIViewController(_ uiViewController: UIImagePickerController, context: UIViewControllerRepresentableContext<ImagePicker>) {
}
}
ContentView
import CoreImage
import CoreImage.CIFilterBuiltins
import SwiftUI
struct ContentView: View {
#State private var image: Image?
#State private var filterIntensity = 0.5
#State private var showingImagePicker = false
#State private var inputImage: UIImage?
#State private var currentFilter = CIFilter.sepiaTone()
let context = CIContext()
var body: some View {
let intensity = Binding<Double>(
get: {
self.filterIntensity
},
set: {
self.filterIntensity = $0
self.applyProcessing()
})
return NavigationView {
VStack {
ZStack {
Rectangle()
.fill(Color.secondary)
if image != nil {
image?
.resizable()
.scaledToFit()
} else {
Text("Tap to select a picture")
.foregroundColor(.white)
.font(.headline)
}
}
.onTapGesture {
self.showingImagePicker = true
}
HStack {
Text("Intensity")
Slider(value: intensity)
}
.padding(.vertical)
HStack {
Button("Change Filter") {
// change filter
}
Spacer()
Button("Save") {
// save the picture
}
}
}
.padding([.horizontal, .bottom])
.navigationBarTitle("Instafilter")
.sheet(isPresented: $showingImagePicker, onDismiss: loadImage) {
ImagePicker(image: self.$inputImage)
}
}
}
func loadImage() {
guard let inputImage = inputImage else { return }
let beginImage = CIImage(image: inputImage)
currentFilter.setValue(beginImage, forKey: kCIInputImageKey)
applyProcessing()
}
func applyProcessing() {
currentFilter.intensity = Float(filterIntensity)
guard let outputImage = currentFilter.outputImage else { return }
if let cgImage = context.createCGImage(outputImage, from: outputImage.extent) {
let uiImage = UIImage(cgImage: cgImage)
image = Image(uiImage: uiImage)
}
}
}

Answer 1

It helps to perform the image filter in a background thread to not block the main queue with the image processing. Also, it would make sense to not react to every single change but to debounce changes: wait for a short amount like 50ms of no change before the actual computation is done - this prevents a large number of image computations to happen if the filter params change often in a short time (like when the user drags the slider around). Both can be achieved using the Combine framework, here is an example based on your code. The effect is separated in a separate class ImageEffect that uses Combine to perform the image processing in the background whenever inputImage/filterIntensity changes, optimized using the debounce operator:
import CoreImage
import CoreImage.CIFilterBuiltins
import SwiftUI
import Combine
class ImageEffect: ObservableObject {
#Published var filterIntensity = 0.5
#Published var inputImage: UIImage?
#Published var outputImage: UIImage?
#Published var currentFilter = CIFilter.sepiaTone()
let context = CIContext()
var subscriptions = Set<AnyCancellable>()
let queue = DispatchQueue(label: "Image processing")
init() {
self.$inputImage
.map { inputImage -> CIImage? in
guard let inputImage = inputImage else { return nil }
return CIImage(image: inputImage)
}
.combineLatest(self.$filterIntensity)
.debounce(for: .milliseconds(50), scheduler: queue)
.map { inputImage, filterIntensity -> UIImage? in
guard let inputImage = inputImage else { return nil }
self.currentFilter.inputImage = inputImage
self.currentFilter.intensity = Float(filterIntensity)
guard let outputImage = self.currentFilter.outputImage else { return nil }
guard let cgImage = self.context.createCGImage(outputImage, from: outputImage.extent) else { return nil }
return UIImage(cgImage: cgImage)
}
.receive(on: RunLoop.main)
.sink { image in
self.outputImage = image
}
.store(in: &self.subscriptions)
}
}
struct ImageEffectView: View {
#ObservedObject var imageEffect = ImageEffect()
#State private var showingImagePicker = false
var body: some View {
NavigationView {
VStack {
ZStack {
Rectangle()
.fill(Color.secondary)
if imageEffect.outputImage != nil {
Image(uiImage: imageEffect.outputImage!)
.resizable()
.scaledToFit()
} else {
Text("Tap to select a picture")
.foregroundColor(.white)
.font(.headline)
}
}
.onTapGesture {
self.showingImagePicker = true
}
HStack {
Text("Intensity")
Slider(value: $imageEffect.filterIntensity)
}
.padding(.vertical)
HStack {
Button("Change Filter") {
// change filter
}
Spacer()
Button("Save") {
// save the picture
}
}
}
.padding([.horizontal, .bottom])
.navigationBarTitle("Instafilter")
.sheet(isPresented: $showingImagePicker) {
ImagePickerView(image: self.$imageEffect.inputImage)
}
}
}
}