Can I apply CIFilter to an ARkit camera channel?

Question

I am trying to apply a blur effect to a real-time camera image in ARSCNView. I checked the WWDC video. They only mentioned personalized rendering with Metal, but I did not find a complete example on the Internet. Any idea how to do this?

---

Updated 1 I tried to apply a filter to the background. It shows the wrong orientation. How can i fix this?

let bg=self.session.currentFrame?.capturedImage

        if(bg != nil){
            let context = CIContext()
            let filter:CIFilter=CIFilter(name:"CIColorInvert")!
            let image:CIImage=CIImage(cvPixelBuffer: bg!)
            filter.setValue(image, forKey: kCIInputImageKey)
            let result=filter.outputImage!
            self.sceneView.scene.background.contents = context.createCGImage(result, from: result.extent)

        }

Answer 1

I found a pretty good solution, which is to simply apply the appropriate geometric transformation to the property backgroundwhen changing the orientation of the device:

func session(_ session: ARSession, didUpdate frame: ARFrame) {
    let image = CIImage(cvPixelBuffer: frame.capturedImage)
    filter.setValue(image, forKey: kCIInputImageKey)

    let context = CIContext()
    if let result = filter.outputImage,
        let cgImage = context.createCGImage(result, from: result.extent) {

        sceneView.scene.background.contents = cgImage
        if let transform = currentScreenTransform() {
            sceneView.scene.background.contentsTransform = transform
        }
    }
}

private func currentScreenTransform() -> SCNMatrix4? {
    switch UIDevice.current.orientation {
    case .landscapeLeft:
        return SCNMatrix4Identity
    case .landscapeRight:
        return SCNMatrix4MakeRotation(.pi, 0, 0, 1)
    case .portrait:
        return SCNMatrix4MakeRotation(.pi / 2, 0, 0, 1)
    case .portraitUpsideDown:
        return SCNMatrix4MakeRotation(-.pi / 2, 0, 0, 1)
    default:
        return nil
    }
}

, UIDevice.current.beginGeneratingDeviceOrientationNotifications() viewDidLoad.

Answer 2

.

let bg = sceneView.session.currentFrame?.capturedImage
    if (bg != nil) {
        let context = CIContext()
        let filter:CIFilter = CIFilter(name: "CIColorInvert")!

        let image:CIImage = CIImage(cvPixelBuffer: bg!, options: nil)

        let cgImage:CGImage = context.createCGImage(image, from: image.extent)!
        let uiImage:UIImage = UIImage.init(cgImage: cgImage)
        let resultImage = CIImage(image: uiImage)?.oriented(forExifOrientation: imageOrientationToDeviceOrientation(value: UIDevice.current.orientation))

        filter.setValue(resultImage, forKey: kCIInputImageKey)

        let result = filter.outputImage!
        self.sceneView.scene.background.contents = context.createCGImage(result, from: result.extent)
    }


func imageOrientationToDeviceOrientation(value: UIDeviceOrientation) -> Int32 {
            switch (value) {
            case .portrait:
                return 6
            case .landscapeLeft:
                return 1
            case .landscapeRight:
                return 3
            default:
                return 6
            }
        }