Stereo AR passthrough view

Sources/MetalScope/StereoView/StereoARRenderer.swift

@@ -0,0 +1,138 @@
+//
+//  StereoARRenderer.swift
+//
+//
+//  Created by Mathijs Bernson on 18/06/2024.
+//
+
+import Metal
+import ARKit
+
+protocol RenderDestinationProvider {
+    var currentRenderPassDescriptor: MTLRenderPassDescriptor? { get }
+    var currentDrawable: CAMetalDrawable? { get }
+    var colorPixelFormat: MTLPixelFormat { get set }
+    var depthStencilPixelFormat: MTLPixelFormat { get set }
+    var sampleCount: Int { get set }
+}
+
+internal final class StereoARRenderer {
+    let session: ARSession
+    let device: MTLDevice
+    var renderDestination: RenderDestinationProvider
+
+    let textureCache: CVMetalTextureCache
+
+    // Metal objects
+    var pipelineState: MTLRenderPipelineState
+    var commandQueue: MTLCommandQueue
+    var currentFrameTexture: MTLTexture?
+
+    private let renderSemaphore = DispatchSemaphore(value: 6)
+    private let eyeRenderingConfigurations: [Eye: EyeRenderingConfiguration]
+
+    // The current viewport size
+    var viewportSize: CGSize = CGSize()
+
+    // Flag for viewport size changes
+    var viewportSizeDidChange: Bool = false
+
+    init(session: ARSession, metalDevice device: MTLDevice, renderDestination: RenderDestinationProvider) throws {
+        self.session = session
+        self.device = device
+        self.renderDestination = renderDestination
+
+        guard let commandQueue = device.makeCommandQueue() else {
+            throw CVError(code: 42)
+        }
+        self.commandQueue = commandQueue
+
+        var cacheOutput: CVMetalTextureCache?
+        let code = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &cacheOutput)
+
+        guard let cache = cacheOutput else {
+            throw CVError(code: code)
+        }
+
+        self.textureCache = cache
+
+        // Load and compile the shaders
+        let defaultLibrary = device.makeDefaultLibrary()
+        guard let vertexFunction = defaultLibrary?.makeFunction(name: "ar_vertex_main") else {
+            print("Missing vertex function")
+            throw CVError(code: 42)
+        }
+        guard let fragmentFunction = defaultLibrary?.makeFunction(name: "ar_fragment_main") else {
+            print("Missing fragment function")
+            throw CVError(code: 42)
+        }
+
+        let pipelineDescriptor = MTLRenderPipelineDescriptor()
+        pipelineDescriptor.vertexFunction = vertexFunction
+        pipelineDescriptor.fragmentFunction = fragmentFunction
+        pipelineDescriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat
+
+        pipelineState = try device.makeRenderPipelineState(descriptor: pipelineDescriptor)
+
+        // TODO: Finish this
+        eyeRenderingConfigurations = [:]
+//        eyeRenderingConfigurations = [
+//            .left: .init(texture: any MTLTexture),
+//            .right: .init(texture: any MTLTexture),
+//        ]
+    }
+
+    func drawRectResized(size: CGSize) {
+        viewportSize = size
+        viewportSizeDidChange = true
+    }
+
+    func update() {
+        if let pixelBuffer = session.currentFrame?.capturedImage {
+            currentFrameTexture = texture(from: pixelBuffer)
+            draw()
+        }
+    }
+
+    private func draw() {
+        guard let currentDrawable = renderDestination.currentDrawable,
+              let renderPassDescriptor = renderDestination.currentRenderPassDescriptor else {
+            return
+        }
+
+        // Create a command buffer
+        let commandBuffer = commandQueue.makeCommandBuffer()
+
+        // Create a render command encoder
+        let renderEncoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor)
+        renderEncoder?.setRenderPipelineState(pipelineState)
+
+        // Set the texture to the render encoder
+        if let texture = currentFrameTexture {
+            renderEncoder?.setFragmentTexture(texture, index: 0)
+        }
+
+        // Draw a quad (assuming vertex buffer and other setup is done)
+        // renderEncoder?.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
+
+        // Finalize rendering
+        renderEncoder?.endEncoding()
+        commandBuffer?.present(currentDrawable)
+        commandBuffer?.commit()
+    }
+
+    private func texture(from pixelBuffer: CVPixelBuffer) -> MTLTexture? {
+        var cvMetalTexture: CVMetalTexture?
+        let width = CVPixelBufferGetWidth(pixelBuffer)
+        let height = CVPixelBufferGetHeight(pixelBuffer)
+        let pixelFormat = MTLPixelFormat.bgra8Unorm
+
+        let result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvMetalTexture)
+
+        guard result == kCVReturnSuccess, let metalTexture = cvMetalTexture else {
+            return nil
+        }
+
+        return CVMetalTextureGetTexture(metalTexture)
+    }
+}

Sources/MetalScope/StereoView/StereoCamera.swift

@@ -0,0 +1,54 @@
+//
+//  StereoCamera.swift
+//
+//
+//  Created by Mathijs Bernson on 18/06/2024.
+//
+
+import Foundation
+import SwiftUI
+import SceneKit
+
+@available(iOS 13.0, *)
+public struct StereoCamera: UIViewControllerRepresentable {
+    let format: MediaFormat
+    let stereoParameters: StereoParameters
+    let technique: SCNTechnique?
+
+    public init(
+        format: MediaFormat = .mono,
+        parameters: StereoParameters = StereoParameters(),
+        technique: SCNTechnique? = nil
+    ) {
+        self.format = format
+        self.stereoParameters = parameters
+        self.technique = technique
+    }
+
+    public func makeUIViewController(context: Context) -> StereoCameraViewController {
+        #if arch(arm) || arch(arm64)
+        let controller = StereoCameraViewController(device: context.coordinator.metalDevice)
+        #else
+        let controller = StereoCameraViewController()
+        #endif
+        return controller
+    }
+
+    public func updateUIViewController(_ controller: StereoCameraViewController, context: Context) {
+    }
+
+    public static func dismantleUIViewController(_ controller: StereoCameraViewController, coordinator: Coordinator) {
+    }
+
+    public func makeCoordinator() -> Coordinator {
+        Coordinator()
+    }
+
+    public class Coordinator {
+        let metalDevice: MTLDevice
+
+        init() {
+            metalDevice = MTLCreateSystemDefaultDevice()!
+        }
+    }
+}

Sources/MetalScope/StereoView/StereoCameraViewController.swift

@@ -0,0 +1,136 @@
+//
+//  StereoCameraViewController.swift
+//
+//
+//  Created by Mathijs Bernson on 18/06/2024.
+//
+
+import Foundation
+import UIKit
+import MetalKit
+import ARKit
+
+extension MTKView: RenderDestinationProvider {
+}
+
+public class StereoCameraViewController: UIViewController, MTKViewDelegate, ARSessionDelegate {
+    var device: MTLDevice?
+    let session = ARSession()
+    var renderer: StereoARRenderer?
+
+    init(device: MTLDevice) {
+        self.device = device
+        super.init(nibName: nil, bundle: nil)
+    }
+
+    init() {
+        super.init(nibName: nil, bundle: nil)
+    }
+
+    required init?(coder: NSCoder) {
+        super.init(coder: coder)
+    }
+
+    public override func loadView() {
+        view = MTKView(frame: .zero, device: device)
+    }
+
+    public override func viewDidLoad() {
+        super.viewDidLoad()
+
+        // Set the view's delegate
+        session.delegate = self
+
+        // Set the view to use the default device
+        if let view = self.view as? MTKView {
+            view.backgroundColor = UIColor.clear
+            view.delegate = self
+
+            guard view.device != nil else {
+                print("Metal is not supported on this device")
+                return
+            }
+
+            do {
+                // Configure the renderer to draw to the view
+                let renderer = try StereoARRenderer(session: session, metalDevice: view.device!, renderDestination: view)
+                renderer.drawRectResized(size: view.bounds.size)
+                self.renderer = renderer
+            } catch {
+                print("Setup error: \(error.localizedDescription)")
+            }
+        }
+
+        let tapGesture = UITapGestureRecognizer(target: self, action: #selector(handleTap))
+        view.addGestureRecognizer(tapGesture)
+    }
+
+    public override func viewWillAppear(_ animated: Bool) {
+        super.viewWillAppear(animated)
+
+        // Create a session configuration
+        let configuration = ARWorldTrackingConfiguration()
+
+        // Run the view's session
+        session.run(configuration)
+    }
+
+    public override func viewWillDisappear(_ animated: Bool) {
+        super.viewWillDisappear(animated)
+
+        // Pause the view's session
+        session.pause()
+    }
+
+    @objc func handleTap(gestureRecognize: UITapGestureRecognizer) {
+        // Create anchor using the camera's current position
+        if let currentFrame = session.currentFrame {
+
+            // Create a transform with a translation of 0.2 meters in front of the camera
+            var translation = matrix_identity_float4x4
+            translation.columns.3.z = -0.2
+            let transform = simd_mul(currentFrame.camera.transform, translation)
+
+            // Add a new anchor to the session
+            let anchor = ARAnchor(transform: transform)
+            session.add(anchor: anchor)
+        }
+    }
+
+    // MARK: - MTKViewDelegate
+
+    // Called whenever view changes orientation or layout is changed
+    public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
+        renderer?.drawRectResized(size: size)
+    }
+
+    // Called whenever the view needs to render
+    public func draw(in view: MTKView) {
+        renderer?.update()
+    }
+
+    // MARK: - ARSessionDelegate
+
+    public func session(_ session: ARSession, didFailWithError error: Error) {
+        // Present an error message to the user
+        presentError(error)
+    }
+
+    public func sessionWasInterrupted(_ session: ARSession) {
+        // Inform the user that the session has been interrupted, for example, by presenting an overlay
+    }
+
+    public func sessionInterruptionEnded(_ session: ARSession) {
+        // Reset tracking and/or remove existing anchors if consistent tracking is required
+    }
+
+    func presentError(_ error: Error) {
+        let alert = UIAlertController(
+            title: "Error occurred",
+            message: error.localizedDescription,
+            preferredStyle: .alert
+        )
+        alert.addAction(UIAlertAction(title: "Ok", style: .cancel))
+        present(alert, animated: true)
+    }
+}

Sources/MetalScope/StereoView/StereoRenderer.swift

@@ -149,14 +149,12 @@ extension MTLCommandBufferStatus: CustomStringConvertible {
     }
 }
 
-private extension StereoRenderer {
-    final class EyeRenderingConfiguration {
-        let texture: MTLTexture
-        var pointOfView: SCNNode?
+final class EyeRenderingConfiguration {
+    let texture: MTLTexture
+    var pointOfView: SCNNode?
 
-        init(texture: MTLTexture) {
-            self.texture = texture
-        }
+    init(texture: MTLTexture) {
+        self.texture = texture
     }
 }