From 8064e80093fa5df6ff2e7e05e950c4458841931f Mon Sep 17 00:00:00 2001 From: Mathijs Bernson Date: Mon, 21 Oct 2024 11:44:37 +0200 Subject: [PATCH] Stereo AR view wip --- .../StereoView/StereoARRenderer.swift | 138 ++++++++++++++++++ .../MetalScope/StereoView/StereoCamera.swift | 54 +++++++ .../StereoCameraViewController.swift | 136 +++++++++++++++++ .../StereoView/StereoRenderer.swift | 12 +- 4 files changed, 333 insertions(+), 7 deletions(-) create mode 100644 Sources/MetalScope/StereoView/StereoARRenderer.swift create mode 100644 Sources/MetalScope/StereoView/StereoCamera.swift create mode 100644 Sources/MetalScope/StereoView/StereoCameraViewController.swift diff --git a/Sources/MetalScope/StereoView/StereoARRenderer.swift b/Sources/MetalScope/StereoView/StereoARRenderer.swift new file mode 100644 index 0000000..c2ca308 --- /dev/null +++ b/Sources/MetalScope/StereoView/StereoARRenderer.swift @@ -0,0 +1,138 @@ +// +// StereoARRenderer.swift +// +// +// Created by Mathijs Bernson on 18/06/2024. +// + +import Metal +import ARKit + +protocol RenderDestinationProvider { + var currentRenderPassDescriptor: MTLRenderPassDescriptor? { get } + var currentDrawable: CAMetalDrawable? { get } + var colorPixelFormat: MTLPixelFormat { get set } + var depthStencilPixelFormat: MTLPixelFormat { get set } + var sampleCount: Int { get set } +} + +internal final class StereoARRenderer { + let session: ARSession + let device: MTLDevice + var renderDestination: RenderDestinationProvider + + let textureCache: CVMetalTextureCache + + // Metal objects + var pipelineState: MTLRenderPipelineState + var commandQueue: MTLCommandQueue + var currentFrameTexture: MTLTexture? + + private let renderSemaphore = DispatchSemaphore(value: 6) + private let eyeRenderingConfigurations: [Eye: EyeRenderingConfiguration] + + // The current viewport size + var viewportSize: CGSize = CGSize() + + // Flag for viewport size changes + var viewportSizeDidChange: Bool = false + + init(session: ARSession, metalDevice device: MTLDevice, renderDestination: RenderDestinationProvider) throws { + self.session = session + self.device = device + self.renderDestination = renderDestination + + guard let commandQueue = device.makeCommandQueue() else { + throw CVError(code: 42) + } + self.commandQueue = commandQueue + + var cacheOutput: CVMetalTextureCache? + let code = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &cacheOutput) + + guard let cache = cacheOutput else { + throw CVError(code: code) + } + + self.textureCache = cache + + // Load and compile the shaders + let defaultLibrary = device.makeDefaultLibrary() + guard let vertexFunction = defaultLibrary?.makeFunction(name: "ar_vertex_main") else { + print("Missing vertex function") + throw CVError(code: 42) + } + guard let fragmentFunction = defaultLibrary?.makeFunction(name: "ar_fragment_main") else { + print("Missing fragment function") + throw CVError(code: 42) + } + + let pipelineDescriptor = MTLRenderPipelineDescriptor() + pipelineDescriptor.vertexFunction = vertexFunction + pipelineDescriptor.fragmentFunction = fragmentFunction + pipelineDescriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat + + pipelineState = try device.makeRenderPipelineState(descriptor: pipelineDescriptor) + + // TODO: Finish this + eyeRenderingConfigurations = [:] +// eyeRenderingConfigurations = [ +// .left: .init(texture: any MTLTexture), +// .right: .init(texture: any MTLTexture), +// ] + } + + func drawRectResized(size: CGSize) { + viewportSize = size + viewportSizeDidChange = true + } + + func update() { + if let pixelBuffer = session.currentFrame?.capturedImage { + currentFrameTexture = texture(from: pixelBuffer) + draw() + } + } + + private func draw() { + guard let currentDrawable = renderDestination.currentDrawable, + let renderPassDescriptor = renderDestination.currentRenderPassDescriptor else { + return + } + + // Create a command buffer + let commandBuffer = commandQueue.makeCommandBuffer() + + // Create a render command encoder + let renderEncoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor) + renderEncoder?.setRenderPipelineState(pipelineState) + + // Set the texture to the render encoder + if let texture = currentFrameTexture { + renderEncoder?.setFragmentTexture(texture, index: 0) + } + + // Draw a quad (assuming vertex buffer and other setup is done) + // renderEncoder?.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + + // Finalize rendering + renderEncoder?.endEncoding() + commandBuffer?.present(currentDrawable) + commandBuffer?.commit() + } + + private func texture(from pixelBuffer: CVPixelBuffer) -> MTLTexture? { + var cvMetalTexture: CVMetalTexture? + let width = CVPixelBufferGetWidth(pixelBuffer) + let height = CVPixelBufferGetHeight(pixelBuffer) + let pixelFormat = MTLPixelFormat.bgra8Unorm + + let result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvMetalTexture) + + guard result == kCVReturnSuccess, let metalTexture = cvMetalTexture else { + return nil + } + + return CVMetalTextureGetTexture(metalTexture) + } +} diff --git a/Sources/MetalScope/StereoView/StereoCamera.swift b/Sources/MetalScope/StereoView/StereoCamera.swift new file mode 100644 index 0000000..fa54a07 --- /dev/null +++ b/Sources/MetalScope/StereoView/StereoCamera.swift @@ -0,0 +1,54 @@ +// +// StereoCamera.swift +// +// +// Created by Mathijs Bernson on 18/06/2024. +// + +import Foundation +import SwiftUI +import SceneKit + +@available(iOS 13.0, *) +public struct StereoCamera: UIViewControllerRepresentable { + let format: MediaFormat + let stereoParameters: StereoParameters + let technique: SCNTechnique? + + public init( + format: MediaFormat = .mono, + parameters: StereoParameters = StereoParameters(), + technique: SCNTechnique? = nil + ) { + self.format = format + self.stereoParameters = parameters + self.technique = technique + } + + public func makeUIViewController(context: Context) -> StereoCameraViewController { + #if arch(arm) || arch(arm64) + let controller = StereoCameraViewController(device: context.coordinator.metalDevice) + #else + let controller = StereoCameraViewController() + #endif + return controller + } + + public func updateUIViewController(_ controller: StereoCameraViewController, context: Context) { + } + + public static func dismantleUIViewController(_ controller: StereoCameraViewController, coordinator: Coordinator) { + } + + public func makeCoordinator() -> Coordinator { + Coordinator() + } + + public class Coordinator { + let metalDevice: MTLDevice + + init() { + metalDevice = MTLCreateSystemDefaultDevice()! + } + } +} diff --git a/Sources/MetalScope/StereoView/StereoCameraViewController.swift b/Sources/MetalScope/StereoView/StereoCameraViewController.swift new file mode 100644 index 0000000..a011428 --- /dev/null +++ b/Sources/MetalScope/StereoView/StereoCameraViewController.swift @@ -0,0 +1,136 @@ +// +// StereoCameraViewController.swift +// +// +// Created by Mathijs Bernson on 18/06/2024. +// + +import Foundation +import UIKit +import MetalKit +import ARKit + +extension MTKView: RenderDestinationProvider { +} + +public class StereoCameraViewController: UIViewController, MTKViewDelegate, ARSessionDelegate { + var device: MTLDevice? + let session = ARSession() + var renderer: StereoARRenderer? + + init(device: MTLDevice) { + self.device = device + super.init(nibName: nil, bundle: nil) + } + + init() { + super.init(nibName: nil, bundle: nil) + } + + required init?(coder: NSCoder) { + super.init(coder: coder) + } + + public override func loadView() { + view = MTKView(frame: .zero, device: device) + } + + public override func viewDidLoad() { + super.viewDidLoad() + + // Set the view's delegate + session.delegate = self + + // Set the view to use the default device + if let view = self.view as? MTKView { + view.backgroundColor = UIColor.clear + view.delegate = self + + guard view.device != nil else { + print("Metal is not supported on this device") + return + } + + do { + // Configure the renderer to draw to the view + let renderer = try StereoARRenderer(session: session, metalDevice: view.device!, renderDestination: view) + renderer.drawRectResized(size: view.bounds.size) + self.renderer = renderer + } catch { + print("Setup error: \(error.localizedDescription)") + } + } + + let tapGesture = UITapGestureRecognizer(target: self, action: #selector(handleTap)) + view.addGestureRecognizer(tapGesture) + } + + public override func viewWillAppear(_ animated: Bool) { + super.viewWillAppear(animated) + + // Create a session configuration + let configuration = ARWorldTrackingConfiguration() + + // Run the view's session + session.run(configuration) + } + + public override func viewWillDisappear(_ animated: Bool) { + super.viewWillDisappear(animated) + + // Pause the view's session + session.pause() + } + + @objc func handleTap(gestureRecognize: UITapGestureRecognizer) { + // Create anchor using the camera's current position + if let currentFrame = session.currentFrame { + + // Create a transform with a translation of 0.2 meters in front of the camera + var translation = matrix_identity_float4x4 + translation.columns.3.z = -0.2 + let transform = simd_mul(currentFrame.camera.transform, translation) + + // Add a new anchor to the session + let anchor = ARAnchor(transform: transform) + session.add(anchor: anchor) + } + } + + // MARK: - MTKViewDelegate + + // Called whenever view changes orientation or layout is changed + public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { + renderer?.drawRectResized(size: size) + } + + // Called whenever the view needs to render + public func draw(in view: MTKView) { + renderer?.update() + } + + // MARK: - ARSessionDelegate + + public func session(_ session: ARSession, didFailWithError error: Error) { + // Present an error message to the user + presentError(error) + } + + public func sessionWasInterrupted(_ session: ARSession) { + // Inform the user that the session has been interrupted, for example, by presenting an overlay + } + + public func sessionInterruptionEnded(_ session: ARSession) { + // Reset tracking and/or remove existing anchors if consistent tracking is required + } + + func presentError(_ error: Error) { + let alert = UIAlertController( + title: "Error occurred", + message: error.localizedDescription, + preferredStyle: .alert + ) + alert.addAction(UIAlertAction(title: "Ok", style: .cancel)) + present(alert, animated: true) + } +} diff --git a/Sources/MetalScope/StereoView/StereoRenderer.swift b/Sources/MetalScope/StereoView/StereoRenderer.swift index 59d9a88..1dae16d 100644 --- a/Sources/MetalScope/StereoView/StereoRenderer.swift +++ b/Sources/MetalScope/StereoView/StereoRenderer.swift @@ -149,14 +149,12 @@ extension MTLCommandBufferStatus: CustomStringConvertible { } } -private extension StereoRenderer { - final class EyeRenderingConfiguration { - let texture: MTLTexture - var pointOfView: SCNNode? +final class EyeRenderingConfiguration { + let texture: MTLTexture + var pointOfView: SCNNode? - init(texture: MTLTexture) { - self.texture = texture - } + init(texture: MTLTexture) { + self.texture = texture } }