A visionOS RealityView view modifier that handles drag gestures

WindowBarGestureModifier.swift

//
//  WindowBarGestureModifier.swift
//
//  Created by Drew Olbrich on 10/25/23.
//  Copyright © 2023 Lunar Skydiving LLC. All rights reserved.
//
//  MIT License
//  
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//  
//  The above copyright notice and this permission notice shall be included in all
//  copies or substantial portions of the Software.
//  
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//  SOFTWARE.
//

import SwiftUI
import RealityKit

extension View {
    
    /// A view modifier that works with `WindowBarEntity` and `WindowBarComponent` to
    /// let the user move `SceneEntityHierarchyController/scenePlacementEntity` around
    /// in the immersive space.
    ///
    /// This view modifier attempts to model the behavior of the visionOS window bar
    /// that appears next to volumetric window groups.
    func windowBarGestureModifier() -> some View {
        self.modifier(WindowBarGestureViewModifier())
    }
    
}

private struct WindowBarGestureViewModifier: ViewModifier {
    
    @Environment(MainNavigationViewModel.self) private var mainNavigationViewModel

    @State private var isDraggingWindowBar = false

    /// `true` if the gesture was cancelled because the immersion style transitioned to
    /// a non-mixed state during the gesture.
    ///
    /// When the immersion style is not mixed, the window bar is hidden and dragging the
    /// scene is disabled. Otherwise, the user would see the immersive space background
    /// move, which would be nauseating.
    @State private var isCancelled = false

    /// The vector from `startLocation3D` to `scenePlacementEntity` in world space.
    @State private var locationToScenePlacementEntityInWorld: SIMD3<Float> = .zero
    
    /// The desired orientation for the scene in world space, such that it faces the camera.
    @State private var targetScenePlacementEntityOrientationInWorld: simd_quatf?

    /// The current elapsed time of the animation.
    @State private var targetScenePlacementOrientationAnimationTime: TimeInterval = 0
    
    /// The intended duration of the animation.
    @State private var targetScenePlacementOrientationAnimationDuration: TimeInterval = 0

    func body(content: Content) -> some View {        
        content
            .gesture(DragGesture(minimumDistance: 0)
                .targetedToEntity(where: .has(WindowBarComponent.self))
                .onChanged { dragGestureValue in
                    guard let sceneViewProxyController = SceneViewProxyControllerManager.shared.sceneViewProxyController else {
                        assertionFailure("sceneViewProxyController is undefined")
                        return
                    }
                    
                    guard let cameraTransform = SceneCameraTransformManager.shared.cameraTransform else {
                        assertionFailure("cameraTransform is undefined")
                        return
                    }

                    let sceneOpacityEntity = sceneViewProxyController.sceneOpacityEntity
                    let windowBarEntity = sceneViewProxyController.windowBarEntity
                    
                    if !isDraggingWindowBar {
                        isCancelled = false

                        // While the user is dragging the scene around, make it transparent. This matches
                        // the visionOS window drag gesture behavior and is an Apple spatial computing
                        // design guideline.
                        sceneOpacityEntity.setOpacity(0.5, animated: true)
                        
                        // In visionOS 1.0 beta 4, when the user moves a volume around, its window bar
                        // temporarily disappears. 
                        //
                        // Perhaps this gives the user the feeling that they are moving the volume's
                        // contents around instead of the window bar. Or, perhaps this is done because if
                        // the volume turns to face the user when they grab the window bar, it feels weird
                        // if the window bar rotates away from the point in space where the user grabbed it.
                        //
                        // Regardless of the reason, we match the behavior of visionOS for consistency.
                        windowBarEntity.didBeginDraggingWindowBar(animated: true)
                                                
                        isDraggingWindowBar = true

                        beginGestureTransform(for: sceneViewProxyController, dragGestureValue: dragGestureValue, cameraTransform: cameraTransform)
                        
                        updateGestureTransform(for: sceneViewProxyController, dragGestureValue: dragGestureValue, cameraTransform: cameraTransform)
                    }
                    
                    guard !isCancelled else {
                        // The gesture was cancelled because the immersion style transitioned to a
                        // non-mixed state during the gesture.
                        return
                    }
                    
                    // This is necessary even for the `!isDraggingWindowBar` case, because otherwise
                    // `targetScenePlacementEntityOrientationInWorld` won't be assigned and the scene
                    // won't auto-rotate if the user touches the window bar but doesn't start a drag
                    // gesture.
                    updateGestureTransform(for: sceneViewProxyController, dragGestureValue: dragGestureValue, cameraTransform: cameraTransform)
                }
                .onEnded { dragGestureValue in
                    isDraggingWindowBar = false
                    isCancelled = false

                    guard let sceneViewProxyController = SceneViewProxyControllerManager.shared.sceneViewProxyController else {
                        assertionFailure("sceneViewProxyController is undefined")
                        return
                    }

                    let sceneOpacityEntity = sceneViewProxyController.sceneOpacityEntity
                    let windowBarEntity = sceneViewProxyController.windowBarEntity

                    sceneOpacityEntity.setOpacity(1, animated: true)
                    windowBarEntity.didEndDraggingWindowBar(animated: true)

                    endGestureTransform(for: sceneViewProxyController, dragGestureValue: dragGestureValue)
                }
            )
            .onChange(of: mainNavigationViewModel.sceneWindowBarIsVisible) { _, sceneWindowBarIsVisible in
                if !sceneWindowBarIsVisible {
                    // When the immersion style transitions from mixed to non-mixed and the window bar
                    // becomes hidden, cancel the gesture. Otherwise, the user would see the immersive
                    // space background move, which would be nauseating.
                    isCancelled = true
                }
            }
    }
    
    private func beginGestureTransform(for sceneViewProxyController: SceneViewProxyController, dragGestureValue: EntityTargetValue<DragGesture.Value>, cameraTransform: Transform) {
        /// The geometry of the scene is parented to this entity. We'll use this to update
        /// the position and orientation of the scene as the user drags it around.
        let scenePlacementEntity = sceneViewProxyController.scenePlacementEntity
        
        /// The custom window bar in front of the scene that the user drags around.
        let windowBarEntity = sceneViewProxyController.windowBarEntity

        /// The start location of the drag gesture in world space.
        let startLocation3D = dragGestureValue.convert(dragGestureValue.startLocation3D, from: .local, to: .scene)

        /// The position of the scene's origin in world space.
        let scenePlacementEntityPositionInWorld = scenePlacementEntity.convert(position: .zero, to: nil)
        
        /// The vector from `startLocation3D` to `scenePlacementEntity` in world space.
        locationToScenePlacementEntityInWorld = scenePlacementEntityPositionInWorld - startLocation3D
        
        /// The desired orientation for the scene in world space, such that it faces the camera.
        targetScenePlacementEntityOrientationInWorld = nil

        /// The scene's initial +Z axis at the start of the drag gesture, in world space,
        /// projected onto the XZ plane.
        let sceneZAxis = normalize(scenePlacementEntity.convert(direction: [0, 0, 1], to: nil).withY(0))
        
        /// A vector from the scene's origin in world space to the camera, projected onto the XZ plane.
        let sceneToCamera = normalize((cameraTransform.translation - scenePlacementEntityPositionInWorld).withY(0))
        
        /// How much the scene is angled away from the camera, in radians.
        let angle = acos(min(1, dot(sceneZAxis, sceneToCamera)))
                
        targetScenePlacementOrientationAnimationTime = 0
        targetScenePlacementOrientationAnimationDuration = 2*TimeInterval(angle)
        
        /// Specify a closure that's called once for each frame during the drag gesture.
        /// We'll use this to animate the scene's orientation over time so that it
        /// continuously turns to face the camera if it's not facing the camera already.
        windowBarEntity.update = { deltaTime in
            updateOrientation(for: sceneViewProxyController, deltaTime: deltaTime)
        }
    }
    
    private func updateGestureTransform(for sceneViewProxyController: SceneViewProxyController, dragGestureValue: EntityTargetValue<DragGesture.Value>, cameraTransform: Transform) {
        let scenePlacementEntity = sceneViewProxyController.scenePlacementEntity
        let windowBarEntity = sceneViewProxyController.windowBarEntity

        /// The current location of the drag gesture in world space.
        let location3D = dragGestureValue.convert(dragGestureValue.location3D, from: .local, to: .scene)
        
        /// The new position of `scenePlacementEntity` in world space, reflecting the
        /// updated drag gesture location.
        let scenePlacementEntityPositionInWorld = location3D + locationToScenePlacementEntityInWorld
        
        /// The desired transform for `scenePlacementEntity` in world space, positioned at
        /// `scenePlacementEntityPositionInWorld` and facing the camera at
        /// `cameraTransform`.
        let sceneTransformInWorld = sceneTransformInWorld(withCameraTransform: cameraTransform, scenePlacementEntityPositionInWorld: scenePlacementEntityPositionInWorld)
        
        /// Update the world space position of the scene.
        scenePlacementEntity.setPosition(sceneTransformInWorld.translation, relativeTo: nil)
        
        /// The orientation in world space that we'd like the scene to eventually have. The
        /// scene will animate to this orientation over time.
        let targetScenePlacementEntityOrientationInWorld = sceneTransformInWorld.rotation
        
        if windowBarEntity.update != nil {
            // The scene's orientation is already animating. Update the target orientation of
            // the animation to reflect the updated gesture.
            self.targetScenePlacementEntityOrientationInWorld = targetScenePlacementEntityOrientationInWorld
        } else {
            // The scene has already reached the desired orientation in the past, so from that
            // point forward, we instantaneously update it without animation. Otherwise, the
            // gesture would feel loose.
            scenePlacementEntity.setOrientation(targetScenePlacementEntityOrientationInWorld, relativeTo: nil)
        }
    }
    
    /// The desired transform for `scenePlacementEntity` in world space, positioned at
    /// `scenePlacementEntityPositionInWorld` and facing the camera at
    /// `cameraTransform`.
    private func sceneTransformInWorld(withCameraTransform cameraTransform: Transform, scenePlacementEntityPositionInWorld: SIMD3<Float>) -> Transform {
        let origin = scenePlacementEntityPositionInWorld
        let zAxis = normalize((cameraTransform.translation - scenePlacementEntityPositionInWorld).withY(0))
        let yAxis: SIMD3<Float> = [0, 1, 0]
        let xAxis = simd_cross(yAxis, zAxis)

        return Transform(columns: (xAxis, yAxis, zAxis, origin))
    }
    
    /// This method is called on every frame to animate the scene as it turns to face the camera.
    private func updateOrientation(for sceneViewProxyController: SceneViewProxyController, deltaTime: TimeInterval) {
        guard let targetScenePlacementEntityOrientationInWorld else {
            return
        }
        
        let scenePlacementEntity = sceneViewProxyController.scenePlacementEntity
        let windowBarEntity = sceneViewProxyController.windowBarEntity

        let currentOrientationInWorld = scenePlacementEntity.orientation(relativeTo: nil)
        
        targetScenePlacementOrientationAnimationTime = min(targetScenePlacementOrientationAnimationDuration, targetScenePlacementOrientationAnimationTime + deltaTime)
        
        let t: Float
        if targetScenePlacementOrientationAnimationDuration == 0 {
            t = 1
        } else {
            t = min(1, Float(targetScenePlacementOrientationAnimationTime/targetScenePlacementOrientationAnimationDuration))
        }
        
        let orientation = simd_slerp(currentOrientationInWorld, targetScenePlacementEntityOrientationInWorld, t)
        
        scenePlacementEntity.setOrientation(orientation, relativeTo: nil)
        
        if t == 1 {
            // Once the orientation animation completes, there's no need to call
            // `updateOrientation(for:deltaTime:)` on every frame.
            windowBarEntity.update = nil
        }
    }
    
    private func endGestureTransform(for sceneViewProxyController: SceneViewProxyController, dragGestureValue: EntityTargetValue<DragGesture.Value>) {
        let windowBarEntity = sceneViewProxyController.windowBarEntity
        
        windowBarEntity.update = nil
        
        targetScenePlacementEntityOrientationInWorld = nil
        targetScenePlacementOrientationAnimationTime = 0
        targetScenePlacementOrientationAnimationDuration = 0
    }
    
}

private extension Transform {
    
    /// Initializes a `Transform` with 4 column vectors.
    init(columns: (xAxis: SIMD3<Float>, yAxis: SIMD3<Float>, zAxis: SIMD3<Float>, origin: SIMD3<Float>)) {
        self.init(matrix: simd_float4x4(columns: (
            SIMD4<Float>(columns.xAxis, 0),
            SIMD4<Float>(columns.yAxis, 0),
            SIMD4<Float>(columns.zAxis, 0),
            SIMD4<Float>(columns.origin, 1)
        )))
    }
    
}

Thanks! That makes sense. Even without it running this gives me some interesting ideas so thank you for posting. I've got a pretty different structure anyway and the main thing I was missing was converting between swiftUI and RealityKit coordinate spaces.