RealityView that detects finger gun gestures and uses ray casting to interact with virtual objects.

HandTrackingRaycastView.swift

import SwiftUI
import RealityKit
import ARKit
import HandVector

struct HandTrackingRaycastView: View {
    @State var jointPositions: [HandSkeleton.JointName: SIMD3<Float>] = [:]
    @State var sphereEntity: ModelEntity?
    @State var laserBeamEntity: ModelEntity?
    @State var gunEntity: ModelEntity?
    let sphereRadius: Float = 0.1
    
    var latestHandTracking: HandVectorManager = .init(left: nil, right: nil)
    let handInfo = String.gunReadyPosition.toModel(HVHandJsonModel.self)!.convertToHVHandInfo()
    let thresholdForFingerGunDetection: Float = 0.95
    
    @State private var isShowingGun: Bool = false
    
    var body: some View {
        RealityView { content in
            let sphere = createSphere()
            content.add(sphere)
            let cylinder = createCylinder()
            sphereEntity = sphere
            laserBeamEntity = cylinder
            content.add(cylinder)
            
            let gunEntity = try! await loadGunEntity()
            content.add(gunEntity)
            self.gunEntity = gunEntity
        }
        .task {
            await setupHandTracking()
        }
    }
    
    func createSphere() -> ModelEntity {
        let sphere = ModelEntity(mesh: .generateSphere(radius: sphereRadius),
                                 materials: [SimpleMaterial(color: .blue, isMetallic: false)])
        sphere.collision = CollisionComponent(shapes: [.generateSphere(radius: sphereRadius)])
        sphere.position = SIMD3<Float>(0.0, 1.3, -2.0)
        sphere.position.z = -2.0
        return sphere
    }
    
    func createCylinder() -> ModelEntity {
        let cylinderMesh = MeshResource.generateCylinder(height: 1, radius: 0.002)
        let material = SimpleMaterial(color: .red, isMetallic: false)
        let entity = ModelEntity(mesh: cylinderMesh, materials: [material])
        entity.components.set(OpacityComponent(opacity: 0.0))
        return entity
    }
    
    private func setupHandTracking() async {
        let session = ARKitSession()
        let handTracking = HandTrackingProvider()
        
        do {
            try await session.run([handTracking])
            
            for await update in handTracking.anchorUpdates {
                let handAnchor = update.anchor
                if handAnchor.chirality == .right {
                    updateJointPositions(for: handAnchor)
                    updateForHandPosition()
                    
                    let handInfo = latestHandTracking.generateHandInfo(from: handAnchor)
                    if let handInfo {
                        await latestHandTracking.updateHandSkeletonEntity(from: handInfo)
                        
                    }
                }
                
                
                let averageAndEachRightScores = latestHandTracking.rightHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: handInfo!)
                let average = averageAndEachRightScores?.0
                if thresholdForFingerGunDetection < average! {
                    print("got the finger gun!")
                    if !isShowingGun {
                        guard let debugCylinder = laserBeamEntity else { return }
                        debugCylinder.components.set(OpacityComponent(opacity: 1.0))
                        gunEntity?.components.set(OpacityComponent(opacity: 1.0))
                        isShowingGun = true
                    }
                } else {
                    if isShowingGun {
                        guard let debugCylinder = laserBeamEntity else { return }
                        debugCylinder.components.set(OpacityComponent(opacity: 0.0))
                        gunEntity?.components.set(OpacityComponent(opacity: 0.0))
                        isShowingGun = false
                    }
                }
            }
        } catch {
            print("Error setting up hand tracking: \(error)")
        }
    }
    
    func updateJointPositions(for handAnchor: HandAnchor) {
        jointPositions = Dictionary(uniqueKeysWithValues:
            HandSkeleton.JointName.allCases.compactMap { jointName in
                guard let joint = handAnchor.handSkeleton?.joint(jointName) else { return nil }
                let worldPosition = handAnchor.originFromAnchorTransform * joint.anchorFromJointTransform.columns.3
                return (jointName, SIMD3<Float>(worldPosition.x, worldPosition.y, worldPosition.z))
            }
        )
    }
    
    func updateForHandPosition() {
        guard let indexTipPosition = jointPositions[.indexFingerTip],
              let indexPIPPosition = jointPositions[.indexFingerIntermediateBase] else {
            return
        }
        let rayDirection = simd_normalize(indexTipPosition - indexPIPPosition)
        let rayLength: Float = 10 // 10 meters long ray
        let rayEnd = indexPIPPosition + rayDirection * rayLength
        
        updateSphere(rayStart: indexPIPPosition, rayDirection: rayDirection)
        updateCylinder(rayStart: indexPIPPosition, rayEnd: rayEnd)
    }
    
    func updateSphere(rayStart: SIMD3<Float>, rayDirection: SIMD3<Float>) {
        guard let sphere = sphereEntity else { return }
        let intersection = rayIntersectsSphere(rayStart: rayStart,
                                               rayDirection: rayDirection,
                                               sphereCenter: sphere.position,
                                               sphereRadius: sphereRadius)
            
        if intersection {
            sphere.model?.materials = [SimpleMaterial(color: .green, isMetallic: false)]
        } else {
            sphere.model?.materials = [SimpleMaterial(color: .blue, isMetallic: false)]
        }
    }
    
    func updateCylinder(rayStart: SIMD3<Float>, rayEnd: SIMD3<Float>) {
        guard let debugCylinder = laserBeamEntity else { return }
        positionAndOrientCylinder(debugCylinder, from: rayStart, to: rayEnd)
        guard let gunEntity = gunEntity else { return }
        updateGunPosition(gunEntity, from: rayStart, to: rayEnd)
    }
    
    private func rayIntersectsSphere(rayStart: SIMD3<Float>, rayDirection: SIMD3<Float>, sphereCenter: SIMD3<Float>, sphereRadius: Float) -> Bool {
        let originToCenter = rayStart - sphereCenter
        let directionMagnitudeSquared = simd_dot(rayDirection, rayDirection)
        let originToCenterProjection = 2.0 * simd_dot(originToCenter, rayDirection)
        let perpDistanceSquared = simd_dot(originToCenter, originToCenter) - sphereRadius * sphereRadius
        let discriminant = originToCenterProjection * originToCenterProjection - 4 * directionMagnitudeSquared * perpDistanceSquared
        return discriminant > 0
    }
    
    private func positionAndOrientCylinder(_ cylinder: ModelEntity, from start: SIMD3<Float>, to end: SIMD3<Float>) {
        let direction = end - start
        let distance = simd_length(direction)
        let midpoint = (start + end) / 2
        
        cylinder.position = midpoint
        cylinder.scale = [1, distance, 1]
        
        let yAxis = simd_normalize(direction)
        let xAxis = simd_normalize(simd_cross([0, 1, 0], yAxis))
        let zAxis = simd_cross(xAxis, yAxis)
        
        let rotationMatrix = simd_float3x3(columns: (xAxis, yAxis, zAxis))
        cylinder.transform.rotation = simd_quaternion(rotationMatrix)
    }
    
    func loadGunEntity(url: URL = URL(string: "https://matt54.github.io/Resources/laser_gun.usdz")!) async throws -> ModelEntity {
        let (downloadedURL, _) = try await URLSession.shared.download(from: url)
        let documentsDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first!
        let destinationURL = documentsDirectory.appendingPathComponent("downloadedModel.usdz")
        if FileManager.default.fileExists(atPath: destinationURL.path) {
            try FileManager.default.removeItem(at: destinationURL)
        }
        try FileManager.default.moveItem(at: downloadedURL, to: destinationURL)
        let entity = try await ModelEntity.init(contentsOf: destinationURL)
        entity.components.set(OpacityComponent(opacity: 0.0))
        return entity
    }
    
    
    func updateGunPosition(_ entity: ModelEntity, from start: SIMD3<Float>, to end: SIMD3<Float>) {
        let direction = end - start
        entity.position = start
        entity.scale = [0.1, 0.1, 0.1]
        
        let yAxis = simd_normalize(direction)
        let xAxis = simd_normalize(simd_cross([0, 1, 0], yAxis))
        let zAxis = simd_cross(xAxis, yAxis)
        
        // fix how the source is not oriented forward
        let rotationX = simd_quatf(angle: .pi*0.5, axis: [0, 0, 1])
        let rotationY = simd_quatf(angle: -.pi*0.5, axis: [1, 0, 0])
        
        let rotationMatrix = simd_float3x3(columns: (xAxis, yAxis, zAxis))
        entity.transform.rotation = simd_quaternion(rotationMatrix) * rotationX * rotationY
    }

}

let headPosition = SIMD3<Float>(0.0, 1.3, 0.0)

// TODO: make pull request with HandVector library to make this extension public
extension String {
    func toModel<T>(_ type: T.Type, using encoding: String.Encoding = .utf8) -> T? where T : Decodable {
        guard let data = self.data(using: encoding) else { return nil }
        do {
            return try JSONDecoder().decode(T.self, from: data)
        } catch {
            print(error)
        }
        return nil
    }
}

extension String {
    static let gunReadyPosition: String = """
{"transform":[[0.03681829,-0.35651627,0.9335633,0],[0.9981043,-0.03298368,-0.051959727,0],[0.049316857,0.9337067,0.35462606,0],[0.119505,1.058816,-0.3033591,1]],"name":"right","joints":[{"transform":[[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]],"name":"wrist","isTracked":true},{"isTracked":true,"transform":[[0.7483124,0.08903434,-0.6573445,0],[0.6232275,0.24503177,0.7426623,0],[0.22719273,-0.96541846,0.12787166,0],[-0.023037225,-0.011565208,0.019217536,1]],"name":"thumbKnuckle"},{"isTracked":true,"transform":[[0.9145838,0.03962274,-0.40245014,0],[0.3839136,0.22763534,0.8948699,0],[0.1270691,-0.9729398,0.19297989,0],[-0.06262271,-0.016338525,0.053467937,0.9999999]],"name":"thumbIntermediateBase"},{"transform":[[0.55769247,-0.09324432,-0.8247935,0],[0.82048357,0.21233602,0.5307732,0],[0.12564181,-0.97273785,0.19492361,0],[-0.09256029,-0.017590031,0.06606347,0.9999999]],"name":"thumbIntermediateTip","isTracked":true},{"isTracked":true,"transform":[[0.5576926,-0.09324434,-0.8247935,0],[0.8204833,0.21233588,0.5307733,0],[0.12564164,-0.9727376,0.1949236,0],[-0.10933509,-0.014713061,0.09040629,0.9999999]],"name":"thumbTip"},{"isTracked":true,"name":"indexFingerMetacarpal","transform":[[0.9873512,0.0007703505,-0.15854767,0],[-0.00018403005,0.99999326,0.0037125493,0],[0.15854947,-0.003636434,0.9873445,0],[-0.02504237,8.650124e-05,0.016333118,1]]},{"name":"indexFingerKnuckle","isTracked":true,"transform":[[0.9271389,0.1894835,0.3232792,0],[-0.20793396,0.9778683,0.023180438,0],[-0.31173214,-0.08871223,0.94601953,0],[-0.09699588,0.00033357737,0.02648428,1]]},{"isTracked":true,"transform":[[0.9415803,0.10521732,0.3199318,0],[-0.12780455,0.9905191,0.050381172,0],[-0.31159747,-0.08832667,0.94610006,0],[-0.13916928,-0.008303657,0.011948585,0.99999994]],"name":"indexFingerIntermediateBase"},{"transform":[[0.94819283,0.034945816,0.31576785,0],[-0.060861282,0.9955039,0.072583534,0],[-0.31181157,-0.08804125,0.9460563,0],[-0.16357674,-0.010998578,0.0036838902,0.99999994]],"isTracked":true,"name":"indexFingerIntermediateTip"},{"transform":[[0.94819283,0.0349457,0.315768,0],[-0.060861256,0.9955037,0.07258351,0],[-0.3118117,-0.08804125,0.9460561,0],[-0.18628134,-0.011772217,-0.0039144484,0.99999994]],"name":"indexFingerTip","isTracked":true},{"isTracked":true,"transform":[[0.9999436,0.00029446578,0.010613574,0],[-0.00033532665,0.9999925,0.0038510533,0],[-0.0106123155,-0.0038544233,0.99993616,0],[-0.027172834,0.0001347065,0.0036730468,1]],"name":"middleFingerMetacarpal"},{"isTracked":true,"name":"middleFingerKnuckle","transform":[[0.31107935,0.9494624,0.0418443,0],[-0.94966394,0.312256,-0.025200296,0],[-0.036992837,-0.031898756,0.99880624,0],[-0.095856436,0.00042155385,0.0019043083,0.99999994]]},{"transform":[[-0.97067386,0.23868601,-0.028664839,0],[-0.23642187,-0.9694015,-0.06607428,0],[-0.04355868,-0.057359572,0.99740297,0],[-0.10820865,-0.04728632,0.0028083322,0.9999999]],"name":"middleFingerIntermediateBase","isTracked":true},{"isTracked":true,"transform":[[-0.98711973,-0.1526101,-0.048010733,0],[0.1551362,-0.98640466,-0.05421107,0],[-0.039084814,-0.06096107,0.9973746,0],[-0.07775817,-0.05483128,0.004591465,0.9999999]],"name":"middleFingerIntermediateTip"},{"name":"middleFingerTip","isTracked":true,"transform":[[-0.98711985,-0.15260991,-0.04801078,0],[0.15513599,-0.9864048,-0.054211125,0],[-0.039084826,-0.06096117,0.99737483,0],[-0.055462185,-0.051387776,0.005703509,0.9999999]]},{"isTracked":true,"transform":[[0.9876838,0.00039916535,0.15646292,0],[-0.001013631,0.999992,0.0038470975,0],[-0.15646014,-0.00395833,0.98767626,0],[-0.02746223,-0.0013832152,-0.00899744,1]],"name":"ringFingerMetacarpal"},{"transform":[[-0.0012533537,0.9747956,-0.22309683,0],[-0.9960826,-0.020943103,-0.08591284,0],[-0.088419706,0.22211517,0.9710031,0],[-0.094665445,-0.0011133702,-0.020322772,0.99999994]],"isTracked":true,"name":"ringFingerKnuckle"},{"name":"ringFingerIntermediateBase","isTracked":true,"transform":[[-0.9931481,0.042646114,-0.10880357,0],[-0.0667791,-0.97115594,0.22890316,0],[-0.09590343,0.23460054,0.96734947,0],[-0.09184222,-0.04280009,-0.008270023,0.9999999]]},{"isTracked":true,"name":"ringFingerIntermediateTip","transform":[[-0.87231886,-0.48775947,0.033920035,0],[0.47993666,-0.84094685,0.24993832,0],[-0.093384825,0.23430538,0.9676676,0],[-0.06364327,-0.044113144,-0.004825471,0.9999999]]},{"transform":[[-0.8723191,-0.4877596,0.03391996,0],[0.4799366,-0.84094703,0.24993841,0],[-0.093384914,0.23430541,0.96766764,0],[-0.044957124,-0.033260565,-0.00605893,0.9999999]],"isTracked":true,"name":"ringFingerTip"},{"transform":[[0.9754438,0.0007679999,0.22024716,0],[-0.0016250212,0.9999917,0.003710361,0],[-0.22024249,-0.003977161,0.9754369,0],[-0.026689336,-0.0029354095,-0.023558915,1]],"isTracked":true,"name":"littleFingerMetacarpal"},{"name":"littleFingerKnuckle","transform":[[-0.021861674,0.9597742,-0.2799214,0],[-0.9730536,-0.08471092,-0.21445592,0],[-0.2295416,0.26769006,0.9357632,0],[-0.08487275,-0.0027343482,-0.037119243,1]],"isTracked":true},{"name":"littleFingerIntermediateBase","transform":[[-0.95487165,0.1191173,-0.2720872,0],[-0.18119034,-0.9594734,0.21582665,0],[-0.23535168,0.2553863,0.9377566,0],[-0.082020074,-0.035506573,-0.025537848,1]],"isTracked":true},{"name":"littleFingerIntermediateTip","transform":[[-0.8006587,-0.59837437,-0.029901683,0],[0.5545437,-0.75905365,0.34105653,0],[-0.22677635,0.25648803,0.93956715,0],[-0.06307931,-0.037934672,-0.01993567,1]],"isTracked":true},{"name":"littleFingerTip","isTracked":true,"transform":[[-0.8006587,-0.59837437,-0.029901683,0],[0.55454355,-0.7590536,0.3410568,0],[-0.22677657,0.2564883,0.93956715,0],[-0.04816789,-0.026222985,-0.020020342,1]]},{"isTracked":true,"transform":[[-0.83367807,-0.33506683,0.4389886,0],[-0.36958972,0.92916626,0.007321467,0],[-0.41034657,-0.15614182,-0.8984629,0],[-2.9802322e-08,1.4901161e-08,-2.9802322e-08,1]],"name":"forearmWrist"},{"name":"forearmArm","transform":[[-0.833678,-0.3350668,0.4389886,0],[-0.36958975,0.92916626,0.0073214276,0],[-0.41034657,-0.15614179,-0.89846283,0],[0.22288692,0.083471484,-0.107541226,1]],"isTracked":true}],"chirality":"right"}
"""
}