Logitech Muse for Apple Vision Pro Tracking, Pressure Sensitive Inputs, and Haptics in RealityKit

AppModel.swift

import SwiftUI

@MainActor
@Observable
class AppModel {
    enum ImmersiveSpaceState {
        case closed
        case inTransition
        case open
    }
    var immersiveSpaceState = ImmersiveSpaceState.closed
    var immersiveSpaceType: ImmersiveSpaceType = .simpleExample
}

enum ImmersiveSpaceType: String, CaseIterable {
    case simpleExample
    
    var title: String {
        return "Simple Example"
    }
}

ImmersiveStylusSimpleExampleView.swift

import CoreHaptics
import GameController
import RealityKit
import SwiftUI

struct ImmersiveStylusSimpleExampleView: View {
    @State private var stylusManager = StylusManager()

    var body: some View {
        RealityView { content in
            let root = Entity()
            content.add(root)
            stylusManager.rootEntity = root
            await stylusManager.handleControllerSetup()
        }
        .task {
            // Don't forget to add the Accessory Tracking capability
            let configuration = SpatialTrackingSession.Configuration(tracking: [.accessory])
            let session = SpatialTrackingSession()
            await session.run(configuration)
        }
    }
}

@MainActor
final class StylusManager {
    var rootEntity: Entity?
    private var hapticEngines: [ObjectIdentifier: CHHapticEngine] = [:]
    private var hapticPlayers: [ObjectIdentifier: CHHapticPatternPlayer] = [:]
    private var inflationHapticPlayers: [ObjectIdentifier: CHHapticPatternPlayer] = [:]
    private var activePressureSphere: ModelEntity?
    private var activeSphereColor: UIColor = .systemRed
    private var maxScale: Float = 0.5
    private let baseRadius: Float = 0.01
    private var tipIndicators: [AnchorEntity: ModelEntity] = [:]

    func handleControllerSetup() async {
        // Existing connections
        let styluses = GCStylus.styli

        for stylus in styluses where stylus.productCategory == GCProductCategorySpatialStylus {
            try? await setupAccessory(stylus: stylus)
        }

        NotificationCenter.default.addObserver(
            forName: NSNotification.Name.GCStylusDidConnect, object: nil, queue: .main
        ) { [weak self] note in
            guard let self,
                  let stylus = note.object as? GCStylus,
                  stylus.productCategory == GCProductCategorySpatialStylus else { return }
            Task { @MainActor in
                try? await self.setupAccessory(stylus: stylus)
            }
        }
    }

    private func setupAccessory(stylus: GCStylus) async throws {
        guard let root = rootEntity else { return }
        let source = try await AnchoringComponent.AccessoryAnchoringSource(device: stylus)
        
        // List available locations (aim and origin)
        print("📍 Available locations: \(source.accessoryLocations)")

        guard let location = source.locationName(named: "aim") else { return }

        let anchor = AnchorEntity(
            .accessory(from: source, location: location),
            trackingMode: .predicted,
            physicsSimulation: .none
        )
        root.addChild(anchor)

        let key = ObjectIdentifier(stylus)

        // Setup haptics if available
        setupHaptics(for: stylus, key: key)
        setupStylusInputs(stylus: stylus, anchor: anchor, key: key)
        addStylusTipIndicator(to: anchor)
    }

    private func setupHaptics(for stylus: GCStylus, key: ObjectIdentifier) {
        guard let deviceHaptics = stylus.haptics else { return }
        
        // Create haptic engine
        let engine = deviceHaptics.createEngine(withLocality: .default)
        do {
            try engine?.start()
            hapticEngines[key] = engine
            
            // Create a simple "tap" pattern for main button press
            let tapPattern = try CHHapticPattern(events: [
                CHHapticEvent(eventType: .hapticTransient, parameters: [
                    CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.85),
                    CHHapticEventParameter(parameterID: .hapticSharpness, value: 0.5)
                ], relativeTime: 0.0)
            ], parameters: [])
            
            let player = try engine?.makePlayer(with: tapPattern)
            hapticPlayers[key] = player
            
            // Create an "inflation" pattern for tip and secondary pressure increase
            let inflationPattern = try CHHapticPattern(events: [
                CHHapticEvent(eventType: .hapticContinuous, parameters: [
                    CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.55),
                    CHHapticEventParameter(parameterID: .hapticSharpness, value: 1.0)
                ], relativeTime: 0.0, duration: 0.1)
            ], parameters: [])
            
            let inflationPlayer = try engine?.makePlayer(with: inflationPattern)
            inflationHapticPlayers[key] = inflationPlayer
        } catch {
            print("❌ Failed to setup haptics: \(error)")
        }
    }
    
    private func playHaptic(for key: ObjectIdentifier) {
        guard let player = hapticPlayers[key] else { return }
        
        do {
            try player.start(atTime: CHHapticTimeImmediate)
        } catch {
            print("❌ Failed to play haptic: \(error)")
        }
    }
    
    private func playInflationHaptic(for key: ObjectIdentifier) {
        guard let player = inflationHapticPlayers[key] else { return }
        
        do {
            try player.start(atTime: CHHapticTimeImmediate)
        } catch {
            print("❌ Failed to play inflation haptic: \(error)")
        }
    }

    private func addStylusTipIndicator(to anchor: AnchorEntity) {
        let tipSphere = ModelEntity(
            mesh: .generateSphere(radius: 0.003),
            materials: [SimpleMaterial(color: .white, isMetallic: false)]
        )
        anchor.addChild(tipSphere)
        tipIndicators[anchor] = tipSphere
    }

    private func setupStylusInputs(stylus: GCStylus, anchor: AnchorEntity, key: ObjectIdentifier) {
        guard let input = stylus.input else { return }
        
        // Handle main button (simple button pressed)
        input.buttons[.stylusPrimaryButton]?.pressedInput.pressedDidChangeHandler = { [weak self] _, _, pressed in
            guard pressed, let self else { return }
            Task { @MainActor in
                self.playHaptic(for: key)
                self.spawnSphere(at: anchor, color: .systemBlue, radius: 0.02)
            }
        }

        // Handle secondary button pressure changes - create and increase scale of red sphere
        input.buttons[.stylusSecondaryButton]?.pressedInput.valueDidChangeHandler = { [weak self] _, _, pressure in
            guard let self else { return }
            Task { @MainActor in
                if pressure > 0.05 {
                    self.updatePressureSphere(at: anchor, pressure: pressure, color: .systemRed, key: key)
                } else if pressure <= 0 {
                    self.dropPressureSphere(at: anchor)
                }
            }
        }

        // Handle tip pressure changes - create and increase scale of green sphere
        input.buttons[.stylusTip]?.pressedInput.valueDidChangeHandler = { [weak self] _, _, pressure in
            guard let self else { return }
            Task { @MainActor in
                if pressure > 0.05 {
                    self.updatePressureSphere(at: anchor, pressure: pressure, color: .systemGreen, key: key)
                } else if pressure <= 0 {
                    self.dropPressureSphere(at: anchor)
                }
            }
        }
    }

    private func spawnSphere(at anchor: AnchorEntity, color: UIColor, radius: Float) {
        guard let root = rootEntity else { return }
        
        let worldTransform = anchor.transformMatrix(relativeTo: nil)
        let worldPosition = SIMD3<Float>(worldTransform.columns.3.x,
                                          worldTransform.columns.3.y,
                                          worldTransform.columns.3.z)
        
        let sphere = ModelEntity(
            mesh: .generateSphere(radius: radius),
            materials: [SimpleMaterial(color: color, isMetallic: false)]
        )
        sphere.position = worldPosition
        
        root.addChild(sphere)
    }
    
    private func updatePressureSphere(at anchor: AnchorEntity, pressure: Float, color: UIColor, key: ObjectIdentifier) {
        if activePressureSphere == nil {
            // Hide the tip indicator while inflating
            tipIndicators[anchor]?.isEnabled = false
            
            // Create new sphere
            let sphere = ModelEntity(
                mesh: .generateSphere(radius: baseRadius),
                materials: [SimpleMaterial(color: color, isMetallic: false)]
            )
            anchor.addChild(sphere)
            activePressureSphere = sphere
            activeSphereColor = color
            maxScale = 0.5  // Reset max scale for new sphere
        }
        
        // Scale the sphere based on pressure (0.0 to 1.0)
        let scale = 0.5 + (pressure * 5.5)
        
        // Only scale UP, never down
        if scale > maxScale {
            maxScale = scale
            activePressureSphere?.scale = SIMD3<Float>(repeating: maxScale)
            
            // Offset the sphere by its scaled radius in the -Z direction (forward from stylus tip)
            // This creates the "balloon blowing" effect
            let offset = baseRadius * maxScale
            activePressureSphere?.position = SIMD3<Float>(0, 0, -offset)
            
            // Play subtle haptic feedback while inflating
            playInflationHaptic(for: key)
            
            let colorEmoji = color == .systemRed ? "🔴" : "🟢"
            print("\(colorEmoji) Pressure: \(String(format: "%.2f", pressure)), Scale: \(String(format: "%.2f", maxScale))")
        }
    }
    
    private func dropPressureSphere(at anchor: AnchorEntity) {
        guard let sphere = activePressureSphere, let root = rootEntity else { return }
        
        // Get the current world position before detaching
        let worldTransform = sphere.transformMatrix(relativeTo: nil)
        let worldPosition = SIMD3<Float>(worldTransform.columns.3.x,
                                          worldTransform.columns.3.y,
                                          worldTransform.columns.3.z)
        let worldScale = sphere.scale
        
        // Remove from anchor
        sphere.removeFromParent()
        
        // Create a new sphere at the world position (dropped)
        let droppedSphere = ModelEntity(
            mesh: .generateSphere(radius: 0.01),
            materials: [SimpleMaterial(color: activeSphereColor, isMetallic: false)]
        )
        droppedSphere.position = worldPosition
        droppedSphere.scale = worldScale
        root.addChild(droppedSphere)
        
        print("💧 Dropped sphere at scale \(worldScale)")
        
        // Show the tip indicator again
        tipIndicators[anchor]?.isEnabled = true
        
        // Clear the reference
        activePressureSphere = nil
    }
}

LogitechMusePlaygroundApp.swift

import SwiftUI

@main
struct LogitechMusePlaygroundApp: App {
    @State private var appModel = AppModel()
    
    var body: some Scene {
        WindowGroup {
            MainWindowView()
                .environment(appModel)
        }
        .windowResizability(.contentSize)

        ImmersiveSpace(id: ImmersiveSpaceType.simpleExample.rawValue) {
            ImmersiveStylusSimpleExampleView()
                .onAppear { appModel.immersiveSpaceState = .open }
                .onDisappear { appModel.immersiveSpaceState = .closed }
        }
        .immersionStyle(selection: .constant(.mixed), in: .mixed)
     }
}

MainWindowView.swift

import RealityKit
import SwiftUI

struct MainWindowView: View {
    @Environment(AppModel.self) private var appModel
    @Environment(\.dismissImmersiveSpace) private var dismissImmersiveSpace
    @Environment(\.openImmersiveSpace) private var openImmersiveSpace
    
    var body: some View {
        @Bindable var appModel = appModel
        
        VStack(spacing: 24) {
            if appModel.immersiveSpaceState != .open {
                Text("Select experience and press start")
                Picker("Experience", selection: $appModel.immersiveSpaceType) {
                    ForEach(ImmersiveSpaceType.allCases, id: \.self) { type in
                        Text(type.title).tag(type)
                    }
                }
            } else {
                SimpleExampleWindowView()
            }
            
            Button(action: {
                toggleImmersiveSpace(type: appModel.immersiveSpaceType)
            },
                   label: {
                Text(appModel.immersiveSpaceState == .open ? "Exit" : "Start")
            })
            .animation(.none, value: 0)
            .fontWeight(.semibold)
            .disabled(appModel.immersiveSpaceState == .inTransition)
        }
        .padding(36)
        .animation(.default, value: appModel.immersiveSpaceState)
    }
    
    func toggleImmersiveSpace(type: ImmersiveSpaceType) {
        Task { @MainActor in
            switch appModel.immersiveSpaceState {
                case .open:
                    appModel.immersiveSpaceState = .inTransition
                    await dismissImmersiveSpace()
                case .closed:
                    appModel.immersiveSpaceState = .inTransition
                    switch await openImmersiveSpace(id: type.rawValue) {
                        case .opened:
                            break

                        case .userCancelled, .error:
                            fallthrough
                        @unknown default:
                            appModel.immersiveSpaceState = .closed
                    }

                case .inTransition:
                    break
            }
        }
    }
}

#Preview(windowStyle: .automatic) {
    MainWindowView()
        .environment(AppModel())
}

SimpleExampleWindowView.swift

import SwiftUI

struct SimpleExampleWindowView: View {
    var body: some View {
        VStack(alignment: .leading, spacing: 12) {
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.white)
                Text("Ready when white sphere appears")
            }
            
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.green)
                ButtonTypeIcon(kind: .pressure)
                Text("Tip pressure increase grows green spheres")
            }
            
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.blue)
                ButtonTypeIcon(kind: .push)
                Text("Main button creates blue spheres")
            }
            
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.red)
                ButtonTypeIcon(kind: .pressure)
                Text("Secondary button pressure increase grows red spheres")
            }
        }
        .frame(width: 500, height: 170)
    }
    
    struct ButtonTypeIcon: View {
        enum Kind {
            case push
            case pressure
        }

        let kind: Kind

        var body: some View {
            switch kind {
            case .push:
                Image(systemName: "cursorarrow.rays")
                    .font(.system(size: 24, weight: .regular))
            case .pressure:
                Image(systemName: "dial.medium")
                    .font(.system(size: 24, weight: .regular))
            }
        }
    }
}

#Preview(windowStyle: .automatic) {
    SimpleExampleWindowView()
}