RealityKit Extruded Text to LowLevelMesh animated by Metal compute shader

animateWavyTextVertices.metal

#include <metal_stdlib>
using namespace metal;

#include "VertexData.h"
#include "WavyTextParams.h"

kernel void animateWavyTextVertices(device VertexData* outVertices [[buffer(0)]],
                                    device const VertexData* inVertices [[buffer(1)]],
                                    constant WavyTextParams& params [[buffer(2)]],
                                    uint id [[thread_position_in_grid]])
{
    VertexData v = inVertices[id];
    float phase = params.phase + v.position.x * params.phaseScale;
    float offset = params.amplitude * sin(phase);
    v.position.x += offset * params.offsetFactors.x;
    v.position.y += offset * params.offsetFactors.y;
    v.position.z += offset * params.offsetFactors.z;
    outVertices[id] = v;
}

ExtrudedTextLowLevelMeshView.swift

import Metal
import RealityKit
import SwiftUI

#Preview { ExtrudedTextLowLevelMeshView() }
struct ExtrudedTextLowLevelMeshView: View {
    @State var lowLevelMesh: LowLevelMesh?
    @State var originalVerticesBuffer: MTLBuffer?
    @State var timer: Timer?

    @State var amplitude: Float = 0.055
    @State var animationRate: Float = 1.0
    @State var animationPhase: Float = 0.0
    @State var scale: Float = 12
    @State var offsetFactors: SIMD3<Float> = .init(x: 0.25, y: 0.75, z: 1)

    let device: MTLDevice
    let commandQueue: MTLCommandQueue
    let computePipelineState: MTLComputePipelineState
    
    let timerUpdateDuration: TimeInterval = 1.0 / 120.0

    init() {
        self.device = MTLCreateSystemDefaultDevice()!
        self.commandQueue = device.makeCommandQueue()!
        let library = device.makeDefaultLibrary()!
        let kernelFunction = library.makeFunction(name: "animateWavyTextVertices")!
        self.computePipelineState = try! device.makeComputePipelineState(function: kernelFunction)
    }
    
    var body: some View {
        VStack {
            RealityView { content in
                let entity = try! getEntity()
                entity.position = .init(x: -0.3, y: 0.0, z: 0)
                content.add(entity)
            }
            
            VStack {
                HStack {
                    Text("Amplitude: \(amplitude, specifier: "%.3f")")
                    Spacer()
                    Slider(value: $amplitude, in: 0.0...0.1)
                        .frame(width: 300)
                }
                HStack {
                    Text("Animation Rate: \(animationRate, specifier: "%.2f")")
                    Spacer()
                    Slider(value: $animationRate, in: 0.1...2)
                        .frame(width: 300)
                }
                
                HStack {
                    Text("Phase Scale: \(scale, specifier: "%.2f")")
                    Spacer()
                    Slider(value: $scale, in: 10...50)
                        .frame(width: 300)
                }

                HStack {
                    Text("X Factor: \(offsetFactors.x, specifier: "%.2f")")
                    Spacer()
                    Slider(value: $offsetFactors.x, in: 0...2)
                        .frame(width: 300)
                }
                
                HStack {
                    Text("Y Factor: \(offsetFactors.y, specifier: "%.2f")")
                    Spacer()
                    Slider(value: $offsetFactors.y, in: 0...2)
                        .frame(width: 300)
                }
                
                HStack {
                    Text("Z Factor: \(offsetFactors.z, specifier: "%.2f")")
                    Spacer()
                    Slider(value: $offsetFactors.z, in: 0...2)
                        .frame(width: 300)
                }
            }
            .frame(width: 500)
            .padding()
            .glassBackgroundEffect()
            
        }
        .onAppear { startTimer() }
        .onDisappear { stopTimer() }
    }
}

// MARK: Entity
extension ExtrudedTextLowLevelMeshView {
    func getEntity() throws -> Entity {
        let resource = try getMeshResource()
        let lowLevelMesh = try createLowLevelMesh(from: resource)

        // Capture original vertices once into a GPU buffer for the compute pass
        lowLevelMesh.withUnsafeBytes(bufferIndex: 0) { buffer in
            let vertices = buffer.bindMemory(to: VertexData.self)
            let originals = Array(vertices)
            let byteCount = originals.count * MemoryLayout<VertexData>.stride
            self.originalVerticesBuffer = device.makeBuffer(bytes: originals, length: byteCount, options: [])
        }

        let swappedMeshResource = try MeshResource(from: lowLevelMesh)
        
        var material = PhysicallyBasedMaterial()
        material.baseColor.tint = .init(red: 1.0, green: 0.9, blue: 0.25, alpha: 1.0)
        material.roughness.scale = 1.0
        material.metallic.scale = 0.0
        
        var material2 = UnlitMaterial()
        material2.color.tint = .init(red: 0.125, green: 0.0, blue: 0.25, alpha: 1.0)

        let modelComponent = ModelComponent(mesh: swappedMeshResource, materials: [material, material])

        let entity = Entity()
        entity.components.set(modelComponent)
        
        self.lowLevelMesh = lowLevelMesh
        
        return entity
    }
}

// MARK: MeshResource extruding AttributedString
extension ExtrudedTextLowLevelMeshView {
    func getMeshResource() throws -> MeshResource {
        var extrusionOptions = MeshResource.ShapeExtrusionOptions()
        extrusionOptions.extrusionMethod = .linear(depth: 1.0)
        extrusionOptions.materialAssignment = .init(front: 0, back: 0, extrusion: 1, frontChamfer: 1, backChamfer: 1)
        extrusionOptions.chamferRadius = 0.1
        return try MeshResource(extruding: attributedString, extrusionOptions: extrusionOptions)
    }
    
    var attributedString: AttributedString {
        var textString = AttributedString("LowLevelMesh")
        textString.font = .systemFont(ofSize: 6)
        let paragraphStyle = NSMutableParagraphStyle()
        paragraphStyle.alignment = .center
        let centerAttributes = AttributeContainer([.paragraphStyle: paragraphStyle])
        textString.mergeAttributes(centerAttributes)
        return textString
    }
}

// MARK: Animation
extension ExtrudedTextLowLevelMeshView {
    func startTimer() {
        timer = Timer.scheduledTimer(withTimeInterval: timerUpdateDuration, repeats: true) { _ in
            animationPhase += animationRate * Float(timerUpdateDuration) * 3
            updateMesh()
        }
    }

    func stopTimer() {
        timer?.invalidate()
        timer = nil
    }
}

// MARK: Update Mesh - Compute Shader Pass
extension ExtrudedTextLowLevelMeshView {
    func updateMesh() {
        guard let mesh = lowLevelMesh,
              let original = originalVerticesBuffer,
              let commandBuffer = commandQueue.makeCommandBuffer(),
              let computeEncoder = commandBuffer.makeComputeCommandEncoder()
        else { return }

        let vertexBuffer = mesh.replace(bufferIndex: 0, using: commandBuffer)

        computeEncoder.setComputePipelineState(computePipelineState)
        computeEncoder.setBuffer(vertexBuffer, offset: 0, index: 0) // output
        computeEncoder.setBuffer(original, offset: 0, index: 1)     // input (originals)

        var params = WavyTextParams(
            amplitude: amplitude,
            phase: animationPhase,
            phaseScale: scale,
            offsetFactors: offsetFactors
        )
        computeEncoder.setBytes(&params, length: MemoryLayout<WavyTextParams>.size, index: 2)

        let threadsPerGrid = MTLSize(width: mesh.vertexCapacity, height: 1, depth: 1)
        let threadsPerThreadgroup = MTLSize(width: 64, height: 1, depth: 1)
        computeEncoder.dispatchThreads(threadsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup)

        computeEncoder.endEncoding()
        commandBuffer.commit()
    }
}

// MARK: MeshResource -> LowLevelMesh
extension ExtrudedTextLowLevelMeshView {
    enum MeshCreationError: Error {
        case modelNotFound, meshPartNotFound
    }
    
    // Complicated by multiple models/parts + instance transforms in MeshResource(extruding:)
    func createLowLevelMesh(from meshResource: MeshResource) throws -> LowLevelMesh {
        let models = meshResource.contents.models
        let instances = meshResource.contents.instances

        guard !models.isEmpty else {
            throw MeshCreationError.meshPartNotFound
        }

        // Collect all parts and associate them with the transform of the instance that references them.
        // If there are no instances, fall back to raw models with the identity transform.
        var resolvedParts: [(part: MeshResource.Part, transform: simd_float4x4)] = []

        // Preferred: use instances
        for instance in instances {
            if let model = models[instance.model] {
                for part in model.parts {
                    resolvedParts.append((part, instance.transform))
                }
            }
        }
        
        // Fallback: no instances, just take the raw models
        if resolvedParts.isEmpty {
            for model in models {
                for part in model.parts {
                    resolvedParts.append((part, matrix_identity_float4x4))
                }
            }
        }

        guard !resolvedParts.isEmpty else {
            throw MeshCreationError.meshPartNotFound
        }

        // Calculate total buffer capacity
        let totalVertexCount = resolvedParts.reduce(0) { $0 + ($1.part[MeshBuffers.positions]?.elements.count ?? 0) }
        let totalIndexCount  = resolvedParts.reduce(0) { $0 + ($1.part.triangleIndices?.elements.count ?? 0) }

        let lowLevelMesh = try VertexData.initializeMesh(
            vertexCapacity: totalVertexCount,
            indexCapacity: totalIndexCount
        )

        // Running offsets into the combined vertex/index buffers
        var vertexOffset = 0
        var indexOffset = 0
        var finalParts: [LowLevelMesh.Part] = []

        for (part, transform) in resolvedParts {
            let positions = part[MeshBuffers.positions]?.elements ?? []
            let normals   = part[MeshBuffers.normals]?.elements ?? []
            let texCoords = part[MeshBuffers.textureCoordinates]?.elements ?? []
            let inputIndices = part.triangleIndices?.elements ?? []

            // Compute the normal matrix from the instance transform (upper-left 3×3 of the 4×4 matrix).
            let normalMatrix = simd_float3x3(
                SIMD3<Float>(transform.columns.0.x, transform.columns.0.y, transform.columns.0.z),
                SIMD3<Float>(transform.columns.1.x, transform.columns.1.y, transform.columns.1.z),
                SIMD3<Float>(transform.columns.2.x, transform.columns.2.y, transform.columns.2.z)
            )

            // Track per-part bounds (world space, after transform)
            var boundsMin = SIMD3<Float>(repeating: .greatestFiniteMagnitude)
            var boundsMax = SIMD3<Float>(repeating: -.greatestFiniteMagnitude)

            // Copy vertex data into the low-level mesh buffer
            lowLevelMesh.withUnsafeMutableBytes(bufferIndex: 0) { rawBuffer in
                let vertexBuffer = rawBuffer.bindMemory(
                    to: (SIMD3<Float>, SIMD3<Float>, SIMD2<Float>).self
                )

                for i in 0..<positions.count {
                    // Transform the position into world space
                    let position4 = transform * SIMD4<Float>(positions[i], 1)
                    let worldPosition = SIMD3<Float>(position4.x, position4.y, position4.z)

                    // Transform the normal into world space (if available)
                    let localNormal = (i < normals.count) ? normals[i] : .zero
                    let worldNormal = (localNormal == .zero) ? .zero : normalize(normalMatrix * localNormal)

                    // Get UV coordinates (if available)
                    let texCoord = (i < texCoords.count) ? texCoords[i] : .zero

                    // Write final vertex into the buffer
                    vertexBuffer[vertexOffset + i] = (worldPosition, worldNormal, texCoord)

                    // Expand bounds
                    boundsMin = simd_min(boundsMin, worldPosition)
                    boundsMax = simd_max(boundsMax, worldPosition)
                }
            }

            // Copy index data into the low-level mesh buffer (offsetting by base vertex index)
            lowLevelMesh.withUnsafeMutableIndices { rawBuffer in
                let indexBuffer = rawBuffer.bindMemory(to: UInt32.self)
                for (localIndex, inputIndex) in inputIndices.enumerated() {
                    indexBuffer[indexOffset + localIndex] = UInt32(vertexOffset) + inputIndex
                }
            }

            // Record this part in the final mesh
            finalParts.append(
                LowLevelMesh.Part(
                    indexOffset: indexOffset * MemoryLayout<UInt32>.stride, // in bytes
                    indexCount: inputIndices.count,
                    topology: .triangle,
                    materialIndex: part.materialIndex,
                    bounds: BoundingBox(min: boundsMin, max: boundsMax)
                )
            )

            // Advance offsets
            vertexOffset += positions.count
            indexOffset  += inputIndices.count
        }

        // Replace all parts in the low-level mesh with our combined list
        lowLevelMesh.parts.replaceAll(finalParts)
        return lowLevelMesh
    }
}

VertexData+Extensions.swift

import Foundation
import RealityKit

extension VertexData {
    static var vertexAttributes: [LowLevelMesh.Attribute] = [
        .init(semantic: .position, format: .float3, offset: MemoryLayout<Self>.offset(of: \.position)!),
        .init(semantic: .normal, format: .float3, offset: MemoryLayout<Self>.offset(of: \.normal)!),
        .init(semantic: .uv0, format: .float2, offset: MemoryLayout<Self>.offset(of: \.uv)!)
    ]
    
    static var vertexLayouts: [LowLevelMesh.Layout] = [
        .init(bufferIndex: 0, bufferStride: MemoryLayout<Self>.stride)
    ]

    static var descriptor: LowLevelMesh.Descriptor {
        var desc = LowLevelMesh.Descriptor()
        desc.vertexAttributes = VertexData.vertexAttributes
        desc.vertexLayouts = VertexData.vertexLayouts
        desc.indexType = .uint32
        return desc
    }
    
    @MainActor static func initializeMesh(vertexCapacity: Int,
                                          indexCapacity: Int) throws -> LowLevelMesh {
        var desc = VertexData.descriptor
        desc.vertexCapacity = vertexCapacity
        desc.indexCapacity = indexCapacity
        return try LowLevelMesh(descriptor: desc)
    }
}

VertexData.h

#include <simd/simd.h>

#ifndef VertexData_h
#define VertexData_h

struct VertexData {
    simd_float3 position;
    simd_float3 normal;
    simd_float2 uv;
};

#endif /* VertexData_h */

WavyTextParams.h

#ifndef WavyTextParams_h
#define WavyTextParams_h

#include <simd/simd.h>

struct WavyTextParams {
    float amplitude;
    float phase;
    float phaseScale;
    simd_float3 offsetFactors;
};

#endif /* WavyTextParams_h */