PLY to RealityKit with Grid-Based Decimation (Example: HD Skeleton Scan)

computeBounds.swift

import Foundation
import RealityKit

/// Computes bounding box for a given vertex array
func computeBounds(for vertices: [SIMD3<Float>]) -> (SIMD3<Float>, SIMD3<Float>) {
    var minV = SIMD3<Float>(repeating: .infinity)
    var maxV = SIMD3<Float>(repeating: -.infinity)
    for v in vertices {
        minV = min(minV, v)
        maxV = max(maxV, v)
    }
    return (minV, maxV)
}

computeNormals.swift

import Foundation
import RealityKit

/// Computes  per-vertex normals for an indexed mesh.
/// Each vertex normal is the average of all face normals that touch it.
nonisolated func computeNormals(mesh: TriangleMesh) -> TriangleMesh {
    // Initialize normals array with zeros
    var normals = [SIMD3<Float>](repeating: .zero, count: mesh.vertexCount)
    
    // Accumulate face normals for each vertex
    for tri in mesh.triangles {
        let i0 = Int(tri.x)
        let i1 = Int(tri.y)
        let i2 = Int(tri.z)
        
        let v0 = mesh.vertices[i0]
        let v1 = mesh.vertices[i1]
        let v2 = mesh.vertices[i2]
        
        // Compute face normal
        let edge1 = v1 - v0
        let edge2 = v2 - v0
        let faceNormal = cross(edge1, edge2)
        
        // Add this face normal to all three vertices
        // (using unnormalized normals weights by triangle area)
        normals[i0] += faceNormal
        normals[i1] += faceNormal
        normals[i2] += faceNormal
    }
    
    // Normalize all vertex normals
    for i in 0..<normals.count {
        let len = length(normals[i])
        if len > 0 {
            normals[i] /= len
        }
    }
    
    return TriangleMesh(vertices: mesh.vertices, triangles: mesh.triangles, normals: normals)
}

decimateGridAveraged.swift

import Foundation

func decimateGridAveraged(mesh: TriangleMesh, cellSize: Float) -> TriangleMesh {
    // Accumulate all vertices per cell
    var accum: [SIMD3<Int>: SIMD3<Float>] = [:]
    var counts: [SIMD3<Int>: Int] = [:]
    
    func key(for v: SIMD3<Float>) -> SIMD3<Int> {
        SIMD3(
            Int(floor(v.x / cellSize)),
            Int(floor(v.y / cellSize)),
            Int(floor(v.z / cellSize))
        )
    }
    
    for v in mesh.vertices {
        let k = key(for: v)
        accum[k, default: .zero] += v
        counts[k, default: 0] += 1
    }
    
    // Compute averaged positions
    var newVertices: [SIMD3<Float>] = []
    var vertexMap: [SIMD3<Int>: Int] = [:]
    
    for (k, sum) in accum {
        let n = Float(counts[k] ?? 1)
        let avg = sum / n
        vertexMap[k] = newVertices.count
        newVertices.append(avg)
    }
    
    // Remap triangles to new vertex indices
    var newTriangles: [SIMD3<Int32>] = []
    newTriangles.reserveCapacity(mesh.triangles.count)
    
    for tri in mesh.triangles {
        guard let i0 = vertexMap[key(for: mesh.vertices[Int(tri.x)])],
              let i1 = vertexMap[key(for: mesh.vertices[Int(tri.y)])],
              let i2 = vertexMap[key(for: mesh.vertices[Int(tri.z)])],
              i0 != i1 && i1 != i2 && i2 != i0
        else { continue }
        
        newTriangles.append(SIMD3<Int32>(Int32(i0), Int32(i1), Int32(i2)))
    }
    
    return TriangleMesh(vertices: newVertices, triangles: newTriangles)
}

estimateCellSize.swift

import Foundation
import RealityKit

/// Estimates an average triangle edge length.
/// We sample a few hundred triangles, measure edge lengths, take the median.
/// `scaleFactor` is a multiplier to make the decimation grid looser/tighter:
/// - 1.0 = very fine detail
/// - 5.0 = very coarse simplification
nonisolated func estimateCellSize(mesh: TriangleMesh, scaleFactor: Float) -> Float {
    guard !mesh.triangles.isEmpty else { return 1.0 }
    
    var lengths: [Float] = []
    lengths.reserveCapacity(500)
    
    // Only sample a subset for speed (first 500 triangles)
    let sampleCount = min(mesh.triangles.count, 500)
    for i in 0..<sampleCount {
        let tri = mesh.triangles[i]
        let a = mesh.vertices[Int(tri.x)]
        let b = mesh.vertices[Int(tri.y)]
        let c = mesh.vertices[Int(tri.z)]
        
        // Add edge lengths
        lengths.append(distance(a, b))
        lengths.append(distance(b, c))
        lengths.append(distance(c, a))
    }
    
    guard !lengths.isEmpty else { return 1.0 }
    
    // Sort and pick median (robust against outliers)
    lengths.sort()
    let median = lengths[lengths.count / 2]
    
    return median * scaleFactor
}

ModelIOLoader.swift

import Foundation
import ModelIO
import simd

nonisolated func loadMeshWithModelIO(from url: URL) throws -> TriangleMesh {
    let asset = MDLAsset(url: url)
    
    guard asset.count > 0 else {
        throw NSError(domain: "ModelIO", code: 1, userInfo: [NSLocalizedDescriptionKey: "No objects found in file"])
    }
    
    guard let mdlMesh = asset.object(at: 0) as? MDLMesh else {
        throw NSError(domain: "ModelIO", code: 2, userInfo: [NSLocalizedDescriptionKey: "First object is not a mesh"])
    }
    
    return try extractTriangleMesh(from: mdlMesh)
}

nonisolated func extractTriangleMesh(from mdlMesh: MDLMesh) throws -> TriangleMesh {
    // Extract vertices
    guard let vertexBuffer = mdlMesh.vertexBuffers.first else {
        throw NSError(domain: "ModelIO", code: 3, userInfo: [NSLocalizedDescriptionKey: "No vertex buffer found"])
    }
    
    let vertexCount = mdlMesh.vertexCount
    var vertices: [SIMD3<Float>] = []
    vertices.reserveCapacity(vertexCount)
    
    // Get vertex descriptor to find position attribute
    let vertexDescriptor = mdlMesh.vertexDescriptor as MDLVertexDescriptor
    
    // Find position attribute
    var positionOffset = 0
    var positionStride = 0
    var foundPosition = false
    
    for i in 0..<vertexDescriptor.attributes.count {
        guard let attribute = vertexDescriptor.attributes[i] as? MDLVertexAttribute else { continue }
        if attribute.name == MDLVertexAttributePosition {
            positionOffset = attribute.offset
            foundPosition = true
            break
        }
    }
    
    guard foundPosition else {
        throw NSError(domain: "ModelIO", code: 5, userInfo: [NSLocalizedDescriptionKey: "No position attribute found"])
    }
    
    // Get stride from layout
    if let layout = vertexDescriptor.layouts[0] as? MDLVertexBufferLayout {
        positionStride = layout.stride
    }
    
    // Read vertices
    let vertexPointer = vertexBuffer.map().bytes
    for i in 0..<vertexCount {
        let offset = i * positionStride + positionOffset
        let ptr = vertexPointer.advanced(by: offset)
        let x = ptr.load(as: Float.self)
        let y = ptr.advanced(by: 4).load(as: Float.self)
        let z = ptr.advanced(by: 8).load(as: Float.self)
        vertices.append(SIMD3<Float>(x, y, z))
    }
    
    // Extract triangles from all submeshes
    var triangles: [SIMD3<Int32>] = []
    
    for submesh in mdlMesh.submeshes ?? [] {
        guard let mdlSubmesh = submesh as? MDLSubmesh else { continue }
        
        // Only process triangle topology
        guard mdlSubmesh.geometryType == .triangles else {
            print("Skipping non-triangle submesh (type: \(mdlSubmesh.geometryType.rawValue))")
            continue
        }
        
        let indexBuffer = mdlSubmesh.indexBuffer
        
        let indexCount = mdlSubmesh.indexCount
        let indexPointer = indexBuffer.map().bytes
        let indexType = mdlSubmesh.indexType
        
        // Read indices based on type
        switch indexType {
        case .uint16:
            for i in stride(from: 0, to: indexCount, by: 3) {
                let i0 = Int32(indexPointer.advanced(by: i * 2).load(as: UInt16.self))
                let i1 = Int32(indexPointer.advanced(by: (i + 1) * 2).load(as: UInt16.self))
                let i2 = Int32(indexPointer.advanced(by: (i + 2) * 2).load(as: UInt16.self))
                triangles.append(SIMD3<Int32>(i0, i1, i2))
            }
            
        case .uint32:
            for i in stride(from: 0, to: indexCount, by: 3) {
                let i0 = Int32(indexPointer.advanced(by: i * 4).load(as: UInt32.self))
                let i1 = Int32(indexPointer.advanced(by: (i + 1) * 4).load(as: UInt32.self))
                let i2 = Int32(indexPointer.advanced(by: (i + 2) * 4).load(as: UInt32.self))
                triangles.append(SIMD3<Int32>(i0, i1, i2))
            }
            
        default:
            print("Unsupported index type: \(indexType.rawValue)")
        }
    }
    
    print("ModelIO loaded: \(vertices.count) vertices, \(triangles.count) triangles")
    
    return TriangleMesh(vertices: vertices, triangles: triangles)
}

PLYToProcessedMeshView.swift

import RealityKit
import SwiftUI

#Preview(windowStyle: .volumetric) { PLYToProcessedMeshView() }
struct PLYToProcessedMeshView: View {
    @State var entity: Entity?
    @State var mesh: LowLevelMesh?
    @State var isLoading: Bool = true
    @State var useLighting: Bool = false
    @State var isMetallic: Bool = false
    @State var useWireframe: Bool = true
    @State var scaleFactor: Float = 0 // (0 = original, 1-5 = decimated)
    @State var originalTriangleCount: Int?
    @State var finalTriangleCount: Int?
    @State var shouldShowSheet: Bool = false
    
    let meshDataStorage = MeshProcessor()
    
    var body: some View {
        GeometryReader3D { geometry in
            RealityView { content in
                try! await meshDataStorage.preloadMeshes(scaleFactors: [0,1,2,3,4,5])
                
                let originalTriangleMesh = await meshDataStorage.getTriangleMeshForScaleFactor(0)
                originalTriangleCount = originalTriangleMesh.triangleCount
                
                let mesh = try! await meshDataStorage.getMeshForScaleFactor(scaleFactor)
                await updateStats()
                
                // Convert to MeshResource for RealityKit rendering.
                if let resource = try? await MeshResource(from: mesh) {
                    let entity = ModelEntity(mesh: resource,
                                             materials: [ materialForCurrentState ])
                    
                    // Orient model upright (PLYs often use Z-up)
                    entity.transform.rotation = simd_quatf(angle: -.pi * 0.5, axis: [1, 0, 0])
                    
                    entity.name = "model"
                    
                    let bounds = entity.model!.mesh.bounds
                    
                    entity.components.set([
                        InputTargetComponent(),
                        CollisionComponent(shapes: [.generateBox(width: bounds.extents.x,
                                                                 height: bounds.extents.y,
                                                                 depth: bounds.extents.z)]),
                    ])
                    
                    entity.positionAndScaleForVolume(content: content, geometry: geometry)
                    
                    content.add(entity)
                    self.entity = entity
                }
                isLoading = false
            } update: { content in
                guard let model = content.entities.first(where: { $0.name == "model" })
                else { return }
                model.positionAndScaleForVolume(content: content, geometry: geometry)
            }
        }
        .onChange(of: useLighting) { applyCurrentMaterialToModel() }
        .onChange(of: isMetallic) { applyCurrentMaterialToModel() }
        .onChange(of: useWireframe) { applyCurrentMaterialToModel() }
        .onChange(of: scaleFactor) {
            Task {
                guard let entity else { return }
                let mesh = try! await meshDataStorage.getMeshForScaleFactor(scaleFactor)
                
                if let resource = try? await MeshResource(from: mesh) {
                    entity.components[ModelComponent.self]?.mesh = resource
                    entity.components[ModelComponent.self]?.materials = [materialForCurrentState]
                }
                
                await updateStats()
            }
        }
        .gesture(
           SpatialTapGesture()
               .targetedToAnyEntity()
               .onEnded { value in
                   shouldShowSheet = true
               }
       )
        .sheet(isPresented: $shouldShowSheet) {
            settingsView
        }
    }
    
    var settingsView: some View {
        VStack(alignment: .leading, spacing: 12) {
            VStack(alignment: .leading, spacing: 4) {
                Text("Decimation Scale Factor: \(scaleFactor == 0 ? "Original" : String(format: "%.1f", scaleFactor))")
                    .font(.caption)
                Slider(value: $scaleFactor, in: 0...5, step: 1.0)
            }
            
            Divider()
            
            if let originalTriangleCount, let finalTriangleCount {
                if scaleFactor != 0 {
                    VStack(alignment: .leading, spacing: 4) {
                        Text("\(originalTriangleCount) -> \(finalTriangleCount) triangles")
                            .font(.caption)
                        Text("\(String(format: "%.1f", Float(finalTriangleCount) / Float(originalTriangleCount) * 100.0))% of original")
                            .font(.caption)
                    }
                }
            }
            
            Divider()
            
            Toggle(isOn: $useWireframe, label: { Text("Use Wireframe") })
            Toggle(isOn: $useLighting, label: { Text("Use Lighting") })
            if useLighting {
                Toggle(isOn: $isMetallic, label: { Text("Is Metallic") })
            }
            
            HStack {
                Spacer()
                Button(action: {
                    shouldShowSheet = false
                }, label: {
                    Text("Close")
                    Image(systemName: "x.circle.fill")
                })
                Spacer()
            }
        }
        .frame(width: 250)
        .padding()
    }
    
    var materialForCurrentState: RealityKit.Material {
        var material: RealityKit.Material
        if useLighting {
            var simpleMaterial = SimpleMaterial(color: .white, isMetallic: isMetallic)
            simpleMaterial.triangleFillMode = useWireframe ? .lines : .fill
            material = simpleMaterial
        } else {
            var unlitMaterial = UnlitMaterial(color: .white)
            unlitMaterial.triangleFillMode = useWireframe ? .lines : .fill
            
            material = unlitMaterial
        }

        return material
    }
    
    func applyCurrentMaterialToModel() {
        guard let entity else { return }
        
        entity.components[ModelComponent.self]?.materials = [materialForCurrentState]
    }
    
    @MainActor
    func updateStats() async {
        let mesh = await meshDataStorage.getTriangleMeshForScaleFactor(scaleFactor)
        finalTriangleCount = mesh.triangleCount
    }
}

actor MeshProcessor {
    var originalMesh: TriangleMesh? = nil
    var processedMeshCache: [Float: TriangleMesh] = [:]
    var lowLevelMeshCache: [Float: LowLevelMesh] = [:]
    
    func loadPLY() async throws -> TriangleMesh {
        
        // Human skeleton HD by Artec 3D (https://www.artec3d.com/portable-3d-scanners)
        // Model url: https://www.artec3d.com/3d-models/human-skeleton-hd
        guard let url = Bundle.main.url(forResource: "HumanSkeleton", withExtension: "ply") else {
            fatalError("download the file here and add it to your project: https://www.artec3d.com/3d-models/human-skeleton-hd")
        }
        
        let mesh = try loadMeshWithModelIO(from: url)
        originalMesh = mesh
        return mesh
    }
    
    func preloadMeshes(scaleFactors: [Float]) async throws {
        let originalMesh = try await loadPLY()
        
        let results = await withTaskGroup(of: (Float, TriangleMesh).self) { group -> [(Float, TriangleMesh)] in
            for scaleFactor in scaleFactors {
                group.addTask {
                    let mesh = self.getProcessedMesh(originalMesh: originalMesh, scaleFactor: scaleFactor)
                    return (scaleFactor, mesh)
                }
            }

            var allResults: [(Float, TriangleMesh)] = []
            for await result in group {
                allResults.append(result)
            }

            return allResults
        }

        for (scaleFactor, mesh) in results {
            processedMeshCache[scaleFactor] = mesh
            let lowLevelMesh = try await self.createLowLevelMesh(mesh: mesh)
            lowLevelMeshCache[scaleFactor] = lowLevelMesh
        }
    }

    nonisolated func getProcessedMesh(originalMesh: TriangleMesh, scaleFactor: Float) -> TriangleMesh {
        var mesh = originalMesh
        
        if scaleFactor != 0 {
            // Decimate using grid-based approach
            let cellSize = estimateCellSize(mesh: mesh, scaleFactor: scaleFactor)
            mesh = decimateGridAveraged(mesh: mesh, cellSize: cellSize)
        }
        
        mesh = computeNormals(mesh: mesh)

        return mesh
    }
    
    @MainActor
    func createLowLevelMesh(mesh: TriangleMesh) throws -> LowLevelMesh {
        let positions = mesh.vertices
        let indices = mesh.triangles.flatMap { [$0.x, $0.y, $0.z].map(UInt32.init) }
        let normals = mesh.normals!

        // Setup mesh descriptor — we must size buffers up front
        var desc = VertexDefinition.descriptor
        desc.vertexCapacity = mesh.vertexCount
        desc.indexCapacity = indices.count  // capacity for wireframe indices

        let mesh = try LowLevelMesh(descriptor: desc)
        
        // Write vertex buffer
        mesh.withUnsafeMutableBytes(bufferIndex: 0) { rawBytes in
            let buffer = rawBytes.bindMemory(to: VertexDefinition.self)
            for i in 0..<positions.count {
                buffer[i] = VertexDefinition(position: positions[i], normal: normals[i])
            }
        }

        // Write index buffer (line topology)
        mesh.withUnsafeMutableIndices { raw in
            let buf = raw.bindMemory(to: UInt32.self)
            for i in 0..<indices.count {
                buf[i] = indices[i]
            }
        }

        // Compute model-space bounding box
        let (minV, maxV) = computeBounds(for: positions)
        let bounds = BoundingBox(min: minV, max: maxV)

        mesh.parts.replaceAll([
            LowLevelMesh.Part(
                indexCount: indices.count,
                topology: .triangle,
                bounds: bounds
            )
        ])
        
        return mesh
    }
    
    nonisolated func getTriangleMeshForScaleFactor(_ scaleFactor: Float) async -> TriangleMesh {
        return await processedMeshCache[scaleFactor]!
    }
    
    nonisolated func getMeshForScaleFactor(_ scaleFactor: Float) async throws -> LowLevelMesh {
        return await lowLevelMeshCache[scaleFactor]!
    }
}

TriangleMesh.swift

import Foundation

nonisolated struct TriangleMesh {
    var vertices: [SIMD3<Float>]
    var triangles: [SIMD3<Int32>]
    var normals: [SIMD3<Float>]?
    
    var vertexCount: Int { vertices.count }
    var triangleCount: Int { triangles.count }
    
    init(vertices: [SIMD3<Float>], triangles: [SIMD3<Int32>], normals: [SIMD3<Float>]? = nil) {
        self.vertices = vertices
        self.triangles = triangles
        self.normals = normals
    }
}

VertexDefinition.swift

import Metal
import RealityKit

struct VertexDefinition {
    var position: SIMD3<Float> = .zero
    var normal: SIMD3<Float> = .zero
    
    static var vertexAttributes: [LowLevelMesh.Attribute] = [
        .init(semantic: .position, format: .float3,
              offset: MemoryLayout<Self>.offset(of: \.position)!),
        .init(semantic: .normal, format: .float3, offset: MemoryLayout<Self>.offset(of: \.normal)!),
    ]
    
    static var vertexLayouts: [LowLevelMesh.Layout] = [
        .init(bufferIndex: 0,
              bufferStride: MemoryLayout<Self>.stride)
    ]
    
    static var descriptor: LowLevelMesh.Descriptor {
        var desc = LowLevelMesh.Descriptor()
        desc.vertexAttributes = vertexAttributes
        desc.vertexLayouts = vertexLayouts
        desc.indexType = .uint32
        return desc
    }
}