CALayer -> MTLTexture renderer (via CARemderer)

CALayerToMetalRenderer.swift

import SWBShared2

import Metal
import AppKit
import CoreImage
import CoreGraphics
import QuartzCore

@globalActor
public struct CALayerToMetalRendererActor {
    public actor ActorType {
    }

    public static let shared: ActorType = ActorType()
}

/*
Render from the layer tree, to a metal texture.

Code ideas from:
https://stackoverflow.com/questions/56150363/rendering-animated-calayer-off-screen-using-carenderer-with-mtltexture
*/

/*
 Related to black frames, and "Core Image defers the rendering until the client requests the access to the frame buffer, i.e. CVPixelBufferLockBaseAddress."
 https://stackoverflow.com/questions/56018503/making-cicontext-renderciimage-cvpixelbuffer-work-with-avassetwriter

 Regarding CARenderer owning the layer:
 https://stackoverflow.com/questions/73467494/carenderer-draws-nothing-into-bound-texture
 */

/*
 Discussion on drawing on a background thread
 https://stackoverflow.com/questions/51812966/is-drawing-to-an-mtkview-or-cametallayer-required-to-take-place-on-the-main-thre#comment90593639_51814181
 */

/*
 Using the queue (kCARendererMetalCommandQueue):
 Below is some code that uses it. It passes the command queue directly to the CARenderer.
 https://github.com/jrmuizel/carenderer-yuv/blob/main/main.mm
 */

/*
 Not directly related to CARenderer, but one of the better articles I've seen on metal in general:
 https://medium.com/@nathan.fooo/real-time-player-with-metal-part-1-3a670f33417d
 */

// for some reason this has to be done on main, else the resulting texture is black/pink (ALL THE TIME)
public class CALayerToMetalRenderer: CountableInstance {
    public private(set) var id = UUID()
    private var device: MTLDevice!
    private var textures: [MTLTexture]!
    private var queue: MTLCommandQueue!
    private var renderers: [CARenderer]!

    private var descriptors: [MTLRenderPassDescriptor]!

    private var renderSize: NSSize
    private var currentTextureTarget = 0


    /*
     CAREFUL: setting this > 1 means we use multiple CARenderers. Each bound to its own texture.
     It also means that EVERY frame, we do an implicit CATransaction flush/commit (to try to have the CARenderer own the layer in question).
        THIS FAILS SILENTLY SOMETIMES, resulting in BLACK/PINK frames.
     */
    private var numberOfBuffersToUse = 1 // 2 = dbl buffering. use = 3 for triple
    private var firstTimeThrough = true

    public init(renderSize: NSSize, numberOfBuffersToUse: Int = 1, device: MTLDevice? = nil) {
        self.renderSize = renderSize
        self.numberOfBuffersToUse = numberOfBuffersToUse
        textures = []
        descriptors = []
        renderers = []

        let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm, width: Int(renderSize.width), height: Int(renderSize.height), mipmapped: false)
//        textureDescriptor.storageMode = .private
        textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]

        if let d = device {
            self.device = d
        } else {
            // Gives us the GPU associated with the main display
            self.device = MTLCreateSystemDefaultDevice()
        }
        for _ in 0..<numberOfBuffersToUse {
            let texture = self.device.makeTexture(descriptor: textureDescriptor)
            textures.append(texture!)
        }

        queue = self.device.makeCommandQueue()

        /*
         The source media (CALayers) is sRGB - I wonder if this is TRUE if you've a P3 display?
         */
        let colorSpace = CGColorSpace(name: CGColorSpace.sRGB)
//        let options: [AnyHashable: Any] = [kCARendererColorSpace: colorSpace as Any, kCARendererMetalCommandQueue: queue as Any]
        let options: [AnyHashable: Any] = [kCARendererColorSpace: colorSpace as Any]
        for index in 0..<numberOfBuffersToUse {
            let target = textures[index]
            renderers.append(CARenderer(mtlTexture: target, options: options))
            descriptors.append(makeDescriptor(forTexture: target))
        }

        Task {
            await InstanceCounter.shared.instanceInit(self)
        }
    }

    deinit {
        renderers = nil
        textures = nil
        descriptors = nil

        InstanceCounter.shared.safeDeinit(self)
    }

    func cleanUp() {
        for renderer in renderers {
            renderer.layer = nil
            CATransaction.commit()
            CATransaction.flush()
        }

        numberOfBuffersToUse = 0
        renderers = []
        textures = []
        descriptors = []
    }

    private func makeDescriptor(forTexture: MTLTexture) -> MTLRenderPassDescriptor {
        let descriptor = MTLRenderPassDescriptor()
        let isKnownDevMachine = NSApplication.isAKnownDevMachine
        descriptor.colorAttachments[0].texture = forTexture
        descriptor.colorAttachments[0].loadAction = .clear
        if isKnownDevMachine {
            // pink! so it is more visible to us
            descriptor.colorAttachments[0].clearColor = MTLClearColorMake(1.0, 0.1, 0.5, 1.0)
        } else {
            descriptor.colorAttachments[0].clearColor = MTLClearColorMake(0.0, 0.0, 0.0, 1.0)
        }
        descriptor.colorAttachments[0].storeAction = .store
        return descriptor
    }

    public func setupRendererWith(layer: CALayer) -> CARenderer {
        let renderer = currentRenderer
//        let target = currentTexture

        if layer != renderer.layer {
            renderer.layer = layer
        }

        let rect = NSMakeRect(0, 0, renderSize.width, renderSize.height)
        if !rect.equalTo(renderer.bounds, tolerance: 1) {
            layer.bounds = rect
            renderer.bounds = rect
        }

//        renderer.setDestination(target)
        // https://stackoverflow.com/questions/73467494/carenderer-draws-nothing-into-bound-texture
        if numberOfBuffersToUse > 1 || firstTimeThrough {
            CATransaction.flush()
            CATransaction.commit()
            firstTimeThrough = true
        }
        V2Logging.rendering.verbose("Rendering to target: \(currentTextureTarget)")
        return renderer
    }

    private var currentRenderer: CARenderer {
        renderers[currentTextureTarget]
    }
    private var currentDescriptor: MTLRenderPassDescriptor {
        descriptors[currentTextureTarget]
    }
    private var currentTexture: MTLTexture {
        textures[currentTextureTarget]
    }

    @CALayerToMetalRendererActor
//    @MainActor
    public func renderLayerToMTLTexture(layer: CALayer, size: NSSize) async -> MTLTexture {

        let target = currentTexture
        let rendererToUse = setupRendererWith(layer: layer)

        /*
         I think only needed if we're trying to use the descriptor to .clear the buffer
         */
        if let renderCommandBuffer: MTLCommandBuffer = queue.makeCommandBuffer() {
            let renderCommandEncoder: MTLRenderCommandEncoder = renderCommandBuffer.makeRenderCommandEncoder(descriptor: currentDescriptor)!
            renderCommandEncoder.endEncoding()
            renderCommandBuffer.commit()
            renderCommandBuffer.waitUntilCompleted()

            rendererToUse.beginFrame(atTime: CACurrentMediaTime(), timeStamp: nil)
            rendererToUse.addUpdate(rendererToUse.bounds)
            rendererToUse.render()
            rendererToUse.endFrame()

         /*
         Trying to sync texture (to get around the black/pink frame problem).
         */
//            if let blitCommandBuffer: MTLCommandBuffer = queue.makeCommandBuffer() {
//                let blitCommandEncoder: MTLBlitCommandEncoder = blitCommandBuffer.makeBlitCommandEncoder()!
//                blitCommandEncoder.synchronize(resource: target)
//                blitCommandEncoder.endEncoding()
//
//                await withCheckedContinuation { [weak self] continuation in
//                    guard let self else {
//                        return
//                    }
//                    blitCommandBuffer.addCompletedHandler { _ in
//                        V2Logging.rendering.debug("blitCommandBuffer \(self.currentTextureTarget) completed")
//                        continuation.resume()
//                    }
//                    blitCommandBuffer.commit()
//                    blitCommandBuffer.waitUntilScheduled()
//                }
//            }
        }

        // This was me showing myself that the texture did NOT have a iosurface
//        DispatchQueue.once {
//            V2Logging.rendering.info("Renderer setup.  textureIO surface: \(texture.iosurface)")
//        }

        currentTextureTarget += 1
        if currentTextureTarget > numberOfBuffersToUse - 1 {
            currentTextureTarget = 0
        }
        return target
    }
}