How to render HTML elements in VR (using react-three/xr)

HTMLInVR.tsx

import React, {useRef, useEffect, useState, ReactNode, useMemo} from 'react';
import * as THREE from 'three';
import {toCanvas} from 'html-to-image';
import {createRoot, Root} from 'react-dom/client';
import {flushSync} from "react-dom";

interface HTMLInVRProps {
	children: ReactNode;
	width?: number;
	height?: number;
	pixelsPerMeter?: number;
	position?: THREE.Vector3 | [number, number, number];
	rotation?: THREE.Euler | [number, number, number];
	canvasPixelRatio?: number;
	backgroundColor?: string;
}

export const HTMLInVR: React.FC<HTMLInVRProps> = (props: HTMLInVRProps)=>{
	const {
		children, width = 1, height = 1, pixelsPerMeter = 100, position = [0, 0, 0], rotation = [0, 0, 0], canvasPixelRatio = 3, backgroundColor = 'transparent',
	} = props;
	const meshRef = useRef<THREE.Mesh>(null);
	const [childRenderContainer, setChildRenderContainer] = useState<HTMLDivElement | null>(null);
	const [childRenderContainerReactRoot, setChildRenderContainerReactRoot] = useState<Root | null>(null);

	// Create a hidden div to render the HTML
	useEffect(()=>{
		if (!childRenderContainer) {
			const div = document.createElement('div');
			div.style.position = 'absolute';
			div.style.left = '-9999px';
			div.style.top = '-9999px';
			div.style.width = `${width * pixelsPerMeter}px`;
			div.style.height = `${height * pixelsPerMeter}px`;
			div.style.overflow = 'hidden';
			div.style.pointerEvents = 'none';
			div.style.backgroundColor = backgroundColor;
			document.body.appendChild(div);
			setChildRenderContainer(div);
			setChildRenderContainerReactRoot(createRoot(div));
		}
		return ()=>{
			if (childRenderContainer) {
				document.body.removeChild(childRenderContainer);
			}
		};
	}, [childRenderContainer, width, height, pixelsPerMeter, backgroundColor]);

	const [texture, setTexture] = useState<THREE.CanvasTexture | null>(null);
	// dispose texture on component unmount
	useEffect(()=>{
		return ()=>{
			if (texture) texture.dispose();
		};
	}, [texture]);

	const childrenHolder_mutatableInterface = useMemo(()=>({latestChildren: children, triggerNextUpdate: null} as HTMLInVR_ChildrenHolder_MutatableInterface), []);
	childrenHolder_mutatableInterface.latestChildren = children;

	// render HTML to the hidden div, then render that to a new canvas, then set canvas-texture to target that canvas as its new image-source

	// NOTE: We detect when the children needs to be rerendered, based on if the child's props json-representation changes.
	// WARNING: This doesn't work in every case! But it works in my project (only 1 child, props all json-serializable); you can customize this check for yours.
	const childrenJSON = JSON.stringify(children?.["props"]);

	useEffect(()=>{
		proceed();
		async function proceed() {
			if (!childRenderContainer) return;
			if (!childRenderContainerReactRoot) return;

			if (childrenHolder_mutatableInterface.triggerNextUpdate == null) {
				childRenderContainerReactRoot.render(<HTMLInVR_ChildrenHolder mutatableInterface={childrenHolder_mutatableInterface}/>);
				for (let i = 0; i < 5000 / 10 && childRenderContainer.childNodes.length == 0; i++) {
					await SleepAsync(10);
				}
			} else {
				childrenHolder_mutatableInterface.triggerNextUpdate!();
			}

			try {
				// Convert HTML to Canvas directly
				if (childRenderContainer.childNodes[0]?.ownerDocument == null) {
					console.error("Error rendering HTML to canvas; childRenderContainer.childNodes[0]?.ownerDocument was null.");
					return;
				}
				const canvas = await toCanvas(childRenderContainer.childNodes[0] as HTMLElement, {
					quality: 1,
					pixelRatio: canvasPixelRatio,
					skipFonts: true,
					//fontEmbedCSS: true,
					backgroundColor,
				});
				
				// update to a new texture, with the new canvas as the source
				const newTexture = new THREE.CanvasTexture(canvas);
				newTexture.minFilter = THREE.LinearFilter;
				newTexture.colorSpace = THREE.SRGBColorSpace;
				newTexture.needsUpdate = true;
				setTexture(newTexture);
			} catch (error) {
				console.error("Error rendering HTML to canvas:", error);
			}
		};
	}, [childRenderContainer, childRenderContainerReactRoot, childrenJSON]);

	return (
		<mesh ref={meshRef} position={position} rotation={rotation}>
			<planeGeometry args={[width, height]}/>
			<meshStandardMaterial map={texture} transparent={backgroundColor === 'transparent'} side={THREE.DoubleSide}/>
		</mesh>
	);
};

export type HTMLInVR_ChildrenHolder_MutatableInterface = {latestChildren: ReactNode, triggerNextUpdate: (()=>any)|n};
export const HTMLInVR_ChildrenHolder = (props: {mutatableInterface: HTMLInVR_ChildrenHolder_MutatableInterface})=>{
	const {mutatableInterface} = props;

	const forceUpdate = useForceUpdate();
	mutatableInterface.triggerNextUpdate = ()=>{
		// flushSync ensures the update is applied immediately (so that the html-to-image can draw it to canvas immediately, reducing delay in it showing up for the user)
		flushSync(()=>{
			forceUpdate();
		});
	};

	return mutatableInterface.latestChildren;
};
	
// utils
export const useForceUpdate = ()=>{
	const [_state, setState] = React.useState(true);
	const forceUpdate = React.useCallback(()=>{
	  setState(s=>!s);
	}, []);
	return forceUpdate;
}
export function SleepAsync(timeMS): Promise<void> {
	return new Promise((resolve, reject)=>{
		WaitXThenRun(timeMS, resolve);
	});
}