Skip to content

Instantly share code, notes, and snippets.

@abernier

abernier/index.html

Created May 26, 2026 12:54
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save abernier/6931c1e39193f416b1d3c3a85dc70fe0 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save abernier/6931c1e39193f416b1d3c3a85dc70fe0 to your computer and use it in GitHub Desktop.
Download ZIP
Pin Art
Raw
index.html
<script type="importmap">
{
"imports": {
"three": "https://cdn.jsdelivr.net/npm/three@0.184.0/build/three.webgpu.js",
"three/webgpu": "https://cdn.jsdelivr.net/npm/three@0.184.0/build/three.webgpu.js",
"three/tsl": "https://cdn.jsdelivr.net/npm/three@0.184.0/build/three.tsl.js",
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.184.0/examples/jsm/",
"@huggingface/transformers": "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.6.3"
}
}
</script>
</head>
<body>
<div id="info">Loading…</div>
<script type="module" src="main.js"></script>
Raw
script.js
import * as THREE from 'three/webgpu';
import { vec2, vec3, vec4, reflect, normalView, positionViewDirection, texture, attribute, luminance, pass, mrt, output } from 'three/tsl';
import { pipeline, RawImage } from '@huggingface/transformers';
import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { ao } from 'three/addons/tsl/display/GTAONode.js';
// ── Config ──
const CAPSULE_RADIUS = 0.07;
const CAPSULE_LENGTH = 3.0;
const GRID_SPACING = 0.16;
const TARGET_GRID_COLS = 73;
const PIN_PUSH = 2.5;
const CAM_FOV = 22;
const DEPTH_CAPTURE_W = 256;
const DEPTH_CAPTURE_H = 192;
const DEPTH_MODEL_SIZE = 196; // multiple of 14 — the model's ViT patch size
const PIN_SMOOTH = 12;
const ROW_SPACING = GRID_SPACING * Math.sqrt( 3 ) / 2; // hex packing
const FRAME_PAD = 1.0;
const BEVEL_SIZE = 0.03; // shared bevel for back plate, glass, and z-offsets
// Back panel sits near the un-extended pin tip: at rest the tip pokes through
// by a small amount, leaving most of the pin body hidden behind the panel.
// As depth pushes a pin, more of its body emerges toward the glass.
const Z_BACK = CAPSULE_LENGTH / 2 - 0.2;
const Z_GLASS = PIN_PUSH + CAPSULE_LENGTH / 2 + CAPSULE_RADIUS + 0.2;
const TOY_CENTER_Z = ( Z_BACK + Z_GLASS ) / 2;
// ── State ──
let camera, scene, renderer, controls;
let renderPipeline;
let capsulesMesh;
let enclosure;
let floor;
let video, videoTexture;
let depthEstimator;
let depthData = null;
let depthWidth = 0, depthHeight = 0;
let captureCanvas, captureCtx;
let processing = false;
let gridCols = 0, gridRows = 0;
let gridW = 0, gridH = 0, outerW = 0, outerH = 0;
let depthFrameId = 0;
let lastAppliedDepthFrameId = - 1;
let pinZ = null;
let pinTargetZ = null;
let pinX = null;
let pinY = null;
let pinGridUv = null;
let infoEl;
init();
async function init() {
infoEl = document.getElementById( 'info' );
camera = new THREE.PerspectiveCamera( CAM_FOV, window.innerWidth / window.innerHeight, 0.1, 100 );
scene = new THREE.Scene();
scene.background = new THREE.Color( 0x111111 );
renderer = new THREE.WebGPURenderer( { antialias: true } );
renderer.setPixelRatio( window.devicePixelRatio );
renderer.setSize( window.innerWidth, window.innerHeight );
renderer.setAnimationLoop( animate );
renderer.shadowMap.enabled = true;
renderer.shadowMap.type = THREE.PCFShadowMap;
renderer.toneMapping = THREE.ACESFilmicToneMapping;
renderer.toneMappingExposure = 0.75;
document.body.appendChild( renderer.domElement );
// PMREMGenerator requires the WebGPU backend to be initialized first.
await renderer.init();
// RoomEnvironment so PBR materials have something realistic to reflect.
const pmremGen = new THREE.PMREMGenerator( renderer );
const envScene = new RoomEnvironment();
scene.environment = pmremGen.fromScene( envScene, 0.04 ).texture;
scene.environmentIntensity = 0.25;
scene.background = scene.environment;
scene.backgroundBlurriness = 0.4;
scene.backgroundIntensity = 1.0;
pmremGen.dispose();
initWebcamSurface();
addLights();
buildGrid();
buildEnclosure();
addFloor();
fitCamera();
setupPostProcessing();
controls = new OrbitControls( camera, renderer.domElement );
controls.target.set( 0, 0, TOY_CENTER_Z );
controls.enableDamping = true;
controls.dampingFactor = 0.08;
controls.minDistance = 4;
controls.maxDistance = 60;
window.addEventListener( 'resize', onResize );
// Scene is now rendering. Request the webcam — the prompt appears with the
// toy already on screen, and pin reflections kick in once permission lands.
try {
await requestWebcam();
} catch ( e ) {
infoEl.textContent = 'Webcam access denied or unavailable';
console.error( e );
return;
}
setupDepth();
}
function initWebcamSurface() {
// Create the video element, VideoTexture, and capture canvas synchronously
// so the rest of the scene can be built and rendered before we prompt for
// the camera. The texture samples as empty until a stream is attached.
video = document.createElement( 'video' );
video.autoplay = true;
video.playsInline = true;
video.muted = true;
videoTexture = new THREE.VideoTexture( video );
videoTexture.colorSpace = THREE.SRGBColorSpace;
videoTexture.wrapS = THREE.ClampToEdgeWrapping;
videoTexture.wrapT = THREE.ClampToEdgeWrapping;
captureCanvas = document.createElement( 'canvas' );
captureCanvas.width = DEPTH_CAPTURE_W;
captureCanvas.height = DEPTH_CAPTURE_H;
captureCtx = captureCanvas.getContext( '2d', { willReadFrequently: true } );
}
async function requestWebcam() {
infoEl.textContent = 'Requesting webcam…';
const stream = await navigator.mediaDevices.getUserMedia( {
video: { width: 640, height: 480, facingMode: 'user' },
audio: false,
} );
video.srcObject = stream;
await video.play();
}
async function setupDepth() {
infoEl.textContent = 'Loading depth model…';
try {
depthEstimator = await pipeline(
'depth-estimation',
'onnx-community/depth-anything-v2-small',
{ device: 'webgpu', dtype: 'fp16' }
);
} catch ( e ) {
console.warn( 'WebGPU pipeline failed, falling back to WASM', e );
try {
depthEstimator = await pipeline(
'depth-estimation',
'onnx-community/depth-anything-v2-small'
);
} catch ( e2 ) {
infoEl.textContent = 'Failed to load depth model';
console.error( e2 );
return;
}
}
const ip = depthEstimator.processor?.image_processor
?? depthEstimator.processor?.feature_extractor;
if ( ip ) {
ip.size = { height: DEPTH_MODEL_SIZE, width: DEPTH_MODEL_SIZE };
ip.do_resize = true;
}
infoEl.textContent = 'Move your face closer to push the pins';
setTimeout( () => { infoEl.style.opacity = 0; }, 3000 );
processLoop();
}
async function processLoop() {
if ( depthEstimator && video.readyState >= 2 && ! processing ) {
processing = true;
try {
captureCtx.save();
captureCtx.translate( DEPTH_CAPTURE_W, 0 );
captureCtx.scale( - 1, 1 );
captureCtx.drawImage( video, 0, 0, DEPTH_CAPTURE_W, DEPTH_CAPTURE_H );
captureCtx.restore();
const img = captureCtx.getImageData( 0, 0, DEPTH_CAPTURE_W, DEPTH_CAPTURE_H );
const raw = new RawImage( img.data, DEPTH_CAPTURE_W, DEPTH_CAPTURE_H, 4 );
const result = await depthEstimator( raw );
depthData = result.depth.data;
depthWidth = result.depth.width;
depthHeight = result.depth.height;
depthFrameId ++;
} catch ( e ) {
console.error( 'Depth inference error', e );
}
processing = false;
}
setTimeout( processLoop, 30 );
}
function setupPostProcessing() {
renderPipeline = new THREE.RenderPipeline( renderer );
const scenePass = pass( scene, camera );
scenePass.setMRT( mrt( {
output: output,
normal: normalView,
} ) );
const scenePassColor = scenePass.getTextureNode( 'output' );
const scenePassDepth = scenePass.getTextureNode( 'depth' );
const scenePassNormal = scenePass.getTextureNode( 'normal' );
const aoPass = ao( scenePassDepth, scenePassNormal, camera );
aoPass.resolutionScale = 0.5; // half-res AO — ~4× fewer pixels, looks fine
aoPass.samples.value = 8; // default 16; halving the tap count
const aoOutput = aoPass.getTextureNode();
renderPipeline.outputNode = scenePassColor.mul( vec4( vec3( aoOutput.r ), 1 ) );
}
function addLights() {
scene.add( new THREE.AmbientLight( 0xffffff, 0.15 ) );
const dir = new THREE.DirectionalLight( 0xffffff, 3.5 );
dir.position.set( 4, 8, 8 );
dir.castShadow = true;
const s = 16;
dir.shadow.camera.left = - s;
dir.shadow.camera.right = s;
dir.shadow.camera.top = s;
dir.shadow.camera.bottom = - s;
dir.shadow.camera.near = 0.1;
dir.shadow.camera.far = 50;
dir.shadow.mapSize.set( 2048, 2048 );
dir.shadow.bias = - 0.0005;
dir.shadow.radius = 4;
scene.add( dir );
}
function addFloor() {
if ( floor ) {
scene.remove( floor );
floor.geometry.dispose();
floor.material.dispose();
}
floor = new THREE.Mesh(
new THREE.CircleGeometry( 40, 64 ),
new THREE.MeshStandardNodeMaterial( {
color: 0xffffff,
roughness: 0.8,
metalness: 0,
} )
);
floor.rotation.x = - Math.PI / 2;
floor.position.y = - outerH / 2;
floor.receiveShadow = true;
scene.add( floor );
}
function computeGridSize() {
const aspect = window.innerWidth / window.innerHeight;
gridCols = TARGET_GRID_COLS;
gridW = ( gridCols - 1 ) * GRID_SPACING + GRID_SPACING / 2;
outerW = gridW + 2 * FRAME_PAD;
// Choose row count so the outer rectangle matches the viewport aspect.
gridRows = Math.max( 8, Math.round( ( outerW / aspect - 2 * FRAME_PAD ) / ROW_SPACING ) + 1 );
gridH = ( gridRows - 1 ) * ROW_SPACING;
outerH = gridH + 2 * FRAME_PAD;
}
function buildGrid() {
if ( capsulesMesh ) {
scene.remove( capsulesMesh );
capsulesMesh.geometry.dispose();
capsulesMesh.material.dispose();
capsulesMesh.dispose();
}
computeGridSize();
const count = gridCols * gridRows;
pinZ = new Float32Array( count );
pinTargetZ = new Float32Array( count );
pinX = new Float32Array( count );
pinY = new Float32Array( count );
pinGridUv = new Float32Array( count * 2 );
// Hex packing — alternate rows shifted right by GRID_SPACING/2
for ( let r = 0; r < gridRows; r ++ ) {
const xShift = ( r % 2 === 1 ) ? GRID_SPACING / 2 : 0;
for ( let c = 0; c < gridCols; c ++ ) {
const i = r * gridCols + c;
pinX[ i ] = c * GRID_SPACING + xShift - gridW / 2;
pinY[ i ] = r * ROW_SPACING - gridH / 2;
pinGridUv[ i * 2 ] = ( pinX[ i ] + gridW / 2 ) / gridW;
pinGridUv[ i * 2 + 1 ] = ( pinY[ i ] + gridH / 2 ) / gridH;
}
}
// Geometry — capsule pointing along +Z
const geo = new THREE.CapsuleGeometry( CAPSULE_RADIUS, CAPSULE_LENGTH, 4, 12 );
geo.rotateX( Math.PI / 2 );
geo.setAttribute( 'aGridUv', new THREE.InstancedBufferAttribute( pinGridUv, 2 ) );
const gridUv = attribute( 'aGridUv', 'vec2' );
// Per-pin reflection parallax — each pin's spherical-reflection center is
// shifted by its grid position so neighbours reflect slightly different
// vantage points across the field.
const REFLECT_PARALLAX = 0.15;
const reflOffset = vec2(
gridUv.x.sub( 0.5 ).mul( REFLECT_PARALLAX ),
gridUv.y.sub( 0.5 ).mul( REFLECT_PARALLAX )
);
const I = positionViewDirection.negate();
const R = reflect( I, normalView );
const m = R.add( vec3( 0, 0, 1 ) ).length().mul( 2 );
const reflUv = vec2(
R.x.div( m ).add( 0.5 ).add( reflOffset.x ).oneMinus(),
R.y.div( m ).add( 0.5 ).add( reflOffset.y )
);
// Grayscale luminance of the webcam reflection so the pins read as chrome
// (lit/shaded by the user's face) rather than colored mirrors.
const reflSample = luminance( texture( videoTexture, reflUv ).rgb );
// Low metalness keeps the diffuse path alive so shadows attenuate the
// reflection; full chrome (metalness=1) zeros diffuse and loses shadows.
const material = new THREE.MeshStandardNodeMaterial( {
metalness: 0.75,
roughness: 0.25,
} );
material.colorNode = reflSample;
capsulesMesh = new THREE.InstancedMesh( geo, material, count );
capsulesMesh.frustumCulled = false;
capsulesMesh.castShadow = true;
capsulesMesh.receiveShadow = true;
scene.add( capsulesMesh );
const dummy = new THREE.Object3D();
for ( let i = 0; i < count; i ++ ) {
dummy.position.set( pinX[ i ], pinY[ i ], 0 );
dummy.updateMatrix();
capsulesMesh.setMatrixAt( i, dummy.matrix );
}
capsulesMesh.instanceMatrix.needsUpdate = true;
}
function roundedRectShape( w, h, r ) {
const shape = new THREE.Shape();
const x = - w / 2, y = - h / 2;
shape.moveTo( x + r, y );
shape.lineTo( x + w - r, y );
shape.absarc( x + w - r, y + r, r, - Math.PI / 2, 0, false );
shape.lineTo( x + w, y + h - r );
shape.absarc( x + w - r, y + h - r, r, 0, Math.PI / 2, false );
shape.lineTo( x + r, y + h );
shape.absarc( x + r, y + h - r, r, Math.PI / 2, Math.PI, false );
shape.lineTo( x, y + r );
shape.absarc( x + r, y + r, r, Math.PI, Math.PI * 1.5, false );
return shape;
}
function buildEnclosure() {
if ( enclosure ) {
scene.remove( enclosure );
enclosure.traverse( o => {
if ( o.geometry ) o.geometry.dispose();
if ( o.material ) o.material.dispose();
} );
}
enclosure = new THREE.Group();
const cornerR = 0.3;
const panelShape = roundedRectShape( outerW, outerH, cornerR );
const bevelOpts = {
bevelEnabled: true,
bevelThickness: BEVEL_SIZE,
bevelSize: BEVEL_SIZE,
bevelSegments: 2,
curveSegments: 12,
};
// Thin black back plate — extrusion goes +Z, so position so the +Z-facing
// cap (including bevel tip) sits at Z_BACK.
const backThickness = 0.12;
const back = new THREE.Mesh(
new THREE.ExtrudeGeometry( panelShape, { depth: backThickness, ...bevelOpts } ),
new THREE.MeshStandardNodeMaterial( { color: 0x0a0a0a, roughness: 0.5, metalness: 0 } )
);
back.position.z = Z_BACK - backThickness - BEVEL_SIZE;
back.receiveShadow = true;
back.castShadow = true;
enclosure.add( back );
// Transparent front panel — same silhouette, sits just in front of the field.
const glassThickness = 0.15;
const glass = new THREE.Mesh(
new THREE.ExtrudeGeometry( panelShape, { depth: glassThickness, ...bevelOpts } ),
new THREE.MeshPhysicalNodeMaterial( {
color: 0xffffff,
transmission: 1.0,
roughness: 0.05,
thickness: 0.3,
ior: 1.5,
metalness: 0,
} )
);
glass.position.z = Z_GLASS - glassThickness - BEVEL_SIZE;
enclosure.add( glass );
// 4 corner support posts connecting the back plate to the glass panel,
// sitting in the padding area between the pin field and the outer edge.
// Posts overhang both panels so they read as bolts going through.
const postRadius = 0.15;
const postOverhang = 0.3;
const postHeight = Z_GLASS - Z_BACK + 2 * postOverhang;
const postZ = ( Z_GLASS + Z_BACK ) / 2;
const postGeo = new THREE.CylinderGeometry( postRadius, postRadius, postHeight, 24 );
postGeo.rotateX( Math.PI / 2 );
const postMat = new THREE.MeshStandardNodeMaterial( {
color: 0x1a1a1a,
roughness: 0.35,
metalness: 0.8,
} );
const xExt = gridW / 2 + FRAME_PAD / 2;
const yExt = gridH / 2 + FRAME_PAD / 2;
for ( const [ sx, sy ] of [[ 1, 1 ], [ - 1, 1 ], [ 1, - 1 ], [ - 1, - 1 ]] ) {
const post = new THREE.Mesh( postGeo, postMat );
post.position.set( sx * xExt, sy * yExt, postZ );
post.castShadow = true;
post.receiveShadow = true;
enclosure.add( post );
}
scene.add( enclosure );
}
function fitCamera() {
const vFov = THREE.MathUtils.degToRad( CAM_FOV / 2 );
const dist = ( outerH / 2 ) / Math.tan( vFov );
// Camera distance is measured from the toy centre, so the breathing-room
// multiplier stays proportional regardless of viewport aspect.
const totalDist = dist * 1.4;
const azimuth = THREE.MathUtils.degToRad( - 25 ); // initial orbit to the left
camera.aspect = window.innerWidth / window.innerHeight;
camera.position.set(
Math.sin( azimuth ) * totalDist,
0,
TOY_CENTER_Z + Math.cos( azimuth ) * totalDist
);
camera.lookAt( 0, 0, TOY_CENTER_Z );
camera.updateProjectionMatrix();
}
function onResize() {
renderer.setSize( window.innerWidth, window.innerHeight );
buildGrid();
buildEnclosure();
addFloor();
fitCamera();
}
// ── Animate ──
const timer = new THREE.Timer();
const _dummy = new THREE.Object3D();
function animate() {
timer.update();
const dt = Math.min( timer.getDelta(), 1 / 30 );
if ( controls ) controls.update();
if ( capsulesMesh && pinZ ) {
const count = pinZ.length;
if ( depthData && depthFrameId !== lastAppliedDepthFrameId ) {
for ( let i = 0; i < count; i ++ ) {
const u = pinGridUv[ i * 2 ];
const v = pinGridUv[ i * 2 + 1 ];
const dx = Math.min( depthWidth - 1, Math.floor( u * depthWidth ) );
const dy = Math.min( depthHeight - 1, Math.floor( ( 1 - v ) * depthHeight ) );
const d = depthData[ dy * depthWidth + dx ] / 255;
pinTargetZ[ i ] = d * PIN_PUSH;
}
lastAppliedDepthFrameId = depthFrameId;
}
const k = 1 - Math.exp( - PIN_SMOOTH * dt );
for ( let i = 0; i < count; i ++ ) {
pinZ[ i ] += ( pinTargetZ[ i ] - pinZ[ i ] ) * k;
_dummy.position.set( pinX[ i ], pinY[ i ], pinZ[ i ] );
_dummy.updateMatrix();
capsulesMesh.setMatrixAt( i, _dummy.matrix );
}
capsulesMesh.instanceMatrix.needsUpdate = true;
}
if ( renderPipeline ) renderPipeline.render();
else renderer.render( scene, camera );
}
Raw
style.css
body {
margin: 0;
overflow: hidden;
background: #000;
font-family: -apple-system, sans-serif;
}
canvas {
display: block;
touch-action: none;
}
#info {
position: absolute;
top: 12px;
left: 12px;
color: #fff;
opacity: 0.8;
font-size: 12px;
pointer-events: none;
transition: opacity 1s;
z-index: 10;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment