Overlay for WebGL

README.md

Overlay for WebGL

Overview

WebGL does not support text. Overlay is a simple class that allows an HTML Canvas rendering 2D to overlay a canvas rendering 3D.

The overlay is able to keep labels synchronized with the 3D view by applying the appropriate view and perspective transformations.

Usage

In your initialization code...

const overlay = new Overlay('overlay', camera, 500, 500);

In your animation loop...

  overlay.clear();

  overlay.text("a", arrowA.h / 2);
  overlay.text("b", arrowB.h / 2);

Modify HTML

<div id='container'>
  <canvas id='canvas3D'></canvas>
  <canvas id='canvas2D' width='500' height='500'></canvas>
</div>
  

Modify CSS

#container {
  position: relative;
}
#canvas3D {
  position: absolute;
  left: 0px;
  top: 0px;
  width: 500px;
  height: 500px;
}
#canvas2D {
  position: absolute;
  left: 0px;
  top: 0px;
  z-index: 10;
  width: 500px;
  height: 500px;
  /**
   * Allow events to go to the other elements
   */
  pointer-events: none;
}

Overlay.ts

export default class Overlay {
  public ctx: CanvasRenderingContext2D;
  constructor(canvas: string, private camera: EIGHT.PerspectiveCamera) {
    const canvasElement = <HTMLCanvasElement>document.getElementById('canvas2D');
    this.ctx = canvasElement.getContext('2d');
    this.ctx.strokeStyle = "#FFFFFF";
    this.ctx.fillStyle = '#ffffff';
    this.ctx.font = '24px Helvetica';
  }
  clear(): void {
    this.ctx.clearRect(0, 0, this.ctx.canvas.width, this.ctx.canvas.height);
  }
  fillText(text: string, X: EIGHT.VectorE3, maxWidth?: number) {
    const coords = this.canvasCoords(X);
    this.ctx.fillText(text, coords.x, coords.y, maxWidth);
  }
  beginPath() {
    this.ctx.beginPath();
  }
  moveTo(X: EIGHT.VectorE3) {
    const coords = this.canvasCoords(X);
    this.ctx.moveTo(coords.x, coords.y);
  }
  lineTo(X: EIGHT.VectorE3) {
    const coords = this.canvasCoords(X);
    this.ctx.lineTo(coords.x, coords.y);
  }
  stroke() {
    this.ctx.stroke();
  }
  private canvasCoords(X: EIGHT.VectorE3): EIGHT.VectorE2 {
    const camera = this.camera;
    const cameraCoords = view(EIGHT.Geometric3.fromVector(X), camera.eye, camera.look, camera.up);
    const N = camera.near;
    const F = camera.far;
    const θ = camera.fov;
    const aspect = camera.aspect;
    const canonCoords = perspective(cameraCoords, N, F, θ, aspect);
    const x = (canonCoords.x + 1) * this.ctx.canvas.width / 2;
    const y = (canonCoords.y - 1) * -this.ctx.canvas.height / 2;
    return {x, y}
  }
}


/**
 * View transformation converts world coordinates to camera frame coordinates.
 */
function view(X: EIGHT.Geometric3, eye: EIGHT.Geometric3, look: EIGHT.Geometric3, up: EIGHT.Geometric3): EIGHT.VectorE3 {

  const n = (eye - look).normalize();
  const u = up.clone().cross(n).normalize();
  const v = n.clone().cross(u);
  
  const du = - eye | u;
  const dv = - eye | v;
  const dn = - eye | n;
  
  const x = (X | u + du).a;
  const y = (X | v + dv).a;
  const z = (X | n + dn).a;
  
  return {x, y, z};
}

/**
 * Perspective transformation projects camera coordinates onto the near plane.
 */
function perspective(X: EIGHT.VectorE3, N: number, F: number, fov: number, aspect: number): EIGHT.VectorE3 {
  const t = N * Math.tan(fov / 2);
  const b = -t;
  const r = aspect * t;
  const l = -r;
  const x = ((2 * N) * X.x + (r + l) * X.z) / (-X.z * (r-l));
  const y = ((2 * N) * X.y + (t + b) * X.z) / (-X.z * (t-b));
  const z = (-(F+N) * X.z - 2 * F * N) / (-X.z * (F-N))
  return {x, y, z}; 
}

displayError.ts

/**
 * Displays an exception by writing it to a <pre> element.
 */
export function displayError(e: any) {
    const stderr = <HTMLPreElement> document.getElementById('error')
    stderr.style.color = "#FF0000"
    stderr.innerHTML = `${e}`
}

HTMLOverlay.ts

export interface Cartesian3 {
    x: number
    y: number
    z: number
}

export interface Camera {
    eye: Cartesian3
    look: Cartesian3
    up: Cartesian3
}

export interface Prism {
    near: number
    far: number
    fov: number
    aspect: number
}

/**
 * An overlay that uses HTML to render text.
 * This has the advantage of supporting MathJax.
 * 
 * 
 * const overlay = new HTMLOverlay('overlay', camera, 500, 500)
 * 
 * overlay.fillText('a', "$$\\hat{i}$$", arrowA.axis)
 */
export class HTMLOverlay {
    public containerElement: HTMLDivElement
    /**
     * A cache of div elements to avoid excessive DOM object creation.
     */
    private labels: { [name: string]: Label } = {}
    /**
     * 
     */
    constructor(containerId: string, private camera: Camera, private prism: Prism) {
        this.containerElement = document.getElementById(containerId) as HTMLDivElement
        this.containerElement.style.position = 'absolute'
        this.containerElement.style.zIndex = '5'
        this.containerElement.style.font = '48px Helvetica'
        this.containerElement.style.color = '#ffffff'
        this.containerElement.style.pointerEvents = 'none'
        this.containerElement.style.top = pixelStyle(0)
        this.containerElement.style.left = pixelStyle(0)
    }
    /**
     * 
     */
    text(name: string, text: string, _maxWidth?: number): Label {
        // Find the projection of the 3D point onto the 2D screen.
        // This could be cached.

        // Find the cached label or create a new one.
        if (!this.labels[name]) {
            const div = document.createElement("div")
            div.className = "floating-div"
            div.style.position = 'absolute'
            const textNode = document.createTextNode("")
            div.appendChild(textNode)
            this.containerElement.appendChild(div)
            this.labels[name] = new Label(this, div)
        }
        const label = this.labels[name]
        const div = label.div
        const firstChild = div.firstChild
        if (firstChild) {
            firstChild.nodeValue = text
        }
        return label
    }
    public coords2D(X: Cartesian3): { x: number; y: number } {
        const camera = this.camera
        const prism = this.prism
        const cameraCoords = view(X, camera.eye, camera.look, camera.up)
        const N = prism.near
        const F = prism.far
        const θ = prism.fov
        const aspect = prism.aspect
        const canonCoords = perspective(cameraCoords, N, F, θ, aspect)
        const x = (canonCoords.x + 1) * parseInt(this.containerElement.style.width, 10) / 2
        const y = (canonCoords.y - 1) * -parseInt(this.containerElement.style.height, 10) / 2
        return { x, y }
    }
}

export class Label {
    constructor(public overlay: HTMLOverlay, public div: HTMLDivElement) {

    }
    moveTo(X: { x: number; y: number; z: number }): void {
        const coords = this.overlay.coords2D(X)
        this.div.style.left = pixelStyle(Math.floor(coords.x))
        this.div.style.top = pixelStyle(Math.floor(coords.y))
    }
}

/**
 * View transformation converts world coordinates to camera frame coordinates.
 */
function view(X: Cartesian3, eye: Cartesian3, look: Cartesian3, up: Cartesian3): Cartesian3 {

    const n = normalize(sub(eye, look))
    const u = normalize(cross(up, n))
    const v = normalize(cross(n, u))

    const dx = - scp(eye, u)
    const dy = - scp(eye, v)
    const dz = - scp(eye, n)

    const x = scp(X, u) + dx
    const y = scp(X, v) + dy
    const z = scp(X, n) + dz

    return { x, y, z }
}

function cross(a: Cartesian3, b: Cartesian3): Cartesian3 {
    return {
        x: a.y * b.z - a.z * b.y,
        y: a.z * b.x - a.x * b.z,
        z: a.x * b.y - a.y * b.x
    }
}

function normalize(v: Cartesian3): Cartesian3 {
    const x = v.x
    const y = v.y
    const z = v.z
    const m = Math.sqrt(x * x + y * y + z * z)
    return { x: v.x / m, y: v.y / m, z: v.z / m }
}

/**
 * Returns the scalar (dot) product of the vectors a and b.
 */
function scp(a: Cartesian3, b: Cartesian3): number {
    return a.x * b.x + a.y * b.y + a.z * b.z
}

function sub(a: Cartesian3, b: Cartesian3): Cartesian3 {
    return { x: a.x - b.x, y: a.y - b.y, z: a.z - b.z }
}

/**
 * Perspective transformation projects camera coordinates onto the near plane.
 */
function perspective(X: Cartesian3, N: number, F: number, fov: number, aspect: number): Cartesian3 {
    const t = N * Math.tan(fov / 2)
    const b = -t
    const r = aspect * t
    const l = -r
    const x = ((2 * N) * X.x + (r + l) * X.z) / (-X.z * (r - l))
    const y = ((2 * N) * X.y + (t + b) * X.z) / (-X.z * (t - b))
    const z = (-(F + N) * X.z - 2 * F * N) / (-X.z * (F - N))
    return { x, y, z }
}

function pixelStyle(pixels: number): string {
    return `${pixels}px`
}

index.html

<!DOCTYPE html>
<html>

<head>
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <script type="text/x-mathjax-config">
        MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});
    </script>
    <script src="https://unpkg.com/[email protected]/dist/system.js"></script>
    <!-- script async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS_CHTML"></script -->
    <link rel='stylesheet' href='style.css'>
</head>

<body>
    <div id='container'>
        <canvas id='canvas3D'></canvas>
        <div id="overlay">
        </div>
    </div>
    <pre id='error'></pre>
    <script>
        System.defaultJSExtensions = true
        System.import('./index')
    </script>
</body>

</html>

index.ts

import { Engine, Capability, Scene } from 'davinci-eight'
import { Facet, PerspectiveCamera, DirectionalLight } from 'davinci-eight'
import { TrackballControls } from 'davinci-eight'
import { Arrow, Box } from 'davinci-eight'
import { Geometric3 } from 'davinci-eight'
import { Color } from 'davinci-eight'
import { HTMLScriptsMaterial, Mesh } from 'davinci-eight'
import { } from 'davinci-eight'
import { windowResizer } from './windowResizer'

import { e1, e2, e3 } from './math'

// Comment out the following line to use the standard window.requestAnimationFrame
import { requestAnimationFrame } from './requestAnimationFrame'
import { WireCube } from './WireCube'
import { HTMLOverlay } from './HTMLOverlay'

/**
 * Wrapper around the WebGLRenderingContext providing the ContextManager interface.
 */
const engine = new Engine('canvas3D')
    .size(500, 500)
    .clearColor(0.0, 0.0, 0.0, 1.0)
    .enable(Capability.DEPTH_TEST)

/**
 * A collection of objects that can be rendered with a single draw method call.
 */
const scene = new Scene(engine)

/**
 * Rendering information that applies to all objects.
 */
const ambients: Facet[] = []

/**
 * Provides the viewing point and perspective transformation.
 */
const camera = new PerspectiveCamera()
camera.eye.copy(e1 + e2 + 2 * e3).normalize().scale(3)
ambients.push(camera)

/**
 * Provides a light color and direction for Lambert shading.
 */
const dirLight = new DirectionalLight()
ambients.push(dirLight)

/**
 * Controls the camera by accumulating mouse movements then moving and rotating the camera.
 */
const trackball = new TrackballControls(camera, window)
trackball.subscribe(engine.canvas)
// trackball.noPan = true

// Create drawables such as Arrow, Box, Curve, Grid, Sphere, Cylinder.
// Add them to the scene here...
const box = new Box(engine, { mode: 'wire', color: Color.white })
box.width = 0.6
box.height = 0.6
box.depth = 0.6
scene.add(box)

const arrowA = new Arrow(engine)
arrowA.color = Color.red
arrowA.axis = e1
scene.add(arrowA)

const arrowB = new Arrow(engine)
arrowB.color = Color.green
arrowB.axis = e2
scene.add(arrowB)

const arrowC = new Arrow(engine)
arrowC.color = Color.blue
arrowC.axis = e3
scene.add(arrowC)

const geometry = new WireCube(engine)
const material = new HTMLScriptsMaterial(engine, ['line-vs', 'line-fs'])
const wireCube = new Mesh(geometry, material, engine)
scene.add(wireCube)

const overlay = new HTMLOverlay('overlay', camera, camera)

const resizer = windowResizer(engine, camera, overlay)

// MathJax doesn't like the text to change rapidly.
// So, we could use the text as a label (caching)
// and find some way to hide until ready
let count = 0

const A = overlay.text('A', `$$A$$`)
const a = overlay.text('a', "$$\\vec{a}$$")
const b = overlay.text('b', "$$\\hat{j}$$")
const c = overlay.text('c', "$$\\hat{k}$$")

/**
 * Animates the scene.
 */
const animate = function(_timestamp: number) {
    trackball.update()

    dirLight.direction.copy(camera.look).sub(camera.eye)

    engine.clear()

    if (count < 10000) {
        A.moveTo(arrowA.axis)
    }
    a.moveTo(Geometric3.fromVector(arrowA.axis) / 2)
    b.moveTo(arrowB.axis)
    c.moveTo(arrowC.axis)

    MathJax.Hub.Queue(["Typeset", MathJax.Hub])

    count += 1

    scene.render(ambients)
    requestAnimationFrame(animate)
}

resizer.resize()
requestAnimationFrame(animate)

line-fs.glsl

varying highp vec4 vColor;
void main(void) {
  gl_FragColor = vColor;
}

line-vs.glsl

attribute vec3 aPosition;
uniform vec3 uColor;
uniform float uOpacity;
uniform mat4 uModel;
uniform mat4 uProjection;
uniform mat4 uView;
varying highp vec4 vColor;
void main(void) {
  gl_Position = uProjection * uView * uModel * vec4(aPosition, 1.0);
  vColor = vec4(uColor, uOpacity);
}

math.ts

import { Geometric3 } from 'davinci-eight'

export const e1 = Geometric3.e1(true)
export const e2 = Geometric3.e2(true)
export const e3 = Geometric3.e3(true)
export const zero = Geometric3.zero(true)

if (!e1.isLocked()) {
    e1.lock()
}

package.json

{
    "description": "Overlay for WebGL",
    "dependencies": {
        "mathjax": "2.7.3",
        "davinci-eight": "7.4.4"
    },
    "operatorOverloading": true,
    "name": "copy-of-eight-starter-template",
    "version": "0.1.0",
    "keywords": [
        "EIGHT",
        "Overlay",
        "2D",
        "WebGL",
        "3D",
        "perspective",
        "view",
        "HTML",
        "mathdoodle"
    ],
    "author": "David Geo Holmes",
    "linting": true,
    "hideConfigFiles": true
}

requestAnimationFrame.ts

import { displayError } from './displayError'

/**
 * Catches exceptions thrown in the animation callback and displays them.
 * This function will have a slight performance impact owing to the try...catch statement.
 * This function may be bypassed for production use by using window.requestAnimationFrame directly.
 */
export function requestAnimationFrame(callback: FrameRequestCallback): number {
    const wrapper: FrameRequestCallback = function(time: number) {
        try {
            callback(time)
        }
        catch (e) {
            displayError(e)
        }
    }
    return window.requestAnimationFrame(wrapper)
}

style.css

body {
    margin: 0;
    background: black;
}

#container {
    position: relative;
    top: 0px;
}

#canvas3D {
    position: absolute;
    left: 0px;
    top: 0px;
    width: 500px;
    height: 500px;
}

#stats {
    position: absolute;
    top: 0;
    left: 0;
}

tsconfig.json

{
    "allowJs": true,
    "checkJs": true,
    "declaration": true,
    "emitDecoratorMetadata": true,
    "experimentalDecorators": true,
    "jsx": "react",
    "module": "system",
    "noImplicitAny": true,
    "noImplicitReturns": true,
    "noImplicitThis": true,
    "noUnusedLocals": true,
    "noUnusedParameters": true,
    "preserveConstEnums": true,
    "removeComments": false,
    "skipLibCheck": true,
    "sourceMap": true,
    "strictNullChecks": true,
    "suppressImplicitAnyIndexErrors": true,
    "target": "es5",
    "traceResolution": true
}

tslint.json

{
    "rules": {
        "array-type": [
            true,
            "array"
        ],
        "curly": false,
        "comment-format": [
            true,
            "check-space"
        ],
        "eofline": true,
        "forin": true,
        "jsdoc-format": true,
        "new-parens": true,
        "no-conditional-assignment": false,
        "no-consecutive-blank-lines": true,
        "no-construct": true,
        "no-for-in-array": true,
        "no-inferrable-types": [
            true
        ],
        "no-magic-numbers": false,
        "no-shadowed-variable": true,
        "no-string-throw": true,
        "no-trailing-whitespace": [
            true,
            "ignore-jsdoc"
        ],
        "no-var-keyword": true,
        "one-variable-per-declaration": [
            true,
            "ignore-for-loop"
        ],
        "prefer-const": true,
        "prefer-for-of": true,
        "prefer-function-over-method": false,
        "prefer-method-signature": true,
        "radix": true,
        "semicolon": [
            true,
            "never"
        ],
        "trailing-comma": [
            true,
            {
                "multiline": "never",
                "singleline": "never"
            }
        ],
        "triple-equals": true,
        "use-isnan": true
    }
}

windowResizer.ts

import { Engine } from 'davinci-eight'
import { PerspectiveCamera } from 'davinci-eight'
import { HTMLOverlay } from './HTMLOverlay'

export interface Resizer {
    /**
     * Force a resize of the displays.
     */
    resize(): this
    /**
     * Stop tracking the window resize event.
     */
    stop(): this
}

/**
 * Creates an object that manages resizing of the output to fit the window.
 */
export function windowResizer(engine: Engine, camera: PerspectiveCamera, overlay: HTMLOverlay): Resizer {
    const callback = function() {
        engine.size(window.innerWidth, window.innerHeight)
        engine.viewport(0, 0, window.innerWidth, window.innerHeight)
        engine.canvas.width = window.innerWidth
        engine.canvas.height = window.innerHeight
        engine.canvas.style.width = `${window.innerWidth}px`
        engine.canvas.style.height = `${window.innerHeight}px`
        overlay.containerElement.style.width = `${window.innerWidth}px`
        overlay.containerElement.style.height = `${window.innerHeight}px`
        camera.aspect = window.innerWidth / window.innerHeight
    }
    window.addEventListener('resize', callback, false)

    const that: Resizer = {
        /**
         * Force a resize of the displays. 
         */
        resize: function() {
            callback()
            return that
        },
        /**
         * Stop watching window resize
         */
        stop: function() {
            window.removeEventListener('resize', callback)
            return that
        }
    }
    return that
}

WireCube.ts

import { Geometry, Engine, Material, Usage, DataType, BeginMode } from 'davinci-eight'
/**
 * A Geometry for rendering a cube made from lines.
 */
export class WireCube implements Geometry {
    private buffer: WebGLBuffer
    public data: Float32Array
    private material: Material
    /**
     * 
     */
    public invalid = true
    constructor(private engine: Engine) {
        const gl = engine.gl
        const size = 1
        const L = size / 2
        this.data = new Float32Array([
            -L, -L, -L, +L, -L, -L,
            -L, +L, -L, +L, +L, -L,
            -L, -L, +L, +L, -L, +L,
            -L, +L, +L, +L, +L, +L,
            -L, +L, +L, -L, +L, -L,
            +L, +L, +L, +L, +L, -L,
            -L, -L, +L, -L, -L, -L,
            +L, -L, +L, +L, -L, -L,
            -L, -L, -L, -L, +L, -L,
            +L, -L, -L, +L, +L, -L,
            -L, -L, +L, -L, +L, +L,
            +L, -L, +L, +L, +L, +L
        ])
        this.buffer = gl.createBuffer() as WebGLBuffer
    }
    bind(material: Material): void {
        const gl = this.engine.gl
        gl.bindBuffer(gl.ARRAY_BUFFER, this.buffer)
        this.material = material
    }
    unbind(_material: Material): void {
        const gl = this.engine.gl
        gl.bindBuffer(gl.ARRAY_BUFFER, null)
        // this.material
    }
    draw(): void {
        const gl = this.engine.gl
        const aPosition = this.material.getAttribLocation('aPosition')

        if (this.invalid) {
            gl.bufferData(gl.ARRAY_BUFFER, this.data, Usage.STATIC_DRAW)
            this.invalid = false
        }

        gl.vertexAttribPointer(aPosition, 3, DataType.FLOAT, true, 0, 0)

        gl.enableVertexAttribArray(aPosition)
        gl.drawArrays(BeginMode.LINES, 0, 24)
        gl.disableVertexAttribArray(aPosition)
    }
}