Skip to content

Instantly share code, notes, and snippets.

@kkdd

kkdd/README.md

Last active April 29, 2022 08:58
Show Gist options
  • Save kkdd/d62a49ada1bcf61af29a3680297ffc97 to your computer and use it in GitHub Desktop.
Save kkdd/d62a49ada1bcf61af29a3680297ffc97 to your computer and use it in GitHub Desktop.
Download ZIP
regl + apache-arrow: fast WebGL scatter plot
Raw
index.html
<!DOCTYPE html>
<html>
<head>
<meta charset=utf-8>
<style>
html,body{
height: 100%;
margin: 0;
}
.axis path {
display: none;
}
.axis line {
stroke-opacity: 0.1;
shape-rendering: crispEdges;
}
svg,
#canvas {
position: absolute;
width: 100%;
height: 100%;
}
#debug {
z-index: 10;
}
</style>
</head>
<body>
<div id="canvas"></div>
<div id='debug' style='padding:4px;background-color:#fffc;position:absolute;right:0;top:0;color:#0af;font-family:courier;font-size:12px;user-select:none'></div>
</body>
<script src="https://npmcdn.com/regl/dist/regl.min.js"></script>
<script src="https://d3js.org/d3.v7.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/apache-arrow/Arrow.es2015.min.js"></script>
<script>
const pointSize = 6, container = "#canvas";
const scaleInitial = 4, nTick = 10, tickPadding = {x: 12, y: 6};
const BYTESIZE = 1 // GL_UNSIGNED_BYTE
const FLOATSIZE = 4 // GL_FLOAT
const THOUSAND = 1000;
const maxPoints = 1 * THOUSAND * THOUSAND
const canvas = document.querySelector(container);
const svg = d3.select(container).append('svg');
const gX = svg.append("g").attr("class", "axis axis--x");
const gY = svg.append("g").attr("class", "axis axis--y");
const regl = createREGL({container: canvas});
const RGBASIZE = BYTESIZE * 4; // vec4 (r, g, b, a); uint8 is normalized and converted to float in regl().
const reglData = {
count: 0, // the number of points to draw
offset: 0,
transform: {}, // set by reglTransform()
x: regl.buffer({
usage: 'dynamic',
type: 'float',
length: FLOATSIZE * maxPoints
}),
y: regl.buffer({
usage: 'dynamic',
type: 'float',
length: FLOATSIZE * maxPoints
}),
color: regl.buffer({
usage: 'dynamic',
type: 'uint8',
length: RGBASIZE * maxPoints
}),
}
const drawNewPoints = (newPoints) => {
const max = (a, b) => a>b?a:b;
const numNew = newPoints.n;
if (numNew > maxPoints) return;
if (numNew + reglData.offset > maxPoints) {
reglData.count = reglData.offset;
reglData.offset = 0;
}
reglData.x.subdata(newPoints.x, reglData.offset * FLOATSIZE);
reglData.y.subdata(newPoints.y, reglData.offset * FLOATSIZE);
reglData.color.subdata(newPoints.color, reglData.offset * RGBASIZE);
reglData.offset += numNew
reglData.count = max(reglData.count, reglData.offset);
displayDebug(`total = ${reglData.count.toLocaleString()}`);
redrawRequested = true;
}
const drawPoints = () => {
const drawReglData = regl({
profile: true,
depth: {enable: false},
stencil: {enable: false},
primitive: 'points',
count: regl.prop('count'),
frag: `
precision mediump float;
varying vec4 fill;
void main() {
gl_FragColor = fill;
}
`,
vert: `
precision mediump float;
attribute float x;
attribute float y;
attribute vec4 color;
uniform vec2 scale;
uniform vec2 offset;
uniform float pointSize;
varying vec4 fill;
void main() {
gl_PointSize = pointSize;
gl_Position = vec4(vec2(x, y)*scale+offset, 0, 1);
fill = color;
}
`,
uniforms: {
scale: regl.prop('transform.scale'),
offset: regl.prop('transform.offset'),
pointSize: pointSize,
},
attributes: {
x: { // float
buffer: regl.prop('x'),
},
y: { // float
buffer: regl.prop('y'),
},
color: { // vec4 (r, g, b, a)
buffer: regl.prop('color'),
normalized: true, // uint8 is normalized and converted to float
}
},
});
regl.clear({depth: 1});
drawReglData(reglData);
}
let d3Transform = transformInitial();
let [widthLatest, heightLatest] = widthHeight(canvas);
let redrawRequested = false;
render();
window.addEventListener('resize', resizeRender);
regl.frame(()=>{
if (redrawRequested) {drawPoints(); redrawRequested = false;}
});
function render() {
const [width, height] = widthHeight(canvas);
const xScale = d3.scaleLinear()
.domain([-width / 2, width / 2])
.range([0, width]);
const yScale = d3.scaleLinear()
.domain([-height / 2, height / 2])
.range([height, 0]);
const xAxis = d3.axisBottom(xScale)
.ticks(width / height * nTick)
.tickSize(height)
.tickPadding(-tickPadding.x);
const yAxis = d3.axisRight(yScale)
.ticks(nTick)
.tickSize(width)
.tickPadding(-width + tickPadding.y);
const zoomed = (event, _) => {
d3Transform = event.transform;
gX.call(xAxis.scale(d3Transform.rescaleX(xScale)));
gY.call(yAxis.scale(d3Transform.rescaleY(yScale)));
reglData.transform = reglTransform();
redrawRequested = true;
}
const zoom = d3.zoom().on("zoom", zoomed);
svg.call(zoom).call(zoom.transform, d3Transform);
};
function resizeRender() {
const [width, height] = widthHeight(canvas);
const v = (1/d3Transform.k-1)/2;
d3Transform = d3Transform.translate((width-widthLatest)*v, (height-heightLatest)*v);
[widthLatest, heightLatest] = [width, height];
render();
}
function reglTransform() {
const [width, height] = widthHeight(canvas);
const scale = [d3Transform.k/width*2, d3Transform.k/height*2];
const offset = [d3Transform.x/width*2 + (d3Transform.k-1), -d3Transform.y/height*2 - (d3Transform.k-1)];
return {scale: scale, offset: offset};
}
function transformInitial(k=scaleInitial) {
const [width, height] = widthHeight(canvas);
return new d3.ZoomTransform(k, -width/2*(k-1), -height/2*(k-1));
}
function widthHeight(canvas) {
const r = canvas.getBoundingClientRect();
return [r.width, r.height];
}
function displayDebug(str) {
document.getElementById("debug").innerText = str;
}
const url = "https://raw.githubusercontent.com/chrisprice/d3fc-webgl-hathi-explorer/master/data.arrows";
const loadData = async (url) => {
const response = await fetch(url);
const reader = await Arrow.RecordBatchReader.from(response);
await reader.open();
for await (const batch of reader) {
drawNewPoints(createPoints(batch));
}
};
loadData(url);
function getValues(arrowBatch, columnName) {
const i = arrowBatch.schema.fields.map(f=>f.name).indexOf(columnName);
if (i < 0) return;
return arrowBatch.data.children[i].values;
}
function createPoints(arrowBatch) {
const rngRGB = () => d3.hsl(Math.random() * 300 - 60, 1, 0.3 + Math.random()*0.3).rgb();
const n = arrowBatch.numRows;
const data = {
n: n,
x: getValues(arrowBatch, 'x'),
y: getValues(arrowBatch, 'y'),
color: new Uint8Array(n*4), // vec4 (r, g, b, a); uint8 is normalized and converted to float in regl().
}
for (let i = 0; i < n; i++) {
const {r, g, b} = rngRGB();
data.color[i*4] = r;
data.color[i*4+1] = g;
data.color[i*4+2] = b;
data.color[i*4+3] = 255;
}
return data;
}
</script>
</html>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment