Skip to content

Instantly share code, notes, and snippets.

@cool-Blue

cool-Blue/SO30322620 idiomatic v3.js

Last active February 28, 2016 20:59
Show Gist options
  • Save cool-Blue/6d7d7a3dfb3fadfc2633 to your computer and use it in GitHub Desktop.
Save cool-Blue/6d7d7a3dfb3fadfc2633 to your computer and use it in GitHub Desktop.
Download ZIP
d3 in webWorkers
Raw
readme.md

Using D3 in web workers

postMessage serialising

Two versions were tried for passing non-transferable objects

  1. Use JSON.stringify to process the object before passing to postMessage
  2. Pass the raw object with methods stripped off

The relative performance varies depending on the browser used, Mozilla and IE don't have significant difference in performance, but Chrome is much slower if the objects are not serialised first.

The final version uses transferable objects.

Raw
index.html
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title></title>
</head>
<body>
<div class="svg-container"></div>
<!--<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>-->
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.0.0-alpha1/jquery.min.js"></script>
<!--<script src="lib/jquery-1.11.1.min.js"></script>-->
<!--<script src=//cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js></script>-->
<!--<script src="lib/d3.min.js"></script>-->
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script src="https://gitcdn.xyz/repo/cool-Blue/d3-lib/master/Output/Output/Output.js"></script>
<script src="https://gitcdn.xyz/repo/cool-Blue/d3-lib/master/elapsedTime/elapsed-time-2.0.js"></script>
<script src="https://gitcdn.xyz/repo/cool-Blue/d3-lib/master/plot/plot-transform.js"></script>
<script src="https://gitcdn.xyz/repo/cool-Blue/d3-lib/master/plot/fps-histogram.js"></script>
<script src="messageObject.js"></script>
<script src="SO30322620 idiomatic v3.js"></script>
</body>
</html>
Raw
messageObject.js
function TransfSelection() {
var buffer, selection, frameLength = 8,
groups;
function frame(d, i, j) {
return [i, j, d.x, d.y, d.indx].concat(d.c);
}
function data(offset){
var frame = buffer.subarray(offset, offset + frameLength);
return {
i: frame[0],
j: frame[1],
d: {
x: frame[2], y: frame[3],
indx: frame[4],
c: [frame[5], frame[6], frame[7]]
}
}
}
function selectionBuffer(){
var k = 0;
buffer = new Float64Array(selection.size()*frameLength);
selection.each(function(d,i,j){
var data = frame(d, i, j);
buffer.set(data, k);
k += data.length;
});
}
function bufferSelection(){
var l = buffer.length, frames = l/frameLength, i, j, k, f,
g = groups, s = new Array(g);
for(i = 0; i < g; i++) s[i] = new Array(frames/g);
for(k = 0; k < l; k += frameLength) {
f = data(k);
s[f.j][f.i] = {__data__: f.d};
}
return s
}
function dataBuffer(){
var k = 0;
buffer = new Float64Array(selection.length*frameLength);
selection.forEach(function(d,i){
var data = frame(d, i);
buffer.set(data, k);
k += data.length;
});
}
function bufferData(){
var l = buffer.length, frames = l/frameLength, i, j, k, f,
d = new Array(frames);
for(k = 0; k < l; k += frameLength) {
f = data(k);
d[f.i] = f.d;
}
return d
}
return {
selectionBuffer: function(_){
if(!_) return buffer;
selection = _;
selectionBuffer();
return {
buffer: buffer.buffer,
groups: selection.length,
frame: frameLength
};
},
selection: function(_){
if(!_) return selection;
buffer = new Float64Array(_.buffer);
groups = _.groups;
frameLength = _.frame;
return bufferSelection();
},
dataBuffer: function(_){
if(!_) return buffer;
selection = _;
dataBuffer();
return {
buffer: buffer.buffer,
frame: frameLength
};
},
data: function(_){
if(!_) return selection;
buffer = new Float64Array(_.buffer);
frameLength = _.frame;
return bufferData();
}
}
}
Raw
SO30322620 idiomatic v3.js
// added limit function to keep squares in play
// changed lock management to d3.map on __data__
// delegated data bind to web worker
// data transfer via JSON serialisation
// lock moved off __data__ onto ___locks
// putting it on __data__ is not thread safe because transitions terminate
// during the worker cycle
$(function () {
var rects,
gridHeight = 500,
gridWidth = 960,
gridExtent, squaresExtent,
cellSize, cellPitch,
cellsColumns = 100,
cellsRows = 50,
cellsCount = cellsColumns * cellsRows,
squares = [],
inputs = d3.select("body").insert("div", '.svg-container')
.attr("id", "metrics"),
elapsedTime = outputs.ElapsedTime("#metrics", {
border: 0, margin: 0, "box-sizing": "border-box",
padding: "0 0 0 6px", background: "#2B303B", "color": "orange"
}),
hist = d3.ui.FpsMeter("#metrics", {display: "inline-block"}, {
height: 10, width: 100,
values: function(d){return 1/d},
domain: [0, 60]
}),
container = d3.select('.svg-container'),
svg = container.append('svg')
.attr('width', gridWidth)
.attr('height', (gridHeight = gridHeight - metrics.clientHeight))
.style({ 'background-color': 'black', opacity: 1 }),
createRandomRGB = function () {
var red = Math.floor((Math.random() * 256)).toString(),
green = Math.floor((Math.random() * 256)).toString(),
blue = Math.floor((Math.random() * 256)).toString(),
rgb = 'rgb(' + red + ',' + green + ',' + blue + ')';
return [red, green, blue];
},
createGrid = function (width, height) {
var scaleHorizontal = d3.scale.ordinal()
.domain(d3.range(cellsColumns))
.rangeBands([0, width], 1 / 15),
rangeHorizontal = scaleHorizontal.range(),
scaleVertical = d3.scale.ordinal()
.domain(d3.range(cellsRows))
.rangeBands([0, height]),
rangeVertical = scaleVertical.range(),
squares = [];
rangeHorizontal.forEach(function (dh, i) {
rangeVertical.forEach(function (dv, j) {
var indx;
squares[indx = i + j * cellsColumns] = { x: dh, y: dv, c: createRandomRGB(), indx: indx }
})
});
cellSize = scaleHorizontal.rangeBand();
cellPitch = {
x: rangeHorizontal[1] - rangeHorizontal[0],
y: rangeVertical[1] - rangeVertical[0]
};
gridExtent = {
x: scaleHorizontal.rangeExtent(),
y: scaleVertical.rangeExtent()
};
squaresExtent = {
x: [gridExtent.x[0], gridExtent.x[1] - cellPitch.x],
y: [gridExtent.y[0], gridExtent.y[1] - cellPitch.y]
}
rects = svg.selectAll("rect")
.data(squares, function (d, i) { return d.indx })
.enter().append('rect')
.attr('class', 'cell')
.attr('width', cellSize)
.attr('height', cellSize)
.attr('x', function (d) { return d.x })
.attr('y', function (d) { return d.y })
.style('fill', function (d) { return "rgb(" + d.c.join(",") + ")"});
return squares;
},
choseRandom = function (options) {
options = options || [true, false];
var max = options.length;
return options[Math.floor(Math.random() * (max))];
},
pickRandomCell = function (selection, group) {
//cells is a group from a selection, i.e. the second dimension of the array
//it may be filtered so use a global cell count as a basis for computing size
var l = cellsCount - locked.count,
r = Math.floor(Math.random() * l);
return l ? d3.select(selection[group][r]).datum().indx : -1;
},
locked = (function () {
var lockedNodeCount = 0;
function l(name, value) {
//<this> is a DOM element
//can be called with name as an object to manage multiple locks
if (typeof name === "string") {
if (value) {
if (!this.___locks) { this.___locks = d3.map(); lockedNodeCount++; }
this.___locks.set(name, value);
} else {
this.___locks.remove(name);
if (this.___locks.empty()) { delete this.___locks; lockedNodeCount--; }
}
} else {
//name is an object, recurse multiple locks
for (var p in name) locked(p, name[p]);
}
};
Object.defineProperty(l, "count", { get: function () { return lockedNodeCount; } })
return l;
})();
function lock(lockClass) {
//<this> is the node
locked.call(this, lockClass, true)
}
function unlock(lockClass) {
//<this> is the node
locked.call(this, lockClass, false)
}
function permutateColours(cells, group, squares) {
var samples = Math.min(10, Math.max(~~(squares.length / 5), 1)), s, ii = [], i, k = 0, c;
while (samples--) {
do i = pickRandomCell(cells, group); while (ii.indexOf(i) > -1 && k++ < 5 && i > -1);
if (k < 10 && i > -1) {
ii.push(i);
s = squares[i];
squares.splice(i, 1, { x: s.x, y: s.y, c: createRandomRGB(), indx: s.indx });
}
}
}
function permutatePositions(cells, group, squares) {
var samples = Math.min(10, Math.max(~~(squares.length / 10), 1)), s, ss = [], d, m, p, k = 0;
while (samples--) {
do s = pickRandomCell(cells, group); while (ss.indexOf(s) > -1 && k++ < 5 && s > -1);
if (k < 10 && s > -1) {
ss.push(s);
d = squares[s];
m = { x: d.x, y: d.y, c: d.c, indx: d.indx };
m[p = choseRandom(["x", "y"])] = limit(m[p] + choseRandom([-1, 1]) * cellPitch[p], squaresExtent[p]);
squares.splice(s, 1, m);
}
}
function limit (value, extent) {
var min = extent[0], max = extent[1];
return Math.min(max, Math.max(value, min))
}
}
function getChanges(rects, squares) {
//use a composite key function to use the exit selection as an attribute update selection
//since its the exit selection, d3 does not bind the new data, this is done with the .each
return rects
.data(squares, function (d, i) { return d.indx + "_" + d.x + "_" + d.y + "_" + d.c.join("_"); })
.exit().each(function (d, i, j) { d3.select(this).datum(squares[i]) })
}
function updateSquaresXY(changes) {
changes
.transition("strokex").duration(600)
.attr("stroke", "white")
.style("stroke-opacity", 0.6)
.transition("x").duration(1500)
.attr('x', function (d) { return d.x })
.each("start", function (d) { lock.call(this, "lockedX") })
.each("end", function (d) { unlock.call(this, "lockedX") })
.transition("strokex").duration(600)
.style("stroke-opacity", 0)
changes
.transition("y").duration(1500)
.attr('y', function (d) { return d.y })
.each("start", function (d) { lock.call(this, "lockedY") })
.each("end", function (d) { unlock.call(this, "lockedY") })
.transition("strokey").duration(600)
.style("stroke-opacity", 0)
}
function updateSquaresX(changes) {
changes
.transition("strokex").duration(600)
.filter(function () { return !this.___locks })
.attr("stroke", "white")
.style("stroke-opacity", 0.6)
.transition("x").duration(1500)
.attr('x', function (d) { return d.x })
.each("start", function (d) { lock.call(this, "lockedX") })
.each("end", function (d) { unlock.call(this, "lockedX") })
.transition("strokex").duration(600)
.style("stroke-opacity", 0)
}
function updateSquaresY(changes) {
changes
.transition("strokey").duration(600)
.filter(function () { return !this.___locks })
.attr("stroke", "white")
.style("stroke-opacity", 0.6)
.transition("y").duration(1500)
.attr('y', function (d) { return d.y })
.each("start", function (d) { lock.call(this, "lockedY") })
.each("end", function (d) { unlock.call(this, "lockedY") })
.transition("strokey").duration(600)
.style("stroke-opacity", 0)
}
function updateSquaresFill(changes) {
changes.style("opacity", 0.6).transition("flash").duration(250).style("opacity", 1)
.transition("fill").duration(800)
.style('fill', function (d, i) { return "rgb(" + d.c.join(",") + ")" })
.each("start", function (d) { lock.call(this, "lockedFill") })
.each("end", function (d) { unlock.call(this, "lockedFill") });
}
squares = createGrid(gridWidth, gridHeight);
var changes, exmpleKeyDescr = { base: squares[0], include: ["indx", "x", "y"] },
rebindX = RebindWorker(["indx", "x"],
function x(changes) {
updateSquaresX(changes);
}),
rebindY = RebindWorker(["indx", "y"],
function y(changes) {
updateSquaresY(changes);
}),
rebindFill = RebindWorker(["indx", "c"],
function fill(changes) {
updateSquaresFill(changes);
});
$.when(rebindX.done, rebindY.done, rebindFill.done).done(function () {
squares_tick(squares)
});
function squares_tick(squares) {
d3.timer(function t () {
var dormantRects = rects.filter(function (d, i) { return !this.___locks }),
_changes,
rectsJSON = {data: null, serialised: true};
permutateColours(dormantRects,0, squares);
rebindFill.postChanges(rects, squares);
permutatePositions(dormantRects,0, squares);
rebindX.postChanges(rects, squares);
rebindY.postChanges(rects, squares);
$.when(rebindX.done, rebindY.done, rebindFill.done).done(function () {
squares_tick(squares)
});
return true
updateSquaresXY(_changes = getChanges(rects, squares));
});
}
function RebindWorker(keyDescriptor, updateThen) {
//dependency jquery Deferred
var dataFrame = TransfSelection(),
rebind = new Worker("updateSquares worker v2.js");
//custom methods
rebind.changes = function (buffer) {
var args;
changes = dataFrame.selection(buffer);
//the message serialisation process truncates trailing null array entries
//re-establish these by adjusting the length of each group in the selection
rects.forEach(function restoreLength(d, i) { changes[i].length = d.length });
//re-bind the d3 selection behaviour to the returned object
Object.keys(d3.selection.prototype).forEach(function (p, i, o) {
changes[p] = d3.selection.prototype[p]
});
//put the new data on the changed nodes
changes.each(function reData(d, i, j) {
d3.select(rects[j][i]).datum(d);
});
//put the dom elements on the newly created changes selection
changes.each(function reNode(d, i, j) {
changes[j][i] = rects[j][i];
});
updateThen(changes);
this.done.resolve(changes);
this.done = $.Deferred();
};
rebind.postChanges = function (rects, squares) {
var rects = dataFrame.selectionBuffer(rects),
squares = dataFrame.dataBuffer(squares),
data = {
method: "changes",
rects: rects,
squares: squares
};
rebind.postMessage(data, [rects.buffer]);
return data
};
rebind.key = function (data) {
this.done.resolve(data);
this.done = $.Deferred();
};
rebind.done = $.Deferred();
//standard methods
rebind.postMessage({
method: "key",
data: keyDescriptor
});
rebind.onmessage = function (e) {
//invoke the method on the data
this[e.data.method](e.data.data);
};
return rebind;
function selectionToBuff(selection) {
return selection.map(function group(g) {
return JSON.stringify(g.map(function node(d) {
return d.__data__
}));
});
}
function selectionFromBuff(selectionJSON) {
return selectionJSON.map(function (g) {
return JSON.parse(g).map(function (d) {
return d ? { __data__: d } : undefined
});
});
}
}
elapsedTime.message(function (value) {
var this_lap = this.lap().lastLap, aveLap = this.aveLap(this_lap);
return 'frame rate: ' + d3.format(" >7,.1f")(1/aveLap)
});
elapsedTime.start(1000);
d3.timer(function () {
elapsedTime.mark();
if(elapsedTime.aveLap.history.length)
hist(elapsedTime.aveLap.history);
})
});
Raw
updateSquares worker v2.js
/// <reference path="d3/d3 CB.js" />
// v2 modified to use indx plus one other property for key
var noop = function () {
return new Function();
};
var window = noop();
window.CSSStyleDeclaration = noop();
window.CSSStyleDeclaration.setProperty = noop();
window.Element = noop();
window.Element.setAttribute = noop();
window.Element.setAttributeNS = noop();
window.navigator = noop();
var document = noop();
document.documentElement = noop();
document.documentElement.style = noop();
document.createElement = function () { return this };
document.style = {};
document.style.setProperty = noop();
importScripts('https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js');
var key = (function () {
var keys, f, k0, kOther;
return function key(d) {
var k = d && d.data;
if (k) {
if (k.base) {
keys = Object.keys(k.base);
keys = k.include ? keys.filter(function (d, i, incl) {
return incl.indexOf(d) != -1;
})
: k.exclude ? keys.filter(function (d, i, ex) {
return ex.indexOf(d) === -1;
}) : keys;
k0 = keys[0]; kOther = keys.slice(1);
f = function _key(d, i) {
var dk0 = d[k0];
return kOther
.reduce(function r (k, p, i) {
return k + "_" + d[p];
}, dk0)
}
} else {
var k3 = k.length === 3;
f = function _key(d, i) {
return d? d[k[0]] + "_" + d[k[1]] + (k3 ? "_" + d[k[1]] : "") : d;
}
}
self.postMessage({ method: "key", data: true });
} else {
return f
}
}
})();
importScripts("messageObject.js");
var dataFrame = TransfSelection()
function changes(d) {
var squares = dataFrame.data(d.squares),
changeSelection = d3.selection.prototype
.data.call(dataFrame.selection(d.rects), squares, key()).exit(),
rects;
//console.log(d.rects.buffer.byteLength/d.rects.frame)
changeSelection.each(function (d, i, j) {
changeSelection[j][i].__data__ = squares[i]
});
self.postMessage({
method: "changes",
data: (rects = dataFrame.selectionBuffer(changeSelection))
}, [rects.buffer]);
}
self.onmessage = function (e) {
self[e.data.method](e.data)
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment