Skip to content

Instantly share code, notes, and snippets.

@chasemc

chasemc/_run.sh

Forked from jexp/_run.sh
Created January 24, 2021 19:14
Show Gist options
  • Save chasemc/6b6e0e52df9313e098c96b3d36094eb8 to your computer and use it in GitHub Desktop.
Save chasemc/6b6e0e52df9313e098c96b3d36094eb8 to your computer and use it in GitHub Desktop.
Download ZIP
Rendering large graphs with vivagraph.js, neo4j-javscript-driver (binary-bolt), meetup dataset and compiled runtime. Oh the joy :)
Raw
_run.sh
npm install neo4j-driver
node test-neo-driver.js
Raw
index.html
<!--
bower install neo4j-driver
bower install vivagraphjs
python -m SimpleHTTPServer 8002
open http://localhost:8002
-->
<html>
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8">
<title>Neo4j NGraph Test</title>
<script src="bower_components/neo4j-driver/lib/browser/neo4j-web.min.js"></script>
<script src="bower_components/vivagraphjs/dist/vivagraph.js"></script>
<script type="text/javascript" charset="utf-8">
function onload() {
var neo = neo4j.v1;
var driver = neo.driver("bolt://localhost", neo.auth.basic("neo4j", "test"));
var session = driver.session();
var dump = {
onNext: function(record) { console.log(record.keys, record.length, record._fields, record._fieldLookup); },
onCompleted: function() { console.log("Completed"); },
onError: console.log
}
// session.run("MATCH (n) RETURN COUNT(*)").subscribe(dump);
var counter = function() {
var start = Date.now();
return {
count : 0,
onNext: function(r) { this.count++; },
onCompleted: function() { console.log("rows",this.count,"took",(Date.now()-start)); }}
};
// session.run("CYPHER runtime=compiled MATCH (n) RETURN id(n)").subscribe(counter());
// var graphGenerator = Viva.Graph.generator();
// var graph = graphGenerator.balancedBinTree(10);
var graph = Viva.Graph.graph();
// graph.addLink(1, 2);
var layout = Viva.Graph.Layout.forceDirected(graph, {
springLength : 30,
springCoeff : 0.0008,
dragCoeff : 0.01,
gravity : -1.2,
theta : 1
});
var nodeColor = 0x009ee8FF, // hex rrggbb
nodeSize = 6;
var colors = {Member:0x008cc1FF, Topic: 0x58b535FF,Group:0xf58220FF};
var graphics = Viva.Graph.View.webglGraphics();
// shader program is overkill and circles make it slow
// first, tell webgl graphics we want to use custom shader
// to render nodes:
// var circleNode = buildCircleNodeShader();
// graphics.setNodeProgram(circleNode);
// second, change the node ui model, which can be understood
// by the custom shader:
graphics.node(function (node) {
var color = colors[node.data] || 0x0f5788;
var degree = node.links.length;
var size = Math.log(degree + 1)*5;
// console.log("color",color,"data",node.data,"size",size,"degree",degree)
return new Viva.Graph.View.webglSquare(size, color);
// return new WebglCircle(nodeSize, nodeColor);
});
graphics.link(function (link) {
return Viva.Graph.View.webglLine(0x909090A0); // light transparent gray
});
var renderer = Viva.Graph.View.renderer(graph,
{
layout : layout,
graphics : graphics,
renderLinks : true,
prerender : true,
container: document.getElementById('graph')
});
var count = 0;
var finished = 0;
var viva = {
onNext: function(record) {
count ++;
var n1 = record._fields[0];
// console.log(n1);
if (record.length == 2) {
graph.addNode(n1);
}
if (record.length == 2) {
var n2 = record._fields[1];
graph.addLink(n1, n2);
}
if (record.length == 4) {
var n2 = record._fields[2];
graph.addNode(n1, record._fields[1])
graph.addNode(n2, record._fields[3])
graph.addLink(n1, n2);
}
if (count % 5000 == 0) console.log("Currently",count,"links");
},
onCompleted: function() {
console.log("Query finished, currently ",count,"links");
// render after all data was added
// renderer.run();
finished ++;
if (finished == 3) {
setTimeout(function() { console.log("Pausing renderer"); renderer.pause(); },10000);
}
}
};
function query(pattern, limit) {
limit = limit||10000;
var statement = "CYPHER runtime=compiled MATCH "+pattern+" RETURN id(from) as n, from.type as nt, id(to) as m, to.type as mt LIMIT "+limit;
console.log("Running",statement);
session.run(statement).subscribe(viva);
}
// session.run("CYPHER runtime=compiled MATCH (n) RETURN id(n) as id LIMIT 10").subscribe(viva);
query("(to:Topic)<--(from:Group)",1000);
query("(to:Group)<--(from:Member)",100000);
query("(to:Topic)<--(from:Member)",100000);
renderer.run(); // render incrementally as data is added
}
// Lets start from the easiest part - model object for node ui in webgl
function WebglCircle(size, color) {
this.size = size;
this.color = color;
}
// Next comes the hard part - implementation of API for custom shader
// program, used by webgl renderer:
function buildCircleNodeShader() {
// For each primitive we need 4 attributes: x, y, color and size.
var ATTRIBUTES_PER_PRIMITIVE = 4,
nodesFS = [
'precision mediump float;',
'varying vec4 color;',
'void main(void) {',
' if ((gl_PointCoord.x - 0.5) * (gl_PointCoord.x - 0.5) + (gl_PointCoord.y - 0.5) * (gl_PointCoord.y - 0.5) < 0.25) {',
' gl_FragColor = color;',
' } else {',
' gl_FragColor = vec4(0);',
' }',
'}'].join('\n'),
nodesVS = [
'attribute vec2 a_vertexPos;',
// Pack color and size into vector. First elemnt is color, second - size.
// Since it's floating point we can only use 24 bit to pack colors...
// thus alpha channel is dropped, and is always assumed to be 1.
'attribute vec2 a_customAttributes;',
'uniform vec2 u_screenSize;',
'uniform mat4 u_transform;',
'varying vec4 color;',
'void main(void) {',
' gl_Position = u_transform * vec4(a_vertexPos/u_screenSize, 0, 1);',
' gl_PointSize = a_customAttributes[1] * u_transform[0][0];',
' float c = a_customAttributes[0];',
' color.b = mod(c, 256.0); c = floor(c/256.0);',
' color.g = mod(c, 256.0); c = floor(c/256.0);',
' color.r = mod(c, 256.0); c = floor(c/256.0); color /= 255.0;',
' color.a = 1.0;',
'}'].join('\n');
var program,
gl,
buffer,
locations,
utils,
nodes = new Float32Array(64),
nodesCount = 0,
canvasWidth, canvasHeight, transform,
isCanvasDirty;
return {
/**
* Called by webgl renderer to load the shader into gl context.
*/
load : function (glContext) {
gl = glContext;
webglUtils = Viva.Graph.webgl(glContext);
program = webglUtils.createProgram(nodesVS, nodesFS);
gl.useProgram(program);
locations = webglUtils.getLocations(program, ['a_vertexPos', 'a_customAttributes', 'u_screenSize', 'u_transform']);
gl.enableVertexAttribArray(locations.vertexPos);
gl.enableVertexAttribArray(locations.customAttributes);
buffer = gl.createBuffer();
},
/**
* Called by webgl renderer to update node position in the buffer array
*
* @param nodeUI - data model for the rendered node (WebGLCircle in this case)
* @param pos - {x, y} coordinates of the node.
*/
position : function (nodeUI, pos) {
var idx = nodeUI.id;
nodes[idx * ATTRIBUTES_PER_PRIMITIVE] = pos.x;
nodes[idx * ATTRIBUTES_PER_PRIMITIVE + 1] = -pos.y;
nodes[idx * ATTRIBUTES_PER_PRIMITIVE + 2] = nodeUI.color;
nodes[idx * ATTRIBUTES_PER_PRIMITIVE + 3] = nodeUI.size;
},
/**
* Request from webgl renderer to actually draw our stuff into the
* gl context. This is the core of our shader.
*/
render : function() {
gl.useProgram(program);
gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
gl.bufferData(gl.ARRAY_BUFFER, nodes, gl.DYNAMIC_DRAW);
if (isCanvasDirty) {
isCanvasDirty = false;
gl.uniformMatrix4fv(locations.transform, false, transform);
gl.uniform2f(locations.screenSize, canvasWidth, canvasHeight);
}
gl.vertexAttribPointer(locations.vertexPos, 2, gl.FLOAT, false, ATTRIBUTES_PER_PRIMITIVE * Float32Array.BYTES_PER_ELEMENT, 0);
gl.vertexAttribPointer(locations.customAttributes, 2, gl.FLOAT, false, ATTRIBUTES_PER_PRIMITIVE * Float32Array.BYTES_PER_ELEMENT, 2 * 4);
gl.drawArrays(gl.POINTS, 0, nodesCount);
},
/**
* Called by webgl renderer when user scales/pans the canvas with nodes.
*/
updateTransform : function (newTransform) {
transform = newTransform;
isCanvasDirty = true;
},
/**
* Called by webgl renderer when user resizes the canvas with nodes.
*/
updateSize : function (newCanvasWidth, newCanvasHeight) {
canvasWidth = newCanvasWidth;
canvasHeight = newCanvasHeight;
isCanvasDirty = true;
},
/**
* Called by webgl renderer to notify us that the new node was created in the graph
*/
createNode : function (node) {
nodes = webglUtils.extendArray(nodes, nodesCount, ATTRIBUTES_PER_PRIMITIVE);
nodesCount += 1;
},
/**
* Called by webgl renderer to notify us that the node was removed from the graph
*/
removeNode : function (node) {
if (nodesCount > 0) { nodesCount -=1; }
if (node.id < nodesCount && nodesCount > 0) {
// we do not really delete anything from the buffer.
// Instead we swap deleted node with the "last" node in the
// buffer and decrease marker of the "last" node. Gives nice O(1)
// performance, but make code slightly harder than it could be:
webglUtils.copyArrayPart(nodes, node.id*ATTRIBUTES_PER_PRIMITIVE, nodesCount*ATTRIBUTES_PER_PRIMITIVE, ATTRIBUTES_PER_PRIMITIVE);
}
},
/**
* This method is called by webgl renderer when it changes parts of its
* buffers. We don't use it here, but it's needed by API (see the comment
* in the removeNode() method)
*/
replaceProperties : function(replacedNode, newNode) {},
};
}
</script>
<style type="text/css" media="screen">
html, body, svg { width: 100%; height: 100%;}
</style>
</head>
<body id="index" onload="onload();">
<div id="graph" style="width:100%;height:100%;background-color: white;"></div>
</body>
</html>
Raw
test-neo-driver.js
var neo4j = require('neo4j-driver').v1;
var driver = neo4j.driver("bolt://localhost", neo4j.auth.basic("neo4j", "test"));
var session = driver.session();
var dump = {
onNext: function(record) { console.log(record.keys, record.length, record._fields, record._fieldLookup); },
onCompleted: function() { console.log("Completed"); },
onError: console.log
}
session.run("MATCH (n) RETURN COUNT(*)").subscribe(dump);
var counter = function() {
var start = Date.now();
return {
count : 0,
onNext: function(r) { this.count++; },
onCompleted: function() { console.log("rows",this.count,"took",(Date.now()-start)); }}
};
session.run("CYPHER runtime=compiled MATCH (n) RETURN id(n)").subscribe(counter());
session.run("CYPHER runtime=compiled MATCH (n)-->(m) RETURN id(n),id(m)").subscribe(counter());
session.run("CYPHER runtime=compiled MATCH (n:User) RETURN id(n)").subscribe(counter());
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment