readme.md

Github Visualization

Run

• git clone
• bundle install
• rackup config.ru &
• navigate to localhost:3000

Description

This visualization tries to display latest events from the github api using a force layout graph from d3 As new events come in, they get added to the graph One event has 4 different types of nodes (event type, user, repo, repo-language) Nodes increment their size through time so its possible to see which nodes are having more activity Labels are used to show new or recently updated nodes The sliders help play around with the graph properties resulting in interesting layout and visual representation Cpu goes high due to the amount of processing, the graph would be better off using processing, easel or some other library that leverages the canvas, it was just an experiment with d3

Example

http://www.flickr.com/photos/calderas/sets/72157629749939356/

Libraries

• d3.js - http://d3js.org/
• jquery ui slider

app.rb

require 'rubygems'      # <-- Added this require
require 'em-websocket'
require 'sinatra/base'
require './github-api.rb'

EventMachine.run do     # <-- Changed EM to EventMachine
  class App < Sinatra::Base
      get '/' do
          File.read(File.join('public', 'index.html'))
      end
  end

  EventMachine::WebSocket.start(:host => '0.0.0.0', :port => 8080) do |ws| # <-- Added |ws|
      # Websocket code here
      @github_client = GithubApi.new
      @github_events = nil
      timer = nil
      ws.onopen {
          #ws.send "connected!!!!"
          puts "Ping supported: #{ws.pingable?}"
          timer = EM.add_periodic_timer(1) {
            @github_events = @github_client.get_json_events
            p ["Sent ping", ws.ping('hello')]
          }   
      }

      ws.onmessage { |msg|
          puts "got message #{msg}"
      }
      
      ws.onpong { |value|
        puts "Received pong: #{value}"
        ws.send(@github_events)
      }

      ws.onclose   {
        EM.cancel_timer(timer)
        ws.send "WebSocket closed"
      }
      
      ws.onerror { |e|
        puts "Error: #{e.message}"
      }

  end

  App.run!({:port => 3000})
end

config.ru

require './app'
run Sinatra::Application

data.js

//underscore find method doesnt provide the index of a found value
//this is the same method but returning the index rather the value
_.findIndex = function(obj, iterator, context) {
    var result;
    _.any(obj, function(value, index, list) {
      if (iterator.call(context, value, index, list)) {
        result = index;
        return true;
      }
    });
    return result;
  };

var github = (function(){
	var api={},
		events=[], 
		nodes=[],
		links,
		freq={}
		minSize=1,
		batchId=1;
	
	function createElements(data){
		//reset  links to return only new ones after the first run
		links=[];
		for (var i=0; i<data.length; i++) {
			var e = data[i];
			var nodeEvent = addOrFindNode(e.event_type, "event");
		 	var nodeUser = addOrFindNode(e.user, "user");
		 	var nodeRepo = addOrFindNode(e.repo, "repo");
		 	var nodeLang = addOrFindNode(e.lang, "lang");
			
			if( nodeEvent != -1 && nodeUser != -1) addLink(nodeEvent, nodeUser);
			if( nodeUser != -1 && nodeRepo != -1) addLink(nodeUser, nodeRepo);
			if( nodeRepo != -1 && nodeLang != -1) addLink(nodeRepo, nodeLang);
		}
	}
	
	function addOrFindNode(nodeName, category){
		var nodeIndex = -1;
		if (typeof nodeName != "undefined"){
			var objFreq = updateFrequency(nodeName, category);
			nodeIndex = _.findIndex(nodes, function(obj){ return obj.name == nodeName})
			
			if (typeof nodeIndex == "undefined"){
				var newNode={
					"name": nodeName, 
					"group": category, 
					"size": objFreq,
					"batch": batchId
				}
				nodes.push(newNode);
				nodeIndex = nodes.length-1;
			}else{
				//update frequency of existing node
				nodes[nodeIndex].size = objFreq;
				nodes[nodeIndex].batch = batchId;
			}
		}
		return nodeIndex;
	}
	
	function updateFrequency(nodeName, category, opt){
		if (typeof freq[category] === "undefined") { 
			freq[category] = {};
		}
		if (typeof freq[category][nodeName] === "undefined"){
				freq[category][nodeName] = minSize;
		}else{
			freq[category][nodeName]++;
		}
		
		return freq[category][nodeName];
	}
	
	function addLink(source, target, value){
		links.push({ 
			"source": source, 
			"target": target, 
			"value": value
		})
	}
	
	api.getGraph = function(data){
		var data = JSON.parse(data);
		
		//maintain a copy of all nodes to have the original index
		events = events.concat(data);
		createElements(data);
		
		var result = {
      		"nodes": nodes,
      		"links": links
    	};

		batchId++;
		
		return result;
	}
	
	api.freq=freq;
	
	return api;
})();

Gemfile

source 'http://rubygems.org'
gem 'rack'
gem 'em-websocket'
gem 'octokit'
gem 'sinatra'
gem 'thin'

Gemfile.lock

GEM
  remote: http://rubygems.org/
  specs:
    addressable (2.2.8)
    daemons (1.1.8)
    em-websocket (0.3.6)
      addressable (>= 2.1.1)
      eventmachine (>= 0.12.9)
    eventmachine (0.12.10)
    faraday (0.8.0)
      multipart-post (~> 1.1)
    faraday_middleware (0.8.7)
      faraday (>= 0.7.4, < 0.9)
    hashie (1.2.0)
    multi_json (1.3.4)
    multipart-post (1.1.4)
    octokit (1.0.5)
      addressable (~> 2.2)
      faraday (~> 0.8)
      faraday_middleware (~> 0.8)
      hashie (~> 1.2)
      multi_json (~> 1.3)
    rack (1.4.1)
    rack-protection (1.2.0)
      rack
    sinatra (1.3.2)
      rack (~> 1.3, >= 1.3.6)
      rack-protection (~> 1.2)
      tilt (~> 1.3, >= 1.3.3)
    thin (1.3.1)
      daemons (>= 1.0.9)
      eventmachine (>= 0.12.6)
      rack (>= 1.0.0)
    tilt (1.3.3)

PLATFORMS
  ruby

DEPENDENCIES
  em-websocket
  octokit
  rack
  sinatra
  thin

github-api.rb

require 'octokit'

class GithubApi
  
  @new_events  
  @old_events
  
  def initialize
    @old_events =[]
  end
  
  def get_json_events
    @new_events = []
    begin
      @github_events = Octokit.public_events
    rescue => e
      puts e
    end
    get_events_repo
    @old_events += @new_events
    @new_events.to_json
  end
  
  #get repo details for each event
  def get_events_repo
    @github_events.each do |event|
      repo = ""
    	if event.repo.name && event.repo.name != "/"
    	  puts event.repo.name
    	  begin
      	  repo = Octokit.repo(event.repo.name)
      	  event.repo.language = repo.language if repo.language
      	  event.repo.watchers = repo.watchers if repo.watchers
      	rescue => e
          puts e
        end
      end
      flatten_event(event, repo)
    end
  end
  
  #create a list of unique events like this: event_id	event_type user repo watchers	language
  def flatten_event(event, repo)
    lang = repo.language if repo != ""
    watchers = repo.watchers if repo != ""
    obj = {
      :event_id => event.id,
      :event_type => event.type,
      :user => event.actor.login,
      :repo => event.repo.name,
      :watchers =>watchers,
      :lang => lang
    }
    @new_events.push(obj) if @old_events.index { |i| i[:event_id] == event.id }.nil?
  end
end

index.html

<!DOCTYPE html>
<html>
	<head>
		<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
		<title>Github Universe</title>
		<!--libs-->
		<link type="text/css" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery.ui.core.css" rel="stylesheet"/>
		<link type="text/css" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery.ui.theme.css" rel="stylesheet"/>
		<link type="text/css" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery.ui.slider.css" rel="stylesheet"/>
		<script src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js' type="text/javascript"></script>
		<script src='http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.18/jquery-ui.min.js' type="text/javascript"></script>
		<script src='http://mbostock.github.com/d3/d3.js?2.7.2'></script>
		<script src='http://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.3.3/underscore-min.js' type="text/javascript"></script>
		
		<!--app-->
		<script src='data.js' type="text/javascript"></script>
		<script src="vis.js" type="text/javascript"></script>
		<style type="text/css">
			body{
				background-color: black;
				font-family: "Trebuchet MS";
				font-size: 1.1em;
			}
			
			text{
				cursor: pointer;
			}
			
			span{
				color:#365B73;
			}
			
			a{
				color:#4193A6;
			}
			
			circle{
				stroke-width: 2;
			}
			
			circle.event{
				fill: #E88D01;
				stroke: #BF8633;	
			}
			
			text.event{
				fill: #BF8633; /*FF7F0E;*/
			}
			
			circle.lang{
				fill: #FFFD32; /*1F893D;*/
				stroke: #B2B000;
			}
			
			text.lang{
				fill: #B2B000; /*2CA02C;*/
			}
			
			circle.repo{
				fill: #4193A6;/*#13409D;*/
				stroke: #365B73;
			}
			
			text.repo{
				fill: #365B73;/*1F77B4;*/
			}
			
			circle.user{
				fill: #A058B3;/*9552A6;*/
				stroke: #7D458C;
			}
			
			text.user{
				fill: #7D458C; /*9467BD;*/
			}
			
			#controls{
				position: absolute;
				padding-left:10px;
				width: 200px;
				color: #fff;
			}
			
			.ui-widget {font-size: 0.1em}
		</style>
	</head>
	<body>
		<div id="controls">
			Gravity: <span id="gravity-label">0.05</span>
			<div id="gravity" class="slider"></div>
			Distance: <span id="distance-label">200</span>
			<div id="distance" class="slider"></div>
			Charge: <span id="charge-label">0</span>
			<div id="charge" class="slider"></div>
			<span id="node-freq"></span> :
			<span id="node-type"></span> :
			<a id="node-link" href="" target="_blank"><span id="node-value"></span></a>
		</div>
		<div id="chart"></div>
	</body>
</html>

vis.js

var w = 2000,
    h = 1024,
    fill = d3.scale.category20(),
    nodes = [],
    links = [],
	color, vis, force,
	minSize = 0;

function initialize(){
	color = d3.scale.category20();

	vis = d3.select("#chart").append("svg")
	    .attr("width", w)
	    .attr("height", h);

   force = d3.layout.force()
	    .nodes(nodes)
	    .links(links)
		.on("tick", tick)
		.gravity(0.05)
		.distance(200)
		.charge(-50)
	    .size([w, h]);
	
	//start with some nodes so it doesnt look sad
	var data = "[{\"event_type\":\"PushEvent\",\"user\":\"GithubAPI\"},{\"event_type\":\"WatchEvent\",\"user\":\"GithubAPI\"},{\"event_type\":\"CreateEvent\",\"user\":\"GithubAPI\"},{\"event_type\":\"GistEvent\",\"user\":\"GithubAPI\"},{\"event_type\":\"PullRequestEvent\",\"user\":\"GithubAPI\"},{\"event_type\":\"IssueCommentEvent\",\"user\":\"GithubAPI\"}]";
	start(github.getGraph(data));
	restart();
}

function tick(){
  vis.selectAll("circle.node")
  	  .attr("cx", function(d) { return d.x; })
      .attr("cy", function(d) { return d.y; })
	  .attr("r", function(d)  { return Math.sqrt(d.size); });
	 	
  vis.selectAll("text.node")
	.attr("x", function(d) { return d.x; })
    .attr("y", function(d) { return d.y; })
	.text(function(d) { 
		if(d.batch >= minSize){ 
			return d.name; 
		}
	});	
};

function onMouseOver(data){
	var link,
		url = "http://github.com/";
	switch(data.group){
		case "event":
			link = "#";
			break;
		case "lang":
			link = url + "languages/" + data.name;
			break;
		default:
			link = url + data.name;
			break;
	}
	d3.select("#node-link").attr("href", link);
	d3.select("#node-type").html(data.group);
	d3.select("#node-value").html(data.name);
	d3.select("#node-freq").html(github.freq[data.group][data.name]);
}

function restart() {

  force
	.nodes(nodes)
	.links(links)
	.start();

  vis.selectAll("circle.node")
      .data(nodes)
     .enter().append("circle")
      .attr("class", function(d) { return "node " + d.group; })
      .attr("cx", function(d) { return d.x; })
      .attr("cy", function(d) { return d.y; })
	  .attr("r", function(d) { return d.size; })
	  .on("mouseover", function(d){onMouseOver(d)})
      .call(force.drag);  

  vis.selectAll("text.node")
	  .data(nodes)
	.enter().append("svg:text")
	 .attr("class", function(d) { return "node " + d.group; })
     .attr("dx", 12)
     .attr("dy", ".25em")
	 .on("mouseover", function(d){onMouseOver(d)})
	 .call(force.drag);

  vis.selectAll("circle.node").data(nodes).exit().remove();	
  vis.selectAll("text.node").data(nodes).exit().remove();

}

function start(data){
	data.nodes.forEach(function(node) {
		//find node if already exists
		var nodeIndex = _.findIndex(nodes, function(obj){ return obj.name === node.name})
		if (typeof nodeIndex === "undefined" ){
			nodes.push(node);
		} else {
			nodes[nodeIndex].size = node.size
		}
	});
	
	data.links.forEach(function(link){
		links.push(link);
	});
	
	minSize++;
	restart();
}

function debug(str){ 
	console.log(str);
};

$(document).ready(function(){
	//vis
	initialize();
	
	//sliders
	$( "#gravity.slider" ).slider({
		value:0.05,
		min: 0.1,
		max: 1.0,
		step: .01,
		slide: function( event, ui ) {
			$( "#gravity-label" ).html( ui.value );
			force.gravity( ui.value );
		}
	});
	
	$( "#distance.slider" ).slider({
		value:200,
		min: 0,
		max: 400,
		step: 10,
		slide: function( event, ui ) {
			$( "#distance-label" ).html( ui.value );
			force.distance( ui.value );
			restart();
		}
	});
	
	$( "#charge.slider" ).slider({
		value:-50,
		min: -150,
		max: 0,
		step: 10,
		slide: function( event, ui ) {
			$( "#charge-label" ).html( ui.value );
			force.charge( ui.value );
			restart();
		}
	});
	
	//websocket
	ws = new WebSocket("ws://0.0.0.0:8080");
	
	ws.onmessage = function(evt) { 
		start(github.getGraph(evt.data));
	};
	ws.onclose = function() { 
		debug("socket closed"); 
	};
	ws.onopen = function() {
		debug("connected...");
		ws.send("hello server");
	};
	
});