Jeremy Freeman freeman-lab

Notes on compiling chromium using an EC2 cluster

This note lays out the machine specifications and configurations used to get a distributed Chome build working on cloud compute (in this case AWS EC2) using the icecc tool.

These links were a good starting point:

but I couldn't fine a full walk through, so hopefully we can make it easier for others.

	package thunder.streaming

	import org.apache.spark.{SparkConf, Logging}
	import org.apache.spark.rdd.RDD
	import org.apache.spark.SparkContext._
	import org.apache.spark.streaming._
	import org.apache.spark.streaming.dstream.DStream
	import org.apache.spark.mllib.clustering.KMeansModel

	import scala.util.Random.nextDouble

	/**
	* bisecting <master> <input> <nNodes> <subIterations>
	*
	* divisive hierarchical clustering using bisecting k-means
	* assumes input is a text file, each row is a data point
	* given as numbers separated by spaces
	*
	*/

	import org.apache.spark.SparkContext

	import java.io.{FileFilter, File}
	import org.apache.hadoop.fs.Path
	import org.apache.hadoop.io.{LongWritable, BytesWritable}
	import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
	import org.apache.hadoop.mapreduce.{JobContext, InputSplit, RecordReader, TaskAttemptContext}

	/**
	* Custom Input Format for reading and splitting flat binary files that contain records, each of which
	* are a fixed size in bytes. The fixed record size is specified through a parameter recordLength
	* in the Hadoop configuration.

	var THREE = require('three.js');
	var _ = require('lodash');


	var ParticleTest = function(selector, data, images, opts) {

	var width = $(selector).width();
	var height = width * 0.7;

	from numpy import array
	from thunder import Images, ThunderContext

	tsc = ThunderContext.start()

	rawpath = ''
	savepath = ''

	data = tsc.loadImages(rawpath, inputFormat='tif', nplanes=1)