Tom Mulder darkseed

#!/usr/bin/env bash

# Assuming OS X Yosemite 10.10.4

# Install XCode and command line tools
# See https://itunes.apple.com/us/app/xcode/id497799835?mt=12#
# See https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man1/xcode-select.1.html
xcode-select --install

General Background and Overview

Probabilistic Data Structures for Web Analytics and Data Mining : A great overview of the space of probabilistic data structures and how they are used in approximation algorithm implementation.
Models and Issues in Data Stream Systems
Philippe Flajolet’s contribution to streaming algorithms : A presentation by Jérémie Lumbroso that visits some of the hostorical perspectives and how it all began with Flajolet
Approximate Frequency Counts over Data Streams by Gurmeet Singh Manku & Rajeev Motwani : One of the early papers on the subject.
[Methods for Finding Frequent Items in Data Streams](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.187.9800&rep=rep1&t

	import breeze.linalg._
	import breeze.stats._
	import scala.math.sqrt


	/**
	* Effecient for sparse vectors. Scales in O(activeSize)
	*/
	// Must take SparseVector, for implicits to be linked correctly
	def pearson(a: SparseVector[Double], b: SparseVector[Double]): Double = {

	import breeze.linalg._
	import breeze.stats
	import breeze.numerics._

	val dataFile = new File(???)
	val userItems: Array[SparseVector[Double]] = loaderUserItemsWithRatings(dataFile, """[ ,:]""".r)
	val itemUsers: Array[SparseVector[Double]] = transpose(userItems) map { vec => normalize(vec, 2) }

	// weights
	val N = DenseVector.fill[Double](itemIndex.size)(userIndex.size) // vector where total numbers of users is repeated

	'''
	DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
	Version 2, December 2004

	Copyright (C) 2004 Sam Hocevar <[email protected]>

	Everyone is permitted to copy and distribute verbatim or modified
	copies of this license document, and changing it is allowed as long
	as the name is changed.

	import com.twitter.algebird.{Aggregator, Semigroup}
	import com.twitter.scalding._

	import scala.util.Random

	/**
	* This job is a tutorial of sorts for scalding's Execution[T] abstraction.
	* It is a simple implementation of Lloyd's algorithm for k-means on 2D data.
	*
	* http://en.wikipedia.org/wiki/K-means_clustering

	from pylab import *
	from numpy.random import dirichlet, rand, binomial, uniform, normal

	def _unit_weight(dim):
	return ones(dim) / float(dim)

	ONE_FRAC = 0.5
	SQRT_TWO_INV = 1.0 / sqrt(2.0)
	def _feature_vec(dim, method="bernoulli"):
	if method == "bernoulli":

	import breeze._
	import breeze.linalg._
	import breeze.numerics._
	import java.awt.image.BufferedImage
	import javax.imageio.ImageIO

	val f = ???
	val img = javax.imageio.ImageIO.read(new File(f))
	val gray = new BufferedImage(img.getWidth, img.getHeight, BufferedImage.TYPE_BYTE_GRAY)
	val g = gray.createGraphics()

	// min-hash

	val fs: Vector[Int => Int] // hash funkce

	items map { it => fs map { f => f(it) } } fold (vectorPairwise(min), initialValue = Vector.fill(infinity))



	// HyperLogLog

	case class ErrorMessage(message: String, cause: String)
	object ErrorMessage {
	import spray.json.DefaultJsonProtocol._
	implicit val errorFormat = jsonFormat2(ErrorMessage.apply)
	}
	import spray.httpx.SprayJsonSupport._
	implicit val jsonRejectionHandler = RejectionHandler {
	case MalformedRequestContentRejection(msg, cause) :: Nil =>
	complete(StatusCodes.BadRequest, ErrorMessage("The request content was malformed", msg))