soumyasd

package redisbenchmark

import java.util.UUID
import java.util.concurrent.ThreadLocalRandom

import akka.actor.ActorSystem
import akka.stream.{MaterializerSettings, OverflowStrategy, FlowMaterializer}
import akka.stream.scaladsl.{TickSource, IterableSource, Source}
import akka.util.ByteString
import redis.RedisClient

soumyasd

1. General Background and Overview

• Probabilistic Data Structures for Web Analytics and Data Mining : A great overview of the space of probabilistic data structures and how they are used in approximation algorithm implementation.
• Models and Issues in Data Stream Systems
• Philippe Flajolet’s contribution to streaming algorithms : A presentation by Jérémie Lumbroso that visits some of the hostorical perspectives and how it all began with Flajolet
• Approximate Frequency Counts over Data Streams by Gurmeet Singh Manku & Rajeev Motwani : One of the early papers on the subject.
• [Methods for Finding Frequent Items in Data Streams](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.187.9800&rep

soumyasd

import akka.actor._
import akka.util.ByteString
import spray.http.HttpEntity.Empty
import spray.http.MediaTypes._
import spray.http._
import spray.routing.{HttpService, RequestContext, SimpleRoutingApp}
 
object StreamingActor {
 
  // helper methods

soumyasd

# using four part files to construct "minitweet"
cat rawtweets/part-0000[1-3] > minitweets

# change log4j properties to WARN to reduce noise during demo
mv conf/log4j.properties.template conf/log4j.properties
vim conf/log4j.properties # Change to WARN

# launch Spark shell REPL
./bin/spark-shell

soumyasd

/* 
 * Object in scala for calculating cosine similarity
 * Reuben Sutton - 2012
 * More information: http://en.wikipedia.org/wiki/Cosine_similarity
 */

object CosineSimilarity {
  
  /*
   * This method takes 2 equal length arrays of integers 

soumyasd

import org.apache.spark.mllib.linalg.distributed.RowMatrix
import org.apache.spark.mllib.linalg._
import org.apache.spark.{SparkConf, SparkContext}

// To use the latest sparse SVD implementation, please build your spark-assembly after this
// change: https://github.com/apache/spark/pull/1378

// Input tsv with 3 fields: rowIndex(Long), columnIndex(Long), weight(Double), indices start with 0
// Assume the number of rows is larger than the number of columns, and the number of columns is
// smaller than Int.MaxValue

soumyasd

/** Find the minimum amount of smoke (second) and resulting color (first) 
    by splitting the sequence at every possible position,
	given `lookup` contains the best mix for subsequence. */
def minSmokeMixtureSingle(s: Seq[Int], lookup: Map[Seq[Int],(Int,Int)]): (Int,Int) = 
	if(s.size == 1)
		(s(0),0)
	else if(s.size == 2)
		mix(s(0),s(1))
	else {
		val splits = (1 to (s.size - 1)).map(s.splitAt)

soumyasd

#!/usr/bin/env scalas

/***
scalaVersion := "2.11.7"
libraryDependencies += "com.typesafe.play" %% "play-netty-server" % "2.4.1"
*/

import play.core.server._
import play.api.routing.sird._
import play.api.mvc._

soumyasd

class HttpClient {
  def get(uri: URI): Future[String]
}

// return Future of next URI to get
def getUri(): Future[URI]

// return's available HttpClient from a pool
def getHttpClient(): Future[HttpClient]

soumyasd

import java.net.{SocketOptions, Inet4Address, InetAddress, Socket}
import java.util.concurrent.TimeoutException

import akka.actor.{ActorRef, Props, ActorSystem}
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.HttpMethods._
import akka.http.scaladsl.model._
import akka.http.scaladsl.model.ws._
import akka.io.Inet
import akka.routing.BroadcastGroup