search_succinct.scala

import edu.berkeley.cs.succinct._


val conf = new SparkConf().setAppName("Ranking Example")

val sc = new SparkContext(conf)


// A large file of raw hip hop lyrics ~ 100 GB
val hipHopRDD = sc.textFile("/hiphopcorpus").map(_.getBytes)

// Persist the data
val hipHopRDDPersisted = hipHopRDD.succcinct.persist()

// Search for all mentions 
val countsLyrics = hipHopRDDPersisted.search("Birthdays were the worst days")

// 3 returns