Scala

- Apache Spark.md

Spark / EMR cookbook

loops.sc

val sample: Array[String] = . . . . .

// 1. foreach style
sample.foreach { println }

// 2. comprehension style
for (name <- sample) println(name)

for (name <- sample if name.startsWith("2"))
     | println(name)

Spark.sc

// Initializing Spark
val conf: SparkConf = new SparkConf().setAppName("myapp").setMaster("local[*]");
val sc: JavaSparkContext = new JavaSparkContext(conf);


val rdd = sc.textFile("/my/path/*.gz")
// Persist into memory for fast re-use
rdd.cache()

// Basic map reduce
val lineLength = rdd.map(l => l.length)
val totalLength = lineLength.reduce((a,b)=>a+b)

//  or simply
rdd.map(s=>s.length).reduce((a,b) => a + b)


// rdd from collection
val data = Array(1, 2, 3, 4, 5)
val distData = sc.parallelize(data)

val accum = sc.longAccumulator
data.map { x => accum.add(x); x }