spark tutorial commands

gistfile1.scala

val data = 1 to 10000
val distData = sc.parallelize(data)
//display values less than 10
distData.filter(_ < 10).collect()

//Word count

val f = sc.textFile("README.md")
val wc = f.flatMap(l => l.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
wc.saveAsTextFile("wc_out")

//how many times spark is used
grep Spark wc_out/part-*

//clk.tsv
2014-03-04	15dfb8e6cc4111e3a5bb600308919594	11
2014-03-06	81da510acc4111e387f3600308919594	61

//reg.tsv
2014-03-02	15dfb8e6cc4111e3a5bb600308919594	1	33.6599436237	-117.958125229
2014-03-04	81da510acc4111e387f3600308919594	2	33.8570099635	-117.855744398

val format = new java.text.SimpleDateFormat("yyyy-MM-dd")
case class Register (d: java.util.Date, uuid: String, cust_id: String, lat: Float, lng: Float)
case class Click (d: java.util.Date, uuid: String, landing_page: Int)
val reg = sc.textFile("reg.tsv").map(_.split("\t")).map(
r => (r(1), Register(format.parse(r(0)), r(1), r(2), r(3).toFloat, r(4).toFloat)))
val clk = sc.textFile("clk.tsv").map(_.split("\t")).map(
 c => (c(1), Click(format.parse(c(0)), c(1), c(2).trim.toInt))
)
reg.join(clk).collect()



val rm = sc.textFile("README.md")
val cl = sc.textFile("RELEASE")

val cl_wc = cl.flatMap(l => l.split(" ")).filter(_ == "Spark").map(word => (word, 1)).reduceByKey(_ + _)
val cl_wc = cl.flatMap(l => l.split(" ")).filter(_ == "Spark").map(word => (word, 1)).reduceByKey(_ + _)

rm_wc.join(cl_wc).collect()