Wordcount from an input stream within 1) a batch 2) a window

sparkStreaming-wordcount-window.scala

import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.{SparkConf, SparkContext}

/*
  Streaming batch size = 10 seconds
  Window size = 20 secs
  Window slide interval = 10 secs
*/

val conf = new SparkConf().setAppName("sbtapp").setMaster("local[3]")
val ssc = new StreamingContext(conf,Seconds(10))
ssc.sparkContext.setLogLevel("ERROR")

val dStream = ssc.socketTextStream("localhost",9999)
val pairs = dStream.map(x=> (x,1)) //create key value

// word count in the batch
val wcount = pairs.reduceByKey((a,n)=> a+n) 
wcount.print()
/ word count in the window
val wcount_window = pairs.reduceByKeyAndWindow((a:Int,n:Int) => (a+n), Seconds(20), Seconds(10))
wcount_window.print()

ssc.start
ssc.stop(true)

/* Output:

-------------------------------------------
Time: 1504424600000 ms
-------------------------------------------
(w,2)
(q,2)

-------------------------------------------
Time: 1504424600000 ms
-------------------------------------------
(w,2)
(q,2)

-------------------------------------------
Time: 1504424610000 ms
-------------------------------------------
(w,2)
(q,2)

-------------------------------------------
Time: 1504424610000 ms
-------------------------------------------
(w,4)
(q,4)
*/

to do: modify to reflect actul wordcount using flatMap