Skip to content

Instantly share code, notes, and snippets.

@j-thepac
Created April 5, 2022 03:47
Show Gist options
  • Save j-thepac/c40c17a5c7b1d6cae126b6ac2ad5d923 to your computer and use it in GitHub Desktop.
Save j-thepac/c40c17a5c7b1d6cae126b6ac2ad5d923 to your computer and use it in GitHub Desktop.
Data into Socket
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.DStream
//nc -l -p 9999
object DStreamSoacket extends App {
val conf = new SparkConf().setMaster("local[*]").setAppName("App name")
val sc = new SparkContext(conf)
val ssc = new StreamingContext(sc, Seconds(5))
val lines: DStream[String] = ssc.socketTextStream(hostname = "localhost", port = 9999)
val wordCounts = lines.flatMap(_.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment