Spark Stream

SimpleStream.scala

package ch06
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.lit
import org.apache.spark.sql.types.StringType
//nc -l -p 9999
object SimpleStream extends App {

  val spark = SparkSession.builder
    .master("local")
    .appName("StructuredNetworkWordCount")
    .getOrCreate()
  spark.sparkContext.setLogLevel("ERROR")
  var max = 0
  val lines = spark.readStream
    .format("socket")
    .option("host", "localhost")
    .option("port", 9999)
    .load()

  val query = lines.writeStream
    .outputMode("append") //complete ,update
    .format("console")
    .start()
    .awaitTermination()

}