Skip to content

Instantly share code, notes, and snippets.

@j-thepac
Created April 5, 2022 03:44
Show Gist options
  • Save j-thepac/166a5118f63ceedab2e255584ff43348 to your computer and use it in GitHub Desktop.
Save j-thepac/166a5118f63ceedab2e255584ff43348 to your computer and use it in GitHub Desktop.
Send RDD in Streaming
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object DStreamSendRDD extends App {
val conf = new SparkConf().setMaster("local[*]").setAppName("StreamingTransformExample")
val ssc = new StreamingContext(conf, Seconds(5))
val rdd1 = ssc.sparkContext.parallelize(Array(1, 2, 3))
val rddQueue = scala.collection.mutable.Queue[RDD[Int]]() //scala.collection.mutable.Queue[RDD[MyObject]]()
rddQueue.enqueue(rdd1)
val numsDStream = ssc.queueStream(rddQueue, true)
val plusOneDStream = numsDStream.map(x => x + 1)
plusOneDStream.print()
ssc.start()
ssc.awaitTermination()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment