Skip to content

Instantly share code, notes, and snippets.

@moust
Created April 24, 2019 10:20
Show Gist options
  • Save moust/ab1d49a9b56a64c85121cb7d5e678dcb to your computer and use it in GitHub Desktop.
Save moust/ab1d49a9b56a64c85121cb7d5e678dcb to your computer and use it in GitHub Desktop.
Spark unit test context
class OptimizerSpec extends FlatSpec with Matchers with SparkSetup {
"solve" should "work with just intercept" in withSparkSession { session =>
...
}
}
trait SparkSetup {
private def ignore(x: Any): Unit = {}
private val conf: SparkConf = new SparkConf()
.setMaster("local[1]")
.setAppName("Spark test")
.set("spark.sql.shuffle.partitions", "1")
def withSparkContext(testMethod: (SparkContext) => Any): Unit = {
val sparkContext = new SparkContext(conf)
try {
ignore(testMethod(sparkContext))
}
finally sparkContext.stop()
}
def withSparkSession(testMethod: (SparkSession) => Any): Unit = {
val sparkSession = SparkSession.builder().config(conf).getOrCreate()
try {
ignore(testMethod(sparkSession))
}
finally sparkSession.stop()
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment