Example of common Spark Test setup

SparkTestSetup.scala

import org.apache.spark.{SparkConf, SparkContext}
import org.scalatest.{BeforeAndAfterAll, Suite}

trait SparkTestSetup extends BeforeAndAfterAll { this: Suite =>

  // Overwrite if it is neccesary
  def master = "local[4]"
  def appName: String
  def defaultSparkConf = new SparkConf().setAppName(appName).setMaster(master)

  var sc: SparkContext = _

  override def beforeAll(): Unit = {
    sc = new SparkContext(defaultSparkConf)
    super.beforeAll()
  }

  override def afterAll(): Unit = {
    try {
      super.afterAll()
    }
    finally {
      if(!sc.isStopped) sc.stop()
    }
  }
}