spark-csv-example.scala

import org.apache.spark.sql.SQLContext

def etl(sqlContext: SQLContext, source: String, destination: String): Unit = {
  val df = sqlContext.read.format("com.databricks.spark.csv").option("header", "true").load(source)
  df.select("year", "model").write.format("com.databricks.spark.csv").save(destination) 
}

// etl(new SQLContext(sc), "cars.csv", "newcars.csv")

val testContext = TestSQLContext()
val testDF = TestDataFrame()

testContext.load("cars.csv") returns testDF
etl(textContext, "cars.csv", "newcars.csv")

there was one(testDF).select("year", "model")
//there was one(testDF).save("newcars.csv")

これはテストする意味ないね。。。