dmpetrov · March 6, 2017 05:07 · iretex · Jan 21, 2020
diff --git a/save_dataframe_in_single_csv.scala b/save_dataframe_in_single_csv.scala
 # Code for blogpost:
 # https://fullstackml.com/2015/12/21/how-to-export-data-frame-from-apache-spark/

 def saveDfToCsv(df: DataFrame, tsvOutput: String,
                sep: String = ",", header: Boolean = false): Unit = {
    val tmpParquetDir = "Posts.tmp.parquet"
  
    df.repartition(1).write.
        format("com.databricks.spark.csv").
        option("header", header.toString).
        option("delimiter", sep).
        save(tmpParquetDir)
  
    val dir = new File(tmpParquetDir)
    val tmpTsvFile = tmpParquetDir + File.separatorChar + "part-00000"
    (new File(tmpTsvFile)).renameTo(new File(tsvOutput))
  
    dir.listFiles.foreach( f => f.delete )
    dir.delete
 }
	# Code for blogpost:
	# https://fullstackml.com/2015/12/21/how-to-export-data-frame-from-apache-spark/

	def saveDfToCsv(df: DataFrame, tsvOutput: String,
	sep: String = ",", header: Boolean = false): Unit = {
	val tmpParquetDir = "Posts.tmp.parquet"

	df.repartition(1).write.
	format("com.databricks.spark.csv").
	option("header", header.toString).
	option("delimiter", sep).
	save(tmpParquetDir)

	val dir = new File(tmpParquetDir)
	val tmpTsvFile = tmpParquetDir + File.separatorChar + "part-00000"
	(new File(tmpTsvFile)).renameTo(new File(tsvOutput))

	dir.listFiles.foreach( f => f.delete )
	dir.delete
	}