Creating DataFrame from weird CSV files

spark_weird_csv.scala

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.io.{LongWritable, Text}
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat

def wcsv_to_df(
    fileName: String, 
    tableName: String, 
    columns: Array[String], 
    fieldTerminator: String, 
    lineTerminator: String
) = {
    val conf = new Configuration(sc.hadoopConfiguration)
    conf.set("textinputformat.record.delimiter", lineTerminator)
    // Read csv
    val rdd = sc.newAPIHadoopFile(fileName, classOf[TextInputFormat], classOf[LongWritable], classOf[Text], conf)
        .map { case (_, text) => text.toString }
        .map(_.replaceAll(fieldTerminator, "\t"))
    // Convert it to DF
    val df = spark.read
        .option("sep", "\t")
        .option("inferSchema", "true")
        .csv(rdd.toDS)
        .toDF(columns: _*)
    df.createOrReplaceTempView(tableName)
    df
}