Finding unique records between ORC file and MySQL rows using Apache Spark

unique_orc_records.scala

import spark.implicits._
import org.apache.spark.sql.SaveMode
val products = spark.sqlContext.read.format("jdbc").option("driver", "com.mysql.jdbc.Driver").option("dbtable", "products").option("user", "gobblin").option("password", "gobblin").option("url", "jdbc:mysql://localhost/mopar_demo").load()

val newProducts = spark.sqlContext.read.format("orc").load("/Users/tilak/gobblin/mopar-demo/output/org/apache/gobblin/copy/user/tilak/pricing.products_1521799535.csv/20180325023900_append/part.task_PullCsvFromS3_1521945534992_0_0.orc")

val newnewProducts = newProducts.except(products)

val dfWriter = newnewProducts.write.mode(SaveMode.Append)
val connectionProperties = new java.util.Properties()
connectionProperties.setProperty("Driver", "com.mysql.jdbc.Driver")
connectionProperties.setProperty("user", "root")
connectionProperties.setProperty("password", "1")
dfWriter.jdbc("jdbc:mysql://localhost/mopar_demo", "products", connectionProperties)