Skip to content

Instantly share code, notes, and snippets.

@ottomata
Created February 14, 2018 19:23
Show Gist options
  • Save ottomata/942331ad6c950426592fd03d490affd6 to your computer and use it in GitHub Desktop.
Save ottomata/942331ad6c950426592fd03d490affd6 to your computer and use it in GitHub Desktop.
it should "convert with case classes and null" in {
case class BA(
b1: String = "b1"
)
case class BB(
b2: String = "b2",
b1: String = "b1"
)
case class Input(
a2: String = "aa2",
b: BA = BA(),
a1: String = "aa1"
)
case class Table(
a1: String = "ab1",
a2: String = "ab2",
b: BB = BB()
)
val sqlContext = new SQLContext(sc)
val tableDf = sqlContext.createDataFrame(sc.parallelize(Seq(Table(), Table(), Table())))
val nullDf = sqlContext.createDataFrame(sc.parallelize(Seq(Input(a2=null), Input(), Input())))
val newDf = tableDf.schema.convertDataFrame(nullDf)
newDf.filter("a2 IS NULL").take(1).foreach(r => {
r.getLong(0) should equal("aa1")
r.isNullAt(1) should equal(true)
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment