concat-df-spec.scala

class ConcatDFSpec extends org.specs2.mutable.Specification {
  def concat(df: DataFrame, inputColName: String, concatColName: String, outputColName: String, dataType: DataType): DataFrame =
    df.withColumn(outputColName, callUDF(_ ++ _, dataType, df(inputColName), df(concatColName)))
  
  // val paramMap: ParamMap
  // val inputCol: Param[String]
  // val outputCol: Param[String]
  // val concatCol: Param[String]
  // concat(df, paramMap(inputCol), paramMap(outputCol), paramMap(concatCol), dataType)
  type A = Map[String, Int]
  forAll { (m: List[(A, A)]) => {
    val df = TestDataFrame("in1" -> m.map(_._1), "in2" -> m.map(_._2))
    val actual = concat(df, "in1", "in2", "out", dataType)
    actual must haveSameElementsWith("out")(m.map(_._1 ++ _._2))
  }
  ).check
}