[iterate over df rdd] how-to iterate over df rdd #dataframe #scala #rdd #iterate

how-to-iterate-over-df-rdd.scala

def process(df: DataFrame): DataFrame = {
  
  val encoder = RowEncoder(df.schema) // provide the Catalyst codegen info about the datatypes of the data to avoid reflection

  df.map(row => {
      val rowIn = row.toArray
      var rowOut = rowIn
 	  // ... do here some kind of rowOut modifications 
      Row.fromSeq(rowOut)
    })(encoder) // much faster than simple rdd iteration , because avoids the reflection overhead

  }