[dataframe pipeline for spark] how-to build a dataframe processing pipeline in scala spark #scala #spark #dataframe #control-flow

scala-spark-dataframe-pipeline.scala

  private def runPipeLine(cnf: Configuration): DataFrame = {
    val dfOut: DataFrame =
         new Phase1(cnf).process()
        .transform(new Phase2(cnf).process)
    return dfOut
  }
  
  
  class Phase1 extends DataFrameStage {
  
  	  override def process(dfIn: DataFrame = None): DataFrame = {
        val dfOut: DataFrame = dfIn.doSomeTransformations()
      
       return dfOut
      }
  }
  
    class Phase2 extends DataFrameStage {
  
  	  override def process(dfIn: DataFrame): DataFrame = {
        val dfOut: DataFrame = dfIn.doSomeTransformations()
      
       return dfOut
      }
  }