ejemplo uso udf

gistfile1.txt

// mydf.count()
// 63385686 

val mydf = sqlContext.read.parquet("ParaMarina/sensEnriched.parquet")

mydf.cache

val r = scala.util.Random

import org.apache.spark.sql.functions.udf

val accum = sc.accumulator(1)

def myNextPositiveNumber():String = {
	accum+=1
    accum.value.toString.concat("D")
}

val myFunction = udf(myNextPositiveNumber _)

val myNewDF = mydf.withColumn("myNewColumn",lit(myNextPositiveNumber))

myNewDF.saveAsParquetFile("ParaMarina/newSensEnriched.parquet")

myNewDF.select("myNewColumn").count

// 63385686

import org.apache.spark.sql.functions.udf

val mydf = sqlContext.read.parquet("some.parquet")

mydf.cache

def myNextPositiveNumber():String = (rand(100) * Integer.MAX_VALUE).cast("bigint").cast("string").toString().concat("D")

// monotonically_increasing_id().toString().concat("D")

val myFunction = udf(myNextPositiveNumber _)

val myNewDF = mydf.withColumn("myNewColumn",expr(myNextPositiveNumber))

myNewDF.saveAsParquetFile("ParaMarina/newSensEnriched.parquet")

myNewDF.select("myNewColumn").show(5,false)

myNewDF.select("myNewColumn").count

// 63385686