Spark DateTime

SparkDateTime.scala

// Imports
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._

// Generate sample data
val rows = Seq((1,"Alpha","10/12/1990 12:10:10"),
                (2,"Beta","11/12/1990 13:10:10"),
                (3,"Tango","12/12/1990 14:10:10"))

// Define Schema for the rows
val schema = StructType(Array(StructField("ID",IntegerType),
                              StructField("name",StringType),
                              StructField("mydatetime",StringType)))

// Map fields in Seq to Rows
val rowRDD = rdd.map(attributes => Row(attributes._1, attributes._2, attributes._3))
val df = spark.createDataFrame(rowRDD,schema)
              .select(col("id").cast("int"),
                      col("name"),
                      to_timestamp($"mydatetime","dd/MM/yyyy HH:mm:ss").alias("rdatetime"))
 
// List Current date time of server
df.select("rdatetime")
  .withColumn("presentdate",current_date())
  .withColumn("presentdatetime",current_timestamp())
  .show

// Add ten days
df.select("rdatetime").withColumn("Add10days", date_add($"rdatetime",10)).show

// Substract ten days
df.select("rdatetime").withColumn("Substract10days", date_sub($"rdatetime",1)).show

// Add One Month
df.select("rdatetime").withColumn("1month", add_months($"rdatetime",1)).show

// Get year component of date
df.select("rdatetime").withColumn("yearOfDate", date_trunc($"rdatetime",1)).show

// Get month component of date
df.select("rdatetime").withColumn("monthOfDate", date_trunc("month",$"rdatetime")).show

// Get day component of date
df.select("rdatetime").withColumn("dayOfDate", date_trunc("day",$"rdatetime")).show

//Format Dates
df.select("rdatetime").withColumn("dateformatted", date_format($"rdatetime","MM-dd-yyyy")).show


https://medium.com/expedia-group-tech/deep-dive-into-apache-spark-datetime-functions-b66de737950a