drop duplicate rows by id, keeping one with latest timestamp

spark_reatain_latest_in_group.scala

// https://www.datasciencemadesimple.com/distinct-value-of-dataframe-in-pyspark-drop-duplicates/
// https://stackoverflow.com/questions/38687212/spark-dataframe-drop-duplicates-and-keep-first

// to deal with ties within window partitions, a tiebreaker column is added

import org.apache.spark.sql.functions._
import org.apache.spark.sql.expressions.Window

val byId = Window.partitionBy("id").orderBy(col("last_updated").desc, col("tiebreak"))

val deduped = df.
    withColumn("tiebreak", monotonically_increasing_id()).
    withColumn("rank", rank().over(byId)).
    filter(col("rank") === 1).
    drop("rank","tiebreak")