PySpark Pandas UDF

gistfile1.txt

from pyspark.sql import functions as F
from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import *
import pandas as pd

df = spark.read\
.option("header", "true")\
.option("inferSchema", "true")\
.csv("yellow_tripdata_2017-06.csv")

def timestamp_to_epoch(t):
return t.dt.strftime("%s").apply(str) # <-- pandas.Series calls

f_timestamp_copy = pandas_udf(timestamp_to_epoch, returnType=StringType())
df = df.withColumn("timestamp_copy", f_timestamp_copy(F.col("tpep_pickup_datetime")))
df.select('timestamp_copy').distinct().count() #=> 2340959 - 9-10 minute runtime (!!)