Skip to content

Instantly share code, notes, and snippets.

@gvyshnya
Created August 17, 2020 14:40
Show Gist options
  • Save gvyshnya/3f1138b28dfcb00a629183501cc224f1 to your computer and use it in GitHub Desktop.
Save gvyshnya/3f1138b28dfcb00a629183501cc224f1 to your computer and use it in GitHub Desktop.
import datetime
from pyspark.sql.functions import year, month, dayofmonth
elevDF = sc.parallelize([
(datetime.datetime(1994, 1, 1, 0, 0), 1, 638.55),
(datetime.datetime(1994, 1, 1, 0, 0), 2, 638.55),
(datetime.datetime(1994, 1, 1, 0, 0), 3, 638.55),
(datetime.datetime(1994, 1, 1, 0, 0), 4, 638.55),
(datetime.datetime(1994, 1, 1, 0, 0), 5, 638.55)
]).toDF(["date", "hour", "value"])
elevDF.select(
year("date").alias('year'),
month("date").alias('month'),
dayofmonth("date").alias('day')
).show()
elevDF1 = sqlContext.createDataFrame(sc.parallelize([
Row(date=datetime.datetime(1994, 1, 1, 0, 0), hour=1, value=638.55),
Row(date=datetime.datetime(1994, 1, 1, 0, 0), hour=2, value=638.55),
Row(date=datetime.datetime(1994, 1, 1, 0, 0), hour=3, value=638.55),
Row(date=datetime.datetime(1994, 1, 1, 0, 0), hour=4, value=638.55),
Row(date=datetime.datetime(1994, 1, 1, 0, 0), hour=5, value=638.55)]))
(elevDF1
.map(lambda (date, hour, value): (date.year, date.month, date.day))
.collect())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment