Hello World for EMR Spark Step Python Pyspark

emr_spark_step_hello_world.py

from pyspark.context import SparkContext
from pyspark.sql import SparkSession

if __name__ == "__main__":

    # Create the spark session
    spark = SparkSession\
        .builder\
        .appName("SparkEMR")\
        .getOrCreate()

    # Create the spark context
    sc = spark.sparkContext

    # Put your bucket and folder here
    s3_bucket = "s3://my-bucket-name/folder-name/hello_world/"

    # Create Hello World Dataframe
    dataframe = spark.createDataFrame([("Hello", "World")])

    # Coalesce the data to 1 file
    # Format as CSV
    # Save the dataframe to your s3_bucket
    # Overwrite what's there
    dataframe.coalesce(1).write.csv(s3_bucket, mode="overwrite")

    # Clean up when done
    sc.stop()