hack_edx_pyspark.py

# insert somewhere in function working with sc directly
sc.stop()
from pyspark import SparkContext
SparkContext.setSystemProperty('spark.executor.memory', '6g') # no sure which one works, use both
SparkContext.setSystemProperty('spark.python.worker.memory', '6g') # no sure which one works, use both
SparkContext.setSystemProperty('spark.shuffle.spill', 'false')
SparkContext.setSystemProperty('spark.driver.memory', '2g')
SparkContext.setSystemProperty('spark.io.compression.codec', 'snappy') # just to be sure
sc = SparkContext("local[8]", "Simple App") # set to your number of cores