PySpark setup script

start-spark.sh

#!/bin/sh

export IPYTHON_OPTS="notebook --profile=yannis_spark"

unset SPARK_JAVA_OPTS

export SPARK_DAEMON_JAVA_OPTS="\
-Dspark.default.parallelism=128   \
-Dspark.cores.max=32              \
-Dspark.worker.timeout=500        \
-Dspark.akka.timeout=500          \
"

#-Dspark.executor.memory=668m       \
#-Dspark.worker.memory=10g        \

export SPARK_DRIVER_MEMORY=20g

#export SPARK_WORKER_MEMORY=1g
#export MASTER=local[16]

nice -n 20 ./spark/bin/pyspark