IPython PySpark Setup Code

ipython_pyspark_setup_code.py

import os
import sys
import os.path
 
spark_home = os.environ.get('SPARK_HOME', None)
if not spark_home:
      raise ValueError('SPARK_HOME environment variable is not set')
sys.path.insert(0, os.path.join(spark_home, 'python'))
sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-src.zip'))
if 'sc' not in vars() and 'sc' not in globals():
    execfile(os.path.join(spark_home, 'python/pyspark/shell.py'))

fileName = 'file:///Users/hadoop/data/millionsong.txt'
numPartitions = 2
rawData = sc.textFile(fileName, numPartitions)
print(rawData.count())