Ipython plus Spark

gistfile1.py

import sys

import os

sys.path.append("/usr/lib/spark-yarn/python")

os.environ["SPARK_HOME"]="/usr/lib/spark-yarn/"

from pyspark import SparkContext 

from pyspark import SparkConf

conf=SparkConf()

conf.set("spark.executor.memory", "10m")

conf.set("spark.cores.max", "2")

sc

sc = SparkContext('local[4]', conf=conf)

sc

words = sc.textFile('hdfs://NAMENODE-1/user/kgorman/testme.txt')