Created
December 11, 2014 21:01
-
-
Save smartkiwi/2cd703e6217695645526 to your computer and use it in GitHub Desktop.
run pyspark in standalone module without bin/pyspark
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import sys | |
# Set the path for spark installation | |
# this is the path where you have built spark using sbt/sbt assembly | |
os.environ['SPARK_HOME'] = "/Users/vvlad/spark/spark-1.0.2" | |
# Append to PYTHONPATH so that pyspark could be found | |
sys.path.append("/Users/vvlad/spark/spark-1.0.2/python") | |
sys.path.append("/Users/vvlad/spark/spark-1.0.2/python/lib/py4j-0.8.1-src.zip") | |
# Now we are ready to import Spark Modules | |
try: | |
from pyspark import SparkContext | |
from pyspark import SparkConf | |
except ImportError as e: | |
raise | |
print ("Error importing Spark Modules", e) | |
sys.exit(1) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment