Last active
November 24, 2015 19:38
-
-
Save peterklipfel/7b2e77c795f43e59b4e6 to your computer and use it in GitHub Desktop.
install spark ubuntu 14.04
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sudo apt-get update | |
sudo apt-get install -y openjdk-7-jdk | |
sudo su -c 'echo "JAVA_HOME=\"/usr/lib/jvm/java-7-openjdk-amd64\"" >> /etc/environment' | |
cd /opt | |
sudo wget http://d3kbcqa49mib13.cloudfront.net/spark-1.0.1-bin-hadoop2.tgz | |
sudo tar -zxf spark-1.0.1-bin-hadoop2.tgz | |
cd spark-1.0.1-bin-hadoop2 | |
# assumes that hostname is set correctly, and the main interface is on eth0, add HOSTNAME to /etc/hosts | |
sudo su -c "echo `ifconfig eth0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'` `cat /etc/hostname` >> /etc/hosts" | |
sbin/start-master.sh | |
grep -i 'spark://' logs/spark-ubuntu-org* |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sudo apt-get update | |
sudo apt-get install -y openjdk-7-jdk | |
sudo su -c 'echo "JAVA_HOME=\"/usr/lib/jvm/java-7-openjdk-amd64\"" >> /etc/environment' | |
cd /opt | |
sudo wget http://d3kbcqa49mib13.cloudfront.net/spark-1.0.1-bin-hadoop2.tgz | |
sudo tar -zxf spark-1.0.1-bin-hadoop2.tgz | |
cd spark-1.0.1-bin-hadoop2 | |
# assumes that hostname is set correctly, and the main interface is on eth0, add HOSTNAME to /etc/hosts | |
sudo su -c "echo `ifconfig eth0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'` `cat /etc/hostname` >> /etc/hosts" | |
echo "spark master url ex: spark://spark-master:1234" | |
read spark_master_uri | |
echo "spark master hostname" | |
read spark_master_hostname | |
echo "spark master ip" | |
read spark_master_ip | |
sudo su -c "echo '$spark_master_ip $spark_master_hostname' >> /etc/hosts" | |
./bin/spark-class org.apache.spark.deploy.worker.Worker spark_master_uri | |
./bin/spark-class org.apache.spark.deploy.worker.Worker spark://IP:PORT |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment