Created
January 10, 2015 06:23
-
-
Save bcambel/b87e9599b5ac538fb260 to your computer and use it in GitHub Desktop.
Set up Hadoop 2.6 in Ubuntu. Based on http://codesfusion.blogspot.nl/2013/10/setup-hadoop-2x-220-on-ubuntu.html updated to 2.6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| sudo apt-get install openjdk-7-jdk | |
| java -version | |
| # java version "1.7.0_25" | |
| # OpenJDK Runtime Environment (IcedTea 2.3.12) (7u25-2.3.12-4ubuntu3) | |
| # OpenJDK 64-Bit Server VM (build 23.7-b01, mixed mode) | |
| cd /usr/lib/jvm | |
| ln -s java-7-openjdk-amd64 jdk | |
| sudo apt-get install openssh-server | |
| sudo addgroup hadoop | |
| sudo adduser --ingroup hadoop hduser | |
| sudo adduser hduser sudo | |
| sudo su hduser | |
| ssh-keygen -t rsa -P '' | |
| cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys | |
| ssh localhost | |
| sudo tar vxzf hadoop-2.2.0.tar.gz -C /usr/local | |
| cd /usr/local | |
| sudo mv hadoop-2.2.0 hadoop | |
| sudo chown -R hduser:hadoop hadoop | |
| # paste following to the end of the BASHRC | |
| #Hadoop variables | |
| export JAVA_HOME=/usr/lib/jvm/jdk/ | |
| export HADOOP_INSTALL=/usr/local/hadoop | |
| export PATH=$PATH:$HADOOP_INSTALL/bin | |
| export PATH=$PATH:$HADOOP_INSTALL/sbin | |
| export HADOOP_MAPRED_HOME=$HADOOP_INSTALL | |
| export HADOOP_COMMON_HOME=$HADOOP_INSTALL | |
| export HADOOP_HDFS_HOME=$HADOOP_INSTALL | |
| export YARN_HOME=$HADOOP_INSTALL | |
| ###end of paste | |
| cd /usr/local/hadoop/etc/hadoop | |
| vi hadoop-env.sh | |
| export JAVA_HOME=/usr/lib/jvm/jdk/ | |
| $ cd /usr/local/hadoop/etc/hadoop | |
| $ vi core-site.xml | |
| #Paste following between <configuration> | |
| <property> | |
| <name>fs.default.name</name> | |
| <value>hdfs://localhost:9000</value> | |
| </property> | |
| $ vi yarn-site.xml | |
| #Paste following between <configuration> | |
| <property> | |
| <name>yarn.nodemanager.aux-services</name> | |
| <value>mapreduce_shuffle</value> | |
| </property> | |
| <property> | |
| <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> | |
| <value>org.apache.hadoop.mapred.ShuffleHandler</value> | |
| </property> | |
| mv mapred-site.xml.template mapred-site.xml | |
| vi mapred-site.xml | |
| <configuration> | |
| <property> | |
| <name>mapreduce.framework.name</name> | |
| <value>yarn</value> | |
| </property> | |
| </configuration> | |
| cd ~ | |
| mkdir -p mydata/hdfs/namenode | |
| mkdir -p mydata/hdfs/datanode | |
| cd /usr/local/hadoop/etc/hadoop | |
| vi hdfs-site.xml | |
| <configuration> | |
| <property> | |
| <name>dfs.replication</name> | |
| <value>1</value> | |
| </property> | |
| <property> | |
| <name>dfs.namenode.name.dir</name> | |
| <value>file:/home/hduser/mydata/hdfs/namenode</value> | |
| </property> | |
| <property> | |
| <name>dfs.datanode.data.dir</name> | |
| <value>file:/home/hduser/mydata/hdfs/datanode</value> | |
| </property> | |
| </configuration> | |
| hdfs namenode -format | |
| # takes a while to start | |
| start-dfs.sh | |
| start-yarn.sh | |
| cd /usr/local/hadoop | |
| #run examples | |
| hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar pi 2 5 | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment