Skip to content

Instantly share code, notes, and snippets.

@bcambel
Created January 10, 2015 06:23
Show Gist options
  • Select an option

  • Save bcambel/b87e9599b5ac538fb260 to your computer and use it in GitHub Desktop.

Select an option

Save bcambel/b87e9599b5ac538fb260 to your computer and use it in GitHub Desktop.
Set up Hadoop 2.6 in Ubuntu. Based on http://codesfusion.blogspot.nl/2013/10/setup-hadoop-2x-220-on-ubuntu.html updated to 2.6
sudo apt-get install openjdk-7-jdk
java -version
# java version "1.7.0_25"
# OpenJDK Runtime Environment (IcedTea 2.3.12) (7u25-2.3.12-4ubuntu3)
# OpenJDK 64-Bit Server VM (build 23.7-b01, mixed mode)
cd /usr/lib/jvm
ln -s java-7-openjdk-amd64 jdk
sudo apt-get install openssh-server
sudo addgroup hadoop
sudo adduser --ingroup hadoop hduser
sudo adduser hduser sudo
sudo su hduser
ssh-keygen -t rsa -P ''
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
ssh localhost
sudo tar vxzf hadoop-2.2.0.tar.gz -C /usr/local
cd /usr/local
sudo mv hadoop-2.2.0 hadoop
sudo chown -R hduser:hadoop hadoop
# paste following to the end of the BASHRC
#Hadoop variables
export JAVA_HOME=/usr/lib/jvm/jdk/
export HADOOP_INSTALL=/usr/local/hadoop
export PATH=$PATH:$HADOOP_INSTALL/bin
export PATH=$PATH:$HADOOP_INSTALL/sbin
export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
export HADOOP_COMMON_HOME=$HADOOP_INSTALL
export HADOOP_HDFS_HOME=$HADOOP_INSTALL
export YARN_HOME=$HADOOP_INSTALL
###end of paste
cd /usr/local/hadoop/etc/hadoop
vi hadoop-env.sh
export JAVA_HOME=/usr/lib/jvm/jdk/
$ cd /usr/local/hadoop/etc/hadoop
$ vi core-site.xml
#Paste following between <configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
$ vi yarn-site.xml
#Paste following between <configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
mv mapred-site.xml.template mapred-site.xml
vi mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
cd ~
mkdir -p mydata/hdfs/namenode
mkdir -p mydata/hdfs/datanode
cd /usr/local/hadoop/etc/hadoop
vi hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hduser/mydata/hdfs/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hduser/mydata/hdfs/datanode</value>
</property>
</configuration>
hdfs namenode -format
# takes a while to start
start-dfs.sh
start-yarn.sh
cd /usr/local/hadoop
#run examples
hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar pi 2 5
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment