Bash script containing all steps required to fire up a CDH cluster

cluster-up.sh

#!/bin/bash
# Hadoop cluster start-up script
# 
# 1. Format the namenode (only required on 1st start!)
# sudo -u hdfs hdfs namenode -format

# 2. Start HDFS
for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do sudo service $x start ; done

# 3. Create the /temp directory
# 3.1 Remove the old /tmp if it exists
# sudo -u hdfs hadoop fs -rm -r /tmp
# 3.2 Create a new /tmp directory and set permissions
# sudo -u hdfs hadoop fs -mkdir /tmp sudo -u hdfs hadoop fs -chmod -R 1777 /tmp

# 4. Create the staging directory and set permissions
sudo -u hdfs hadoop fs -mkdir /tmp/hadoop-yarn/staging sudo -u hdfs hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging
# 4.1 Create the done_intermediate directory under the staging directory and set permissions
sudo -u hdfs hadoop fs -mkdir /tmp/hadoop-yarn/staging/history/done_intermediate sudo -u hdfs hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging/history/done_intermediate
# 4.2 Change ownership on the staging directory and subdirectory
sudo -u hdfs hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn/staging
# 4.3 Create the /var/log/hadoop-yarn directory and set ownership
sudo -u hdfs hadoop fs -mkdir /var/log/hadoop-yarn sudo -u hdfs hadoop fs -chown yarn:mapred /var/log/hadoop-yarn

# 5. Verify the HDFS file structure
sudo -u hdfs hadoop fs -ls -R /

# 6. Start YARN
sudo service hadoop-yarn-resourcemanager start sudo service hadoop-yarn-nodemanager start sudo service hadoop-mapreduce-historyserver start

# 7. Create user directories
sudo -u hdfs hadoop fs -mkdir /user/<user> $ sudo -u hdfs hadoop fs -chown <user> /user/<user>

# 8. Set HADOOP_MAPRED_HOME for user
export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce