hdp_hadoop_configure.sh

# configure hadoop

wget http://public-repo-1.hortonworks.com/HDP/tools/2.0.6.0/hdp_manual_install_rpm_helper_files-2.0.6.76.tar.gz
tar xf hdp_manual_install_rpm_helper_files-2.0.6.76.tar.gz

(TODO ...)

hdp_hadoop_env.sh

# users and groups
HDFS_USER=hdfs
YARN_USER=yarn
MAPRED_USER=mapred
HADOOP_GROUP=hadoop

# directories
DFS_NAME_DIR="/data0/hadoop/hdfs/nn /data1/hadoop/hdfs/nn /data2/hadoop/hdfs/nn /data3/hadoop/hdfs/nn /data0/hadoop/hdfs/nn /data4/hadoop/hdfs/nn"
DFS_DATA_DIR="/data0/hadoop/hdfs/dn /data1/hadoop/hdfs/dn /data2/hadoop/hdfs/dn /data3/hadoop/hdfs/dn /data4/hadoop/hdfs/dn"
FS_CHECKPOINT_DIR="/data0/hadoop/hdfs/snn /data1/hadoop/hdfs/snn /data2/hadoop/hdfs/snn /data3/hadoop/hdfs/snn /data4/hadoop/hdfs/snn"
HDFS_LOG_DIR="/var/log/hadoop/hdfs"
HDFS_PID_DIR="/var/run/hadoop/hdfs"
HADOOP_CONF_DIR="/etc/hadoop/conf"
YARN_LOCAL_DIR="/data0/hadoop/yarn/local /data1/hadoop/yarn/local /data2/hadoop/yarn/local /data3/hadoop/yarn/local /data4/hadoop/yarn/local "
YARN_LOG_DIR="/var/log/hadoop/yarn"
YARN_LOCAL_LOG_DIR="/data0/hadoop/yarn/logs /data1/hadoop/yarn/logs /data2/hadoop/yarn/logs /data3/hadoop/yarn/logs /data4/hadoop/yarn/logs"
YARN_PID_DIR="/var/run/hadoop/yarn"
MAPRED_LOG_DIR="/var/log/hadoop/mapred"
MAPRED_PID_DIR="/var/run/hadoop/mapred"

# configurations
NAMENODE-HOSTNAME="nd12-rack2-cloud"
NAMENODE-HEAPSIZE="10240m"

SECONDARYNAMENODE-HOSTNAME="nd12-rack2-cloud"
JOBHISTORYNODE-HOSTNAME="nd12-rack2-cloud"
RMNODE-HOSTNAME="nd12-rack2-cloud"

hdp_hadoop_install.sh

# install hadoop hdfs/yarn
# require root (may execute with salt)

source hdp_hadoop_env.sh

# install!
wget -nv http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.0.6.0/hdp.repo -O /etc/yum.repos.d/hdp.repo
umask 0022
yum install hadoop hadoop-hdfs hadoop-libhdfs hadoop-yarn hadoop-mapreduce hadoop-client openssl
yum install snappy snappy-devel
ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/.
yum install lzo lzo-devel hadoop-lzo hadoop-lzo-native

mkdir -p $DFS_NAME_DIR
chown -R $HDFS_USER:$HADOOP_GROUP $DFS_NAME_DIR
chmod -R 755 $DFS_NAME_DIR

mkdir -p $FS_CHECKPOINT_DIR
chown -R $HDFS_USER:$HADOOP_GROUP $FS_CHECKPOINT_DIR
chmod -R 755 $FS_CHECKPOINT_DIR

mkdir -p $DFS_DATA_DIR
chown -R $HDFS_USER:$HADOOP_GROUP $DFS_DATA_DIR
chmod -R 750 $DFS_DATA_DIR

mkdir -p $YARN_LOCAL_DIR
chown -R $YARN_USER:$HADOOP_GROUP $YARN_LOCAL_DIR
chmod -R 755 $YARN_LOCAL_DIR

mkdir -p $YARN_LOCAL_LOG_DIR
chown -R $YARN_USER:$HADOOP_GROUP $YARN_LOCAL_LOG_DIR
chmod -R 755 $YARN_LOCAL_LOG_DIR

mkdir -p $HDFS_LOG_DIR
chown -R $HDFS_USER:$HADOOP_GROUP $HDFS_LOG_DIR
chmod -R 755 $HDFS_LOG_DIR

mkdir -p $YARN_LOG_DIR
chown -R $YARN_USER:$HADOOP_GROUP $YARN_LOG_DIR
chmod -R 755 $YARN_LOG_DIR

mkdir -p $HDFS_PID_DIR
chown -R $HDFS_USER:$HADOOP_GROUP $HDFS_PID_DIR
chmod -R 755 $HDFS_PID_DIR

mkdir -p $YARN_PID_DIR
chown -R $YARN_USER:$HADOOP_GROUP $YARN_PID_DIR
chmod -R 755 $YARN_PID_DIR

mkdir -p $MAPRED_LOG_DIR
chown -R $MAPRED_USER:$HADOOP_GROUP $MAPRED_LOG_DIR
chmod -R 755 $MAPRED_LOG_DIR

mkdir -p $MAPRED_PID_DIR
chown -R $MAPRED_USER:$HADOOP_GROUP $MAPRED_PID_DIR
chmod -R 755 $MAPRED_PID_DIR

# for health_check script
chown -R root:hadoop /usr/lib/hadoop-yarn/bin/container-executor
chmod -R 6050 /usr/lib/hadoop-yarn/bin/container-executor

hdp_install_zk.sh

ZOOKEEPER_USER=zookeeper
HADOOP_GROUP=hadoop

ZOOKEEPER_DATA_DIR="/data0/hadoop/zookeeper/data"
ZOOKEEPER_CONF_DIR="/etc/zookeeper/conf"
ZOOKEEPER_LOG_DIR="/var/log/zookeeper"
ZOOKEEPER_PID_DIR="/var/run/zookeeper"

yum install zookeeper

mkdir -p $ZOOKEEPER_LOG_DIR;
chown -R $ZOOKEEPER_USER:$HADOOP_GROUP $ZOOKEEPER_LOG_DIR;
chmod -R 755 $ZOOKEEPER_LOG_DIR;

mkdir -p $ZOOKEEPER_PID_DIR;
chown -R $ZOOKEEPER_USER:$HADOOP_GROUP $ZOOKEEPER_PID_DIR;
chmod -R 755 $ZOOKEEPER_PID_DIR;

mkdir -p $ZOOKEEPER_DATA_DIR;
chmod -R 755 $ZOOKEEPER_DATA_DIR;
chown -R $ZOOKEEPER_USER:$HADOOP_GROUP $ZOOKEEPER_DATA_DIR

hdp_start_historyserver.sh

# start mapreduce history server

sudo -u hdfs hadoop fs -mkdir -p /mr-history/tmp
sudo -u hdfs hadoop fs -chmod -R 1777 /mr-history/tmp
sudo -u hdfs hadoop fs -mkdir -p /mr-history/done
sudo -u hdfs hadoop fs -chmod -R 1777 /mr-history/done
sudo -u hdfs hadoop fs -chown -R $MAPRED_USER:$HDFS_USER /mr-history

sudo -u hdfs hadoop fs -mkdir -p /app-logs
sudo -u hdfs hadoop fs -chmod -R 1777 /app-logs 
sudo -u hdfs hadoop fs -chown yarn /app-logs


#!!! below executed with user mapred
sudo su - mapred
export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec/
/usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh --config /etc/hadoop/conf start historyserver
exit

hdp_start_nm.sh

# start yarn nodemanager

sudo su - yarn
yarn export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec;
/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf start nodemanager
exit

hdp_start_rm.sh

# start yarn resourcemanager (run as user yarn)

sudo su - yarn
export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf start resourcemanager
exit

hdp_start_zk.sh

sudo su - zookeeper
export ZOO_LOG_DIR=/var/log/zookeeper
/usr/lib/zookeeper/bin/zkServer.sh start /etc/zookeeper/zoo.cfg
exit