pseudo_distributed_hadoop.sh

#!/bin/bash

set -u
set -e

run_cmd() {
  echo "$ $@"
  eval "$@"
  if [ $? -ne 0 ] ; then
    echo "Failed to run $@, exiting..."
    exit -1
  fi
}

if [ $# -ne 1 ]; then
  echo "Usage: $0 hadoop_home_directory"
  echo "  e.g. $0 /home/test/hadoop-2.4.1"
  exit -1
fi

export HADOOP_HOME=$1
if [ "${HADOOP_HOME:0:1}" != "/" ]; then
  HADOOP_HOME="$(pwd)/$HADOOP_HOME"
fi
export RECOVERY_DIR="$HADOOP_HOME/recovery"

echo "Installing to $HADOOP_HOME"

mkdir -p $HADOOP_HOME/local1
mkdir -p $HADOOP_HOME/local2
mkdir -p $HADOOP_HOME/local3
mkdir -p $HADOOP_HOME/log1
mkdir -p $HADOOP_HOME/log2
mkdir -p $HADOOP_HOME/log3

# configure core-site.xml
echo "
<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost:9000</value>
  </property>
</configuration>" > $HADOOP_HOME/etc/hadoop/core-site.xml

# configure hdfs-site.xml
echo "
<configuration>
 <property>
   <name>dfs.replication</name>
   <value>1</value>
 </property>
</configuration>" > $HADOOP_HOME/etc/hadoop/hdfs-site.xml

# configure mapred-site.xml
echo "
<configuration>
<property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
 </property>
</configuration>" > $HADOOP_HOME/etc/hadoop/mapred-site.xml

# configure yarn-site.sml
# Reference: http://www.cloudera.com/content/www/en-us/documentation/enterprise/5-3-x/topics/cdh_hag_rm_ha_config.html
echo "
<configuration>
<property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
 </property>
 <property>
   <name>yarn.resourcemanager.scheduler.class</name>
   <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
 </property>
 <property>
   <name>yarn.resourcemanager.ha.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
   <value>true</value>
 </property>
 <property>
   <name>yarn.resourcemanager.cluster-id</name>
   <value>test-jun</value>
 </property>
 <property>
   <name>yarn.resourcemanager.ha.rm-ids</name>
   <value>rm1,rm2</value>
 </property>
 <property>
   <name>yarn.resourcemanager.ha.id</name>
   <value>rm1</value>
 </property>
 <property>
    <name>yarn.resourcemanager.recovery.enabled</name>
    <value>true</value>
 </property>
 <property>
   <name>yarn.resourcemanager.store.class</name>
   <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
 </property>
 <property>
   <name>yarn.resourcemanager.zk-address</name>
   <value>localhost:2181</value>
 </property>
 <property>
   <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
   <value>5000</value>
 </property>
 <property>
   <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>yarn.resourcemanager.am.max-attempts</name>
   <value>2</value>
 </property>

<!-- NM restart: https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html -->
 <property>
   <name>yarn.nodemanager.recovery.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>yarn.nodemanager.recovery.dir</name>
   <value>$RECOVERY_DIR</value>
 </property>
 <property>
   <name>yarn.nodemanager.address</name>
   <value>localhost:8888</value>
 </property>
 <property>
   <name>yarn.nodemanager.recovery.supervised</name>
   <value>true</value>
 </property>
 <property>
   <name>yarn.nodemanager.local-dirs</name>
   <value>$HADOOP_HOME/local1,$HADOOP_HOME/local2,$HADOOP_HOME/local3</value>
 </property>
 <property>
   <name>yarn.nodemanager.log-dirs</name>
   <value>$HADOOP_HOME/log1,$HADOOP_HOME/log2,$HADOOP_HOME/log3</value>
 </property>
 <property>
   <name>yarn.nodemanager.delete.debug-delay-sec</name>
   <value>3000000</value>
 </property>

<!-- RM1 configs -->
 <property>
   <name>yarn.resourcemanager.address.rm1</name>
   <value>localhost:23140</value>
 </property>
 <property>
   <name>yarn.resourcemanager.scheduler.address.rm1</name>
   <value>localhost:23130</value>
 </property>
 <property>
   <name>yarn.resourcemanager.webapp.https.address.rm1</name>
   <value>localhost:23189</value>
 </property>
 <property>
   <name>yarn.resourcemanager.webapp.address.rm1</name>
   <value>localhost:23188</value>
 </property>
 <property>
   <name>yarn.resourcemanager.resource-tracker.address.rm1</name>
   <value>localhost:23125</value>
 </property>
 <property>
   <name>yarn.resourcemanager.admin.address.rm1</name>
   <value>localhost:23141</value>
 </property>

 <!-- RM2 configs -->
 <property>
   <name>yarn.resourcemanager.address.rm2</name>
   <value>localhost:23140</value>
 </property>
 <property>
   <name>yarn.resourcemanager.scheduler.address.rm2</name>
   <value>localhost:23130</value>
 </property>
 <property>
   <name>yarn.resourcemanager.webapp.https.address.rm2</name>
   <value>localhost:23189</value>
 </property>
 <property>
   <name>yarn.resourcemanager.webapp.address.rm2</name>
   <value>localhost:23188</value>
 </property>
 <property>
   <name>yarn.resourcemanager.resource-tracker.address.rm2</name>
   <value>localhost:23125</value>
 </property>
 <property>
   <name>yarn.resourcemanager.admin.address.rm2</name>
   <value>localhost:23141</value>
 </property>
</configuration>" > $HADOOP_HOME/etc/hadoop/yarn-site.xml


BIN=$HADOOP_HOME/bin
SBIN=$HADOOP_HOME/sbin
echo "Start service : "
run_cmd "$BIN/hdfs namenode -format >/dev/null"
run_cmd "$SBIN/start-dfs.sh >/dev/null"
run_cmd "$SBIN/start-yarn.sh >/dev/null"
run_cmd "$SBIN/yarn-daemon.sh start resourcemanager >/dev/null"
jps

# Generate stop-all.sh: stop-dfs.sh + stop-yarn.sh
echo "$SBIN/stop-dfs.sh
$SBIN/stop-yarn.sh
$SBIN/yarn-daemon.sh stop resourcemanager
rm -rf $HADOOP_HOME/logs/*
rm -rf $RECOVERY_DIR
rm -rf /private/tmp/hadoop-jungong" > stop-all.sh
chmod +x stop-all.sh