Configure Hadoop Cluster

configureCluster.sh

ips=`cat ips.txt`
masterPublic=`echo $ips | cut -f1 -d " "`
masterPrivate=`echo $ips | cut -f2 -d " "`

rm -rf hostsFile
echo "$masterPrivate hmaster" > hostsFile
rm -rf slavesFile
touch slavesFile

iparray=(`cat ips.txt`)
ipCount=`cat ips.txt |wc -w`
ipCount=`expr $ipCount - 2`
echo "Slave Count " $ipCount

echo "Configuring Master $masterPublic...Setting hostname, replication count..."
ssh -i ~/ssk.pem ubuntu@$masterPublic 'sudo hostname hmaster'
ssh -i ~/ssk.pem ubuntu@$masterPublic "sed -i \"s/#replication/$ipCount/g\" ~/image/hadoop-0.20.2-cdh3u4/conf/hdfs-site.xml"

idx=0
for i in "${iparray[@]}"
do
   echo $idx
   if [ $idx -ne 0 ] && [ $idx -ne 1 ] 
   then
     slavePrivate=$i
     slaveHN=hslave`expr $idx - 2`
     echo "Configuring Slave...$slavePrivate $slaveHN...Setting hostname"

     echo "$slavePrivate $slaveHN" >> hostsFile
     echo "$slavePrivate" >> slavesFile

     ssh -i ~/ssk.pem ubuntu@$masterPublic "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$slavePrivate 'sudo hostname $slaveHN'"

     #remove slave hdfs data 
     ssh -i ~/ssk.pem ubuntu@$masterPublic "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$slavePrivate rm -rf ~/hdfs/data"
   fi
   idx=`expr $idx + 1`
done

#Copy hosts file to Master
echo "Copying Hosts file to Master..."
scp -i ~/ssk.pem hostsFile ubuntu@$masterPublic:/tmp
ssh -i ~/ssk.pem ubuntu@$masterPublic 'sudo cat /tmp/hostsFile >> /etc/hosts'


#Copy Hosts file to all slaves
idx=0
for i in "${iparray[@]}"
do
   echo $idx
   if [ $idx -ne 0 ] && [ $idx -ne 1 ]
   then
      slavePrivate=$i
      echo "copying hosts file to slave " $slavePrivate
      ssh -i ~/ssk.pem ubuntu@$masterPublic "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/hostsFile ubuntu@$slavePrivate:/tmp"
      ssh -i ~/ssk.pem ubuntu@$masterPublic "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$slavePrivate \"sudo cat /tmp/hostsFile >> /etc/hosts\""
   fi
   idx=`expr $idx + 1`
done

#Copy slaves file to Master
echo "copying Slaves file to Master..."
scp -i ~/ssk.pem slavesFile ubuntu@$masterPublic:~/image/hadoop-0.20.2-cdh3u4/conf/slaves

echo "Format HDFS..."
ssh -i ~/ssk.pem ubuntu@$masterPublic  "echo \"Y\" | /home/ubuntu/image/hadoop-0.20.2-cdh3u4/bin/hadoop namenode -format"

echo "Starting Hadoop Cluster..."
ssh -i ~/ssk.pem ubuntu@$masterPublic  '/home/ubuntu/image/hadoop-0.20.2-cdh3u4/bin/start-all.sh'