Archivo temporal para la autoconfiguracion del cluster de hadoo de la FEUD

autoconfigure_hadoop_feud.sh

#!/bin/bash
# run as root

#Variables por ambiente
nodo1="nodo1"
nodo2="nodo2"
nodo3="nodo3"
ip1="10.0.2.15"
ip2=""
ip3=""
current_ip=$(ip -4 addr show scope global | awk '$1 == "inet" {print $2}' | awk -F"/" '{print $1}')

#date
current_date=$(date +%Y-%m-%d)

cp /etc/bash.bashrc "/etc/bash.bashrc_bk$current_date"
#echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> /etc/bash.bashrc
sed -i '1s/^/export JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\/\n/' /etc/bash.bashrc
#echo "export HADOOP_HOME=/opt/hadoop-2.8.4" >> /etc/bash.bashrc
sed -i '1s/^/export HADOOP_HOME=\/opt\/hadoop-2.8.5\n/' /etc/bash.bashrc

source /etc/bash.bashrc

echo "Validar JAVA HOME"
ls $JAVA_HOME

cd /opt
wget http://192.168.102.254/hadoop-2.8.4-a.tar.gz

tar xzf hadoop-2.8.4-a.tar.gz -C .

chown -R usuario:usuario hadoop-2.8.4
cp  $HADOOP_HOME/etc/hadoop/core-site.xml $HADOOP_HOME/etc/hadoop/core-site_bk.xml

cat <<< '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://nodo1:9000</value>
	</property>
	<property>
		<name>hadoop.tmp.dir</name>
        <value>/opt/hadoop-2.8.4/tmp</value>
	</property>

</configuration>' > $HADOOP_HOME/etc/hadoop/core-site.xml

#nano  $HADOOP_HOME/etc/hadoop/hdfs-site.xml
cp  $HADOOP_HOME/etc/hadoop/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site_bk.xml
cat <<< '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>2</value>
    </property>
</configuration>' > $HADOOP_HOME/etc/hadoop/hdfs-site.xml

cp $HADOOP_HOME/etc/hadoop/mapred-site.xml.template  $HADOOP_HOME/etc/hadoop/mapred-site.xml

#nano  $HADOOP_HOME/etc/hadoop/mapred-site.xml
cat <<< '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>' > $HADOOP_HOME/etc/hadoop/mapred-site.xml


#nano  $HADOOP_HOME/etc/hadoop/yarn-site.xml

cp $HADOOP_HOME/etc/hadoop/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site_bk.xml

cat <<< '<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->

<property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>nodo1</value>
    </property>
</configuration>' > $HADOOP_HOME/etc/hadoop/yarn-site.xml


##nano  $HADOOP_HOME/etc/hadoop/hadoop-env.sh
#	export HADOOP_HEAPSIZE=256
######
cp $HADOOP_HOME/etc/hadoop/hadoop-env.sh $HADOOP_HOME/etc/hadoop/hadoop-env_bk.sh
sed -i 's/.*export HADOOP_HEAPSIZE=.*/export HADOOP_HEAPSIZE=256/g' $HADOOP_HOME/etc/hadoop/hadoop-env.sh


cp $HADOOP_HOME/etc/hadoop/slaves $HADOOP_HOME/etc/hadoop/slaves_bk
cat > $HADOOP_HOME/etc/hadoop/slaves <<EOF
$nodo1
$nodo2
$nodo3
EOF
#nano /etc/hostname
if [ "$ip1" == "$current_ip" ] ; then
  echo  $nodo1 > /etc/hostname
elif [ "$ip2" == "$current_ip" ] ; then
  echo $nodo2 > /etc/hostname
elif [ "$ip3" == "$current_ip" ] ; then
  echo $nodo3 > /etc/hostname
fi

#nano /etc/hosts
echo $ip1 $nodo1 >> /etc/hosts
echo $ip2 $nodo2 >> /etc/hosts
echo $ip3 $nodo3 >> /etc/hosts

chown -R usuario:usuario /etc/hadoop-2.8.4 

cat <<< 'unalias fs &> /dev/null
alias fs="hadoop fs"
unalias hls &> /dev/null
alias hls="fs -ls"

lzohead () {
    hadoop fs -cat $1 | lzop -dc | head -1000 | less
}
export PATH=$PATH:$HADOOP_HOME/bin' > /etc/bash.bashrc

su - usuario


ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa

ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo1


if [ "$ip1" == "$current_ip" ] ; then
  ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo2
  ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo3
elif [ "$ip2" == "$current_ip" ] ; then
  ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo1
  ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo3
elif [ "$ip3" == "$current_ip" ] ; then
  ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo2
  ssh-copy-id -i ~/.ssh/id_rsa usuario@$nodo1
fi


$HADOOP_HOME/bin/hadoop namenode -format