シェルクスリプトでログ回収

log_retrieve.sh

#!/bin/sh

BINPATH=$(cd $(dirname $0) && pwd)
FILE=$(basename $0)
FILEPATH=$BINPATH/$FILE

DATE=`date +"%Y%m%d"`

LOGPATH=/var/log/data-mining
ACCESSLOG=/access_log.$DATE
ACTIONLOG=/action_log.$DATE

HADOOPLOGPATH=/log
NAMENODE=hdfs://192.168.20.10:54310/

SERVERS=`cat $BINPATH/servers`

START_TIME=`date +%s`

for SERVER in $SERVERS
do
   echo -e "\e[33m>${SERVER} \e[m"
   if [ ! -d $BINPATH/$SERVER ]; then
       mkdir $BINPATH/$SERVER
   fi
   scp -i $HOME/.ssh/id_rsa data-mining@$SERVER:$LOGPATH$ACCESSLOG $BINPATH/$SERVER &
   #scp -i $HOME/.ssh/id_rsa data-mining@$SERVER:$LOGPATH$ACTIONLOG $BINPATH/$SERVER &
done

wait

echo "Log retrieve complete."

for SERVER in $SERVERS
do
   sudo -u hdfs hadoop fs -mkdir /log/$SERVER
   echo "sudo -u hdfs hadoop fs -fs $NAMENODE -moveFromLocal $LOGPATH/$SERVER$ACCESSLOG /log/$SERVER/"
   sudo -u hdfs hadoop fs -fs $NAMENODE -moveFromLocal $LOGPATH/$SERVER$ACCESSLOG /log/$SERVER/ &
   #sudo -u hdfs hadoop fs -fs $NAMENODE -moveFromLocal $LOGPATH/$SERVER$ACTIONLOG /log/$SERVER/ &
done

wait

END_TIME=`date +%s`

echo "Done."

SS=`expr ${END_TIME} - ${START_TIME}`

HH=`expr ${SS} / 3600`
SS=`expr ${SS} % 3600`
MM=`expr ${SS} / 60`
SS=`expr ${SS} % 60`

echo "Total Time: ${HH}:${MM}:${SS} (h:m:s)"