hatools.sh

#!/bin/bash
# Author  Xiang Wang
# Date    10/13/2010

declare -a H_ARRAY
declare -a W_ARRAY
declare -a L_ARRAY
declare -a R_ARRAY
declare -a S_ARRAY
####### check Parameter

if [[ ! ("$1" == "install" ||  "$1" == "uninstall" || "$1" == "Manual" || "$1" == "HTTP" || "$1" == "FTP" || "$1" == "HAAGENT") ]]; then
    echo "Usage : `basename $0` install/uninstall"
    echo "        `basename $0` Manual/HTTP/FTP/HAAGENT"
    exit 1
fi
# change to executable script
chmod u+x $0
## installation and uninstallation

case $1 in
    'install')
        echo "Install HA tools !"
        # check deploymode
        if [ ! -f "/etc/iscan/cluster.ini" ];then
            echo "Current Deploy mode was not HA !"
            echo "You can not install this tool in none HA mode !"
            exit 0
        fi
        #check two ndoes
        if [ ! "`cat /etc/iscan/cluster.ini | grep "cluster_nodes" | gawk -F\; '{print $2}'`" ];then
            echo "Current Only One Node in HA Mode"
            echo "You have to Run this script in TWO Nodes Environment"
        fi
        # check the path of the script
        if [ ! -e "/etc/iscan/`basename $0`" ];then
            cp -f $0 /etc/iscan/
        fi
        # change the attr of the script
        chmod u+x /etc/iscan/hatools.sh
        crontab -l > /tmp/crontjob

if [ "`cat /tmp/crontjob | grep "hatools"| grep -v "grep"`" ]; then
            echo "HA tolls has been installed"
            rm -f /tmp/crontjob
            exit 0
        fi
        scriptname="/etc/iscan/`basename $0`"
        echo "0 2 * * * $scriptname Manual " >> /tmp/crontjob
        echo "15 2 * * * $scriptname HTTP " >> /tmp/crontjob
        echo "30 2 * * * $scriptname FTP "  >> /tmp/crontjob
        echo "45 2 * * * $scriptname HAAGENT " >> /tmp/crontjob
        crontab -u root /tmp/crontjob
        /etc/init.d/crond restart
        rm /tmp/crontjob
        exit 0
        ;;
    'uninstall')
        crontab -l > /tmp/crontjob
        sed -i '/hatools/d' /tmp/crontjob
        crontab /tmp/crontjob
        rm /tmp/crontjob
        echo "HA tools has been uninstalled"
        exit 0
        ;;
esac

# set up the log path

LOGFILE="/var/log/hatool.log.`date +"%F" | gawk -F- '{print $1 "." $2 "." $3}'`"
case $1 in
    'Manual')
        LOGFILE=$LOGFILE.001
        ;;
    'HTTP')
        LOGFILE=$LOGFILE.002
        ;;
    'FTP')
        LOGFILE=$LOGFILE.003
        ;;
    'HAAGENT')
        LOGFILE=$LOGFILE.004
        ;;
    esac

TIMESTAMP="`date` :"

# check Deploy mode here
if [ ! -f "/etc/iscan/cluster.ini" ];then
    echo "$TIMESTAMP Current Deploy mode was not HA !" >> $LOGFILE
    exit 0
fi
##

## check two node here

is_twonodes=`cat /etc/iscan/cluster.ini | grep "cluster_nodes" | gawk -F\; '{print $2}'`
if [ ! "$is_twonodes" ];then
    echo "$TIMESTAMP Current Only One Node in HA Mode" >> $LOGFILE
    exit 0
fi
#

preemption=`sed -n '/preemption/p' /etc/iscan/cluster.ini | gawk -F= '{print $2}'`

loadHAInfo()
{
    echo "Loading HA Information"
    for tag in hostname weight host_ip heartbeat_ip is_local role status
    do
        OUT=`/usr/iwss/haagntcmd GetClusterInfo |grep  $tag | tr -d '\t' | sed 's/^<.*>\([^<].*\)<.*>$/\1/' `
        eval ${tag}=`echo -ne \""${OUT}"\"`
    done

    H_ARRAY=( `echo ${hostname}` )
    W_ARRAY=( `echo ${weight}`   )
    L_ARRAY=( `echo ${is_local}` )
    I_ARRAY=( `echo ${host_ip}` )
    R_ARRAY=( `echo ${role}` )
    S_ARRAY=( `echo ${status}` )
}

logHADebug()
{
    echo "$TIMESTAMP" >> $LOGFILE
    echo "Preemption Mode = $preemption" >> $LOGFILE
    if [[ "${L_ARRAY[0]}" =~ "true" ]];then
        echo "Local Peer  :" >> $LOGFILE
        echo "Hostname    : ${H_ARRAY[0]}" >> $LOGFILE
        echo "Weight      : ${W_ARRAY[0]}" >> $LOGFILE
        echo "Host IP     : ${I_ARRAY[0]}" >> $LOGFILE
        echo "Role        : ${R_ARRAY[0]}" >> $LOGFILE
        echo "Status      : ${S_ARRAY[0]}" >> $LOGFILE
        echo "">> $LOGFILE
        echo "Remote Peer :" >> $LOGFILE
        echo "Hostname    : ${H_ARRAY[1]}" >> $LOGFILE
        echo "Weight      : ${W_ARRAY[1]}" >> $LOGFILE
        echo "Host IP     : ${I_ARRAY[1]}" >> $LOGFILE
        echo "Role        : ${R_ARRAY[1]}" >> $LOGFILE
        echo "Status      : ${S_ARRAY[1]}" >> $LOGFILE
    elif [[ "${L_ARRAY[1]}" =~ "true" ]]; then
        echo "Local Peer:" >> $LOGFILE
        echo "Hostname    : ${H_ARRAY[1]}" >> $LOGFILE
        echo "Weight      : ${W_ARRAY[1]}" >> $LOGFILE
        echo "Host IP     : ${I_ARRAY[1]}" >> $LOGFILE
        echo "Role        : ${R_ARRAY[1]}" >> $LOGFILE
        echo "Status      : ${S_ARRAY[1]}" >> $LOGFILE
        echo "">> $LOGFILE
        echo "Remote Peer :" >> $LOGFILE
        echo "Hostname    : ${H_ARRAY[0]}" >> $LOGFILE
        echo "Weight      : ${W_ARRAY[0]}" >> $LOGFILE
        echo "Host IP     : ${I_ARRAY[0]}" >> $LOGFILE
        echo "Role        : ${R_ARRAY[0]}" >> $LOGFILE
        echo "Status      : ${S_ARRAY[0]}" >> $LOGFILE
    fi
    echo "" >> $LOGFILE
}

checkENV()
{
    for pid in haagent svcmonitor iwssd isftpd
    do
        #pidof command can not work in cronjob
        pids=`ps -ef | grep -v "grep" | grep $pid`
        #echo "pids = $pids">> $LOGFILE
        if [ ! "$pids" ];then
            echo "$pid service was done"
            echo "$TIMESTAMP $pid service was done" >> $LOGFILE
            case $pid in
                'haagent')
                    echo "$TIMESTAMP HA Agent was down" >> $LOGFILE
                    /etc/iscan/S99IShaagent start
                    echo "$TIMESTAMP HA Agent started" >> $LOGFILE
                    sleep 10
                    ;;
                'svcmonitor')
                    echo "$TIMESTAMP SVC Monitor was down" >> $LOGFILE
                    /etc/iscan/S99ISsvcmonitor start
                    echo "$TIMESTAMP SVC Monitor started" >> $LOGFILE
                    sleep 10
                    ;;
                'iwssd')
                    echo "$TIMESTAMP HTTP Daemon was down" >> $LOGFILE
                    #`/etc/iscan/S99ISproxy stop`
                    /etc/iscan/S99ISproxy start
                    echo "$TIMESTAMP HTTP Daemon started" >> $LOGFILE
                    sleep 10
                    ;;
                'isftpd')
                    echo "$TIMESTAMP FTP Daemon was down" >> $LOGFILE
                    #`/etc/iscan/S99ISftp stop`
                    /etc/iscan/S99ISftp start
                    echo "$TIMESTAMP FTP Daemon started" >> $LOGFILE
                    sleep 10
                    ;;
            esac
        else
            echo "$TIMESTAMP $pid service was fine"
            sleep 1
        fi

    done

    # Check HA Status

    loadHAInfo

    if [[ ( ! "${R_ARRAY[0]}" =~ "parent" &&  ! "${R_ARRAY[0]}" =~ "child" ) || ( ! "${R_ARRAY[1]}" =~ "parent" && ! "${R_ARRAY[1]}" =~ "child" ) ]];then
        echo " HA Role Info was wrong"
        echo "$TIMESTAMP HA Role Info was wrong ! " >> $LOGFILE
        # Write Debug log
        logHADebug
        exit 1
    else
        echo "HA Environment was fine"
    fi
}

checkENV

if [[ ("${L_ARRAY[0]}" =~  "true"  && "${R_ARRAY[0]}" =~ "parent") || ( "${L_ARRAY[1]}" =~  "true"  && "${R_ARRAY[1]}" =~ "parent") ]];then
    echo "Current Node is Parent" >> $LOGFILE
    echo "Prepare to Trigger Script" >> $LOGFILE
    echo "Plese wait for 20 secs" >> $LOGFILE
    sleep 20
    echo "----------------------------------" >> $LOGFILE
    echo " Before Tool Trigger " >> $LOGFILE
    logHADebug
    case $1 in
        'Manual')
            echo "$TIMESTAMP Manual Switch Over" >> $LOGFILE
            /usr/iwss/haagntcmd Switch
            sleep 10
            ;;
        'HTTP')
            echo "$TIMESTAMP HTTP Force Switch Over " >> $LOGFILE
            killall -9 iwssd
            sleep 10
            ;;
        'FTP')
            echo "$TIMESTAMP FTP Force Switch Over" >> $LOGFILE
            killall -9 isftpd
            sleep 10
            ;;
        'HAAGENT')
            echo "$TIMESTAMP HA AGENT Force Switch Over" >> $LOGFILE
            /etc/iscan/S99IShaagent stop
            sleep 5
            /etc/iscan/S99IShaagent start
            echo "HA Agent restarted"
            sleep 10
            ;;
    esac
    echo "----------------------------------" >> $LOGFILE
    echo " After  Tool Trigger " >> $LOGFILE
    loadHAInfo
    logHADebug

    echo "----------------------------------" >> $LOGFILE
    echo "" >> $LOGFILE
    sleep 10
    checkENV
    exit 0
else
    echo "Child Node"
    echo "---------------------------------" >> $LOGFILE
    echo "Current Node was Child" >> $LOGFILE
    echo "---------------------------------" >> $LOGFILE
    logHADebug
    echo "---------------------------------" >> $LOGFILE
    exit 0
fi