Inotify + rsync script

exfile-test

- *.log
- *.swp

inotify-rsync.sh

#!/bin/bash
#
# echo 30000000 > /proc/sys/fs/inotify/max_user_watches
# echo 327679 > /proc/sys/fs/inotify/max_queued_events
# /path/to/inotify-rsync.sh > /path/to/log/inotify.log 2>&1
#
# /etc/sysctl.conf
# fs.inotify.max_user_watches = 30000000
# fs.inotify.max_user_instances = 5120
# fs.inotify.max_queued_events = 32767900

base_dir=/path/to/base_dir

passwd_file=/path/to/rsync.secrets
passwd_cmd=
if [ -f ${passwd_file} ]; then
    passwd_cmd="--password-file=${passwd_file}"
fi

###########################
# dir,name
dirs[1001]='/path/to/test/,sync_test'

sync_test[0]='/path/to/test/,[email protected],test'
sync_test[1]='/path/to/test/,[email protected],test'
sync_test[2]='/path/to/test/,[email protected],test'

# sync_name[1]='/path/to/local/dir,host,rsync_module'
###########################

for dirval in ${dirs[@]}; do {

dir=`echo ${dirval} | awk -F"," '{print $1}'`
name=`echo ${dirval} | awk -F"," '{print $2}'`
length=`echo ${dirval} | awk -F"," '{print $3}'`

declare -a 'sync=("${'"$name"'[@]}")'

inotifywait --exclude '(.*/*\.log|.*/*\.swp|.*/*\.swx|.*/*~$|.*data/.*cache/.*|.*data/sessions/.*)' \
 -mrq --timefmt '%d/%m/%y %H:%M' --format  '%T %w%f %e' \
 --event CLOSE_WRITE,create,move,delete $dir | while read date time file event
    do {
        d1=`date '+%Y%m'`
        d2=`date '+%Y%m%d'`
        d3=`date '+%Y-%m-%d:%H:%M:%S'`

        logdir=${base_dir}/log/${d1}
        mkdir -p ${logdir}
        #echo "[$d3] [$event] $file" >> ${logdir}/tasksh-${d2}.log
        case $event in
            CLOSE_WRITE,CLOSE|MODIFY|CREATE|MOVE|MOVED_TO|MOVED_TO,ISDIR|MOVE,ISDIR|CREATE,ISDIR|MODIFY,ISDIR)
                if [ "${file: -4}" != '4913' ]  && [ "${file: -1}" != '~' ]; then
                    echo "[${d3}] ["$event'] '$file >> ${logdir}/tasksh-${d2}.log
                    for item in ${sync[@]}; do {
                        src=`echo $item | awk -F"," '{print $1}'`
                        dest=`echo $item | awk -F"," '{print $2}'`
                        module=`echo $item | awk -F"," '{print $3}'`

                        replace="${src//\//\\/}"
                        relfile=`echo ${file} | sed -e "s/${replace}//"`
                        if [ -d ${file} ]; then
                            params=az
                            relfile=${relfile}/
                            file=${file}/
                        else
                            params=lptgoDz
                        fi
                        exfile=${base_dir}/task/exfile-${module}
                        exfile_cmd=
                        if [ -f ${exfile} ]; then
                            exfile_cmd="--exclude-from=${exfile}"
                        fi
                        #cmd="rsync -az ${exfile_cmd} --exclude=* --include='"$file"' ${passwd_cmd} $src $dest::$module"
                        cmd="rsync -${params} ${exfile_cmd} --include='"$file"' ${passwd_cmd} $file $dest::$module/${relfile}"
                        echo "[$d3] [$event] sync to $dest::$module" >> ${logdir}/tasksh-${d2}.log
                        #echo "[$d3] [$event] "$cmd >> ${logdir}/tasksh-${d2}.log
                        $cmd  >> ${logdir}/tasksh-${d2}.log
                    } &
                    done
                fi
                ;;

            MOVED_FROM|MOVED_FROM,ISDIR|DELETE|DELETE,ISDIR)
                if [ "${file: -4}" != '4913' ]  && [ "${file: -1}" != '~' ]; then
                    echo "[${d3}] ["$event'] '$file >> ${logdir}/tasksh-${d2}.log
                    for item in ${sync[@]}; do {
                        src=`echo $item | awk -F"," '{print $1}'`
                        dest=`echo $item | awk -F"," '{print $2}'`
                        module=`echo $item | awk -F"," '{print $3}'`

                        replace="${src//\//\\/}"
                        pdir=`dirname ${file}`
                        pdir=${pdir}/
                        reldir=`echo ${pdir} | sed -e "s/${replace}//"`
                        params=az
                        exfile=${base_dir}/task/exfile-${module}
                        exfile_cmd=
                        if [ -f ${exfile} ]; then
                            exfile_cmd="--exclude-from=${exfile}"
                        fi
                        #cmd="rsync -az --delete-delay ${exfile_cmd} ${passwd_cmd} $src $dest::$module"
                        cmd="rsync -${params} --del ${exfile_cmd} ${passwd_cmd} $pdir $dest::$module/${reldir}"
                        echo "[$d3] [$event] sync to $dest::$module" >> ${logdir}/tasksh-${d2}.log
                        #echo "[$d3] [$event] "$cmd >> ${logdir}/tasksh-${d2}.log
                        $cmd >> ${logdir}/tasksh-${d2}.log
                    } &
                    done
                fi
                ;;
        esac } &
    done } &
done

WAITPIDS="$WAITPIDS "$!;
wait $WAITPIDS

inotifyd

#!/bin/sh
### BEGIN INIT INFO
# Provides:
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start daemon at boot time
# Description:       Enable service provided by daemon.
### END INIT INFO

dir=""
user="root"
cmd="/path/to/inotify-rsync.sh"

name=`basename $0`
pid_file="/var/run/$name.pid"
stdout_log="/var/log/$name.log"
stderr_log="/var/log/$name.err"

get_pid() {
    cat "$pid_file"
}

is_running() {
    [ -f "$pid_file" ] && ps `get_pid` > /dev/null 2>&1
}

kill_tree() {
    local PID=$1
    if [ -z $PID ];
    then
        echo "No pid specified"
    fi

    local PPLIST=$PID
    local CHILD_LIST=`pgrep -P $PPLIST -d,`

    while [ ! -z "$CHILD_LIST" ]
    do
        PPLIST="$PPLIST,$CHILD_LIST"
        CHILD_LIST=`pgrep -P $CHILD_LIST -d,`
    done

    local SIGNAL=$2

    if [ -z $SIGNAL ]
    then
        SIGNAL="TERM"
    fi
    #do substring from comma to space
    kill -$SIGNAL ${PPLIST//,/ }
}

case "$1" in
    start)
    if is_running; then
        echo "Already started"
    else
        echo "Starting $name"
        cd "$dir"
            sudo -u "$user" $cmd > "$stdout_log" 2> "$stderr_log" \
        & echo $! > "$pid_file"
        if ! is_running; then
        echo "Unable to start, see $stdout_log and $stderr_log"
        exit 1
        fi
    fi
    ;;
    stop)
    if is_running; then
        echo "Stopping $name"
        kill_tree `get_pid`
        rm "$pid_file"
    else
        echo "Not running"
    fi
    ;;
    restart)
    $0 stop
    $0 start
    ;;
    status)
    if is_running; then
        echo "Running"
    else
        echo "Stopped"
        exit 1
    fi
    ;;
    *)
    echo "Usage: $0 {start|stop|restart|status}"
    exit 1
    ;;
esac

exit 0