Yet Another rsync Script

yars

#!/bin/bash
# This script makes incremental or full backups of a given folder using rsync.
#     Copyright (C) 2013  Arne Ludwig <[email protected]>
#
#     This program is free software: you can redistribute it and/or modify it under the terms of the
#     GNU General Public License as published by the Free Software Foundation, either version 3 of
#     the License, or (at your option) any later version.
#
#     This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
#     without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
#     the GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License along with this program. If
#     not, see <http://www.gnu.org/licenses/>.

# You may specify the following options in a seperate file in bash syntax, eg. './YArS.sh backup.config'.

# Specify folder(s) to make a backup of. If you want to backup only the contents of a folder you
# have to precede it with a blackslash ('/'); otherwise the folder itself will be copied.
: ${SOURCES:=()}

# This is the destination folder. It can be any local or remote folder (only ssh supported). Make
# sure it exists and you have write permissions. For each backup a subfolder named
# 'YYYY-mm-dd-MM-HH-SS' will be created. If the creation fails the script will exit with a non-zero
# status.
: ${DESTINATION:=''}

# TODO use exclude file instead
# Exclude file patterns from the backup (see --exclude option from rsync). You may specify multiple
# patterns separated by colons (':').
: ${EXCLUDE:=''}

# This provides a way to control rsync. This OVERRIDES the default options (see RSYNC_OPTIONS
# below).
: ${RSYNC_OPTIONS:=''}

# If a log file is given a new line for each run will be generated; unless the call results in the
# help or version being shown.
: ${LOG:=''}


# The following options control the magnitude of the overall backup. Magnitude relates to number or
# age. All deletion takes place AFTER the current backup unless the --delete-only options is
# present.

# If you set this option to a positive integer only that much backups will be kept -- oldest are
# deleted first.
: ${KEEP_NUM:=0}

# TODO make a adaptive history, eg:
#     - the last 60 minutes
#     - the last 24 hour
#     - the last 30 days
#     - the last 12 month
#     - the last 2 years

# If you set this option to a to some abitrary date only backups newer than that
# date will be kept; if the date lies in the future nothing is deleted and a
# warning is issued.
#
# The date string must be understood by 'date --date=STRING', eg '3 months ago'.
# Refer to `man date` and `info coreutils 'date input formats'` for details.
: ${KEEP_AFTER:=''}


# EDIT ANYTHING AFTER THIS POINT ON YOUR OWN RISK.
#
# --- SCRIPT BEGIN -------------------------------------------------------------

VERSION="v0.1.7a 2013-10-16"

# Matches a timestamp as produced by the function with the same name and an
# optional trailing backslash ('/')
TIMESTAMP_REGEX="^[0-9]{4}(-[0-5][0-9]){5}\/?$"

# TODO fetch paths from command line as last two params
# TODO fetch EXCLUDE, LOG, KEEP_* from command line
function print_help {
  (
    echo -n "USAGE $(basename "$0") [-f|--full|-d|--delete-only] [-s|--suppress-clutter] "
       echo "[-q|--quiet] [-h|--help] [-v|--version]"
    echo "  -f, --full              Run a full backup (instead of incremental)"
    echo "  -d, --delete-only       Just delete backups to meet magnitude requirements"
    echo "  -s, --suppress-clutter  Delete backup if no changes were detected"
    echo "  -q, --quiet             Suppress output except for errors"
    echo "  -v, --version           Display version number, copyright info and exit"
    echo "  -h, --help, --usage     Display this help and exit"
    echo
    echo "Configure this script to match your own needs by:"
    echo "- providing a configuration file on stdin,"
    echo "- editing this file or"
    echo "- by setting the appropriate environment variables"
  ) 1>&2
}

function print_version {
  (
    echo "$(basename "$0") ${VERSION}"
    echo
    echo "Copyright (C) 2013  Arne Ludwig <[email protected]>"
    echo "This program comes with ABSOLUTELY NO WARRANTY."
    echo "This is free software, and you are welcome to redistribute it"
    echo "under certain conditions; for details look into the source code."
  ) 1>&2
}

function read_config {
  if [[ -f "${CONFIG_FILE}" ]]; then
    source "${CONFIG_FILE}"
  fi
}

function report {
  (( IS_QUIET )) || echo "$@"
}

function report_unkown_option {
  echo "error: unkown option '$1'" 1>&2
}

function log {
  if [[ -n ${LOG} ]]; then
    echo "$(date +'%F %T') $(whoami)@$(hostname) $0[$$] $1" >> "${LOG}"
  fi
}

function escaped {
  local QUOT="'\''"
  echo "'${1//\'/$QUOT}'"
}

function is_remote {
  [[ "$1" =~ [^:]:[^:] ]]
}

function get_remote_shell {
  # TODO extract from $2 if appropriate; see -e|--rsh
  echo "ssh $(get_host "$1")"
}

function remote {
  $SHELL "$(</dev/stdin)"
}

function get_host {
  is_remote "$1" && [[ "$1" =~ (^.*[^:]):[^:] ]] && echo "${BASH_REMATCH[1]}"
}

function get_local_destination {
  if is_remote "$1"; then
    [[ "$1" =~ [^:]:([^:].*$) ]] && echo "${BASH_REMATCH[1]}"
  else
    echo "$1"
  fi
}

function timestamp {
  date +%Y-%m-%d-%H-%M-%S --date="${1:-now}"
}

DEFINE_FORCE_REMOVAL='
  function force_removal {
    rm -rf "$@" ||
    ( chmod o+rwx "$@" && rm -rf "$@" ) ||
    ( [[ -t 0 ]] && sudo rm -rf "$@" )
  }'

DEFINE_IS_BEFORE='
  function is_before {
    local DATE1;
    local DATE2;
    if [[ "$1" =~ '"${TIMESTAMP_REGEX}"' ]]; then
      # remove hyphens and backslashes
      DATE1=${1//[\/-]/}
    else
      DATE1=$(date +%Y%m%d%H%M%S --date="${1:-'"$(date -Iseconds)"'}")
    fi

    if [[ "$2" =~ '"${TIMESTAMP_REGEX}"' ]]; then
      # remove hyphens and backslashes
      DATE2=${2//[\/-]/}
    else
      DATE2=$(date +%Y%m%d%H%M%S --date="${2:-'"$(date -Iseconds)"'}")
    fi

    (( DATE1 < DATE2 ))
  }'
eval "${DEFINE_IS_BEFORE}"


# Command line processing
for ARG in "$@"; do
  case "${ARG}" in
    -f|--full)
      REQUIRE_FULL=1
      ;;
    -d|--delete-only)
      DELETE_ONLY=1
      ;;
    -s|--suppress-clutter)
      SUPPRESS_CLUTTER=1
      ;;
    -q|--quiet)
      IS_QUIET=1
      ;;
    -v|--version)
      SHOW_VERSION=1
      ;;
    -h|--help)
      HELP=1
      ;;
    *)
      if [[ -f "${ARG}" && -z "$CONFIG_FILE" ]]; then
        CONFIG_FILE="${ARG}"
      else
        report_unkown_option "${ARG}"
        OPTIONS_ERROR=1
      fi
      ;;
  esac
done

if (( $# == 0 )); then
  OPTIONS_ERROR=1
fi

if (( REQUIRE_FULL && DELETE_ONLY )); then
  echo "error: --full and --delete-only are mutual exclusive" 1>&2
  OPTIONS_ERROR=1
fi

# If necessary print usage or version and exit
if (( HELP || OPTIONS_ERROR )); then
  print_help
  exit ${OPTIONS_ERROR}
elif (( SHOW_VERSION )); then
  print_version
  exit 0
fi

read_config

# Determine shell and set LOCAL_DESTINATION
if is_remote "${DESTINATION}"; then
  SHELL=$(get_remote_shell "${DESTINATION}" "${RSYNC_OPTIONS}")
  LOCAL_DESTINATION="$(get_local_destination "${DESTINATION}")"
  report "Using remote shell '$SHELL'."
else
  SHELL="$SHELL -c"
  LOCAL_DESTINATION="$DESTINATION"
fi

# Pin KEEP_AFTER_DATE to avoid confusion due to timing
if [[ -n ${KEEP_AFTER} ]]; then
  KEEP_AFTER_DATE="$(date -Iseconds --date "${KEEP_AFTER}")"
fi

if (( ! DELETE_ONLY )); then
  # Assert DESTINATION exists
  # REMOTE_ACTION
  remote <<< "mkdir -p ${LOCAL_DESTINATION}"

  # Lists subdirectories of DESTINATION in descending order and trailing '/' for
  # directories. This format is then parsed by grep (see above).
  # REMOTE_ACTION
  PREVIOUS_BACKUP=$(remote <<< "ls -1Fr $(escaped "${LOCAL_DESTINATION}") | \
                                grep -m1 -E $(escaped "${TIMESTAMP_REGEX}")")
  HAVE_PREVIOUS_BACKUP=$(( $? == 0 ))

  PREVIOUS_BACKUP="${LOCAL_DESTINATION}/${PREVIOUS_BACKUP}"
  CURRENT_BACKUP="${DESTINATION}/$(timestamp)"

  LOCAL_RSYNC_OPTIONS=''

  # Include the patterns from EXCLUDE
  OLD_IFS="${IFS}"
  IFS=":"
  for EXLUDE_PATH in $EXCLUDE; do
    LOCAL_RSYNC_OPTIONS+=" --exclude ${EXLUDE_PATH}"
  done
  IFS="${OLD_IFS}"

  # Default options (see 'man rsync' for details):
  #   -a                 archive mode; equals -rlptgoD (no -H,-A,-X)
  #   -h                 output numbers in a human-readable format
  #   --delete           delete extraneous files from dest dirs
  : ${RSYNC_OPTIONS:='-ah --delete'}
  # -savPh --delete --numeric-ids --stats

  # Make incremental backup by default ...
  # but only if we have a previous backup and no full backup is required
  if (( HAVE_PREVIOUS_BACKUP && ! REQUIRE_FULL )); then
    LOCAL_RSYNC_OPTIONS+=" --link-dest=${PREVIOUS_BACKUP}"
  fi

  # Report parameters
  report -e '\033[1;37mSource(s):\033[00m'
  for SRC in "${SOURCES[@]}"; do
    report -e "\t${SRC}"
  done
  report -e '\033[1;37mDestination:\033[00m'
  report -e "\t${DESTINATION}"
  report -e '\033[1;37mRsync-Options:\033[00m'
  report -e "\t${LOCAL_RSYNC_OPTIONS} ${RSYNC_OPTIONS}"

  # Dry run rsync to make check for differences ...
  if (( SUPPRESS_CLUTTER )); then
    CHANGES=$(rsync --itemize-changes --dry-run ${LOCAL_RSYNC_OPTIONS} ${RSYNC_OPTIONS}\
              ${SOURCES[@]} ${CURRENT_BACKUP})
  fi

  # Check if there were changes
  if [[ -n ${CHANGES} ]] || ! (( SUPPRESS_CLUTTER )); then
    # Finally, run rsync to make the backup ...
    rsync ${LOCAL_RSYNC_OPTIONS} ${RSYNC_OPTIONS} "${SOURCES[@]}" "${CURRENT_BACKUP}"

    if (( $? == 0 )); then
      log 'backup done'
      report -e '\e[1;32mBackup successfully done.\033[00m'
    else
      log 'backup failed'
      report -e '\e[1;31mBackup failed.\033[00m'
    fi
  else
    log 'everything in sync; backup done'
    report -e '\e[1;32mNo changes since last backup.\033[00m'
  fi
fi


# Now, let's delete old backups if desired!

# Count the present backups (see PREVIOUS_BACKUP for details)
NUM_BACKUPS=$(remote <<< "ls -1F $(escaped "${LOCAL_DESTINATION}") | \
                          grep -cE $(escaped "${TIMESTAMP_REGEX}")")

if (( KEEP_NUM > 0 )); then
  report "Removing $((KEEP_NUM < NUM_BACKUPS ? NUM_BACKUPS - KEEP_NUM : 0)) old backups."
  remote <<EOF
    ${DEFINE_FORCE_REMOVAL}
    KEEP_NUM=${KEEP_NUM}
    NUM_BACKUPS=${NUM_BACKUPS}
    LOCAL_DESTINATION=$(escaped "${LOCAL_DESTINATION}")
    TIMESTAMP_REGEX=$(escaped "${TIMESTAMP_REGEX}")
    OLDEST_BACKUP=$(escaped "${OLDEST_BACKUP}")

    # Delete the oldest backup until the count drops below NUM_BACKUPS
    until (( NUM_BACKUPS <= KEEP_NUM )); do
      # Get the oldest backup's name (see PREVIOUS_BACKUP for details)
      OLDEST_BACKUP="\$(ls -1F "\${LOCAL_DESTINATION}" | grep -m1 -E "\${TIMESTAMP_REGEX}")"
      OLDEST_BACKUP="\${LOCAL_DESTINATION}/\${OLDEST_BACKUP}"

      force_removal "\${OLDEST_BACKUP}"
      NUM_BACKUPS=\$(( NUM_BACKUPS - 1 ))
    done
EOF
fi

if [[ -n ${KEEP_AFTER} ]]; then
  if is_before "${KEEP_AFTER_DATE}"; then
    report "Removing backups older than ${KEEP_AFTER}, ie before $(date --date=${KEEP_AFTER_DATE})."
    remote <<EOF
      ${DEFINE_FORCE_REMOVAL}
      ${DEFINE_IS_BEFORE}
      LOCAL_DESTINATION=$(escaped "${LOCAL_DESTINATION}")
      TIMESTAMP_REGEX=$(escaped "${TIMESTAMP_REGEX}")
      KEEP_AFTER_DATE=$(escaped "${KEEP_AFTER_DATE}")
      OLDEST_BACKUP=${OLDEST_BACKUP}

      for BACKUP in \$(ls -1F "\${LOCAL_DESTINATION}" | grep -E "\${TIMESTAMP_REGEX}"); do
        if is_before "\${BACKUP}" "\${KEEP_AFTER_DATE}"; then
          force_removal "\${LOCAL_DESTINATION}/\${BACKUP}"
        fi
      done
EOF
  else
    echo 'warning: KEEP_AFTER lies in the future; doing nothing' >&2
    log 'warning: KEEP_AFTER lies in the future; doing nothing'
  fi
fi