proof of concept error detection for back-in-time

backintime-alert.sh

#!/bin/bash

# STATUS: proof of concept.  TODO: scan syslog as well, with
# forward-compatibility for systemd's journalctl syslog also
# WARNING: this may have bugs.

# CONFIGURE THIS VARIABLE ...based on BIT config file, but here
# it is hard coded for rapid prototyping
DEPTH=6

# This script assumes you're doing daily backups.  It could be
# augmented to automatically detect the schedule via reading
# the back-in-time config file.

# pipe all output of this script to splunk or nagios (whatever)
# which will see any lines beginning with "ERROR:" and send
# alerts as appropriate...or just pipe to email and disable
# the lines that echo informational details, beginning with "INFO:"

PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin

# get list of configured backup jobs ("profiles") Output example: 
# /path/to/backups/backintime/<hostname>/root/1

JOBS=( $( find /backups -maxdepth $DEPTH -mindepth $DEPTH -type d ) );

# get bash array length => ${#JOBS[@]}
# loop over bash array => for ${JOBS[@]};

for JOB in ${JOBS[@]};
do

   # get list of backups that should have happened on this date 
   # (There should always be at least one).
   BACKUPS=( $( find $JOB -maxdepth 1 -mindepth 1 -type d -name "$( date +%Y%m%d )*" ) );

   if [[ ${#BACKUPS[@]} -eq 0 ]];
   then
      echo "ERROR: One of our scheduled backups was missed.  You should check `hostname`:$JOB"
      ls -l $JOB

      continue
   fi

   for BACKUP in ${BACKUPS[@]};
   do
      if [[ ! -e $BACKUP/takesnapshot.log.bz2 ]];
      then
         echo "ERROR: No log found for one of today's backups! You should check `hostname`:$BACKUP"

         continue
      else
         echo "INFO: Scanning backup log in $BACKUP/takesnapshot.log.bz2"
      fi

      ERRORS=$( bzgrep '^\[E\] Error' $BACKUP/takesnapshot.log.bz2 2>&1 )

      if [[ "$ERRORS" != "" ]];
      then
         echo -e "Problems detected in this backup.  See system message:\n$ERRORS" | \

         while read LINE; 
         do
            echo 'ERROR: ' $line;
         done
      fi
   done
done