Skip to content

Instantly share code, notes, and snippets.

@deajan

deajan/backup_tool_script

Last active November 29, 2018 10:38
Show Gist options
  • Save deajan/b8bec02a3dc33e90d19c8b300e027aab to your computer and use it in GitHub Desktop.
Save deajan/b8bec02a3dc33e90d19c8b300e027aab to your computer and use it in GitHub Desktop.
Download ZIP
backup_tool_script v0.4.4
Raw
backup_tool_script
#!/usr/bin/env bash
PROGRAM="backup_tool_script"
PROGRAM_VERSION=0.4.4
PROGRAM_BUILD=2018110501
AUTHOR="(C) 2017-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - [email protected]"
IS_STABLE=yes
## backup_tool_script - A script to check burp backup sanity
## backup_tool_script can verify a given number of backups for each client. It can run verifiy operations in parallel.
## Verify operations are timed in order to stop them after a given amount of time, leaving the system performance ready for backup operations.
## The script can also list clients that have outdated backups. It uses two different methods to list clients in order to detect rogue clients.
## It can also ensure that the burp server service is running properly, relaunch it if needed, on a scheduled basis.
## The script can send a warning / error when problems are found, even while operating.
## The script can send a warning / error when disk quotas exceed, even while operating.
## backup_tool_script can also launch vss_strip for each file found in a given directory.
## The script can also send mails directly to the client, if an email address is given in client config file as 'label = email_address : [email protected]'
## When exiting, backup_tool_script ensures that no forked burp processes remain, without touching other burp processes that didn't belong to backup_tool_script.
## Set an unique identifier for the script which will be used for logs and alert mails
INSTANCE_ID="base"
## Backup verifications timers
## After how much time (in seconds) for a single verification a warning should be logged (defaults to 3 hours)
SOFT_MAX_EXEC_TIME_PER_VERIFY=10800
## After how much time (in seconds) for a single verification the process should be stopped (defaults to 5 hours)
HARD_MAX_EXEC_TIME_PER_VERIFY=18000
## After how much seconds of execution of all steps a warning should be logged (defaults to 10 hours)
SOFT_MAX_EXEC_TIME=64800
## After how much seconds of execution of all steps a verification process should be stopped (defaults to 12 hours)
HARD_MAX_EXEC_TIME=82800
# Verify operations checks
## When a client isn't idle, we can postpone the it's backup verification process. How many times should we retry the verification command. Set this to 0 to disable operation postponing
POSTPONE_RETRY=2
## When postponed, how much time (in seconds) before next try (defauls to 1 hour)
POSTPONE_TIME=3600
## Backup executable (can be set to /usr/sbin/burp, /usr/local/sbin/burp, or autodetect via $(type -p burp))
BACKUP_EXECUTABLE=/usr/local/sbin/burp # PROD = @sbindir@/@name@
## @name@ service type (can be "initv" or "systemd")
SERVICE_TYPE=initv # PROD = initv
## How many simultaneous verify operations should be launched (please check I/O and CPU usage before increasing this)
PARELLEL_VERIFY_CONCURRENCY=2
# ------------ Mail alert settings -------------
## General alert mail subject
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
## Optional change of mail body encoding (using iconv)
## By default, all mails are sent in UTF-8 format without headers
## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle) for maximal compatibility
MAIL_BODY_CHARSET="ISO-8859-1"
# ------------ Client specific alert email template --------------
## Email subject
CLIENT_ALERT_SUBJECT="burp backup - Warning about your backup"
## Valid message body placeholders are
## [NUMBERDAYS] is the number of days we check.
## [QUOTAEXCEED] is the size of the actual backup versus the quota.
## [CLIENT] is the client name
## [INSTANCE] is the current instance name
## Message sent directly to client email address when no recent backups are found.
CLIENT_ALERT_BODY_OUTDATED="Hello
No valid backup sets found in the last [NUMBERDAYS] day(s) for client [CLIENT].
Please leave your computer online enough time for a backup to occur.
Contact your system administrator for further information.
burp backup server $INSTANCE.
"
# Message sent directly to client email address when quota exceeded.
CLIENT_ALERT_BODY_QUOTA="Hello,
Your backup disk quota has been exceeded ([QUOTAEXCEED]) for client [CLIENT].
Contact your system administrator for further information.
burp backup server $INSTANCE.
"
# ------------ Do not modify under this line unless you have great cow powers --------------
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
fi
export LC_ALL=C
_LOGGER_SILENT=false
_LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date"
if [ "$KEEP_LOGGING" == "" ]; then
KEEP_LOGGING=1801
fi
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=false
WARN_ALERT=false
LOCAL_USER=$(whoami)
LOCAL_HOST=$(hostname)
SCRIPT_PID=$$
# Get a random number on Windows BusyBox alike, also works on most Unixes that have dd, if dd is not found, then return $RANDOM
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
local isFirst=true
if type dd >/dev/null 2>&1; then
# Some read bytes can't be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
if [ $isFirst == false ] || [ $(printf "%d" "'$char") != "0" ]; then
number=$number$(printf "%d" "'$char")
isFirst=false
fi
if [ ${#number} -ge $digits ]; then
echo ${number:0:$digits}
break;
fi
done
elif [ "$RANDOM" -ne 0 ]; then
echo $RANDOM
else
Logger "Cannot generate random number." "ERROR"
fi
}
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"
## Default log file until config file is loaded
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.log"
elif [ -w . ]; then
LOG_FILE="./$PROGRAM.log"
else
LOG_FILE="/tmp/$PROGRAM.log"
fi
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
RUN_DIR=/var/tmp
else
RUN_DIR=.
fi
#### DEBUG SUBSET ####
## allow function call checks #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG
_DEBUG=yes #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no
_LOGGER_VERBOSE=false
else
trap 'TrapError ${LINENO} $?' ERR
_LOGGER_VERBOSE=true
fi
if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.1
fi
#### DEBUG SUBSET END ####
#### Logger SUBSET ####
#### RemoteLogger SUBSET ####
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
local IFS="$1"; shift; echo "$*";
}
# Sub function of Logger
function _Logger {
local logValue="${1}" # Log to file
local stdValue="${2}" # Log to screeen
local toStdErr="${3:-false}" # Log to stderr instead of stdout
if [ "$logValue" != "" ]; then
echo -e "$logValue" >> "$LOG_FILE"
# Current log file
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP"
fi
if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
if [ $toStdErr == true ]; then
# Force stderr color in subshell
(>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi
}
# General log function with log levels:
# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
function Logger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date) - "
else
prefix=""
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "$prefix$value" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
else
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}
#### Logger SUBSET END ####
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
if kill -0 "$pid" > /dev/null 2>&1; then
# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
}
function KillAllChilds {
local pids="${1}" # List of parent pids to kill separated by semi-colon
local self="${2:-false}" # Should parent be killed too ?
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
local errorcount=0
IFS=';' read -a pidsArray <<< "$pids"
for pid in "${pidsArray[@]}"; do
KillChilds $pid $self
if [ $? != 0 ]; then
errorcount=$((errorcount+1))
fi
done
return $errorcount
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
__CheckArguments 0-1 $# "$@" #__WITH_PARANOIA_DEBUG
local attachment
local attachmentFile
local subject
local body
if [ "$DESTINATION_MAILS" == "" ]; then
return 0
fi
if [ "$_DEBUG" == "yes" ]; then
Logger "Debug mode, no warning mail will be sent." "NOTICE"
return 0
fi
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
attachment=false
else
attachment=true
fi
body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of log"$'\n\n'"$(tail -n 1000 $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)"
if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID"
elif [ $WARN_ALERT == true ]; then
subject="Warning alert for $INSTANCE_ID"
else
subject="Alert for $INSTANCE_ID"
fi
if [ $runAlert == true ]; then
subject="Currently runing - $subject"
else
subject="Finished run - $subject"
fi
if [ "$attachment" == true ]; then
attachmentFile="$ALERT_LOG_FILE"
fi
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
# Delete tmp log file
if [ "$attachment" == true ]; then
if [ -f "$ALERT_LOG_FILE" ]; then
rm -f "$ALERT_LOG_FILE"
fi
fi
}
# Generic email sending function.
# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or ""
# SendEmail "subject" "Body text" "[email protected] [email protected]" "/path/to/attachment.file"
# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend)
# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or ""
# smtp_server.domain.tld is mandatory, as is smtpPort (should be 25, 465 or 587)
# encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "[email protected] [email protected]" "/path/to/attachment.file" "[email protected]" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"
# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc
#set ttycharset=iso-8859-1
#set sendcharsets=iso-8859-1
#set encoding=8bit
function SendEmail {
local subject="${1}"
local message="${2}"
local destinationMails="${3}"
local attachment="${4}"
local senderMail="${5}"
local smtpServer="${6}"
local smtpPort="${7}"
local encryption="${8}"
local smtpUser="${9}"
local smtpPassword="${10}"
__CheckArguments 3-10 $# "$@" #__WITH_PARANOIA_DEBUG
local mail_no_attachment=
local attachment_command=
local encryption_string=
local auth_string=
local i
if [ ! -f "$attachment" ]; then
attachment_command="-a $attachment"
mail_no_attachment=1
else
mail_no_attachment=0
fi
for i in "${destinationMails[@]}"; do
if [ $(CheckRFC822 "$i") -ne 1 ]; then
Logger "Given email [$i] does not seem to be valid." "WARN"
fi
done
# Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then
if type iconv > /dev/null 2>&1; then
echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP"
message="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP)"
else
Logger "iconv utility not installed. Will not convert email charset." "NOTICE"
fi
fi
if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then
if [ "$smtpPort" == "" ]; then
Logger "Missing smtp port, assuming 25." "WARN"
smtpPort=25
fi
if type sendmail > /dev/null 2>&1; then
if [ "$encryption" == "tls" ]; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
elif [ "$encryption" == "ssl" ]; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
else
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
fi
if [ $? != 0 ]; then
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
# Don't bother try other mail systems with busybox
return 1
else
return 0
fi
else
Logger "Sendmail not present. Won't send any mail" "WARN"
return 1
fi
fi
if type mutt > /dev/null 2>&1 ; then
# We need to replace spaces with comma in order for mutt to be able to process multiple destinations
echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mutt) !!!" "WARN"
else
Logger "Sent mail using mutt." "NOTICE"
return 0
fi
fi
if type mail > /dev/null 2>&1 ; then
# We need to detect which version of mail is installed
if ! $(type -p mail) -V > /dev/null 2>&1; then
# This may be MacOS mail program
attachment_command=""
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $attachment"
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a$attachment"
else
attachment_command=""
fi
echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
echo "$message" | $(type -p mail) -s "$subject" "$destinationMails"
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN"
else
Logger "Sent mail using mail command without attachment." "NOTICE"
return 0
fi
else
Logger "Sent mail using mail command." "NOTICE"
return 0
fi
fi
if type sendmail > /dev/null 2>&1 ; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destinationMails"
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN"
else
Logger "Sent mail using sendmail command without attachment." "NOTICE"
return 0
fi
fi
# Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$senderMail" == "" ]; then
Logger "Missing sender email." "ERROR"
return 1
fi
if [ "$smtpServer" == "" ]; then
Logger "Missing smtp port." "ERROR"
return 1
fi
if [ "$smtpPort" == "" ]; then
Logger "Missing smtp port, assuming 25." "WARN"
smtpPort=25
fi
if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then
Logger "Bogus smtp encryption, assuming none." "WARN"
encryption_string=
elif [ "$encryption" == "tls" ]; then
encryption_string=-starttls
elif [ "$encryption" == "ssl" ]:; then
encryption_string=-ssl
fi
if [ "$smtpUser" != "" ] && [ "$smtpPassword" != "" ]; then
auth_string="-auth -user \"$smtpUser\" -pass \"$smtpPassword\""
fi
$(type mailsend.exe) -f "$senderMail" -t "$destinationMails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtpServer" -port "$smtpPort" $encryption_string $auth_string
if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0
fi
fi
# pfSense specific
if [ -f /usr/local/bin/mail.php ]; then
echo "$message" | /usr/local/bin/mail.php -s="$subject"
if [ $? != 0 ]; then
Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
else
Logger "Sent mail using pfSense mail.php." "NOTICE"
return 0
fi
fi
# If function has not returned 0 yet, assume it is critical that no alert can be sent
Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
}
#### TrapError SUBSET ####
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_LOGGER_SILENT == false ]; then
(>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m")
fi
}
#### TrapError SUBSET END ####
# Quick and dirty performance logger only used for debugging
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
local i #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG
done #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
}
# Checks email address validity
function CheckRFC822 {
local mail="${1}"
local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$"
if [[ $mail =~ $rfc822 ]]; then
echo 1
else
echo 0
fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : '^[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[0-9]+$ ]]; then
echo 1
else
echo 0
fi
fi
}
# Usage [ $(IsNumeric $var) -eq 1 ]
function IsNumeric {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
fi
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
echo $(IsNumeric "$value")
}
## Modified version of http://stackoverflow.com/a/8574392
## Usage: [ $(ArrayContains "needle" "${haystack[@]}") -eq 1 ]
function ArrayContains () {
local needle="${1}"
local haystack="${2}"
local e
if [ "$needle" != "" ] && [ "$haystack" != "" ]; then
for e in "${@:2}"; do
if [ "$e" == "$needle" ]; then
echo 1
return
fi
done
fi
echo 0
return
}
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0
else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0
fi
}
## Main asynchronous execution function
## Function can work in:
## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for
## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed.
## Example of improved wait $!
## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false
## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
## sleep 15 &
## pid=$!
## sleep 20 &
## pid2=$!
## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions"
## Bear in mind that given commands and conditions need to be quoted
## ExecTasks has the following ofunctions subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds
## Full call
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
function ExecTasks {
# Mandatory arguments
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll)
for i in "${booleans[@]}"; do
test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries)
for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
fi
# Expand validExitCodes into array
IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
# ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing pids of commands currently run
local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed
local temp
# Common variables
local pid # Current pid working on
local pidState # State of the process
local mainItemCount=0 # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true)
local counter=0
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
local functionMode
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3)
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
# Init function variables depending on mode
if [ $numberOfProcesses -gt 0 ]; then
functionMode=ParallelExec
else
functionMode=WaitForTaskCompletion
fi
if [ $readFromFile == false ]; then
if [ $functionMode == "WaitForTaskCompletion" ]; then
IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount="${#pidsArray[@]}"
else
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount="${#commandsArray[@]}"
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount="${#commandsConditionArray[@]}"
fi
else
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
Logger "Cannot read main file [$mainInput]." "WARN"
fi
if [ "$auxInput" != "" ]; then
if [ -f "$auxInput" ]; then
auxItemCount=$(wc -l < "$auxInput")
else
Logger "Cannot read aux file [$auxInput]." "WARN"
fi
fi
fi
if [ $functionMode == "WaitForTaskCompletion" ]; then
# Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode
counter=$mainItemCount
fi
Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
function _ExecTasksTimeCheck {
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$((SECONDS - seconds_begin))
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
softAlert=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
if [ $noErrorLogsAtAll != true ]; then
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
done
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount
else
return 129
fi
fi
}
function _ExecTasksPidsCheck {
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
# Check if pid hasn't run more than soft/hard perProcessTime
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then
if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time [$softPerProcessTime] exceeded for pid [$pid]." "WARN"
if [ "${commandsArrayPid[$pid]}]" != "" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
fi
softAlert=true
SendAlert true
fi
fi
if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then
if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then
Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR"
if [ "${commandsArrayPid[$pid]}]" != "" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
fi
fi
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Command with pid [$pid] stopped successfully." "NOTICE"
else
if [ $noErrorLogsAtAll != true ]; then
Logger "Could not stop command with pid [$pid]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
fi
newPidsArray+=($pid)
fi
else
# pid is dead, get its exit code from wait command
wait $pid
retval=$?
# Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$failedPidsList" == "" ]; then
failedPidsList="$pid:$retval"
else
failedPidsList="$failedPidsList;$pid:$retval"
fi
else
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
fi
fi
hasPids=true ##__WITH_PARANOIA_DEBUG
fi
done
# hasPids can be false on last iteration in ParallelExec mode
if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
}
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do
_ExecTasksTimeCheck
retval=$?
if [ $retval -ne 0 ]; then
return $retval;
fi
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do
_ExecTasksTimeCheck
retval=$?
if [ $retval -ne 0 ]; then
return $retval;
fi
executeCommand=false
isPostponedCommand=false
currentCommand=""
currentCommandCondition=""
needsPostponing=false
if [ $readFromFile == true ]; then
# awk identifies first line as 1 instead of 0 so we need to increase counter
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
fi
# Check if we need to fetch postponed commands
if [ "$currentCommand" == "" ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP")
isPostponedCommand=true
fi
else
currentCommand="${commandsArray[$counter]}"
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition="${commandsConditionArray[$counter]}"
fi
if [ "$currentCommand" == "" ]; then
currentCommand="${postponedCommandsArray[$postponedCounter]}"
currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}"
isPostponedCommand=true
fi
fi
# Check if we execute postponed commands, or if we delay them
if [ $isPostponedCommand == true ]; then
# Get first value before '@'
postponedExecTime="${currentCommand%%@*}"
postponedExecTime=$((SECONDS-postponedExecTime))
# Get everything after first '@'
temp="${currentCommand#*@}"
# Get first value before '@'
postponedRetryCount="${temp%%@*}"
# Replace currentCommand with actual filtered currentCommand
currentCommand="${temp#*@}"
# Since we read a postponed command, we may decrase postponedItemCounter
postponedItemCount=$((postponedItemCount-1))
#Since we read one line, we need to increase the counter
postponedCounter=$((postponedCounter+1))
else
postponedRetryCount=0
postponedExecTime=0
fi
if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $((minTimeBetweenRetries)) ]) || [ $isPostponedCommand == false ]; then
if [ "$currentCommandCondition" != "" ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true
subRetval=$?
if [ $subRetval -ne 0 ]; then
# is postponing enabled ?
if [ $maxPostponeRetries -gt 0 ]; then
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE"
postponedRetryCount=$((postponedRetryCount+1))
if [ $postponedRetryCount -ge $maxPostponeRetries ]; then
Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE"
else
needsPostponing=true
fi
postponedExecTime=0
else
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE"
fi
else
executeCommand=true
fi
else
executeCommand=true
fi
else
needsPostponing=true
fi
if [ $needsPostponing == true ]; then
postponedItemCount=$((postponedItemCount+1))
if [ $readFromFile == true ]; then
echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP"
echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP"
else
postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand")
postponedCommandsConditionArray+=("$currentCommandCondition")
fi
fi
if [ $executeCommand == true ]; then
Logger "Running current check [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$currentCommand"
# Initialize pid execution time array
pidsTimeArray[$pid]=0
else
Logger "Skipping command [$currentCommand]." "DEBUG"
fi
if [ $isPostponedCommand == false ]; then
counter=$((counter+1))
fi
_ExecTasksPidsCheck
done
fi
_ExecTasksPidsCheck
done
Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\""
if [ $mainItemCount -eq 1 ]; then
return $retval
else
return $errorcount
fi
}
function CleanUp {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
############################################ END OF OFUNCTIONS CODE
function TrapQuit {
local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
fi
if [ $ERROR_ALERT == true ]; then
Logger "$PROGRAM finished with errors." "ERROR"
if [ "$_DEBUG" != "yes" ]
then
SendAlert
else
Logger "Debug mode, no alert mail will be sent." "NOTICE"
fi
exitcode=1
elif [ $WARN_ALERT == true ]; then
Logger "$PROGRAM finished with warnings." "WARN"
if [ "$_DEBUG" != "yes" ]
then
SendAlert
else
Logger "Debug mode, no alert mail will be sent." "NOTICE"
fi
exitcode=2 # Warning exit code must not force daemon mode to quit
else
Logger "$PROGRAM finished." "ALWAYS"
exitcode=0
fi
CleanUp
KillChilds $SCRIPT_PID > /dev/null 2>&1
Logger "Elapsed [$SECONDS] seconds." "DEBUG"
exit $exitcode
}
# Takes as many file arguments as needed
function InterleaveFiles {
local counter=0
local hasLine=true
local i
while [ $hasLine == true ]; do
hasLine=false
for i in "$@"; do
line=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$i")
if [ -n "$line" ]; then
echo "$line"
hasLine=true
fi
done
counter=$((counter+1))
done
}
function ListClients {
local backupDir="${1}"
local configFile="${2}"
local clientConfDir="${3}"
__CheckArguments 0-3 $# "$@" #__WITH_PARANOIA_DEBUG
local clientIsIncluded
local clientIsExcluded
local excludeArray
local client
local clientEmail
local configString
local i
local j
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
if [ "$clientConfDir" == "" ]; then
clientConfDir="/etc/burp/clientconfdir"
fi
if [ -d "$backupDir" ]; then
# File 'backup_stats' is there only when a backup is finished
find "$backupDir" -mindepth 3 -maxdepth 3 -type f -name "backup_stats" | grep -e '.*' > /dev/null
if [ $? != 0 ]; then
Logger "The directory [$backupDir] does not seem to be a burp folder. Please check the path. Additionnaly, protocol 2 directores need to specify the dedup group directory and the client subfolder." "ERROR"
Logger "This message may also show if the current user running this script does not have sufficient privileges on the burp folder." "ERROR"
fi
fi
# Autodetect clients or use provided client with --client or -C option
if [ "$GIVEN_CLIENT" == "" ]; then
# Using both burp -a S list and find method in order to find maximum backup clients
cmd="$BACKUP_EXECUTABLE $configString -a S | grep \"last backup\" | awk '{print \$1}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP\""
Logger "Running cmd [$cmd]." "DEBUG"
eval "$cmd" &
ExecTasks $! "${FUNCNAME[0]}_lastbackup" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Enumerating burp clients via [$BACKUP_EXECUTABLE $configString -a S] failed. Check provided burp client config file." "ERROR"
else
Logger "burp detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP)" "DEBUG"
fi
# First exp removes everything before last '/'
# sed tested on linux and BSD (should work on MacOS too)
if [ -d "$backupDir" ]; then
find "$backupDir" -mindepth 1 -maxdepth 1 -type d | sed -e "s/\(.*\)\/\(.*\)/\2/g" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP"
while IFS=$'\n' read -r client; do
find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | grep -e '.*' > /dev/null 2>&1
if [ $? == 0 ]; then
echo "$client" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
fi
done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP"
fi
if [ ! -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" ]; then
touch "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
fi
Logger "Backup file detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG"
# Merge all clients found by burp executable and manual check
sort "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" | uniq > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
else
echo "$GIVEN_CLIENT" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
fi
while IFS=$'\n' read -r client; do
clientIsIncluded=false
clientIsExcluded=false
IFS=',' read -a includeArray <<< "$INCLUDE_CLIENTS"
for i in "${includeArray[@]}"; do
echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1
if [ $? == 0 ]; then
clientIsIncluded=true
fi
done
IFS=',' read -a excludeArray <<< "$EXCLUDE_CLIENTS"
for i in "${excludeArray[@]}"; do
echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1
if [ $? == 0 ]; then
clientIsExcluded=true
fi
done
if ([ $clientIsIncluded == false ] && [ $clientIsExcluded == true ]); then
Logger "Ommiting client [$client]." "NOTICE"
else
if [ -f "$backupDir$client/current/timestamp" ]; then
Logger "Found client [$client] backup directory." "NOTICE"
elif [ -d "$backupDir" ]; then
Logger "Client [$client] does not seem to have any backups (check provided backup directory)." "WARN"
fi
CLIENT_LIST+=("$client")
# Client email is a label in client config file
# Check whether we can fetch the default value
if [ -f "$clientConfDir/$client" ]; then
clientEmail=$(egrep "^label( )?=( )?email_address( )?:( )?" "$clientConfDir/$client")
if [ "$clientEmail" != "" ]; then
clientEmail="${clientEmail#*:}"
# Remove eventual spaces
clientEmail="${clientEmail/ /}"
for j in $clientEmail; do
if [ $(CheckRFC822 "$j") -eq 1 ]; then
if [ "${CLIENT_EMAIL["$client"]}" == "" ]; then
CLIENT_EMAIL["$client"]="$j"
else
CLIENT_EMAIL["$client"]=${CLIENT_EMAIL["$client"]}" $j"
fi
else
Logger "Client [$client] has a bogus mail address [$j]." "WARN"
fi
done
else
Logger "Client [$client] has no mail address set." "NOTICE"
fi
elif ([ "$CLIENTS_ALERT_QUOTAS" == true ] || [ "$CLIENTS_ALERT_OUTDATED" == true ]); then
Logger "Cannot find client config file [$clientConfDir/$client] to fetch email address." "ERROR"
fi
fi
done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
}
function IsClientIdle {
local client="${1}"
local configFile="${2}"
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local exitCode
local configString
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
Logger "Checking if client [$client] is currently idle." "DEBUG"
cmd="$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1"
ExecTasks $! "${FUNCNAME[0]}_idle" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING
exitCode=$?
if [ $exitCode -ne 0 ]; then
Logger "Client [$client] is currently backing up." "NOTICE"
return $exitCode
else
return $exitCode
fi
}
function VerifyBackups {
local backupDir="${1}"
local numberToVerify="${2}"
local configFile="${3}"
__CheckArguments 2-3 $# "$@" #__WITH_PARANOIA_DEBUG
local backupNumber
local exitCode
local client
local configString
local interleaveFileArgs=()
local interleaveConditionsFileArgs=()
local safeExitCodes="0;2;3" # 0: success, 2: warnings, 3: timer conditions not met
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
for client in "${CLIENT_LIST[@]}"; do
# Only backups containing file backup_stats are valid
find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | sort -nr | head -n $numberToVerify | sed -e 's/.*\([0-9]\{7\}\).*/\1/' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP"
Logger "Can check $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP | wc -l) backups for [$client]." "NOTICE"
while IFS=$'\n' read -r backupNumber; do
# sed here removes all lines containing only block logs (64 chars + number)
# sed regex isn't complete (lacks \+$ because BSD / macOS sed does not like extended regex)
Logger "Preparing verification of backup [$backupNumber] for client [$client]." "NOTICE"
echo "$BACKUP_EXECUTABLE $configString -C $client -a v -b $backupNumber | sed '/^[BbfFvud]\{64\} [0-9]/d' >> \"$LOG_FILE\" 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP"
echo "$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP"
done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP" ]; then
interleaveFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP")
fi
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP" ]; then
interleaveConditionsFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP")
fi
done
InterleaveFiles "${interleaveFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
InterleaveFiles "${interleaveConditionsFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP"
Logger "Now running backup verifications, concurrency set to $PARELLEL_VERIFY_CONCURRENCY." "NOTICE"
Logger "Executing parallel commands\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG"
ExecTasks "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" "${FUNCNAME[0]}" true $SOFT_MAX_EXEC_TIME_PER_VERIFY $HARD_MAX_EXEC_TIME_PER_VERIFY $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING true false false $PARELLEL_VERIFY_CONCURRENCY "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP" $POSTPONE_RETRY $POSTPONE_TIME "$safeExitCodes"
exitCode=$?
if [ $exitCode -ne 0 ]; then
Logger "Client backup verification produced errors [$exitCode]." "ERROR"
else
Logger "Client backup verification succeed." "NOTICE"
fi
Logger "Backup verification done." "NOTICE"
}
function ListOutdatedClients {
local backupDir="${1}"
local oldDays="${2}"
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local found=false
local clientAlertBody
Logger "Checking for outdated clients." "NOTICE"
for client in "${CLIENT_LIST[@]}"; do
recentBackups=$(find "$backupDir$client" -maxdepth 2 -name "backup_stats" -and ! -path "*working*" -ctime -$oldDays | wc -l)
if [ $recentBackups -le 0 ]; then
Logger "Client [$client] has no backups newer than [$oldDays] days." "ERROR"
if [ $CLIENTS_ALERT_OUTDATED == true ]; then
clientAlertBody="${CLIENT_ALERT_BODY_OUTDATED/"[CLIENT]"/$client}"
clientAlertBody="${clientAlertBody/"[NUMBERDAYS]"/$oldDays}"
clientAlertBody="${clientAlertBody/"[INSTANCE]"/$INSTANCE_ID}"
if [ "${CLIENT_EMAIL[$client]}" != "" ]; then
SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}"
Logger "Sent outdated client mail to [${CLIENT_EMAIL[$client]}]." "NOTICE"
else
Logger "Client [$client] does not have a mail address. Cannot send notification." "ERROR"
fi
fi
found=true
fi
done
if [ $found == false ]; then
Logger "No outdated clients found." "NOTICE"
else
Logger "Outdated client checks done." "NOTICE"
fi
}
function ListQuotaExceedClients {
local backupDir="${1}"
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
local found=false
local lastBackupDir
local clientAlertBody
local bytesEstimated
local quotaExceed
local quotaDiff
Logger "Checking for clients with exceeded quota." "NOTICE"
for client in "${CLIENT_LIST[@]}"; do
lastBackupDir=$(find "$backupDir$client" -maxdepth 2 -name "*current")
if zcat "$lastBackupDir/log.gz" | grep quota > /dev/null 2>&1; then
bytesEstimated=$(zcat "$lastBackupDir/log.gz" | grep "Bytes estimated")
bytesEstimated="${bytesEstimated##*:}"
# Remove leading spaces
bytesEstimated="${bytesEstimated# *}"
quotaExceed=$(zcat "$lastBackupDir/log.gz" | grep "quota")
quotaExceed="${quotaExceed##*:}"
quotaDiff="$bytesEstimated /$quotaExceed)"
Logger "Client [$client] quota exceed ($quotaDiff)." "WARN"
if [ $CLIENTS_ALERT_QUOTAS == true ]; then
clientAlertBody="${CLIENT_ALERT_BODY_QUOTA/"[CLIENT]"/$client}"
clientAlertBody="${clientAlertBody/"[QUOTAEXCEED]"/$quotaDiff}"
if [ "${CLIENT_EMAIL[$client]}" != "" ]; then
SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}"
Logger "Sent quota exceeded mail to [${CLIENT_EMAIL[$client]}]." "NOTICE"
else
Logger "Client [$client] does not have a mail address. Cannot send notification" "ERROR"
fi
fi
found=true
fi
done
if [ $found == false ]; then
Logger "No clients with exceeded quota found." "NOTICE"
else
Logger "Quota checks done." "NOTICE"
fi
}
function VerifyLastWarnings {
local backupDir="${1}"
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
local found=false
Logger "Checking for warnings in last backups." "NOTICE"
for client in "${CLIENT_LIST[@]}"; do
if [ -f "$backupDir$client/current/log.gz" ]; then
if zcat "$backupDir$client/current/log.gz" | grep "WARNING" > /dev/null 2>&1; then
Logger "Client [$client] has the following warnings:" "WARN"
Logger "$(zcat $backupDir$client/current/log.gz | grep WARNING)" "WARN"
found=true
fi
elif [ -f "$backupDir$client/current/log" ]; then
if cat "$backupDir$client/current/log" | grep "WARNING" > /dev/null 2>&1; then
Logger "Client [$client] has the following warnings:" "WARN"
Logger "$(grep WARNING $backupDir$client/current/log)" "WARN"
found=true
fi
else
Logger "No log file found for analysis in [$backupDir$client/current]." "WARN"
fi
done
if [ $found == false ]; then
Logger "No warnings found in last backups." "NOTICE"
fi
}
function UnstripVSS {
local path="${1}"
__CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG
# We need to have a modular temp extension so we will not overwrite potential existing files
local tempExtension="$SCRIPT_PID.$TSTAMP.old"
if ! type vss_strip > /dev/null 2>&1; then
Logger "Could not find vss_strip binary. Please check your path variable." "CRITICAL"
exit 1
fi
find "$path" -type f -print0 | while IFS= read -r -d $'\0' file; do
Logger "Unstripping file [$file]." "NOTICE"
mv -f "$file" "$file.$tempExtension"
if [ $? -ne 0 ]; then
Logger "Could not move [$file] to [$file.$tempExtension] for processing." "WARN"
continue
else
vss_strip -i "$file.$tempExtension" -o "$file"
if [ $? -ne 0 ]; then
Logger "Could not vss_strip [$file.$tempExtension] to [$file]." "WARN"
mv -f "$file.$tempExtension" "$file"
if [ $? -ne 0 ]; then
Logger "Coult not move back [$file.$tempExtension] to [$file]." "WARN"
fi
else
rm -f "$file.$tempExtension"
if [ $? -ne 0 ]; then
Logger "Could not delete temporary file [$file.$tempExtension]." "WARN"
continue
fi
fi
fi
done
# Cannot get exitcode since find uses a subshell. Getting exit code from Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
return 2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
return 1
fi
return 0
}
function VerifyService {
local serviceName="${1}"
local serviceType="${2}"
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local serviceNameArray
local serviceStatusCommand
local serviceStartCommand
local i
IFS=',' read -a serviceNameArray <<< "$serviceName"
for i in "${serviceNameArray[@]}"; do
if [ "$serviceType" == "initv" ]; then
serviceStatusCommand="service $i status"
serviceStartCommand="service $i start"
elif [ "$serviceType" == "systemd" ]; then
serviceStatusCommand="systemctl status $i"
serviceStartCommand="systemctl start $i"
else
serviceStatusCommand="service $i status"
serviceStartCommand="systemctl start $i"
Logger "No valid service type given [$serviceType]. Trying default initV style." "ERROR"
fi
eval "$serviceStatusCommand" > /dev/null 2>&1 &
ExecTasks $! "${FUNCNAME[0]}_statuscmd" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
Logger "Service [$i] is not started. Trying to start it." "WARN"
eval "$serviceStartCommand" > /dev/null 2>&1 &
ExecTasks $! "${FUNCNAME[0]}_startcmd" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
Logger "Cannot start service [$i]." "CRITICAL"
SendAlert
else
Logger "Service [$i] was successfuly started." "WARN"
SendAlert
fi
else
Logger "Service [$i] is running." "NOTICE"
fi
done
}
function DisplayBackupCalendar {
local configFile="${1}"
local client
local cmd
local backups
local months
local days
local days_expr
local cdate
local month_count=1
if [ -f "$configFile" ]; then
configString="-c \"$configFile\""
fi
for client in "${CLIENT_LIST[@]}"; do
cmd="$BACKUP_EXECUTABLE $configString -a l -C \"$client\" | grep \"^Backup: \" |awk '{print \$3}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\""
Logger "Running cmd [$cmd]." "DEBUG"
eval "$cmd" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Failed to enumerate backups for client [$client]." "ERROR"
elif [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
backups="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")"
months=$(echo "$backups" | cut -d'-' -f1-2 | sort -V | uniq | tail -n $month_count)
Logger "Calendar for client [$client]:" "NOTICE"
for month in $months; do
days=$(echo "$backups" | grep "$month" | cut -d'-' -f3 | sort | uniq )
days_expr=$(echo "$days" | tr '\n' '|'|rev|cut -c2-|rev)
cdate="$(date --date="$month-01" +"%m %Y")"
cal $cdate | head -1
cal $cdate | tail -n +2 | sed -r 's/\x5f\x08//g;s/ ([0-9]( |$))/0\1/g' | grep --color -EC10 "((^| )$days_expr( |$))"
done
else
Logger "No backups were found for client [$client]" "NOTICE"
fi
done
}
function Init {
# Set error exit code if a piped command fails
set -o pipefail
set -o errtrace
trap TrapQuit TERM EXIT HUP QUIT
}
function Usage {
if [ "$IS_STABLE" != "yes" ]; then
echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m"
fi
echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD"
echo "$AUTHOR"
echo "$CONTACT"
echo ""
echo "Usage:"
echo "$0 [OPTIONS]"
echo ""
echo "[OPTIONS]"
echo "-d, --backup-dir=\"\" The directory where the client backup directories are"
echo "-o, --check-outdated-clients=n Check for clients that don't have backups newer than n days"
echo "-v, --verify-last-backups=n Verify the last n backups of all clients"
echo "-q, --verify-quotas Check for client quotas"
echo "-i, --include-clients=\"\" Comma separated list of clients to include. This list takes grep -e compatible regular expressions, includes prevail excludes"
echo "-e, --exclude-clients=\"\" Comma separated list of clients to exclude. This list takes grep -e compatible regular expressions"
echo "-c, --config-file=\"\" Path to optional burp client configuration file (defaults to /etc/burp/burp.conf)"
echo "-s, --vss-strip-path=\"\" Run vss_strip for all files in given path"
echo "-j, --verify-service=\"\" Comma separated list of burp services to check and restart if they aren't running"
echo "-w, --verify-warnings Check for warnings in last backup logs"
echo "-A, --clients-alert-quotas Use email defined in client config to alert users about exceeded disk quotas (see email template in header)"
echo "-a, --clients-alert-outdated Use email defined in client config to alert users about outdated backups (see email template in header)"
echo "-z, --clientconfdir=\"\" Path of clientconfdir in order to fetch email addresses when client alerts are used (defaults to /etc/burp/clientconfdir)"
echo "-C, --client=\"\" Specify specific client instead of detecting clients."
echo "--calendar Show backup calendar"
echo ""
echo "Examples:"
echo "$0 -d /path/to/burp/protocol1 -v 3 -c /etc/burp/burp.conf"
echo "$0 -d /path/to/burp/protocol2/global/clients --check-outdated-clients7 --exclude-clients=restoreclient,burp-ui.local"
echo "$0 --vss-strip-path=/path/to/restored/files"
echo "$0 -j burp.service"
echo "Exclude via regex all clients beginning with 'cli' and otherclient1/2:"
echo "$0 --backup-dir=/path/to/burp/protocol1 --exclude-clients=cli.*,otherclient1,otherclient2"
echo ""
echo "Additionnal options"
echo "--no-maxtime Don't stop checks after the configured maximal time in script"
echo "-s, --silent Don't output to stdout, log file only"
echo "--errors-only Don't output anything but errors."
echo ""
echo "--destination-mails=\"\" Space separated list of email adresses where to send warning and error mails"
echo "--instance-id=\"\" Arbitrary identifier for log files and alert mails"
exit 128
}
#### SCRIPT ENTRY POINT
DESTINATION_MAILS=""
no_maxtime=false
ERROR_ALERT=false
WARN_ALERT=false
CONFIG_FILE=""
BACKUP_DIR=""
VERIFY_BACKUPS=""
INCLUDE_CLIENTS=""
EXCLUDE_CLIENTS=""
OUTDATED_DAYS=""
CLIENT_LIST=()
GIVEN_CLIENT=""
declare -A CLIENT_EMAIL
VSS_STRIP_DIR=""
VERIFY_SERVICE=""
VERIFY_WARNINGS=false
VERIFY_QUOTAS=false
CLIENTS_ALERT_QUOTAS=false
CLIENTS_ALERT_OUTDATED=false
CLIENT_CONF_DIR=""
SHOW_BACKUP_CALENDAR=false
function GetCommandlineArguments {
local isFirstArgument=true
if [ $# -eq 0 ]
then
Usage
fi
while [ $# -gt 0 ]; do
## Options name is $1, argument is $2 unless there is a separator other than space
case $1 in
--instance-id=*)
INSTANCE_ID="${1##*=}"
;;
--silent)
_LOGGER_SILENT=true
;;
--verbose)
_LOGGER_VERBOSE=true
;;
--no-maxtime)
no_maxtime=true
;;
--help|-h|--version)
Usage
;;
--backup-dir=*)
BACKUP_DIR="${1##*=}"
;;
-d)
BACKUP_DIR="${2}"
shift
;;
--check-outdated-clients=*)
OUTDATED_DAYS="${1##*=}"
;;
-o)
OUTDATED_DAYS="${2}"
shift
;;
--verify-last-backups=*)
VERIFY_BACKUPS="${1##*=}"
;;
-v)
VERIFY_BACKUPS="${2}"
shift
;;
-q|--verify-quotas)
VERIFY_QUOTAS=true
;;
--include-clients=*)
INCLUDE_CLIENTS="${1##*=}"
;;
-i)
INCLUDE_CLIENTS="${2}"
shift
;;
--exclude-clients=*)
EXCLUDE_CLIENTS="${1##*=}"
;;
-e)
EXCLUDE_CLIENTS="${2}"
shift
;;
--config-file=*)
CONFIG_FILE="${1##*=}"
;;
-c)
CONFIG_FILE="${2}"
shift
;;
-C)
GIVEN_CLIENT="${2}"
shift
;;
--client=*)
GIVEN_CLIENT="${1##*=}"
;;
--calendar)
SHOW_BACKUP_CALENDAR=true
;;
--vss-strip-path=*)
VSS_STRIP_DIR="${1##*=}"
;;
-s)
VSS_STRIP_DIR="${2}"
shift
;;
-j)
VERIFY_SERVICE="${2}"
shift
;;
-A|--client-alert-quotas)
CLIENTS_ALERT_QUOTAS=true
;;
-a|--client-alert-outdated)
CLIENTS_ALERT_OUTDATED=true
;;
-z)
CLIENT_CONF_DIR="${2}"
shift
;;
--clientconfdir=*)
CLIENT_CONF_DIR="${1##*=}"
;;
--verify-service=*)
VERIFY_SERVICE="${1##*=}"
;;
-w|--verify-warnings)
VERIFY_WARNINGS=true
;;
--errors-only)
_LOGGER_ERR_ONLY=true
;;
--destination-mails=*)
DESTINATION_MAILS="${1##*=}"
;;
--no-maxtime)
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
;;
*)
if [ $isFirstArgument == false ]; then
Logger "Unknown option '${1}'" "CRITICAL"
Usage
fi
;;
esac
shift
isFirstArgument=false
done
}
GetCommandlineArguments "$@"
Init
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
DATE=$(date)
Logger "---------------------------------------------------------------------" "NOTICE"
Logger "$DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS"
Logger "---------------------------------------------------------------------" "NOTICE"
Logger "Instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME_PER_VERIFY=0
HARD_MAX_EXEC_TIME_PER_VERIFY=0
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
fi
if ! type -p "$BACKUP_EXECUTABLE" > /dev/null 2>&1; then
Logger "Cannot find [$BACKUP_EXECUTABLE]. Please modify binary path in $0 script header." "CRITICAL"
exit 126
fi
if [ "$VSS_STRIP_DIR" != "" ]; then
if [ -d "$VSS_STRIP_DIR" ]; then
UnstripVSS "$VSS_STRIP_DIR"
exit $?
else
Logger "Bogus path given to unstrip [$VSS_STRIP_DIR]." "CRITICAL"
exit 1
fi
fi
if [ "$VERIFY_SERVICE" != "" ]; then
VerifyService "$VERIFY_SERVICE" "$SERVICE_TYPE"
fi
if [ "$BACKUP_DIR" != "" ]; then
if [ ! -d "$BACKUP_DIR" ]; then
Logger "Backup dir [$BACKUP_DIR] doesn't exist." "CRITICAL"
exit 1
else
# Make sure there is only one trailing slash on path
BACKUP_DIR="${BACKUP_DIR%/}/"
fi
fi
if [ "$CONFIG_FILE" != "" ]; then
if [ ! -f "$CONFIG_FILE" ]; then
Logger "Bogus configuration file [$CONFIG_FILE] given." "CRITICAL"
exit 1
fi
fi
if [ "$CLIENT_CONF_DIR" != "" ]; then
if [ ! -d "$CLIENT_CONF_DIR" ]; then
Logger "Bogus clientconfdir [$CLIENT_CONF_DIR] given." "CRITICAL"
exit 1
fi
fi
ListClients "$BACKUP_DIR" "$CONFIG_FILE" "$CLIENT_CONF_DIR"
if [ $SHOW_BACKUP_CALENDAR == true ]; then
DisplayBackupCalendar "$CONFIG_FILE"
fi
if [ $VERIFY_WARNINGS == true ]; then
VerifyLastWarnings "$BACKUP_DIR"
fi
if [ $VERIFY_QUOTAS == true ]; then
ListQuotaExceedClients "$BACKUP_DIR"
fi
if [ "$OUTDATED_DAYS" != "" ]; then
if [ $(IsInteger "$OUTDATED_DAYS") -ne 0 ]; then
ListOutdatedClients "$BACKUP_DIR" $OUTDATED_DAYS
else
Logger "Bogus --check-outdated-clients value [$OUTDATED_DAYS]." "CRITICAL"
exit 1
fi
fi
if [ "$VERIFY_BACKUPS" != "" ]; then
if [ $(IsInteger "$VERIFY_BACKUPS") -ne 0 ]; then
VerifyBackups "$BACKUP_DIR" $VERIFY_BACKUPS "$CONFIG_FILE"
else
Logger "Bogus --verify-last-backups value [$VERIFY_BACKUPS]." "CRITICAL"
exit 1
fi
fi
# v0.4.4
# - More explicit log messages
# - Fix for missing date %N in BSD / MacOS
# - Ported some minor fixes from osync project
# v0.4.2
# - Added Hakong's backup calendar display (using --calendar)
# - Added [INSTANCE] placeholder for email sending
# - Client detection now also happens when no data directory is set
# - Fixed warning not shown when email address cannot be fetched and client alerts are enabled
# - Fixed multiple email adresses not being checked against RFC822
# v0.4.0 first public release, merged into burp codebase
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment