Backup some directories on an EC2 instance to S3 as a tarball, prune old archives and publish result to an SNS topic

backup-ec2.sh

#!/bin/bash

# INVOCATION: bash $0
# LOG LOCATION: $storagedir/$prefix-DATESTAMP.log
# NOTES:
# - Install this in your crontab on an appropriate schedule. Filenames are to minute resolution.
# - Depending on the method in which AWS CLI was installed, it may not be in $PATH when cron executes. This will be evident in your local user mail and the script log

# CONFIGS 
storagedir="$HOME/backups"
backupdirs=("$HOME/anope-2.0.2/run/conf" "$HOME/anope-2.0.2/run/data" "$HOME/inspircd-2.0.20/run/conf")
prefix='ircserver' # filename prefix for backup archives
s3path='BUCKET/SUBDIR/'
awspath="$(which aws)"
awsregion='us-west-1' # for the sns publish, this has to be set right in the aws cli (~/.aws/config)
s3options="--sse --debug --region $awsregion" # add some params for the s3 cp operation here - debug for verbose log output
snstopic='arn:aws:sns:REGION:ACCOUNTNUMBER:TOPICNAME' # publish the result of this job to which SNS topic?
min_long_run_time=10 # minimum seconds before a warning code will be sent even if backup was successful
max_long_run_time=30 # next trigger for a bigger warning (seconds)
prune_s3=1 # to delete backups older than $remote_backup_retention or not (boolean)
remote_backup_retention=30 # number of archives to keep in s3 at any given time
local_backup_retention=7 # number of archives to keep on local disk at any given time
# END CONFIG

# Create the local backup directory
if [ ! -d ${storagedir} ] ; then mkdir ${storagedir} ; fi
cd ${storagedir}

# Get some metadata and create some variables
instanceid=$(curl http://169.254.169.254/latest/meta-data/instance-id) 
curdate=$(date -u '+%Y%m%d%H%M')
archivename="${prefix}-${curdate}.tgz"
archivepath="${storagedir}/${archivename}"

# Logfile generated 
logfile="${storagedir}/${prefix}-${curdate}.log"
date > $logfile

# Assume it will all go well
failcode=0

# Start timer
total_start_time=$(date +%s)

# Generate md5sum based on metadata of files in backup targets
echo "[+] Generating md5..." >> $logfile
backupmd5=$(ls -la ${backupdirs[@]} | md5sum | cut -d ' ' -f 1)

# Compare archive to previous -- do not upload this one if hash comparison shows that they are identical
lastarchive=$(ls -1 ${prefix}-*.tgz | tac | head -n2 | awk 'NR == 2 { print }' | sed 's/\.tgz$//')
if [[ ! -e "${storagedir}/${lastarchive}.md5" ]] || [[ "$(cat ${storagedir}/${lastarchive}.md5)" != "${backupmd5}" ]] ; then
	echo "[!] Processing attachment backups..." >> $logfile
	echo "[+] Archiving..." >> $logfile
	tar cfz ${archivename} ${backupdirs[@]}
	echo $backupmd5 > "${prefix}-${curdate}.md5"
	echo "[+] Uploading to S3..." >> $logfile
	result=$(${awspath} s3 cp ${s3options} ./${archivename} s3://${s3path} 2>&1)
	
	# trap errors
	if [ $(echo "$result" | grep -Pc "(does not match MD5 checksum|exit status is '1')") -eq 1 ] ; then
		failcode=$(($failcode + 1))
		echo "$result" >> $logfile
	fi

	# Delete archives in S3 if there are more than the allotted number to be retained
	# This isn't necessary if you use a bucket policy to remove files of a certain age
	if [ $prune_s3 -eq 1 ]	; then
		curremotebackups=(`${awspath} s3 ls s3://${s3path} | grep ${prefix}- | sort | awk '{print $4}' | cut -d '/' -f 5`)
		if [ ${#curremotebackups[@]} -ge $remote_backup_retention ] ; then
			echo "[+] Deleting oldest backup archive in S3..." >> $logfile
			numoldbackups=$((${#curremotebackups[@]} - $remote_backup_retention))
			if [ $numoldbackups -lt 0 ] ; then numoldbackups=0 ; fi
			oldremotebackups=(`echo ${curremotebackups[@]} | cut -d' ' -f -${numoldbackups}`)
			for oldfile in ${oldremotebackups[@]} ; do
				${awspath} s3 rm --region $awsregion s3://${s3path}${oldfile}
			done
		fi
	fi

	# Delete local backups in ${storagedir} if there are more than the allotted number to be retained
	curlocalbackups=(`ls -tr | grep -Po "${prefix}-(\d{12})" | uniq`)
	if [ ${#curlocalbackups[@]} -ge $local_backup_retention ] ; then
		numoldbackups=$((${#curlocalbackups[@]} - $local_backup_retention))
		if [ $numoldbackups -lt 0 ] ; then numoldbackups=0 ; fi
		oldlocalbackups=($(echo ${curlocalbackups[@]} | cut -d' ' -f -${numoldbackups}))
		echo "[+] Deleting oldest backup archive(s) locally..." >> $logfile
		for oldfile in ${oldlocalbackups[@]} ; do
			rm ${oldfile}*
		done
	fi
else
	echo "[!] Won't backup attachments as nothing has changed..." >> $logfile
fi

total_finish_time=$(date +%s)
total_execution_time=$(($total_finish_time - $total_start_time))

# If script takes longer than a specified period, warn, and if even longer, error out!
if [[ $total_execution_time -ge $min_long_run_time && $total_execution_time -lt $max_long_run_time ]] ; then
	failcode=$(($failcode + 2))
elif [ $total_execution_time -ge $max_long_run_time ] ; then
	failcode=$(($failcode + 4))
fi

# Construct the message we will post to an SNS topic on the result of this backup job
postmsg="Backup job on $instanceid completed with"

# Determine job completion status and construct error message if necessary
if [[ $(($failcode & 1)) -eq 1 ]] ; then 
	postmsg="$postmsg error (S3 upload failed)!"
	resultcode='failed'
else
	postmsg="$postmsg success!"
	resultcode='success'
fi

# Finally add how long the job took to compelte
postmsg="${postmsg} Job ran "
if [[ $(($failcode & 2)) -eq 2 ]] ; then 
	postmsg="$postmsg a bit slow"
elif [[ $(($failcode & 4)) -eq 4 ]] ; then
	postmsg"$postmsg quite slow"
fi
postmsg="$postmsg in ${total_execution_time}s."

echo "[+] $postmsg" >> $logfile

## Publish our message to an SNS topic
# echo $postmsg
echo "[!] Sending message to SNS topic!" >> $logfile
${awspath} sns publish --topic-arn $snstopic --subject "Server backup job notification ($resultcode)" --message "$postmsg" 2>&1 >> $logfile

echo "[!] Done!" >> $logfile

if [ $failcode -eq 1 ] ; then
        exit 1 
else
        exit 0
fi

iam-policy.json

{

    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:GetBucketLocation"
            ],
            "Resource": "arn:aws:s3:::*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket"
            ],
            "Resource": "arn:aws:s3:::BUCKETNAME",
            "Condition": {
                "StringEquals": {
                    "s3:prefix": [
                        "",
                        "SURBIR/"
                    ],
                    "s3:delimiter": [
                        "/"
                    ]
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": "s3:ListBucket",
            "Resource": "arn:aws:s3:::BUCKETNAME",
            "Condition": {
                "StringLike": {
                    "s3:prefix": [
                        "SUBDIR/*"
                    ]
                }
            }
        },
        {
            "Effect": "Allow",
            "Action": "s3:*",
            "Resource": "arn:aws:s3:::BUCKETNAME/SUBDIR/*"
        },
        {
            "Effect": "Allow",
            "Action": "sns:Publish",
            "Resource": "arn:aws:sns:REGION:ACCOUNTNUM:TOPICNAM"
        }
    ]
}