v2 of the lvsbackup.sh script, now using S3 for storage

lvs-backup-v2.sh

#!/bin/bash
#
# lvsbackup.sh - backup a mongo secondary by rsync/tgz
#
# LICENSE
#
# Copyright (c) 2013, Thanos Angelatos [email protected]
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#    * Neither the name of the Thanos Angelatos nor the
#      names of its contributors may be used to endorse or promote products
#      derived from this software without specific prior written permission.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#DISCLAIMED. IN NO EVENT SHALL Thanos Angelatos BE LIABLE FOR ANY
#DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
#ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# DESCRIPTION
#
# This script will backup a mongo db instance to a different location
# using rsync or other methods. To be extra-safe, enabling PARANOIDMODE will make the script
# ask mongodb to "fsync and lock" - which will flush all journal and files
# to disk, making it a 100% safe backup. Use PARANOIDMODE only on a secondary
# of a replica set. WARNING: Using "fsync and lock" on a primary will make your
# primary queue and eventually reject writes given a certain load. Usually
# a snapshot at LVM level is not more than 1-2 seconds but you have been warned.
#
# Ideally this script should be executed at night via cron - you need to monitor and
# make sure that backup is complete before your "normal backup procedure" takes over.
# The script will remove the snapshot when done, so you may check if the snapshot
# is still alive as an indication that the script is still working.
#

#
# ****** CONFIGURATION
#

#set the locale correctly
export LANG=en_US.UTF-8

#
# setting the paranoid mode to "Y" will make this script want to
# perform an fsyncLock() on your mongo process. Note that if
# your mongo is a PRIMARY you're effectively locking out all writes
# for the duration of the snapshot creation (1-30 sec). Some
# apps may not handle this well.
PARANOIDMODE="Y"

#
# setting this to Y will make the script fail if this is not a
# secondary. This is a necessary precaution since doing this on a
# primary will make the system deny writes for a period of time
FAILIFPRIMARY="Y"


# day used for the snapshot location info
DATE=`date +"%a"|tr '[A-Z]' '[a-z]'`

# BACKUPSRC is the location of the data (inside the snapshot volume)
BACKUPSRC=/data/mongo-node

# BACKUPVOL is the volume to snapshot
BACKUPVOL=/dev/vg0/lv_data

# BACKUPSNAP is the new snapshot, created out of the $BACKUPVOL plus the day
BACKUPSNAP="${BACKUPVOL}_snap_${DATE}"

# BACKUPSNAPMOUNT is where to mount the snapshot (directory will be created)
BACKUPSNAPMOUNT=/ephemeral/backup-data/snap

# BACKUPTARGET is the mountpoint of your EBS backup volume. This should be
# mounted and have space for your backup.
BACKUPTARGET=/ephemeral/backup-data

# SNAPSIZE is the size of the SNAPSHOT. Check lvm documentation and size accordingly
SNAPSIZE="50GB"

# OPTS contains options for rsync.
# consult the man page of rsync for details.
RSYNCOPTS="-aWHPrx --delete --inplace -p -vv " # remove -vv if too verbose for you

# mongo executable location
MONGO="/opt/mongodb/bin/mongo"

# host and port to connect. /admin is appended, so something should be here.
HOSTPORT="localhost:27017"
# OPTIONAL - if another mongo (e.g. Config server) runs at the same location (BACKUPVOL).
# It will be backed up also, so needs to also freeze.
HOSTPORTC="localhost:27019"

# mount options
# XFS - if nouuid is not used, it will not mount
# http://www.miljan.org/main/2009/11/16/lvm-snapshots-and-xfs/
MOUNTOPTS=" -t xfs -o nouuid "


#################################
## S3 STUFF
##
#################################
S3BUCKET="ec2-mcs-ie-backups"

# location of aws executable:
AWS=/root/aws


if [ ! -f "$AWS" ];
then
	echo " "
	echo "lvsbackup needs AWS script for AWS connectivity. Install via the following commands:"
	echo "# curl https://raw.github.com/timkay/aws/master/aws -o aws "
	echo "# chmod +x aws"
	exit 200
else
	if [ ! -f ~/.awssecret ];
	then
		echo "ERROR: .awssecret not configured"
		exit 201
	fi
fi

echo "********************************************************************"
echo " Backup script using lvm snapshots on mongo"
echo " "
echo " WARNING !!! USE THIS ONLY ON A SECONDARY NODE !!! WARNING"
echo " "
echo " Configured: "
echo "     PARANOID=$PARANOIDMODE (Y enables fsync+lock, N goes topless)"
echo "     FAILIFPRIMARY=$FAILIFPRIMARY (Y fails if this node is a Primary)"
echo "     Source $BACKUPSRC volume $BACKUPVOL"
echo "     Target $BACKUPTARGET snapshot mount $BACKUPSNAPMOUNT"
echo "     Snapshot size to use $SNAPSIZE"
echo "     main mongo command : '$MONGO $HOSTPORT/admin'"
echo "     aws executable : $AWS"
echo " "
echo "********************************************************************"
echo " "



#
# function to unmount and kill the snapshot,
# removing the used space in the process.
# Remember: LVM works on COW principles, so you dont need
# the full space for a snapshot - only differences are stored.
#
unmountSnapshot() {
	if [ -e $BACKUPSNAP ]
	then
		/bin/umount $BACKUPSNAPMOUNT
		echo "Snapshot $BACKUPSNAP unmounted from $BACKUPSNAPMOUNT"
  		/sbin/lvremove -f $BACKUPSNAP
		echo "Snapshot $BACKUPSNAP removed."
	else
		echo "Snapshot not there, nothing to clean"
	fi
}

#
# if PARANOIDMODE is "Y" then this will ask
# your mongo to clean up and flush everything to disk
# This is the safest option by far - but in the backup you still
# need to remove the mongod.lock file. Check docs.
#
fsyncLock() {
	if [ "x$PARANOIDMODE" == "xY" ]
	then
		echo "db.fsyncLock(); " > script.js
		$MONGO $1/admin script.js
		RET=$?
		if [ $RET = 0 ]
		then
 			echo "PARANOID: Locked mongodb on $1"
		else
			echo "********************************************************************"
			echo "PARANOID: Failed to lock $1, (got $RET back) PANIC!"
			echo "********************************************************************"
			exit $RET
		fi
	fi
}

#
# if PARANOIDMODE is "Y" this will unlock the DB.
# In case of error, it will fail the script - and your DB
# will be locked. Make it notify you via email if this happens
#
unlock() {
	if [ "x$PARANOIDMODE" == "xY" ]
	then
		echo "db.fsyncUnlock(); " > script.js
		$MONGO $1/admin script.js
		RET=$?
		if [ $RET = 0 ]
		then
 			echo "PARANOID: Unlocked mongodb" on $1
		else
			echo "********************************************************************"
			echo "PARANOID: Failed to unlock $1, (got $RET back) PANIC!"
			echo "********************************************************************"
			exit $RET
		fi
	fi

}

#
# creates and mounts the snapshot.
#
createAndMountSnapshot() {

	fsyncLock "$HOSTPORT"
	fsyncLock "$HOSTPORTC"

	/sbin/lvcreate -L$SNAPSIZE -s -n $BACKUPSNAP $BACKUPVOL
	RET=$?
	if [ $RET = 0 ]
 	then
		echo "Snapshot $BACKUPSNAP created"
		unlock "$HOSTPORT"
		unlock "$HOSTPORTC"
	else
		echo "Snapshot $BACKUPSNAP failed!!"
		unlock "$HOSTPORT"
		unlock "$HOSTPORTC"
		exit $RET
	fi
	if [ ! -d $BACKUPSNAPMOUNT ]
	then
		mkdir $BACKUPSNAPMOUNT
	fi
	mount $MOUNTOPTS $BACKUPSNAP $BACKUPSNAPMOUNT
	RET=$?
	if [ $RET = 0 ]
	then
		echo "Mounted $BACKUPSNAP on $BACKUPSNAPMOUNT successfully"
	else
		echo "$BACKUPSNAP cannot be mount to $BACKUPSNAPMOUNT"
		exit $RET
	fi
}

tgzBackup() {
	#check if #backup folder exists
	mkdir -p $BACKUPTARGET/backup
	rm -fr $BACKUPTARGET/backup/current.tgz
	tar cvfz $BACKUPTARGET/backup/current.tgz $BACKUPSNAPMOUNT/*
	if [ $? = 0 ]
	then
		echo "tgz backup completed at $BACKUPTARGET/backup/current.tgz"
		ORIGFILE="$BACKUPTARGET/backup/current.tgz"
		FNAME="backup-`hostname -s`-$DATE.tgz"
		FINALFILE="$BACKUPTARGET/backup/$FNAME"
		rm -fr $FINALFILE
		mv "$ORIGFILE" "$FINALFILE"
	else
		echo " "
		echo "********************************************************************"
		echo "WARNING! Backup was interrupted and volume not copied completely!"
		echo "********************************************************************"
		echo " "
	fi

}



s3copy() {
	echo "About to copy $FINALFILE to S3"
	cd $BACKUPTARGET/backup
	if [ ! -f $FNAME ]
	then
		echo "$FNAME not found, fail to copy!!!"
		exit 205
	fi
	$AWS put "$S3BUCKET" "$FNAME"
	if [ $? = 0 ]
	then
		echo "s3put succeeded! File $FINALFILE is now in $S3BUCKET "
	else
		echo "s3put failed!!!!!!! check errors!"
	fi
}


rsyncBackup() {

	/usr/bin/rsync $RSYNCOPTS $BACKUPSNAPMOUNT $BACKUPTARGET/backup
	if [ $? = 0 ]
	then
		echo "Rsync backup completed."
	else
		echo " "
		echo "********************************************************************"
		echo "WARNING! Backup was interrupted and volume not copied completely!"
		echo "********************************************************************"
		echo " "
	fi

}

checkIfPrimary() {

	is_secondary=$($MONGO $HOSTPORT/admin --quiet --eval 'rs.isMaster().secondary')
	if [ "$is_secondary" == "true" ]
	then
		echo "Secondary node - backup can proceed"
	else
		if [ "$FAILIFPRIMARY" == "Y" ]
		then
			echo "PRIMARY !!! Primary backup is dangerous, bailing out "
			exit 50
		else
			echo "PRIMARY !!! WARNING !!! BACKUP WILL CONTINUE !!! DANGER !!!"
		fi
	fi
}


echo " "
echo -n "Checking node $HOSTPORT status: "
checkIfPrimary
echo "Starting backup at `date`"
echo " "
unmountSnapshot
echo " "
createAndMountSnapshot
echo " "
echo "about to backup the volume...."
echo " "
tgzBackup
echo " "
echo "volume backup complete"
echo " "
unmountSnapshot
echo " "
echo "Copy to S3 started (file=$FINALFILE) at `date`"
s3copy
echo "Copy to S3 completed at `date`"
echo " "
echo "All done - finished at `date`."