mongo backup

mongo_backup.sh

#!/bin/bash
#
#=====================================================================
#=====================================================================

# Username to access the mongo server e.g. dbuser
# DBUSERNAME=""

# Username to access the mongo server e.g. password
# DBPASSWORD=""

# Host name (or IP address) of mongo server e.g localhost
DBHOST="localhost"

# Port that mongo is listening on
DBPORT="27017"


# Backup directory location e.g /backups
BACKUPDIR="/mnt/dbbackup"

# Data path. Used for snapshots
DATADIR="/mnt/sql/shard"

# Which day do you want weekly backups? (1 to 7 where 1 is Monday)
DOWEEKLY=6

# Choose Compression type. (gzip or bzip2)
COMP="gzip"

# Choose if the uncompressed folder should be deleted after compression has completed
CLEANUP="yes"

# Use oplog for point-in-time snapshotting.
OPLOG="yes"

# Snapshot or mongodump option. Possible values are (dump, snapshot)
BACKUPTYPE="dump"

PATH=/usr/local/bin:/usr/bin:/bin

DATE=`date +%Y%m%d-%H%M`                          # Datestamp e.g 20020921-1200
DOW=`date +%A`                                    # Day of the week e.g. Monday
DNOW=`date +%u`                                   # Day number of the week 1 to 7 where 1 represents Monday
DOM=`date +%d`                                    # Date of the Month e.g. 27
M=`date +%B`                                      # Month e.g January
W=`date +%V`                                      # Week Number e.g 37


shardList=""


# Database name if you want to backup specific database
#DATABASE="elek_archive"
DATABASE="haruntest"

CREDENTIALS=""

#=====================================================================

shellout () {
    if [ -n "$1" ]; then
        echo $1
        exit 1
    fi
    exit 0
}

#=====================================================================


# Do we need to use a username/password?
if [ "$DBUSERNAME" ]; then
    CREDENTIALS="--username=$DBUSERNAME --password=$DBPASSWORD"
fi

# Create required directories
mkdir -p $BACKUPDIR/{daily,weekly,monthly} || shellout 'failed to create directories'

# Functions

# Database dump function
dump () {
    
    # Is there any specific database?
    DB="--db $DATABASE"
    if [ "$3" ]; then
        DB="--db $3"
    fi

    mongodump --host $1 $DB --out $2 $CREDENTIALS
    echo "Dump of $1 has been taken to: $2"
    return 1
}

# LVM Snashot
snapshot () {
    lvcreate --size 200G --snapshot --name $1 $DATADIR
}

#
# Select first available Secondary member in the Replica Sets and show its
# host name and port.
#
function select_secondary_member {

    # Return list of with all replica set members
    members=( $(mongo --quiet --host $1 $CREDENTIALS --eval 'rs.conf().members.forEach(function(x){ print(x.host) })') )

    # Check each replset member to see if it's a secondary and return it.
    if [ ${#members[@]} -gt 1 ]; then
        for member in "${members[@]}"; do

            is_secondary=$(mongo --quiet --host $member $CREDENTIALS --eval 'rs.isMaster().secondary')
            case "$is_secondary" in
                'true')     # First secondary wins ...
                    secondary=$member
                    break
                ;;
                'false')    # Skip particular member if it is a Primary.
                    continue
                ;;
                *)          # Skip irrelevant entries.  Should not be any anyway ...
                    continue
                ;;
            esac
        done
    fi
}

# Compression function plus latest copy
compression () {
    SUFFIX=""
    dir=$(dirname $1)
    file=$(basename $1)
    if [ -n "$COMP" ]; then
        [ "$COMP" = "gzip" ] && SUFFIX=".tgz"
        [ "$COMP" = "bzip2" ] && SUFFIX=".tar.bz2"
        echo Tar and $COMP to "$file$SUFFIX"
        cd "$dir" && tar -cf - "$file" | $COMP -c > "$file$SUFFIX"
        cd - >/dev/null || return 1
    else
        echo "No compression option set, check advanced settings"
    fi

    if [ "$CLEANUP" = "yes" ]; then
        echo Cleaning up folder at "$1"
        rm -rf "$1"
    fi

    return 0
}
                                   
# Stop Balancer
function stop_balancer () {
    mongo --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'sh.stopBalancer()'
}

# Check Balancer State
function check_balancer_state () {
    echo "Checking if the balancer is stopped"
    is_running=$(mongo --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'sh.isBalancerRunning()')
    i=0
    while [ $is_running != "false" ];  do
        is_running=$(mongo --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'sh.isBalancerRunning()')

        sleep 1

        i=$((i+1))
        if [ $i -eq 5 ]; then
                echo "Balancer is still running. Couldn't stopped."
                return 1
        fi
    done
    echo "Balancer is not running now."
    return 0
}

# Start Balancer
function start_balancer () {
    mongo --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'sh.startBalancer()'
    echo "Balancer started"
}

# Lock Secondary in a shard
function lock_db () {
    # disable profiling before locking.
    profile=$(mongo --quiet $1/admin $CREDENTIALS --eval 'db.setProfilingLevel(0)')
    echo "Profiling is disabled"
    locked=$(mongo --quiet $1 $CREDENTIALS --eval 'printjson(db.fsyncLock())' | grep "ok" | cut -d":" -f2)
    if [ $locked -ne "1"  ]; then
        start_balancer
        echo "Couldn't lock the db $1"
        exit 1
    fi
    echo "Locked the db $1"
}

# Unlock Secondary in a shard
function unlock_db () {
    unlocked=$(mongo --quiet $1/admin $CREDENTIALS --eval 'printjson(db.fsyncUnlock())' | grep "ok" | cut -d":" -f2)
    unlocked=${unlocked%,*}
    if [ $unlocked -ne "1"  ]; then
        echo "Couldn't unlock the db $1"
        exit 1
    fi
    echo "Unlocked the db $1"
}

# Find config servers and return last one to take a backup of config server
function find_config_server () {
    # comma-seperated list of config servers
    last_host=$(mongo admin --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'db.runCommand( "getCmdLineOpts" ).parsed.configdb' | awk -F"," '{ print $NF }')
    echo $last_host
}

# find a host from each shards
function find_shards () {
    # json object
    shardList=$(mongo admin --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'db.runCommand({listshards:1}).shards.forEach(function(x){ print(x.host.replace(x._id+"/","")) })')

    #shardlist=$(mongo admin --quiet --host $DBHOST:$DBPORT $CREDENTIALS --eval 'printjson(db.runCommand({listshards:1}).shards)')
    #echo $shardlist | php -r '$jsonResult=json_decode(trim(file_get_contents("php://stdin")),true); foreach ($jsonResult as $shard) { echo  end(explode(",", str_replace( $shard["_id"]."/" , "", $shard["host"] ) ) )."\n"; }'
    #echo $shardlist | php -r '$jsonResult=json_decode(trim(file_get_contents("php://stdin")),true); foreach ($jsonResult as $shard) { echo str_replace( $shard["_id"]."/" , "", $shard["host"] ) ."\n"; }'
}

echo
echo Backup of Database Server
echo ======================================================================

echo Backup Start `date`
echo ======================================================================
# Monthly Full Backup of all Databases
if [ $DOM = "01" ]; then
    echo Monthly Full Backup

    find $BACKUPDIR/monthly/ -mtime +366 -exec rm -rf \{\} \;
    FILE="$BACKUPDIR/monthly/$DATE.$M"

# Weekly Backup
elif [ $DNOW = $DOWEEKLY ]; then
    echo Weekly Backup
    echo
    echo Rotating 5 weeks Backups...

    find $BACKUPDIR/weekly/ -mtime +8 -exec rm -rf \{\} \;


    if [ "$W" -le 05 ]; then
        REMW=`expr 48 + $W`
    elif [ "$W" -lt 15 ]; then
        REMW=0`expr $W - 5`
    else
        REMW=`expr $W - 5`
    fi
    rm -f $BACKUPDIR/weekly/week.$REMW.*
    echo
    FILE="$BACKUPDIR/weekly/week.$W.$DATE"

# Daily Backup
else
    echo Daily Backup of Databases
    echo Rotating last weeks Backup...
    echo
    rm -f $BACKUPDIR/daily/*.$DOW.*
    echo
    FILE="$BACKUPDIR/daily/$DATE.$DOW"
fi

###########################################################################
## Backup Process
###########################################################################
echo "===================================================================="

stop_balancer

check_balancer_state "false"

configServer=$(find_config_server)

dump $configServer "$FILE.cfg" "config" && compression "$FILE.cfg"

find_shards
    for shard in $shardList; do
        s=$((s+1))
        lastHost=${shard##*,} #find last host
        echo "Shards $s: $shard"
        secondary=""
        select_secondary_member $lastHost
        if [ $secondary ]; then
                lock_db $secondary
                if [ $BACKUPTYPE == "dump" ]; then
                    # mongodump
                    dump $secondary "$FILE.srd$s" && compression "$FILE.srd$s"
                    echo
                elif [ $BACKUPTYPE == "snapshot" ]; then
                    #lvm snapshot
                    #snapshot $FILE
                echo
                else
                echo Wrong backup type. Please choose dump or snapshot
                    exit 1
                fi
        else
                echo "No secondary found for $shard"
        fi
        unlock_db $secondary

    done

start_balancer

###########################################################################

echo ----------------------------------------------------------------------
echo Backup End Time `date`
echo ======================================================================

echo Total disk space used for backup storage..
echo Size - Location
echo `du -hs "$BACKUPDIR"`
echo
echo ======================================================================