Mongodb scripts for incremental backup

README.md

Introduction

I can't take credit for much of the work here -- I adapted it from this blog post: https://tech.willhaben.at/mongodb-incremental-backups-dff4c8f54d58

My main contribution was to make it a little easier to use with docker as well as numerous little cleanup tasks. I also made it gzip the oplog backups and added support for SSL connections

Note that I havne't yet tested the point in time restore script; it likely needs work, particularly to make things work with the gzipped oplog files

full.sh

#!/bin/bash

source $(dirname "$0")/initParams.inc.sh

mkdir -p -v "${OUTPUT_DIRECTORY}"

DEST_PATH=${OUTPUT_DIRECTORY}/full/$(date \+\%Y\%m\%d_\%s)

log $LOG_MESSAGE_INFO "[INFO] starting full backup of ${MONGODB_URI} to ${DEST_PATH}"

mongodump --uri="$MONGODB_URI" $SSL_OPTS --oplog --gzip -o="${DEST_PATH}" 2>> $LOG_FILE
RET_CODE=$?

if [ $RET_CODE -ne 0 ]; then
   log $LOG_MESSAGE_ERROR "[ERROR] full backup of ${MONGODB_URI} failed with return code $RET_CODE"
#    rm -Rfv ${DEST_PATH}
else
   log $LOG_MESSAGE_INFO "[INFO] completed full backup of ${MONGODB_URI} to ${DEST_PATH}"
fi

gistfile1.txt

#!/bin/bash
IMG=mongo:latest
SCRIPT_BASE=$(basename $1)
CNAME=$2
if [ -z $CNAME ]; then
    CNAME=${SCRIPT_BASE%.*}
fi

SCRIPT_PATH=`cd $(dirname $0) && pwd`


docker pull $IMG
docker run --rm \
   -v ${SCRIPT_PATH}/db_backup:/backup \
   -v ${SCRIPT_PATH}:/app \
   -v /path/to/certs:/certs \
   -e LOG_DIR=/app/logs \
   -e MONGODB_URI="mongodb://your.host.name:27017/" \
   -e MONGODB_LOCAL_URI="mongodb://your.host.name:27017/local" \
   -e SSL_CAFILE=/certs/your_ca.crt \
   -e SSL_PEMKEYFILE=/certs/mongodb_cert.pem \
   --name backup_task_$CNAME \
   $IMG /app/$1

incremental.sh

#!/bin/bash

source $(dirname "$0")/initParams.inc.sh

log $LOG_MESSAGE_INFO "[INFO] starting incremental backup of oplog"

mkdir -p -v $OUTPUT_DIRECTORY/oplogs

LAST_OPLOG_DUMP=`ls -t ${OUTPUT_DIRECTORY}/oplogs/*.bson.gz  2> /dev/null | head -1`

if [ "$LAST_OPLOG_DUMP" != "" ]; then
   log $LOG_MESSAGE_DEBUG "[DEBUG] last incremental oplog backup is $LAST_OPLOG_DUMP"
   set -o xtrace
   LAST_OPLOG_ENTRY=`zcat ${LAST_OPLOG_DUMP} | bsondump 2>> $LOG_FILE | grep ts | tail -1`
   set +o xtrace
   if [ "$LAST_OPLOG_ENTRY" == "" ]; then
      log $LOG_MESSAGE_ERROR "[ERROR] evaluating last backed-up oplog entry with bsondump failed"
      exit 1
   else
      TIMESTAMP_LAST_OPLOG_ENTRY=`echo $LAST_OPLOG_ENTRY | jq '.ts[].t'`
      INC_NUMBER_LAST_OPLOG_ENTRY=`echo $LAST_OPLOG_ENTRY | jq '.ts[].i'`
      START_TIMESTAMP="{\"\$timestamp\":{\"t\":${TIMESTAMP_LAST_OPLOG_ENTRY},\"i\":${INC_NUMBER_LAST_OPLOG_ENTRY}}}"
      log $LOG_MESSAGE_DEBUG "[DEBUG] dumping everything newer than $START_TIMESTAMP"
   fi
   log $LOG_MESSAGE_DEBUG "[DEBUG] last backed-up oplog entry: $LAST_OPLOG_ENTRY"
else
   log $LOG_MESSAGE_WARN "[WARN] no backed-up oplog available. creating initial backup"
   TIMESTAMP_LAST_OPLOG_ENTRY=0000000000
   INC_NUMBER_LAST_OPLOG_ENTRY=0
fi

OPLOG_OUTFILE=${OUTPUT_DIRECTORY}/oplogs/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson.gz

if [ "$LAST_OPLOG_ENTRY" != "" ]; then
   OPLOG_QUERY="{ \"ts\" : { \"\$gt\" : $START_TIMESTAMP } }"
   set -o xtrace
   mongodump --uri="$MONGODB_LOCAL_URI" $SSL_OPTS -c oplog.rs --query "${OPLOG_QUERY}" -o - | gzip -9 > $OPLOG_OUTFILE 2>> $LOG_FILE
   set +o xtrace
   RET_CODE=$?
else 
   set -o xtrace
   mongodump --uri="$MONGODB_LOCAL_URI" $SSL_OPTS -c oplog.rs -o - | gzip -9 > $OPLOG_OUTFILE 2>> $LOG_FILE
   set +o xtrace
   RET_CODE=$?
fi

if [ $RET_CODE -gt 0 ]; then
   log $LOG_MESSAGE_ERROR "[ERROR] incremental backup of oplog with mongodump failed with return code $RET_CODE"
fi

FILESIZE=`stat --printf="%s" ${OPLOG_OUTFILE}`

# Note that I found many times when I had a failure I still had a 20 byte file;
# I figured anything smaller than 50 bytes isn't big enough to matter regardless
if [ $FILESIZE -lt 50 ]; then
   log $LOG_MESSAGE_WARN "[WARN] no documents have been dumped with incremental backup (no changes in mongodb since last backup?). Deleting ${OPLOG_OUTFILE}"
   rm -f ${OPLOG_OUTFILE}
else
   log $LOG_MESSAGE_INFO "[INFO] finished incremental backup of oplog to ${OPLOG_OUTFILE}"
fi

initParams.inc.sh

function initStaticParams
{
    MONGODB_URI=${MONGODB_URI:-mongodb://127.0.0.1:27017}
    MONGODB_LOCAL_URI=${MONGODB_LOCAL_URI:-mongodb://127.0.0.1:27017/local}
    OUTPUT_DIRECTORY=${OUTPUT_DIR:-/backup}
    LOG_DIRECTORY=${LOG_DIR:-/logs}
    LOG_FILE="${LOG_DIR}/backup.log"

    SSL_OPTS=""
    if [ -n "$SSL_CAFILE" ]; then
        SSL_OPTS="${SSL_OPTS} --sslCAFile=\"${SSL_CAFILE}\""
    fi
    if [ -n "$SSL_PEMKEYFILE" ]; then
        SSL=1
        SSL_OPTS="${SSL_OPTS} --sslPEMKeyFile=\"${SSL_PEMKEYFILE}\""
    fi
    if [ -n "$SSL_PEMKEYPASS" ]; then
        SSL=1
        SSL_OPTS="${SSL_OPTS} --sslPEMKeyPassword=\"${SSL_PEMKEYPASS}\""
    fi

    if [ "$SSL" -eq 1 ]; then
        SSL_OPTS="--ssl ${SSL_OPTS}"
    else
        SSL_OPTS=""
    fi

    mkdir -p -v "$LOG_DIR"
    LOG_MESSAGE_ERROR=1
    LOG_MESSAGE_WARN=2
    LOG_MESSAGE_INFO=3
    LOG_MESSAGE_DEBUG=4
    LOG_LEVEL=$LOG_MESSAGE_DEBUG
    SCRIPT=`readlink -f ${BASH_SOURCE[0]}`
    ABSOLUTE_SCRIPT_PATH=$(cd `dirname "$SCRIPT"` && pwd)
}

function log
{
   MESSAGE_LEVEL=$1
   shift
   MESSAGE="$@"

   if [ $MESSAGE_LEVEL -le $LOG_LEVEL ]; then
      echo "`date +'%Y-%m-%dT%H:%M:%S.%3N'` $MESSAGE" >> $LOG_FILE
   fi
}

initStaticParams

restore.sh

#!/bin/bash -xe

# OPLOG_LIMIT: only include oplog entries before the provided Timestamp
# The timestamp is a unix timestamp 
# If your desaster happened for example on 2017-18-10 12:20 and you want to restore until 12:19
# your timestamp is 'date -d "2017-10-18 12:19:00" +%s'

source $(dirname "$0")/initParams.inc.sh

FULL_DUMP_DIRECTORY=$1
OPLOGS_DIRECTORY=$2
OPLOG_LIMIT=$3

if [ "$FULL_DUMP_DIRECTORY" == "" ]; then
   echo "usage: restore.sh [NAME_OF_DIRECTORY_OF_FULL_DUMP] [DIRECTORY_TOP_OPLOGS]"
   exit 1
fi

if [ "$OPLOGS_DIRECTORY" == "" ]; then
   echo "usage: restore.sh [NAME_OF_DIRECTORY_OF_FULL_DUMP] [DIRECTORY_TOP_OPLOGS]"
   exit 1
fi

if [ "$OPLOG_LIMIT" == "" ]; then
   echo "usage: restore.sh [NAME_OF_DIRECTORY_OF_FULL_DUMP] [DIRECTORY_TOP_OPLOGS] [OPLOG_LIMIT]"
   exit 1
fi



FULL_DUMP_TIMESTAMP=`echo $FULL_DUMP_DIRECTORY | cut -d "_" -f 2 | cut -d "/" -f 1`
LAST_OPLOG=""
ALREADY_APPLIED_OPLOG=0

mkdir -p /tmp/emptyDirForOpRestore

for OPLOG in `ls $OPLOGS_DIRECTORY/*.bson.gz`; do
   OPLOG_TIMESTAMP=`echo $OPLOG | rev | cut -d "/" -f 1 | rev | cut -d "_" -f 1`
   if [ $OPLOG_TIMESTAMP -gt $FULL_DUMP_TIMESTAMP ]; then
      if [ $ALREADY_APPLIED_OPLOG -eq 0 ]; then
         ALREADY_APPLIED_OPLOG=1
         echo "applying oplog $LAST_OPLOG"
         mongorestore --uri="$MONGODB_URI" $SSL_OPTS --oplogFile $LAST_OPLOG --oplogReplay --dir /tmp/emptyDirForOpRestore --oplogLimit=$OPLOG_LIMIT
         echo "applying oplog $OPLOG"
         mongorestore --uri="$MONGODB_URI" $SSL_OPTS --oplogFile $OPLOG --oplogReplay --dir /tmp/emptyDirForOpRestore --oplogLimit=$OPLOG_LIMIT
      else
         echo "applying oplog $OPLOG"
         mongorestore --uri="$MONGODB_URI" $SSL_OPTS --oplogFile $OPLOG --oplogReplay --dir /tmp/emptyDirForOpRestore --oplogLimit=$OPLOG_LIMIT
      fi
   else
      LAST_OPLOG=$OPLOG 
   fi
done

if [ $ALREADY_APPLIED_OPLOG -eq 0 ]; then
   if [ "$LAST_OPLOG" != "" ]; then
         echo "applying oplog $LAST_OPLOG"
	 mongorestore --uri="$MONGODB_URI" $SSL_OPTS --oplogFile $LAST_OPLOG --oplogReplay --dir $OPLOGS_DIRECTORY --oplogLimit=$OPLOG_LIMIT
   fi
fi