mongodb_incremental_backup.sh

#!/bin/bash

function initStaticParams
{
   MONGODB_SERVER=127.0.0.1
   MONOGDB_PORT=27017
   MONGODB_USER=
   MONGODB_PWD=
   OUTPUT_DIRECTORY=/mnt/backups/oplogs
   LOG_FILE="/appl/mongo-backup/logs/backup.log"

   LOG_MESSAGE_ERROR=1
   LOG_MESSAGE_WARN=2
   LOG_MESSAGE_INFO=3
   LOG_MESSAGE_DEBUG=4
   LOG_LEVEL=$LOG_MESSAGE_DEBUG
   SCRIPT=`readlink -f ${BASH_SOURCE[0]}`
   ABSOLUTE_SCRIPT_PATH=$(cd `dirname "$SCRIPT"` && pwd)
}

function log
{
   MESSAGE_LEVEL=$1
   shift
   MESSAGE="$@"

   if [ $MESSAGE_LEVEL -le $LOG_LEVEL ]; then
      echo "`date +'%Y-%m-%dT%H:%M:%S.%3N'` $MESSAGE" >> $LOG_FILE
   fi
}

initStaticParams

log $LOG_MESSAGE_INFO "[INFO] starting incremental backup of oplog"

mkdir -p $OUTPUT_DIRECTORY

LAST_OPLOG_DUMP=`ls -t ${OUTPUT_DIRECTORY}/*.bson  2> /dev/null | head -1`

if [ "$LAST_OPLOG_DUMP" != "" ]; then
   log $LOG_MESSAGE_DEBUG "[DEBUG] last incremental oplog backup is $LAST_OPLOG_DUMP"
   LAST_OPLOG_ENTRY=`bsondump ${LAST_OPLOG_DUMP} 2>> $LOG_FILE | grep ts | tail -1`
   if [ "$LAST_OPLOG_ENTRY" == "" ]; then
      log $LOG_MESSAGE_ERROR "[ERROR] evaluating last backuped oplog entry with bsondump failed"
      exit 1
   else
      TIMESTAMP_LAST_OPLOG_ENTRY=`echo $LAST_OPLOG_ENTRY | jq '.ts[].t'`
      INC_NUMBER_LAST_OPLOG_ENTRY=`echo $LAST_OPLOG_ENTRY | jq '.ts[].i'`
      START_TIMESTAMP="Timestamp( ${TIMESTAMP_LAST_OPLOG_ENTRY}, ${INC_NUMBER_LAST_OPLOG_ENTRY} )"
      log $LOG_MESSAGE_DEBUG "[DEBUG] dumping everything newer than $START_TIMESTAMP"
   fi
   log $LOG_MESSAGE_DEBUG "[DEBUG] last backuped oplog entry: $LAST_OPLOG_ENTRY"
else
   log $LOG_MESSAGE_WARN "[WARN] no backuped oplog available. creating initial backup"
fi

if [ "$LAST_OPLOG_ENTRY" != "" ]; then
   mongodump -h $MONGODB_SERVER --u $MONGODB_USER -p $MONGODB_PWD -authenticationDatabase=admin -d local -c oplog.rs --query "{ \"ts\" : { \"\$gt\" : $START_TIMESTAMP } }" -o - > ${OUTPUT_DIRECTORY}/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson 2>> $LOG_FILE
   RET_CODE=$?
else 
   TIMESTAMP_LAST_OPLOG_ENTRY=0000000000
   INC_NUMBER_LAST_OPLOG_ENTRY=0
   mongodump -h $MONGODB_SERVER -u $MONGODB_USER -p $MONGODB_PWD --authenticationDatabase=admin -d local -c oplog.rs -o - > ${OUTPUT_DIRECTORY}/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson 2>> $LOG_FILE
   RET_CODE=$?
fi

if [ $RET_CODE -gt 0 ]; then
   log $LOG_MESSAGE_ERROR "[ERROR] incremental backup of oplog with mongodump failed with return code $RET_CODE"
fi

FILESIZE=`stat --printf="%s" ${OUTPUT_DIRECTORY}/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson`

if [ $FILESIZE -eq 0 ]; then
   log $LOG_MESSAGE_WARN "[WARN] no documents have been dumped with incremental backup (no changes in mongodb since last backup?). Deleting ${OUTPUT_DIRECTORY}/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson"
   rm -f ${OUTPUT_DIRECTORY}/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson
else
   log $LOG_MESSAGE_INFO "[INFO] finished incremental backup of oplog to ${OUTPUT_DIRECTORY}/${TIMESTAMP_LAST_OPLOG_ENTRY}_${INC_NUMBER_LAST_OPLOG_ENTRY}_oplog.bson"
fi

I had to make some fixes to work:

• I had to change the format of the creation of the functions, from function initStaticParams to initStaticParams()
• On line 58 change from --u to -u
• On line 58 change from -authenticationDatabase to --authenticationDatabase

I have followed your medium article.

But I am unable to find the location of logs i.e. LOG_FILE=”/appl/mongo-backup/logs/backup.log”. Can you please give me a hint so that I can proceed with successful backup.

@usmanghani599 as I can see you need to replace this value with our own suitable value for log file location

I had to make some fixes to work:

• I had to change the format of the creation of the functions, from function initStaticParams to initStaticParams()
• On line 58 change from --u to -u
• On line 58 change from -authenticationDatabase to --authenticationDatabase

Nice!