Upgrading MongoDB across versions.

upgrade_mongodb.py

#!/usr/bin/python

import os
import sys
import tarfile
import subprocess
from datetime import datetime
from pymongo import MongoClient

class UpgradeMongo():

	DB_LOCATION = "/data/db/"
	RELEASE_INFO_LOC = "/home/swateek/workspace/db/RELEASE_INFO/"
	UPGRD_SEQ = ['3.4', '3.6', '4.0', '4.2']
	MONGODB_REPO = "/mongo_manager/tools/manage_versions/repo/"
	DB_BACKUP_LOCATION= "/home/swateek/workspace/db/store/backup/"
	TOOLS_LOGGER = "/home/swateek/workspace/logs/mongodb/docker_db.log"

	def __init__(self):
		pass

	def run(self):
		curr_mongo_version = '3.4' # default, because that's when we did all this
		is_fresh_install = self.__check_if_fresh_install()

		if is_fresh_install:
			print('Exiting Upgrade.. this is a fresh installation!')
			curr_mongo_version = self.UPGRD_SEQ[len(self.UPGRD_SEQ)-1]
			prev_mongo_version = self.UPGRD_SEQ[len(self.UPGRD_SEQ)-2] # one above this for consistency
			subprocess.check_output("echo " + curr_mongo_version + " > " + self.RELEASE_INFO_LOC + "mongodb.version_current", stderr=subprocess.STDOUT, shell=True)
			subprocess.check_output("echo " + prev_mongo_version + " > " + self.RELEASE_INFO_LOC + "mongodb.version_previous", stderr=subprocess.STDOUT, shell=True)
			sys.exit() # run the native mongo, no upgrades
		else:
			print('...Finding currently installed version...')
			curr_mongo_version = self.__find_current_version()

		to_run_upgrades = self.__find_upgrades_to_run(curr_mongo_version)
		upgrade_filemap = {}

		if len(to_run_upgrades) < 1:
			print('...Exiting Upgrade.. System is already on latest version of MongoDB!...')
			sys.exit() # Already on latest Mongo
		else:
			upgrade_filemap = self.__find_mongodb_repo_list(to_run_upgrades)

		print('...Backing up existing production database folder...')
		self.__backup_db_folder()
		print('...Beginning launching upgrade missiles...')
		self.upgrade_system_mongodb(to_run_upgrades, upgrade_filemap)
		print('...Finished running the upgrade script...')

	def upgrade_system_mongodb(self, to_run_upgrades, upgrade_filemap):
		for version in to_run_upgrades:
			# Step 1: Untar the file
			# Step 2: Run Mongo from the bin inside
			# Step 3: make pymongo to run command for setting compatibility
			# Step 4: Stop Mongo
			# Step 5: remove the release info file from location, and create one for current version and another for previous version
			# Step 6: delete the untarred file
			print("Running upgrade for - " + version)

			# Step 1
			mongo_tar = self.MONGODB_REPO + upgrade_filemap[version]
			mongo_untar = self.MONGODB_REPO + upgrade_filemap[version][:-4]
			subprocess.check_output("tar -xvzf " + mongo_tar + " -C " + self.MONGODB_REPO, stderr=subprocess.STDOUT, shell=True)

			# Step 2
			result = subprocess.check_output(mongo_untar+"/bin/mongod --fork --logpath "+self.TOOLS_LOGGER+" --logappend", stderr=subprocess.STDOUT, shell=True)
			#print(result)

			# Step 3
			client = MongoClient()
			db = client.admin
			db.command({ "setFeatureCompatibilityVersion": version})
			result = db.command({ 'getParameter': 1, 'featureCompatibilityVersion': 1 })
			print(result)

			# Step 4
			result = subprocess.check_output(mongo_untar + "/bin/mongod --shutdown", stderr=subprocess.STDOUT, shell=True)
			print(result)

			# Step 5
			subprocess.check_output("rm " + self.RELEASE_INFO_LOC + "mongodb.version_current", stderr=subprocess.STDOUT, shell=True)
			subprocess.check_output("touch " + self.RELEASE_INFO_LOC + "mongodb.version_current", stderr=subprocess.STDOUT, shell=True)
			subprocess.check_output("echo " + version + " > " + self.RELEASE_INFO_LOC + "mongodb.version_current", stderr=subprocess.STDOUT, shell=True)

			version_index = self.UPGRD_SEQ.index(version)
			if version_index > 0:
				previous_version = self.UPGRD_SEQ[version_index-1]
				if os.path.exists(self.RELEASE_INFO_LOC+"mongodb.version_previous"):
					subprocess.check_output("rm " + self.RELEASE_INFO_LOC + "mongodb.version_previous", stderr=subprocess.STDOUT, shell=True)
					subprocess.check_output("touch " + self.RELEASE_INFO_LOC + "mongodb.version_previous", stderr=subprocess.STDOUT, shell=True)
				else:
					subprocess.check_output("touch " + self.RELEASE_INFO_LOC + "mongodb.version_previous", stderr=subprocess.STDOUT, shell=True)
				subprocess.check_output("echo " + previous_version + " > " + self.RELEASE_INFO_LOC + "mongodb.version_previous", stderr=subprocess.STDOUT, shell=True)

			# Step 6
			subprocess.check_output("rm -rf "+mongo_untar, stderr=subprocess.STDOUT, shell=True)

		return True

	def __backup_db_folder(self):
		try:
			result = subprocess.check_output("cd /data/db/ && tar cvzf " + self.DB_BACKUP_LOCATION + "mongodb_backup_ug.tar.gz *", stderr=subprocess.STDOUT, shell=True)
			print(result)
			return True
		except Exception as e:
			print(e)
			return False

	def __find_mongodb_repo_list(self, to_run_upgrades):
		files_at_loc = []
		repo_map = {}
		try:
			files_at_loc = os.listdir(self.MONGODB_REPO)
			for direc in files_at_loc:
				abs_path = os.path.join(self.MONGODB_REPO, direc)
				if os.path.isdir(abs_path):
					files_at_loc.remove(direc)
					subprocess.check_output("rm -rf " + abs_path, stderr=subprocess.STDOUT, shell=True)
		except Exception as e:
			print(e)

		for upgrd in to_run_upgrades:
			for f in files_at_loc:
				if upgrd in f:
					repo_map[upgrd] = f

		return repo_map

	def __find_upgrades_to_run(self, curr_mongo_version):
		TO_RUN_UPGRD_SEQ = self.UPGRD_SEQ
		release_indx = self.UPGRD_SEQ.index(curr_mongo_version)

		while release_indx >= 0:
			TO_RUN_UPGRD_SEQ = TO_RUN_UPGRD_SEQ[1:]
			release_indx = release_indx - 1

		if len(TO_RUN_UPGRD_SEQ) == 1: # if latest, then we don't need to run
			if TO_RUN_UPGRD_SEQ[0] == curr_mongo_version:
				TO_RUN_UPGRD_SEQ = []

		return TO_RUN_UPGRD_SEQ

	def __find_current_version(self):
		rel_info_file = False # check if the mongodb file is present
		curr_version = ''
		rel_info_file = os.path.exists(self.RELEASE_INFO_LOC+"mongodb.version_current")
		# mongodb.version_current file if not found.. need to put a default file

		if rel_info_file:
			try:
				with open(self.RELEASE_INFO_LOC+"mongodb.version_current") as f:
					curr_version = f.read()
			except Exception as e:
				print(e)
				raise
		else:
			try:
				with open(self.RELEASE_INFO_LOC+"mongodb.version_current", 'a+') as f:
					f.write("3.4") # it was in 3.4 we realised how to correctly upgrade
					curr_version = "3.4"
			except Exception as e:
				print(e)
				raise

		return curr_version.strip()

	def __check_if_fresh_install(self):
		files_at_loc = []
		try:
			files_at_loc = os.listdir(self.DB_LOCATION)
		except Exception as e:
			print(e)

		if len(files_at_loc) > 0:
			return False
		else:
			return True

UpgradeMongo().run()