mongodb.rolling_upgrade Fabric Task

mongodb.rolling_upgrade.py

@task
def rolling_upgrade(hostname):
        """ Rolling upgrade of mongodb """
        users.root()
        ## Make initial connection to determine which member is the primary node 
        conn = Connection('%s'%(hostname))
        db = conn.local
        isMaster = db.command("isMaster")
        primary = (isMaster['primary'])
        primary = primary.replace(":27017","")
        ## Connect to the primary node to check health on the secondary nodes
        conn = Connection('%s'%(primary))
        db = conn.admin
        replSet = db.command("replSetGetStatus")
        ## Loop through the secondary hosts
        for secondary in replSet['members']:
                stateStr = str(secondary['stateStr'])
                host = str(secondary['name'])
                host = host.replace(":27017","")
                if stateStr == 'SECONDARY':
                        print (blue("Upgrading Host:" + blue(host)))
                        print " "
                        # Pull in updated binaries using puppet
                        execute(puppet.puppetd_test, host='%s' %(host))
                        # Restart mongodb
                        execute(mongod.restart, host='%s' %(host))
                        # Check to make sure the secondary node is recovered before moving on to the next node
                        # Sleep 5 just to make sure that the master has been notified that the node is down
                        time.sleep(5)
                        goodstatus = 'false'
                        count = 0
                        while goodstatus != 'true':
                                # Make sure that the secondary node actually recovers if it doesn't exit the python script and don't continue
                                if count < 100:
                                        replSet = db.command("replSetGetStatus")
                                        for sec in db.command("replSetGetStatus")['members']:
                                                h = str(sec['name'])
                                                h = h.replace(":27017","")
                                                if h == host:
                                                        s = str(sec['state'])
                                                        if s == '2':
                                                                goodstatus = 'true'
                                                        else:
                                                                goodstatus = 'false'
                                                                count += 1
                                                                time.sleep(2)
                                else:
                                                print (red("PROBLEM: Node '%s' Didn't Upgrade")%(host))
                                                sys.exit(1)
        for primary in replSet['members']:
                stateStr = str(primary['stateStr'])
                host = str(primary['name'])
                host = host.replace(":27017","")
                state = str(primary['state'])
                if stateStr == 'PRIMARY':
                        execute(mongod.stop, host='%s' %(host))
                        execute(puppet.puppetd_test, host='%s' %(host))
                        execute(mongod.start, host='%s' %(host))