Fabric Pull Database

fabfile.py

from fabric.api import *
from fabric.context_managers import hide
from fabric.operations import local, put, prompt, get

from time import time
import json
import os
import re
import sys
from string import strip

def prod_server():
    env.user  = 'root'
    env.hosts = []
    env.json  = open('config.json')
    env.json = json.load(env.json)    

def get_config_details(key=None):
    if key is None:
        return env.json
    elif key in env.json:     
        return env.json[key]
    else:
        return False

def pull_db():
    remote_db_sql = _dump_remote_database()
    
    with hide('output'):
        remote_db_sql_size = run('du -b ' + remote_db_sql).replace(remote_db_sql, '').strip()

    if int(remote_db_sql_size) > 524288000: #500mb
        print "------------ Remote DB is > 500 MB --- Tarring for transfer ---------"

        with cd('/var/backup/fabric'):
            remote_tar_file = run('tar -zcvf ' + remote_db_sql + '.tar.gz ' + get_config_details('master_db_name') + '.fabric.sql')
        
        if get(remote_db_sql + '.tar.gz', './' + get_config_details('master_db_name') + '.fabric.sql.tar.gz'):
            run('rm -f ' + remote_db_sql)
            run('rm -f ' + remote_db_sql + '.tar.gz')
            
            with hide('output'):
                local('tar -zxvf ' + get_config_details('master_db_name') + '.fabric.sql.tar.gz')
                
            local('rm -f ' + './' + get_config_details('master_db_name') + '.fabric.sql.tar.gz')
    else:   
        if get(remote_db_sql, "./" + get_config_details('master_db_name') + '.fabric.sql'):
            run('rm -f ' + remote_db_sql)
                    
    local_sql_filename = "./" + get_config_details('master_db_name') + ".fabric.sql"
    
    """ At this point, regardless of tarring, we have no files except a single local .fabric.sql """
    master_url = get_config_details('master_project_url')
    slave_url  = get_config_details('slave_project_url')

    local('sed -e "s/' + master_url + '/' + slave_url + '/g" ' + local_sql_filename + ' > ' + local_sql_filename + '.replaced')

    local(_mysql_import_string(get_config_details('slave_db_username'),
                               get_config_details('slave_db_password'),
                               get_config_details('slave_db_name'),
                               local_sql_filename + '.replaced'))

    local('rm -f ' + local_sql_filename + ' ' + local_sql_filename + '.replaced')

"""
Returns a basic mysql import string, but in the process is checks is false,
prepends text to the file to disable checks, and appends text to the end to re-enable them
after the contents of the sql file.
"""
def _mysql_import_string(user, password, name, filename, esc_pass=True, checks=False):
    if esc_pass is True:
        password = re.escape(password)

    if not checks:
        print "CHECKS DISABLED -------------------------------------"
        local("sed -i '1i SET AUTOCOMMIT=0; SET UNIQUE_CHECKS=0; SET FOREIGN_KEY_CHECKS=0;' " + filename)
        local("echo 'SET FOREIGN_KEY_CHECKS=1; SET UNIQUE_CHECKS=1; SET AUTOCOMMIT=1; COMMIT;' >> " + filename)
    
    mysql_import = "mysql -u" + user + " -p" + password + " " + name + " < " + filename

    return mysql_import

def _mysqldump_string(user, password, database, output_filename):
    mysqldump =  "mysqldump --quick --extended-insert -u" + str(user)
    mysqldump += " -p" + str(password)
    mysqldump += " " + str(database)
    mysqldump += " > " + output_filename

    return mysqldump

def _dump_remote_database():
    with cd('/var/backup/fabric'):
        cmd = _mysqldump_string(get_config_details('master_db_username'),
                                get_config_details('master_db_password'),
                                get_config_details('master_db_name'),
                                get_config_details('master_db_name') + str('.fabric.sql'))

        run(cmd)

        return "/var/backup/fabric/"  + get_config_details('master_db_name') + '.fabric.sql'