gistfile1.py

import os, sys, re, commands
from fabric.api import *
from fabric.contrib.console import *
from fabric.contrib.project import *
from fabric.contrib.files import contains, exists
import xmlrpclib
import pip
from itertools import izip_longest
    
from fabric.version import VERSION
if VERSION > (0, 9, 2, "final", 0):
    abort("Fabric > 0.9.2: Check argument order of fabric.contrib.files.contains(...)")

env.local_app = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
env.local_build = os.path.join(env.local_app, 'build')
env.local_solr = os.path.join(env.local_app, 'solr')
env.local_site_media = os.path.join(env.local_app, 'src/atizo/site_media')

env.remote_virtualenv = '/srv/atizo-platform'
env.remote_app = env.remote_virtualenv + '/srv'
env.remote_transconf = env.remote_virtualenv + '/.translation'
env.remote_tomcatconf = '/etc/tomcat5/Catalina/localhost'
env.remote_solr = '/srv/solr'

env.user = 'root'
env.current_branch = commands.getoutput("git branch | grep '*' | sed 's/* //'")
env.ext_closure_compiler = 'http://closure-compiler.googlecode.com/files/compiler-latest.zip'
env.rsync_deploy_args = \
  ' --exclude=/log' +\
  ' --exclude=/docs' +\
  ' --exclude=/.translation' +\
  ' --exclude=/lib/atizo/site_media/survey' +\
  ' --exclude=/lib/atizo/site_media/staffstore'
env.rsync_build_args = \
  ' --include=/site_media/uploads/containers/wander-ag' +\
  ' --include=/site_media/uploads/containers/www' +\
  ' --include=/site_media/uploads/containers/energie-wasser-bern' +\
  ' --exclude=/site_media/uploads/profile/*' +\
  ' --exclude=/site_media/uploads/containers/*'
env.solr_version = '3.1.0'
env.solr_mirror = 'http://mirror.switch.ch/mirror/apache/dist/lucene/solr'

#
# banners
def _banner_prod():
    print """
#########################################################
#                                                       #
#             WARNING: About to deploy to               #
#                     \x1b[5;31mPRODUCTION\x1b[0;39m                        #
#                                                       #
#########################################################
    """

#
# environments

def stage():
    env.env = 'stage'
    env.hosts = ['fred.lan.atizo.org']

def test():
    env.env = 'test'
    env.user = 'hudson'
    env.remote_virtualenv = '/srv/atizo-platform-test'
    env.remote_app = env.remote_virtualenv + '/srv'
    env.hosts = ['fred.lan.atizo.org']

def prod():
    env.env = 'prod'
    env.hosts = ['gordon.rz.atizo.org']
    env.rsync_deploy_args += " --exclude='/lib/atizo/site_media/uploads'"
    env.source_branch = 'stage'
    env.target_branch = 'prod'
    env.hotfix_branch = 'hotfix'
    _banner_prod()
    prompt("enter 'halle luja' to continue", validate=r'^halle luja$')

def db():
    env.env = 'db'
    env.hosts = ['liam.atizo.com:12903']
    env.db = 'platform_prod'

#
# local targets

def clean():
    with settings(hide('warnings'),warn_only=True):
        if local('rm -rf %(local_build)s' % env):
            return True

def checkjs():
    env['js_files'] = ''
    for dirname, dirnames, filenames in os.walk('%(local_app)s/jstruct/' % env):        
        if dirname.endswith('jplugins'):
            continue
        for filename in filenames:
            if os.path.splitext(filename)[1] == '.js':
                env['js_files'] += os.path.join(dirname, filename) + " "
    local('! java -jar %(local_app)s/scripts/tools/jslint/js.jar %(local_app)s/scripts/tools/jslint/jslint-check.js %(js_files)s | grep -B 100 "Error(s) found."' % env, capture=False)

def _update_artifacts():
    local("compass _0.11.5_ clean %(local_app)s/compass" % env)
    local("compass _0.11.5_ compile %(local_app)s/compass" % env)
    local("%(local_app)s/scripts/jstruct.py --no-daemonize" % env)
    if env.env != 'test':
        with cd('%(local_app)s/src/atizo' % env): local("./manage.py cmessages")

def _setup_closure_compiler():
    if not os.path.exists("/tmp/closure-compiler/compiler.jar"):
        local("mkdir -p /tmp/closure-compiler")
        local("cd /tmp/closure-compiler && wget %(ext_closure_compiler)s && unzip compiler-latest.zip && rm compiler-latest.zip" % env)

def _mediaprocessor():
    local("%(local_app)s/scripts/mediaprocessor.py -f %(local_app)s/src/atizo/environments/%(env)s/mediaprocessor.conf %(local_build)s" % env)
    
def _clear_pycs():
    local("find '%(local_build)s' -type f -name '*.pyc' -print0 | xargs -0 rm -f")

def _build_environment_files(filenames, concat=False):
    if concat:
        redirector = '>>'
    else:
        redirector = '>'
    for filename in filenames:
        source = False
        params = { 'local_app':env.local_app, 'local_build':env.local_build, 'env':env.env, 'filename':filename, 'redirector':redirector, }
        if os.path.exists("%(local_app)s/src/atizo/environments/%(filename)s" % params):
            local("cat '%(local_app)s/src/atizo/environments/%(filename)s' %(redirector)s '%(local_build)s/lib/atizo/%(filename)s'" % params)
        if os.path.exists("%(local_app)s/src/atizo/environments/%(env)s/%(filename)s" % params):
            local("cat '%(local_app)s/src/atizo/environments/%(env)s/%(filename)s' %(redirector)s '%(local_build)s/lib/atizo/%(filename)s'" % params)

    local("rm -rf '%(local_build)s'/lib/atizo/environments" % env)

def _merge_release(merge='true'):
    local("git checkout %(target_branch)s" % env)
    local("git pull")
    if merge == 'hotfix':
        tag_msg = 'hotfix'
        local("git merge %(hotfix_branch)s" % env)
    elif merge == 'true':
        tag_msg = ''
        local("git merge %(source_branch)s" % env)
    else:
        tag_msg = ''

    latest_tag = commands.getoutput('git tag | egrep "[0-9]\.[0-9]+" | sort -V | tail -1')
    matches = re.search(r"""^(?P<majors>[\d\.]+\.)(?P<minor>\d+)$""", latest_tag)
    new_tag = matches.group('majors') + unicode(int(matches.group('minor')) + 1)
    local("git tag -m '%s' %s" % (tag_msg, new_tag))
    local("git push upstream %(target_branch)s" % env)
    local("git push upstream %(target_branch)s --tags" % env)
    if merge == 'hotfix':
        local("git checkout %(source_branch)s" % env)
        local("git merge %(hotfix_branch)s" % env)
    local("git checkout %(current_branch)s" % env)

def build(merge='true'):
    require('env')
    if env.env != 'stage' and env.env != 'test':
        if merge == 'hotfix':
            local("git checkout %(hotfix_branch)s" % env)
        elif merge == 'true':
            local("git checkout %(source_branch)s" % env)
        else:
            local("git checkout %(target_branch)s" % env)
        if commands.getstatusoutput("git status | grep -q 'working directory clean'")[0]:
            abort('working directory not clean')
    clean()
    _setup_closure_compiler()
    _update_artifacts()
    checkjs()
    local("mkdir -p '%(local_build)s/lib'" % env)
    local("cp -a '%(local_app)s/lib/'* '%(local_build)s/lib/'" % env)
    local("rsync -a %(rsync_build_args)s '%(local_app)s/src/atizo/' '%(local_build)s/lib/atizo/'" % env)
    _build_environment_files(['settings_local.py'])
    _build_environment_files(['dependencies.pip'], concat=True)
    _mediaprocessor()
    _clear_pycs()

#
# deploy targets

def deploy(merge='true'):
    require('env')
    build(merge=merge)
    if contains("1", env.remote_transconf):
        env.rsync_deploy_args += " --exclude='/lib/atizo/locale'"
    run("mkdir -p '%(remote_app)s/lib/atizo'" % env)
    put("%(local_build)s/lib/atizo/dependencies.pip" % env, "%(remote_app)s/lib/atizo/dependencies.pip" % env)
    _update_packages()
    local("rsync -a --delete %(rsync_deploy_args)s '%(local_build)s/' %(user)s@%(host)s:'%(remote_app)s/'" % env)
    if env.env != 'test':
        with cd('%(remote_app)s' % env):
            run("chown root:root .")
            run("mkdir -p log")
            run("chown -R root:root lib")
            run("chown -R uwsgi:uwsgi log lib/atizo/site_media/uploads lib/atizo/site_media/staffstore")
            run("find -type d -print0 | xargs -0 chmod 755")
            run("find -type f -print0 | xargs -0 chmod 644")
            run("chmod 755 lib/atizo/manage.py")
            if contains("1", env.remote_transconf):
                run("chown -R uwsgi:uwsgi lib/atizo/locale/*/LC_MESSAGES/*")
            run("/etc/init.d/uwsgi reload")
            run("/etc/init.d/celeryd restart")
    if env.env == 'test':
        with cd('%(remote_app)s' % env):
            run("mkdir -p lib/atizo/site_media/uploads/profile")
            run("chown -R %(user)s:%(user)s lib/atizo/site_media/uploads/profile" % env)
    clean()
    if env.env != 'test':
        run("echo flush_all | nc localhost 11211")
    if env.env != 'stage' and env.env != 'test':
        _merge_release(merge)

def deploy_solr():
    require('env')
    run("mkdir -p %(remote_solr)s/bin" % env)
    run("mkdir -p %(remote_solr)s/data" % env)
    local("rsync -a --delete '%(local_solr)s/conf/' %(user)s@%(host)s:'%(remote_solr)s/conf/'" % env)
    run("chown -R tomcat:tomcat %(remote_solr)s/*" % env)
    
    run("wget -q %(solr_mirror)s/%(solr_version)s/apache-solr-%(solr_version)s.tgz -O %(remote_solr)s/apache-solr-%(solr_version)s.tgz" % env)
    run("tar xzf %(remote_solr)s/apache-solr-%(solr_version)s.tgz apache-solr-%(solr_version)s/dist/apache-solr-%(solr_version)s.war" % env)
    run("mv apache-solr-%(solr_version)s/dist/apache-solr-%(solr_version)s.war %(remote_solr)s/ && rm -rf apache-solr-%(solr_version)s/ && rm %(remote_solr)s/apache-solr-%(solr_version)s.tgz" % env)
    
    put("%(local_solr)s/tomcat/solr.xml" % env, "%(remote_tomcatconf)s/solr.xml" % env)
    run("/etc/init.d/tomcat5 restart")

def deploy_containers_zonefile():
    require('env')
    with cd('%(remote_app)s/lib/atizo' % env):
        run('PYTHONPATH=%(remote_app)s/lib python -u manage.py deploy_containers_zonefile' % env)

#
# virtualenv targets

def _virtualenv(command):
    with cd(env.remote_virtualenv):
        run("source %s/bin/activate && %s" % (env.remote_virtualenv, command))

def _update_packages():
    _virtualenv('pip install -r %(remote_app)s/lib/atizo/dependencies.pip' % env)

#
# translation commands

def translation_start():
    require('env')
    if env.env != 'stage': abort("only available for 'stage'")
    if contains("1", env.remote_transconf): abort("ongoing translation")
    put("%(local_app)s/src/atizo/environments/translation-backups.cron" % env, 
        "/etc/cron.d/translation-backups.cron", 0600)
    with cd('%(local_app)s/src/atizo' % env):
        local("./manage.py makemessages -a" % env)
        local("./manage.py makemessages -d djangojs -a" % env)
    deploy()
    run("chown -R uwsgi:uwsgi %(remote_app)s/lib/atizo/locale/*/LC_MESSAGES/*" % env)
    run("echo 1 > %(remote_transconf)s;" % env)

def translation_stop():
    require('env')
    if env.env != 'stage': abort("only available for 'stage'")
    if not contains("1", env.remote_transconf): abort("no translation active")
    run("echo 0 > %(remote_transconf)s" % env)
    run("rm /etc/cron.d/translation-backups.cron")
    local("rsync -a %(user)s@%(host)s:'%(remote_app)s/lib/atizo/locale/' '%(local_app)s/src/atizo/locale/'" % env)
    run("chown -R root:root %(remote_app)s/lib/atizo/locale/*/LC_MESSAGES/*" % env)
    with cd('%(local_app)s/src/atizo' % env):
        local("./manage.py makemessages -a" % env)
        local("./manage.py makemessages -d djangojs -a" % env)
        local("./manage.py cleanmessages" % env)
        local("./manage.py cmessages" % env)
    print "remember to commit the translated gettext files..."

def translation_status():
    require('env')
    if env.env != 'stage': abort("only available for 'stage'")
    if contains("1", env.remote_transconf):
        print "ongoing translation"
    else:
        print "no translation active"

#
# django management commands

def resetload():
    require('env')
    if env.env != 'stage': abort("I'm sorry, Dave. I'm afraid I can't do that.")
    with cd('%(remote_app)s/lib/atizo' % env):
        run('export PYTHONPATH=%(remote_app)s/lib/; python -u manage.py resetload' % env)

def rebuild_index():
    require('env')
    with cd('%(remote_app)s/lib/atizo' % env):
        run('PYTHONPATH=%(remote_app)s/lib/ python -u manage.py rebuild_index --noinput' % env)

#
# db server commands

def dumpdb():
    require('env')
    if env.env != 'db': abort("only available for env 'db'")
    cmd = '/usr/pgsql-9.0/bin/pg_dump -U postgres %(db)s > /var/lib/pgsql/9.0/backups/%(db)s_devels.out; /usr/pgsql-9.0/bin/pg_dump -s -U postgres %(db)s > /var/lib/pgsql/9.0/backups/%(db)s_devels_schema.out' % env
    run(cmd)

#
# helpers

def sync_media():
    require('env')
    local('rsync -av --progress %(user)s@%(host)s:%(remote_app)s/lib/atizo/site_media/uploads/ %(local_site_media)s/uploads/' % env)
    
def check_for_updates():
    def _compare_version(version_local, version_remote):
        status = 0
        for l, r in izip_longest(version_local.split('.'), version_remote.split('.')):
            
            if l is None:
                status = 1
                break
            elif r is None:
                status = 2
                break
            
            if l.isdigit() and r.isdigit():
                l = int(l)
                r = int(r)
            if l < r:
                status = 1
                break
            elif l > r:
                status = 2
                break

        if status == 1:
            return '\x1b[0;32m%s available\x1b[0;39m' % version_remote
        elif status == 2:
            return 'ahead (%s >= %s)' % (version_local, version_remote)
        else:
            return 'up to date'
            
    pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    print 'check for updates (local):\n'
    for dist in pip.get_installed_distributions():
        available = pypi.package_releases(dist.project_name)
        if not available:
            # Try to capitalize pkg name
            available = pypi.package_releases(dist.project_name.capitalize())
        if not available:
            # Try to lower
            available = pypi.package_releases(dist.project_name.lower())
        
        if not available:
            msg = 'no releases at pypi'
        else:
            msg = _compare_version(dist.version, available[0])
            
        pkg_info = '{dist.project_name} {dist.version}'.format(dist=dist)
        if msg != 'up to date':
            print '\t{pkg_info:40} {msg}'.format(pkg_info=pkg_info, msg=msg)