fabfile with google compute engine (GCE) support + releases + virtualenv + supervisor

fabfile.py

from fabric.api import env, run, sudo, local, runs_once
# Uncomment this two lines  if you need debug mode
# import paramiko
# paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG)

# gce conf
GCE_PROJECT = 'my-project'
GCE_ZONE = 'europe-west1-b'
GCE_HOSTMATCH = 'frontend.*'
GCE_GS_URL = 'gs://bucket/dir/'
# vcs conf
VCS_REPOSITORY = 'repository-url'
VCS_BRANCH = None  # None for master, otherwise branch name 
# remote conf
REMOTE_BASE_DIR = '/opt/hosting/'


def APP_DOMAIN():
    """Preconf APP_DOMAIN to do action 
    Deploy with `fab APP_DOMAIN deploy`    
    """
    env.domain_name = "APP_DOMAIN"
    gcloud()
    env.hosts = gcloud_hosts(GCE_PROJECT, GCE_ZONE, GCE_HOSTMATCH)

def gcloud_hosts(project, zone, match):
    """Retrieve from google sdk tool the hosts we want to deploy code
    Extracts all public ip addresses from hosts in project/zone 
    and name matching pattern (hosts in group should have groupname in hostname)
    """
    hosts = []
    data = {'project': project, 'zone': zone, 'match': match}
    gcloud = local('gcloud --project "%(project)s" compute instances list --zone "%(zone)s" -r "%(match)s"' % data, capture=True)
    for line in gcloud.split('\n')[1:]:
        items = filter(lambda x: x, line.split(' '))
        hosts.append(items[-2])
    return hosts

@runs_once
def upload_to_gs():
    """After successful deploy, upload a tarball to Google Storage
    Google Apis full access is needed in VM instances (write in google storage 
    at least). This process is really useful for autoscaling -> new hosts take  
    packages from google storage (instead of master or branch).
    """
    sudo("cd %(releases_path)s/%(current_revision)s/ && tar czf /tmp/%(domain_name)s.tgz *" % {'releases_path': env.releases_path, 'current_revision': env.current_revision, 'domain_name': env.domain_name})
    sudo("gsutil cp /tmp/%(domain_name)s.tgz %(gs_url)s" % {'domain_name': env.domain_name, 'gs_url': GCE_GS_URL})


def gcloud():
    """Defines google compute engine environment. Deploys will be done with 
    current user and current gcloud sdk auth.
    """
    env.user = local('whoami', capture=True)
    env.password = ''
    env.key_filename = '~/.ssh/google_compute_engine'
    env.base_dir = REMOTE_BASE_DIR
    env.domain_path = "%(base_dir)s/%(domain_name)s" % {'base_dir': env.base_dir, 'domain_name': env.domain_name}
    env.current_path = "%(domain_path)s/current" % {'domain_path': env.domain_path}
    env.releases_path = "%(domain_path)s/releases" % {'domain_path': env.domain_path}
    env.shared_path = "%(domain_path)s/shared" % {'domain_path': env.domain_path}
    env.git_clone = VCS_REPOSITORY
    env.env_file = "requirements.txt"
    env.branch = VCS_BRANCH


def releases():
    """List a releases made"""
    env.releases = sorted(run('ls -x %(releases_path)s' % {'releases_path': env.releases_path}).split())
    if len(env.releases) >= 1:
        env.current_revision = env.releases[-1]
        env.current_release = "%(releases_path)s/%(current_revision)s" % {'releases_path': env.releases_path, 'current_revision': env.current_revision}
    if len(env.releases) > 1:
        env.previous_revision = env.releases[-2]
        env.previous_release = "%(releases_path)s/%(previous_revision)s" % {'releases_path': env.releases_path, 'previous_revision': env.previous_revision}


def start():
    """Start the application servers"""
    sudo("supervisorctl start %(domain_name)s:*" % {'domain_name': env.domain_name}, shell=False)


def restart():
    """Restarts your application"""
    sudo("supervisorctl restart %(domain_name)s:*" % {'domain_name': env.domain_name}, shell=False)


def stop():
    """Stop the application servers"""
    sudo("supervisorctl stop %(domain_name)s:*" % {'domain_name': env.domain_name}, shell=False)


def permissions():
    """Make the release group-writable"""
    # sudo("chmod -R g+w %(domain_path)s" % { 'domain_path':env.domain_path }, shell=False)
    # sudo("chown deploy:deploy -R %(domain_path)s" % {'domain_path': env.domain_path}, shell=False)


def setup():
    """Prepares one or more servers for deployment"""
    run("mkdir -p %(domain_path)s/{releases,shared}" % {'domain_path': env.domain_path})
    run("mkdir -p %(shared_path)s/log" % {'shared_path': env.shared_path})
    permissions()


def checkout():
    """Checkout code to the remote servers"""
    from time import time
    env.current_release = "%(releases_path)s/%(time).0f" % {'releases_path': env.releases_path, 'time': time()}
    if env.branch:
        run("cd %(releases_path)s; git clone -q %(git_clone)s %(current_release)s" % {'releases_path': env.releases_path, 'git_clone': env.git_clone, 'current_release': env.current_release})
        run("cd %(current_release)s && git checkout %(branch)s && cd .." % {'current_release': env.current_release, 'branch': env.branch})
    else:
        run("cd %(releases_path)s; git clone -q -o deploy --depth 1 %(git_clone)s %(current_release)s" % {'releases_path': env.releases_path, 'git_clone': env.git_clone, 'current_release': env.current_release})

    run("rm -rf %(current_release)s/.git/" % {'current_release': env.current_release})
    run("cp %(current_release)s/conf/settings-%(file)s.py %(current_release)s/settings.py" % {'current_release': env.current_release, 'file': env.domain_name})


def update():
    """Copies your project and updates environment and symlink"""
    update_code()
    update_env()
    symlink()
    permissions()


def update_code():
    """Copies your project to the remote servers"""
    checkout()
    permissions()


def symlink():
    """Updates the symlink to the most recently deployed version"""
    if 'current_release' not in env:
        releases()
    sudo("ln -nfs %(current_release)s %(current_path)s" % {'current_release': env.current_release, 'current_path': env.current_path})
    sudo("ln -nfs %(shared_path)s/log %(current_release)s/log" % {'shared_path': env.shared_path, 'current_release': env.current_release})


def update_env():
    """Update servers environment on the remote servers"""
    if 'current_release' not in env:
        releases()
    run("cd %(current_release)s; virtualenv -p /usr/bin/python2.7 env" % {'current_release': env.current_release})
    run("source %(current_release)s/env/bin/activate; cd %(current_release)s; pip -q install -r %(current_release)s/%(env_file)s; pip -q install aerospike --install-option=\"--lua-system-path=env/local/\"" % {'current_release': env.current_release, 'env_file': env.env_file})
    permissions()


def cleanup():
    """Clean up old releases"""
    if 'releases' not in env:
        releases()
    if len(env.releases) > 5:
        directories = env.releases
        directories.reverse()
        del directories[:5]
        env.directories = ' '.join(["%(releases_path)s/%(release)s" % {'releases_path': env.releases_path, 'release': release} for release in directories])
        sudo("rm -rf %(directories)s" % {'directories': env.directories})


def rollback_code():
    """Rolls back to the previously deployed version"""
    if 'releases' not in env:
        releases()
    if len(env.releases) >= 2:
        env.current_release = env.releases[-1]
        env.previous_revision = env.releases[-2]
        env.current_release = "%(releases_path)s/%(current_revision)s" % {'releases_path': env.releases_path, 'current_revision': env.current_revision}
        env.previous_release = "%(releases_path)s/%(previous_revision)s" % {'releases_path': env.releases_path, 'previous_revision': env.previous_revision}
        sudo("rm %(current_path)s; ln -s %(previous_release)s %(current_path)s && rm -rf %(current_release)s" % {'current_release': env.current_release, 'previous_release': env.previous_release, 'current_path': env.current_path})


def rollback():
    """Rolls back to a previous version and restarts"""
    rollback_code()
    restart()


def deploy():
    """Deploys your project. This calls both `update' and `restart'"""
    setup()
    update()
    restart()
    cleanup()
    upload_to_gs()

This gist looks like it'd solve my problems. Do you give others permission to use it, eg under some permissive license like MIT?
https://opensource.org/licenses/MIT

Thanks in advance!
Danny

@dgboy2000 of course, do whatever you want with this piece of code

Thanks a lot for this gist!

Note that the compute instances list parameter evolved a bit since then, I am now currently using 'gcloud --project "%s" compute instances list --filter "NAME=(%s) zone:(%s)"' % (project, match, zone), hope that helps some others :)