linux tools for commcarehq

common.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Server layout:
    ~/www/
        This folder contains the code, python environment, and logs
        for each environment (staging, production, etc) running on the server.
        Each environment has its own subfolder named for its evironment
        (i.e. ~/www/staging/log and ~/www/production/log).

    ~/www/<environment>/releases/<YYYY-MM-DD-HH.SS>
        This folder contains a release of commcarehq. Each release has its own
        virtual environment that can be found in `python_env`.

    ~/www/<environment>/current
        This path is a symlink to the release that is being run
        (~/www/<environment>/releases<YYYY-MM-DD-HH.SS>).

    ~/www/<environment>/current/services/
        This contains two subfolders
            /supervisor/
        which hold the configurations for these applications
        for each environment (staging, production, etc) running on the server.
        Theses folders are included in the global /etc/apache2 and
        /etc/supervisor configurations.

"""
import datetime
import os
import posixpath
import yaml
import pipes
from distutils.util import strtobool
from getpass import getpass

from fabric import utils
from fabric.api import run, roles, execute, task, sudo, env, parallel
from fabric.colors import blue, red, magenta
from fabric.context_managers import cd
from fabric.contrib import files, console
from fabric.decorators import runs_once
from fabric.operations import require


PROJECT_ROOT = os.path.dirname(__file__)

if env.ssh_config_path and os.path.isfile(os.path.expanduser(env.ssh_config_path)):
    env.use_ssh_config = True

env.abort_exception = Exception
env.linewise = True
env.colorize_errors = True
env.captain_user = None
env.always_use_pty = False
env['sudo_prefix'] += '-H '

if not hasattr(env, 'code_branch'):
    print ("code_branch not specified, using 'master'. "
           "You can set it with '--set code_branch=<branch>'")
    env.code_branch = 'master'


if not hasattr(env, 'force'):
    env.force = False  # --set force=true to override blocking warnings (e.g. stale pillow checkpoints)


env.roledefs = {
    'django_celery': [],
    'django_app': [],
    # for now combined with celery
    'django_pillowtop': [],
    'sms_queue': [],
    'reminder_queue': [],
    'pillow_retry_queue': [],
    # 'django_celery, 'django_app', and 'django_pillowtop' all in one
    # use this ONLY for single server config,
    # otherwise deploy() will run multiple times in parallel causing issues
    'django_monolith': [],

    'formsplayer': [],
    'staticfiles': [],

    # package level configs that are not quite config'ed yet in this fabfile
    'couch': [],
    'pg': [],
    'rabbitmq': [],
    'lb': [],
    # need a special 'deploy' role to make deploy only run once
    'deploy': [],
}


def _require_target():
    require('root', 'code_root', 'hosts', 'environment',
            provided_by=('staging', 'production', 'softlayer'))


def _setup_path():
    # using posixpath to ensure unix style slashes.
    # See bug-ticket: http://code.fabfile.org/attachments/61/posixpath.patch
    env.root = posixpath.join(env.home, 'www', env.environment)
    env.log_dir = posixpath.join(env.home, 'www', env.environment, 'log')
    env.releases = posixpath.join(env.root, 'releases')
    env.code_current = posixpath.join(env.root, 'current')
    env.code_root = posixpath.join(env.releases, env.deploy_metadata.timestamp)
    env.project_root = posixpath.join(env.code_root, env.project)
    env.project_media = posixpath.join(env.code_root, 'media')
    env.virtualenv_current = posixpath.join(env.code_current, 'python_env')
    env.virtualenv_root = posixpath.join(env.code_root, 'python_env')
    env.services = posixpath.join(env.code_root, 'services')
    env.jython_home = '/usr/local/lib/jython'
    env.db = '%s_%s' % (env.project, env.environment)
    env.offline_code_dir = posixpath.join('/home/{}/releases/{}'.format(env.user, env.deploy_metadata.timestamp))


def load_env(env_name):
    def get_env_dict(path):
        if os.path.isfile(path):
            with open(path) as f:
                try:
                    return yaml.load(f)
                except Exception:
                    print 'Error in file {}'.format(path)
                    raise
        else:
            raise Exception("Environment file not found: {}".format(path))



@task
def swiss():
    env.inventory = os.path.join(PROJECT_ROOT, 'inventory', 'swiss')
    load_env('swiss')
    env.force = True  # don't worry about kafka checkpoints on swiss
    execute(env_common)


@task
def india():
    softlayer()


@task
def softlayer():
    env.inventory = os.path.join(PROJECT_ROOT, 'inventory', 'softlayer')
    load_env('softlayer')
    execute(env_common)


@task
def icds():
    env.inventory = os.path.join(PROJECT_ROOT, 'inventory', 'icds')
    load_env('icds')
    env.user = 'ansible'
    env.password = getpass('Enter the password for then ansbile user: ')
    env.force = True  # don't worry about kafka checkpoints on icds
    execute(env_common)


@task
def production():
    """www.commcarehq.org"""
    if env.code_branch != 'master':
        branch_message = (
            "Woah there bud! You're using branch {env.code_branch}. "
            "ARE YOU DOING SOMETHING EXCEPTIONAL THAT WARRANTS THIS?"
        ).format(env=env)
        if not console.confirm(branch_message, default=False):
            utils.abort('Action aborted.')

    load_env('production')
    env.inventory = os.path.join(PROJECT_ROOT, 'inventory', 'production')
    execute(env_common)


@task
def staging():
    """staging.commcarehq.org"""
    if env.code_branch == 'master':
        env.code_branch = 'autostaging'
        print ("using default branch of autostaging. you can override this "
               "with --set code_branch=<branch>")

    env.force = True  # don't worry about kafka checkpoints on staging
    env.inventory = os.path.join(PROJECT_ROOT, 'inventory', 'staging')
    load_env('staging')
    execute(env_common)


def read_inventory_file(filename):
    """
    filename is a path to an ansible inventory file

    returns a mapping of group names ("webworker", "proxy", etc.)
    to lists of hosts (ip addresses)

    """
    from ansible.inventory.ini import InventoryParser

    return {name: [host.name for host in group.get_hosts()]
            for name, group in InventoryParser(filename).groups.items()}


@task
def development():
    """
    Must pass in the 'inventory' env variable,
    which is the path to an ansible inventory file
    and an 'environment' env variable,
    which is the name of the directory to be used under /home/cchq/www/

    Example command:

        fab development awesome_deploy \
        --set inventory=/path/to/commcarehq-ansible/ansible/inventories/development,environment=dev

    """
    load_env('development')
    execute(env_common)


def env_common():
    require('inventory')
    servers = read_inventory_file(env.inventory)
    print servers

    env.is_monolith = len(servers['all']) == 1

    deploy = servers.get('deploy', servers['postgresql'])[:1]
    #env.hosts = env.roledefs['deploy']
    env.hosts = servers['all']

fabfile.py

#!/usr/bin/env python

import os

from fabric import utils
from fabric.api import run, roles, execute, task, sudo, env, parallel
from fabric.colors import blue, red, magenta
from fabric.context_managers import cd
from fabric.contrib import files, console
from fabric.decorators import runs_once
from fabric.operations import require



PROJECT_ROOT = os.path.dirname(__file__)
from common import development, softlayer, icds

@task
def df():
    run("hostname")
    run("df -h")

@task
def old_ps():
    run("ps axh -o vsize,etimes,user,pid,pcpu,pmem,stat,start,time,cmd | sort -n -k 2 -r|awk '$3 != \"root\" && $1 > 0 && $2 > ((180*24*3600))'")

@task
def fix_apt():
    sudo("apt-get -y install -f")