gistfile1.txt

^C
dliman@dev-full-i-06d10a21916387f7f:/opt/developer_api/log$ exit a
logout
-bash: exit: a: numeric argument required
Connection to 10.0.11.114 closed.
dliman@admin-bastion-i-dd0e5846:~$ cat fabfile.py
import os, sys, re, boto, urllib2,  itertools, time
from boto.ec2 import connect_to_region
from fabric.api import env, run, cd, settings, sudo, open_shell
from fabric.api import parallel

env.user = "dliman"
#env.user = "ubuntu"
#env.user = getpass.getuser()

env.connection_attempts = 1
env.timeout = 2
env.skip_bad_hosts = True
env.warn_only = True
env.disable_known_hosts = True

def memoize(function):
  memo = {}
  def wrapper(*args):
    if args in memo:
      return memo[args]
    else:
      rv = function(*args)
      memo[args] = rv
      return rv
  return wrapper

@memoize
def all_ec2_hosts():
  connections = map(_create_connection, ['us-east-1', 'us-west-2'])
  instances = []
  for conn in connections:
    # print "conn:", conn
    reservations = conn.get_all_instances()
    for res in reservations:
      instances.append(res.instances)
  instances = itertools.chain(*instances)
  ec2_hosts = sorted([i.tags['Name'] for i in instances])
  # print(ec2_hosts)
  return ec2_hosts

def chef_cleanup(delete=False):
  api = chef.autoconfigure()
  chef_cleanup_nodes(delete)
  chef_cleanup_clients(delete)

def chef_cleanup_clients(delete=False):
  clients = sorted(chef.Search('client'), key=lambda x: x['name'])
  ec2_hosts = all_ec2_hosts()
  not_in_ec2 = 0
  for client in clients:
    is_in_ec2 = client['name'] in ec2_hosts
    is_local = re.match(r'^local-', client['name']) != None
    is_chef_internal = re.match(r'chef-', client['name']) != None
    if not is_in_ec2: not_in_ec2 += 1
    print "in_ec2: %5s is_local: %5s is_chef: %5s client: %s" % (is_in_ec2, is_local, is_chef_internal, client['name'])
    if not is_in_ec2 and not is_local and not is_chef_internal:
      print "Delete", client['name']
      n = chef.Client(client['name'])
      if delete: n.delete()
  print "admin.chef01.chef.clients.not_in_ec2 %d %d" % (not_in_ec2, time.time())

def chef_cleanup_nodes(delete=False):
  ec2_hosts = all_ec2_hosts()
  nodes = sorted(chef.Search('node'), key=lambda x: x['name'])
  not_in_ec2 = 0
  for node in nodes:
    is_in_ec2 = node['name'] in ec2_hosts
    is_local = re.match(r'^local-', node['name']) != None
    if not is_in_ec2: not_in_ec2 += 1
    print "in_ec2: %5s is_local: %5s node: %s" % (is_in_ec2, is_local, node['name'])
    if not is_in_ec2 and not is_local:
      print "Delete", node['name']
      n = chef.Node(node['name'])
      if delete: n.delete()
  print "admin.chef01.chef.nodes.not_in_ec2 %d %d" % (not_in_ec2, time.time())

@parallel
def update_os():
  sudo('apt-get update -y')
  sudo('apt-get upgrade -y')

@parallel
def security_update():
  sudo('apt-get update -y')
  sudo('unattended-upgrades')


def restart_erl_services(timeout=5.0):
  sudo('stop erl_services')
  time.sleep(1)
  cmd('pgrep -fl beam.*erl_services.*console')
  sudo('pkill -9 -f beam.*erl_services.*console')
  time.sleep(0.5)
  sudo('start erl_services')
  time.sleep(float(timeout))

def restart_xmpp(timeout=10.0):
  sudo('stop devapi')
  sudo('stop xmpp')
  time.sleep(1)
  cmd('pgrep -fl beam.*developer_api.*console')
  cmd('pgrep -fl beam.*xmpp_server.*console')
  sudo('pkill -9 -f beam.*developer_api.*console')
  sudo('pkill -9 -f beam.*xmpp_server.*console')
  time.sleep(0.5)
  sudo('start xmpp')
  time.sleep(0.5)
  sudo('start devapi')
  time.sleep(float(timeout))

def restart_cn():
  sudo('/etc/init.d/web-cn* restart')
  time.sleep(60)

def es_enable_allocation():
  sudo("curl -XPUT 'localhost:9200/_settings' -d '{ \"index.routing.allocation.disable_allocation\": false }'")

@parallel
def cmd(cmd='uptime'):
  run(cmd)

@parallel
def uptime():
  run('uptime')

@parallel
def cassandra_repair_inc():
    sudo('/usr/local/cassandra/bin/nodetool repair -inc -par')

@parallel
def cassandra_repair():
    sudo('/usr/local/cassandra/bin/nodetool repair')

@parallel
def sensu():
  sudo('rm /etc/sensu/conf.d/checks/cassandra_balance.json')
  sudo('service sensu-client stop')
  sudo('service sensu-client start')

@parallel
def drop_caches(level=3):
  sudo('sync; echo %s > /proc/sys/vm/drop_caches' % (level))
  run('free -m')

@parallel
def chef_gem(gem='nokogiri'):
  sudo("/opt/chef/embedded/bin/gem install %s --no-rdoc --no-ri" % (gem))

@parallel
def package(pkg):
  sudo('apt-get install -y %s' % (pkg))

def shell():
  open_shell()

@parallel
def chef_would():
  run('grep Would /var/log/chef-client.formatter.log; echo > /dev/null', warn_only=True)

@parallel
def chef_client(args=''):
  sudo('/opt/chef/bin/chef-client %s' % (args))

@parallel
def chef_fix():
  chef_client()
  chef_client(args='-W')

@parallel
def load_balance_xmpp(action='disable', xmpp_host='', region='us-east-1'):
  filter = {'tag:Service':'loadbalancer','tag:Environment':'production*','tag:Role':'*lb_*'}
  ec2(region, **filter)
  print "hosts: %s" % (env.hosts)
  for lb_host in env.hosts:
    with settings(host_string=lb_host):
      haproxy(action, xmpp_host)

@parallel
def haproxy(action='disable', host=''):
  sudo("haproxyctl %s all %s" % (action, host))

# Fabric task to set env.hosts based on tag key-value pairs
# Examples:

# $ fab ec2   ## will default to every machine in us-east-1

# $ fab ec2:us-west-2  ## will use every machine in us-west-2

# $ fab ec2:instance-id=i-234ks12lkj  ## will use the filter {instance-id => i-123lkjdsa}

# $ fab ec2:tag:Service=xmpp  ## will use every machine in us-east-1 with tag Service == xmpp

# $ fab ec2:tag:Service=xmpp,tag:Environment=development ## uses multiple filters
def ec2(region='us-east-1', **kwargs):
  filters = kwargs

  print "filters"
  for k,v in filters.iteritems():
    print "  %s => %s" % (k,v)

  print "region:", region

  env.hosts = []
  connection = _create_connection(region)
  reservations = connection.get_all_instances(filters = filters)
  for reservation in reservations:
    for i in reservation.instances:
      if 'Environment' in i.tags:
        hostname = i.tags['Name']
        print "%25s\t%15s" % (i.private_ip_address, hostname)
        #env.hosts.append(str(i.tags['Name'] or i.private_ip_address))
        env.hosts.append(i.private_ip_address)

# Private method for getting AWS connection
def _create_connection(region):
#  if region == 'us-east-1':
#    env.gateway = 'aries.tigertext.me'
#  else:
#    env.gateway = 'leo.tigertext.me'

 # print "gateway:", env.gateway
 # print ""
  conn = connect_to_region(
    region_name = region,
    aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
    aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY")
  )

  print "Connection with AWS established"
  return conn


dliman@admin-bastion-i-dd0e5846:~$