manage_asg.py

import argparse
import boto3
import logging
import sys
from botocore.exceptions import ClientError, WaiterError

def main():
    logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
    parser = argparse.ArgumentParser(description='')

    parser.add_argument(
        '--new_asg',
        required=True,
        help='New ASG to be attach to the ELB'
    )
    parser.add_argument(
        '--elb_name',
        required=True,
        help='name of the ELB to attach/detach from the ASGs'
    )
    args = parser.parse_args()

    logging.info('Initializating AWS Connection...')
    try:
        # initialize client
        asg = boto3.client('autoscaling')

        # Find all the ASG(s) currently attached to ELB
        # In some corner cases, there may be more than one attached
        response = asg.describe_auto_scaling_groups()
        old_asgs_attached_to_lb = []
        for autoscaling_group in response["AutoScalingGroups"]:
            if args.elb_name in autoscaling_group['LoadBalancerNames']:
                old_asgs_attached_to_lb.append(autoscaling_group['AutoScalingGroupName'])

        logging.info('Current ASG(s): %s' % (old_asgs_attached_to_lb))

        # attach *NEW* ASG to ELB
        asg.attach_load_balancers(
            AutoScalingGroupName=args.new_asg,
            LoadBalancerNames=[args.elb_name]
        )

        # get the instance ids of the new ASG
        new_asg_instances=list()
        response = asg.describe_auto_scaling_groups(
            AutoScalingGroupNames=[args.new_asg])

        for instance in response["AutoScalingGroups"][0]["Instances"]:
            new_asg_instances.append({'InstanceId' : instance["InstanceId"]})

        print('Successfully attached ELB %s to the ASG %s.' % (args.elb_name, args.new_asg))
        logging.info('Successfully attached ELB %s to the ASG %s.' % (args.elb_name, args.new_asg))

        # Check and wait the health of instances
        print('Waiting for the ELB to have all health instances before removing old ASG...')
        logging.info('Waiting for the ELB to have all health instances before removing old ASG...')
        try:
            elb = boto3.client('elb')
            waiter = elb.get_waiter('instance_in_service')
            waiter.wait(
                LoadBalancerName=args.elb_name,
                WaiterConfig={
                    'Delay':1,
                    'MaxAttempts': 600
                },
                Instances=new_asg_instances
            )

            # if waiter succeeded delete detach/delete the old asg
            detach_delete_asgs(old_asgs_attached_to_lb, args.elb_name)
        except WaiterError as we:
            logging.error('Received Waiter error: %s' % we)
            logging.info('Describing Both ASG to get who is unhealthy')

            bad_asgs = set()
            unhealthy_instances = list()

            elb_instances_health = elb.describe_instance_health(
                LoadBalancerName = args.elb_name
            )
            for instance in elb_instances_health['InstanceStates']:
                if instance['State'] == 'OutOfService' or instance['State'] == 'Unknown':
                    logging.error('Instance: %s is %s due to: %s' % ( instance['InstanceId'], instance['State'], instance['Description']))
                    # add it to the list
                    unhealthy_instances.append(instance['InstanceId'])

            asg_instances = asg.describe_auto_scaling_instances(
                InstanceIds=unhealthy_instances
            )

            for instance in asg_instances['AutoScalingInstances']:
                # double checking if the health of the instance is bad ?
                if instance['HealthStatus'] == "Unhealthy":
                    logging.error('Instance: %s is %s ' % ( instance['InstanceId'], instance['HealthStatus']))

                    bad_asgs.add(instance['AutoScalingGroupName'])

            # if we have more than one item delete the old ones
            if len(bad_asgs) > 1 or not bad_asgs:
                detach_delete_asgs(old_asgs_attached_to_lb, args.elb_name)

            elif len(bad_asgs) == 1:
                if bad_asgs[0] in old_asgs_attached_to_lb:
                    # One of the old ones is unhealthy
                    # detach/delete all the old ones, the newly deployed one is healthy, use that one
                    detach_delete_asgs(old_asgs_attached_to_lb, args.elb_name)
                else:
                    # the unhealthy one is the new one we just attached
                    # to avoid zero-healthy hosts, detach/delete the new ASG
                    detach_delete_asg(bad_asgs[0], args.elb_name)

            sys.exit(1)

    except ClientError as e:
        logging.error('Received Client error: %s' % e)
        sys.exit(1)

def detach_delete_asgs(asg_names, elb_name):
    for asg_name in asg_names:
        detach_delete_asg(asg_name, elb_name)

def detach_delete_asg(asg_name, elb_name):
    asg = boto3.client('autoscaling')

    logging.info('Detaching {} from {}, then deleting'.format(asg_name, elb_name))

    asg.detach_load_balancers(
        AutoScalingGroupName=asg_name,
        LoadBalancerNames=[elb_name]
    )
    print('Successfully detached ELB %s from the ASG %s.' % (elb_name, asg_name))
    logging.info('Successfully detached ELB %s from the ASG %s.' % (elb_name,asg_name))

    # get the instances from the old asg
    instances_list=list()
    response = asg.describe_auto_scaling_groups(
        AutoScalingGroupNames=[asg_name])

    for instance in response["AutoScalingGroups"][0]["Instances"]:
        instances_list.append({'InstanceId' : instance["InstanceId"]})

    # wait for the asg instances to be deregistered
    elb = boto3.client('elb')
    waiter = elb.get_waiter('instance_deregistered')
    waiter.wait(
        LoadBalancerName=elb_name,
        Instances=instances_list,
        WaiterConfig={
                    'Delay':1,
                    'MaxAttempts': 600
                }
    )


    # delete old ASG
    asg.delete_auto_scaling_group(
        AutoScalingGroupName=asg_name,
        ForceDelete=True
    )

    print('Successfully deleted ASG: %s' % asg_name)
    logging.info('Successfully deleted ASG: %s' % asg_name)

if __name__ == "__main__":
    main()