Invoke Tasks to create RDS and ElasticCache instances

rds-cache-tasks.py

import os
import sys
import time
import boto3
import botocore
import pprint
from invoke import run, task

DB_CONFIG = {
    # SecurityGroup='sg-name" with ingress on port 5432 and source from itself
    # .ebextensions/02_ec2.config should have:
    #   - namespace: aws:autoscaling:launchconfiguration
    #   option_name: SecurityGroups
    #   value: 'sg-name'
    'SECURITY_GROUP': 'sg-xxxxx', 
    'DB_ID': 'rds-db1',
    'DB_NAME': 'mydb',
    'DB_VERSION': '9.5',
    'DB_TYPE': 'postgres',
    'DB_USER': 'ebroot',
    'DB_PASSWORD': 'super.secure', # TODO: Hide this
    'DB_INSTANCE_TYPE': 'db.t2.micro',  # TODO: Change to db.t2.medium or db.m4.large
    'DB_STORAGE': 5, # TODO: Should be 20 or more
    'DB_IOPS': 1000 # Not used
}

CACHE_CONFIG = {
    # SecurityGroup='sg-name' with ingress on port 6379 and source from itself
    # .ebextensions/02_ec2.config should have:
    #   - namespace: aws:autoscaling:launchconfiguration
    #   option_name: SecurityGroups
    #   value: 'sg-name'
    'SECURITY_GROUP': 'sg-xxxxxxxx',  
    'REDIS_ID': 'redis-cache1',
    'REDIS_NUM_NODES': 1,
    'REDIS_INSTANCE_TYPE': 'cache.t2.micro',
    'REDIS_VERSION': '2.8.24',
    'REDIS_PORT': 6379,
    'NOTIFICATION': 'arn:aws:sns:us-east-1:xxxxxxxx:MyNotifications'

}

AWS_SERVER_TAGS = [{
    'Key': 'Project',
    'Value': 'Strato'
}, {
    'Key': 'Stage',
    'Value': 'Production'
}]


@task
def report_rds_instance(ctx, all=False):
    print('**** report_rds_instance ****')
    rds = boto3.client('rds')

    running = True
    while running:
        response = rds.describe_db_instances()

        db_instances = response['DBInstances']
        if not all and len(db_instances) != 1:
            raise Exception('Whoa cowboy! More than one DB (or None) instance returned; this should never happen')

        for db_instance in db_instances:
            #pprint.pprint(db_instance)

            interesting_data = [
                'DBInstanceIdentifier', 'DBInstanceClass', 'Engine', 'EngineVersion',
                'DBName', 'MasterUsername',
                'AllocatedStorage', 'MultiAZ', 'StorageEncrypted', 'StorageType'
            ]
            for key in interesting_data:
                print('{0:<20} = {1}'.format(key, db_instance[key]))
            print('')

            status = db_instance['DBInstanceStatus']

            print('Last DB status: {0}'.format(status))

            time.sleep(10)
            if status == 'available':
                endpoint = db_instance['Endpoint']
                host = endpoint['Address']
                # port = endpoint['Port']

                print('DB instance ready with host: {}'.format(host))
                running = False
            print('----------------')

@task
def report_redis_cluster(ctx, all=False):
    print('**** report_redis_cluster ****')
    redis = boto3.client('elasticache')

    response = redis.describe_cache_clusters()
    #pprint.pprint(response)

    clusters = response['CacheClusters']
    for cluster in clusters:
        interesting_data = [
            'CacheClusterId', 'CacheClusterStatus', 'CacheNodeType', 'NumCacheNodes',
            'Engine', 'EngineVersion',
        ]
        for key in interesting_data:
            print('{0:<20} = {1}'.format(key, cluster[key]))
        print('')

        groups = redis.describe_replication_groups()
        if len(groups['ReplicationGroups']):
            pprint.pprint(groups['ReplicationGroups'])

    print('----------------')


@task
def create_rds_instance(ctx):
    rds = boto3.client('rds')
    try:
        rds.create_db_instance(
            DBInstanceIdentifier=DB_CONFIG['DB_ID'],
            AllocatedStorage=DB_CONFIG['DB_STORAGE'],
            DBName=DB_CONFIG['DB_NAME'],
            Engine=DB_CONFIG['DB_TYPE'],
            EngineVersion=DB_CONFIG['DB_VERSION'],
            AutoMinorVersionUpgrade=True,
            # General purpose SSD
            StorageType='gp2', # TODO: Is this ok?
            StorageEncrypted=False, # TODO: Change to True for Production
            MultiAZ=False, # TODO: Change to True for Production
            MasterUsername=DB_CONFIG['DB_USER'],
            MasterUserPassword=DB_CONFIG['DB_PASSWORD'],
            VpcSecurityGroupIds=[DB_CONFIG['SECURITY_GROUP'],],
            DBInstanceClass=DB_CONFIG['DB_INSTANCE_TYPE'],
            Tags=AWS_SERVER_TAGS
        )
        print('Starting RDS instance with ID: {0}'.format(DB_CONFIG['DB_ID']))
    except botocore.exceptions.ClientError as e:
        print(str(e))
        if e and 'DBInstanceAlreadyExists' in e.message:
            print('DB instance {0} exists already, continuing to poll ...'.format(DB_CONFIG['DB_ID']))
        else:
            raise
        sys.exit()

    report_rds_instance(ctx)

@task
def create_redis_cluster(ctx):
    redis = boto3.client('elasticache')
    try:
        response = redis.create_cache_cluster(
            CacheClusterId=CACHE_CONFIG['REDIS_ID'],
            AZMode='single-az',  # TODO: Change for production: 'cross-az',
            NumCacheNodes=CACHE_CONFIG['REDIS_NUM_NODES'],
            CacheNodeType=CACHE_CONFIG['REDIS_INSTANCE_TYPE'],
            Engine='redis',
            EngineVersion=CACHE_CONFIG['REDIS_VERSION'],
            SecurityGroupIds=[CACHE_CONFIG['SECURITY_GROUP'],],
            Tags=AWS_SERVER_TAGS,
            Port=CACHE_CONFIG['REDIS_PORT'],
            NotificationTopicArn=CACHE_CONFIG['NOTIFICATION'],
            AutoMinorVersionUpgrade=True,
        )
        print('Starting Redus instance with ID: {0}'.format(CACHE_CONFIG['REDIS_ID']))
    except botocore.exceptions.ClientError as e:
        print(str(e))
        sys.exit()

    report_redis_cluster(ctx)