Script to collect data from EC2 instances

ec2_metrics.py

import sys

import boto3, json
from datetime import datetime, timedelta
import dateutil.parser
from time import sleep

# pbpaste | jq -r '.[] | [.BucketName, .StorageType, .SizeBytes, .ObjectCount, .Timestamp] | @csv'| sed s/\"//g
# based on http://www.quora.com/Amazon-S3/What-is-the-fastest-way-to-measure-the-total-size-of-an-S3-bucket

# assumes you've already configured your access id & secret key
session = boto3.Session(profile_name='bkp-default', region_name='us-east-1')
ec2 = session.client('ec2')

def getValue(key, list):
    for item in list:
        return item[key]
    return ""

def getItem(key, field, list):
    if field in list:
        for item in list[field]:
            if item["Key"] == key:
                return item["Value"]
    return ""

def get_ec2_instances():
    '''Given a bucket name, retrieve the size of each key in the bucket
    and sum them together. Returns the size in gigabytes and
    the number of objects.'''

    response = ec2.describe_instances()

    ec2Instances = []
    # For each metric get data
    for instances in response['Reservations']:
        for instance in instances['Instances']:
            instanceId = instance['InstanceId']
            print("Getting data for ec2 instance: {}".format(instanceId))
            dateIso8601 = ""
            timestamp = instance['LaunchTime']
            if timestamp:
                dateIso8601 = timestamp.isoformat()
            ec2Instances.append({
                "InstanceId": instanceId,
                "InstanceType": instance['InstanceType'],
                "KeyName": instance['KeyName'],
                "State":  instance['State']['Name'],
                "CreateTime": dateIso8601,
                "VpcId": instance['VpcId'],
                "Name": getItem('Name', 'Tags', instance),
                "Onwer": getItem('Owner', 'Tags', instance),
                "Application": getItem('Application', 'Tags', instance),
                "EmrClusterId": getItem('aws:elasticmapreduce:job-flow-id', 'Tags', instance),
                "EmrNodeType": getItem('aws:elasticmapreduce:instance-group-role', 'Tags', instance)
            })

    return ec2Instances


if __name__ == '__main__':
    bucket_sizes = get_ec2_instances()
    text_file = open("/Users/admartins/LocalDocuments/notes/ec2_instances.json", "w")
    text_file.write(json.dumps(bucket_sizes, ensure_ascii=False))
    text_file.close()