aws_inventory.py

#!/usr/bin/python
import skew, json, placebo, os, sys
from pprint import pprint
from datetime import datetime
from elasticsearch import Elasticsearch

# parse command line args
arn_list = {

    # logging
    "cloudwatch": "arn:aws:logs:*:*:log-group/*",
    "cloudtrail": "arn:aws:cloudtrail:*:*:trail/*",
    "flow_logs": "arn:aws:ec2:*:*:flow-log/*",

    # network, external
    "hosted_zone": "arn:aws:route53:*:*:hostedzone/*",
    "elb":     "arn:aws:elb:*:*:loadbalancer/*",
    "eip":    "arn:aws:ec2:*:*:address/*",

    # network, internal
    "vpc":    "arn:aws:ec2:*:*:vpc/*",
    "ig":     "arn:aws:ec2:*:*:internet-gateway/*",
    "route_table": "arn:aws:ec2:*:*:route-table/*",
    "nat_gateway": "arn:aws:ec2:*:*:natgateway/*",
    "network_acl": "arn:aws:ec2:*:*:network-acl/*",
    "subnet": "arn:aws:ec2:*:*:subnet/*",
    "sg":     "arn:aws:ec2:*:*:security-group/*",

    # compute
    "ec2":      "arn:aws:ec2:*:*:instance/*",
    "ami":      "arn:aws:ec2:*:*:image/*",
    "volume":   "arn:aws:ec2:*:*:volume/*",
    "snapshot": "arn:aws:ec2:*:*:snapshot/*",
    "key_pair": "arn:aws:ec2:*:*:key-pair/*",
    "asg":      "arn:aws:autoscaling:*:*:autoScalingGroup/*",
    # "target_group"
    # "alb_listener"
    "lc": "arn:aws:autoscaling:*:*:launchConfiguration/*",

    # identity
    "iam_user":        "arn:aws:iam:*:*:user/*",
    "iam_group":        "arn:aws:iam:*:*:group/*",
    "iam_role":         "arn:aws:iam:*:*:role/*",
    "iam_inst-profile": "arn:aws:iam:*:*:instance-profile/*",
    "iam_policy":     "arn:aws:iam:*:*:policy/*",

    # file store
    "s3":     "arn:aws:s3:*:*:bucket/*",

    # database
    "rds":    "arn:aws:rds:*:*:db/*",
    "rds_security-group": "arn:aws:rds:*:*:secgrp/*",
    "rds_subnet-group": "arn:aws:rds:*:*:subgrp/*",
    # "rds_parameter-group": "arn:aws:rds:*:*:pg/*",
    # "rds_option-group": "arn:aws:rds:*:*:og/*",
    # "rds_snapshot": "arn:aws:rds:*:*:snapshot/*",
    # "rds_cluster": "arn:aws:rds:*:*:cluster/*",
    # "rds_cluster-snapshot": "arn:aws:rds:*:*:cluster-snapshot/*",
    # "rds_cluster-parameter-group": "arn:aws:rds:*:*:cluster-pg/*",
    # "rds_subscription": "arn:aws:rds:*:*:es/*",

    # sns
    # sqs

    # secrets store
    # kms
    # ssm
    # parameter_store
}

arn_list = {
    # "elb":     "arn:aws:elb:us-east-1:*:loadbalancer/*",
    # "cloudwatch": "arn:aws:logs:us-east-1:*:log-group/CloudTrail/DefaultLogGroup",
    # "cloudtrail": "arn:aws:cloudtrail:us-east-1:*:trail/*",
    # "cloudtrail": "arn:aws:cloudtrail:us-east-1:*:trail/awslog",
    # "flow_logs": "arn:aws:ec2:us-east-1:*:flow-log/*",
    # "flow_logs": "arn:aws:ec2:us-east-1:*:flow-log/fl-3217fe5b",
    "iam_user":        "arn:aws:iam:*:*:user/*",
    # "iam_policies":     "arn:aws:iam:*:*:policy/*",
}


def get_response_path(test_case="default"):
    p = os.path.join(os.path.dirname(__file__), 'responses')
    return os.path.join(p, test_case)


for arn_type, arn_str in arn_list.items():
    placebo_cfg = {
        'placebo': placebo,
        'placebo_dir': get_response_path(arn_type),
        'placebo_mode': 'record'}
    arn = skew.scan(arn_str, **placebo_cfg)
    l = list(arn)
    print(l[0].tags)
    pprint(l[0].data)
    sys.exit(0)

# setup ES connection
# https://elasticsearch-py.readthedocs.io/en/master/api.html
# es = Elasticsearch(['http://127.0.0.1:9200'])

# set the ES index, based on the current date
# es_index = "aws-6.6.2-" + datetime.now().strftime("%Y.%m.%d")
# print(es_index)

# clear out ES
# es.indices.delete(index="aws-6.6.2-*", ignore=[400, 404])

# run inventory scripts
for arn_type, arn_str in arn_list.items():
    for res in skew.scan(arn_str):

        data = {
            res.resourcetype: res.data,
            'arn': res.arn,
            'resourcetype': res.resourcetype,
            'account_id': res._client.account_id,
            'region_name': res._client.region_name,
            'tags': res.tags,
            'vpc_id': ""
        }
        pprint(data)
        break

        # pull-up the VpcId when possible
        if 'VpcId' in res.data:
            data['vpc_id'] = res.data['VpcId']

        if 'VPCId' in res.data:
            data['vpc_id'] = res.data['VPCId']

        # @todo convert vpc_id to an array?
        if arn_type is "ig":
            data['vpc_id'] = res.data['Attachments'][0]['VpcId']

        # ignore defaul AWS IAM policies
        if arn_type is "iam_policies" and (\
            data['arn'].startswith('arn:aws:iam::aws') \
            or data['policy']['AttachmentCount'] < 1):
            continue

        # enumerate S3 objects
        # if arn_type is "s3":
        #    for obj in res:
        #        break
        #        pprint(obj)
        pprint(data)
        # break

        # save to ES
        # res = es.index(index=es_index, doc_type='aws', body=data)

# es.indices.refresh(index=es_index)