RouteScraper

route_scraper.py

# generate a yaml representation of active route53 traffic policies
# specify -j to get json output to process w/ jq
# the output from the route53 api calls is a bit messy, this script
# consolidates everything into a single struct.
import boto3
import collections
import json
import yaml
from optparse import OptionParser

# https://stackoverflow.com/questions/1254454/fastest-way-to-convert-a-dicts-keys-values-from-unicode-to-str
def convert(data):
    if isinstance(data, basestring):
        return str(data)
    elif isinstance(data, collections.Mapping):
        return dict(map(convert, data.iteritems()))
    elif isinstance(data, collections.Iterable):
        return type(data)(map(convert, data))
    else:
        return data

def get_all_policies():
    alltps = {}
    tpis = r53.list_traffic_policy_instances()
    for tpi in tpis["TrafficPolicyInstances"]:
        thistp = {}
        alltps[tpi["Name"]] = thistp
        thistp["Name"] = tpi["Name"]
        thistp["TrafficPolicyType"] = tpi["TrafficPolicyType"]
        thistp["TrafficPolicyVersion"] = tpi["TrafficPolicyVersion"]

        tp = r53.get_traffic_policy(Id=tpi["TrafficPolicyId"],
                               Version=tpi["TrafficPolicyVersion"])

        policy = tp["TrafficPolicy"]
        rawdoc = policy["Document"]
        doc = json.loads(rawdoc)
        endpoints = doc["Endpoints"]

        # this won't work for complex traffic policies!
        first_rule = doc["Rules"].keys()[0]
        rule_contents = doc["Rules"][first_rule]

        regions = rule_contents["Regions"]
        thistp["Regions"] = {}
        for region in regions:
            thisregion = {}
            thisregion["Region"] = region["Region"]
            thisregion["EvaluateTargetHealth"] = region["EvaluateTargetHealth"]
            epref = region["EndpointReference"]
            if thisregion["EvaluateTargetHealth"] == True:
                hcid = region["HealthCheck"]
                rawhc = r53.get_health_check(HealthCheckId=hcid)
                hc = {}
                hc["Config"] = rawhc["HealthCheck"]["HealthCheckConfig"]
                hc["Version"] = rawhc["HealthCheck"]["HealthCheckVersion"]
                thisregion["HealthCheck"] = hc
            myep = endpoints[epref]
            thisregion["Endpoint"] = myep
            thistp["Regions"][region["Region"]] = thisregion
    return alltps

#session = boto3.Session(profile_name='foo')
session = boto3.Session()
r53 = session.client('route53')

parser = OptionParser()
parser.add_option("-j", "--json", action="store_true", help="output json", default=False)
(options, args) = parser.parse_args()

alltps = convert(get_all_policies())
if options.json:
    print(json.dumps(alltps))
else:
    print(yaml.dump(alltps))

# show only endpoints for all regions
# jq '.[] | {Name, Endpoints: [.Regions[].Endpoint.Value]}' < foo.json

# show endpoints for a given region
#jq '.[] | select(.Name == "bedrock-prod.moz.works.") | {Name, Endpoints: [.Regions[].Endpoint.Value]}' < foo.json